otsdaq  v2_04_01
otsdaq_import_fhicl.cc
1 #include <dirent.h>
2 #include <cassert>
3 #include <iostream>
4 #include <memory>
5 #include <string>
6 #include "artdaq/Application/LoadParameterSet.hh"
7 #include "otsdaq-core/ConfigurationInterface/ConfigurationInterface.h"
8 #include "otsdaq-core/ConfigurationInterface/ConfigurationManagerRW.h"
9 
10 // usage:
11 // otsdaq_import_system_aliases <pathOfDatabaseToImport> <(optional) prependLabel>
12 //
13 // all system aliases are imported to current db and are prepended with a label
14 //
15 
16 using namespace ots;
17 
18 void usage()
19 {
20  std::cout << "\n\nusage: Two arguments:\n\n\t otsdaq_import_system_aliases "
21  "<path_to_import_database_folder> <path_to_active_groups_file> "
22  "<import_prepend_base_name (optional)> \n\n"
23  << "\t\t Default values: \n\t\t\timport_prepend_base_name = \""
24  << prependBaseName << "\" "
25  << "\n\n"
26  << "\t\tfor example:\n\n"
27  << "\t\t\totsdaq_import_system_aliases "
28  "~/databaseToImport/filesystemdb/test_db "
29  "~/UserDataToImport/ServiceData/ActiveTableGroups.cfg"
30  << __E__;
31 
32  std::cout << "\n\n\tExample active groups file content:\n\n"
33  << "testContext\n2\nTableEditWizBackbone\n3\ndefaultConfig\n1\n\n"
34  << __E__;
35 
36  std::cout << "\n\nNote: This assumes artdaq db file type interface. "
37  << "The current database/ will be backed up to database_<linuxtime>/ "
38  << "before importing the active groups.\n\n"
39  << __E__;
40 }
41 
42 void ImportSystemAliasTableGroups(fhicl::ParameterSet pset)
43 {
44  __COUT__ << "=================================================\n";
45  __COUT__ << "=================================================\n";
46  __COUT__ << "=================================================\n";
47  __COUT__ << "Importing External System Aliases!" << std::endl;
48 
49  std::string prependBaseName = pset.get<std::string>("prepend_name", "Imported");
50  const std::string groupAliasesTableName =
51  ConfigurationManager::GROUP_ALIASES_TABLE_NAME;
52  const std::string versionAliasesTableName =
53  ConfigurationManager::VERSION_ALIASES_TABLE_NAME;
54 
55  // determine if help was requested
56  if(pset.get<bool>("do_help", false) || !pset.has_key("current_database_path") ||
57  !pset.has_key("active_groups_file"))
58  {
59  usage();
60  return;
61  }
62 
63  std::string pathToImportDatabase = pset.get<std::string>("current_database_path");
64  std::string pathToImportActiveGroups = pset.get<std::string>("active_groups_file");
65 
66  __COUTV__(pathToImportDatabase);
67  __COUTV__(pathToImportActiveGroups);
68  __COUTV__(prependBaseName);
69 
70  //==============================================================================
71  // Define environment variables
72  // Note: normally these environment variables are set by StartOTS.sh
73 
74  // These are needed by
75  // otsdaq/otsdaq-core/ConfigurationDataFormats/ConfigurationInfoReader.cc [207]
76  setenv("CONFIGURATION_TYPE", "File", 1); // Can be File, Database, DatabaseTest
77  setenv("CONFIGURATION_DATA_PATH",
78  (std::string(__ENV__("USER_DATA")) + "/ConfigurationDataExamples").c_str(),
79  1);
80  setenv(
81  "TABLE_INFO_PATH", (std::string(__ENV__("USER_DATA")) + "/TableInfo").c_str(), 1);
83 
84  // Some configuration plug-ins use __ENV__("SERVICE_DATA_PATH") in init() so define it
85  setenv("SERVICE_DATA_PATH",
86  (std::string(__ENV__("USER_DATA")) + "/ServiceData").c_str(),
87  1);
88 
89  // Some configuration plug-ins use __ENV__("OTSDAQ_LIB") and
90  // __ENV__("OTSDAQ_UTILITIES_LIB") in init() so define it to a non-sense place is ok
91  setenv("OTSDAQ_LIB", (std::string(__ENV__("USER_DATA")) + "/").c_str(), 1);
92  setenv("OTSDAQ_UTILITIES_LIB", (std::string(__ENV__("USER_DATA")) + "/").c_str(), 1);
93 
94  // Some configuration plug-ins use __ENV__("OTS_MAIN_PORT") in init() so define it
95  setenv("OTS_MAIN_PORT", "2015", 1);
96 
97  // also xdaq envs for XDAQContextTable
98  setenv("XDAQ_CONFIGURATION_DATA_PATH",
99  (std::string(__ENV__("USER_DATA")) + "/XDAQConfigurations").c_str(),
100  1);
101  setenv("XDAQ_CONFIGURATION_XML", "otsConfigurationNoRU_CMake", 1);
103 
104  //==============================================================================
105 
106  // Steps:
107  //
108  // -- create empty map of import alias to original groupName & groupKey
109  // -- create empty map of original groupName & groupKey to new groupKey
110  //
111  // -- create empty map of import alias to original tableName & tableVersion
112  // -- create empty map of original tableName & tableVersion to new tableVersion
113  //
114  // -- in current-db extract info
115  // - get set of existing group aliases
116  // . load current aliases from current-db
117  // - get set of existing table aliases
118  // . load current aliases from current-db
119  //
120  // -- swap to import-db, clear cache, and create vector of groups to import
121  // - get active groups from user data file
122  // - get aliases from active backbone group
123  // . check basename+alias for collision with existing group aliases and throw
124  // error if collision
125  // . add map record of basename+alias to original groupName & groupKey
126  // . check basename+alias for collision with existing table aliases and throw
127  // error if collision
128  // . add map record of basename+alias to original tableName & tableVersion
129  //
130  // -- for each group in set
131  // - swap to import-db
132  // - load/activate group
133  // - swap to current-db
134  // - for each member table in group
135  // . check map of original tableName & tableVersion to new tableName &
136  // tableVersion, to prevent making a new table
137  // . if needed, save active tables as next table version (using interface)
138  // , add map record of original tableName & tableVersion to new tableName
139  //& tableVersion
140  // - save new group with associated table versions
141  // . add map record of original groupName & groupKey to new groupName &
142  // groupKey
143  //
144  //
145  // -- in current-db after loop...
146  // - destroy and reload active groups
147  // -- insert new aliases for imported groups
148  // - should be basename+alias connection to (hop through maps) new groupName &
149  // groupKey
150  // -- insert new aliases for imported tables
151  // - should be basename+alias connection to (hop through maps) new tableName &
152  // tableVersion
153  // -- save new backbone tables and save new backbone group
154  // -- backup the file ConfigurationManager::ACTIVE_GROUPS_FILENAME with time
155  // -- activate the new backbone group
156  //
157 
158  //==============================================================================
159 
160  // create objects
161 
162  std::map<std::string /*importGroupAlias*/,
163  /*original*/ std::pair<std::string /*groupName*/, TableGroupKey>>
164  originalGroupAliasMap;
165 
166  std::map</*original*/ std::pair<std::string /*groupName*/, TableGroupKey>,
167  /*new*/ TableGroupKey>
168  groupSet;
169 
170  std::map<std::string /*importTableAlias*/,
171  /*original*/ std::pair<std::string /*tableName*/, TableVersion>>
172  originalTableAliasMap;
173 
174  std::map</*original*/ std::pair<std::string /*tableName*/, TableVersion>,
175  /*new*/ TableVersion>
176  newTableVersionMap;
177 
178  std::map</*original*/ std::pair<std::string, TableGroupKey>,
179  std::string /*error string*/>
180  groupErrors;
181 
182  // get prepared with initial source db
183 
184  // ConfigurationManager instance immediately loads active groups
185  __COUT__ << "Getting started..." << std::endl;
186  ConfigurationManagerRW cfgMgrInst("import_aliases");
187  ConfigurationManagerRW* cfgMgr = &cfgMgrInst;
188  bool importedDbInPlace = false;
189 
190  __COUT__ << "Configuration manager initialized." << __E__;
191 
192  std::string nowTime = std::to_string(time(0));
193  std::string currentDir = __ENV__("ARTDAQ_DATABASE_URI");
194 
195  if(currentDir.find("filesystemdb://") != 0)
196  {
197  __SS__ << "filesystemdb:// was not found in $ARTDAQ_DATABASE_URI!" << std::endl;
198  __COUT_ERR__ << "\n" << ss.str();
199  __SS_THROW__;
200  }
201 
202  currentDir = currentDir.substr(std::string("filesystemdb://").length());
203  while(currentDir.length() &&
204  currentDir[currentDir.length() - 1] == '/') // remove trailing '/'s
205  currentDir = currentDir.substr(0, currentDir.length() - 1);
206 
207  __COUTV__(currentDir);
208 
209  std::string backupDir = currentDir + "_" + nowTime;
210  std::string importDir = pathToImportDatabase + "_" + nowTime;
211  std::string tmpCurrentDir = currentDir + "_tmp_" + nowTime;
212  // std::string tmpImportDir = pathToImportDatabase + "_tmp_" + nowTime;
213 
214  // -- get set of existing aliases
215  std::map<std::string /*table name*/,
216  std::map<std::string /*version alias*/, TableVersion /*aliased version*/>>
217  existingTableAliases = cfgMgr->ConfigurationManager::getVersionAliases();
218  std::map<std::string /*alias*/, std::pair<std::string /*group name*/, TableGroupKey>>
219  existingGroupAliases = cfgMgr->getActiveGroupAliases();
220 
221  // -- swap to import-db and clear cache
222  {
223  // back up current directory now
224  __COUT__ << "Backing up current database at '" << currentDir << "' to '"
225  << backupDir << "'" << __E__;
226  std::system(("cp -r " + currentDir + " " + backupDir).c_str());
227 
228  __COUT__ << "Backing up current database at '" << pathToImportDatabase << "' to '"
229  << importDir << "'" << __E__;
230  std::system(("cp -r " + pathToImportDatabase + " " + importDir).c_str());
231 
232  cfgMgr->destroy();
233 
234  __COUT__ << "Swap to import-db" << std::endl;
235  if(rename(currentDir.c_str(), tmpCurrentDir.c_str()) < 0)
236  {
237  __SS__ << "Problem!" << std::endl;
238  __SS_THROW__;
239  }
240  if(rename(importDir.c_str(), currentDir.c_str()) < 0)
241  {
242  __SS__ << "Problem!" << std::endl;
243  __SS_THROW__;
244  }
245  importedDbInPlace = true;
246  }
247 
248  try
249  {
250  // - get active groups from user data file
251  cfgMgr->restoreActiveTableGroups(true /*throwErrors*/, pathToImportActiveGroups);
252 
253  // add active groups to set
254  std::map<std::string, std::pair<std::string, TableGroupKey>> activeGroupsMap =
255  cfgMgr->getActiveTableGroups();
256 
257  for(const auto& activeGroup : activeGroupsMap)
258  {
259  if(activeGroup.second.second.isInvalid())
260  continue;
261  if(activeGroup.second.first == "")
262  continue;
263 
264  __COUTV__(activeGroup.second.first);
265  __COUTV__(activeGroup.second.second);
266 
267  groupSet.insert(
268  std::pair<std::pair<std::string, TableGroupKey>, TableGroupKey>(
269  std::pair<std::string, TableGroupKey>(activeGroup.second.first,
270  activeGroup.second.second),
271  TableGroupKey()));
272  }
273 
274  // add system alias groups to set
275  std::map<std::string, TableVersion> activeVersions = cfgMgr->getActiveVersions();
276  if(activeVersions.find(groupAliasesTableName) == activeVersions.end())
277  {
278  __SS__ << "\nActive version of " << groupAliasesTableName << " missing! "
279  << groupAliasesTableName
280  << " is a required member of the Backbone configuration group."
281  << "\n\nLikely you need to activate a valid Backbone group."
282  << std::endl;
283  __SS_THROW__;
284  }
285  if(activeVersions.find(versionAliasesTableName) == activeVersions.end())
286  {
287  __SS__ << "\nActive version of " << versionAliasesTableName << " missing! "
288  << versionAliasesTableName
289  << " is a required member of the Backbone configuration group."
290  << "\n\nLikely you need to activate a valid Backbone group."
291  << std::endl;
292  __SS_THROW__;
293  }
294 
295  std::vector<std::pair<std::string, ConfigurationTree>> aliasNodePairs =
296  cfgMgr->getNode(groupAliasesTableName).getChildren();
297  std::string aliasName;
298  for(auto& aliasPair : aliasNodePairs)
299  {
300  if(TableGroupKey(aliasPair.second.getNode("GroupKey").getValueAsString())
301  .isInvalid())
302  continue;
303  if(aliasPair.second.getNode("GroupName").getValueAsString() == "")
304  continue;
305 
306  aliasName = aliasPair.second.getNode("GroupKeyAlias").getValueAsString();
307  if(aliasName == "")
308  continue;
309 
310  if(aliasName[0] >= 'a' && aliasName[0] <= 'z') // capitalize the name
311  aliasName[0] -= 32;
312  aliasName = prependBaseName + aliasName;
313  __COUTV__(aliasName);
314  __COUTV__(aliasPair.second.getNode("GroupName").getValueAsString());
315  __COUTV__(aliasPair.second.getNode("GroupKey").getValueAsString());
316 
317  // . check basename+alias for collision with existing group aliases
318  // and throw error if collision
319  if(existingGroupAliases.find(aliasName) != existingGroupAliases.end())
320  {
321  __SS__ << "Conflicting group alias '" << aliasName << "' found!" << __E__;
322  __SS_THROW__;
323  }
324 
325  groupSet.insert(
326  std::pair<std::pair<std::string, TableGroupKey>, TableGroupKey>(
327  std::pair<std::string, TableGroupKey>(
328  aliasPair.second.getNode("GroupName").getValueAsString(),
329  TableGroupKey(
330  aliasPair.second.getNode("GroupKey").getValueAsString())),
331  TableGroupKey()));
332 
333  originalGroupAliasMap[aliasName] = std::pair<std::string, TableGroupKey>(
334  aliasPair.second.getNode("GroupName").getValueAsString(),
335  TableGroupKey(aliasPair.second.getNode("GroupKey").getValueAsString()));
336  } // end group aliases loop
337 
338  aliasNodePairs = cfgMgr->getNode(versionAliasesTableName).getChildren();
339  for(auto& aliasPair : aliasNodePairs)
340  {
341  if(TableVersion(aliasPair.second.getNode("Version").getValueAsString())
342  .isInvalid())
343  continue;
344  if(aliasPair.second.getNode("TableName").getValueAsString() == "")
345  continue;
346  aliasName = aliasPair.second.getNode("VersionAlias").getValueAsString();
347  if(aliasName == "")
348  continue;
349 
350  if(aliasName[0] >= 'a' && aliasName[0] <= 'z') // capitalize the name
351  aliasName[0] -= 32;
352  aliasName = prependBaseName + aliasName;
353  __COUTV__(aliasPair.second.getNode("TableName").getValueAsString());
354  __COUTV__(aliasName);
355  __COUTV__(aliasPair.second.getNode("Version").getValueAsString());
356 
357  // . check basename+alias for collision with existing table aliases
358  // and throw error if collision
359  if(existingTableAliases.find(aliasName) != existingTableAliases.end())
360  {
361  __SS__ << "Conflicting table version alias '" << aliasName << "' found!"
362  << __E__;
363  __SS_THROW__;
364  }
365 
366  originalTableAliasMap[aliasName] = std::pair<std::string, TableVersion>(
367  aliasPair.second.getNode("TableName").getValueAsString(),
368  TableVersion(aliasPair.second.getNode("Version").getValueAsString()));
369  } // end table aliases loop
370  } // end extract group set
371  catch(const std::runtime_error& e)
372  {
373  __COUT_ERR__ << "There was a fatal error: " << e.what() << __E__;
374 
375  __COUT__ << "Run the following to return to your previous database structure:"
376  << std::endl;
377  __COUT__ << "\t otsdaq_flatten_system_aliases -1 " << backupDir << "\n\n"
378  << std::endl;
379 
380  return;
381  }
382 
383  __COUTV__(StringMacros::mapToString(existingGroupAliases));
384  __COUTV__(StringMacros::mapToString(existingTableAliases));
385 
386  __COUT__ << "Identified groups:" << std::endl;
387  for(auto& group : groupSet)
388  __COUT__ << "\t" << group.first.first << " " << group.first.second << std::endl;
389 
390  __COUT__ << "Identified group aliases:" << std::endl;
391  for(auto& groupAlias : originalGroupAliasMap)
392  __COUT__ << "\t" << groupAlias.first << " ==> " << groupAlias.second.first << "-"
393  << groupAlias.second.second << std::endl;
394 
395  //==============================================================================
396  // -- for each group in set
397 
398  ConfigurationInterface* theInterface_ = ConfigurationInterface::getInstance(
399  false); // true for File interface, false for artdaq database;
400  TableView* cfgView;
401  TableBase* config;
402  TableVersion newVersion;
403  TableGroupKey newKey;
404 
405  bool errDetected;
406  std::string accumulateErrors = "";
407  int count = 0;
408 
409  std::map<std::string, TableVersion> memberMap;
410  std::map<std::string /*name*/, std::string /*alias*/> groupAliases;
411  std::string groupComment;
412  std::string groupAuthor;
413  std::string groupCreateTime;
414  time_t groupCreateTime_t;
415  TableBase* groupMetadataTable = cfgMgr->getMetadataTable();
416 
417  __COUT__ << "Proceeding with handling of identified groups..." << __E__;
418 
419  // -- for each group in set
420  for(auto& groupPair : groupSet)
421  {
422  cfgMgr->destroy();
423 
424  errDetected = false;
425 
426  //- swap to import-db (first time already in import-db
427  //- load/activate group
428 
429  __COUTV__(importedDbInPlace);
430  // -- swap to import-db
431  if(!importedDbInPlace)
432  {
433  __COUT__ << "Swap to import-db" << std::endl;
434  if(rename(currentDir.c_str(), tmpCurrentDir.c_str()) < 0)
435  {
436  __SS__ << "Problem!" << std::endl;
437  __SS_THROW__;
438  }
439  if(rename(importDir.c_str(), currentDir.c_str()) < 0)
440  {
441  __SS__ << "Problem!" << std::endl;
442  __SS_THROW__;
443  }
444  importedDbInPlace = true;
445  }
446 
447  // cfgMgr->restoreActiveTableGroups(true
448  // /*throwErrors*/,pathToImportActiveGroups);
449 
450  __COUT__ << "****************************" << std::endl;
451  __COUT__ << "Loading members for " << groupPair.first.first << "("
452  << groupPair.first.second << ")" << std::endl;
453  __COUTV__(count);
454 
455  //=========================
456  // load group, group metadata, and tables from original DB
457  try
458  {
459  cfgMgr->loadTableGroup(groupPair.first.first,
460  groupPair.first.second,
461  true /*doActivate*/,
462  &memberMap /*memberMap*/,
463  0 /*progressBar*/,
464  &accumulateErrors,
465  &groupComment,
466  &groupAuthor,
467  &groupCreateTime,
468  false /*doNotLoadMember*/,
469  0 /*groupTypeString*/,
470  &groupAliases);
471  }
472  catch(std::runtime_error& e)
473  {
474  __COUT__ << "Error was caught loading members for " << groupPair.first.first
475  << "(" << groupPair.first.second << ")" << std::endl;
476  __COUT__ << e.what() << std::endl;
477  errDetected = true;
478  }
479  catch(...)
480  {
481  __COUT__ << "Error was caught loading members for " << groupPair.first.first
482  << "(" << groupPair.first.second << ")" << std::endl;
483  errDetected = true;
484  }
485 
486  //=========================
487 
488  // if(count == 2) break;
489 
490  // exit loop if any (loading) failure
491  if(errDetected)
492  {
493  // goto CLEAN_UP;
494 
495  // power on if group failed
496  // and record error
497 
498  groupErrors.insert(
499  std::pair<std::pair<std::string, TableGroupKey>, std::string>(
500  std::pair<std::string, TableGroupKey>(groupPair.first.first,
501  groupPair.first.second),
502  "Error caught loading the group."));
503  continue;
504  }
505 
506  // -- swap to current-db
507  if(importedDbInPlace)
508  {
509  __COUT__ << "Swap to current-db" << std::endl;
510  if(rename(currentDir.c_str(), importDir.c_str()) < 0)
511  {
512  __SS__ << "Problem!" << std::endl;
513  __SS_THROW__;
514  }
515  if(rename(tmpCurrentDir.c_str(), currentDir.c_str()) < 0)
516  {
517  __SS__ << "Problem!" << std::endl;
518  __SS_THROW__;
519  }
520  importedDbInPlace = false;
521  }
522 
523  //=========================
524  // save group and its tables with new key and versions!
525  try
526  {
527  // saving tables
528  for(auto& memberPair : memberMap)
529  {
530  __COUT__ << memberPair.first << ":v" << memberPair.second << std::endl;
531 
532  // check if table has already been modified by a previous group
533  // (i.e. two groups using the same version of a table)
534  if(newTableVersionMap.find(std::pair<std::string, TableVersion>(
535  memberPair.first, memberPair.second)) != newTableVersionMap.end())
536  {
537  __COUT__ << "Table was already modified!" << std::endl;
538  memberPair.second =
539  newTableVersionMap[std::pair<std::string, TableVersion>(
540  memberPair.first, memberPair.second)];
541  __COUT__ << "\t to...\t" << memberPair.first << ":v"
542  << memberPair.second << std::endl;
543  continue;
544  }
545 
546  // change the version of the active view to next available version and
547  // save it
548  config = cfgMgr->getTableByName(memberPair.first);
549  cfgView = config->getViewP();
550  // newVersion = theInterface_->saveNewVersion(config, temporaryVersion);
551  newVersion = TableVersion::getNextVersion(
552  theInterface_->findLatestVersion(config));
553  __COUTV__(newVersion);
554  cfgView->setVersion(newVersion);
555  theInterface_->saveActiveVersion(config);
556 
557  // set it back for the table so that future groups can re-use cached
558  // version
559  // FIXME -- RAR note: I do not understand why this was important.. seems
560  // like it will not help since the cache is destroyed for each group
561  // cfgView->setVersion(memberPair.second); //IMPORTANT
562 
563  // save new version to modifiedTables
564  newTableVersionMap.insert(
565  std::pair<std::pair<std::string, TableVersion>, TableVersion>(
566  std::pair<std::string, TableVersion>(memberPair.first,
567  memberPair.second),
568  newVersion));
569 
570  memberPair.second = newVersion; // change version in the member map
571 
572  __COUT__ << "\t to...\t" << memberPair.first << ":v" << memberPair.second
573  << std::endl;
574  } // end member map loop
575 
576  __COUT__ << "Member map completed" << __E__;
577  __COUTV__(StringMacros::mapToString(memberMap));
578 
579  // Note: this code copies actions in ConfigurationManagerRW::saveNewTableGroup
580 
581  // add meta data
582  __COUTV__(StringMacros::mapToString(groupAliases));
583  __COUTV__(groupComment);
584  __COUTV__(groupAuthor);
585  __COUTV__(groupCreateTime);
586  sscanf(groupCreateTime.c_str(), "%ld", &groupCreateTime_t);
587  __COUTV__(groupCreateTime_t);
588 
589  // to compensate for unusual errors upstream, make sure the metadata table has
590  // one row
591  while(groupMetadataTable->getViewP()->getNumberOfRows() > 1)
592  groupMetadataTable->getViewP()->deleteRow(0);
593  if(groupMetadataTable->getViewP()->getNumberOfRows() == 0)
594  groupMetadataTable->getViewP()->addRow();
595 
596  // columns are uid,comment,author,time
597  // ConfigurationManager::METADATA_COL_ALIASES TODO
598  groupMetadataTable->getViewP()->setValue(
599  StringMacros::mapToString(
600  groupAliases, "," /*primary delimiter*/, ":" /*secondary delimeter*/),
601  0,
602  ConfigurationManager::METADATA_COL_ALIASES);
603  groupMetadataTable->getViewP()->setValue(
604  groupComment, 0, ConfigurationManager::METADATA_COL_COMMENT);
605  groupMetadataTable->getViewP()->setValue(
606  groupAuthor, 0, ConfigurationManager::METADATA_COL_AUTHOR);
607  groupMetadataTable->getViewP()->setValue(
608  groupCreateTime_t, 0, ConfigurationManager::METADATA_COL_TIMESTAMP);
609 
610  // set version of metadata table
611  newVersion = TableVersion::getNextVersion(
612  theInterface_->findLatestVersion(groupMetadataTable));
613  __COUTV__(newVersion);
614  groupMetadataTable->getViewP()->setVersion(newVersion);
615  theInterface_->saveActiveVersion(groupMetadataTable);
616 
617  // force groupMetadataTable_ to be a member for the group
618  memberMap[groupMetadataTable->getTableName()] =
619  groupMetadataTable->getViewVersion();
620 
621  // memberMap should now consist of members with new flat version, so save
622  // group
623  newKey = TableGroupKey::getNextKey(
624  theInterface_->findLatestGroupKey(groupPair.first.first));
625 
626  __COUTV__(newKey);
627 
628  // memberMap should now consist of members with new flat version, so save
629  theInterface_->saveTableGroup(
630  memberMap,
631  TableGroupKey::getFullGroupString(groupPair.first.first, newKey));
632 
633  // and modify groupSet and activeGroupKeys keys
634  groupPair.second = newKey;
635  }
636  catch(std::runtime_error& e)
637  {
638  __COUT__ << "Error was caught saving group " << groupPair.first.first << " ("
639  << groupPair.first.second << ") " << std::endl;
640  __COUT__ << e.what() << std::endl;
641 
642  groupErrors.insert(
643  std::pair<std::pair<std::string, TableGroupKey>, std::string>(
644  std::pair<std::string, TableGroupKey>(groupPair.first.first,
645  groupPair.first.second),
646  "Error caught saving the group."));
647  }
648  catch(...)
649  {
650  __COUT__ << "Error was caught saving group " << groupPair.first.first << " ("
651  << groupPair.first.second << ") " << std::endl;
652 
653  groupErrors.insert(
654  std::pair<std::pair<std::string, TableGroupKey>, std::string>(
655  std::pair<std::string, TableGroupKey>(groupPair.first.first,
656  groupPair.first.second),
657  "Error caught saving the group."));
658  }
659  //=========================
660 
661  // increment
662  ++count;
663  } // end group loop
664 
665  __COUT__ << "Completed group and table saving for " << count << " groups." << __E__;
666  __COUT__ << "Created tables:" << std::endl;
667  for(auto& tablePair : newTableVersionMap)
668  __COUT__ << "\t" << tablePair.first.first << "-v" << tablePair.first.second
669  << " ==> " << tablePair.second << std::endl;
670 
671  __COUT__ << "Created groups:" << std::endl;
672  for(auto& group : groupSet)
673  __COUT__ << "\t" << group.first.first << "(" << group.first.second << ") ==> "
674  << group.second << std::endl;
675 
676  // -- in current-db after loop...
677  // -- swap to current-db
678  if(importedDbInPlace)
679  {
680  __COUT__ << "Swap to current-db" << std::endl;
681  if(rename(currentDir.c_str(), importDir.c_str()) < 0)
682  {
683  __SS__ << "Problem!" << std::endl;
684  __SS_THROW__;
685  }
686  if(rename(tmpCurrentDir.c_str(), currentDir.c_str()) < 0)
687  {
688  __SS__ << "Problem!" << std::endl;
689  __SS_THROW__;
690  }
691  importedDbInPlace = false;
692  }
693 
694  // record in readme for current-db
695  {
696  FILE* fp = fopen((currentDir + "/README_otsdaq_import.txt").c_str(), "a");
697 
698  if(!fp)
699  __COUT__ << "\tError opening README file!" << std::endl;
700  else
701  {
702  time_t rawtime;
703  struct tm* timeinfo;
704  char buffer[200];
705 
706  time(&rawtime);
707  timeinfo = localtime(&rawtime);
708  strftime(buffer, 200, "%b %d, %Y %I:%M:%S%p %Z", timeinfo);
709 
710  fprintf(fp,
711  "This database...\t %s \t received an import from...\t %s \t at this "
712  "time \t %lu \t %s\n\n",
713  currentDir.c_str(),
714  pathToImportDatabase.c_str(),
715  time(0),
716  buffer);
717  fclose(fp);
718  }
719  }
720 
721  // -- in current-db after loop...
722  // - destroy and reload active groups
723  cfgMgr->destroy();
724  cfgMgr->restoreActiveTableGroups(true /*throwErrors*/);
725 
726  // -- insert new aliases for imported groups
727  // - should be basename+alias connection to (hop through maps) new groupName &
728  // groupKey
729  // -- insert new aliases for imported tables
730  // - should be basename+alias connection to (hop through maps) new tableName &
731  // tableVersion
732  // -- save new backbone tables and save new backbone group
733 
734  __COUT__ << "Modifying the active Backbone table to reflect new table versions and "
735  "group keys."
736  << std::endl;
737 
738  try
739  {
740  // modify Group Aliases Table and Version Aliases Table to
741  // include new groups and tables
742 
743  std::string activeBackboneGroupName =
744  cfgMgr->getActiveGroupName(ConfigurationManager::ACTIVE_GROUP_NAME_BACKBONE);
745  cfgMgr->loadTableGroup(
746  activeBackboneGroupName,
747  cfgMgr->getActiveGroupKey(ConfigurationManager::ACTIVE_GROUP_NAME_BACKBONE),
748  true,
749  &memberMap,
750  0,
751  &accumulateErrors);
752 
753  std::map<std::string, TableVersion> activeMap = cfgMgr->getActiveVersions();
754 
755  // modify Group Aliases Table
756  if(activeMap.find(groupAliasesTableName) != activeMap.end())
757  {
758  __COUT__ << "\n\nModifying " << groupAliasesTableName << std::endl;
759  config = cfgMgr->getTableByName(groupAliasesTableName);
760  cfgView = config->getViewP();
761 
762  unsigned int col0 = cfgView->findCol("GroupKeyAlias");
763  unsigned int col1 = cfgView->findCol("GroupName");
764  unsigned int col2 = cfgView->findCol("GroupKey");
765  unsigned int row;
766 
767  cfgView->print();
768 
769  // -- insert new aliases for imported groups
770  // - should be basename+alias connection to (hop through maps) new
771  // groupName & groupKey
772  for(auto& aliasPair : originalGroupAliasMap)
773  {
774  auto groupIt = groupSet.find(std::pair<std::string, TableGroupKey>(
775  aliasPair.second.first, aliasPair.second.second));
776 
777  if(groupIt == groupSet.end())
778  {
779  __COUT__
780  << "Error! Could not find the new entry for the original group "
781  << aliasPair.second.first << "(" << aliasPair.second.second << ")"
782  << __E__;
783  continue;
784  }
785  row = cfgView->addRow("import_aliases", true /*incrementUniqueData*/);
786  cfgView->setValue(aliasPair.first, row, col0);
787  cfgView->setValue(aliasPair.second.first, row, col1);
788  cfgView->setValue(groupIt->second.toString(), row, col2);
789  } // end group alias edit
790 
791  cfgView->print();
792  }
793 
794  // modify Version Aliases Table
795  if(activeMap.find(versionAliasesTableName) != activeMap.end())
796  {
797  __COUT__ << "\n\nModifying " << versionAliasesTableName << std::endl;
798  config = cfgMgr->getTableByName(versionAliasesTableName);
799  cfgView = config->getViewP();
800  unsigned int col0 = cfgView->findCol("VersionAlias");
801  unsigned int col1 = cfgView->findCol("TableName");
802  unsigned int col2 = cfgView->findCol("Version");
803 
804  unsigned int row;
805 
806  cfgView->print();
807 
808  // -- insert new aliases for imported tables
809  // - should be basename+alias connection to (hop through maps) new
810  // tableName & tableVersion
811  for(auto& aliasPair : originalTableAliasMap)
812  {
813  auto tableIt =
814  newTableVersionMap.find(std::pair<std::string, TableVersion>(
815  aliasPair.second.first, aliasPair.second.second));
816 
817  if(tableIt == newTableVersionMap.end())
818  {
819  __COUT__
820  << "Error! Could not find the new entry for the original table "
821  << aliasPair.second.first << "(" << aliasPair.second.second << ")"
822  << __E__;
823  continue;
824  }
825  row = cfgView->addRow("import_aliases", true /*incrementUniqueData*/);
826  cfgView->setValue(aliasPair.first, row, col0);
827  cfgView->setValue(aliasPair.second.first, row, col1);
828  cfgView->setValue(tableIt->second.toString(), row, col2);
829  } // end group alias edit
830 
831  cfgView->print();
832  }
833 
834  // save new Group Aliases Table and Version Aliases Table
835 
836  // change the version of the active view to flatVersion and save it
837  config = cfgMgr->getTableByName(groupAliasesTableName);
838  cfgView = config->getViewP();
839  newVersion =
840  TableVersion::getNextVersion(theInterface_->findLatestVersion(config));
841  __COUTV__(newVersion);
842  cfgView->setVersion(newVersion);
843  theInterface_->saveActiveVersion(config);
844 
845  memberMap[groupAliasesTableName] = newVersion; // change version in the member
846  // map
847 
848  __COUT__ << "\t to...\t" << groupAliasesTableName << ":v"
849  << memberMap[groupAliasesTableName] << std::endl;
850 
851  __COUT__ << versionAliasesTableName << ":v" << memberMap[versionAliasesTableName]
852  << std::endl;
853  // change the version of the active view to flatVersion and save it
854  config = cfgMgr->getTableByName(versionAliasesTableName);
855  cfgView = config->getViewP();
856  newVersion =
857  TableVersion::getNextVersion(theInterface_->findLatestVersion(config));
858  __COUTV__(newVersion);
859  cfgView->setVersion(newVersion);
860  theInterface_->saveActiveVersion(config);
861 
862  memberMap[versionAliasesTableName] =
863  newVersion; // change version in the member map
864 
865  __COUT__ << "\t to...\t" << versionAliasesTableName << ":v"
866  << memberMap[versionAliasesTableName] << std::endl;
867 
868  __COUT__ << "Backbone member map completed" << __E__;
869  __COUTV__(StringMacros::mapToString(memberMap));
870 
871  newKey = TableGroupKey::getNextKey(
872  theInterface_->findLatestGroupKey(activeBackboneGroupName));
873 
874  __COUTV__(newKey);
875 
876  // memberMap should now consist of members with new flat version, so save
877  theInterface_->saveTableGroup(
878  memberMap,
879  TableGroupKey::getFullGroupString(activeBackboneGroupName, newKey));
880 
881  std::string renameFile =
882  ConfigurationManager::ACTIVE_GROUPS_FILENAME + "." + nowTime;
883  rename(ConfigurationManager::ACTIVE_GROUPS_FILENAME.c_str(), renameFile.c_str());
884 
885  __COUT__ << "Backing up '" << ConfigurationManager::ACTIVE_GROUPS_FILENAME
886  << "' to ... '" << renameFile << "'" << std::endl;
887 
888  cfgMgr->activateTableGroup(activeBackboneGroupName,
889  newKey); // and write to active group file
890 
891  } // end try
892  catch(const std::runtime_error& e)
893  {
894  __COUT_ERR__ << "There was a fatal error during backbone modification: "
895  << e.what() << __E__;
896 
897  goto CLEAN_UP;
898  }
899 
900  // print resulting all groups
901 
902  std::cout << "\n\n" << __COUT_HDR_FL__ << "Resulting Groups:" << std::endl;
903  for(const auto& group : groupSet)
904  __COUT__ << "\t" << group.first.first << ": " << group.first.second << " => "
905  << group.second << std::endl;
906  std::cout << "\n\n" << __COUT_HDR_FL__ << "Resulting Groups end." << std::endl;
907 
908 CLEAN_UP:
909  //==============================================================================
910  __COUT__ << "End of Importing Active Table Groups!\n\n\n" << std::endl;
911 
912  __COUT__ << "****************************" << std::endl;
913  __COUT__ << "There were " << groupSet.size() << " groups considered, and there were "
914  << groupErrors.size() << " errors found handling those groups." << std::endl;
915  __COUT__ << "The following errors were found handling the groups:" << std::endl;
916  for(auto& groupErr : groupErrors)
917  __COUT__ << "\t" << groupErr.first.first << " " << groupErr.first.second << ": \t"
918  << groupErr.second << std::endl;
919  __COUT__ << "End of errors.\n\n" << std::endl;
920 
921  __COUT__ << "Run the following to return to your previous database structure:"
922  << std::endl;
923  __COUT__ << "\t otsdaq_flatten_system_aliases -1 " << backupDir << "\n\n"
924  << std::endl;
925 
926  return;
927 }
928 
929 int main(int argc, char* argv[])
930 {
931  auto pset = artdaq::LoadParameterSet(argc,
932  argv,
933  "otsdaq_import_fhicl",
934  "Import a ParameterSet to the otsdaq database");
935  ImportSystemAliasTableGroups(pset);
936  return 0;
937 }
938 // BOOST_AUTO_TEST_SUITE_END()