otsdaq  v2_03_00
otsdaq_import_system_aliases.cc
1 #include <dirent.h>
2 #include <cassert>
3 #include <iostream>
4 #include <memory>
5 #include <string>
6 #include "otsdaq-core/ConfigurationInterface/ConfigurationInterface.h"
7 #include "otsdaq-core/ConfigurationInterface/ConfigurationManagerRW.h"
8 
9 // usage:
10 // otsdaq_import_system_aliases <pathOfDatabaseToImport> <(optional) prependLabel>
11 //
12 // all system aliases are imported to current db and are prepended with a label
13 //
14 
15 using namespace ots;
16 
17 void ImportSystemAliasTableGroups(int argc, char* argv[])
18 {
19  __COUT__ << "=================================================\n";
20  __COUT__ << "=================================================\n";
21  __COUT__ << "=================================================\n";
22  __COUT__ << "Importing External System Aliases!" << std::endl;
23 
24  std::string prependBaseName = "Imported";
25  const std::string groupAliasesTableName =
26  ConfigurationManager::GROUP_ALIASES_TABLE_NAME;
27  const std::string versionAliasesTableName =
28  ConfigurationManager::VERSION_ALIASES_TABLE_NAME;
29 
30  std::cout << "\n\nusage: Two arguments:\n\n\t otsdaq_import_system_aliases "
31  "<path_to_import_database_folder> <path_to_active_groups_file> "
32  "<import_prepend_base_name (optional)> \n\n"
33  << "\t\t Default values: \n\t\t\timport_prepend_base_name = \""
34  << prependBaseName << "\" "
35  << "\n\n"
36  << "\t\tfor example:\n\n"
37  << "\t\t\totsdaq_import_system_aliases "
38  "~/databaseToImport/filesystemdb/test_db "
39  "~/UserDataToImport/ServiceData/ActiveTableGroups.cfg"
40  << __E__;
41 
42  std::cout << "\n\n\tExample active groups file content:\n\n"
43  << "testContext\n2\nTableEditWizBackbone\n3\ndefaultConfig\n1\n\n"
44  << __E__;
45 
46  std::cout << "\n\nNote: This assumes artdaq db file type interface. "
47  << "The current database/ will be backed up to database_<linuxtime>/ "
48  << "before importing the active groups.\n\n"
49  << __E__;
50 
51  __COUTV__(argc);
52  for(int i = 0; i < argc; i++)
53  std::cout << "argv[" << i << "] = " << argv[i] << std::endl;
54 
55  if(argc < 3)
56  {
57  __COUT__ << "Must provide at least two parameters (the database and active "
58  "groups files for the import).\n\n\n"
59  << __E__;
60  return;
61  }
62 
63  // remove trailing /'s
64  while(strlen(argv[1]) && argv[1][strlen(argv[1]) - 1] == '/')
65  argv[1][strlen(argv[1]) - 1] = '\0';
66 
67  // determine if "h"elp was first parameter
68  std::string pathToImportDatabase = argv[1];
69  if(pathToImportDatabase.size() &&
70  (pathToImportDatabase[0] == 'h' || pathToImportDatabase[0] == '-'))
71  {
72  __COUT__
73  << "Recognized parameter 1. as a 'help' option. Usage was printed. Exiting."
74  << std::endl;
75  return;
76  }
77 
78  std::string pathToImportActiveGroups = "";
79  if(argc >= 3)
80  pathToImportActiveGroups = argv[2];
81  if(argc >= 4)
82  prependBaseName = argv[3];
83 
84  __COUTV__(pathToImportDatabase);
85  __COUTV__(pathToImportActiveGroups);
86  __COUTV__(prependBaseName);
87 
88  //==============================================================================
89  // Define environment variables
90  // Note: normally these environment variables are set by StartOTS.sh
91 
92  // These are needed by
93  // otsdaq/otsdaq-core/ConfigurationDataFormats/ConfigurationInfoReader.cc [207]
94  setenv("CONFIGURATION_TYPE", "File", 1); // Can be File, Database, DatabaseTest
95  setenv("CONFIGURATION_DATA_PATH",
96  (std::string(getenv("USER_DATA")) + "/ConfigurationDataExamples").c_str(),
97  1);
98  setenv(
99  "TABLE_INFO_PATH", (std::string(getenv("USER_DATA")) + "/TableInfo").c_str(), 1);
101 
102  // Some configuration plug-ins use getenv("SERVICE_DATA_PATH") in init() so define it
103  setenv("SERVICE_DATA_PATH",
104  (std::string(getenv("USER_DATA")) + "/ServiceData").c_str(),
105  1);
106 
107  // Some configuration plug-ins use getenv("OTSDAQ_LIB") and
108  // getenv("OTSDAQ_UTILITIES_LIB") in init() so define it to a non-sense place is ok
109  setenv("OTSDAQ_LIB", (std::string(getenv("USER_DATA")) + "/").c_str(), 1);
110  setenv("OTSDAQ_UTILITIES_LIB", (std::string(getenv("USER_DATA")) + "/").c_str(), 1);
111 
112  // Some configuration plug-ins use getenv("OTS_MAIN_PORT") in init() so define it
113  setenv("OTS_MAIN_PORT", "2015", 1);
114 
115  // also xdaq envs for XDAQContextTable
116  setenv("XDAQ_CONFIGURATION_DATA_PATH",
117  (std::string(getenv("USER_DATA")) + "/XDAQConfigurations").c_str(),
118  1);
119  setenv("XDAQ_CONFIGURATION_XML", "otsConfigurationNoRU_CMake", 1);
121 
122  //==============================================================================
123 
124  // Steps:
125  //
126  // -- create empty map of import alias to original groupName & groupKey
127  // -- create empty map of original groupName & groupKey to new groupKey
128  //
129  // -- create empty map of import alias to original tableName & tableVersion
130  // -- create empty map of original tableName & tableVersion to new tableVersion
131  //
132  // -- in current-db extract info
133  // - get set of existing group aliases
134  // . load current aliases from current-db
135  // - get set of existing table aliases
136  // . load current aliases from current-db
137  //
138  // -- swap to import-db, clear cache, and create vector of groups to import
139  // - get active groups from user data file
140  // - get aliases from active backbone group
141  // . check basename+alias for collision with existing group aliases and throw
142  // error if collision
143  // . add map record of basename+alias to original groupName & groupKey
144  // . check basename+alias for collision with existing table aliases and throw
145  // error if collision
146  // . add map record of basename+alias to original tableName & tableVersion
147  //
148  // -- for each group in set
149  // - swap to import-db
150  // - load/activate group
151  // - swap to current-db
152  // - for each member table in group
153  // . check map of original tableName & tableVersion to new tableName &
154  // tableVersion, to prevent making a new table
155  // . if needed, save active tables as next table version (using interface)
156  // , add map record of original tableName & tableVersion to new tableName
157  //& tableVersion
158  // - save new group with associated table versions
159  // . add map record of original groupName & groupKey to new groupName &
160  // groupKey
161  //
162  //
163  // -- in current-db after loop...
164  // - destroy and reload active groups
165  // -- insert new aliases for imported groups
166  // - should be basename+alias connection to (hop through maps) new groupName &
167  // groupKey
168  // -- insert new aliases for imported tables
169  // - should be basename+alias connection to (hop through maps) new tableName &
170  // tableVersion
171  // -- save new backbone tables and save new backbone group
172  // -- backup the file ConfigurationManager::ACTIVE_GROUPS_FILENAME with time
173  // -- activate the new backbone group
174  //
175 
176  //==============================================================================
177 
178  // create objects
179 
180  // std::map<std::string /*tableName*/,
181  // TableVersion /*next persisten version*/>
182  // tableNextVersionMap;
183 
184  std::map<std::string /*importGroupAlias*/,
185  /*original*/ std::pair<std::string /*groupName*/, TableGroupKey>>
186  originalGroupAliasMap;
187 
188  std::map</*original*/ std::pair<std::string /*groupName*/, TableGroupKey>,
189  /*new*/ TableGroupKey>
190  groupSet;
191 
192  std::map<std::string /*importTableAlias*/,
193  /*original*/ std::pair<std::string /*tableName*/, TableVersion>>
194  originalTableAliasMap;
195 
196  std::map</*original*/ std::pair<std::string /*tableName*/, TableVersion>,
197  /*new*/ TableVersion>
198  newTableVersionMap;
199 
200  std::map</*original*/ std::pair<std::string, TableGroupKey>,
201  std::string /*error string*/>
202  groupErrors;
203 
204  // get prepared with initial source db
205 
206  // ConfigurationManager instance immediately loads active groups
207  __COUT__ << "Getting started..." << std::endl;
208  ConfigurationManagerRW cfgMgrInst("import_aliases");
209  ConfigurationManagerRW* cfgMgr = &cfgMgrInst;
210  bool importedDbInPlace = false;
211 
212  __COUT__ << "Configuration manager initialized." << __E__;
213 
214  std::string nowTime = std::to_string(time(0));
215  std::string currentDir = getenv("ARTDAQ_DATABASE_URI");
216 
217  if(currentDir.find("filesystemdb://") != 0)
218  {
219  __SS__ << "filesystemdb:// was not found in $ARTDAQ_DATABASE_URI!" << std::endl;
220  __COUT_ERR__ << "\n" << ss.str();
221  __SS_THROW__;
222  }
223 
224  currentDir = currentDir.substr(std::string("filesystemdb://").length());
225  while(currentDir.length() &&
226  currentDir[currentDir.length() - 1] == '/') // remove trailing '/'s
227  currentDir = currentDir.substr(0, currentDir.length() - 1);
228 
229  __COUTV__(currentDir);
230 
231  std::string backupDir = currentDir + "_" + nowTime;
232  std::string importDir = pathToImportDatabase + "_" + nowTime;
233  std::string tmpCurrentDir = currentDir + "_tmp_" + nowTime;
234  // std::string tmpImportDir = pathToImportDatabase + "_tmp_" + nowTime;
235 
236  // -- get set of existing aliases
237  std::map<std::string /*table name*/,
238  std::map<std::string /*version alias*/, TableVersion /*aliased version*/>>
239  existingTableAliases = cfgMgr->ConfigurationManager::getVersionAliases();
240  std::map<std::string /*alias*/, std::pair<std::string /*group name*/, TableGroupKey>>
241  existingGroupAliases = cfgMgr->getActiveGroupAliases();
242 
243  // //NOT NEEDED -- just ask db on fly for next version // - fill map of table name
244  // to next persistent table version
245  // {
246  // const std::map<std::string, TableInfo>& allCfgInfo =
247  // cfgMgr->getAllConfigurationInfo(); for(auto& mapPair : allCfgInfo)
248  // {
249  // tableNextVersionMap[mapPair.first] =
250  // mapPair.second.versions_.size()?
251  // *(mapPair.second.versions_.rbegin()):
252  // TableVersion(TableVersion::DEFAULT);
253  // }
254  // }
255  //
256  // __COUTV__(StringMacros::mapToString(tableNextVersionMap));
257 
258  // -- swap to import-db and clear cache
259  {
260  // back up current directory now
261  __COUT__ << "Backing up current database at '" << currentDir << "' to '"
262  << backupDir << "'" << __E__;
263  std::system(("cp -r " + currentDir + " " + backupDir).c_str());
264 
265  __COUT__ << "Backing up current database at '" << pathToImportDatabase << "' to '"
266  << importDir << "'" << __E__;
267  std::system(("cp -r " + pathToImportDatabase + " " + importDir).c_str());
268 
269  cfgMgr->destroy();
270 
271  __COUT__ << "Swap to import-db" << std::endl;
272  if(rename(currentDir.c_str(), tmpCurrentDir.c_str()) < 0)
273  {
274  __SS__ << "Problem!" << std::endl;
275  __SS_THROW__;
276  }
277  if(rename(importDir.c_str(), currentDir.c_str()) < 0)
278  {
279  __SS__ << "Problem!" << std::endl;
280  __SS_THROW__;
281  }
282  importedDbInPlace = true;
283  }
284 
285  try
286  {
287  // - get active groups from user data file
288  cfgMgr->restoreActiveTableGroups(true /*throwErrors*/, pathToImportActiveGroups);
289 
290  // add active groups to set
291  std::map<std::string, std::pair<std::string, TableGroupKey>> activeGroupsMap =
292  cfgMgr->getActiveTableGroups();
293 
294  for(const auto& activeGroup : activeGroupsMap)
295  {
296  if(activeGroup.second.second.isInvalid())
297  continue;
298  if(activeGroup.second.first == "")
299  continue;
300 
301  __COUTV__(activeGroup.second.first);
302  __COUTV__(activeGroup.second.second);
303 
304  groupSet.insert(
305  std::pair<std::pair<std::string, TableGroupKey>, TableGroupKey>(
306  std::pair<std::string, TableGroupKey>(activeGroup.second.first,
307  activeGroup.second.second),
308  TableGroupKey()));
309  }
310 
311  // add system alias groups to set
312  std::map<std::string, TableVersion> activeVersions = cfgMgr->getActiveVersions();
313  if(activeVersions.find(groupAliasesTableName) == activeVersions.end())
314  {
315  __SS__ << "\nActive version of " << groupAliasesTableName << " missing! "
316  << groupAliasesTableName
317  << " is a required member of the Backbone configuration group."
318  << "\n\nLikely you need to activate a valid Backbone group."
319  << std::endl;
320  __SS_THROW__;
321  }
322  if(activeVersions.find(versionAliasesTableName) == activeVersions.end())
323  {
324  __SS__ << "\nActive version of " << versionAliasesTableName << " missing! "
325  << versionAliasesTableName
326  << " is a required member of the Backbone configuration group."
327  << "\n\nLikely you need to activate a valid Backbone group."
328  << std::endl;
329  __SS_THROW__;
330  }
331 
332  std::vector<std::pair<std::string, ConfigurationTree>> aliasNodePairs =
333  cfgMgr->getNode(groupAliasesTableName).getChildren();
334  std::string aliasName;
335  for(auto& aliasPair : aliasNodePairs)
336  {
337  if(TableGroupKey(aliasPair.second.getNode("GroupKey").getValueAsString())
338  .isInvalid())
339  continue;
340  if(aliasPair.second.getNode("GroupName").getValueAsString() == "")
341  continue;
342 
343  aliasName = aliasPair.second.getNode("GroupKeyAlias").getValueAsString();
344  if(aliasName == "")
345  continue;
346 
347  if(aliasName[0] >= 'a' && aliasName[0] <= 'z') // capitalize the name
348  aliasName[0] -= 32;
349  aliasName = prependBaseName + aliasName;
350  __COUTV__(aliasName);
351  __COUTV__(aliasPair.second.getNode("GroupName").getValueAsString());
352  __COUTV__(aliasPair.second.getNode("GroupKey").getValueAsString());
353 
354  // . check basename+alias for collision with existing group aliases
355  // and throw error if collision
356  if(existingGroupAliases.find(aliasName) != existingGroupAliases.end())
357  {
358  __SS__ << "Conflicting group alias '" << aliasName << "' found!" << __E__;
359  __SS_THROW__;
360  }
361 
362  groupSet.insert(
363  std::pair<std::pair<std::string, TableGroupKey>, TableGroupKey>(
364  std::pair<std::string, TableGroupKey>(
365  aliasPair.second.getNode("GroupName").getValueAsString(),
366  TableGroupKey(
367  aliasPair.second.getNode("GroupKey").getValueAsString())),
368  TableGroupKey()));
369 
370  originalGroupAliasMap[aliasName] = std::pair<std::string, TableGroupKey>(
371  aliasPair.second.getNode("GroupName").getValueAsString(),
372  TableGroupKey(aliasPair.second.getNode("GroupKey").getValueAsString()));
373  } // end group aliases loop
374 
375  aliasNodePairs = cfgMgr->getNode(versionAliasesTableName).getChildren();
376  for(auto& aliasPair : aliasNodePairs)
377  {
378  if(TableVersion(aliasPair.second.getNode("Version").getValueAsString())
379  .isInvalid())
380  continue;
381  if(aliasPair.second.getNode("TableName").getValueAsString() == "")
382  continue;
383  aliasName = aliasPair.second.getNode("VersionAlias").getValueAsString();
384  if(aliasName == "")
385  continue;
386 
387  if(aliasName[0] >= 'a' && aliasName[0] <= 'z') // capitalize the name
388  aliasName[0] -= 32;
389  aliasName = prependBaseName + aliasName;
390  __COUTV__(aliasPair.second.getNode("TableName").getValueAsString());
391  __COUTV__(aliasName);
392  __COUTV__(aliasPair.second.getNode("Version").getValueAsString());
393 
394  // . check basename+alias for collision with existing table aliases
395  // and throw error if collision
396  if(existingTableAliases.find(aliasName) != existingTableAliases.end())
397  {
398  __SS__ << "Conflicting table version alias '" << aliasName << "' found!"
399  << __E__;
400  __SS_THROW__;
401  }
402 
403  originalTableAliasMap[aliasName] = std::pair<std::string, TableVersion>(
404  aliasPair.second.getNode("TableName").getValueAsString(),
405  TableVersion(aliasPair.second.getNode("Version").getValueAsString()));
406  } // end table aliases loop
407  } // end extract group set
408  catch(const std::runtime_error& e)
409  {
410  __COUT_ERR__ << "There was a fatal error: " << e.what() << __E__;
411 
412  __COUT__ << std::endl;
413  __COUT__ << std::endl;
414  __COUT__ << "Run the following to return to your previous database structure:"
415  << std::endl;
416  __COUT__ << "\t otsdaq_flatten_system_aliases -1 " << backupDir << "\n\n"
417  << std::endl;
418  __COUT__ << std::endl;
419  __COUT__ << std::endl;
420  return;
421  }
422 
423  __COUTV__(StringMacros::mapToString(existingGroupAliases));
424  __COUTV__(StringMacros::mapToString(existingTableAliases));
425 
426  __COUT__ << std::endl;
427  __COUT__ << std::endl;
428  __COUT__ << "Identified groups:" << std::endl;
429  for(auto& group : groupSet)
430  __COUT__ << "\t" << group.first.first << " " << group.first.second << std::endl;
431  __COUT__ << std::endl;
432  __COUT__ << std::endl;
433 
434  __COUT__ << "Identified group aliases:" << std::endl;
435  for(auto& groupAlias : originalGroupAliasMap)
436  __COUT__ << "\t" << groupAlias.first << " ==> " << groupAlias.second.first << "-"
437  << groupAlias.second.second << std::endl;
438  __COUT__ << std::endl;
439  __COUT__ << std::endl;
440 
441  //==============================================================================
442  // -- for each group in set
443 
444  ConfigurationInterface* theInterface_ = ConfigurationInterface::getInstance(
445  false); // true for File interface, false for artdaq database;
446  TableView* cfgView;
447  TableBase* config;
448  TableVersion newVersion;
449  TableGroupKey newKey;
450 
451  bool errDetected;
452  std::string accumulateErrors = "";
453  int count = 0;
454 
455  std::map<std::string, TableVersion> memberMap;
456  std::map<std::string /*name*/, std::string /*alias*/> groupAliases;
457  std::string groupComment;
458  std::string groupAuthor;
459  std::string groupCreateTime;
460  time_t groupCreateTime_t;
461  TableBase* groupMetadataTable = cfgMgr->getMetadataTable();
462 
463  __COUT__ << "Proceeding with handling of identified groups..." << __E__;
464 
465  // -- for each group in set
466  for(auto& groupPair : groupSet)
467  {
468  cfgMgr->destroy();
469 
470  errDetected = false;
471 
472  //- swap to import-db (first time already in import-db
473  //- load/activate group
474 
475  __COUTV__(importedDbInPlace);
476  // -- swap to import-db
477  if(!importedDbInPlace)
478  {
479  __COUT__ << "Swap to import-db" << std::endl;
480  if(rename(currentDir.c_str(), tmpCurrentDir.c_str()) < 0)
481  {
482  __SS__ << "Problem!" << std::endl;
483  __SS_THROW__;
484  }
485  if(rename(importDir.c_str(), currentDir.c_str()) < 0)
486  {
487  __SS__ << "Problem!" << std::endl;
488  __SS_THROW__;
489  }
490  importedDbInPlace = true;
491  }
492 
493  // cfgMgr->restoreActiveTableGroups(true
494  // /*throwErrors*/,pathToImportActiveGroups);
495 
496  __COUT__ << "****************************" << std::endl;
497  __COUT__ << "Loading members for " << groupPair.first.first << "("
498  << groupPair.first.second << ")" << std::endl;
499  __COUTV__(count);
500 
501  //=========================
502  // load group, group metadata, and tables from original DB
503  try
504  {
505  cfgMgr->loadTableGroup(groupPair.first.first,
506  groupPair.first.second,
507  true /*doActivate*/,
508  &memberMap /*memberMap*/,
509  0 /*progressBar*/,
510  &accumulateErrors,
511  &groupComment,
512  &groupAuthor,
513  &groupCreateTime,
514  false /*doNotLoadMember*/,
515  0 /*groupTypeString*/,
516  &groupAliases);
517  }
518  catch(std::runtime_error& e)
519  {
520  __COUT__ << "Error was caught loading members for " << groupPair.first.first
521  << "(" << groupPair.first.second << ")" << std::endl;
522  __COUT__ << e.what() << std::endl;
523  errDetected = true;
524  }
525  catch(...)
526  {
527  __COUT__ << "Error was caught loading members for " << groupPair.first.first
528  << "(" << groupPair.first.second << ")" << std::endl;
529  errDetected = true;
530  }
531 
532  //=========================
533 
534  // if(count == 2) break;
535 
536  // exit loop if any (loading) failure
537  if(errDetected)
538  {
539  // goto CLEAN_UP;
540 
541  // power on if group failed
542  // and record error
543 
544  groupErrors.insert(
545  std::pair<std::pair<std::string, TableGroupKey>, std::string>(
546  std::pair<std::string, TableGroupKey>(groupPair.first.first,
547  groupPair.first.second),
548  "Error caught loading the group."));
549  continue;
550  }
551 
552  // -- swap to current-db
553  if(importedDbInPlace)
554  {
555  __COUT__ << "Swap to current-db" << std::endl;
556  if(rename(currentDir.c_str(), importDir.c_str()) < 0)
557  {
558  __SS__ << "Problem!" << std::endl;
559  __SS_THROW__;
560  }
561  if(rename(tmpCurrentDir.c_str(), currentDir.c_str()) < 0)
562  {
563  __SS__ << "Problem!" << std::endl;
564  __SS_THROW__;
565  }
566  importedDbInPlace = false;
567  }
568 
569  //=========================
570  // save group and its tables with new key and versions!
571  try
572  {
573  // saving tables
574  for(auto& memberPair : memberMap)
575  {
576  __COUT__ << memberPair.first << ":v" << memberPair.second << std::endl;
577 
578  // check if table has already been modified by a previous group
579  // (i.e. two groups using the same version of a table)
580  if(newTableVersionMap.find(std::pair<std::string, TableVersion>(
581  memberPair.first, memberPair.second)) != newTableVersionMap.end())
582  {
583  __COUT__ << "Table was already modified!" << std::endl;
584  memberPair.second =
585  newTableVersionMap[std::pair<std::string, TableVersion>(
586  memberPair.first, memberPair.second)];
587  __COUT__ << "\t to...\t" << memberPair.first << ":v"
588  << memberPair.second << std::endl;
589  continue;
590  }
591 
592  // change the version of the active view to next available version and
593  // save it
594  config = cfgMgr->getTableByName(memberPair.first);
595  cfgView = config->getViewP();
596  // newVersion = theInterface_->saveNewVersion(config, temporaryVersion);
597  newVersion = TableVersion::getNextVersion(
598  theInterface_->findLatestVersion(config));
599  __COUTV__(newVersion);
600  cfgView->setVersion(newVersion);
601  theInterface_->saveActiveVersion(config);
602 
603  // set it back for the table so that future groups can re-use cached
604  // version
605  // FIXME -- RAR note: I do not understand why this was important.. seems
606  // like it will not help since the cache is destroyed for each group
607  // cfgView->setVersion(memberPair.second); //IMPORTANT
608 
609  // save new version to modifiedTables
610  newTableVersionMap.insert(
611  std::pair<std::pair<std::string, TableVersion>, TableVersion>(
612  std::pair<std::string, TableVersion>(memberPair.first,
613  memberPair.second),
614  newVersion));
615 
616  memberPair.second = newVersion; // change version in the member map
617 
618  __COUT__ << "\t to...\t" << memberPair.first << ":v" << memberPair.second
619  << std::endl;
620  } // end member map loop
621 
622  __COUT__ << "Member map completed" << __E__;
623  __COUTV__(StringMacros::mapToString(memberMap));
624 
625  // Note: this code copies actions in ConfigurationManagerRW::saveNewTableGroup
626 
627  // add meta data
628  __COUTV__(StringMacros::mapToString(groupAliases));
629  __COUTV__(groupComment);
630  __COUTV__(groupAuthor);
631  __COUTV__(groupCreateTime);
632  sscanf(groupCreateTime.c_str(), "%ld", &groupCreateTime_t);
633  __COUTV__(groupCreateTime_t);
634 
635  // to compensate for unusual errors upstream, make sure the metadata table has
636  // one row
637  while(groupMetadataTable->getViewP()->getNumberOfRows() > 1)
638  groupMetadataTable->getViewP()->deleteRow(0);
639  if(groupMetadataTable->getViewP()->getNumberOfRows() == 0)
640  groupMetadataTable->getViewP()->addRow();
641 
642  // columns are uid,comment,author,time
643  // ConfigurationManager::METADATA_COL_ALIASES TODO
644  groupMetadataTable->getViewP()->setValue(
645  StringMacros::mapToString(
646  groupAliases, "," /*primary delimiter*/, ":" /*secondary delimeter*/),
647  0,
648  ConfigurationManager::METADATA_COL_ALIASES);
649  groupMetadataTable->getViewP()->setValue(
650  groupComment, 0, ConfigurationManager::METADATA_COL_COMMENT);
651  groupMetadataTable->getViewP()->setValue(
652  groupAuthor, 0, ConfigurationManager::METADATA_COL_AUTHOR);
653  groupMetadataTable->getViewP()->setValue(
654  groupCreateTime_t, 0, ConfigurationManager::METADATA_COL_TIMESTAMP);
655 
656  // set version of metadata table
657  newVersion = TableVersion::getNextVersion(
658  theInterface_->findLatestVersion(groupMetadataTable));
659  __COUTV__(newVersion);
660  groupMetadataTable->getViewP()->setVersion(newVersion);
661  theInterface_->saveActiveVersion(groupMetadataTable);
662 
663  // force groupMetadataTable_ to be a member for the group
664  memberMap[groupMetadataTable->getTableName()] =
665  groupMetadataTable->getViewVersion();
666 
667  // memberMap should now consist of members with new flat version, so save
668  // group
669  newKey = TableGroupKey::getNextKey(
670  theInterface_->findLatestGroupKey(groupPair.first.first));
671 
672  __COUTV__(newKey);
673 
674  // memberMap should now consist of members with new flat version, so save
675  theInterface_->saveTableGroup(
676  memberMap,
677  TableGroupKey::getFullGroupString(groupPair.first.first, newKey));
678 
679  // and modify groupSet and activeGroupKeys keys
680  groupPair.second = newKey;
681  }
682  catch(std::runtime_error& e)
683  {
684  __COUT__ << "Error was caught saving group " << groupPair.first.first << " ("
685  << groupPair.first.second << ") " << std::endl;
686  __COUT__ << e.what() << std::endl;
687 
688  groupErrors.insert(
689  std::pair<std::pair<std::string, TableGroupKey>, std::string>(
690  std::pair<std::string, TableGroupKey>(groupPair.first.first,
691  groupPair.first.second),
692  "Error caught saving the group."));
693  }
694  catch(...)
695  {
696  __COUT__ << "Error was caught saving group " << groupPair.first.first << " ("
697  << groupPair.first.second << ") " << std::endl;
698 
699  groupErrors.insert(
700  std::pair<std::pair<std::string, TableGroupKey>, std::string>(
701  std::pair<std::string, TableGroupKey>(groupPair.first.first,
702  groupPair.first.second),
703  "Error caught saving the group."));
704  }
705  //=========================
706 
707  // increment
708  ++count;
709  } // end group loop
710 
711  __COUT__ << "Completed group and table saving for " << count << " groups." << __E__;
712  __COUT__ << "Created tables:" << std::endl;
713  for(auto& tablePair : newTableVersionMap)
714  __COUT__ << "\t" << tablePair.first.first << "-v" << tablePair.first.second
715  << " ==> " << tablePair.second << std::endl;
716  __COUT__ << std::endl;
717  __COUT__ << std::endl;
718  __COUT__ << "Created groups:" << std::endl;
719  for(auto& group : groupSet)
720  __COUT__ << "\t" << group.first.first << "(" << group.first.second << ") ==> "
721  << group.second << std::endl;
722 
723  // -- in current-db after loop...
724  // -- swap to current-db
725  if(importedDbInPlace)
726  {
727  __COUT__ << "Swap to current-db" << std::endl;
728  if(rename(currentDir.c_str(), importDir.c_str()) < 0)
729  {
730  __SS__ << "Problem!" << std::endl;
731  __SS_THROW__;
732  }
733  if(rename(tmpCurrentDir.c_str(), currentDir.c_str()) < 0)
734  {
735  __SS__ << "Problem!" << std::endl;
736  __SS_THROW__;
737  }
738  importedDbInPlace = false;
739  }
740 
741  // record in readme for current-db
742  {
743  FILE* fp = fopen((currentDir + "/README_otsdaq_import.txt").c_str(), "a");
744 
745  if(!fp)
746  __COUT__ << "\tError opening README file!" << std::endl;
747  else
748  {
749  time_t rawtime;
750  struct tm* timeinfo;
751  char buffer[200];
752 
753  time(&rawtime);
754  timeinfo = localtime(&rawtime);
755  strftime(buffer, 200, "%b %d, %Y %I:%M:%S%p %Z", timeinfo);
756 
757  fprintf(fp,
758  "This database...\t %s \t received an import from...\t %s \t at this "
759  "time \t %lu \t %s\n\n",
760  currentDir.c_str(),
761  pathToImportDatabase.c_str(),
762  time(0),
763  buffer);
764  fclose(fp);
765  }
766  }
767 
768  // -- in current-db after loop...
769  // - destroy and reload active groups
770  cfgMgr->destroy();
771  cfgMgr->restoreActiveTableGroups(true /*throwErrors*/);
772 
773  // -- insert new aliases for imported groups
774  // - should be basename+alias connection to (hop through maps) new groupName &
775  // groupKey
776  // -- insert new aliases for imported tables
777  // - should be basename+alias connection to (hop through maps) new tableName &
778  // tableVersion
779  // -- save new backbone tables and save new backbone group
780 
781  __COUT__ << "Modifying the active Backbone table to reflect new table versions and "
782  "group keys."
783  << std::endl;
784 
785  try
786  {
787  // modify Group Aliases Table and Version Aliases Table to
788  // include new groups and tables
789 
790  std::string activeBackboneGroupName =
791  cfgMgr->getActiveGroupName(ConfigurationManager::ACTIVE_GROUP_NAME_BACKBONE);
792  cfgMgr->loadTableGroup(
793  activeBackboneGroupName,
794  cfgMgr->getActiveGroupKey(ConfigurationManager::ACTIVE_GROUP_NAME_BACKBONE),
795  true,
796  &memberMap,
797  0,
798  &accumulateErrors);
799 
800  std::map<std::string, TableVersion> activeMap = cfgMgr->getActiveVersions();
801 
802  // modify Group Aliases Table
803  if(activeMap.find(groupAliasesTableName) != activeMap.end())
804  {
805  __COUT__ << "\n\nModifying " << groupAliasesTableName << std::endl;
806  config = cfgMgr->getTableByName(groupAliasesTableName);
807  cfgView = config->getViewP();
808 
809  unsigned int col0 = cfgView->findCol("GroupKeyAlias");
810  unsigned int col1 = cfgView->findCol("GroupName");
811  unsigned int col2 = cfgView->findCol("GroupKey");
812  unsigned int row;
813 
814  cfgView->print();
815 
816  // -- insert new aliases for imported groups
817  // - should be basename+alias connection to (hop through maps) new
818  // groupName & groupKey
819  for(auto& aliasPair : originalGroupAliasMap)
820  {
821  auto groupIt = groupSet.find(std::pair<std::string, TableGroupKey>(
822  aliasPair.second.first, aliasPair.second.second));
823 
824  if(groupIt == groupSet.end())
825  {
826  __COUT__
827  << "Error! Could not find the new entry for the original group "
828  << aliasPair.second.first << "(" << aliasPair.second.second << ")"
829  << __E__;
830  continue;
831  }
832  row = cfgView->addRow("import_aliases", true /*incrementUniqueData*/);
833  cfgView->setValue(aliasPair.first, row, col0);
834  cfgView->setValue(aliasPair.second.first, row, col1);
835  cfgView->setValue(groupIt->second.toString(), row, col2);
836  } // end group alias edit
837 
838  cfgView->print();
839  }
840 
841  // modify Version Aliases Table
842  if(activeMap.find(versionAliasesTableName) != activeMap.end())
843  {
844  __COUT__ << "\n\nModifying " << versionAliasesTableName << std::endl;
845  config = cfgMgr->getTableByName(versionAliasesTableName);
846  cfgView = config->getViewP();
847  unsigned int col0 = cfgView->findCol("VersionAlias");
848  unsigned int col1 = cfgView->findCol("TableName");
849  unsigned int col2 = cfgView->findCol("Version");
850 
851  unsigned int row;
852 
853  cfgView->print();
854 
855  // -- insert new aliases for imported tables
856  // - should be basename+alias connection to (hop through maps) new
857  // tableName & tableVersion
858  for(auto& aliasPair : originalTableAliasMap)
859  {
860  auto tableIt =
861  newTableVersionMap.find(std::pair<std::string, TableVersion>(
862  aliasPair.second.first, aliasPair.second.second));
863 
864  if(tableIt == newTableVersionMap.end())
865  {
866  __COUT__
867  << "Error! Could not find the new entry for the original table "
868  << aliasPair.second.first << "(" << aliasPair.second.second << ")"
869  << __E__;
870  continue;
871  }
872  row = cfgView->addRow("import_aliases", true /*incrementUniqueData*/);
873  cfgView->setValue(aliasPair.first, row, col0);
874  cfgView->setValue(aliasPair.second.first, row, col1);
875  cfgView->setValue(tableIt->second.toString(), row, col2);
876  } // end group alias edit
877 
878  cfgView->print();
879  }
880 
881  // save new Group Aliases Table and Version Aliases Table
882 
883  // change the version of the active view to flatVersion and save it
884  config = cfgMgr->getTableByName(groupAliasesTableName);
885  cfgView = config->getViewP();
886  newVersion =
887  TableVersion::getNextVersion(theInterface_->findLatestVersion(config));
888  __COUTV__(newVersion);
889  cfgView->setVersion(newVersion);
890  theInterface_->saveActiveVersion(config);
891 
892  memberMap[groupAliasesTableName] = newVersion; // change version in the member
893  // map
894 
895  __COUT__ << "\t to...\t" << groupAliasesTableName << ":v"
896  << memberMap[groupAliasesTableName] << std::endl;
897 
898  __COUT__ << versionAliasesTableName << ":v" << memberMap[versionAliasesTableName]
899  << std::endl;
900  // change the version of the active view to flatVersion and save it
901  config = cfgMgr->getTableByName(versionAliasesTableName);
902  cfgView = config->getViewP();
903  newVersion =
904  TableVersion::getNextVersion(theInterface_->findLatestVersion(config));
905  __COUTV__(newVersion);
906  cfgView->setVersion(newVersion);
907  theInterface_->saveActiveVersion(config);
908 
909  memberMap[versionAliasesTableName] =
910  newVersion; // change version in the member map
911 
912  __COUT__ << "\t to...\t" << versionAliasesTableName << ":v"
913  << memberMap[versionAliasesTableName] << std::endl;
914 
915  __COUT__ << "Backbone member map completed" << __E__;
916  __COUTV__(StringMacros::mapToString(memberMap));
917 
918  newKey = TableGroupKey::getNextKey(
919  theInterface_->findLatestGroupKey(activeBackboneGroupName));
920 
921  __COUTV__(newKey);
922 
923  // memberMap should now consist of members with new flat version, so save
924  theInterface_->saveTableGroup(
925  memberMap,
926  TableGroupKey::getFullGroupString(activeBackboneGroupName, newKey));
927 
928  std::string renameFile =
929  ConfigurationManager::ACTIVE_GROUPS_FILENAME + "." + nowTime;
930  rename(ConfigurationManager::ACTIVE_GROUPS_FILENAME.c_str(), renameFile.c_str());
931 
932  __COUT__ << "Backing up '" << ConfigurationManager::ACTIVE_GROUPS_FILENAME
933  << "' to ... '" << renameFile << "'" << std::endl;
934 
935  cfgMgr->activateTableGroup(activeBackboneGroupName,
936  newKey); // and write to active group file
937 
938  } // end try
939  catch(const std::runtime_error& e)
940  {
941  __COUT_ERR__ << "There was a fatal error during backbone modification: "
942  << e.what() << __E__;
943 
944  goto CLEAN_UP;
945  }
946 
947  // print resulting all groups
948 
949  std::cout << "\n\n" << __COUT_HDR_FL__ << "Resulting Groups:" << std::endl;
950  for(const auto& group : groupSet)
951  __COUT__ << "\t" << group.first.first << ": " << group.first.second << " => "
952  << group.second << std::endl;
953  std::cout << "\n\n" << __COUT_HDR_FL__ << "Resulting Groups end." << std::endl;
954 
955 CLEAN_UP:
956  //==============================================================================
957  __COUT__ << "End of Importing Active Table Groups!\n\n\n" << std::endl;
958 
959  __COUT__ << "****************************" << std::endl;
960  __COUT__ << "There were " << groupSet.size() << " groups considered, and there were "
961  << groupErrors.size() << " errors found handling those groups." << std::endl;
962  __COUT__ << "The following errors were found handling the groups:" << std::endl;
963  for(auto& groupErr : groupErrors)
964  __COUT__ << "\t" << groupErr.first.first << " " << groupErr.first.second << ": \t"
965  << groupErr.second << std::endl;
966  __COUT__ << "End of errors.\n\n" << std::endl;
967 
968  __COUT__ << "Run the following to return to your previous database structure:"
969  << std::endl;
970  __COUT__ << "\t otsdaq_flatten_system_aliases -1 " << backupDir << "\n\n"
971  << std::endl;
972 
973  return;
974 }
975 
976 int main(int argc, char* argv[])
977 {
978  ImportSystemAliasTableGroups(argc, argv);
979  return 0;
980 }
981 // BOOST_AUTO_TEST_SUITE_END()