otsdaq  v2_04_02
ConfigurationManagerRW.cc
1 #include "otsdaq/ConfigurationInterface/ConfigurationManagerRW.h"
2 
3 // backbone includes
4 //#include "otsdaq/TablePlugins/ConfigurationAliases.h"
5 //#include "otsdaq/TablePlugins/Configurations.h"
6 //#include "otsdaq/TablePlugins/DefaultConfigurations.h"
7 //#include "otsdaq/TablePlugins/VersionAliases.h"
8 
9 //#include "otsdaq/ConfigurationInterface/ConfigurationInterface.h"//All configurable
10 // objects are included here
11 
12 //
13 //#include "otsdaq/TablePlugins/DetectorTable.h"
14 //#include "otsdaq/TablePlugins/MaskTable.h"
15 //#include "otsdaq/TablePlugins/DetectorToFETable.h"
16 //
17 
18 //#include "otsdaq/ConfigurationInterface/DACStream.h"
19 //#include "otsdaq/ConfigurationDataFormats/TableGroupKey.h"
20 //
21 //#include "otsdaq/ConfigurationInterface/FileConfigurationInterface.h"
22 //
23 //#include <cassert>
24 
25 #include <dirent.h>
26 
27 using namespace ots;
28 
29 #undef __MF_SUBJECT__
30 #define __MF_SUBJECT__ "ConfigurationManagerRW"
31 
32 #define TABLE_INFO_PATH std::string(__ENV__("TABLE_INFO_PATH")) + "/"
33 #define TABLE_INFO_EXT "Info.xml"
34 
35 #define CORE_TABLE_INFO_FILENAME \
36  ((getenv("SERVICE_DATA_PATH") == NULL) \
37  ? (std::string(__ENV__("USER_DATA")) + "/ServiceData") \
38  : (std::string(__ENV__("SERVICE_DATA_PATH")))) + \
39  "/CoreTableInfoNames.dat"
40 
41 //==============================================================================
42 // ConfigurationManagerRW
43 ConfigurationManagerRW::ConfigurationManagerRW(const std::string& username)
44  : ConfigurationManager(username) // for use as author of new views
45 {
46  __COUT__ << "Using Config Mgr with Write Access! (for " << username << ")" << __E__;
47 
48  // FIXME only necessarily temporarily while Lore is still using fileSystem xml
49  theInterface_ =
50  ConfigurationInterface::getInstance(false); // false to use artdaq DB
51  // FIXME -- can delete this change of
52  // interface once RW and regular use
53  // same interface instance
54 
55  //=========================
56  // dump names of core tables (so UpdateOTS.sh can copy core tables for user)
57  // only if table does not exist
58  {
59  const std::set<std::string>& contextMemberNames = getContextMemberNames();
60  const std::set<std::string>& backboneMemberNames = getBackboneMemberNames();
61  const std::set<std::string>& iterateMemberNames = getIterateMemberNames();
62 
63  FILE* fp = fopen((CORE_TABLE_INFO_FILENAME).c_str(), "r");
64 
65  __COUT__ << "Updating core tables table..." << __E__;
66 
67  if(fp) // check for all core table names in file, and force their presence
68  {
69  std::vector<unsigned int> foundVector;
70  char line[100];
71  for(const auto& name : contextMemberNames)
72  {
73  foundVector.push_back(false);
74  rewind(fp);
75  while(fgets(line, 100, fp))
76  {
77  if(strlen(line) < 1)
78  continue;
79  line[strlen(line) - 1] = '\0'; // remove endline
80  if(strcmp(line, ("ContextGroup/" + name).c_str()) == 0) // is match?
81  {
82  foundVector.back() = true;
83  //__COUTV__(name);
84  break;
85  }
86  }
87  }
88 
89  for(const auto& name : backboneMemberNames)
90  {
91  foundVector.push_back(false);
92  rewind(fp);
93  while(fgets(line, 100, fp))
94  {
95  if(strlen(line) < 1)
96  continue;
97  line[strlen(line) - 1] = '\0'; // remove endline
98  if(strcmp(line, ("BackboneGroup/" + name).c_str()) == 0) // is match?
99  {
100  foundVector.back() = true;
101  //__COUTV__(name);
102  break;
103  }
104  }
105  }
106 
107  for(const auto& name : iterateMemberNames)
108  {
109  foundVector.push_back(false);
110  rewind(fp);
111  while(fgets(line, 100, fp))
112  {
113  if(strlen(line) < 1)
114  continue;
115  line[strlen(line) - 1] = '\0'; // remove endline
116  if(strcmp(line, ("IterateGroup/" + name).c_str()) == 0) // is match?
117  {
118  foundVector.back() = true;
119  //__COUTV__(name);
120  break;
121  }
122  }
123  }
124 
125  fclose(fp);
126 
127  // for(const auto &found:foundVector)
128  // __COUTV__(found);
129 
130  // open file for appending the missing names
131  fp = fopen((CORE_TABLE_INFO_FILENAME).c_str(), "a");
132  if(fp)
133  {
134  unsigned int i = 0;
135  for(const auto& name : contextMemberNames)
136  {
137  if(!foundVector[i])
138  fprintf(fp, "\nContextGroup/%s", name.c_str());
139 
140  ++i;
141  }
142  for(const auto& name : backboneMemberNames)
143  {
144  if(!foundVector[i])
145  fprintf(fp, "\nBackboneGroup/%s", name.c_str());
146 
147  ++i;
148  }
149  for(const auto& name : iterateMemberNames)
150  {
151  if(!foundVector[i])
152  fprintf(fp, "\nIterateGroup/%s", name.c_str());
153 
154  ++i;
155  }
156  fclose(fp);
157  }
158  else
159  {
160  __SS__ << "Failed to open core table info file for appending: "
161  << CORE_TABLE_INFO_FILENAME << __E__;
162  __SS_THROW__;
163  }
164  }
165  else
166  {
167  fp = fopen((CORE_TABLE_INFO_FILENAME).c_str(), "w");
168  if(fp)
169  {
170  for(const auto& name : contextMemberNames)
171  fprintf(fp, "\nContextGroup/%s", name.c_str());
172  for(const auto& name : backboneMemberNames)
173  fprintf(fp, "\nBackboneGroup/%s", name.c_str());
174  for(const auto& name : iterateMemberNames)
175  fprintf(fp, "\nIterateGroup/%s", name.c_str());
176  fclose(fp);
177  }
178  else
179  {
180  __SS__ << "Failed to open core table info file: "
181  << CORE_TABLE_INFO_FILENAME << __E__;
182  __SS_THROW__;
183  }
184  }
185  } // end dump names of core tables
186 } // end constructor
187 
188 //==============================================================================
189 // getAllTableInfo()
190 // Used by ConfigurationGUISupervisor to get all the info for the existing configurations
191 //
192 // if(accumulatedErrors)
193 // this implies allowing column errors and accumulating such errors in given string
194 const std::map<std::string, TableInfo>& ConfigurationManagerRW::getAllTableInfo(
195  bool refresh, std::string* accumulatedErrors, const std::string& errorFilterName)
196 {
197  // allTableInfo_ is container to be returned
198 
199  // if(accumulatedErrors)
200  // *accumulatedErrors = "";
201 
202  if(!refresh)
203  return allTableInfo_;
204 
205  // else refresh!
206  allTableInfo_.clear();
207  allGroupInfo_.clear();
208 
209  TableBase* table;
210 
211  // existing configurations are defined by which infos are in TABLE_INFO_PATH
212  // can test that the class exists based on this
213  // and then which versions
214  __COUT__ << "======================================================== "
215  "getAllTableInfo start"
216  << __E__;
217  __COUT__ << "Refreshing all! Extracting list of tables..." << __E__;
218  DIR* pDIR;
219  struct dirent* entry;
220  std::string path = TABLE_INFO_PATH;
221  char fileExt[] = TABLE_INFO_EXT;
222  const unsigned char MIN_TABLE_NAME_SZ = 3;
223  if((pDIR = opendir(path.c_str())) != 0)
224  {
225  while((entry = readdir(pDIR)) != 0)
226  {
227  // enforce table name length
228  if(strlen(entry->d_name) < strlen(fileExt) + MIN_TABLE_NAME_SZ)
229  continue;
230 
231  // find file names with correct file extenstion
232  if(strcmp(&(entry->d_name[strlen(entry->d_name) - strlen(fileExt)]),
233  fileExt) != 0)
234  continue; // skip different extentions
235 
236  entry->d_name[strlen(entry->d_name) - strlen(fileExt)] =
237  '\0'; // remove file extension to get table name
238 
239  //__COUT__ << entry->d_name << __E__;
240 
241  // 0 will force the creation of new instance (and reload from Info)
242  table = 0;
243 
244  try // only add valid table instances to maps
245  {
246  theInterface_->get(table, entry->d_name, 0, 0,
247  true); // dont fill
248  }
249  catch(cet::exception const&)
250  {
251  if(table)
252  delete table;
253  table = 0;
254 
255  __COUT__ << "Skipping! No valid class found for... " << entry->d_name
256  << "\n";
257  continue;
258  }
259  catch(std::runtime_error& e)
260  {
261  if(table)
262  delete table;
263  table = 0;
264 
265  __COUT__ << "Skipping! No valid class found for... " << entry->d_name
266  << "\n";
267  __COUT__ << "Error: " << e.what() << __E__;
268 
269  // for a runtime_error, it is likely that columns are the problem
270  // the Table Editor needs to still fix these.. so attempt to
271  // proceed.
272  if(accumulatedErrors)
273  {
274  if(errorFilterName == "" || errorFilterName == entry->d_name)
275  {
276  *accumulatedErrors += std::string("\nIn table '") +
277  entry->d_name + "'..." +
278  e.what(); // global accumulate
279 
280  __SS__ << "Attempting to allow illegal columns!" << __E__;
281  *accumulatedErrors += ss.str();
282  }
283 
284  // attempt to recover and build a mock-up
285  __COUT__ << "Attempting to allow illegal columns!" << __E__;
286 
287  std::string returnedAccumulatedErrors;
288  try
289  {
290  // table = new TableBase(entry->d_name,
291  // &returnedAccumulatedErrors);
292  table = new TableBase(entry->d_name, &returnedAccumulatedErrors);
293  }
294  catch(...)
295  {
296  __COUT__
297  << "Skipping! Allowing illegal columns didn't work either... "
298  << entry->d_name << "\n";
299  continue;
300  }
301  __COUT__ << "Error (but allowed): " << returnedAccumulatedErrors
302  << __E__;
303 
304  if(errorFilterName == "" || errorFilterName == entry->d_name)
305  *accumulatedErrors +=
306  std::string("\nIn table '") + entry->d_name + "'..." +
307  returnedAccumulatedErrors; // global accumulate
308  }
309  else
310  continue;
311  }
312 
313  //__COUT__ << "Instance created: " << entry->d_name << "\n"; //found!
314 
315  if(nameToTableMap_[entry->d_name]) // handle if instance existed
316  {
317  // copy the temporary versions! (or else all is lost)
318  std::set<TableVersion> versions =
319  nameToTableMap_[entry->d_name]->getStoredVersions();
320  for(auto& version : versions)
321  if(version.isTemporaryVersion())
322  {
323  //__COUT__ << "copying tmp = " << version << __E__;
324 
325  try // do NOT let TableView::init() throw here
326  {
327  nameToTableMap_[entry->d_name]->setActiveView(version);
328  table->copyView( // this calls TableView::init()
329  nameToTableMap_[entry->d_name]->getView(),
330  version,
331  username_);
332  }
333  catch(...) // do NOT let invalid temporary version throw at this
334  // point
335  {
336  } // just trust configurationBase throws out the failed version
337  }
338  //__COUT__ << "deleting: " << entry->d_name << "\n"; //found!
339  delete nameToTableMap_[entry->d_name];
340  nameToTableMap_[entry->d_name] = 0;
341  }
342 
343  nameToTableMap_[entry->d_name] = table;
344 
345  allTableInfo_[entry->d_name].tablePtr_ = table;
346  allTableInfo_[entry->d_name].versions_ = theInterface_->getVersions(table);
347 
348  // also add any existing temporary versions to all table info
349  // because the interface wont find those versions
350  std::set<TableVersion> versions =
351  nameToTableMap_[entry->d_name]->getStoredVersions();
352  for(auto& version : versions)
353  if(version.isTemporaryVersion())
354  {
355  //__COUT__ << "surviving tmp = " << version << __E__;
356  allTableInfo_[entry->d_name].versions_.emplace(version);
357  }
358  }
359  closedir(pDIR);
360  }
361  __COUT__ << "Extracting list of tables complete. Now initializing..." << __E__;
362 
363  // call init to load active versions by default
364  init(accumulatedErrors);
365 
366  __COUT__
367  << "======================================================== getAllTableInfo end"
368  << __E__;
369 
370  // get Group Info too!
371  try
372  {
373  // build allGroupInfo_ for the ConfigurationManagerRW
374 
375  std::set<std::string /*name*/> tableGroups =
376  theInterface_->getAllTableGroupNames();
377  __COUT__ << "Number of Groups: " << tableGroups.size() << __E__;
378 
379  TableGroupKey key;
380  std::string name;
381  for(const auto& fullName : tableGroups)
382  {
383  TableGroupKey::getGroupNameAndKey(fullName, name, key);
384  cacheGroupKey(name, key);
385  }
386 
387  // for each group get member map & comment, author, time, and type for latest key
388  for(auto& groupInfo : allGroupInfo_)
389  {
390  try
391  {
392  loadTableGroup(groupInfo.first /*groupName*/,
393  groupInfo.second.getLatestKey(),
394  false /*doActivate*/,
395  &groupInfo.second.latestKeyMemberMap_ /*groupMembers*/,
396  0 /*progressBar*/,
397  0 /*accumulateErrors*/,
398  &groupInfo.second.latestKeyGroupComment_,
399  &groupInfo.second.latestKeyGroupAuthor_,
400  &groupInfo.second.latestKeyGroupCreationTime_,
401  true /*doNotLoadMember*/,
402  &groupInfo.second.latestKeyGroupTypeString_);
403  }
404  catch(...)
405  {
406  __COUT_WARN__
407  << "Error occurred loading latest group info into cache for '"
408  << groupInfo.first << "'..." << __E__;
409  groupInfo.second.latestKeyGroupComment_ = "UNKNOWN";
410  groupInfo.second.latestKeyGroupAuthor_ = "UNKNOWN";
411  groupInfo.second.latestKeyGroupCreationTime_ = "0";
412  groupInfo.second.latestKeyGroupTypeString_ = "UNKNOWN";
413  }
414  } // end group info loop
415  } // end get group info
416  catch(const std::runtime_error& e)
417  {
418  __SS__ << "A fatal error occurred reading the info for all table groups. Error: "
419  << e.what() << __E__;
420  __COUT_ERR__ << "\n" << ss.str();
421  if(accumulatedErrors)
422  *accumulatedErrors += ss.str();
423  else
424  throw;
425  }
426  catch(...)
427  {
428  __SS__ << "An unknown fatal error occurred reading the info for all table groups."
429  << __E__;
430  __COUT_ERR__ << "\n" << ss.str();
431  if(accumulatedErrors)
432  *accumulatedErrors += ss.str();
433  else
434  throw;
435  }
436 
437  return allTableInfo_;
438 } // end getAllTableInfo
439 
440 //==============================================================================
441 // getVersionAliases()
442 // get version aliases organized by table, for currently active backbone tables
443 // add scratch versions to the alias map returned by ConfigurationManager
444 std::map<std::string /*table name*/,
445  std::map<std::string /*version alias*/, TableVersion /*aliased version*/> >
446 ConfigurationManagerRW::getVersionAliases(void) const
447 {
448  //__COUT__ << "getVersionAliases()" << __E__;
449  std::map<std::string /*table name*/,
450  std::map<std::string /*version alias*/, TableVersion /*aliased version*/> >
451  retMap = ConfigurationManager::getVersionAliases();
452 
453  // always have scratch alias for each table that has a scratch version
454  // overwrite map entry if necessary
455  if(!ConfigurationInterface::isVersionTrackingEnabled())
456  for(const auto& tableInfo : allTableInfo_)
457  for(const auto& version : tableInfo.second.versions_)
458  if(version.isScratchVersion())
459  retMap[tableInfo.first][ConfigurationManager::SCRATCH_VERSION_ALIAS] =
460  TableVersion(TableVersion::SCRATCH);
461 
462  return retMap;
463 } // end getVersionAliases()
464 
465 //==============================================================================
466 // setActiveGlobalConfiguration
467 // load table group and activate
468 // deactivates previous table group of same type if necessary
469 void ConfigurationManagerRW::activateTableGroup(const std::string& configGroupName,
470  TableGroupKey tableGroupKey,
471  std::string* accumulatedTreeErrors)
472 {
473  try
474  {
475  __COUTV__(accumulatedTreeErrors);
476  loadTableGroup(configGroupName,
477  tableGroupKey,
478  true, // loads and activates
479  0, // no members needed
480  0, // no progress bar
481  accumulatedTreeErrors); // accumulate warnings or not
482  }
483  catch(...)
484  {
485  __COUT_ERR__ << "There were errors, so de-activating group: " << configGroupName
486  << " (" << tableGroupKey << ")" << __E__;
487  try // just in case any lingering pieces, let's deactivate
488  {
489  destroyTableGroup(configGroupName, true);
490  }
491  catch(...)
492  {
493  }
494  throw; // re-throw original exception
495  }
496 
497  __COUT_INFO__ << "Updating persistent active groups to "
498  << ConfigurationManager::ACTIVE_GROUPS_FILENAME << " ..." << __E__;
499  __MOUT_INFO__ << "Updating persistent active groups to "
500  << ConfigurationManager::ACTIVE_GROUPS_FILENAME << " ..." << __E__;
501 
502  std::string fn = ConfigurationManager::ACTIVE_GROUPS_FILENAME;
503  FILE* fp = fopen(fn.c_str(), "w");
504  if(!fp)
505  {
506  __SS__ << "Fatal Error! Unable to open the file "
507  << ConfigurationManager::ACTIVE_GROUPS_FILENAME
508  << " for editing! Is there a permissions problem?" << __E__;
509  __COUT_ERR__ << ss.str();
510  __SS_THROW__;
511  return;
512  }
513 
514  __MCOUT_INFO__("Active Context table group: "
515  << theContextTableGroup_ << "("
516  << (theContextTableGroupKey_
517  ? theContextTableGroupKey_->toString().c_str()
518  : "-1")
519  << ")" << __E__);
520  __MCOUT_INFO__("Active Backbone table group: "
521  << theBackboneTableGroup_ << "("
522  << (theBackboneTableGroupKey_
523  ? theBackboneTableGroupKey_->toString().c_str()
524  : "-1")
525  << ")" << __E__);
526  __MCOUT_INFO__("Active Iterate table group: "
527  << theIterateTableGroup_ << "("
528  << (theIterateTableGroupKey_
529  ? theIterateTableGroupKey_->toString().c_str()
530  : "-1")
531  << ")" << __E__);
532  __MCOUT_INFO__("Active Configuration table group: "
533  << theConfigurationTableGroup_ << "("
534  << (theConfigurationTableGroupKey_
535  ? theConfigurationTableGroupKey_->toString().c_str()
536  : "-1")
537  << ")" << __E__);
538 
539  fprintf(fp, "%s\n", theContextTableGroup_.c_str());
540  fprintf(
541  fp,
542  "%s\n",
543  theContextTableGroupKey_ ? theContextTableGroupKey_->toString().c_str() : "-1");
544  fprintf(fp, "%s\n", theBackboneTableGroup_.c_str());
545  fprintf(
546  fp,
547  "%s\n",
548  theBackboneTableGroupKey_ ? theBackboneTableGroupKey_->toString().c_str() : "-1");
549  fprintf(fp, "%s\n", theIterateTableGroup_.c_str());
550  fprintf(
551  fp,
552  "%s\n",
553  theIterateTableGroupKey_ ? theIterateTableGroupKey_->toString().c_str() : "-1");
554  fprintf(fp, "%s\n", theConfigurationTableGroup_.c_str());
555  fprintf(fp,
556  "%s\n",
557  theConfigurationTableGroupKey_
558  ? theConfigurationTableGroupKey_->toString().c_str()
559  : "-1");
560  fclose(fp);
561 
562 } // end activateTableGroup()
563 
564 //==============================================================================
565 // createTemporaryBackboneView
566 // sourceViewVersion of INVALID is from MockUp, else from valid view version
567 // returns temporary version number (which is always negative)
568 TableVersion ConfigurationManagerRW::createTemporaryBackboneView(
569  TableVersion sourceViewVersion)
570 {
571  __COUT_INFO__ << "Creating temporary backbone view from version " << sourceViewVersion
572  << __E__;
573 
574  // find common available temporary version among backbone members
575  TableVersion tmpVersion =
576  TableVersion::getNextTemporaryVersion(); // get the default temporary version
577  TableVersion retTmpVersion;
578  auto backboneMemberNames = ConfigurationManager::getBackboneMemberNames();
579  for(auto& name : backboneMemberNames)
580  {
581  retTmpVersion =
582  ConfigurationManager::getTableByName(name)->getNextTemporaryVersion();
583  if(retTmpVersion < tmpVersion)
584  tmpVersion = retTmpVersion;
585  }
586 
587  __COUT__ << "Common temporary backbone version found as " << tmpVersion << __E__;
588 
589  // create temporary views from source version to destination temporary version
590  for(auto& name : backboneMemberNames)
591  {
592  retTmpVersion =
593  getTableByName(name)->createTemporaryView(sourceViewVersion, tmpVersion);
594  if(retTmpVersion != tmpVersion)
595  {
596  __SS__ << "Failure! Temporary view requested was " << tmpVersion
597  << ". Mismatched temporary view created: " << retTmpVersion << __E__;
598  __COUT_ERR__ << ss.str();
599  __SS_THROW__;
600  }
601  }
602 
603  return tmpVersion;
604 }
605 
606 //==============================================================================
607 TableBase* ConfigurationManagerRW::getTableByName(const std::string& tableName)
608 {
609  if(nameToTableMap_.find(tableName) == nameToTableMap_.end())
610  {
611  __SS__ << "Table not found with name: " << tableName << __E__;
612  size_t f;
613  if((f = tableName.find(' ')) != std::string::npos)
614  ss << "There was a space character found in the table name needle at "
615  "position "
616  << f << " in the string (was this intended?). " << __E__;
617  __COUT_ERR__ << "\n" << ss.str();
618  __SS_THROW__;
619  }
620  return nameToTableMap_[tableName];
621 }
622 
623 //==============================================================================
624 // getVersionedTableByName
625 // Used by table GUI to load a particular table-version pair as the active version.
626 // This table instance must already exist and be owned by ConfigurationManager.
627 // return null pointer on failure, on success return table pointer.
628 TableBase* ConfigurationManagerRW::getVersionedTableByName(
629  const std::string& tableName,
630  TableVersion version,
631  bool looseColumnMatching /* =false */,
632  std::string* accumulatedErrors /* =0 */)
633 {
634  auto it = nameToTableMap_.find(tableName);
635  if(it == nameToTableMap_.end())
636  {
637  __SS__ << "\nCan not find table named '" << tableName
638  << "'\n\n\n\nYou need to load the table before it can be used."
639  << "It probably is missing from the member list of the Table "
640  "Group that was loaded?\n\n\n\n\n"
641  << __E__;
642  __SS_THROW__;
643  }
644  TableBase* table = it->second;
645  theInterface_->get(table,
646  tableName,
647  0,
648  0,
649  false, // fill w/version
650  version,
651  false, // do not reset
652  looseColumnMatching,
653  accumulatedErrors);
654  return table;
655 }
656 
657 //==============================================================================
658 // saveNewTable
659 // saves version, makes the new version the active version, and returns new version
660 TableVersion ConfigurationManagerRW::saveNewTable(const std::string& tableName,
661  TableVersion temporaryVersion,
662  bool makeTemporary) //,
663 // bool saveToScratchVersion)
664 {
665  TableVersion newVersion(temporaryVersion);
666 
667  // set author of version
668  TableBase* table = getTableByName(tableName);
669  table->getTemporaryView(temporaryVersion)->setAuthor(username_);
670  // NOTE: author is assigned to permanent versions when saved to DBI
671 
672  if(!makeTemporary) // saveNewVersion makes the new version the active version
673  newVersion = theInterface_->saveNewVersion(table, temporaryVersion);
674  else // make the temporary version active
675  table->setActiveView(newVersion);
676 
677  // if there is a problem, try to recover
678  while(!makeTemporary && !newVersion.isScratchVersion() &&
679  allTableInfo_[tableName].versions_.find(newVersion) !=
680  allTableInfo_[tableName].versions_.end())
681  {
682  __COUT_ERR__ << "What happenened!?? ERROR::: new persistent version v"
683  << newVersion
684  << " already exists!? How is it possible? Retrace your steps and "
685  "tell an admin."
686  << __E__;
687 
688  // create a new temporary version of the target view
689  temporaryVersion = table->createTemporaryView(newVersion);
690 
691  if(newVersion.isTemporaryVersion())
692  newVersion = temporaryVersion;
693  else
694  newVersion = TableVersion::getNextVersion(newVersion);
695 
696  __COUT_WARN__ << "Attempting to recover and use v" << newVersion << __E__;
697 
698  if(!makeTemporary) // saveNewVersion makes the new version the active version
699  newVersion =
700  theInterface_->saveNewVersion(table, temporaryVersion, newVersion);
701  else // make the temporary version active
702  table->setActiveView(newVersion);
703  }
704 
705  if(newVersion.isInvalid())
706  {
707  __SS__ << "Something went wrong saving the new version v" << newVersion
708  << ". What happened?! (duplicates? database error?)" << __E__;
709  __COUT_ERR__ << "\n" << ss.str();
710  __SS_THROW__;
711  }
712 
713  // update allTableInfo_ with the new version
714  allTableInfo_[tableName].versions_.insert(newVersion);
715 
716  __COUT__ << "New version added to info " << newVersion << __E__;
717 
718  // table->getView().print();
719  return newVersion;
720 }
721 
722 //==============================================================================
723 // eraseTemporaryVersion
724 // if version is invalid then erases ALL temporary versions
725 //
726 // maintains allTableInfo_ also while erasing
727 void ConfigurationManagerRW::eraseTemporaryVersion(const std::string& tableName,
728  TableVersion targetVersion)
729 {
730  TableBase* table = getTableByName(tableName);
731 
732  table->trimTemporary(targetVersion);
733 
734  // if allTableInfo_ is not setup, then done
735  if(allTableInfo_.find(tableName) == allTableInfo_.end())
736  return;
737  // else cleanup table info
738 
739  if(targetVersion.isInvalid())
740  {
741  // erase all temporary versions!
742  for(auto it = allTableInfo_[tableName].versions_.begin();
743  it != allTableInfo_[tableName].versions_.end();
744  /*no increment*/)
745  {
746  if(it->isTemporaryVersion())
747  {
748  __COUT__ << "Removing version info: " << *it << __E__;
749  allTableInfo_[tableName].versions_.erase(it++);
750  }
751  else
752  ++it;
753  }
754  }
755  else // erase target version only
756  {
757  __COUT__ << "Removing version info: " << targetVersion << __E__;
758  auto it = allTableInfo_[tableName].versions_.find(targetVersion);
759  if(it == allTableInfo_[tableName].versions_.end())
760  {
761  __COUT__ << "Target version was not found in info versions..." << __E__;
762  return;
763  }
764  allTableInfo_[tableName].versions_.erase(
765  allTableInfo_[tableName].versions_.find(targetVersion));
766  __COUT__ << "Target version was erased from info." << __E__;
767  }
768 }
769 
770 //==============================================================================
771 // clearCachedVersions
772 // clear ALL cached persistent versions (does not erase temporary versions)
773 //
774 // maintains allTableInfo_ also while erasing (trivial, do nothing)
775 void ConfigurationManagerRW::clearCachedVersions(const std::string& tableName)
776 {
777  TableBase* table = getTableByName(tableName);
778 
779  table->trimCache(0);
780 }
781 
782 //==============================================================================
783 // clearAllCachedVersions
784 // clear ALL cached persistent versions (does not erase temporary versions)
785 //
786 // maintains allTableInfo_ also while erasing (trivial, do nothing)
787 void ConfigurationManagerRW::clearAllCachedVersions()
788 {
789  for(auto configInfo : allTableInfo_)
790  configInfo.second.tablePtr_->trimCache(0);
791 }
792 
793 //==============================================================================
794 // copyViewToCurrentColumns
795 TableVersion ConfigurationManagerRW::copyViewToCurrentColumns(
796  const std::string& tableName, TableVersion sourceVersion)
797 {
798  getTableByName(tableName)->reset();
799 
800  // make sure source version is loaded
801  // need to load with loose column rules!
802  TableBase* table =
803  getVersionedTableByName(tableName, TableVersion(sourceVersion), true);
804 
805  // copy from source version to a new temporary version
806  TableVersion newTemporaryVersion =
807  table->copyView(table->getView(), TableVersion(), username_);
808 
809  // update allTableInfo_ with the new version
810  allTableInfo_[tableName].versions_.insert(newTemporaryVersion);
811 
812  return newTemporaryVersion;
813 }
814 
815 //==============================================================================
816 // cacheGroupKey
817 void ConfigurationManagerRW::cacheGroupKey(const std::string& groupName,
818  TableGroupKey key)
819 {
820  allGroupInfo_[groupName].keys_.emplace(key);
821 }
822 
823 //==============================================================================
824 // getGroupInfo
825 // the interface is slow when there are a lot of groups..
826 // so plan is to maintain local cache of recent group info
827 const GroupInfo& ConfigurationManagerRW::getGroupInfo(const std::string& groupName)
828 {
829  // //NOTE: seems like this filter is taking the long amount of time
830  // std::set<std::string /*name*/> fullGroupNames =
831  // theInterface_->getAllTableGroupNames(groupName); //db filter by
832  // group name
833 
834  // so instead caching ourselves...
835  auto it = allGroupInfo_.find(groupName);
836  if(it == allGroupInfo_.end())
837  {
838  __SS__ << "Group name '" << groupName
839  << "' not found in group info! (creating empty info)" << __E__;
840  __COUT_WARN__ << ss.str();
841  //__SS_THROW__;
842  return allGroupInfo_[groupName];
843  }
844  return it->second;
845 }
846 
847 //==============================================================================
848 // findTableGroup
849 // return group with same name and same members and same aliases
850 // else return invalid key
851 //
852 // Note: if aliases, then member alias is matched (not member
853 //
854 // Note: this is taking too long when there are a ton of groups.
855 // Change to going back only a limited number.. (but the order also comes in alpha order
856 // from theInterface_->getAllTableGroupNames which is a problem for choosing
857 // the most recent to check. )
858 TableGroupKey ConfigurationManagerRW::findTableGroup(
859  const std::string& groupName,
860  const std::map<std::string, TableVersion>& groupMemberMap,
861  const std::map<std::string /*name*/, std::string /*alias*/>& groupAliases)
862 {
863  // //NOTE: seems like this filter is taking the long amount of time
864  // std::set<std::string /*name*/> fullGroupNames =
865  // theInterface_->getAllTableGroupNames(groupName); //db filter by
866  // group name
867  const GroupInfo& groupInfo = getGroupInfo(groupName);
868 
869  // std::string name;
870  // TableGroupKey key;
871  std::map<std::string /*name*/, TableVersion /*version*/> compareToMemberMap;
872  std::map<std::string /*name*/, std::string /*alias*/> compareToGroupAliases;
873  bool isDifferent;
874 
875  const unsigned int MAX_DEPTH_TO_CHECK = 20;
876  unsigned int keyMinToCheck = 0;
877 
878  if(groupInfo.keys_.size())
879  keyMinToCheck = groupInfo.keys_.rbegin()->key();
880  if(keyMinToCheck > MAX_DEPTH_TO_CHECK)
881  {
882  keyMinToCheck -= MAX_DEPTH_TO_CHECK;
883  __COUT__ << "Checking groups back to key... " << keyMinToCheck << __E__;
884  }
885  else
886  {
887  keyMinToCheck = 0;
888  __COUT__ << "Checking all groups." << __E__;
889  }
890 
891  // have min key to check, now loop through and check groups
892  // std::string fullName;
893  for(const auto& key : groupInfo.keys_)
894  {
895  // TableGroupKey::getGroupNameAndKey(fullName,name,key);
896 
897  if(key.key() < keyMinToCheck)
898  continue; // skip keys that are too old
899 
900  // fullName = TableGroupKey::getFullGroupString(groupName,key);
901  //
902  // __COUT__ << "checking group... " << fullName << __E__;
903  //
904  // compareToMemberMap =
905  // theInterface_->getTableGroupMembers(fullName);
906 
907  loadTableGroup(groupName,
908  key,
909  false /*doActivate*/,
910  &compareToMemberMap /*memberMap*/,
911  0,
912  0,
913  0,
914  0,
915  0, /*null pointers*/
916  true /*doNotLoadMember*/,
917  0 /*groupTypeString*/,
918  &compareToGroupAliases);
919 
920  isDifferent = false;
921  for(auto& memberPair : groupMemberMap)
922  {
923  //__COUT__ << memberPair.first << " - " << memberPair.second << __E__;
924 
925  if(groupAliases.find(memberPair.first) != groupAliases.end())
926  {
927  // handle this table as alias, not version
928  if(compareToGroupAliases.find(memberPair.first) ==
929  compareToGroupAliases.end() || // alias is missing
930  groupAliases.at(memberPair.first) !=
931  compareToGroupAliases.at(memberPair.first))
932  { // then different
933  //__COUT__ << "alias mismatch found!" << __E__;
934  isDifferent = true;
935  break;
936  }
937  else
938  continue;
939  } // else check if compareTo group is using an alias for table
940  else if(compareToGroupAliases.find(memberPair.first) !=
941  compareToGroupAliases.end())
942  {
943  // then different
944  //__COUT__ << "alias mismatch found!" << __E__;
945  isDifferent = true;
946  break;
947 
948  } // else handle as table version comparison
949  else if(compareToMemberMap.find(memberPair.first) ==
950  compareToMemberMap.end() || // name is missing
951  memberPair.second !=
952  compareToMemberMap.at(memberPair.first)) // or version mismatch
953  { // then different
954  //__COUT__ << "mismatch found!" << __E__;
955  isDifferent = true;
956  break;
957  }
958  }
959  if(isDifferent)
960  continue;
961 
962  // check member size for exact match
963  if(groupMemberMap.size() != compareToMemberMap.size())
964  continue; // different size, so not same (groupMemberMap is a subset of
965  // memberPairs)
966 
967  __COUT__ << "Found exact match with key: " << key << __E__;
968  // else found an exact match!
969  return key;
970  }
971  __COUT__ << "No match found - this group is new!" << __E__;
972  // if here, then no match found
973  return TableGroupKey(); // return invalid key
974 }
975 
976 //==============================================================================
977 // saveNewTableGroup
978 // saves new group and returns the new group key
979 // if previousVersion is provided, attempts to just bump that version
980 // else, bumps latest version found in db
981 //
982 // Note: groupMembers map will get modified with group metadata table version
983 TableGroupKey ConfigurationManagerRW::saveNewTableGroup(
984  const std::string& groupName,
985  std::map<std::string, TableVersion>& groupMembers,
986  const std::string& groupComment,
987  std::map<std::string /*table*/, std::string /*alias*/>* groupAliases)
988 {
989  // steps:
990  // determine new group key
991  // verify group members
992  // verify groupNameWithKey
993  // verify store
994 
995  if(groupMembers.size() == 0) // do not allow empty groups
996  {
997  __SS__ << "Empty group member list. Can not create a group without members!"
998  << __E__;
999  __SS_THROW__;
1000  }
1001 
1002  // determine new group key
1003  TableGroupKey newKey =
1004  TableGroupKey::getNextKey(theInterface_->findLatestGroupKey(groupName));
1005 
1006  __COUT__ << "New Key for group: " << groupName << " found as " << newKey << __E__;
1007 
1008  // verify group members
1009  // - use all table info
1010  std::map<std::string, TableInfo> allCfgInfo = getAllTableInfo();
1011  for(auto& memberPair : groupMembers)
1012  {
1013  // check member name
1014  if(allCfgInfo.find(memberPair.first) == allCfgInfo.end())
1015  {
1016  __COUT_ERR__ << "Group member \"" << memberPair.first
1017  << "\" not found in database!";
1018 
1019  if(groupMetadataTable_.getTableName() == memberPair.first)
1020  {
1021  __COUT_WARN__
1022  << "Looks like this is the groupMetadataTable_ '"
1023  << ConfigurationInterface::GROUP_METADATA_TABLE_NAME
1024  << ".' Note that this table is added to the member map when groups "
1025  "are saved."
1026  << "It should not be part of member map when calling this function."
1027  << __E__;
1028  __COUT__ << "Attempting to recover." << __E__;
1029  groupMembers.erase(groupMembers.find(memberPair.first));
1030  }
1031  else
1032  {
1033  __SS__ << ("Group member not found!") << __E__;
1034  __SS_THROW__;
1035  }
1036  }
1037  // check member version
1038  if(allCfgInfo[memberPair.first].versions_.find(memberPair.second) ==
1039  allCfgInfo[memberPair.first].versions_.end())
1040  {
1041  __SS__ << "Group member \"" << memberPair.first << "\" version \""
1042  << memberPair.second << "\" not found in database!";
1043  __SS_THROW__;
1044  }
1045  } // end verify members
1046 
1047  // verify group aliases
1048  if(groupAliases)
1049  {
1050  for(auto& aliasPair : *groupAliases)
1051  {
1052  // check for alias table in member names
1053  if(groupMembers.find(aliasPair.first) == groupMembers.end())
1054  {
1055  __COUT_ERR__ << "Group member \"" << aliasPair.first
1056  << "\" not found in group member map!";
1057 
1058  __SS__ << ("Alias table not found in member list!") << __E__;
1059  __SS_THROW__;
1060  }
1061  }
1062  } // end verify group aliases
1063 
1064  // verify groupNameWithKey and attempt to store
1065  try
1066  {
1067  // save meta data for group; reuse groupMetadataTable_
1068  std::string groupAliasesString = "";
1069  if(groupAliases)
1070  groupAliasesString = StringMacros::mapToString(
1071  *groupAliases, "," /*primary delimeter*/, ":" /*secondary delimeter*/);
1072  __COUT__ << "Metadata: " << username_ << " " << time(0) << " " << groupComment
1073  << " " << groupAliasesString << __E__;
1074 
1075  // to compensate for unusual errors upstream, make sure the metadata table has one
1076  // row
1077  while(groupMetadataTable_.getViewP()->getNumberOfRows() > 1)
1078  groupMetadataTable_.getViewP()->deleteRow(0);
1079  if(groupMetadataTable_.getViewP()->getNumberOfRows() == 0)
1080  groupMetadataTable_.getViewP()->addRow();
1081 
1082  // columns are uid,comment,author,time
1083  groupMetadataTable_.getViewP()->setValue(
1084  groupAliasesString, 0, ConfigurationManager::METADATA_COL_ALIASES);
1085  groupMetadataTable_.getViewP()->setValue(
1086  groupComment, 0, ConfigurationManager::METADATA_COL_COMMENT);
1087  groupMetadataTable_.getViewP()->setValue(
1088  username_, 0, ConfigurationManager::METADATA_COL_AUTHOR);
1089  groupMetadataTable_.getViewP()->setValue(
1090  time(0), 0, ConfigurationManager::METADATA_COL_TIMESTAMP);
1091 
1092  // set version to first available persistent version
1093  groupMetadataTable_.getViewP()->setVersion(TableVersion::getNextVersion(
1094  theInterface_->findLatestVersion(&groupMetadataTable_)));
1095 
1096  // groupMetadataTable_.print();
1097 
1098  theInterface_->saveActiveVersion(&groupMetadataTable_);
1099 
1100  // force groupMetadataTable_ to be a member for the group
1101  groupMembers[groupMetadataTable_.getTableName()] =
1102  groupMetadataTable_.getViewVersion();
1103 
1104  theInterface_->saveTableGroup(
1105  groupMembers, TableGroupKey::getFullGroupString(groupName, newKey));
1106  __COUT__ << "Created table group: " << groupName << ":" << newKey << __E__;
1107  }
1108  catch(std::runtime_error& e)
1109  {
1110  __COUT_ERR__ << "Failed to create table group: " << groupName << ":" << newKey
1111  << __E__;
1112  __COUT_ERR__ << "\n\n" << e.what() << __E__;
1113  throw;
1114  }
1115  catch(...)
1116  {
1117  __COUT_ERR__ << "Failed to create table group: " << groupName << ":" << newKey
1118  << __E__;
1119  throw;
1120  }
1121 
1122  // store cache of recent groups
1123  cacheGroupKey(groupName, newKey);
1124 
1125  // at this point succeeded!
1126  return newKey;
1127 }
1128 
1129 //==============================================================================
1130 // saveNewBackbone
1131 // makes the new version the active version and returns new version number
1132 // INVALID will give a new backbone from mockup
1133 TableVersion ConfigurationManagerRW::saveNewBackbone(TableVersion temporaryVersion)
1134 {
1135  __COUT_INFO__ << "Creating new backbone from temporary version " << temporaryVersion
1136  << __E__;
1137 
1138  // find common available temporary version among backbone members
1139  TableVersion newVersion(TableVersion::DEFAULT);
1140  TableVersion retNewVersion;
1141  auto backboneMemberNames = ConfigurationManager::getBackboneMemberNames();
1142  for(auto& name : backboneMemberNames)
1143  {
1144  retNewVersion = ConfigurationManager::getTableByName(name)->getNextVersion();
1145  __COUT__ << "New version for backbone member (" << name << "): " << retNewVersion
1146  << __E__;
1147  if(retNewVersion > newVersion)
1148  newVersion = retNewVersion;
1149  }
1150 
1151  __COUT__ << "Common new backbone version found as " << newVersion << __E__;
1152 
1153  // create new views from source temporary version
1154  for(auto& name : backboneMemberNames)
1155  {
1156  // saveNewVersion makes the new version the active version
1157  retNewVersion = getConfigurationInterface()->saveNewVersion(
1158  getTableByName(name), temporaryVersion, newVersion);
1159  if(retNewVersion != newVersion)
1160  {
1161  __SS__ << "Failure! New view requested was " << newVersion
1162  << ". Mismatched new view created: " << retNewVersion << __E__;
1163  __COUT_ERR__ << ss.str();
1164  __SS_THROW__;
1165  }
1166  }
1167 
1168  return newVersion;
1169 }
1170 
1171 //==============================================================================
1172 void ConfigurationManagerRW::testXDAQContext()
1173 {
1174  try
1175  {
1176  __COUT__ << "Loading table..." << __E__;
1177  loadTableGroup("FETest", TableGroupKey(2)); // Context_1
1178  ConfigurationTree t = getNode("/FETable/DEFAULT/FrontEndType");
1179 
1180  std::string v;
1181 
1182  __COUT__ << __E__;
1183  t.getValue(v);
1184  __COUT__ << "Value: " << v << __E__;
1185  __COUT__ << "Value index: " << t.getValue<int>() << __E__;
1186 
1187  return;
1188  }
1189  catch(...)
1190  {
1191  __COUT__ << "Failed to load table..." << __E__;
1192  }
1193 }