otsdaq  v2_00_00
ConfigurationManagerRW.cc
1 #include "otsdaq-core/ConfigurationInterface/ConfigurationManagerRW.h"
2 
3 //backbone includes
4 //#include "otsdaq-core/ConfigurationPluginDataFormats/ConfigurationAliases.h"
5 //#include "otsdaq-core/ConfigurationPluginDataFormats/Configurations.h"
6 //#include "otsdaq-core/ConfigurationPluginDataFormats/DefaultConfigurations.h"
7 //#include "otsdaq-core/ConfigurationPluginDataFormats/VersionAliases.h"
8 
9 //#include "otsdaq-core/ConfigurationInterface/ConfigurationInterface.h"//All configurable objects are included here
10 
11 
12 //
13 //#include "otsdaq-core/ConfigurationPluginDataFormats/DetectorConfiguration.h"
14 //#include "otsdaq-core/ConfigurationPluginDataFormats/MaskConfiguration.h"
15 //#include "otsdaq-core/ConfigurationPluginDataFormats/DetectorToFEConfiguration.h"
16 //
17 
18 //#include "otsdaq-core/ConfigurationInterface/DACStream.h"
19 //#include "otsdaq-core/ConfigurationDataFormats/ConfigurationGroupKey.h"
20 //
21 //#include "otsdaq-core/ConfigurationInterface/FileConfigurationInterface.h"
22 //
23 //#include <cassert>
24 
25 #include<dirent.h>
26 
27 using namespace ots;
28 
29 
30 #undef __MF_SUBJECT__
31 #define __MF_SUBJECT__ "ConfigurationManagerRW"
32 
33 #define CONFIGURATION_INFO_PATH std::string(getenv("CONFIGURATION_INFO_PATH")) + "/"
34 #define CONFIGURATION_INFO_EXT "Info.xml"
35 
36 //==============================================================================
37 //ConfigurationManagerRW
38 ConfigurationManagerRW::ConfigurationManagerRW(std::string username)
39 : ConfigurationManager(username) //for use as author of new views
40 {
41  __COUT__ << "Using Config Mgr with Write Access! (for " << username << ")" << std::endl;
42 
43 
44  //FIXME only necessarily temporarily while Lore is still using fileSystem xml
45  theInterface_ = ConfigurationInterface::getInstance(false); //false to use artdaq DB
46  // FIXME -- can delete this change of interface once RW and regular use same interface instance
47 }
48 
49 //==============================================================================
50 //getAllConfigurationInfo()
51 // Used by ConfigurationGUISupervisor to get all the info for the existing configurations
52 //
53 //if(accumulatedErrors)
54 // this implies allowing column errors and accumulating such errors in given string
55 const std::map<std::string, ConfigurationInfo>& ConfigurationManagerRW::getAllConfigurationInfo(
56  bool refresh, std::string *accumulatedErrors, const std::string &errorFilterName)
57 {
58  //allConfigurationInfo_ is container to be returned
59 
60  if(accumulatedErrors) *accumulatedErrors = "";
61 
62  if(!refresh) return allConfigurationInfo_;
63 
64  //else refresh!
65  allConfigurationInfo_.clear();
66  allGroupInfo_.clear();
67 
68  ConfigurationBase* configuration;
69 
70  //existing configurations are defined by which infos are in CONFIGURATION_INFO_PATH
71  //can test that the class exists based on this
72  //and then which versions
73  __COUT__ << "======================================================== getAllConfigurationInfo start" << std::endl;
74  __COUT__ << "Refreshing all! Extracting list of Configuration tables..." << std::endl;
75  DIR *pDIR;
76  struct dirent *entry;
77  std::string path = CONFIGURATION_INFO_PATH;
78  char fileExt[] = CONFIGURATION_INFO_EXT;
79  const unsigned char MIN_CONFIG_NAME_SZ = 3;
80  if( (pDIR=opendir( path.c_str() )) != 0 )
81  {
82  while((entry = readdir(pDIR)) != 0)
83  {
84  //enforce configuration name length
85  if(strlen(entry->d_name) < strlen(fileExt)+MIN_CONFIG_NAME_SZ)
86  continue;
87 
88  //find file names with correct file extenstion
89  if( strcmp(
90  &(entry->d_name[strlen(entry->d_name) - strlen(fileExt)]),
91  fileExt) != 0)
92  continue; //skip different extentions
93 
94 
95  entry->d_name[strlen(entry->d_name) - strlen(fileExt)] = '\0'; //remove file extension to get config name
96 
97  //__COUT__ << entry->d_name << std::endl;
98 
99  //0 will force the creation of new instance (and reload from Info)
100  configuration = 0;
101 
102  try //only add valid configuration instances to maps
103  {
104  theInterface_->get(configuration,
105  entry->d_name,
106  0, 0,
107  true); //dont fill
108  }
109  catch(cet::exception)
110  {
111  if(configuration) delete configuration;
112  configuration = 0;
113 
114  __COUT__ << "Skipping! No valid class found for... " << entry->d_name << "\n";
115  continue;
116  }
117  catch(std::runtime_error &e)
118  {
119  if(configuration) delete configuration;
120  configuration = 0;
121 
122  __COUT__ << "Skipping! No valid class found for... " << entry->d_name << "\n";
123  __COUT__ << "Error: " << e.what() << std::endl;
124 
125 
126  //for a runtime_error, it is likely that columns are the problem
127  // the Table Editor needs to still fix these.. so attempt to
128  // proceed.
129  if(accumulatedErrors)
130  {
131 
132  if(errorFilterName == "" ||
133  errorFilterName == entry->d_name)
134  {
135  *accumulatedErrors += std::string("\nIn table '") + entry->d_name +
136  "'..." + e.what(); //global accumulate
137 
138  __SS__ << "Attempting to allow illegal columns!" << std::endl;
139  *accumulatedErrors += ss.str();
140  }
141 
142  //attempt to recover and build a mock-up
143  __COUT__ << "Attempting to allow illegal columns!" << std::endl;
144 
145 
146  std::string returnedAccumulatedErrors;
147  try
148  {
149  //configuration = new ConfigurationBase(entry->d_name, &returnedAccumulatedErrors);
150  configuration = new ConfigurationBase(entry->d_name, &returnedAccumulatedErrors);
151  }
152  catch(...)
153  {
154  __COUT__ << "Skipping! Allowing illegal columns didn't work either... " <<
155  entry->d_name << "\n";
156  continue;
157  }
158  __COUT__ << "Error (but allowed): " << returnedAccumulatedErrors << std::endl;
159 
160  if(errorFilterName == "" ||
161  errorFilterName == entry->d_name)
162  *accumulatedErrors += std::string("\nIn table '") + entry->d_name +
163  "'..." + returnedAccumulatedErrors; //global accumulate
164  }
165  else
166  continue;
167  }
168 
169  //__COUT__ << "Instance created: " << entry->d_name << "\n"; //found!
170 
171  if(nameToConfigurationMap_[entry->d_name]) //handle if instance existed
172  {
173  //copy the temporary versions! (or else all is lost)
174  std::set<ConfigurationVersion> versions =
175  nameToConfigurationMap_[entry->d_name]->getStoredVersions();
176  for(auto &version:versions)
177  if(version.isTemporaryVersion())
178  {
179  __COUT__ << "copying tmp = " << version << std::endl;
180 
181  try //do NOT let ConfigurationView::init() throw here
182  {
183  nameToConfigurationMap_[entry->d_name]->setActiveView(version);
184  configuration->copyView( //this calls ConfigurationView::init()
185  nameToConfigurationMap_[entry->d_name]->getView(),
186  version,
187  username_);
188  }
189  catch(...) //do NOT let invalid temporary version throw at this point
190  {} //just trust configurationBase throws out the failed version
191  }
192  //__COUT__ << "deleting: " << entry->d_name << "\n"; //found!
193  delete nameToConfigurationMap_[entry->d_name];
194  nameToConfigurationMap_[entry->d_name] = 0;
195  }
196 
197  nameToConfigurationMap_[entry->d_name] = configuration;
198 
199  allConfigurationInfo_[entry->d_name].configurationPtr_ = configuration;
200  allConfigurationInfo_[entry->d_name].versions_ = theInterface_->getVersions(configuration);
201 
202  //also add any existing temporary versions to all config info
203  // because the interface wont find those versions
204  std::set<ConfigurationVersion> versions =
205  nameToConfigurationMap_[entry->d_name]->getStoredVersions();
206  for(auto &version:versions)
207  if(version.isTemporaryVersion())
208  {
209  __COUT__ << "surviving tmp = " << version << std::endl;
210  allConfigurationInfo_[entry->d_name].versions_.emplace(version);
211  }
212  }
213  closedir(pDIR);
214  }
215  __COUT__ << "Extracting list of Configuration tables complete" << std::endl;
216 
217  //call init to load active versions by default
218  init(accumulatedErrors);
219 
220  __COUT__ << "======================================================== getAllConfigurationInfo end" << std::endl;
221 
222 
223  //get Group Info too!
224  {
225  std::set<std::string /*name*/> configGroups = theInterface_->getAllConfigurationGroupNames();
226  __COUT__ << "Number of Groups: " << configGroups.size() << std::endl;
227 
229  std::string name;
230  for(const auto& fullName:configGroups)
231  {
232  ConfigurationGroupKey::getGroupNameAndKey(fullName,name,key);
233  cacheGroupKey(name,key);
234  }
235 
236  //for each group get member map & comment, author, time, and type for latest key
237  for(auto& groupInfo:allGroupInfo_)
238  {
239  try
240  {
241  loadConfigurationGroup(
242  groupInfo.first /*groupName*/,
243  groupInfo.second.getLatestKey(),
244  false /*doActivate*/,
245  &groupInfo.second.latestKeyMemberMap_ /*groupMembers*/,
246  0 /*progressBar*/,0 /*accumulateErrors*/,
247  &groupInfo.second.latestKeyGroupComment_,
248  &groupInfo.second.latestKeyGroupAuthor_,
249  &groupInfo.second.latestKeyGroupCreationTime_,
250  true /*doNotLoadMember*/,
251  &groupInfo.second.latestKeyGroupTypeString_);
252  }
253  catch(...)
254  {
255  __COUT_WARN__ << "Error occurred loading latest group info into cache for '" <<
256  groupInfo.first << "'..." << __E__;
257  groupInfo.second.latestKeyGroupComment_ = "UNKNOWN";
258  groupInfo.second.latestKeyGroupAuthor_ = "UNKNOWN";
259  groupInfo.second.latestKeyGroupCreationTime_ = "0";
260  groupInfo.second.latestKeyGroupTypeString_ = "UNKNOWN";
261  }
262  } //end group info loop
263  } //end get group info
264  return allConfigurationInfo_;
265 }
266 
267 //==============================================================================
268 //getActiveAliases()
269 // get active version aliases organized by table
270 std::map<std::string,std::map<std::string,ConfigurationVersion> >
271 ConfigurationManagerRW::getActiveVersionAliases(void) const
272 {
273  __COUT__ << "getActiveVersionAliases()" << std::endl;
274  std::map<std::string,std::map<std::string,ConfigurationVersion> > retMap;
275 
276  std::map<std::string, ConfigurationVersion> activeVersions = getActiveVersions();
277  std::string versionAliasesTableName = "VersionAliasesConfiguration";
278  if(activeVersions.find(versionAliasesTableName) == activeVersions.end())
279  {
280  __SS__ << "Active version of VersionAliases missing!" <<
281  "Make sure you have a valid active Backbone Group." << std::endl;
282  __COUT_WARN__ << "\n" << ss.str();
283  return retMap;
284  }
285 
286  __COUT__ << "activeVersions[\"VersionAliasesConfiguration\"]=" <<
287  activeVersions[versionAliasesTableName] << std::endl;
288 
289  //always have scratch alias for each table that has a scratch version
290  if(!ConfigurationInterface::isVersionTrackingEnabled())
291  for(const auto &tableInfo:allConfigurationInfo_)
292  for(const auto &version:tableInfo.second.versions_)
293  if(version.isScratchVersion())
294  retMap[tableInfo.first][ConfigurationManager::SCRATCH_VERSION_ALIAS] =
295  ConfigurationVersion(ConfigurationVersion::SCRATCH);
296 
297  std::vector<std::pair<std::string,ConfigurationTree> > aliasNodePairs =
298  getNode(versionAliasesTableName).getChildren();
299 
300  //create map
301  // add the first of each configName, versionAlias pair encountered
302  // ignore any repeats (Note: this also prevents overwriting of Scratch alias)
303  std::string configName, versionAlias;
304  for(auto& aliasNodePair:aliasNodePairs)
305  {
306  configName = aliasNodePair.second.getNode(
307  "ConfigurationName").getValueAsString();
308  versionAlias = aliasNodePair.second.getNode(
309  "VersionAlias").getValueAsString();
310 
311  if(retMap.find(configName) != retMap.end() &&
312  retMap[configName].find(versionAlias) != retMap[configName].end())
313  continue; //skip repeats (Note: this also prevents overwriting of Scratch alias)
314 
315  //else add version to map
316  retMap[configName][versionAlias] = ConfigurationVersion(
317  aliasNodePair.second.getNode("Version").getValueAsString());
318  }
319 
320  return retMap;
321 }
322 
323 //==============================================================================
324 //setActiveGlobalConfiguration
325 // load config group and activate
326 // deactivates previous config group of same type if necessary
327 void ConfigurationManagerRW::activateConfigurationGroup(const std::string &configGroupName,
328  ConfigurationGroupKey configGroupKey, std::string *accumulatedTreeErrors)
329 {
330  loadConfigurationGroup(configGroupName,configGroupKey,
331  true, //loads and activates
332  0, //no members needed
333  0, //no progress bar
334  accumulatedTreeErrors); //accumulate warnings or not
335 
336  if(accumulatedTreeErrors &&
337  *accumulatedTreeErrors != "")
338  {
339  __COUT_ERR__ << "Errors were accumulated so de-activating group: " <<
340  configGroupName << " (" << configGroupKey << ")" << std::endl;
341  try //just in case any lingering pieces, lets deactivate
342  { destroyConfigurationGroup(configGroupName,true); }
343  catch(...){}
344  }
345 
346  __COUT_INFO__ << "Updating persistent active groups to " <<
347  ConfigurationManager::ACTIVE_GROUP_FILENAME << " ..." << std::endl;
348  __MOUT_INFO__ << "Updating persistent active groups to " <<
349  ConfigurationManager::ACTIVE_GROUP_FILENAME << " ..." << std::endl;
350 
351  std::string fn = ConfigurationManager::ACTIVE_GROUP_FILENAME;
352  FILE *fp = fopen(fn.c_str(),"w");
353  if(!fp)
354  {
355  __SS__ << "Fatal Error! Unable to open the file " <<
356  ConfigurationManager::ACTIVE_GROUP_FILENAME << " for editing! Is there a permissions problem?" << std::endl;
357  __COUT_ERR__ << ss.str();
358  throw std::runtime_error(ss.str());
359  return;
360  }
361 
362  __COUT_INFO__ << "Active Context: " << theContextGroup_ << "(" <<
363  (theContextGroupKey_?theContextGroupKey_->toString().c_str():"-1") << ")" << std::endl;
364  __COUT_INFO__ << "Active Backbone: " << theBackboneGroup_ << "(" <<
365  (theBackboneGroupKey_?theBackboneGroupKey_->toString().c_str():"-1") << ")" << std::endl;
366  __COUT_INFO__ << "Active Iterate: " << theIterateGroup_ << "(" <<
367  (theIterateGroupKey_?theIterateGroupKey_->toString().c_str():"-1") << ")" << std::endl;
368  __COUT_INFO__ << "Active Configuration: " << theConfigurationGroup_ << "(" <<
369  (theConfigurationGroupKey_?theConfigurationGroupKey_->toString().c_str():"-1") << ")" << std::endl;
370 
371  __MOUT_INFO__ << "Active Context: " << theContextGroup_ << "(" <<
372  (theContextGroupKey_?theContextGroupKey_->toString().c_str():"-1") << ")" << std::endl;
373  __MOUT_INFO__ << "Active Backbone: " << theBackboneGroup_ << "(" <<
374  (theBackboneGroupKey_?theBackboneGroupKey_->toString().c_str():"-1") << ")" << std::endl;
375  __MOUT_INFO__ << "Active Iterate: " << theIterateGroup_ << "(" <<
376  (theIterateGroupKey_?theIterateGroupKey_->toString().c_str():"-1") << ")" << std::endl;
377  __MOUT_INFO__ << "Active Configuration: " << theConfigurationGroup_ << "(" <<
378  (theConfigurationGroupKey_?theConfigurationGroupKey_->toString().c_str():"-1") << ")" << std::endl;
379 
380  fprintf(fp,"%s\n",theContextGroup_.c_str());
381  fprintf(fp,"%s\n",theContextGroupKey_?theContextGroupKey_->toString().c_str():"-1");
382  fprintf(fp,"%s\n",theBackboneGroup_.c_str());
383  fprintf(fp,"%s\n",theBackboneGroupKey_?theBackboneGroupKey_->toString().c_str():"-1");
384  fprintf(fp,"%s\n",theIterateGroup_.c_str());
385  fprintf(fp,"%s\n",theIterateGroupKey_?theIterateGroupKey_->toString().c_str():"-1");
386  fprintf(fp,"%s\n",theConfigurationGroup_.c_str());
387  fprintf(fp,"%s\n",theConfigurationGroupKey_?theConfigurationGroupKey_->toString().c_str():"-1");
388  fclose(fp);
389 }
390 
391 //==============================================================================
392 //createTemporaryBackboneView
393 // sourceViewVersion of INVALID is from MockUp, else from valid view version
394 // returns temporary version number (which is always negative)
395 ConfigurationVersion ConfigurationManagerRW::createTemporaryBackboneView(ConfigurationVersion sourceViewVersion)
396 {
397  __COUT_INFO__ << "Creating temporary backbone view from version " <<
398  sourceViewVersion << std::endl;
399 
400  //find common available temporary version among backbone members
401  ConfigurationVersion tmpVersion = ConfigurationVersion::getNextTemporaryVersion(); //get the default temporary version
402  ConfigurationVersion retTmpVersion;
403  auto backboneMemberNames = ConfigurationManager::getBackboneMemberNames();
404  for (auto& name : backboneMemberNames)
405  {
406  retTmpVersion = ConfigurationManager::getConfigurationByName(name)->getNextTemporaryVersion();
407  if(retTmpVersion < tmpVersion)
408  tmpVersion = retTmpVersion;
409  }
410 
411  __COUT__ << "Common temporary backbone version found as " <<
412  tmpVersion << std::endl;
413 
414  //create temporary views from source version to destination temporary version
415  for (auto& name : backboneMemberNames)
416  {
417  retTmpVersion = getConfigurationByName(name)->createTemporaryView(sourceViewVersion, tmpVersion);
418  if(retTmpVersion != tmpVersion)
419  {
420  __SS__ << "Failure! Temporary view requested was " <<
421  tmpVersion << ". Mismatched temporary view created: " << retTmpVersion << std::endl;
422  __COUT_ERR__ << ss.str();
423  throw std::runtime_error(ss.str());
424  }
425  }
426 
427  return tmpVersion;
428 }
429 
430 
431 
432 //==============================================================================
433 ConfigurationBase* ConfigurationManagerRW::getConfigurationByName(const std::string &configurationName)
434 {
435  if(nameToConfigurationMap_.find(configurationName) == nameToConfigurationMap_.end())
436  {
437  __SS__ << "Configuration not found with name: " << configurationName << std::endl;
438  size_t f;
439  if((f=configurationName.find(' ')) != std::string::npos)
440  ss << "There was a space character found in the configuration name needle at position " <<
441  f << " in the string (was this intended?). " << std::endl;
442  __COUT_ERR__ << "\n" << ss.str();
443  throw std::runtime_error(ss.str());
444  }
445  return nameToConfigurationMap_[configurationName];
446 }
447 
448 //==============================================================================
449 //getVersionedConfigurationByName
450 // Used by configuration GUI to load a particular configuration-version pair as the active version.
451 // This configuration instance must already exist and be owned by ConfigurationManager.
452 // return null pointer on failure, on success return configuration pointer.
453 ConfigurationBase* ConfigurationManagerRW::getVersionedConfigurationByName(const std::string &configurationName,
454  ConfigurationVersion version, bool looseColumnMatching)
455 {
456  auto it = nameToConfigurationMap_.find(configurationName);
457  if(it == nameToConfigurationMap_.end())
458  {
459  __SS__ << "\nCan not find configuration named '" <<
460  configurationName <<
461  "'\n\n\n\nYou need to load the configuration before it can be used." <<
462  "It probably is missing from the member list of the Configuration Group that was loaded?\n\n\n\n\n"
463  << std::endl;
464  throw std::runtime_error(ss.str());
465  }
466  ConfigurationBase* configuration = it->second;
467  theInterface_->get(configuration, configurationName, 0 , 0,
468  false, //fill w/version
469  version,
470  false, //do not reset
471  looseColumnMatching);
472  return configuration;
473 }
474 
475 //==============================================================================
476 //saveNewConfiguration
477 // saves version, makes the new version the active version, and returns new version
478 ConfigurationVersion ConfigurationManagerRW::saveNewConfiguration(
479  const std::string &configurationName,
480  ConfigurationVersion temporaryVersion,
481  bool makeTemporary)//,
482  //bool saveToScratchVersion)
483 {
484  ConfigurationVersion newVersion(temporaryVersion);
485 
486  //set author of version
487  ConfigurationBase *config = getConfigurationByName(configurationName);
488  config->getTemporaryView(temporaryVersion)->setAuthor(username_);
489  //NOTE: somehow? author is assigned to permanent versions when saved to DBI?
490 
491  if(!makeTemporary) //saveNewVersion makes the new version the active version
492  newVersion = theInterface_->saveNewVersion(config, temporaryVersion);
493  else //make the temporary version active
494  config->setActiveView(newVersion);
495 
496  //if there is a problem, try to recover
497  while(!makeTemporary && !newVersion.isScratchVersion() &&
498  allConfigurationInfo_[configurationName].versions_.find(newVersion) !=
499  allConfigurationInfo_[configurationName].versions_.end())
500  {
501  __COUT_ERR__ << "What happenened!?? ERROR::: new persistent version v" << newVersion <<
502  " already exists!? How is it possible? Retrace your steps and tell an admin." << std::endl;
503 
504  //create a new temporary version of the target view
505  temporaryVersion = config->createTemporaryView(newVersion);
506 
507  if(newVersion.isTemporaryVersion())
508  newVersion = temporaryVersion;
509  else
510  newVersion = ConfigurationVersion::getNextVersion(newVersion);
511 
512  __COUT_WARN__ << "Attempting to recover and use v" << newVersion << std::endl;
513 
514 
515  if(!makeTemporary) //saveNewVersion makes the new version the active version
516  newVersion = theInterface_->saveNewVersion(config, temporaryVersion, newVersion);
517  else //make the temporary version active
518  config->setActiveView(newVersion);
519  }
520 
521  if(newVersion.isInvalid())
522  {
523  __SS__ << "Something went wrong saving the new version v" << newVersion <<
524  ". What happened?! (duplicates? database error?)" << std::endl;
525  __COUT_ERR__ << "\n" << ss.str();
526  throw std::runtime_error(ss.str());
527  }
528 
529  //update allConfigurationInfo_ with the new version
530  allConfigurationInfo_[configurationName].versions_.insert(newVersion);
531 
532  __COUT__ << "New version added to info " << newVersion << std::endl;
533 
534  return newVersion;
535 }
536 
537 //==============================================================================
538 //eraseTemporaryVersion
539 // if version is invalid then erases ALL temporary versions
540 //
541 // maintains allConfigurationInfo_ also while erasing
542 void ConfigurationManagerRW::eraseTemporaryVersion(const std::string &configurationName,
543  ConfigurationVersion targetVersion)
544 {
545  ConfigurationBase* config = getConfigurationByName(configurationName);
546 
547  config->trimTemporary(targetVersion);
548 
549  //if allConfigurationInfo_ is not setup, then done
550  if(allConfigurationInfo_.find(configurationName) ==
551  allConfigurationInfo_.end()) return;
552  //else cleanup config info
553 
554  if(targetVersion.isInvalid())
555  {
556  //erase all temporary versions!
557  for(auto it = allConfigurationInfo_[configurationName].versions_.begin();
558  it != allConfigurationInfo_[configurationName].versions_.end(); /*no increment*/)
559  {
560  if(it->isTemporaryVersion())
561  {
562  __COUT__ << "Removing version info: " << *it << std::endl;
563  allConfigurationInfo_[configurationName].versions_.erase(it++);
564  }
565  else
566  ++it;
567  }
568  }
569  else //erase target version only
570  {
571  __COUT__ << "Removing version info: " << targetVersion << std::endl;
572  auto it = allConfigurationInfo_[configurationName].versions_.find(targetVersion);
573  if(it == allConfigurationInfo_[configurationName].versions_.end())
574  {
575  __COUT__ << "Target version was not found in info versions..." << std::endl;
576  return;
577  }
578  allConfigurationInfo_[configurationName].versions_.erase(
579  allConfigurationInfo_[configurationName].versions_.find(targetVersion));
580  __COUT__ << "Target version was erased from info." << std::endl;
581  }
582 }
583 
584 //==============================================================================
585 //clearCachedVersions
586 // clear ALL cached persistent versions (does not erase temporary versions)
587 //
588 // maintains allConfigurationInfo_ also while erasing (trivial, do nothing)
589 void ConfigurationManagerRW::clearCachedVersions(const std::string &configurationName)
590 {
591  ConfigurationBase* config = getConfigurationByName(configurationName);
592 
593  config->trimCache(0);
594 }
595 
596 //==============================================================================
597 //clearAllCachedVersions
598 // clear ALL cached persistent versions (does not erase temporary versions)
599 //
600 // maintains allConfigurationInfo_ also while erasing (trivial, do nothing)
601 void ConfigurationManagerRW::clearAllCachedVersions()
602 {
603  for(auto configInfo: allConfigurationInfo_)
604  configInfo.second.configurationPtr_->trimCache(0);
605 }
606 
607 //==============================================================================
608 //copyViewToCurrentColumns
609 ConfigurationVersion ConfigurationManagerRW::copyViewToCurrentColumns(const std::string &configurationName,
610  ConfigurationVersion sourceVersion)
611 {
612  getConfigurationByName(configurationName)->reset();
613 
614  //make sure source version is loaded
615  //need to load with loose column rules!
616  ConfigurationBase *config = getVersionedConfigurationByName(configurationName,
617  ConfigurationVersion(sourceVersion), true);
618 
619  //copy from source version to a new temporary version
620  ConfigurationVersion newTemporaryVersion = config->copyView(config->getView(),
621  ConfigurationVersion(),username_);
622 
623  //update allConfigurationInfo_ with the new version
624  allConfigurationInfo_[configurationName].versions_.insert(newTemporaryVersion);
625 
626  return newTemporaryVersion;
627 }
628 
629 //==============================================================================
630 //cacheGroupKey
631 void ConfigurationManagerRW::cacheGroupKey(const std::string &groupName,
633 {
634  allGroupInfo_[groupName].keys_.emplace(key);
635 }
636 
637 //==============================================================================
638 //getGroupInfo
639 // the interface is slow when there are a lot of groups..
640 // so plan is to maintain local cache of recent group info
641 const GroupInfo& ConfigurationManagerRW::getGroupInfo(const std::string &groupName)
642 {
643 
644  // //NOTE: seems like this filter is taking the long amount of time
645  // std::set<std::string /*name*/> fullGroupNames =
646  // theInterface_->getAllConfigurationGroupNames(groupName); //db filter by group name
647 
648  //so instead caching ourselves...
649  auto it = allGroupInfo_.find(groupName);
650  if(it == allGroupInfo_.end())
651  {
652  __SS__ << "Group name '" << groupName << "' not found in group info! (creating empty info)" << __E__;
653  __COUT_WARN__ << ss.str();
654  //throw std::runtime_error(ss.str());
655  return allGroupInfo_[groupName];
656  }
657  return it->second;
658 }
659 
660 //==============================================================================
661 //findConfigurationGroup
662 // return group with same name and same members
663 // else return invalid key
664 //
665 // Note: this is taking too long when there are a ton of groups.
666 // Change to going back only a limited number.. (but the order also comes in alpha order from
667 // theInterface_->getAllConfigurationGroupNames which is a problem for choosing the
668 // most recent to check. )
669 ConfigurationGroupKey ConfigurationManagerRW::findConfigurationGroup(const std::string &groupName,
670  const std::map<std::string, ConfigurationVersion> &groupMemberMap)
671 {
672  // //Taking too long..... taking out check for now
673  // //if here, then no match found
674  // return ConfigurationGroupKey(); //return invalid key
675 
676  // //NOTE: seems like this filter is taking the long amount of time
677  // std::set<std::string /*name*/> fullGroupNames =
678  // theInterface_->getAllConfigurationGroupNames(groupName); //db filter by group name
679 
680  const GroupInfo& groupInfo = getGroupInfo(groupName);
681 
682  //std::string name;
683  //ConfigurationGroupKey key;
684  std::map<std::string /*name*/, ConfigurationVersion /*version*/> compareToMemberMap;
685  bool isDifferent;
686 
687  const unsigned int MAX_DEPTH_TO_CHECK = 20;
688  unsigned int keyMinToCheck = 0;
689 
690  if(groupInfo.keys_.size())
691  keyMinToCheck = groupInfo.keys_.rbegin()->key();
692  if(keyMinToCheck > MAX_DEPTH_TO_CHECK)
693  {
694  keyMinToCheck -= MAX_DEPTH_TO_CHECK;
695  __COUT__ << "Checking groups back to key... " << keyMinToCheck << std::endl;
696  }
697  else
698  {
699  keyMinToCheck = 0;
700  __COUT__ << "Checking all groups." << std::endl;
701  }
702 
703 
704  //determine min key to check
705 // if(groupInfo.keys_.size() > MAX_DEPTH_TO_CHECK)
706 // {
707 // //find keyMinToCheck to avoid looking at all groups
708 // for(const auto& key: groupInfo.keys_)
709 // {
710 // //ConfigurationGroupKey::getGroupNameAndKey(fullName,name,key);
711 // if(key.key() > keyMinToCheck)
712 // keyMinToCheck = key.key();
713 // }
714 //
715 // //just in case of craziness, check not crossing 0
716 // if(keyMinToCheck >= MAX_DEPTH_TO_CHECK - 1)
717 // keyMinToCheck -= MAX_DEPTH_TO_CHECK - 1;
718 // else
719 // keyMinToCheck = 0;
720 //
721 // __COUT__ << "Checking groups back to key... " << keyMinToCheck << std::endl;
722 // }
723 // else
724 // __COUT__ << "Checking all groups." << std::endl;
725 
726  //have min key to check, now loop through and check groups
727  std::string fullName;
728  for(const auto& key: groupInfo.keys_)
729  {
730  //ConfigurationGroupKey::getGroupNameAndKey(fullName,name,key);
731 
732  if(key.key() < keyMinToCheck) continue; //skip keys that are too old
733 
734  //__COUT__ << fullName << " has name " << name << " ==? " << groupName << std::endl;
735  //if( name != groupName) continue; // superfluous check, but just to make sure the db filter worked
736 
737  fullName = ConfigurationGroupKey::getFullGroupString(groupName,key);
738 
739  __COUT__ << "checking group... " << fullName << std::endl;
740  compareToMemberMap = theInterface_->getConfigurationGroupMembers(fullName);
741 
742  isDifferent = false;
743  for(auto &memberPair: groupMemberMap)
744  {
745  //__COUT__ << memberPair.first << " - " << memberPair.second << std::endl;
746  if(compareToMemberMap.find(memberPair.first) == compareToMemberMap.end() || //name is missing
747  memberPair.second != compareToMemberMap[memberPair.first]) //or version mismatch
748  { //then different
749  //__COUT__ << "mismatch found!" << std::endl;
750  isDifferent = true;
751  break;
752  }
753  }
754  if(isDifferent) continue;
755 
756  //check member size for exact match
757  if(groupMemberMap.size() != compareToMemberMap.size()) continue; //different size, so not same (groupMemberMap is a subset of memberPairs)
758 
759  __COUT__ << "Found exact match with key: " << key << std::endl;
760  //else found an exact match!
761  return key;
762  }
763  __COUT__ << "No match found - this group is new!" << std::endl;
764  //if here, then no match found
765  return ConfigurationGroupKey(); //return invalid key
766 }
767 
768 
769 //==============================================================================
770 //saveNewConfigurationGroup
771 // saves new group and returns the new group key
772 // if previousVersion is provided, attempts to just bump that version
773 // else, bumps latest version found in db
774 //
775 // Note: groupMembers map will get modified with group metadata table version
776 ConfigurationGroupKey ConfigurationManagerRW::saveNewConfigurationGroup(
777  const std::string &groupName,
778  std::map<std::string, ConfigurationVersion> &groupMembers,
779  const std::string &groupComment)
780 {
781  //steps:
782  // determine new group key
783  // verify group members
784  // verify groupNameWithKey
785  // verify store
786 
787 
788  if(groupMembers.size() == 0) //do not allow empty groups
789  {
790  __SS__ << "Empty group member list. Can not create a group without members!" << std::endl;
791  __COUT_ERR__ << ss.str();
792  throw std::runtime_error(ss.str());
793  }
794 
795  //determine new group key
796  ConfigurationGroupKey newKey;
797  //CHANGED do not allow bumping, it is too risky.. because database allows it.. and then it corrupts database
798  //if(!previousVersion.isInvalid()) //if previous provided, bump that
799  // newKey = ConfigurationGroupKey::getNextKey(previousVersion);
800  //else //get latest key from db, and bump
801  {
802  std::set<ConfigurationGroupKey> keys = theInterface_->getKeys(groupName);
803  if(keys.size()) //if keys exist, bump the last
804  newKey = ConfigurationGroupKey::getNextKey(*(keys.crbegin()));
805  else //else, take default
806  newKey = ConfigurationGroupKey::getDefaultKey();
807  }
808 
809  __COUT__ << "New Key for group: " << groupName << " found as " << newKey << std::endl;
810 
811  // verify group members
812  // - use all config info
813  std::map<std::string, ConfigurationInfo> allCfgInfo = getAllConfigurationInfo();
814  for(auto &memberPair : groupMembers )
815  {
816  //check member name
817  if(allCfgInfo.find(memberPair.first) == allCfgInfo.end())
818  {
819  __COUT_ERR__ << "Group member \"" << memberPair.first << "\" not found in configuration!";
820 
821  if(groupMetadataTable_.getConfigurationName() ==
822  memberPair.first)
823  {
824  __COUT_WARN__ << "Looks like this is the groupMetadataTable_. " <<
825  "Note that this table is added to the member map when groups are saved." <<
826  "It should not be part of member map when calling this function." << std::endl;
827  __COUT__ << "Attempting to recover." << std::endl;
828  groupMembers.erase(groupMembers.find(memberPair.first));
829  }
830  else
831  {
832  __SS__ << ("Group member not found!") << std::endl;
833  __COUT_ERR__ << ss.str();
834  throw std::runtime_error(ss.str());
835  }
836  }
837  //check member version
838  if(allCfgInfo[memberPair.first].versions_.find(memberPair.second) ==
839  allCfgInfo[memberPair.first].versions_.end())
840  {
841  __SS__ << "Group member \"" << memberPair.first << "\" version \"" <<
842  memberPair.second << "\" not found in configuration!";
843  __COUT_ERR__ << ss.str();
844  throw std::runtime_error(ss.str());
845  }
846  }
847 
848  // verify groupNameWithKey and attempt to store
849  try
850  {
851  //save meta data for group; reuse groupMetadataTable_
852 
853  __COUT__ << username_ << " " << time(0) << " " << groupComment << std::endl;
854  //columns are uid,comment,author,time
855  groupMetadataTable_.getViewP()->setValue(groupComment,0,1);
856  groupMetadataTable_.getViewP()->setValue(username_,0,2);
857  groupMetadataTable_.getViewP()->setValue(time(0),0,3);
858 
859  //set version to first available persistent version
860  groupMetadataTable_.getViewP()->setVersion(
861  ConfigurationVersion::getNextVersion(theInterface_->findLatestVersion(&groupMetadataTable_))
862  );
863 
864  //groupMetadataTable_.print();
865 
866  theInterface_->saveActiveVersion(&groupMetadataTable_);
867 
868  //force groupMetadataTable_ to be a member for the group
869  groupMembers[groupMetadataTable_.getConfigurationName()] =
870  groupMetadataTable_.getViewVersion();
871 
872  theInterface_->saveConfigurationGroup(groupMembers,
873  ConfigurationGroupKey::getFullGroupString(groupName,newKey));
874  __COUT__ << "Created config group: " << groupName << ":" << newKey << std::endl;
875  }
876  catch(std::runtime_error &e)
877  {
878  __COUT_ERR__ << "Failed to create config group: " << groupName << ":" << newKey << std::endl;
879  __COUT_ERR__ << "\n\n" << e.what() << std::endl;
880  throw;
881  }
882  catch(...)
883  {
884  __COUT_ERR__ << "Failed to create config group: " << groupName << ":" << newKey << std::endl;
885  throw;
886  }
887 
888 
889  //store cache of recent groups
890  cacheGroupKey(groupName,newKey);
891 
892  // at this point succeeded!
893  return newKey;
894 }
895 
896 //==============================================================================
897 //saveNewBackbone
898 // makes the new version the active version and returns new version number
899 // INVALID will give a new backbone from mockup
900 ConfigurationVersion ConfigurationManagerRW::saveNewBackbone(ConfigurationVersion temporaryVersion)
901 {
902  __COUT_INFO__ << "Creating new backbone from temporary version " <<
903  temporaryVersion << std::endl;
904 
905  //find common available temporary version among backbone members
906  ConfigurationVersion newVersion(ConfigurationVersion::DEFAULT);
907  ConfigurationVersion retNewVersion;
908  auto backboneMemberNames = ConfigurationManager::getBackboneMemberNames();
909  for (auto& name : backboneMemberNames)
910  {
911  retNewVersion = ConfigurationManager::getConfigurationByName(name)->getNextVersion();
912  __COUT__ << "New version for backbone member (" << name << "): " <<
913  retNewVersion << std::endl;
914  if(retNewVersion > newVersion)
915  newVersion = retNewVersion;
916  }
917 
918  __COUT__ << "Common new backbone version found as " <<
919  newVersion << std::endl;
920 
921  //create new views from source temporary version
922  for (auto& name : backboneMemberNames)
923  {
924  //saveNewVersion makes the new version the active version
925  retNewVersion = getConfigurationInterface()->saveNewVersion(
926  getConfigurationByName(name), temporaryVersion, newVersion);
927  if(retNewVersion != newVersion)
928  {
929  __SS__ << "Failure! New view requested was " <<
930  newVersion << ". Mismatched new view created: " << retNewVersion << std::endl;
931  __COUT_ERR__ << ss.str();
932  throw std::runtime_error(ss.str());
933  }
934  }
935 
936  return newVersion;
937 }
938 
939 //==============================================================================
940 void ConfigurationManagerRW::testXDAQContext()
941 {
942 // //test creating config group with author, create time, and comment
943 // {
944 // __COUT__ << std::endl;
945 //
946 // std::string groupName = "testGroup";
947 // ConfigurationGroupKey newKey;
948 // //ConfigurationGroupMetadata
949 //
950 // try
951 // {
952 // {
953 // std::map<std::string, ConfigurationVersion> members;
954 // members["ARTDAQAggregatorConfiguration"] = ConfigurationVersion(1);
955 //
956 // //ConfigurationGroupKey
957 // // saveNewConfigurationGroup
958 // // (const std::string &groupName, const std::map<std::string,
959 // // ConfigurationVersion> &groupMembers, ConfigurationGroupKey previousVersion=ConfigurationGroupKey());
960 // newKey =
961 // saveNewConfigurationGroup(groupName,members);
962 // }
963 //
964 // //std::map<std::string, ConfigurationVersion >
965 // // loadConfigurationGroup
966 // // (const std::string &configGroupName,
967 // // ConfigurationGroupKey configGroupKey, bool doActivate=false, ProgressBar* progressBar=0, std::string *accumulateWarnings=0);
968 // {
969 // std::map<std::string, ConfigurationVersion > memberMap =
970 // loadConfigurationGroup(groupName,newKey);
971 // __COUT__ << "Group members:" << std::endl;
972 // for(const auto &member: memberMap)
973 // __COUT__ << member.first << " " << member.second << std::endl;
974 // }
975 //
976 //
977 //
978 // //do it again
979 // {
980 // std::map<std::string, ConfigurationVersion> members;
981 // members["ARTDAQAggregatorConfiguration"] = ConfigurationVersion(1);
982 //
983 // //ConfigurationGroupKey
984 // // saveNewConfigurationGroup
985 // // (const std::string &groupName, const std::map<std::string,
986 // // ConfigurationVersion> &groupMembers, ConfigurationGroupKey previousVersion=ConfigurationGroupKey());
987 // newKey =
988 // saveNewConfigurationGroup(groupName,members);
989 // }
990 //
991 // //std::map<std::string, ConfigurationVersion >
992 // // loadConfigurationGroup
993 // // (const std::string &configGroupName,
994 // // ConfigurationGroupKey configGroupKey, bool doActivate=false, ProgressBar* progressBar=0, std::string *accumulateWarnings=0);
995 // {
996 // std::map<std::string, ConfigurationVersion > memberMap =
997 // loadConfigurationGroup(groupName,newKey);
998 // __COUT__ << "Group members:" << std::endl;
999 // for(const auto &member: memberMap)
1000 // __COUT__ << member.first << " " << member.second << std::endl;
1001 // }
1002 //
1003 //
1004 // }
1005 // catch(std::runtime_error &e)
1006 // {
1007 // __COUT_ERR__ << "Failed to create config group: " << groupName << ":" << newKey << std::endl;
1008 // __COUT_ERR__ << "\n\n" << e.what() << std::endl;
1009 // }
1010 // catch(...)
1011 // {
1012 // __COUT_ERR__ << "Failed to create config group: " << groupName << ":" << newKey << std::endl;
1013 // }
1014 //
1015 // return;
1016 // }
1017 
1018 // //test creating config group and reading
1019 // {
1020 // __COUT__ << std::endl;
1021 //
1022 // auto gcfgs = theInterface_->getAllConfigurationGroupNames();
1023 // __COUT__ << "Global config size: " << gcfgs.size() << std::endl;
1024 // for(auto &g:gcfgs)
1025 // {
1026 // __COUT__ << "Global config " << g << std::endl;
1027 // auto gcMap = theInterface_->getConfigurationGroupMembers(g);
1028 //
1029 // for(auto &cv:gcMap)
1030 // __COUT__ << "\tMember config " << cv.first << ":" << cv.second << std::endl;
1031 // }
1032 //
1033 // auto cfgs = theInterface_->getAllConfigurationNames();
1034 // __COUT__ << "Sub-config size: " << cfgs.size() << std::endl;
1035 // for(auto &c:cfgs)
1036 // {
1037 // __COUT__ << "config " << c << std::endl;
1038 // auto vs = theInterface_->getVersions(getConfigurationByName(c));
1039 // for(auto &v:vs)
1040 // __COUT__ << "\tversion " << v << std::endl;
1041 // }
1042 //
1043 // if(0) //create a global config group (storeGlobalConfiguration)
1044 // {
1045 // //storeGlobalConfiguration with latest of each
1046 // std::map<std::string /*name*/, ConfigurationVersion /*version*/> gcMap;
1047 //
1048 // for(auto &c:cfgs) //for each sub-config, take latest version
1049 // {
1050 // auto vs = theInterface_->getVersions(getConfigurationByName(c));
1051 // if(1 && vs.rbegin() != vs.rend()) //create latest!
1052 // {
1053 // gcMap[c] = *(vs.rbegin());
1054 // __COUT__ << "Adding config " << c << ":" << gcMap[c] << std::endl;
1055 // }
1056 // else if(vs.begin() != vs.end()) //create oldest!
1057 // {
1058 // gcMap[c] = *(vs.begin());
1059 // __COUT__ << "Adding config " << c << ":" << gcMap[c] << std::endl;
1060 // }
1061 // }
1062 //
1063 // int i = 0;
1064 // bool done = false;
1065 // char gcname[100];
1066 // bool found;
1067 // while(!done && i<1)
1068 // {
1069 // do //avoid conflicting global config names
1070 // {
1071 // found = false;
1072 // sprintf(gcname,"GConfig_v%d",i++);
1073 // for(auto &g:gcfgs)
1074 // if(g == gcname) {found = true; break;}
1075 // }
1076 // while(found);
1077 // __COUT__ << "Trying Global config: " << gcname<< std::endl;
1078 //
1079 // try
1080 // {
1081 // theInterface_->saveConfigurationGroup(gcMap,gcname);
1082 // done = true;
1083 // __COUT__ << "Created Global config: " << gcname<< std::endl;
1084 // }
1085 // catch(...) {++i;} //repeat names are not allowed, so increment name
1086 // }
1087 //
1088 // }
1089 //
1090 // //return;
1091 // }
1092 //
1093  //this is to test config tree
1094  try
1095  {
1096  __COUT__ << "Loading config..." << std::endl;
1097  loadConfigurationGroup("FETest",ConfigurationGroupKey(2)); // Context_1
1098  ConfigurationTree t = getNode("/FEConfiguration/DEFAULT/FrontEndType");
1099 
1100  std::string v;
1101 
1102  __COUT__ << std::endl;
1103  t.getValue(v);
1104  __COUT__ << "Value: " << v << std::endl;
1105  __COUT__ << "Value index: " << t.getValue<int>() << std::endl;
1106 
1107  return;
1108 
1109  //ConfigurationTree t = getNode("XDAQContextConfiguration");
1110  //ConfigurationTree t = getNode("/FEConfiguration/OtsUDPFSSR3/FrontEndType");
1111  //ConfigurationTree t = getNode("/FEConfiguration").getNode("OtsUDPFSSR3");
1112 // ConfigurationTree t = getNode("/XDAQContextConfiguration/testContext/");
1113 //
1114 // __COUT__ << std::endl;
1115 // t.getValue(v);
1116 // __COUT__ << "Value: " << v << std::endl;
1117 //
1118 // if(!t.isValueNode())
1119 // {
1120 // auto C = t.getChildrenNames();
1121 // for(auto &c: C)
1122 // __COUT__ << "\t+ " << c << std::endl;
1123 //
1124 // std::stringstream ss;
1125 // t.print(-1,ss);
1126 // __COUT__ << "\n" << ss.str() << std::endl;
1127 //
1128 // try
1129 // {
1130 // ConfigurationTree tt = t.getNode("OtsUDPFSSR3");
1131 // tt.getValue(v);
1132 // __COUT__ << "Value: " << v << std::endl;
1133 //
1134 // C = tt.getChildrenNames();
1135 // for(auto &c: C)
1136 // __COUT__ << "\t+ " << c << std::endl;
1137 // }
1138 // catch(...)
1139 // {
1140 // __COUT__ << "Failed to find extra node." << std::endl;
1141 // }
1142 // }
1143 
1144  }
1145  catch(...)
1146  {
1147  __COUT__ << "Failed to load config..." << std::endl;
1148  }
1149 
1150 
1151 }
1152 
1153 
1154 
1155 
1156 
1157 
1158 
1159 
1160 
1161 
1162 
1163 
1164