otsdaq  v2_04_01
otsdaq_flatten_active_to_version.cc
1 #include <dirent.h>
2 #include <cassert>
3 #include <iostream>
4 #include <memory>
5 #include <string>
6 #include "otsdaq-core/ConfigurationInterface/ConfigurationInterface.h"
7 #include "otsdaq-core/ConfigurationInterface/ConfigurationManagerRW.h"
8 //#include "artdaq-database/StorageProviders/FileSystemDB/provider_filedb_index.h"
9 //#include "artdaq-database/JsonDocument/JSONDocument.h"
10 
11 // usage:
12 // otsdaq_flatten_active_to_version <flatVersion> <pathToSwapIn (optional)>
13 //
14 // if flatVersion is invalid or temporary nothing is saved in the new db
15 // (Note: this can be used to swap dbs using pathToSwapIn)
16 
17 using namespace ots;
18 
19 void FlattenActiveTableGroups(int argc, char* argv[])
20 {
21  std::cout << "=================================================\n";
22  std::cout << "=================================================\n";
23  std::cout << "=================================================\n";
24  __COUT__ << "\nFlattening Active Table Groups!" << std::endl;
25 
26  std::cout << "\n\nusage: Two arguments:\n\t pathToSwapIn <flatVersion> <pathToSwapIn "
27  "(optional)> \n\n"
28  << "\t Default values: flatVersion = 0, pathToSwapIn = \"\" \n\n"
29  << std::endl;
30 
31  std::cout << "\n\nNote: you can optionally just swap databases (and not modify their "
32  "contents at all)"
33  << " by providing an invalid flatVersion of -1.\n\n"
34  << std::endl;
35 
36  std::cout << "\n\nNote: This assumes artdaq db file type interface. "
37  << "The current database/ will be moved to database_<linuxtime>/ "
38  << "and if a pathToSwapIn is specified it will be copied to database/ "
39  << "before saving the currently active groups.\n\n"
40  << std::endl;
41 
42  std::cout << "argc = " << argc << std::endl;
43  for(int i = 0; i < argc; i++)
44  std::cout << "argv[" << i << "] = " << argv[i] << std::endl;
45 
46  if(argc < 2)
47  {
48  std::cout << "Must provide at least one parameter.";
49  return;
50  }
51 
52  // determine if "h"elp was first parameter
53  std::string flatVersionStr = argv[1];
54  if(flatVersionStr.find('h') != std::string::npos)
55  {
56  std::cout
57  << "Recognized parameter 1. as a 'help' option. Usage was printed. Exiting."
58  << std::endl;
59  return;
60  }
61 
62  int flatVersion = 0;
63  std::string pathToSwapIn = "";
64  if(argc >= 2)
65  sscanf(argv[1], "%d", &flatVersion);
66  if(argc >= 3)
67  pathToSwapIn = argv[2];
68 
69  __COUT__ << "flatVersion = " << flatVersion << std::endl;
70  __COUT__ << "pathToSwapIn = " << pathToSwapIn << std::endl;
71 
72  //==============================================================================
73  // Define environment variables
74  // Note: normally these environment variables are set by StartOTS.sh
75 
76  // These are needed by
77  // otsdaq/otsdaq-core/ConfigurationDataFormats/ConfigurationInfoReader.cc [207]
78  setenv("CONFIGURATION_TYPE", "File", 1); // Can be File, Database, DatabaseTest
79  setenv("CONFIGURATION_DATA_PATH",
80  (std::string(__ENV__("USER_DATA")) + "/ConfigurationDataExamples").c_str(),
81  1);
82  setenv(
83  "TABLE_INFO_PATH", (std::string(__ENV__("USER_DATA")) + "/TableInfo").c_str(), 1);
85 
86  // Some configuration plug-ins use __ENV__("SERVICE_DATA_PATH") in init() so define it
87  setenv("SERVICE_DATA_PATH",
88  (std::string(__ENV__("USER_DATA")) + "/ServiceData").c_str(),
89  1);
90 
91  // also xdaq envs for XDAQContextTable
92  setenv("XDAQ_CONFIGURATION_DATA_PATH",
93  (std::string(__ENV__("USER_DATA")) + "/XDAQConfigurations").c_str(),
94  1);
95  setenv("XDAQ_CONFIGURATION_XML", "otsConfigurationNoRU_CMake", 1);
97 
98  //==============================================================================
99  // get prepared with initial source db
100 
101  // ConfigurationManager instance immediately loads active groups
102  ConfigurationManagerRW cfgMgrInst("flatten_admin");
103  ConfigurationManagerRW* cfgMgr = &cfgMgrInst;
104 
105  __COUT__ << "\n\n\nLoading activeGroupsMap..." << std::endl;
106 
107  // save group members to reform groups
108  std::map<std::string, std::pair<std::string, TableGroupKey> > activeGroupsMap =
109  cfgMgr->getActiveTableGroups();
110 
111  std::map<std::string, std::map<std::string, TableVersion> > activeGroupMembersMap;
112  std::map<std::string, std::map<std::string /*name*/, std::string /*alias*/> >
113  activeGroupAliasesMap;
114  std::map<std::string, std::string> activeGroupCommentMap;
115  std::map<std::string, std::string> activeGroupAuthorMap;
116  std::string groupCreateTime;
117  std::map<std::string, time_t> activeGroupCreateTimeMap;
118  TableBase* groupMetadataTable = cfgMgr->getMetadataTable();
119 
120  for(auto& activeGroupPair : activeGroupsMap)
121  {
122  if(activeGroupPair.second.second.isInvalid())
123  {
124  __COUT__ << "Skipping invalid " << activeGroupPair.first << std::endl;
125  continue;
126  }
127 
128  __COUT__ << "Loading members for " << activeGroupPair.first << "\t"
129  << activeGroupPair.second.first << "(" << activeGroupPair.second.second
130  << ")" << std::endl;
131 
132  // //old way before metadata
133  // activeGroupMembersMap[activeGroupPair.second.first] =
134  // cfgMgr->getConfigurationInterface()->getTableGroupMembers(
135  // TableGroupKey::getFullGroupString(
136  // activeGroupPair.second.first,activeGroupPair.second.second));
137 
138  //=========================
139  // load group, group metadata, and tables from original DB
140  try
141  {
142  cfgMgr->loadTableGroup(
143  activeGroupPair.second.first,
144  activeGroupPair.second.second,
145  true /*doActivate*/,
146  &activeGroupMembersMap[activeGroupPair.second.first] /*memberMap*/,
147  0 /*progressBar*/,
148  &accumulateErrors,
149  &activeGroupCommentMap[activeGroupPair.second.first],
150  &activeGroupAuthorMap[activeGroupPair.second.first],
151  &groupCreateTime,
152  false /*doNotLoadMember*/,
153  0 /*groupTypeString*/,
154  &activeGroupAliasesMap[activeGroupPair.second.first]);
155  sscanf(groupCreateTime.c_str(),
156  "%ld",
157  &activeGroupCreateTimeMap[activeGroupPair.second.first]);
158  }
159  catch(std::runtime_error& e)
160  {
161  __COUT__ << "Error was caught loading members for " << groupPair.first.first
162  << "(" << groupPair.first.second << ")" << std::endl;
163  __COUT__ << e.what() << std::endl;
164  errDetected = true;
165  }
166  catch(...)
167  {
168  __COUT__ << "Error was caught loading members for " << groupPair.first.first
169  << "(" << groupPair.first.second << ")" << std::endl;
170  errDetected = true;
171  }
172 
173  //=========================
174  }
175 
176  //==============================================================================
177  // manipulate directories
178  std::string currentDir = __ENV__("ARTDAQ_DATABASE_URI");
179 
180  if(currentDir.find("filesystemdb://") != 0)
181  {
182  __SS__ << "filesystemdb:// was not found in $ARTDAQ_DATABASE_URI!" << std::endl;
183  __COUT_ERR__ << "\n" << ss.str();
184  __SS_THROW__;
185  }
186 
187  currentDir = currentDir.substr(std::string("filesystemdb://").length());
188  while(currentDir.length() &&
189  currentDir[currentDir.length() - 1] == '/') // remove trailing '/'s
190  currentDir = currentDir.substr(0, currentDir.length() - 1);
191  std::string moveToDir = currentDir + "_" + std::to_string(time(0));
192 
193  __COUT__ << "Moving current directory: \t" << currentDir << std::endl;
194  __COUT__ << "\t... to: \t\t" << moveToDir << std::endl;
195 
196  if(argc < 2)
197  {
198  __SS__ << ("Aborting move! Must at least give version argument to flatten to!")
199  << std::endl;
200  __COUT_ERR__ << "\n" << ss.str();
201  __SS_THROW__;
202  }
203 
204  rename(currentDir.c_str(), moveToDir.c_str());
205  FILE* fp = fopen((moveToDir + "/README_otsdaq_flatten.txt").c_str(), "a");
206  if(!fp)
207  __COUT__ << "\tError opening README file!" << std::endl;
208  else
209  {
210  time_t rawtime;
211  struct tm* timeinfo;
212  char buffer[200];
213 
214  time(&rawtime);
215  timeinfo = localtime(&rawtime);
216  strftime(buffer, 200, "%b %d, %Y %I:%M%p %Z", timeinfo);
217 
218  fprintf(fp,
219  "This database was moved from...\n\t %s \nto...\n\t %s \nat this time "
220  "\n\t %lu \t %s\n\n\n",
221  currentDir.c_str(),
222  moveToDir.c_str(),
223  time(0),
224  buffer);
225 
226  fclose(fp);
227  }
228 
229  if(pathToSwapIn != "")
230  {
231  DIR* dp;
232  if((dp = opendir(pathToSwapIn.c_str())) == 0)
233  {
234  __COUT__ << "ERROR:(" << errno << "). Can't open directory: " << pathToSwapIn
235  << std::endl;
236  exit(0);
237  }
238  closedir(dp);
239 
240  __COUT__ << "Swapping in directory: \t" << pathToSwapIn << std::endl;
241  __COUT__ << "\t.. to: \t\t" << currentDir << std::endl;
242 
243  rename(pathToSwapIn.c_str(), currentDir.c_str());
244  FILE* fp = fopen((currentDir + "/README_otsdaq_flatten.txt").c_str(), "a");
245  if(!fp)
246  __COUT__ << "\tError opening README file!" << std::endl;
247  else
248  {
249  time_t rawtime;
250  struct tm* timeinfo;
251  char buffer[200];
252 
253  time(&rawtime);
254  timeinfo = localtime(&rawtime);
255  strftime(buffer, 200, "%b %d, %Y %I:%M:%S%p %Z", timeinfo);
256 
257  fprintf(fp,
258  "This database was moved from...\t %s \t to...\t %s at this time \t "
259  "%lu \t %s\n\n",
260  pathToSwapIn.c_str(),
261  currentDir.c_str(),
262  time(0),
263  buffer);
264  fclose(fp);
265  }
266  }
267 
268  //==============================================================================
269  // save current active versions with flatVersion
270  // Note: do not try this at home kids.
271  ConfigurationInterface* theInterface_ = ConfigurationInterface::getInstance(
272  false); // true for File interface, false for artdaq database;
273  TableView* cfgView;
274  TableBase* config;
275 
276  std::map<std::string, TableVersion> activeMap = cfgMgr->getActiveVersions();
277 
278  // modify Group Aliases Table and Version Aliases Table to point
279  // at DEFAULT and flatVersion respectively
280 
281  const std::string groupAliasesName = ConfigurationManager::GROUP_ALIASES_TABLE_NAME;
282  const std::string versionAliasesName =
283  ConfigurationManager::VERSION_ALIASES_TABLE_NAME;
284 
285  // don't do anything more if flatVersion is not persistent
286  if(TableVersion(flatVersion).isInvalid() ||
287  TableVersion(flatVersion).isTemporaryVersion())
288  {
289  __COUT__ << "\n\nflatVersion " << TableVersion(flatVersion)
290  << " is an invalid or temporary version. Skipping to end!" << std::endl;
291  goto CLEAN_UP;
292  }
293 
294  // modify Group Aliases Table
295  if(activeMap.find(groupAliasesName) != activeMap.end())
296  {
297  __COUT__ << "\n\nModifying " << groupAliasesName << std::endl;
298  config = cfgMgr->getTableByName(groupAliasesName);
299  cfgView = config->getViewP();
300 
301  unsigned int col1 = cfgView->findCol("GroupName");
302  unsigned int col2 = cfgView->findCol("GroupKey");
303 
304  // change all key entries to active groups to DEFAULT
305  for(unsigned int row = 0; row < cfgView->getNumberOfRows(); ++row)
306  for(auto& activeGroupPair : activeGroupsMap)
307  if(activeGroupPair.second.second.isInvalid())
308  continue;
309  else if(cfgView->getDataView()[row][col1] ==
310  activeGroupPair.second.first &&
311  cfgView->getDataView()[row][col2] ==
312  activeGroupPair.second.second.toString())
313  {
314  // found a matching group/key pair
315  __COUT__ << "Changing row " << row << " for "
316  << cfgView->getDataView()[row][col1]
317  << " key=" << cfgView->getDataView()[row][col2]
318  << " to DEFAULT="
319  << TableGroupKey(TableGroupKey::getDefaultKey())
320  << std::endl;
321  cfgView->setValue(
322  TableGroupKey(TableGroupKey::getDefaultKey()).toString(),
323  row,
324  col2);
325  break;
326  }
327  }
328 
329  // modify Version Aliases Table
330  if(activeMap.find(versionAliasesName) != activeMap.end())
331  {
332  __COUT__ << "\n\nModifying " << versionAliasesName << std::endl;
333  config = cfgMgr->getTableByName(versionAliasesName);
334  cfgView = config->getViewP();
335  unsigned int col1 = cfgView->findCol("TableName");
336  unsigned int col2 = cfgView->findCol("Version");
337 
338  // change all version entries to active tables to flatVersion
339  for(unsigned int row = 0; row < cfgView->getNumberOfRows(); ++row)
340  for(auto& activePair : activeMap)
341  if(cfgView->getDataView()[row][col1] == activePair.first &&
342  cfgView->getDataView()[row][col2] == activePair.second.toString())
343  {
344  // found a matching group/key pair
345  __COUT__ << "Changing row " << row << " for "
346  << cfgView->getDataView()[row][col1]
347  << " version=" << cfgView->getDataView()[row][col2]
348  << " to flatVersion=" << TableVersion(flatVersion)
349  << std::endl;
350  cfgView->setValue(TableVersion(flatVersion).toString(), row, col2);
351  break;
352  }
353  }
354 
355  __COUT__ << "\n\nChanging versions... " << std::endl;
356 
357  for(auto& activePair : activeMap)
358  {
359  __COUT__ << activePair.first << ":v" << activePair.second << std::endl;
360  // change the version of the active view to flatVersion and save it
361  config = cfgMgr->getTableByName(activePair.first);
362  cfgView = config->getViewP();
363  cfgView->setVersion(TableVersion(flatVersion));
364  theInterface_->saveActiveVersion(config);
365  }
366 
367  //==============================================================================
368  // save the active groups with the new member flatVersions
369  // Note: do not try this at home kids.
370  for(auto& activeGroupMembersPair : activeGroupMembersMap)
371  {
372  __COUT__ << "Group " << activeGroupMembersPair.first << std::endl;
373 
374  for(auto& groupMemberPair : activeGroupMembersPair.second)
375  {
376  __COUT__ << "\t from...\t" << groupMemberPair.first << ":v"
377  << groupMemberPair.second << std::endl;
378  groupMemberPair.second = flatVersion;
379  }
380 
381  for(auto& groupMemberPair : activeGroupMembersPair.second)
382  {
383  __COUT__ << "\t to...\t" << groupMemberPair.first << ":v"
384  << groupMemberPair.second << std::endl;
385  }
386 
387  // Note: this code copies actions in ConfigurationManagerRW::saveNewTableGroup
388 
389  // add meta data
390  __COUTV__(StringMacros::mapToString(
391  activeGroupAliasesMap[activeGroupMembersPair.first]));
392  __COUTV__(activeGroupCommentMap[activeGroupMembersPair.first]);
393  __COUTV__(activeGroupAuthorMap[activeGroupMembersPair.first]);
394  __COUTV__(activeGroupCreateTimeMap[activeGroupMembersPair.first]);
395 
396  // to compensate for unusual errors upstream, make sure the metadata table has one
397  // row
398  while(groupMetadataTable->getViewP()->getNumberOfRows() > 1)
399  groupMetadataTable->getViewP()->deleteRow(0);
400  if(groupMetadataTable->getViewP()->getNumberOfRows() == 0)
401  groupMetadataTable->getViewP()->addRow();
402 
403  // columns are uid,comment,author,time
404  // ConfigurationManager::METADATA_COL_ALIASES TODO
405  groupMetadataTable->getViewP()->setValue(
406  StringMacros::mapToString(activeGroupAliasesMap[activeGroupMembersPair.first],
407  "," /*primary delimiter*/,
408  ":" /*secondary delimeter*/),
409  0,
410  ConfigurationManager::METADATA_COL_ALIASES);
411  groupMetadataTable->getViewP()->setValue(
412  activeGroupCommentMap[activeGroupMembersPair.first],
413  0,
414  ConfigurationManager::METADATA_COL_COMMENT);
415  groupMetadataTable->getViewP()->setValue(
416  activeGroupAuthorMap[activeGroupMembersPair.first],
417  0,
418  ConfigurationManager::METADATA_COL_AUTHOR);
419  groupMetadataTable->getViewP()->setValue(
420  activeGroupCreateTimeMap[activeGroupMembersPair.first],
421  0,
422  ConfigurationManager::METADATA_COL_TIMESTAMP);
423 
424  // set version of metadata table
425  groupMetadataTable->getViewP()->setVersion(TableVersion(flatVersion));
426  theInterface_->saveActiveVersion(groupMetadataTable);
427 
428  // force groupMetadataTable_ to be a member for the group
429  activeGroupMembersPair.second[groupMetadataTable->getTableName()] =
430  groupMetadataTable->getViewVersion();
431 
432  // memberMap should now consist of members with new flat version, so save group
433  theInterface_->saveTableGroup(activeGroupMembersPair.second,
434  TableGroupKey::getFullGroupString(
435  activeGroupMembersPair.first,
436  TableGroupKey(TableGroupKey::getDefaultKey())));
437  }
438 
439 CLEAN_UP:
440  //==============================================================================
441  __COUT__ << "\n\nEnd of Flattening Active Table Groups!\n\n\n" << std::endl;
442 
443  __COUT__ << "Run the following to return to your previous database structure:"
444  << std::endl;
445  __COUT__ << "\t otsdaq_flatten_active_to_version -1 " << moveToDir << "\n\n"
446  << std::endl;
447 
448  return;
449 }
450 
451 int main(int argc, char* argv[])
452 {
453  FlattenActiveTableGroups(argc, argv);
454  return 0;
455 }
456 // BOOST_AUTO_TEST_SUITE_END()