$treeview $search $mathjax $extrastylesheet
otsdaq
v2_03_00
$projectbrief
|
$projectbrief
|
$searchbox |
00001 #include <dirent.h> 00002 #include <cassert> 00003 #include <iostream> 00004 #include <memory> 00005 #include <string> 00006 #include "otsdaq-core/ConfigurationInterface/ConfigurationInterface.h" 00007 #include "otsdaq-core/ConfigurationInterface/ConfigurationManagerRW.h" 00008 //#include "artdaq-database/StorageProviders/FileSystemDB/provider_filedb_index.h" 00009 //#include "artdaq-database/JsonDocument/JSONDocument.h" 00010 00011 // usage: 00012 // otsdaq_flatten_active_to_version <flatVersion> <pathToSwapIn (optional)> 00013 // 00014 // if flatVersion is invalid or temporary nothing is saved in the new db 00015 // (Note: this can be used to swap dbs using pathToSwapIn) 00016 00017 using namespace ots; 00018 00019 void FlattenActiveTableGroups(int argc, char* argv[]) 00020 { 00021 std::cout << "=================================================\n"; 00022 std::cout << "=================================================\n"; 00023 std::cout << "=================================================\n"; 00024 __COUT__ << "\nFlattening Active Table Groups!" << std::endl; 00025 00026 std::cout << "\n\nusage: Two arguments:\n\t pathToSwapIn <flatVersion> <pathToSwapIn " 00027 "(optional)> \n\n" 00028 << "\t Default values: flatVersion = 0, pathToSwapIn = \"\" \n\n" 00029 << std::endl; 00030 00031 std::cout << "\n\nNote: you can optionally just swap databases (and not modify their " 00032 "contents at all)" 00033 << " by providing an invalid flatVersion of -1.\n\n" 00034 << std::endl; 00035 00036 std::cout << "\n\nNote: This assumes artdaq db file type interface. " 00037 << "The current database/ will be moved to database_<linuxtime>/ " 00038 << "and if a pathToSwapIn is specified it will be copied to database/ " 00039 << "before saving the currently active groups.\n\n" 00040 << std::endl; 00041 00042 std::cout << "argc = " << argc << std::endl; 00043 for(int i = 0; i < argc; i++) 00044 std::cout << "argv[" << i << "] = " << argv[i] << std::endl; 00045 00046 if(argc < 2) 00047 { 00048 std::cout << "Must provide at least one parameter."; 00049 return; 00050 } 00051 00052 // determine if "h"elp was first parameter 00053 std::string flatVersionStr = argv[1]; 00054 if(flatVersionStr.find('h') != std::string::npos) 00055 { 00056 std::cout 00057 << "Recognized parameter 1. as a 'help' option. Usage was printed. Exiting." 00058 << std::endl; 00059 return; 00060 } 00061 00062 int flatVersion = 0; 00063 std::string pathToSwapIn = ""; 00064 if(argc >= 2) 00065 sscanf(argv[1], "%d", &flatVersion); 00066 if(argc >= 3) 00067 pathToSwapIn = argv[2]; 00068 00069 __COUT__ << "flatVersion = " << flatVersion << std::endl; 00070 __COUT__ << "pathToSwapIn = " << pathToSwapIn << std::endl; 00071 00072 //============================================================================== 00073 // Define environment variables 00074 // Note: normally these environment variables are set by StartOTS.sh 00075 00076 // These are needed by 00077 // otsdaq/otsdaq-core/ConfigurationDataFormats/ConfigurationInfoReader.cc [207] 00078 setenv("CONFIGURATION_TYPE", "File", 1); // Can be File, Database, DatabaseTest 00079 setenv("CONFIGURATION_DATA_PATH", 00080 (std::string(getenv("USER_DATA")) + "/ConfigurationDataExamples").c_str(), 00081 1); 00082 setenv( 00083 "TABLE_INFO_PATH", (std::string(getenv("USER_DATA")) + "/TableInfo").c_str(), 1); 00085 00086 // Some configuration plug-ins use getenv("SERVICE_DATA_PATH") in init() so define it 00087 setenv("SERVICE_DATA_PATH", 00088 (std::string(getenv("USER_DATA")) + "/ServiceData").c_str(), 00089 1); 00090 00091 // also xdaq envs for XDAQContextTable 00092 setenv("XDAQ_CONFIGURATION_DATA_PATH", 00093 (std::string(getenv("USER_DATA")) + "/XDAQConfigurations").c_str(), 00094 1); 00095 setenv("XDAQ_CONFIGURATION_XML", "otsConfigurationNoRU_CMake", 1); 00097 00098 //============================================================================== 00099 // get prepared with initial source db 00100 00101 // ConfigurationManager instance immediately loads active groups 00102 ConfigurationManagerRW cfgMgrInst("flatten_admin"); 00103 ConfigurationManagerRW* cfgMgr = &cfgMgrInst; 00104 00105 __COUT__ << "\n\n\nLoading activeGroupsMap..." << std::endl; 00106 00107 // save group members to reform groups 00108 std::map<std::string, std::pair<std::string, TableGroupKey> > activeGroupsMap = 00109 cfgMgr->getActiveTableGroups(); 00110 00111 std::map<std::string, std::map<std::string, TableVersion> > activeGroupMembersMap; 00112 std::map<std::string, std::map<std::string /*name*/, std::string /*alias*/> > 00113 activeGroupAliasesMap; 00114 std::map<std::string, std::string> activeGroupCommentMap; 00115 std::map<std::string, std::string> activeGroupAuthorMap; 00116 std::string groupCreateTime; 00117 std::map<std::string, time_t> activeGroupCreateTimeMap; 00118 TableBase* groupMetadataTable = cfgMgr->getMetadataTable(); 00119 00120 for(auto& activeGroupPair : activeGroupsMap) 00121 { 00122 if(activeGroupPair.second.second.isInvalid()) 00123 { 00124 __COUT__ << "Skipping invalid " << activeGroupPair.first << std::endl; 00125 continue; 00126 } 00127 00128 __COUT__ << "Loading members for " << activeGroupPair.first << "\t" 00129 << activeGroupPair.second.first << "(" << activeGroupPair.second.second 00130 << ")" << std::endl; 00131 00132 // //old way before metadata 00133 // activeGroupMembersMap[activeGroupPair.second.first] = 00134 // cfgMgr->getConfigurationInterface()->getTableGroupMembers( 00135 // TableGroupKey::getFullGroupString( 00136 // activeGroupPair.second.first,activeGroupPair.second.second)); 00137 00138 //========================= 00139 // load group, group metadata, and tables from original DB 00140 try 00141 { 00142 cfgMgr->loadTableGroup( 00143 activeGroupPair.second.first, 00144 activeGroupPair.second.second, 00145 true /*doActivate*/, 00146 &activeGroupMembersMap[activeGroupPair.second.first] /*memberMap*/, 00147 0 /*progressBar*/, 00148 &accumulateErrors, 00149 &activeGroupCommentMap[activeGroupPair.second.first], 00150 &activeGroupAuthorMap[activeGroupPair.second.first], 00151 &groupCreateTime, 00152 false /*doNotLoadMember*/, 00153 0 /*groupTypeString*/, 00154 &activeGroupAliasesMap[activeGroupPair.second.first]); 00155 sscanf(groupCreateTime.c_str(), 00156 "%ld", 00157 &activeGroupCreateTimeMap[activeGroupPair.second.first]); 00158 } 00159 catch(std::runtime_error& e) 00160 { 00161 __COUT__ << "Error was caught loading members for " << groupPair.first.first 00162 << "(" << groupPair.first.second << ")" << std::endl; 00163 __COUT__ << e.what() << std::endl; 00164 errDetected = true; 00165 } 00166 catch(...) 00167 { 00168 __COUT__ << "Error was caught loading members for " << groupPair.first.first 00169 << "(" << groupPair.first.second << ")" << std::endl; 00170 errDetected = true; 00171 } 00172 00173 //========================= 00174 } 00175 00176 //============================================================================== 00177 // manipulate directories 00178 std::string currentDir = getenv("ARTDAQ_DATABASE_URI"); 00179 00180 if(currentDir.find("filesystemdb://") != 0) 00181 { 00182 __SS__ << "filesystemdb:// was not found in $ARTDAQ_DATABASE_URI!" << std::endl; 00183 __COUT_ERR__ << "\n" << ss.str(); 00184 __SS_THROW__; 00185 } 00186 00187 currentDir = currentDir.substr(std::string("filesystemdb://").length()); 00188 while(currentDir.length() && 00189 currentDir[currentDir.length() - 1] == '/') // remove trailing '/'s 00190 currentDir = currentDir.substr(0, currentDir.length() - 1); 00191 std::string moveToDir = currentDir + "_" + std::to_string(time(0)); 00192 00193 __COUT__ << "Moving current directory: \t" << currentDir << std::endl; 00194 __COUT__ << "\t... to: \t\t" << moveToDir << std::endl; 00195 00196 if(argc < 2) 00197 { 00198 __SS__ << ("Aborting move! Must at least give version argument to flatten to!") 00199 << std::endl; 00200 __COUT_ERR__ << "\n" << ss.str(); 00201 __SS_THROW__; 00202 } 00203 00204 rename(currentDir.c_str(), moveToDir.c_str()); 00205 FILE* fp = fopen((moveToDir + "/README_otsdaq_flatten.txt").c_str(), "a"); 00206 if(!fp) 00207 __COUT__ << "\tError opening README file!" << std::endl; 00208 else 00209 { 00210 time_t rawtime; 00211 struct tm* timeinfo; 00212 char buffer[200]; 00213 00214 time(&rawtime); 00215 timeinfo = localtime(&rawtime); 00216 strftime(buffer, 200, "%b %d, %Y %I:%M%p %Z", timeinfo); 00217 00218 fprintf(fp, 00219 "This database was moved from...\n\t %s \nto...\n\t %s \nat this time " 00220 "\n\t %lu \t %s\n\n\n", 00221 currentDir.c_str(), 00222 moveToDir.c_str(), 00223 time(0), 00224 buffer); 00225 00226 fclose(fp); 00227 } 00228 00229 if(pathToSwapIn != "") 00230 { 00231 DIR* dp; 00232 if((dp = opendir(pathToSwapIn.c_str())) == 0) 00233 { 00234 __COUT__ << "ERROR:(" << errno << "). Can't open directory: " << pathToSwapIn 00235 << std::endl; 00236 exit(0); 00237 } 00238 closedir(dp); 00239 00240 __COUT__ << "Swapping in directory: \t" << pathToSwapIn << std::endl; 00241 __COUT__ << "\t.. to: \t\t" << currentDir << std::endl; 00242 00243 rename(pathToSwapIn.c_str(), currentDir.c_str()); 00244 FILE* fp = fopen((currentDir + "/README_otsdaq_flatten.txt").c_str(), "a"); 00245 if(!fp) 00246 __COUT__ << "\tError opening README file!" << std::endl; 00247 else 00248 { 00249 time_t rawtime; 00250 struct tm* timeinfo; 00251 char buffer[200]; 00252 00253 time(&rawtime); 00254 timeinfo = localtime(&rawtime); 00255 strftime(buffer, 200, "%b %d, %Y %I:%M:%S%p %Z", timeinfo); 00256 00257 fprintf(fp, 00258 "This database was moved from...\t %s \t to...\t %s at this time \t " 00259 "%lu \t %s\n\n", 00260 pathToSwapIn.c_str(), 00261 currentDir.c_str(), 00262 time(0), 00263 buffer); 00264 fclose(fp); 00265 } 00266 } 00267 00268 //============================================================================== 00269 // save current active versions with flatVersion 00270 // Note: do not try this at home kids. 00271 ConfigurationInterface* theInterface_ = ConfigurationInterface::getInstance( 00272 false); // true for File interface, false for artdaq database; 00273 TableView* cfgView; 00274 TableBase* config; 00275 00276 std::map<std::string, TableVersion> activeMap = cfgMgr->getActiveVersions(); 00277 00278 // modify Group Aliases Table and Version Aliases Table to point 00279 // at DEFAULT and flatVersion respectively 00280 00281 const std::string groupAliasesName = ConfigurationManager::GROUP_ALIASES_TABLE_NAME; 00282 const std::string versionAliasesName = 00283 ConfigurationManager::VERSION_ALIASES_TABLE_NAME; 00284 00285 // don't do anything more if flatVersion is not persistent 00286 if(TableVersion(flatVersion).isInvalid() || 00287 TableVersion(flatVersion).isTemporaryVersion()) 00288 { 00289 __COUT__ << "\n\nflatVersion " << TableVersion(flatVersion) 00290 << " is an invalid or temporary version. Skipping to end!" << std::endl; 00291 goto CLEAN_UP; 00292 } 00293 00294 // modify Group Aliases Table 00295 if(activeMap.find(groupAliasesName) != activeMap.end()) 00296 { 00297 __COUT__ << "\n\nModifying " << groupAliasesName << std::endl; 00298 config = cfgMgr->getTableByName(groupAliasesName); 00299 cfgView = config->getViewP(); 00300 00301 unsigned int col1 = cfgView->findCol("GroupName"); 00302 unsigned int col2 = cfgView->findCol("GroupKey"); 00303 00304 // change all key entries to active groups to DEFAULT 00305 for(unsigned int row = 0; row < cfgView->getNumberOfRows(); ++row) 00306 for(auto& activeGroupPair : activeGroupsMap) 00307 if(activeGroupPair.second.second.isInvalid()) 00308 continue; 00309 else if(cfgView->getDataView()[row][col1] == 00310 activeGroupPair.second.first && 00311 cfgView->getDataView()[row][col2] == 00312 activeGroupPair.second.second.toString()) 00313 { 00314 // found a matching group/key pair 00315 __COUT__ << "Changing row " << row << " for " 00316 << cfgView->getDataView()[row][col1] 00317 << " key=" << cfgView->getDataView()[row][col2] 00318 << " to DEFAULT=" 00319 << TableGroupKey(TableGroupKey::getDefaultKey()) 00320 << std::endl; 00321 cfgView->setValue( 00322 TableGroupKey(TableGroupKey::getDefaultKey()).toString(), 00323 row, 00324 col2); 00325 break; 00326 } 00327 } 00328 00329 // modify Version Aliases Table 00330 if(activeMap.find(versionAliasesName) != activeMap.end()) 00331 { 00332 __COUT__ << "\n\nModifying " << versionAliasesName << std::endl; 00333 config = cfgMgr->getTableByName(versionAliasesName); 00334 cfgView = config->getViewP(); 00335 unsigned int col1 = cfgView->findCol("TableName"); 00336 unsigned int col2 = cfgView->findCol("Version"); 00337 00338 // change all version entries to active tables to flatVersion 00339 for(unsigned int row = 0; row < cfgView->getNumberOfRows(); ++row) 00340 for(auto& activePair : activeMap) 00341 if(cfgView->getDataView()[row][col1] == activePair.first && 00342 cfgView->getDataView()[row][col2] == activePair.second.toString()) 00343 { 00344 // found a matching group/key pair 00345 __COUT__ << "Changing row " << row << " for " 00346 << cfgView->getDataView()[row][col1] 00347 << " version=" << cfgView->getDataView()[row][col2] 00348 << " to flatVersion=" << TableVersion(flatVersion) 00349 << std::endl; 00350 cfgView->setValue(TableVersion(flatVersion).toString(), row, col2); 00351 break; 00352 } 00353 } 00354 00355 __COUT__ << "\n\nChanging versions... " << std::endl; 00356 00357 for(auto& activePair : activeMap) 00358 { 00359 __COUT__ << activePair.first << ":v" << activePair.second << std::endl; 00360 // change the version of the active view to flatVersion and save it 00361 config = cfgMgr->getTableByName(activePair.first); 00362 cfgView = config->getViewP(); 00363 cfgView->setVersion(TableVersion(flatVersion)); 00364 theInterface_->saveActiveVersion(config); 00365 } 00366 00367 //============================================================================== 00368 // save the active groups with the new member flatVersions 00369 // Note: do not try this at home kids. 00370 for(auto& activeGroupMembersPair : activeGroupMembersMap) 00371 { 00372 __COUT__ << "Group " << activeGroupMembersPair.first << std::endl; 00373 00374 for(auto& groupMemberPair : activeGroupMembersPair.second) 00375 { 00376 __COUT__ << "\t from...\t" << groupMemberPair.first << ":v" 00377 << groupMemberPair.second << std::endl; 00378 groupMemberPair.second = flatVersion; 00379 } 00380 00381 for(auto& groupMemberPair : activeGroupMembersPair.second) 00382 { 00383 __COUT__ << "\t to...\t" << groupMemberPair.first << ":v" 00384 << groupMemberPair.second << std::endl; 00385 } 00386 00387 // Note: this code copies actions in ConfigurationManagerRW::saveNewTableGroup 00388 00389 // add meta data 00390 __COUTV__(StringMacros::mapToString( 00391 activeGroupAliasesMap[activeGroupMembersPair.first])); 00392 __COUTV__(activeGroupCommentMap[activeGroupMembersPair.first]); 00393 __COUTV__(activeGroupAuthorMap[activeGroupMembersPair.first]); 00394 __COUTV__(activeGroupCreateTimeMap[activeGroupMembersPair.first]); 00395 00396 // to compensate for unusual errors upstream, make sure the metadata table has one 00397 // row 00398 while(groupMetadataTable->getViewP()->getNumberOfRows() > 1) 00399 groupMetadataTable->getViewP()->deleteRow(0); 00400 if(groupMetadataTable->getViewP()->getNumberOfRows() == 0) 00401 groupMetadataTable->getViewP()->addRow(); 00402 00403 // columns are uid,comment,author,time 00404 // ConfigurationManager::METADATA_COL_ALIASES TODO 00405 groupMetadataTable->getViewP()->setValue( 00406 StringMacros::mapToString(activeGroupAliasesMap[activeGroupMembersPair.first], 00407 "," /*primary delimiter*/, 00408 ":" /*secondary delimeter*/), 00409 0, 00410 ConfigurationManager::METADATA_COL_ALIASES); 00411 groupMetadataTable->getViewP()->setValue( 00412 activeGroupCommentMap[activeGroupMembersPair.first], 00413 0, 00414 ConfigurationManager::METADATA_COL_COMMENT); 00415 groupMetadataTable->getViewP()->setValue( 00416 activeGroupAuthorMap[activeGroupMembersPair.first], 00417 0, 00418 ConfigurationManager::METADATA_COL_AUTHOR); 00419 groupMetadataTable->getViewP()->setValue( 00420 activeGroupCreateTimeMap[activeGroupMembersPair.first], 00421 0, 00422 ConfigurationManager::METADATA_COL_TIMESTAMP); 00423 00424 // set version of metadata table 00425 groupMetadataTable->getViewP()->setVersion(TableVersion(flatVersion)); 00426 theInterface_->saveActiveVersion(groupMetadataTable); 00427 00428 // force groupMetadataTable_ to be a member for the group 00429 activeGroupMembersPair.second[groupMetadataTable->getTableName()] = 00430 groupMetadataTable->getViewVersion(); 00431 00432 // memberMap should now consist of members with new flat version, so save group 00433 theInterface_->saveTableGroup(activeGroupMembersPair.second, 00434 TableGroupKey::getFullGroupString( 00435 activeGroupMembersPair.first, 00436 TableGroupKey(TableGroupKey::getDefaultKey()))); 00437 } 00438 00439 CLEAN_UP: 00440 //============================================================================== 00441 __COUT__ << "\n\nEnd of Flattening Active Table Groups!\n\n\n" << std::endl; 00442 00443 __COUT__ << "Run the following to return to your previous database structure:" 00444 << std::endl; 00445 __COUT__ << "\t otsdaq_flatten_active_to_version -1 " << moveToDir << "\n\n" 00446 << std::endl; 00447 00448 return; 00449 } 00450 00451 int main(int argc, char* argv[]) 00452 { 00453 FlattenActiveTableGroups(argc, argv); 00454 return 0; 00455 } 00456 // BOOST_AUTO_TEST_SUITE_END()