$treeview $search $mathjax $extrastylesheet
otsdaq
v2_03_00
$projectbrief
|
$projectbrief
|
$searchbox |
00001 #include "otsdaq-core/ConfigurationInterface/DatabaseConfigurationInterface.h" 00002 #include "otsdaq-core/Macros/CoutMacros.h" 00003 #include "otsdaq-core/MessageFacility/MessageFacility.h" 00004 00005 #include <algorithm> 00006 #include <iostream> 00007 #include <iterator> 00008 #include <string> 00009 00010 #include "artdaq-database/BasicTypes/basictypes.h" 00011 #include "artdaq-database/ConfigurationDB/configurationdbifc.h" 00012 #include "otsdaq-core/TableCore/TableBase.h" 00013 00014 #include "artdaq-database/ConfigurationDB/configuration_common.h" 00015 #include "artdaq-database/ConfigurationDB/dispatch_common.h" 00016 #include "artdaq-database/StorageProviders/FileSystemDB/provider_filedb.h" 00017 #include "artdaq-database/StorageProviders/FileSystemDB/provider_filedb_index.h" 00018 00019 using namespace ots; 00020 00021 using artdaq::database::basictypes::FhiclData; 00022 using artdaq::database::basictypes::JsonData; 00023 00024 using ots::DatabaseConfigurationInterface; 00025 using config_version_map_t = ots::DatabaseConfigurationInterface::config_version_map_t; 00026 00027 namespace db = artdaq::database::configuration; 00028 using VersionInfoList_t = db::ConfigurationInterface::VersionInfoList_t; 00029 00030 constexpr auto default_dbprovider = "filesystem"; 00031 constexpr auto default_entity = "OTSROOT"; 00032 00033 //============================================================================== 00034 DatabaseConfigurationInterface::DatabaseConfigurationInterface() 00035 { 00036 #ifdef DEBUG_ENABLE 00037 // to enable debugging 00038 if(0) 00039 { 00040 artdaq::database::configuration::debug::ExportImport(); 00041 artdaq::database::configuration::debug::ManageAliases(); 00042 artdaq::database::configuration::debug::ManageConfigs(); 00043 artdaq::database::configuration::debug::ManageDocuments(); 00044 artdaq::database::configuration::debug::Metadata(); 00045 00046 artdaq::database::configuration::debug::detail::ExportImport(); 00047 artdaq::database::configuration::debug::detail::ManageAliases(); 00048 artdaq::database::configuration::debug::detail::ManageConfigs(); 00049 artdaq::database::configuration::debug::detail::ManageDocuments(); 00050 artdaq::database::configuration::debug::detail::Metadata(); 00051 00052 artdaq::database::configuration::debug::options::OperationBase(); 00053 artdaq::database::configuration::debug::options::BulkOperations(); 00054 artdaq::database::configuration::debug::options::ManageDocuments(); 00055 artdaq::database::configuration::debug::options::ManageConfigs(); 00056 artdaq::database::configuration::debug::options::ManageAliases(); 00057 00058 artdaq::database::configuration::debug::MongoDB(); 00059 artdaq::database::configuration::debug::UconDB(); 00060 artdaq::database::configuration::debug::FileSystemDB(); 00061 00062 artdaq::database::filesystem::index::debug::enable(); 00063 00064 artdaq::database::filesystem::debug::enable(); 00065 artdaq::database::mongo::debug::enable(); 00066 00067 artdaq::database::docrecord::debug::JSONDocumentBuilder(); 00068 artdaq::database::docrecord::debug::JSONDocument(); 00069 00070 // debug::registerUngracefullExitHandlers(); 00071 // artdaq::database::useFakeTime(true); 00072 } 00073 #endif 00074 } 00075 00076 //============================================================================== 00077 // read configuration from database 00078 // version = -1 means latest version 00079 void DatabaseConfigurationInterface::fill(TableBase* configuration, 00080 TableVersion version) const 00081 throw(std::runtime_error) 00082 { 00083 auto ifc = db::ConfigurationInterface{default_dbprovider}; 00084 00085 auto versionstring = version.toString(); 00086 00087 //__COUTV__(versionstring); 00088 // configuration->getViewP()->setUniqueStorageIdentifier(storageUID); 00089 00090 auto result = ifc.template loadVersion<decltype(configuration), JsonData>( 00091 configuration, versionstring, default_entity); 00092 00093 if(result.first) 00094 { 00095 // make sure version is set.. not clear it was happening in loadVersion 00096 configuration->getViewP()->setVersion(version); 00097 return; 00098 } 00099 __SS__ << "\n\nDBI Error while filling '" << configuration->getTableName() 00100 << "' version '" << versionstring << "' - are you sure this version exists?\n" 00101 << "Here is the error:\n\n" 00102 << result.second << __E__; 00103 __SS_THROW__; 00104 } 00105 00106 //============================================================================== 00107 // write configuration to database 00108 void DatabaseConfigurationInterface::saveActiveVersion(const TableBase* configuration, 00109 bool overwrite) const 00110 throw(std::runtime_error) 00111 { 00112 auto ifc = db::ConfigurationInterface{default_dbprovider}; 00113 00114 // configuration->getView().getUniqueStorageIdentifier() 00115 00116 auto versionstring = configuration->getView().getVersion().toString(); 00117 //__COUT__ << "versionstring: " << versionstring << "\n"; 00118 00119 // auto result = 00120 // ifc.template storeVersion<decltype(configuration), JsonData>(configuration, 00121 // versionstring, default_entity); 00122 auto result = overwrite 00123 ? ifc.template overwriteVersion<decltype(configuration), JsonData>( 00124 configuration, versionstring, default_entity) 00125 : ifc.template storeVersion<decltype(configuration), JsonData>( 00126 configuration, versionstring, default_entity); 00127 00128 if(result.first) 00129 return; 00130 00131 __SS__ << "DBI Error:" << result.second << __E__; 00132 __COUT__ << "\n" << ss.str(); 00133 __SS_THROW__; 00134 } 00135 00136 //============================================================================== 00137 // find the latest configuration version by configuration type 00138 TableVersion DatabaseConfigurationInterface::findLatestVersion( 00139 const TableBase* table) const noexcept 00140 { 00141 auto versions = getVersions(table); 00142 00143 __COUT__ << "Config Name: " << table->getTableName() << __E__; 00144 __COUT__ << "All Versions: "; 00145 for(auto& v : versions) 00146 std::cout << v << " "; 00147 std::cout << __E__; 00148 00149 if(!versions.size()) 00150 return TableVersion(); // return INVALID 00151 00152 return *(versions.rbegin()); 00153 } 00154 00155 //============================================================================== 00156 // find all configuration versions by configuration type 00157 std::set<TableVersion> DatabaseConfigurationInterface::getVersions( 00158 const TableBase* table) const noexcept try 00159 { 00160 auto ifc = db::ConfigurationInterface{default_dbprovider}; 00161 auto result = ifc.template getVersions<decltype(table)>(table, default_entity); 00162 00163 auto resultSet = std::set<TableVersion>{}; 00164 for(std::string const& version : result) 00165 resultSet.insert(TableVersion(std::stol(version, 0, 10))); 00166 00167 // auto to_set = [](auto const& inputList) 00168 // { 00169 // auto resultSet = std::set<TableVersion>{}; 00170 // std::for_each(inputList.begin(), inputList.end(), 00171 // [&resultSet](std::string const& version) 00172 // { resultSet.insert(std::stol(version, 0, 10)); }); 00173 // return resultSet; 00174 // }; 00175 00176 // auto vs = to_set(result); 00177 // for(auto &v:vs) 00178 // __COUT__ << "\tversion " << v << __E__; 00179 00180 return resultSet; // to_set(result); 00181 } 00182 catch(std::exception const& e) 00183 { 00184 __COUT__ << "DBI Exception:" << e.what() << "\n"; 00185 return {}; 00186 } 00187 00188 //============================================================================== 00189 // returns a list of all configuration names 00190 std::set<std::string /*name*/> DatabaseConfigurationInterface::getAllTableNames() const 00191 throw(std::runtime_error) try 00192 { 00193 auto ifc = db::ConfigurationInterface{default_dbprovider}; 00194 00195 auto collection_name_prefix = std::string{}; 00196 00197 return ifc.listCollections(collection_name_prefix); 00198 } 00199 catch(std::exception const& e) 00200 { 00201 __SS__ << "DBI Exception:" << e.what() << "\n"; 00202 __SS_THROW__; 00203 } 00204 catch(...) 00205 { 00206 __SS__ << "DBI Unknown exception.\n"; 00207 __SS_THROW__; 00208 } 00209 00210 //============================================================================== 00211 // find all configuration groups in database 00212 std::set<std::string /*name*/> DatabaseConfigurationInterface::getAllTableGroupNames( 00213 const std::string& filterString) const throw(std::runtime_error) try 00214 { 00215 auto ifc = db::ConfigurationInterface{default_dbprovider}; 00216 00217 if(filterString == "") 00218 return ifc.findGlobalConfigurations("*"); // GConfig will return all GConfig* 00219 // with filesystem db.. for mongodb 00220 // would require reg expr 00221 else 00222 return ifc.findGlobalConfigurations(filterString + "*"); // GConfig will return 00223 // all GConfig* with 00224 // filesystem db.. for 00225 // mongodb would require 00226 // reg expr 00227 } 00228 catch(std::exception const& e) 00229 { 00230 __SS__ << "Filter string '" << filterString << "' yielded DBI Exception:" << e.what() 00231 << "\n"; 00232 __COUT_ERR__ << ss.str(); 00233 __SS_THROW__; 00234 } 00235 catch(...) 00236 { 00237 __SS__ << "Filter string '" << filterString << "' yielded DBI Unknown exception.\n"; 00238 __COUT_ERR__ << ss.str(); 00239 __SS_THROW__; 00240 } 00241 00242 //============================================================================== 00243 // find the latest configuration group key by group name 00244 // if not found, return invalid 00245 TableGroupKey DatabaseConfigurationInterface::findLatestGroupKey( 00246 const std::string& groupName) const noexcept 00247 { 00248 std::set<TableGroupKey> keys = DatabaseConfigurationInterface::getKeys(groupName); 00249 if(keys.size()) // if keys exist, bump the last 00250 return *(keys.crbegin()); 00251 00252 // else, return invalid 00253 return TableGroupKey(); 00254 } 00255 00256 //============================================================================== 00257 // find all configuration groups in database 00258 std::set<TableGroupKey /*key*/> DatabaseConfigurationInterface::getKeys( 00259 const std::string& groupName) const 00260 { 00261 std::set<TableGroupKey> retSet; 00262 std::set<std::string /*name*/> names = getAllTableGroupNames(); 00263 for(auto& n : names) 00264 if(n.find(groupName) == 0) 00265 retSet.insert(TableGroupKey(n)); 00266 return retSet; 00267 } 00268 00269 //============================================================================== 00270 // return the contents of a configuration group 00271 config_version_map_t DatabaseConfigurationInterface::getTableGroupMembers( 00272 std::string const& tableGroup, bool includeMetaDataTable) const 00273 throw(std::runtime_error) try 00274 { 00275 auto ifc = db::ConfigurationInterface{default_dbprovider}; 00276 auto result = ifc.loadGlobalConfiguration(tableGroup); 00277 00278 // for(auto &item:result) 00279 // __COUT__ << "====================>" << item.configuration << ": " << 00280 // item.version << __E__; 00281 00282 auto to_map = [](auto const& inputList, bool includeMetaDataTable) { 00283 auto resultMap = config_version_map_t{}; 00284 00285 std::for_each(inputList.begin(), inputList.end(), [&resultMap](auto const& info) { 00286 resultMap[info.configuration] = std::stol(info.version, 0, 10); 00287 }); 00288 00289 if(!includeMetaDataTable) 00290 { 00291 // remove special meta data table from member map 00292 auto metaTable = resultMap.find(GROUP_METADATA_TABLE_NAME); 00293 if(metaTable != resultMap.end()) 00294 resultMap.erase(metaTable); 00295 } 00296 return resultMap; 00297 }; 00298 00299 return to_map(result, includeMetaDataTable); 00300 } // end getTableGroupMembers() 00301 catch(std::exception const& e) 00302 { 00303 __SS__ << "DBI Exception getting Group's member tables for '" << tableGroup 00304 << "':\n\n" 00305 << e.what() << "\n"; 00306 __COUT_ERR__ << ss.str(); 00307 __SS_THROW__; 00308 } 00309 catch(...) 00310 { 00311 __SS__ << "DBI Unknown exception getting Group's member tables for '" << tableGroup 00312 << ".'\n"; 00313 __COUT_ERR__ << ss.str(); 00314 __SS_THROW__; 00315 } 00316 00317 //============================================================================== 00318 // create a new configuration group from the contents map 00319 void DatabaseConfigurationInterface::saveTableGroup( 00320 config_version_map_t const& configurationMap, 00321 std::string const& configurationGroup) const throw(std::runtime_error) try 00322 { 00323 auto ifc = db::ConfigurationInterface{default_dbprovider}; 00324 00325 auto to_list = [](auto const& inputMap) { 00326 auto resultList = VersionInfoList_t{}; 00327 std::transform( 00328 inputMap.begin(), 00329 inputMap.end(), 00330 std::back_inserter(resultList), 00331 [](auto const& mapEntry) { 00332 return VersionInfoList_t::value_type{ 00333 mapEntry.first, mapEntry.second.toString(), default_entity}; 00334 }); 00335 00336 return resultList; 00337 }; 00338 00339 auto result = 00340 ifc.storeGlobalConfiguration(to_list(configurationMap), configurationGroup); 00341 00342 if(result.first) 00343 return; 00344 00345 __THROW__(result.second); 00346 } // end saveTableGroup() 00347 catch(std::exception const& e) 00348 { 00349 __SS__ << "DBI Exception:" << e.what() << "\n"; 00350 __COUT_ERR__ << ss.str(); 00351 __SS_THROW__; 00352 } 00353 catch(...) 00354 { 00355 __SS__ << "DBI Unknown exception.\n"; 00356 __COUT_ERR__ << ss.str(); 00357 __SS_THROW__; 00358 }