otsdaq  v2_04_01
DatabaseConfigurationInterface.cc
1 #include "otsdaq-core/ConfigurationInterface/DatabaseConfigurationInterface.h"
2 #include "otsdaq-core/Macros/CoutMacros.h"
3 #include "otsdaq-core/MessageFacility/MessageFacility.h"
4 
5 #include <algorithm>
6 #include <iostream>
7 #include <iterator>
8 #include <string>
9 
10 #include "artdaq-database/BasicTypes/basictypes.h"
11 #include "artdaq-database/ConfigurationDB/configurationdbifc.h"
12 #include "otsdaq-core/TableCore/TableBase.h"
13 
14 #include "artdaq-database/ConfigurationDB/configuration_common.h"
15 #include "artdaq-database/ConfigurationDB/dispatch_common.h"
16 #include "artdaq-database/StorageProviders/FileSystemDB/provider_filedb.h"
17 #include "artdaq-database/StorageProviders/FileSystemDB/provider_filedb_index.h"
18 
19 using namespace ots;
20 
21 using artdaq::database::basictypes::FhiclData;
22 using artdaq::database::basictypes::JsonData;
23 
25 using config_version_map_t = ots::DatabaseConfigurationInterface::config_version_map_t;
26 
27 namespace db = artdaq::database::configuration;
28 using VersionInfoList_t = db::ConfigurationInterface::VersionInfoList_t;
29 
30 constexpr auto default_dbprovider = "filesystem";
31 constexpr auto default_entity = "OTSROOT";
32 
33 //==============================================================================
34 DatabaseConfigurationInterface::DatabaseConfigurationInterface()
35 {
36 #ifdef DEBUG_ENABLE
37  // to enable debugging
38  if(0)
39  {
40  artdaq::database::configuration::debug::ExportImport();
41  artdaq::database::configuration::debug::ManageAliases();
42  artdaq::database::configuration::debug::ManageConfigs();
43  artdaq::database::configuration::debug::ManageDocuments();
44  artdaq::database::configuration::debug::Metadata();
45 
46  artdaq::database::configuration::debug::detail::ExportImport();
47  artdaq::database::configuration::debug::detail::ManageAliases();
48  artdaq::database::configuration::debug::detail::ManageConfigs();
49  artdaq::database::configuration::debug::detail::ManageDocuments();
50  artdaq::database::configuration::debug::detail::Metadata();
51 
52  artdaq::database::configuration::debug::options::OperationBase();
53  artdaq::database::configuration::debug::options::BulkOperations();
54  artdaq::database::configuration::debug::options::ManageDocuments();
55  artdaq::database::configuration::debug::options::ManageConfigs();
56  artdaq::database::configuration::debug::options::ManageAliases();
57 
58  artdaq::database::configuration::debug::MongoDB();
59  artdaq::database::configuration::debug::UconDB();
60  artdaq::database::configuration::debug::FileSystemDB();
61 
62  artdaq::database::filesystem::index::debug::enable();
63 
64  artdaq::database::filesystem::debug::enable();
65  artdaq::database::mongo::debug::enable();
66 
67  artdaq::database::docrecord::debug::JSONDocumentBuilder();
68  artdaq::database::docrecord::debug::JSONDocument();
69 
70  // debug::registerUngracefullExitHandlers();
71  // artdaq::database::useFakeTime(true);
72  }
73 #endif
74 }
75 
76 //==============================================================================
77 // read configuration from database
78 // version = -1 means latest version
79 void DatabaseConfigurationInterface::fill(TableBase* configuration,
80  TableVersion version) const
81 
82 {
83  auto ifc = db::ConfigurationInterface{default_dbprovider};
84 
85  auto versionstring = version.toString();
86 
87  //__COUTV__(versionstring);
88  // configuration->getViewP()->setUniqueStorageIdentifier(storageUID);
89 
90  auto result = ifc.template loadVersion<decltype(configuration), JsonData>(
91  configuration, versionstring, default_entity);
92 
93  if(result.first)
94  {
95  // make sure version is set.. not clear it was happening in loadVersion
96  configuration->getViewP()->setVersion(version);
97  return;
98  }
99  __SS__ << "\n\nDBI Error while filling '" << configuration->getTableName()
100  << "' version '" << versionstring << "' - are you sure this version exists?\n"
101  << "Here is the error:\n\n"
102  << result.second << __E__;
103  __SS_THROW__;
104 }
105 
106 //==============================================================================
107 // write configuration to database
108 void DatabaseConfigurationInterface::saveActiveVersion(const TableBase* configuration,
109  bool overwrite) const
110 
111 {
112  auto ifc = db::ConfigurationInterface{default_dbprovider};
113 
114  // configuration->getView().getUniqueStorageIdentifier()
115 
116  auto versionstring = configuration->getView().getVersion().toString();
117  //__COUT__ << "versionstring: " << versionstring << "\n";
118 
119  // auto result =
120  // ifc.template storeVersion<decltype(configuration), JsonData>(configuration,
121  // versionstring, default_entity);
122  auto result = overwrite
123  ? ifc.template overwriteVersion<decltype(configuration), JsonData>(
124  configuration, versionstring, default_entity)
125  : ifc.template storeVersion<decltype(configuration), JsonData>(
126  configuration, versionstring, default_entity);
127 
128  if(result.first)
129  return;
130 
131  __SS__ << "DBI Error:" << result.second << __E__;
132  __COUT__ << "\n" << ss.str();
133  __SS_THROW__;
134 }
135 
136 //==============================================================================
137 // find the latest configuration version by configuration type
138 TableVersion DatabaseConfigurationInterface::findLatestVersion(
139  const TableBase* table) const noexcept
140 {
141  auto versions = getVersions(table);
142 
143  __COUT__ << "Config Name: " << table->getTableName() << __E__;
144  __COUT__ << "All Versions: ";
145  for(auto& v : versions)
146  std::cout << v << " ";
147  std::cout << __E__;
148 
149  if(!versions.size())
150  return TableVersion(); // return INVALID
151 
152  return *(versions.rbegin());
153 }
154 
155 //==============================================================================
156 // find all configuration versions by configuration type
157 std::set<TableVersion> DatabaseConfigurationInterface::getVersions(
158  const TableBase* table) const noexcept try
159 {
160  auto ifc = db::ConfigurationInterface{default_dbprovider};
161  auto result = ifc.template getVersions<decltype(table)>(table, default_entity);
162 
163  auto resultSet = std::set<TableVersion>{};
164  for(std::string const& version : result)
165  resultSet.insert(TableVersion(std::stol(version, 0, 10)));
166 
167  // auto to_set = [](auto const& inputList)
168  // {
169  // auto resultSet = std::set<TableVersion>{};
170  // std::for_each(inputList.begin(), inputList.end(),
171  // [&resultSet](std::string const& version)
172  // { resultSet.insert(std::stol(version, 0, 10)); });
173  // return resultSet;
174  // };
175 
176  // auto vs = to_set(result);
177  // for(auto &v:vs)
178  // __COUT__ << "\tversion " << v << __E__;
179 
180  return resultSet; // to_set(result);
181 }
182 catch(std::exception const& e)
183 {
184  __COUT__ << "DBI Exception:" << e.what() << "\n";
185  return {};
186 }
187 
188 //==============================================================================
189 // returns a list of all configuration names
190 std::set<std::string /*name*/> DatabaseConfigurationInterface::getAllTableNames() const
191  try
192 {
193  auto ifc = db::ConfigurationInterface{default_dbprovider};
194 
195  auto collection_name_prefix = std::string{};
196 
197  return ifc.listCollections(collection_name_prefix);
198 }
199 catch(std::exception const& e)
200 {
201  __SS__ << "DBI Exception:" << e.what() << "\n";
202  __SS_THROW__;
203 }
204 catch(...)
205 {
206  __SS__ << "DBI Unknown exception.\n";
207  __SS_THROW__;
208 }
209 
210 //==============================================================================
211 // find all configuration groups in database
212 std::set<std::string /*name*/> DatabaseConfigurationInterface::getAllTableGroupNames(
213  const std::string& filterString) const try
214 {
215  auto ifc = db::ConfigurationInterface{default_dbprovider};
216 
217  if(filterString == "")
218  return ifc.findGlobalConfigurations("*"); // GConfig will return all GConfig*
219  // with filesystem db.. for mongodb
220  // would require reg expr
221  else
222  return ifc.findGlobalConfigurations(filterString + "*"); // GConfig will return
223  // all GConfig* with
224  // filesystem db.. for
225  // mongodb would require
226  // reg expr
227 }
228 catch(std::exception const& e)
229 {
230  __SS__ << "Filter string '" << filterString << "' yielded DBI Exception:" << e.what()
231  << "\n";
232  __COUT_ERR__ << ss.str();
233  __SS_THROW__;
234 }
235 catch(...)
236 {
237  __SS__ << "Filter string '" << filterString << "' yielded DBI Unknown exception.\n";
238  __COUT_ERR__ << ss.str();
239  __SS_THROW__;
240 }
241 
242 //==============================================================================
243 // find the latest configuration group key by group name
244 // if not found, return invalid
245 TableGroupKey DatabaseConfigurationInterface::findLatestGroupKey(
246  const std::string& groupName) const noexcept
247 {
248  std::set<TableGroupKey> keys = DatabaseConfigurationInterface::getKeys(groupName);
249  if(keys.size()) // if keys exist, bump the last
250  return *(keys.crbegin());
251 
252  // else, return invalid
253  return TableGroupKey();
254 }
255 
256 //==============================================================================
257 // find all configuration groups in database
258 std::set<TableGroupKey /*key*/> DatabaseConfigurationInterface::getKeys(
259  const std::string& groupName) const
260 {
261  std::set<TableGroupKey> retSet;
262  std::set<std::string /*name*/> names = getAllTableGroupNames();
263  for(auto& n : names)
264  if(n.find(groupName) == 0)
265  retSet.insert(TableGroupKey(n));
266  return retSet;
267 }
268 
269 //==============================================================================
270 // return the contents of a configuration group
271 config_version_map_t DatabaseConfigurationInterface::getTableGroupMembers(
272  std::string const& tableGroup, bool includeMetaDataTable) const try
273 {
274  auto ifc = db::ConfigurationInterface{default_dbprovider};
275  auto result = ifc.loadGlobalConfiguration(tableGroup);
276 
277  // for(auto &item:result)
278  // __COUT__ << "====================>" << item.configuration << ": " <<
279  // item.version << __E__;
280 
281  auto to_map = [](auto const& inputList, bool includeMetaDataTable) {
282  auto resultMap = config_version_map_t{};
283 
284  std::for_each(inputList.begin(), inputList.end(), [&resultMap](auto const& info) {
285  resultMap[info.configuration] = std::stol(info.version, 0, 10);
286  });
287 
288  if(!includeMetaDataTable)
289  {
290  // remove special meta data table from member map
291  auto metaTable = resultMap.find(GROUP_METADATA_TABLE_NAME);
292  if(metaTable != resultMap.end())
293  resultMap.erase(metaTable);
294  }
295  return resultMap;
296  };
297 
298  return to_map(result, includeMetaDataTable);
299 } // end getTableGroupMembers()
300 catch(std::exception const& e)
301 {
302  __SS__ << "DBI Exception getting Group's member tables for '" << tableGroup
303  << "':\n\n"
304  << e.what() << "\n";
305  __COUT_ERR__ << ss.str();
306  __SS_THROW__;
307 }
308 catch(...)
309 {
310  __SS__ << "DBI Unknown exception getting Group's member tables for '" << tableGroup
311  << ".'\n";
312  __COUT_ERR__ << ss.str();
313  __SS_THROW__;
314 }
315 
316 //==============================================================================
317 // create a new configuration group from the contents map
318 void DatabaseConfigurationInterface::saveTableGroup(
319  config_version_map_t const& configurationMap,
320  std::string const& configurationGroup) const try
321 {
322  auto ifc = db::ConfigurationInterface{default_dbprovider};
323 
324  auto to_list = [](auto const& inputMap) {
325  auto resultList = VersionInfoList_t{};
326  std::transform(
327  inputMap.begin(),
328  inputMap.end(),
329  std::back_inserter(resultList),
330  [](auto const& mapEntry) {
331  return VersionInfoList_t::value_type{
332  mapEntry.first, mapEntry.second.toString(), default_entity};
333  });
334 
335  return resultList;
336  };
337 
338  auto result =
339  ifc.storeGlobalConfiguration(to_list(configurationMap), configurationGroup);
340 
341  if(result.first)
342  return;
343 
344  __THROW__(result.second);
345 } // end saveTableGroup()
346 catch(std::exception const& e)
347 {
348  __SS__ << "DBI Exception:" << e.what() << "\n";
349  __COUT_ERR__ << ss.str();
350  __SS_THROW__;
351 }
352 catch(...)
353 {
354  __SS__ << "DBI Unknown exception.\n";
355  __COUT_ERR__ << ss.str();
356  __SS_THROW__;
357 }