otsdaq  v2_03_00
DatabaseConfigurationInterface.cc
1 #include "otsdaq-core/ConfigurationInterface/DatabaseConfigurationInterface.h"
2 #include "otsdaq-core/Macros/CoutMacros.h"
3 #include "otsdaq-core/MessageFacility/MessageFacility.h"
4 
5 #include <algorithm>
6 #include <iostream>
7 #include <iterator>
8 #include <string>
9 
10 #include "artdaq-database/BasicTypes/basictypes.h"
11 #include "artdaq-database/ConfigurationDB/configurationdbifc.h"
12 #include "otsdaq-core/TableCore/TableBase.h"
13 
14 #include "artdaq-database/ConfigurationDB/configuration_common.h"
15 #include "artdaq-database/ConfigurationDB/dispatch_common.h"
16 #include "artdaq-database/StorageProviders/FileSystemDB/provider_filedb.h"
17 #include "artdaq-database/StorageProviders/FileSystemDB/provider_filedb_index.h"
18 
19 using namespace ots;
20 
21 using artdaq::database::basictypes::FhiclData;
22 using artdaq::database::basictypes::JsonData;
23 
25 using config_version_map_t = ots::DatabaseConfigurationInterface::config_version_map_t;
26 
27 namespace db = artdaq::database::configuration;
28 using VersionInfoList_t = db::ConfigurationInterface::VersionInfoList_t;
29 
30 constexpr auto default_dbprovider = "filesystem";
31 constexpr auto default_entity = "OTSROOT";
32 
33 //==============================================================================
34 DatabaseConfigurationInterface::DatabaseConfigurationInterface()
35 {
36 #ifdef DEBUG_ENABLE
37  // to enable debugging
38  if(0)
39  {
40  artdaq::database::configuration::debug::ExportImport();
41  artdaq::database::configuration::debug::ManageAliases();
42  artdaq::database::configuration::debug::ManageConfigs();
43  artdaq::database::configuration::debug::ManageDocuments();
44  artdaq::database::configuration::debug::Metadata();
45 
46  artdaq::database::configuration::debug::detail::ExportImport();
47  artdaq::database::configuration::debug::detail::ManageAliases();
48  artdaq::database::configuration::debug::detail::ManageConfigs();
49  artdaq::database::configuration::debug::detail::ManageDocuments();
50  artdaq::database::configuration::debug::detail::Metadata();
51 
52  artdaq::database::configuration::debug::options::OperationBase();
53  artdaq::database::configuration::debug::options::BulkOperations();
54  artdaq::database::configuration::debug::options::ManageDocuments();
55  artdaq::database::configuration::debug::options::ManageConfigs();
56  artdaq::database::configuration::debug::options::ManageAliases();
57 
58  artdaq::database::configuration::debug::MongoDB();
59  artdaq::database::configuration::debug::UconDB();
60  artdaq::database::configuration::debug::FileSystemDB();
61 
62  artdaq::database::filesystem::index::debug::enable();
63 
64  artdaq::database::filesystem::debug::enable();
65  artdaq::database::mongo::debug::enable();
66 
67  artdaq::database::docrecord::debug::JSONDocumentBuilder();
68  artdaq::database::docrecord::debug::JSONDocument();
69 
70  // debug::registerUngracefullExitHandlers();
71  // artdaq::database::useFakeTime(true);
72  }
73 #endif
74 }
75 
76 //==============================================================================
77 // read configuration from database
78 // version = -1 means latest version
79 void DatabaseConfigurationInterface::fill(TableBase* configuration,
80  TableVersion version) const
81  throw(std::runtime_error)
82 {
83  auto ifc = db::ConfigurationInterface{default_dbprovider};
84 
85  auto versionstring = version.toString();
86 
87  //__COUTV__(versionstring);
88  // configuration->getViewP()->setUniqueStorageIdentifier(storageUID);
89 
90  auto result = ifc.template loadVersion<decltype(configuration), JsonData>(
91  configuration, versionstring, default_entity);
92 
93  if(result.first)
94  {
95  // make sure version is set.. not clear it was happening in loadVersion
96  configuration->getViewP()->setVersion(version);
97  return;
98  }
99  __SS__ << "\n\nDBI Error while filling '" << configuration->getTableName()
100  << "' version '" << versionstring << "' - are you sure this version exists?\n"
101  << "Here is the error:\n\n"
102  << result.second << __E__;
103  __SS_THROW__;
104 }
105 
106 //==============================================================================
107 // write configuration to database
108 void DatabaseConfigurationInterface::saveActiveVersion(const TableBase* configuration,
109  bool overwrite) const
110  throw(std::runtime_error)
111 {
112  auto ifc = db::ConfigurationInterface{default_dbprovider};
113 
114  // configuration->getView().getUniqueStorageIdentifier()
115 
116  auto versionstring = configuration->getView().getVersion().toString();
117  //__COUT__ << "versionstring: " << versionstring << "\n";
118 
119  // auto result =
120  // ifc.template storeVersion<decltype(configuration), JsonData>(configuration,
121  // versionstring, default_entity);
122  auto result = overwrite
123  ? ifc.template overwriteVersion<decltype(configuration), JsonData>(
124  configuration, versionstring, default_entity)
125  : ifc.template storeVersion<decltype(configuration), JsonData>(
126  configuration, versionstring, default_entity);
127 
128  if(result.first)
129  return;
130 
131  __SS__ << "DBI Error:" << result.second << __E__;
132  __COUT__ << "\n" << ss.str();
133  __SS_THROW__;
134 }
135 
136 //==============================================================================
137 // find the latest configuration version by configuration type
138 TableVersion DatabaseConfigurationInterface::findLatestVersion(
139  const TableBase* table) const noexcept
140 {
141  auto versions = getVersions(table);
142 
143  __COUT__ << "Config Name: " << table->getTableName() << __E__;
144  __COUT__ << "All Versions: ";
145  for(auto& v : versions)
146  std::cout << v << " ";
147  std::cout << __E__;
148 
149  if(!versions.size())
150  return TableVersion(); // return INVALID
151 
152  return *(versions.rbegin());
153 }
154 
155 //==============================================================================
156 // find all configuration versions by configuration type
157 std::set<TableVersion> DatabaseConfigurationInterface::getVersions(
158  const TableBase* table) const noexcept try
159 {
160  auto ifc = db::ConfigurationInterface{default_dbprovider};
161  auto result = ifc.template getVersions<decltype(table)>(table, default_entity);
162 
163  auto resultSet = std::set<TableVersion>{};
164  for(std::string const& version : result)
165  resultSet.insert(TableVersion(std::stol(version, 0, 10)));
166 
167  // auto to_set = [](auto const& inputList)
168  // {
169  // auto resultSet = std::set<TableVersion>{};
170  // std::for_each(inputList.begin(), inputList.end(),
171  // [&resultSet](std::string const& version)
172  // { resultSet.insert(std::stol(version, 0, 10)); });
173  // return resultSet;
174  // };
175 
176  // auto vs = to_set(result);
177  // for(auto &v:vs)
178  // __COUT__ << "\tversion " << v << __E__;
179 
180  return resultSet; // to_set(result);
181 }
182 catch(std::exception const& e)
183 {
184  __COUT__ << "DBI Exception:" << e.what() << "\n";
185  return {};
186 }
187 
188 //==============================================================================
189 // returns a list of all configuration names
190 std::set<std::string /*name*/> DatabaseConfigurationInterface::getAllTableNames() const
191  throw(std::runtime_error) try
192 {
193  auto ifc = db::ConfigurationInterface{default_dbprovider};
194 
195  auto collection_name_prefix = std::string{};
196 
197  return ifc.listCollections(collection_name_prefix);
198 }
199 catch(std::exception const& e)
200 {
201  __SS__ << "DBI Exception:" << e.what() << "\n";
202  __SS_THROW__;
203 }
204 catch(...)
205 {
206  __SS__ << "DBI Unknown exception.\n";
207  __SS_THROW__;
208 }
209 
210 //==============================================================================
211 // find all configuration groups in database
212 std::set<std::string /*name*/> DatabaseConfigurationInterface::getAllTableGroupNames(
213  const std::string& filterString) const throw(std::runtime_error) try
214 {
215  auto ifc = db::ConfigurationInterface{default_dbprovider};
216 
217  if(filterString == "")
218  return ifc.findGlobalConfigurations("*"); // GConfig will return all GConfig*
219  // with filesystem db.. for mongodb
220  // would require reg expr
221  else
222  return ifc.findGlobalConfigurations(filterString + "*"); // GConfig will return
223  // all GConfig* with
224  // filesystem db.. for
225  // mongodb would require
226  // reg expr
227 }
228 catch(std::exception const& e)
229 {
230  __SS__ << "Filter string '" << filterString << "' yielded DBI Exception:" << e.what()
231  << "\n";
232  __COUT_ERR__ << ss.str();
233  __SS_THROW__;
234 }
235 catch(...)
236 {
237  __SS__ << "Filter string '" << filterString << "' yielded DBI Unknown exception.\n";
238  __COUT_ERR__ << ss.str();
239  __SS_THROW__;
240 }
241 
242 //==============================================================================
243 // find the latest configuration group key by group name
244 // if not found, return invalid
245 TableGroupKey DatabaseConfigurationInterface::findLatestGroupKey(
246  const std::string& groupName) const noexcept
247 {
248  std::set<TableGroupKey> keys = DatabaseConfigurationInterface::getKeys(groupName);
249  if(keys.size()) // if keys exist, bump the last
250  return *(keys.crbegin());
251 
252  // else, return invalid
253  return TableGroupKey();
254 }
255 
256 //==============================================================================
257 // find all configuration groups in database
258 std::set<TableGroupKey /*key*/> DatabaseConfigurationInterface::getKeys(
259  const std::string& groupName) const
260 {
261  std::set<TableGroupKey> retSet;
262  std::set<std::string /*name*/> names = getAllTableGroupNames();
263  for(auto& n : names)
264  if(n.find(groupName) == 0)
265  retSet.insert(TableGroupKey(n));
266  return retSet;
267 }
268 
269 //==============================================================================
270 // return the contents of a configuration group
271 config_version_map_t DatabaseConfigurationInterface::getTableGroupMembers(
272  std::string const& tableGroup, bool includeMetaDataTable) const
273  throw(std::runtime_error) try
274 {
275  auto ifc = db::ConfigurationInterface{default_dbprovider};
276  auto result = ifc.loadGlobalConfiguration(tableGroup);
277 
278  // for(auto &item:result)
279  // __COUT__ << "====================>" << item.configuration << ": " <<
280  // item.version << __E__;
281 
282  auto to_map = [](auto const& inputList, bool includeMetaDataTable) {
283  auto resultMap = config_version_map_t{};
284 
285  std::for_each(inputList.begin(), inputList.end(), [&resultMap](auto const& info) {
286  resultMap[info.configuration] = std::stol(info.version, 0, 10);
287  });
288 
289  if(!includeMetaDataTable)
290  {
291  // remove special meta data table from member map
292  auto metaTable = resultMap.find(GROUP_METADATA_TABLE_NAME);
293  if(metaTable != resultMap.end())
294  resultMap.erase(metaTable);
295  }
296  return resultMap;
297  };
298 
299  return to_map(result, includeMetaDataTable);
300 } // end getTableGroupMembers()
301 catch(std::exception const& e)
302 {
303  __SS__ << "DBI Exception getting Group's member tables for '" << tableGroup
304  << "':\n\n"
305  << e.what() << "\n";
306  __COUT_ERR__ << ss.str();
307  __SS_THROW__;
308 }
309 catch(...)
310 {
311  __SS__ << "DBI Unknown exception getting Group's member tables for '" << tableGroup
312  << ".'\n";
313  __COUT_ERR__ << ss.str();
314  __SS_THROW__;
315 }
316 
317 //==============================================================================
318 // create a new configuration group from the contents map
319 void DatabaseConfigurationInterface::saveTableGroup(
320  config_version_map_t const& configurationMap,
321  std::string const& configurationGroup) const throw(std::runtime_error) try
322 {
323  auto ifc = db::ConfigurationInterface{default_dbprovider};
324 
325  auto to_list = [](auto const& inputMap) {
326  auto resultList = VersionInfoList_t{};
327  std::transform(
328  inputMap.begin(),
329  inputMap.end(),
330  std::back_inserter(resultList),
331  [](auto const& mapEntry) {
332  return VersionInfoList_t::value_type{
333  mapEntry.first, mapEntry.second.toString(), default_entity};
334  });
335 
336  return resultList;
337  };
338 
339  auto result =
340  ifc.storeGlobalConfiguration(to_list(configurationMap), configurationGroup);
341 
342  if(result.first)
343  return;
344 
345  __THROW__(result.second);
346 } // end saveTableGroup()
347 catch(std::exception const& e)
348 {
349  __SS__ << "DBI Exception:" << e.what() << "\n";
350  __COUT_ERR__ << ss.str();
351  __SS_THROW__;
352 }
353 catch(...)
354 {
355  __SS__ << "DBI Unknown exception.\n";
356  __COUT_ERR__ << ss.str();
357  __SS_THROW__;
358 }