artdaq  v3_01_00
RootDAQOutFile.cc
1 #include "artdaq/ArtModules/RootDAQOutput/RootDAQOutFile.h"
2 // vim: set sw=2:
3 
4 #include "Rtypes.h"
5 #include "TBranchElement.h"
6 #include "TClass.h"
7 #include "TFile.h"
8 #include "TTree.h"
9 #include "art/Framework/Core/FileBlock.h"
10 #if ART_HEX_VERSION >= 0x20703
11 # include "art/Framework/Core/OutputFileGranularity.h"
12 # include "art/Framework/Services/System/DatabaseConnection.h"
13 # include "art/Persistency/RootDB/TKeyVFSOpenPolicy.h"
14 # include "cetlib/sqlite/Ntuple.h"
15 # include "cetlib/sqlite/Transaction.h"
16 # include "cetlib/sqlite/create_table.h"
17 # include "cetlib/sqlite/exec.h"
18 # include "cetlib/sqlite/insert.h"
19 #else
20 # include "art/Framework/Services/Registry/ServiceHandle.h"
21 # include "cetlib/Ntuple/Transaction.h"
22 # include "art/Persistency/RootDB/SQLite3Wrapper.h"
23 #endif
24 #include "art/Framework/IO/FileStatsCollector.h"
25 #include "art/Framework/IO/Root/DropMetaData.h"
26 #include "art/Framework/IO/Root/GetFileFormatEra.h"
27 #include "art/Framework/IO/Root/GetFileFormatVersion.h"
28 #include "artdaq/ArtModules/RootDAQOutput/detail/KeptProvenance.h"
29 #include "artdaq/ArtModules/RootDAQOutput/RootDAQOutFile.h"
30 #include "art/Framework/Principal/EventPrincipal.h"
31 #include "art/Framework/Principal/ResultsPrincipal.h"
32 #include "art/Framework/Principal/RunPrincipal.h"
33 #include "art/Framework/Principal/SubRunPrincipal.h"
34 #include "art/Persistency/Provenance/BranchIDListRegistry.h"
35 #include "art/Persistency/Provenance/ProcessHistoryRegistry.h"
36 #include "art/Persistency/Provenance/ProductMetaData.h"
37 #include "art/Persistency/RootDB/SQLErrMsg.h"
38 #include "art/Version/GetReleaseVersion.h"
39 #include "boost/date_time/posix_time/posix_time.hpp"
40 #include "canvas/Persistency/Provenance/rootNames.h"
41 #include "canvas/Persistency/Provenance/BranchChildren.h"
42 #include "canvas/Persistency/Provenance/BranchID.h"
43 #include "canvas/Persistency/Provenance/BranchIDList.h"
44 #include "canvas/Persistency/Provenance/BranchType.h"
45 #include "canvas/Persistency/Provenance/EventAuxiliary.h"
46 #include "canvas/Persistency/Provenance/EventID.h"
47 #include "canvas/Persistency/Provenance/FileFormatVersion.h"
48 #include "canvas/Persistency/Provenance/History.h"
49 #include "canvas/Persistency/Provenance/ParameterSetBlob.h"
50 #include "canvas/Persistency/Provenance/Parentage.h"
51 #include "canvas/Persistency/Provenance/ParentageRegistry.h"
52 #include "canvas/Persistency/Provenance/ProcessHistoryID.h"
53 #include "canvas/Persistency/Provenance/ProductStatus.h"
54 #include "canvas/Persistency/Provenance/ResultsAuxiliary.h"
55 #include "canvas/Persistency/Provenance/RunAuxiliary.h"
56 #include "canvas/Persistency/Provenance/SubRunAuxiliary.h"
57 #include "canvas/Utilities/Exception.h"
58 #include "cetlib/canonical_string.h"
59 #include "cetlib/container_algorithms.h"
60 #include "cetlib/exempt_ptr.h"
61 #include "fhiclcpp/ParameterSet.h"
62 #include "fhiclcpp/ParameterSetID.h"
63 #include "fhiclcpp/ParameterSetRegistry.h"
64 #define TRACE_NAME "RootDAQOutFile"
65 #include "trace.h" // TRACE
66 
67 #include <algorithm>
68 #include <iomanip>
69 #include <sstream>
70 #include <utility>
71 #include <utility>
72 #include <vector>
73 
74 using namespace cet;
75 using namespace std;
76 using art::BranchType;
78 using art::rootNames::metaBranchRootName;
79 
80 namespace {
81 
82  void create_table(sqlite3* const db,
83  std::string const& name,
84  std::vector<std::string> const& columns,
85  std::string const& suffix = {})
86  {
87  if (columns.empty())
88  throw art::Exception(art::errors::LogicError)
89  << "Number of sqlite columns specified for table: "
90  << name << '\n'
91  << "is zero.\n";
92 
93  sqlite::Transaction txn {db};
94  art::SQLErrMsg errMsg;
95  std::string ddl =
96  "DROP TABLE IF EXISTS " + name + "; "
97  "CREATE TABLE " + name +
98  "("+columns.front();
99  std::for_each(columns.begin()+1, columns.end(),
100  [&ddl](auto const& col) {
101  ddl += ","+col;
102  } );
103  ddl += ") ";
104  ddl += suffix;
105  ddl += ";";
106  sqlite3_exec(db, ddl.c_str(), nullptr, nullptr, errMsg);
107  errMsg.throwIfError();
108  txn.commit();
109  }
110 
111  void
112  insert_eventRanges_row(sqlite3_stmt* stmt,
113  art::SubRunNumber_t const sr,
114  art::EventNumber_t const b,
115  art::EventNumber_t const e)
116  {
117  sqlite3_bind_int64(stmt, 1, sr);
118  sqlite3_bind_int64(stmt, 2, b);
119  sqlite3_bind_int64(stmt, 3, e);
120  sqlite3_step(stmt);
121  sqlite3_reset(stmt);
122  }
123 
124  void
125  insert_rangeSets_row(sqlite3_stmt* stmt,
126  art::RunNumber_t const r)
127  {
128  sqlite3_bind_int64(stmt, 1, r);
129  sqlite3_step(stmt);
130  sqlite3_reset(stmt);
131  }
132 
133  void
134  insert_rangeSets_eventSets_row(sqlite3_stmt* stmt,
135  unsigned const rsid,
136  unsigned const esid)
137  {
138  sqlite3_bind_int64(stmt, 1, rsid);
139  sqlite3_bind_int64(stmt, 2, esid);
140  sqlite3_step(stmt);
141  sqlite3_reset(stmt);
142  }
143 
144  int
145  found_rowid(sqlite3_stmt* stmt)
146  {
147  return sqlite3_step(stmt) == SQLITE_ROW ?
148  sqlite3_column_int64(stmt,0) :
149  throw art::Exception(art::errors::SQLExecutionError)
150  << "ROWID not found for EventRanges.\n"
151  << "Contact artists@fnal.gov.\n";
152  }
153 
154  unsigned
155  getNewRangeSetID(sqlite3* db,
156  art::BranchType const bt,
157  art::RunNumber_t const r)
158  {
159  sqlite::Transaction txn {db};
160  sqlite3_stmt* stmt {nullptr};
161  std::string const ddl {"INSERT INTO " + art::BranchTypeToString(bt) + "RangeSets(Run) VALUES(?);"};
162  sqlite3_prepare_v2(db, ddl.c_str(), -1, &stmt, nullptr);
163  insert_rangeSets_row(stmt, r);
164  unsigned const rsID = sqlite3_last_insert_rowid(db);
165  sqlite3_finalize(stmt);
166  txn.commit();
167  return rsID;
168  }
169 
170  vector<unsigned>
171  getExistingRangeSetIDs(sqlite3* db, art::RangeSet const& rs)
172  {
173  vector<unsigned> rangeSetIDs;
174  for (auto const& range : rs) {
175  sqlite::Transaction txn {db};
176  sqlite3_stmt* stmt {nullptr};
177  std::string const ddl {"SELECT ROWID FROM EventRanges WHERE "
178  "SubRun=" + std::to_string(range.subRun()) + " AND "
179  "begin=" + std::to_string(range.begin()) + " AND "
180  "end=" + std::to_string(range.end()) + ";"};
181  sqlite3_prepare_v2(db, ddl.c_str(), -1, &stmt, nullptr);
182  rangeSetIDs.push_back(found_rowid(stmt));
183  sqlite3_finalize(stmt);
184  txn.commit();
185  }
186  return rangeSetIDs;
187  }
188 
189  void
190  insertIntoEventRanges(sqlite3* db, art::RangeSet const& rs)
191  {
192  sqlite::Transaction txn {db};
193  sqlite3_stmt* stmt {nullptr};
194  std::string const ddl {"INSERT INTO EventRanges(SubRun, begin, end) "
195  "VALUES(?, ?, ?);"};
196  sqlite3_prepare_v2(db, ddl.c_str(), -1, &stmt, nullptr);
197  for (auto const& range : rs) {
198  insert_eventRanges_row(stmt, range.subRun(), range.begin(), range.end());
199  }
200  sqlite3_finalize(stmt);
201  txn.commit();
202  }
203 
204  void
205  insertIntoJoinTable(sqlite3* db,
206  art::BranchType const bt,
207  unsigned const rsID,
208  vector<unsigned> const& eventRangesIDs)
209  {
210  sqlite::Transaction txn {db};
211  sqlite3_stmt* stmt {nullptr};
212  std::string const ddl {"INSERT INTO "+art::BranchTypeToString(bt) +
213  "RangeSets_EventRanges(RangeSetsID, EventRangesID) Values(?,?);"};
214  sqlite3_prepare_v2(db, ddl.c_str(), -1, &stmt, nullptr);
215  cet::for_all(eventRangesIDs,
216  [stmt,rsID](auto const eventRangeID) {
217  insert_rangeSets_eventSets_row(stmt, rsID, eventRangeID);
218  });
219  sqlite3_finalize(stmt);
220  txn.commit();
221  }
222 
223  void
224  maybeInvalidateRangeSet(BranchType const bt,
225  art::RangeSet const& principalRS,
226  art::RangeSet& productRS)
227  {
228  if (!productRS.is_valid())
229  return;
230 
231  assert(principalRS.is_sorted());
232  assert(productRS.is_sorted());
233 
234  if (bt == art::InRun && productRS.is_full_run()) return;
235  if (bt == art::InSubRun && productRS.is_full_subRun()) return;
236  assert(!productRS.ranges().empty());
237 
238  auto const r = productRS.run();
239  auto const& productFront = productRS.ranges().front();
240  if (!principalRS.contains(r, productFront.subRun(), productFront.begin()))
241  productRS = art::RangeSet::invalid();
242  }
243 
244  using art::detail::RangeSetsSupported;
245 
246  // The purpose of 'maybeInvalidateRangeSet' is to support the
247  // following situation. Suppose process 1 creates three files with
248  // one Run product each, all corresponding to the same Run. Let's
249  // call the individual Run product instances in the three separate
250  // files as A, B, and C. Now suppose that the three files serve as
251  // inputs to process 2, where a concatenation is being performed AND
252  // ALSO an output file switch. Process 2 results in two output
253  // files, and now, in process 3, we concatenate the outputs from
254  // process 2. The situation would look like this:
255  //
256  // Process 1: [A] [B] [C]
257  // \ / \ /
258  // Process 2: [A + B] [B + C]
259  // \ / \ /
260  // D=agg(A,B) | | E=agg(B,C)
261  // \ /
262  // Process 3: [D + E]
263  //
264  // Notice the complication in process 3: product 'B' will be
265  // aggregated twice: once with A, and once with C. Whenever the
266  // output from process 3 is read as input to another process, the
267  // fetched product will be equivalent to A+2B+C.
268  //
269  // To avoid this situation, we compare the RangeSet of the product
270  // with the RangeSet of the in-memory RunAuxiliary. If the
271  // beginning of B's RangeSet is not contained within the auxiliary's
272  // RangeSet, then a dummy product with an invalid RangeSet is
273  // written to disk. Instead of the diagram above, we have:
274  //
275  // Process 1: [A] [B] [C]
276  // \ / \ /
277  // Process 2: [A + B] [x + C]
278  // \ / \ /
279  // D=agg(A,B) | | E=agg(x,C)=C
280  // \ /
281  // Process 3: [D + E]
282  //
283  // where 'x' represent a dummy product. Upon aggregating D and E,
284  // we obtain the correctly formed A+B+C product.
285  template <BranchType BT>
286  std::enable_if_t<RangeSetsSupported<BT>::value, art::RangeSet>
287  getRangeSet(art::OutputHandle const& oh,
288  art::RangeSet const& principalRS,
289  bool const producedInThisProcess)
290  {
291  auto rs = oh.isValid() ? oh.rangeOfValidity() : art::RangeSet::invalid();
292  // Because a user can specify (e.g.):
293  // r.put(std::move(myProd), art::runFragment(myRangeSet));
294  // products that are produced in this process can have valid, yet
295  // arbitrary RangeSets. We therefore never invalidate a RangeSet
296  // that corresponds to a product produced in this process.
297  //
298  // It is possible for a user to specify a RangeSet which does not
299  // correspond AT ALL to the in-memory auxiliary RangeSet. In that
300  // case, users should not expect to be able to retrieve products
301  // for which no corresponding events or sub-runs were processed.
302  if (!producedInThisProcess) {
303  maybeInvalidateRangeSet(BT, principalRS, rs);
304  }
305  return rs;
306  }
307 
308  template <BranchType BT>
309  std::enable_if_t<!RangeSetsSupported<BT>::value, art::RangeSet>
310  getRangeSet(art::OutputHandle const&,
311  art::RangeSet const& /*principalRS*/,
312  bool const /*producedInThisProcess*/)
313  {
314  return art::RangeSet::invalid();
315  }
316 
317  template <BranchType BT>
318  std::enable_if_t<!RangeSetsSupported<BT>::value>
319  setProductRangeSetID(art::RangeSet const& /*rs*/,
320  sqlite3*,
321  art::EDProduct*,
322  std::map<unsigned,unsigned>& /*checksumToIndexLookup*/)
323  {}
324 
325  template <BranchType BT>
326  std::enable_if_t<RangeSetsSupported<BT>::value>
327  setProductRangeSetID(art::RangeSet const& rs,
328  sqlite3* db,
329  art::EDProduct* product,
330  std::map<unsigned,unsigned>& checksumToIndexLookup)
331  {
332  if (!rs.is_valid()) // Invalid range-sets not written to DB
333  return;
334 
335  // Set range sets for SubRun and Run products
336  auto it = checksumToIndexLookup.find(rs.checksum());
337  if (it != checksumToIndexLookup.cend()) {
338  product->setRangeSetID(it->second);
339  }
340  else {
341  unsigned const rsID = getNewRangeSetID(db, BT, rs.run());
342  product->setRangeSetID(rsID);
343  checksumToIndexLookup.emplace(rs.checksum(), rsID);
344  insertIntoEventRanges(db, rs);
345  auto const& eventRangesIDs = getExistingRangeSetIDs(db, rs);
346  insertIntoJoinTable(db, BT, rsID, eventRangesIDs);
347  }
348  }
349 
350 } // unnamed namespace
351 
352 art::
353 RootDAQOutFile::
354 RootDAQOutFile(OutputModule* om,
355  string const& fileName,
356  ClosingCriteria const& fileSwitchCriteria,
357  int const compressionLevel,
358  int64_t const saveMemoryObjectThreshold,
359  int64_t const treeMaxVirtualSize,
360  int const splitLevel,
361  int const basketSize,
362  DropMetaData dropMetaData,
363  bool const dropMetaDataForDroppedData,
364  bool const fastCloning)
365  : om_{om}
366  , file_{fileName}
367  , fileSwitchCriteria_{fileSwitchCriteria}
368  , compressionLevel_{compressionLevel}
369  , saveMemoryObjectThreshold_{saveMemoryObjectThreshold}
370  , treeMaxVirtualSize_{treeMaxVirtualSize}
371  , splitLevel_{splitLevel}
372  , basketSize_{basketSize}
373  , dropMetaData_{dropMetaData}
374  , dropMetaDataForDroppedData_{dropMetaDataForDroppedData}
375  , fastCloning_{fastCloning}
376  , filePtr_{TFile::Open(file_.c_str(), "recreate", "", compressionLevel)}
377  , treePointers_ { // Order (and number) must match BranchTypes.h!
378  std::make_unique<RootOutputTree>(
379 # if ART_HEX_VERSION < 0x20703
380  static_cast<EventPrincipal*>(nullptr),
381 # endif
382  filePtr_.get(), InEvent, pEventAux_,
383  pEventProductProvenanceVector_, basketSize, splitLevel,
384  treeMaxVirtualSize, saveMemoryObjectThreshold),
385  std::make_unique<RootOutputTree>(
386 # if ART_HEX_VERSION < 0x20703
387  static_cast<SubRunPrincipal*>(nullptr),
388 # endif
389  filePtr_.get(), InSubRun, pSubRunAux_,
390  pSubRunProductProvenanceVector_, basketSize, splitLevel,
391  treeMaxVirtualSize, saveMemoryObjectThreshold),
392  std::make_unique<RootOutputTree>(
393 # if ART_HEX_VERSION < 0x20703
394  static_cast<RunPrincipal*>(nullptr),
395 # endif
396  filePtr_.get(), InRun, pRunAux_,
397  pRunProductProvenanceVector_, basketSize, splitLevel,
398  treeMaxVirtualSize, saveMemoryObjectThreshold),
399  std::make_unique<RootOutputTree>(
400 # if ART_HEX_VERSION < 0x20703
401  static_cast<ResultsPrincipal*>(nullptr),
402 # endif
403  filePtr_.get(), InResults, pResultsAux_,
404  pResultsProductProvenanceVector_, basketSize, splitLevel,
405  treeMaxVirtualSize, saveMemoryObjectThreshold) }
406 # if ART_HEX_VERSION < 0x20703
407  , rootFileDB_{filePtr_.get(), "RootFileDB", SQLITE_OPEN_CREATE | SQLITE_OPEN_READWRITE}
408 # else
409  , rootFileDB_{ServiceHandle<DatabaseConnection>{}->get<TKeyVFSOpenPolicy>("RootFileDB",
410  filePtr_.get(),
411  SQLITE_OPEN_CREATE|SQLITE_OPEN_READWRITE)}
412 # endif
413 {
414  // Don't split metadata tree or event description tree
415  metaDataTree_ = RootOutputTree::makeTTree(filePtr_.get(), rootNames::metaDataTreeName(), 0);
416  fileIndexTree_ = RootOutputTree::makeTTree(filePtr_.get(), rootNames::fileIndexTreeName(), 0);
417  parentageTree_ = RootOutputTree::makeTTree(filePtr_.get(), rootNames::parentageTreeName(), 0);
418  // Create the tree that will carry (event) History objects.
419  eventHistoryTree_ = RootOutputTree::makeTTree(filePtr_.get(), rootNames::eventHistoryTreeName(), splitLevel);
420  if (!eventHistoryTree_) {
421  throw art::Exception(art::errors::FatalRootError)
422  << "Failed to create the tree for History objects\n";
423  }
424 
425 
426  pHistory_ = new History;
427  if (!eventHistoryTree_->Branch(rootNames::eventHistoryBranchName().c_str(), &pHistory_, basketSize, 0)) {
428  throw art::Exception(art::errors::FatalRootError)
429  << "Failed to create a branch for History in the output file\n";
430  }
431  delete pHistory_;
432  pHistory_ = nullptr;
433 
434  createDatabaseTables();
435 }
436 
437 art::
438 RootDAQOutFile::OutputItem::Sorter::
439 Sorter(TTree* tree)
440 {
441  // Fill a map mapping branch names to an index specifying
442  // the order in the tree.
443  if (!tree) {
444  return;
445  }
446  auto branches = tree->GetListOfBranches();
447  for (int i = 0, sz = branches->GetEntries(); i != sz; ++i) {
448  auto br = reinterpret_cast<TBranchElement*>(branches->At(i));
449  treeMap_.emplace(string(br->GetName()), i);
450  }
451 }
452 
453 bool
454 art::
455 RootDAQOutFile::OutputItem::Sorter::
456 operator()(OutputItem const& lh, OutputItem const& rh) const
457 {
458  // Provides a comparison for sorting branches according
459  // to the index values in treeMap_. Branches not found
460  // are always put at the end, i.e. <not-found> is greater
461  // than <found>.
462  if (treeMap_.empty()) {
463  return lh < rh;
464  }
465  auto const& lname = lh.branchDescription_->branchName();
466  auto const& rname = rh.branchDescription_->branchName();
467  auto lit = treeMap_.find(lname);
468  auto rit = treeMap_.find(rname);
469  bool lfound = (lit != treeMap_.end());
470  bool rfound = (rit != treeMap_.end());
471  if (lfound && rfound) {
472  return lit->second < rit->second;
473  }
474  if (lfound) {
475  return true;
476  }
477  if (rfound) {
478  return false;
479  }
480  return lh < rh;
481 }
482 
483 void
484 art::RootDAQOutFile::createDatabaseTables()
485 {
486 
487  // // FileCatalog metadata
488  // create_table(rootFileDB_, "FileCatalog_metadata",
489  // {"ID INTEGER PRIMARY KEY", "Name", "Value"});
490 
491  // Event ranges
492  create_table(rootFileDB_, "EventRanges",
493  {"SubRun INTEGER", "begin INTEGER", "end INTEGER", "UNIQUE (SubRun,begin,end) ON CONFLICT IGNORE"});
494 
495  // SubRun range sets
496  create_table(rootFileDB_, "SubRunRangeSets",
497  {"Run INTEGER"});
498  create_table(rootFileDB_, "SubRunRangeSets_EventRanges",
499  {"RangeSetsID INTEGER", "EventRangesID INTEGER", "PRIMARY KEY(RangeSetsID,EventRangesID)"},
500  "WITHOUT ROWID");
501 
502  // Run range sets
503  create_table(rootFileDB_, "RunRangeSets",
504  {"Run INTEGER"});
505  create_table(rootFileDB_, "RunRangeSets_EventRanges",
506  {"RangeSetsID INTEGER", "EventRangesID INTEGER", "PRIMARY KEY(RangeSetsID,EventRangesID)"},
507  "WITHOUT ROWID");
508 }
509 
510 void
511 art::
512 RootDAQOutFile::
513 selectProducts(FileBlock const& fb)
514 {
515  for (int i = InEvent; i < NumBranchTypes; ++i) {
516  auto bt = static_cast<BranchType>(i);
517  auto& items = selectedOutputItemList_[bt];
518  for (auto const& val : items) {
519  treePointers_[bt]->resetOutputBranchAddress(*val.branchDescription_);
520  }
521  items.clear();
522  for (auto const& bd : om_->keptProducts()[bt]) {
523  if ((bt < InResults) || bd->produced()) {
524  items.emplace_back(bd);
525  }
526  }
527  if ((bt == InEvent) && (fb.tree() != nullptr)) {
528  // Only care about sorting event tree because that's the only one
529  // we might fast clone.
530  sort(items.begin(), items.end(), OutputItem::Sorter(fb.tree()));
531  }
532  for (auto const& val : items) {
533  treePointers_[bt]->addOutputBranch(*val.branchDescription_, val.product_);
534  }
535  }
536 }
537 
538 void
539 art::
540 RootDAQOutFile::
541 beginInputFile(FileBlock const& fb, bool fastClone)
542 {
543  selectProducts(fb);
544  auto const origCurrentlyFastCloning = currentlyFastCloning_;
545  currentlyFastCloning_ = fastCloning_ && fastClone;
546  if (currentlyFastCloning_ &&
547  !treePointers_[InEvent]->checkSplitLevelAndBasketSize(fb.tree())) {
548  mf::LogWarning("FastCloning")
549  << "Fast cloning deactivated for this input file due to "
550  << "splitting level and/or basket size.";
551  currentlyFastCloning_ = false;
552  } else if (fb.tree() && fb.tree()->GetCurrentFile()->GetVersion() < 60001) {
553  mf::LogWarning("FastCloning")
554  << "Fast cloning deactivated for this input file due to "
555  << "ROOT version used to write it (< 6.00/01)\n"
556  "having a different splitting policy.";
557  currentlyFastCloning_ = false;
558  }
559 
560  if (currentlyFastCloning_ && fb.fileFormatVersion().value_ < 9) {
561  mf::LogWarning("FastCloning")
562  << "Fast cloning deactivated for this input file due to "
563  << "reading in file that does not support RangeSets.";
564  currentlyFastCloning_ = false;
565  }
566 
567  if (currentlyFastCloning_ && !origCurrentlyFastCloning) {
568  mf::LogWarning("FastCloning")
569  << "Fast cloning reactivated for this input file.";
570  }
571  treePointers_[InEvent]->beginInputFile(currentlyFastCloning_);
572  treePointers_[InEvent]->fastCloneTree(fb.tree());
573 }
574 
575 void
576 art::RootDAQOutFile::incrementInputFileNumber()
577 {
578 # if ART_HEX_VERSION >= 0x20703
579  fp_.update<Granularity::InputFile>();
580 # else
581  fp_.update<Boundary::InputFile>();
582 # endif
583 }
584 
585 void
586 art::
587 RootDAQOutFile::
588 respondToCloseInputFile(FileBlock const&)
589 {
590  cet::for_all(treePointers_, [](auto const& p){ p->setEntries(); });
591 }
592 
593 bool
594 art::
595 RootDAQOutFile::
596 requestsToCloseFile()
597 {
598  using namespace std::chrono;
599  unsigned int constexpr oneK {1024u};
600  TLOG(10) << "RootDAQOutFile::requestsToCloseFile start" ;
601  unsigned sz=filePtr_->GetSize();
602  TLOG(10) << "RootDAQOutFile::requestsToCloseFile after filePtr_->GetSize()";
603  fp_.updateSize(sz/oneK);
604  fp_.updateAge(duration_cast<seconds>(steady_clock::now() - beginTime_));
605  bool ret=fileSwitchCriteria_.should_close(fp_);
606  TLOG(10) << "RootDAQOutFile::requestsToCloseFile done/return" ;
607  return ret;
608 }
609 
610 void art::RootDAQOutFile::writeOne(EventPrincipal const& e)
611 {
612  TLOG(10) << "RootDAQOutFile::writeOne begin" ;
613  // Auxiliary branch.
614  // Note: pEventAux_ must be set before calling fillBranches
615  // since it gets written out in that routine.
616  pEventAux_ = &e.aux();
617  // Because getting the data may cause an exception to be
618  // thrown we want to do that first before writing anything
619  // to the file about this event.
620  fillBranches<InEvent>(e, pEventProductProvenanceVector_);
621  // History branch.
622  History historyForOutput {e.history()};
623  historyForOutput.addEventSelectionEntry(om_->selectorConfig());
624  pHistory_ = &historyForOutput;
625  int sz = eventHistoryTree_->Fill();
626  if (sz <= 0) {
627  throw art::Exception(art::errors::FatalRootError)
628  << "Failed to fill the History tree for event: "
629  << e.id()
630  << "\nTTree::Fill() returned "
631  << sz
632  << " bytes written."
633  << endl;
634  }
635  // Add the dataType to the job report if it hasn't already been done
636  if (!dataTypeReported_) {
637  string dataType {"MC"};
638  if (pEventAux_->isRealData()) {
639  dataType = "Data";
640  }
641  dataTypeReported_ = true;
642  }
643  pHistory_ = &e.history();
644  // Add event to index
645  fileIndex_.addEntry(pEventAux_->id(), fp_.eventEntryNumber());
646 # if ART_HEX_VERSION >= 0x20703
647  fp_.update<Granularity::Event>(status_);
648 # else
649  fp_.update<Boundary::Event>(status_);
650 # endif
651  TLOG(10) << "RootDAQOutFile::writeOne done/return" ;
652 }
653 
654 void
655 art::
656 RootDAQOutFile::
657 writeSubRun(SubRunPrincipal const& sr)
658 {
659  pSubRunAux_ = &sr.aux();
660  pSubRunAux_->setRangeSetID(subRunRSID_);
661  fillBranches<InSubRun>(sr, pSubRunProductProvenanceVector_);
662  fileIndex_.addEntry(EventID::invalidEvent(pSubRunAux_->id()), fp_.subRunEntryNumber());
663 # if ART_HEX_VERSION >= 0x20703
664  fp_.update<Granularity::SubRun>(status_);
665 # else
666  fp_.update<Boundary::SubRun>(status_);
667 # endif
668 }
669 
670 void
671 art::
672 RootDAQOutFile::
673 writeRun(RunPrincipal const& r)
674 {
675  pRunAux_ = &r.aux();
676  pRunAux_->setRangeSetID(runRSID_);
677  fillBranches<InRun>(r, pRunProductProvenanceVector_);
678  fileIndex_.addEntry(EventID::invalidEvent(pRunAux_->id()), fp_.runEntryNumber());
679 # if ART_HEX_VERSION >= 0x20703
680  fp_.update<Granularity::Run>(status_);
681 # else
682  fp_.update<Boundary::Run>(status_);
683 # endif
684 }
685 
686 void
687 art::
688 RootDAQOutFile::
689 writeParentageRegistry()
690 {
691  ParentageID const* hash = new ParentageID;
692  if (!parentageTree_->Branch(rootNames::parentageIDBranchName().c_str(),
693  &hash, basketSize_, 0)) {
694  throw art::Exception(art::errors::FatalRootError)
695  << "Failed to create a branch for ParentageIDs in the output file";
696  }
697  delete hash;
698  hash = nullptr;
699  Parentage const* desc = new Parentage;
700  if (!parentageTree_->Branch(rootNames::parentageBranchName().c_str(), &desc,
701  basketSize_, 0)) {
702  throw art::Exception(art::errors::FatalRootError)
703  << "Failed to create a branch for Parentages in the output file";
704  }
705  delete desc;
706  desc = nullptr;
707  for (auto const& pr : ParentageRegistry::get()) {
708  hash = &pr.first;
709  desc = &pr.second;
710  parentageTree_->Fill();
711  }
712  parentageTree_->SetBranchAddress(rootNames::parentageIDBranchName().c_str(), nullptr);
713  parentageTree_->SetBranchAddress(rootNames::parentageBranchName().c_str(), nullptr);
714 }
715 
716 void
717 art::
718 RootDAQOutFile::
719 writeFileFormatVersion()
720 {
721  FileFormatVersion ver(getFileFormatVersion(), getFileFormatEra());
722  FileFormatVersion* pver = &ver;
723  TBranch* b = metaDataTree_->Branch(metaBranchRootName<FileFormatVersion>(),
724  &pver, basketSize_, 0);
725  // FIXME: Turn this into a throw!
726  assert(b);
727  b->Fill();
728 }
729 
730 void
731 art::
732 RootDAQOutFile::
733 writeFileIndex()
734 {
735  fileIndex_.sortBy_Run_SubRun_Event();
736  FileIndex::Element elem {};
737  auto findexElemPtr = &elem;
738  TBranch* b = fileIndexTree_->Branch(metaBranchRootName<FileIndex::Element>(),
739  &findexElemPtr, basketSize_, 0);
740  // FIXME: Turn this into a throw!
741  assert(b);
742  for (auto& entry: fileIndex_) {
743  findexElemPtr = &entry;
744  b->Fill();
745  }
746  b->SetAddress(0);
747 }
748 
749 void
750 art::
751 RootDAQOutFile::
752 writeEventHistory()
753 {
754  RootOutputTree::writeTTree(eventHistoryTree_);
755 }
756 
757 void
758 art::
759 RootDAQOutFile::
760 writeProcessConfigurationRegistry()
761 {
762  // We don't do this yet; currently we're storing a slightly
763  // bloated ProcessHistoryRegistry.
764 }
765 
766 void
767 art::
768 RootDAQOutFile::
769 writeProcessHistoryRegistry()
770 {
771 # if ART_HEX_VERSION >= 0x20703
772  ProcessHistoryMap pHistMap;
773  for (auto const& pr : ProcessHistoryRegistry::get()) {
774  pHistMap.emplace(pr);
775  }
776  auto const* p = &pHistMap;
777 # else
778  ProcessHistoryMap const& r = ProcessHistoryRegistry::get();
779  ProcessHistoryMap* p = &const_cast<ProcessHistoryMap&>(r);
780 # endif
781  TBranch* b = metaDataTree_->Branch(metaBranchRootName<ProcessHistoryMap>(),
782  &p, basketSize_, 0);
783  if (b != nullptr) {
784  b->Fill();
785  } else {
786  throw Exception(errors::LogicError)
787  << "Unable to locate required ProcessHistoryMap branch in output metadata tree.\n";
788  }
789 }
790 
791 void
792 art::
793 RootDAQOutFile::
794 writeBranchIDListRegistry()
795 {
796 # if ART_HEX_VERSION >= 0x20703
797  BranchIDLists const* p = &BranchIDListRegistry::instance().data();
798 # else
799  BranchIDLists* p = &BranchIDListRegistry::instance()->data();
800 # endif
801  TBranch* b = metaDataTree_->Branch(metaBranchRootName<BranchIDLists>(), &p,
802  basketSize_, 0);
803  // FIXME: Turn this into a throw!
804  assert(b);
805  b->Fill();
806 }
807 
808 void
809 art::
810 RootDAQOutFile::
811 writeFileCatalogMetadata(FileStatsCollector const& stats,
812  FileCatalogMetadata::collection_type const& md,
813  FileCatalogMetadata::collection_type const& ssmd)
814 {
815 # if ART_HEX_VERSION >= 0x20703
816  using namespace cet::sqlite;
817  Ntuple<std::string,std::string> fileCatalogMetadata {rootFileDB_, "FileCatalog_metadata", {"Name","Value"}, true};
818  Transaction txn {rootFileDB_};
819  for (auto const& kv : md) {
820  fileCatalogMetadata.insert(kv.first, kv.second);
821  }
822  // Add our own specific information: File format and friends.
823  fileCatalogMetadata.insert("file_format", "\"artroot\"");
824  fileCatalogMetadata.insert("file_format_era", cet::canonical_string(getFileFormatEra()));
825  fileCatalogMetadata.insert("file_format_version", to_string(getFileFormatVersion()));
826 # else
827  ntuple::Ntuple<std::string,std::string> fileCatalogMetadata {rootFileDB_,
828  "FileCatalog_metadata",
829  {"Name","Value"},
830  true};
831  using namespace sqlite;
832  Transaction txn {rootFileDB_};
833  for (auto const& kv : md) {
834  insert_into(fileCatalogMetadata).values(kv.first, kv.second);
835  }
836  // Add our own specific information: File format and friends.
837  insert_into(fileCatalogMetadata).values("file_format", "\"artroot\"");
838  insert_into(fileCatalogMetadata).values("file_format_era", cet::canonical_string(getFileFormatEra()));
839  insert_into(fileCatalogMetadata).values("file_format_version", to_string(getFileFormatVersion()));
840 # endif
841 
842  // File start time.
843  namespace bpt = boost::posix_time;
844  auto formatted_time = [](auto const& t){ return cet::canonical_string(bpt::to_iso_extended_string(t)); };
845 # if ART_HEX_VERSION >= 0x20703
846  fileCatalogMetadata.insert("start_time", formatted_time(stats.outputFileOpenTime()));
847  // File "end" time: now, since file is not actually closed yet.
848  fileCatalogMetadata.insert("end_time", formatted_time(boost::posix_time::second_clock::universal_time()));
849 # else
850  insert_into(fileCatalogMetadata).values("start_time", formatted_time(stats.outputFileOpenTime()));
851  // File "end" time: now, since file is not actually closed yet.
852  insert_into(fileCatalogMetadata).values("end_time", formatted_time(boost::posix_time::second_clock::universal_time()));
853 # endif
854 
855  // Run/subRun information.
856  if (!stats.seenSubRuns().empty()) {
857 
858  auto I = find_if(md.crbegin(), md.crend(),
859  [](auto const& p){ return p.first == "run_type"; } );
860 
861  if (I != md.crend()) {
862  ostringstream buf;
863  buf << "[ ";
864  for (auto const& srid : stats.seenSubRuns()) {
865  buf << "[ "
866  << srid.run()
867  << ", "
868  << srid.subRun()
869  << ", "
870  << cet::canonical_string(I->second)
871  << " ], ";
872  }
873  // Rewind over last delimiter.
874  buf.seekp(-2, ios_base::cur);
875  buf << " ]";
876 # if ART_HEX_VERSION >= 0x20703
877  fileCatalogMetadata.insert("runs", buf.str());
878 # else
879  insert_into(fileCatalogMetadata).values("runs", buf.str());
880 # endif
881  }
882  }
883  // Number of events.
884 # if ART_HEX_VERSION >= 0x20703
885  fileCatalogMetadata.insert("event_count", to_string(stats.eventsThisFile()));
886 # else
887  insert_into(fileCatalogMetadata).values("event_count", to_string(stats.eventsThisFile()));
888 # endif
889  // first_event and last_event.
890  auto eidToTuple = [](EventID const & eid)->string {
891  ostringstream eidStr;
892  eidStr << "[ "
893  << eid.run()
894  << ", "
895  << eid.subRun()
896  << ", "
897  << eid.event()
898  << " ]";
899  return eidStr.str();
900  };
901 # if ART_HEX_VERSION >= 0x20703
902  fileCatalogMetadata.insert("first_event", eidToTuple(stats.lowestEventID()));
903  fileCatalogMetadata.insert("last_event", eidToTuple(stats.highestEventID()));
904 # else
905  insert_into(fileCatalogMetadata).values("first_event", eidToTuple(stats.lowestEventID()));
906  insert_into(fileCatalogMetadata).values("last_event", eidToTuple(stats.highestEventID()));
907 # endif
908  // File parents.
909  if (!stats.parents().empty()) {
910  ostringstream pstring;
911  pstring << "[ ";
912  for (auto const& parent : stats.parents()) {
913  pstring << cet::canonical_string(parent) << ", ";
914  }
915  // Rewind over last delimiter.
916  pstring.seekp(-2, ios_base::cur);
917  pstring << " ]";
918 # if ART_HEX_VERSION >= 0x20703
919  fileCatalogMetadata.insert("parents", pstring.str());
920 # else
921  insert_into(fileCatalogMetadata).values("parents", pstring.str());
922 # endif
923  }
924  // Incoming stream-specific metadata overrides.
925  for (auto const& kv : ssmd) {
926 # if ART_HEX_VERSION >= 0x20703
927  fileCatalogMetadata.insert(kv.first, kv.second);
928 # else
929  insert_into(fileCatalogMetadata).values(kv.first, kv.second);
930 # endif
931  }
932  txn.commit();
933 }
934 
935 void
936 art::
937 RootDAQOutFile::
938 writeParameterSetRegistry()
939 {
940  fhicl::ParameterSetRegistry::exportTo(rootFileDB_);
941 }
942 
943 void
944 art::
945 RootDAQOutFile::
946 writeProductDescriptionRegistry()
947 {
948  // Make a local copy of the MasterProductRegistry's ProductList,
949  // removing any transient or pruned products.
950  auto end = branchesWithStoredHistory_.end();
951 
952  ProductRegistry reg;
953  for ( auto const& pr : ProductMetaData::instance().productList() ) {
954  if ( branchesWithStoredHistory_.find(pr.second.branchID()) == end ){
955  continue;
956  }
957  reg.productList_.emplace_hint(reg.productList_.end(),pr);
958  }
959 
960  auto* regp = &reg;
961  TBranch* b = metaDataTree_->Branch(metaBranchRootName<ProductRegistry>(),
962  &regp, basketSize_, 0);
963  // FIXME: Turn this into a throw!
964  assert(b);
965  b->Fill();
966 }
967 
968 void
969 art::
970 RootDAQOutFile::
971 writeProductDependencies()
972 {
973  BranchChildren& pDeps = const_cast<BranchChildren&>(om_->branchChildren());
974  BranchChildren* ppDeps = &pDeps;
975  TBranch* b = metaDataTree_->Branch(metaBranchRootName<BranchChildren>(),
976  &ppDeps, basketSize_, 0);
977  // FIXME: Turn this into a throw!
978  assert(b);
979  b->Fill();
980 }
981 
982 void
983 art::
984 RootDAQOutFile::
985 writeResults(ResultsPrincipal & resp)
986 {
987  pResultsAux_ = &resp.aux();
988  fillBranches<InResults>(resp, pResultsProductProvenanceVector_);
989 }
990 
991 void
992 # if ART_HEX_VERSION >= 0x20703
993 art::RootDAQOutFile::writeTTrees()
994 # else
995 RootDAQOutFile::
996 finishEndFile()
997 # endif
998 {
999 # if ART_HEX_VERSION < 0x20703
1000  metaDataTree_->SetEntries(-1);
1001 # endif
1002  RootOutputTree::writeTTree(metaDataTree_);
1003  RootOutputTree::writeTTree(fileIndexTree_);
1004  RootOutputTree::writeTTree(parentageTree_);
1005  // Write out the tree corresponding to each BranchType
1006  for (int i = InEvent; i < NumBranchTypes; ++i) {
1007  auto const branchType = static_cast<BranchType>(i);
1008  treePointers_[branchType]->writeTree();
1009  }
1010 # if ART_HEX_VERSION < 0x20703
1011  // Write out DB -- the d'tor of the SQLite3Wrapper calls
1012  // sqlite3_close. For the tkeyvfs, closing the DB calls
1013  // rootFile->Write("",TObject::kOverwrite).
1014  rootFileDB_.reset();
1015  // Close the file.
1016  filePtr_->Close();
1017  filePtr_.reset();
1018 # endif
1019 }
1020 
1021 template <art::BranchType BT>
1022 void art::RootDAQOutFile::fillBranches(Principal const& principal,
1023  vector<ProductProvenance>* vpp)
1024 {
1025  TLOG(11) << "RootDAQOutFile::fillBranches begin" ;
1026  bool const fastCloning = (BT == InEvent) && currentlyFastCloning_;
1027  detail::KeptProvenance keptProvenance {dropMetaData_, dropMetaDataForDroppedData_, branchesWithStoredHistory_};
1028  map<unsigned,unsigned> checksumToIndex;
1029 
1030  auto const& principalRS = principal.seenRanges();
1031 
1032  for (auto const& val : selectedOutputItemList_[BT]) {
1033  auto const* bd = val.branchDescription_;
1034  auto const bid = bd->branchID();
1035  branchesWithStoredHistory_.insert(bid);
1036  bool const produced {bd->produced()};
1037  bool const resolveProd = (produced || !fastCloning ||
1038  treePointers_[BT]->uncloned(bd->branchName()));
1039 
1040  // Update the kept provenance
1041  bool const keepProvenance = (dropMetaData_ == DropMetaData::DropNone ||
1042  (dropMetaData_ == DropMetaData::DropPrior &&
1043  produced));
1044  auto const& oh = principal.getForOutput(bid, resolveProd);
1045 
1046  unique_ptr<ProductProvenance> prov {nullptr};
1047  if (keepProvenance) {
1048  if (oh.productProvenance()) {
1049  prov = std::make_unique<ProductProvenance>(keptProvenance.insert(*oh.productProvenance()));
1050  keptProvenance.insertAncestors(*oh.productProvenance(), principal);
1051  }
1052  else {
1053  // No provenance, product was either not produced,
1054  // or was dropped, create provenance to remember that.
1055  auto const status = produced ? productstatus::neverCreated() : productstatus::dropped();
1056  prov = std::make_unique<ProductProvenance>(keptProvenance.emplace(bid, status));
1057  }
1058  }
1059 
1060  // Resolve the product if necessary
1061  if (resolveProd) {
1062  auto const& rs = getRangeSet<BT>(oh, principalRS, produced);
1063  if (RangeSetsSupported<BT>::value && !rs.is_valid()) {
1064  // Unfortunately, 'unknown' is the only viable product status
1065  // when this condition is triggered (due to the assert
1066  // statement in ProductStatus::setNotPresent). Whenever the
1067  // metadata revolution comes, this should be revised.
1068  keptProvenance.setStatus(*prov, productstatus::unknown());
1069  }
1070 
1071  auto const* product = getProduct<BT>(oh, rs, bd->wrappedName());
1072  setProductRangeSetID<BT>(rs,
1073  rootFileDB_,
1074  const_cast<EDProduct*>(product),
1075  checksumToIndex);
1076  val.product_ = product;
1077  }
1078  }
1079  vpp->assign(keptProvenance.begin(), keptProvenance.end());
1080  treePointers_[BT]->fillTree(); // write happes here
1081  vpp->clear();
1082  TLOG(11) << "RootDAQOutFile::fillBranches done/return" ;
1083 }
1084 
1085 void
1086 art::
1087 RootDAQOutFile::
1088 setSubRunAuxiliaryRangeSetID(RangeSet const& ranges)
1089 {
1090  subRunRSID_ = getNewRangeSetID(rootFileDB_, InSubRun, ranges.run());
1091  insertIntoEventRanges(rootFileDB_, ranges);
1092  auto const& eventRangesIDs = getExistingRangeSetIDs(rootFileDB_, ranges);
1093  insertIntoJoinTable(rootFileDB_, InSubRun, subRunRSID_, eventRangesIDs);
1094 }
1095 
1096 void
1097 art::
1098 RootDAQOutFile::
1099 setRunAuxiliaryRangeSetID(RangeSet const& ranges)
1100 {
1101  runRSID_ = getNewRangeSetID(rootFileDB_, InRun, ranges.run());
1102  insertIntoEventRanges(rootFileDB_, ranges);
1103  auto const& eventRangesIDs = getExistingRangeSetIDs(rootFileDB_, ranges);
1104  insertIntoJoinTable(rootFileDB_, InRun, runRSID_, eventRangesIDs);
1105 }
1106 
1107 template <BranchType BT>
1108 std::enable_if_t<!RangeSetsSupported<BT>::value, art::EDProduct const*>
1109 art::RootDAQOutFile::getProduct(art::OutputHandle const& oh,
1110  art::RangeSet const& /*prunedProductRS*/,
1111  std::string const& wrappedName)
1112 {
1113  if (oh.isValid())
1114  return oh.wrapper();
1115  else
1116  return dummyProductCache_.product(wrappedName);
1117 }
1118 
1119 template <BranchType BT>
1120 std::enable_if_t<RangeSetsSupported<BT>::value, art::EDProduct const*>
1121 art::RootDAQOutFile::getProduct(art::OutputHandle const& oh,
1122  art::RangeSet const& prunedProductRS,
1123  std::string const& wrappedName)
1124 {
1125  if (oh.isValid() && prunedProductRS.is_valid())
1126  return oh.wrapper();
1127  else
1128  return dummyProductCache_.product(wrappedName);
1129 }
An output module which allows specifying the output filename exactly. Used for testing (file -&gt; /dev/...