artdaq  v3_06_01
RootDAQOutFile.cc
1 #include "artdaq/ArtModules/RootDAQOutput-s85/RootDAQOutFile.h"
2 // vim: set sw=2:
3 
4 #include "art/Framework/IO/FileStatsCollector.h"
5 #include "art/Framework/IO/Root/DropMetaData.h"
6 #include "art/Framework/IO/Root/GetFileFormatEra.h"
7 #include "art/Framework/IO/Root/GetFileFormatVersion.h"
8 #include "art/Framework/IO/Root/RootDB/SQLErrMsg.h"
9 #include "art/Framework/IO/Root/RootDB/TKeyVFSOpenPolicy.h"
10 #include "art/Framework/IO/Root/RootFileBlock.h"
11 #include "art/Framework/IO/Root/checkDictionaries.h"
12 #include "art/Framework/IO/Root/detail/getObjectRequireDict.h"
13 #include "art/Framework/IO/Root/detail/readFileIndex.h"
14 #include "art/Framework/Principal/EventPrincipal.h"
15 #include "art/Framework/Principal/ResultsPrincipal.h"
16 #include "art/Framework/Principal/RunPrincipal.h"
17 #include "art/Framework/Principal/SubRunPrincipal.h"
18 #include "art/Framework/Services/System/DatabaseConnection.h"
19 #include "art/Persistency/Provenance/ProcessHistoryRegistry.h"
20 #include "art/Persistency/Provenance/ProductMetaData.h"
21 #include "art/Version/GetReleaseVersion.h"
22 #include "boost/date_time/posix_time/posix_time.hpp"
23 #include "canvas/Persistency/Provenance/BranchChildren.h"
24 #include "canvas/Persistency/Provenance/BranchType.h"
25 #include "canvas/Persistency/Provenance/EventAuxiliary.h"
26 #include "canvas/Persistency/Provenance/EventID.h"
27 #include "canvas/Persistency/Provenance/FileFormatVersion.h"
28 #include "canvas/Persistency/Provenance/History.h"
29 #include "canvas/Persistency/Provenance/ParameterSetBlob.h"
30 #include "canvas/Persistency/Provenance/Parentage.h"
31 #include "canvas/Persistency/Provenance/ParentageRegistry.h"
32 #include "canvas/Persistency/Provenance/ProcessHistoryID.h"
33 #include "canvas/Persistency/Provenance/ProductID.h"
34 #include "canvas/Persistency/Provenance/ProductProvenance.h"
35 #include "canvas/Persistency/Provenance/ProductStatus.h"
36 #include "canvas/Persistency/Provenance/ResultsAuxiliary.h"
37 #include "canvas/Persistency/Provenance/RunAuxiliary.h"
38 #include "canvas/Persistency/Provenance/SubRunAuxiliary.h"
39 #include "canvas/Persistency/Provenance/rootNames.h"
40 #include "canvas/Utilities/Exception.h"
41 #include "canvas_root_io/Utilities/DictionaryChecker.h"
42 #include "cetlib/canonical_string.h"
43 #include "cetlib/container_algorithms.h"
44 #include "cetlib/exempt_ptr.h"
45 #include "cetlib/sqlite/Ntuple.h"
46 #include "cetlib/sqlite/Transaction.h"
47 #include "cetlib/sqlite/create_table.h"
48 #include "cetlib/sqlite/exec.h"
49 #include "cetlib/sqlite/insert.h"
50 #include "fhiclcpp/ParameterSet.h"
51 #include "fhiclcpp/ParameterSetID.h"
52 #include "fhiclcpp/ParameterSetRegistry.h"
53 
54 #include "TFile.h"
55 #include "TTree.h"
56 #include "artdaq/DAQdata/Globals.hh"
57 #include "tracemf.h" // TLOG
58 #define TRACE_NAME (app_name + "_RootDAQOutFile").c_str()
59 
60 #include <algorithm>
61 #include <map>
62 #include <set>
63 #include <utility>
64 #include <vector>
65 #include <fcntl.h> // posix_fadvise POSIX_FADV_DONTNEED
66 #include <sys/sysinfo.h> // sysinfo(sysinfo*)
67 
68 using namespace cet;
69 using namespace std;
70 using art::BranchType;
72 using art::rootNames::metaBranchRootName;
73 
74 namespace {
75 
76  void
77  create_table(sqlite3* const db,
78  std::string const& name,
79  std::vector<std::string> const& columns,
80  std::string const& suffix = {})
81  {
82  if (columns.empty())
83  throw art::Exception(art::errors::LogicError)
84  << "Number of sqlite columns specified for table: " << name << '\n'
85  << "is zero.\n";
86 
87  std::string ddl = "DROP TABLE IF EXISTS " + name +
88  "; "
89  "CREATE TABLE " +
90  name + "(" + columns.front();
91  std::for_each(columns.begin() + 1, columns.end(), [&ddl](auto const& col) {
92  ddl += "," + col;
93  });
94  ddl += ") ";
95  ddl += suffix;
96  ddl += ";";
97  sqlite::exec(db, ddl);
98  }
99 
100  void
101  insert_eventRanges_row(sqlite3_stmt* stmt,
102  art::SubRunNumber_t const sr,
103  art::EventNumber_t const b,
104  art::EventNumber_t const e)
105  {
106  sqlite3_bind_int64(stmt, 1, sr);
107  sqlite3_bind_int64(stmt, 2, b);
108  sqlite3_bind_int64(stmt, 3, e);
109  sqlite3_step(stmt);
110  sqlite3_reset(stmt);
111  }
112 
113  void
114  insert_rangeSets_eventSets_row(sqlite3_stmt* stmt,
115  unsigned const rsid,
116  unsigned const esid)
117  {
118  sqlite3_bind_int64(stmt, 1, rsid);
119  sqlite3_bind_int64(stmt, 2, esid);
120  sqlite3_step(stmt);
121  sqlite3_reset(stmt);
122  }
123 
124  unsigned
125  getNewRangeSetID(sqlite3* db,
126  art::BranchType const bt,
127  art::RunNumber_t const r)
128  {
129  sqlite::insert_into(db, art::BranchTypeToString(bt) + "RangeSets")
130  .values(r);
131  return sqlite3_last_insert_rowid(db);
132  }
133 
134  vector<unsigned>
135  getExistingRangeSetIDs(sqlite3* db, art::RangeSet const& rs)
136  {
137  vector<unsigned> rangeSetIDs;
138  cet::transform_all(
139  rs, std::back_inserter(rangeSetIDs), [db](auto const& range) {
140  sqlite::query_result<unsigned> r;
141  r << sqlite::select("ROWID")
142  .from(db, "EventRanges")
143  .where("SubRun=" + std::to_string(range.subRun()) +
144  " AND "
145  "begin=" +
146  std::to_string(range.begin()) +
147  " AND "
148  "end=" +
149  std::to_string(range.end()));
150  return unique_value(r);
151  });
152  return rangeSetIDs;
153  }
154 
155  void
156  insertIntoEventRanges(sqlite3* db, art::RangeSet const& rs)
157  {
158  sqlite::Transaction txn{db};
159  sqlite3_stmt* stmt{nullptr};
160  std::string const ddl{"INSERT INTO EventRanges(SubRun, begin, end) "
161  "VALUES(?, ?, ?);"};
162  sqlite3_prepare_v2(db, ddl.c_str(), -1, &stmt, nullptr);
163  for (auto const& range : rs) {
164  insert_eventRanges_row(stmt, range.subRun(), range.begin(), range.end());
165  }
166  sqlite3_finalize(stmt);
167  txn.commit();
168  }
169 
170  void
171  insertIntoJoinTable(sqlite3* db,
172  art::BranchType const bt,
173  unsigned const rsID,
174  vector<unsigned> const& eventRangesIDs)
175  {
176  sqlite::Transaction txn{db};
177  sqlite3_stmt* stmt{nullptr};
178  std::string const ddl{
179  "INSERT INTO " + art::BranchTypeToString(bt) +
180  "RangeSets_EventRanges(RangeSetsID, EventRangesID) Values(?,?);"};
181  sqlite3_prepare_v2(db, ddl.c_str(), -1, &stmt, nullptr);
182  cet::for_all(eventRangesIDs, [stmt, rsID](auto const eventRangeID) {
183  insert_rangeSets_eventSets_row(stmt, rsID, eventRangeID);
184  });
185  sqlite3_finalize(stmt);
186  txn.commit();
187  }
188 
189  void
190  maybeInvalidateRangeSet(BranchType const bt,
191  art::RangeSet const& principalRS,
192  art::RangeSet& productRS)
193  {
194  assert(principalRS.is_sorted());
195  assert(productRS.is_sorted());
196 
197  if (!productRS.is_valid()) {
198  return;
199  }
200  if (bt == art::InRun && productRS.is_full_run()) {
201  return;
202  }
203  if (bt == art::InSubRun && productRS.is_full_subRun()) {
204  return;
205  }
206  if (productRS.ranges().empty()) {
207  return;
208  }
209  auto const r = productRS.run();
210  auto const& productFront = productRS.ranges().front();
211  if (!principalRS.contains(r, productFront.subRun(), productFront.begin())) {
212  productRS = art::RangeSet::invalid();
213  }
214  }
215 
216  using art::detail::RangeSetsSupported;
217 
218  // The purpose of 'maybeInvalidateRangeSet' is to support the
219  // following situation. Suppose process 1 creates three files with
220  // one Run product each, all corresponding to the same Run. Let's
221  // call the individual Run product instances in the three separate
222  // files as A, B, and C. Now suppose that the three files serve as
223  // inputs to process 2, where a concatenation is being performed AND
224  // ALSO an output file switch. Process 2 results in two output
225  // files, and now, in process 3, we concatenate the outputs from
226  // process 2. The situation would look like this:
227  //
228  // Process 1: [A] [B] [C]
229  // \ / \ /
230  // Process 2: [A + B] [B + C]
231  // \ / \ /
232  // D=agg(A,B) | | E=agg(B,C)
233  // \ /
234  // Process 3: [D + E]
235  //
236  // Notice the complication in process 3: product 'B' will be
237  // aggregated twice: once with A, and once with C. Whenever the
238  // output from process 3 is read as input to another process, the
239  // fetched product will be equivalent to A+2B+C.
240  //
241  // To avoid this situation, we compare the RangeSet of the product
242  // with the RangeSet of the in-memory RunAuxiliary. If the
243  // beginning of B's RangeSet is not contained within the auxiliary's
244  // RangeSet, then a dummy product with an invalid RangeSet is
245  // written to disk. Instead of the diagram above, we have:
246  //
247  // Process 1: [A] [B] [C]
248  // \ / \ /
249  // Process 2: [A + B] [x + C]
250  // \ / \ /
251  // D=agg(A,B) | | E=agg(x,C)=C
252  // \ /
253  // Process 3: [D + E]
254  //
255  // where 'x' represent a dummy product. Upon aggregating D and E,
256  // we obtain the correctly formed A+B+C product.
257  template <BranchType BT>
258  std::enable_if_t<RangeSetsSupported<BT>::value, art::RangeSet>
259  getRangeSet(art::OutputHandle const& oh,
260  art::RangeSet const& principalRS,
261  bool const producedInThisProcess)
262  {
263  auto rs = oh.isValid() ? oh.rangeOfValidity() : art::RangeSet::invalid();
264  // Because a user can specify (e.g.):
265  // r.put(std::move(myProd), art::runFragment(myRangeSet));
266  // products that are produced in this process can have valid, yet
267  // arbitrary RangeSets. We therefore never invalidate a RangeSet
268  // that corresponds to a product produced in this process.
269  //
270  // It is possible for a user to specify a RangeSet which does not
271  // correspond AT ALL to the in-memory auxiliary RangeSet. In that
272  // case, users should not expect to be able to retrieve products
273  // for which no corresponding events or sub-runs were processed.
274  if (!producedInThisProcess) {
275  maybeInvalidateRangeSet(BT, principalRS, rs);
276  }
277  return rs;
278  }
279 
280  template <BranchType BT>
281  std::enable_if_t<!RangeSetsSupported<BT>::value, art::RangeSet>
282  getRangeSet(art::OutputHandle const&,
283  art::RangeSet const& /*principalRS*/,
284  bool const /*producedInThisProcess*/)
285  {
286  return art::RangeSet::invalid();
287  }
288 
289  template <BranchType BT>
290  std::enable_if_t<!RangeSetsSupported<BT>::value>
291  setProductRangeSetID(art::RangeSet const& /*rs*/,
292  sqlite3*,
293  art::EDProduct*,
294  std::map<unsigned, unsigned>& /*checksumToIndexLookup*/)
295  {}
296 
297  template <BranchType BT>
298  std::enable_if_t<RangeSetsSupported<BT>::value>
299  setProductRangeSetID(art::RangeSet const& rs,
300  sqlite3* db,
301  art::EDProduct* product,
302  std::map<unsigned, unsigned>& checksumToIndexLookup)
303  {
304  if (!rs.is_valid()) {
305  // Invalid range-sets not written to DB
306  return;
307  }
308  // Set range sets for SubRun and Run products
309  auto it = checksumToIndexLookup.find(rs.checksum());
310  if (it != checksumToIndexLookup.cend()) {
311  product->setRangeSetID(it->second);
312  } else {
313  unsigned const rsID = getNewRangeSetID(db, BT, rs.run());
314  product->setRangeSetID(rsID);
315  checksumToIndexLookup.emplace(rs.checksum(), rsID);
316  insertIntoEventRanges(db, rs);
317  auto const& eventRangesIDs = getExistingRangeSetIDs(db, rs);
318  insertIntoJoinTable(db, BT, rsID, eventRangesIDs);
319  }
320  }
321 
322 } // unnamed namespace
323 
324 art::RootDAQOutFile::RootDAQOutFile(OutputModule* om,
325  string const& fileName,
326  ClosingCriteria const& fileSwitchCriteria,
327  int const compressionLevel,
328  unsigned freePercent,
329  unsigned freeMB,
330  int64_t const saveMemoryObjectThreshold,
331  int64_t const treeMaxVirtualSize,
332  int const splitLevel,
333  int const basketSize,
334  DropMetaData dropMetaData,
335  bool const dropMetaDataForDroppedData,
336  bool const fastCloningRequested,
337  bool const parentageEnabled,
338  bool const rangesEnabled,
339  bool const dbEnabled)
340  : om_{om}
341  , file_{fileName}
342  , fileSwitchCriteria_{fileSwitchCriteria}
343  , compressionLevel_{compressionLevel}
344  , freePercent_{freePercent}
345  , freeMB_{freeMB}
346  , saveMemoryObjectThreshold_{saveMemoryObjectThreshold}
347  , treeMaxVirtualSize_{treeMaxVirtualSize}
348  , splitLevel_{splitLevel}
349  , basketSize_{basketSize}
350  , dropMetaData_{dropMetaData}
351  , dropMetaDataForDroppedData_{dropMetaDataForDroppedData}
352  , fastCloningEnabledAtConstruction_{fastCloningRequested}
353  , filePtr_{TFile::Open(file_.c_str(), "recreate", "", compressionLevel)}
354  , treePointers_{{// Order (and number) must match BranchTypes.h!
355  std::make_unique<RootOutputTree>(
356  filePtr_.get(),
357  InEvent,
358  pEventAux_,
359  pEventProductProvenanceVector_,
360  basketSize,
361  splitLevel,
362  treeMaxVirtualSize,
363  saveMemoryObjectThreshold),
364  std::make_unique<RootOutputTree>(
365  filePtr_.get(),
366  InSubRun,
367  pSubRunAux_,
368  pSubRunProductProvenanceVector_,
369  basketSize,
370  splitLevel,
371  treeMaxVirtualSize,
372  saveMemoryObjectThreshold),
373  std::make_unique<RootOutputTree>(
374  filePtr_.get(),
375  InRun,
376  pRunAux_,
377  pRunProductProvenanceVector_,
378  basketSize,
379  splitLevel,
380  treeMaxVirtualSize,
381  saveMemoryObjectThreshold),
382  std::make_unique<RootOutputTree>(
383  filePtr_.get(),
384  InResults,
385  pResultsAux_,
386  pResultsProductProvenanceVector_,
387  basketSize,
388  splitLevel,
389  treeMaxVirtualSize,
390  saveMemoryObjectThreshold)}}
391  , parentageEnabled_{parentageEnabled}
392  , rangesEnabled_{rangesEnabled}
393  , dbEnabled_{dbEnabled}
394 {
395  if (dbEnabled_) {
396  rootFileDB_ = ServiceHandle<DatabaseConnection> {}
397  ->get<TKeyVFSOpenPolicy>(
398  "RootFileDB", filePtr_.get(), SQLITE_OPEN_CREATE | SQLITE_OPEN_READWRITE);
399  }
400  // Don't split metadata tree or event description tree
401  metaDataTree_ =
402  RootOutputTree::makeTTree(filePtr_.get(), rootNames::metaDataTreeName(), 0);
403  fileIndexTree_ = RootOutputTree::makeTTree(
404  filePtr_.get(), rootNames::fileIndexTreeName(), 0);
405  {
406  FileIndex::Element elem;
407  auto const* findexElemPtr = &elem;
408  TBranch* b = fileIndexTree_->Branch(
409  metaBranchRootName<FileIndex::Element>(), &findexElemPtr, basketSize_, 0);
410  b->ResetAddress();
411  }
412  // Create the tree that will carry (event) History objects.
413  eventHistoryTree_ = RootOutputTree::makeTTree(
414  filePtr_.get(), rootNames::eventHistoryTreeName(), splitLevel);
415  if (!eventHistoryTree_) {
416  throw art::Exception(art::errors::FatalRootError)
417  << "Failed to create the tree for History objects\n";
418  }
419  {
420  pHistory_ = new History;
421  TBranch* b = eventHistoryTree_->Branch(
422  rootNames::eventHistoryBranchName().c_str(), &pHistory_, basketSize, 0);
423  if (!b) {
424  throw art::Exception(art::errors::FatalRootError)
425  << "Failed to create a branch for History in the output file\n";
426  }
427  delete pHistory_;
428  pHistory_ = nullptr;
429  }
430  // Check that dictionaries for the auxiliaries exist
431  root::DictionaryChecker checker{};
432  checker.checkDictionaries<EventAuxiliary>();
433  checker.checkDictionaries<SubRunAuxiliary>();
434  checker.checkDictionaries<RunAuxiliary>();
435  checker.checkDictionaries<ResultsAuxiliary>();
436  checker.reportMissingDictionaries();
437  if (dbEnabled_) {
438  createDatabaseTables();
439  }
440  TLOG(3) << "RootDAQOutFile ctor complete";
441  }
442 
443 art::RootDAQOutFile::~RootDAQOutFile()
444 {
445  struct sysinfo info;
446  int sts=sysinfo(&info);
447  unsigned free_percent=(unsigned)(info.freeram*100/info.totalram);
448  unsigned free_MB=(unsigned)(info.freeram*info.mem_unit >> 20); // round down (1024.9 => 1024 MB)
449  TRACE( 3,"~RootDAQOutFile free %%%u %.1fMB (%u) buffers=%fGB mem_unit=%u"
450  ,free_percent
451  ,(float)info.freeram *info.mem_unit/(1024*1024)
452  , free_MB
453  ,(float)info.bufferram*info.mem_unit/(1024*1024*1024)
454  ,info.mem_unit );
455  if (free_percent < freePercent_ || free_MB < freeMB_ ) {
456  TLOG(3) << "RootDAQOutFile Flush/DONTNEED";
457  filePtr_->Flush();
458  sts = posix_fadvise( filePtr_->GetFd(), 0,0/*len,0=all*/, POSIX_FADV_DONTNEED );
459  }
460  TLOG(3) << "~RootDAQOutFile complete sts="<<sts;
461 }
462 
463 void
464 art::RootDAQOutFile::createDatabaseTables()
465 {
466  if (!dbEnabled_) {
467  return;
468  }
469  // Event ranges
470  create_table(rootFileDB_,
471  "EventRanges",
472  {"SubRun INTEGER",
473  "begin INTEGER",
474  "end INTEGER",
475  "UNIQUE (SubRun,begin,end) ON CONFLICT IGNORE"});
476  // SubRun range sets
477  using namespace cet::sqlite;
478  create_table(rootFileDB_, "SubRunRangeSets", column<int>{"Run"});
479  create_table(rootFileDB_,
480  "SubRunRangeSets_EventRanges",
481  {"RangeSetsID INTEGER",
482  "EventRangesID INTEGER",
483  "PRIMARY KEY(RangeSetsID,EventRangesID)"},
484  "WITHOUT ROWID");
485  // Run range sets
486  create_table(rootFileDB_, "RunRangeSets", column<int>{"Run"});
487  create_table(rootFileDB_,
488  "RunRangeSets_EventRanges",
489  {"RangeSetsID INTEGER",
490  "EventRangesID INTEGER",
491  "PRIMARY KEY(RangeSetsID,EventRangesID)"},
492  "WITHOUT ROWID");
493 }
494 
495 void
496 art::RootDAQOutFile::selectProducts()
497 {
498  for (int i = InEvent; i < NumBranchTypes; ++i) {
499  auto const bt = static_cast<BranchType>(i);
500  auto& items = selectedOutputItemList_[bt];
501  for (auto const& pd : om_->keptProducts()[bt]) {
502  // Persist Results products only if they have been produced by
503  // the current process.
504  if (bt == InResults && !pd->produced()) {
505  continue;
506  }
507  checkDictionaries(*pd);
508  // Although the transient flag is already checked when
509  // OutputModule::doSelectProducts is called, it can be flipped
510  // to 'true' after the BranchDescription transients have been
511  // fluffed, which happens during the checkDictionaries call.
512  if (pd->transient()) {
513  continue;
514  }
515  items.emplace(pd);
516  }
517  for (auto const& val : items) {
518  treePointers_[bt]->addOutputBranch(*val.branchDescription_, val.product_);
519  }
520  }
521 }
522 
523 void
524 art::RootDAQOutFile::beginInputFile(RootFileBlock const* rfb,
525  bool const fastCloneFromOutputModule)
526 {
527  // FIXME: the logic here is nasty.
528  bool shouldFastClone{fastCloningEnabledAtConstruction_ &&
529  fastCloneFromOutputModule && rfb};
530  // Create output branches, and then redo calculation to determine if
531  // fast cloning should be done.
532  selectProducts();
533  if (shouldFastClone &&
534  !treePointers_[InEvent]->checkSplitLevelAndBasketSize(rfb->tree())) {
535  mf::LogWarning("FastCloning")
536  << "Fast cloning deactivated for this input file due to "
537  << "splitting level and/or basket size.";
538  shouldFastClone = false;
539  } else if (rfb && rfb->tree() &&
540  rfb->tree()->GetCurrentFile()->GetVersion() < 60001) {
541  mf::LogWarning("FastCloning")
542  << "Fast cloning deactivated for this input file due to "
543  << "ROOT version used to write it (< 6.00/01)\n"
544  "having a different splitting policy.";
545  shouldFastClone = false;
546  }
547  if (shouldFastClone && rfb->fileFormatVersion().value_ < 10) {
548  mf::LogWarning("FastCloning")
549  << "Fast cloning deactivated for this input file due to "
550  << "reading in file that has a different ProductID schema.";
551  shouldFastClone = false;
552  }
553  if (shouldFastClone && !fastCloningEnabledAtConstruction_) {
554  mf::LogWarning("FastCloning")
555  << "Fast cloning reactivated for this input file.";
556  }
557  treePointers_[InEvent]->beginInputFile(shouldFastClone);
558  auto tree = (rfb && rfb->tree()) ? rfb->tree() : nullptr;
559  wasFastCloned_ = treePointers_[InEvent]->fastCloneTree(tree);
560 }
561 
562 void
563 art::RootDAQOutFile::incrementInputFileNumber()
564 {
565  fp_.update<Granularity::InputFile>();
566 }
567 
568 void
569 art::RootDAQOutFile::respondToCloseInputFile(FileBlock const&)
570 {
571  cet::for_all(treePointers_, [](auto const& p) { p->setEntries(); });
572 }
573 
574 bool
575 art::RootDAQOutFile::requestsToCloseFile()
576 {
577  using namespace std::chrono;
578  unsigned constexpr oneK{1024U};
579  if (fileSwitchCriteria_.fileProperties().size() !=
580  art::Defaults::size_max()) {
581  // Note: TFile::GetSize() is very, very slow, so we only call it
582  // if we actually need to.
583  fp_.updateSize(filePtr_->GetSize() / oneK);
584  }
585  fp_.updateAge(duration_cast<seconds>(steady_clock::now() - beginTime_));
586  return fileSwitchCriteria_.should_close(fp_);
587 }
588 
589 void
590 art::RootDAQOutFile::writeOneEvent_EventPart(EventPrincipal const& e)
591 {
592  TLOG(TLVL_TRACE) << "Start of RootDAQOutFile::writeOne";
593  // Auxiliary branch.
594  // Note: pEventAux_ must be set before calling fillBranches
595  // since it gets written out in that routine.
596  pEventAux_ = &e.aux();
597  // Because getting the data may cause an exception to be
598  // thrown we want to do that first before writing anything
599  // to the file about this event.
600  fillBranches<InEvent>(e, pEventProductProvenanceVector_);
601 }
602 
603 void
604 art::RootDAQOutFile::writeOneEvent_HistoryPart(EventPrincipal const& e)
605 {
606  // History branch.
607  History historyForOutput{e.history()};
608  historyForOutput.addEventSelectionEntry(om_->selectorConfig());
609  pHistory_ = &historyForOutput;
610  int sz = eventHistoryTree_->Fill();
611  pHistory_ = &e.history();
612  if (sz <= 0) {
613  throw art::Exception(art::errors::FatalRootError)
614  << "Failed to fill the History tree for event: " << e.id()
615  << "\nTTree::Fill() returned " << sz << " bytes written." << endl;
616  }
617 }
618 
619 void
620 art::RootDAQOutFile::writeOneEvent_FileIndexPart(EventPrincipal const& e)
621 {
622  // Add event to index
623  {
624  FileIndex::Element elem{e.aux().id(), fp_.eventEntryNumber()};
625  auto const* findexElemPtr = &elem;
626  fileIndexTree_->SetBranchAddress(metaBranchRootName<FileIndex::Element>(),
627  &findexElemPtr);
628  fileIndexTree_->Fill();
629  fileIndexTree_->ResetBranchAddress(
630  fileIndexTree_->GetBranch(metaBranchRootName<FileIndex::Element>()));
631  }
632 }
633 
634 void
635 art::RootDAQOutFile::writeOne(EventPrincipal const& e)
636 {
637  writeOneEvent_EventPart(e);
638  writeOneEvent_HistoryPart(e);
639  // Add the dataType to the job report if it hasn't already been done
640  if (!dataTypeReported_) {
641  string dataType{"MC"};
642  if (e.aux().isRealData()) {
643  dataType = "Data";
644  }
645  dataTypeReported_ = true;
646  }
647  writeOneEvent_FileIndexPart(e);
648  fp_.update<Granularity::Event>(status_);
649  TLOG(TLVL_TRACE) << "End of RootDAQOutFile::writeOne";
650 }
651 
652 void
653 art::RootDAQOutFile::writeSubRun_FileIndexPart(SubRunPrincipal const& sr)
654 {
655  {
656  FileIndex::Element elem{EventID::invalidEvent(sr.aux().id()),
657  fp_.subRunEntryNumber()};
658  auto const* findexElemPtr = &elem;
659  fileIndexTree_->SetBranchAddress(metaBranchRootName<FileIndex::Element>(),
660  &findexElemPtr);
661  fileIndexTree_->Fill();
662  fileIndexTree_->ResetBranchAddress(
663  fileIndexTree_->GetBranch(metaBranchRootName<FileIndex::Element>()));
664  }
665 }
666 
667 void
668 art::RootDAQOutFile::writeSubRun(SubRunPrincipal const& sr)
669 {
670  pSubRunAux_ = &sr.aux();
671  if (rangesEnabled_) {
672  pSubRunAux_->setRangeSetID(subRunRSID_);
673  }
674  fillBranches<InSubRun>(sr, pSubRunProductProvenanceVector_);
675  writeSubRun_FileIndexPart(sr);
676  fp_.update<Granularity::SubRun>(status_);
677 }
678 
679 void
680 art::RootDAQOutFile::writeRun_FileIndexPart(RunPrincipal const& r)
681 {
682  {
683  FileIndex::Element elem{EventID::invalidEvent(r.aux().id()),
684  fp_.runEntryNumber()};
685  auto const* findexElemPtr = &elem;
686  fileIndexTree_->SetBranchAddress(metaBranchRootName<FileIndex::Element>(),
687  &findexElemPtr);
688  fileIndexTree_->Fill();
689  fileIndexTree_->ResetBranchAddress(
690  fileIndexTree_->GetBranch(metaBranchRootName<FileIndex::Element>()));
691  }
692 }
693 
694 void
695 art::RootDAQOutFile::writeRun(RunPrincipal const& r)
696 {
697  pRunAux_ = &r.aux();
698  if (rangesEnabled_) {
699  pRunAux_->setRangeSetID(runRSID_);
700  }
701  fillBranches<InRun>(r, pRunProductProvenanceVector_);
702  writeRun_FileIndexPart(r);
703  fp_.update<Granularity::Run>(status_);
704 }
705 
706 void
707 art::RootDAQOutFile::writeParentageRegistry()
708 {
709  auto pid = root::getObjectRequireDict<ParentageID>();
710  ParentageID const* hash = &pid;
711  TBranch* b = parentageTree_->Branch(
712  rootNames::parentageIDBranchName().c_str(), &hash, basketSize_, 0);
713  if (!b) {
714  throw Exception(errors::FatalRootError)
715  << "Failed to create a branch for ParentageIDs in the output file";
716  }
717  b = nullptr;
718  hash = nullptr;
719  auto par = root::getObjectRequireDict<Parentage>();
720  Parentage const* desc = &par;
721  b = parentageTree_->Branch(
722  rootNames::parentageBranchName().c_str(), &desc, basketSize_, 0);
723  if (!b) {
724  throw art::Exception(art::errors::FatalRootError)
725  << "Failed to create a branch for Parentages in the output file";
726  }
727  desc = nullptr;
728  for (auto const& pr : ParentageRegistry::get()) {
729  hash = &pr.first;
730  desc = &pr.second;
731  parentageTree_->Fill();
732  }
733  parentageTree_->SetBranchAddress(rootNames::parentageIDBranchName().c_str(),
734  reinterpret_cast<void*>(-1L));
735  parentageTree_->SetBranchAddress(rootNames::parentageBranchName().c_str(),
736  reinterpret_cast<void*>(-1L));
737 }
738 
739 void
740 art::RootDAQOutFile::writeFileFormatVersion()
741 {
742  FileFormatVersion const ver{getFileFormatVersion(), getFileFormatEra()};
743  auto const* pver = &ver;
744  TBranch* b = metaDataTree_->Branch(
745  metaBranchRootName<FileFormatVersion>(), &pver, basketSize_, 0);
746  assert(b);
747  b->Fill();
748 }
749 
750 void
751 art::RootDAQOutFile::writeFileIndex()
752 {
753  // Read file index.
754  auto findexPtr = &fileIndex_;
755  detail::readFileIndex(filePtr_.get(), fileIndexTree_, findexPtr);
756  // Sort it.
757  fileIndex_.sortBy_Run_SubRun_Event();
758  // Make replacement tree.
759  fileIndexTree_ = RootOutputTree::makeTTree(
760  filePtr_.get(), rootNames::fileIndexTreeName(), 0);
761  // Make FileIndex branch.
762  FileIndex::Element elem;
763  auto findexElemPtr = &elem;
764  auto b = fileIndexTree_->Branch(
765  metaBranchRootName<FileIndex::Element>(), &findexElemPtr, basketSize_, 0);
766  // Now write out the new sorted FileIndex.
767  for (auto& entry : fileIndex_) {
768  findexElemPtr = &entry;
769  fileIndexTree_->Fill();
770  }
771  b->ResetAddress();
772 }
773 
774 void
775 art::RootDAQOutFile::writeEventHistory()
776 {
777  RootOutputTree::writeTTree(eventHistoryTree_);
778 }
779 
780 void
781 art::RootDAQOutFile::writeProcessConfigurationRegistry()
782 {
783  // We don't do this yet; currently we're storing a slightly
784  // bloated ProcessHistoryRegistry.
785 }
786 
787 void
788 art::RootDAQOutFile::writeProcessHistoryRegistry()
789 {
790  ProcessHistoryMap pHistMap;
791  for (auto const& pr : ProcessHistoryRegistry::get()) {
792  pHistMap.emplace(pr);
793  }
794  auto const* p = &pHistMap;
795  TBranch* b = metaDataTree_->Branch(
796  metaBranchRootName<ProcessHistoryMap>(), &p, basketSize_, 0);
797  if (!b) {
798  throw Exception(errors::LogicError)
799  << "Unable to locate required ProcessHistoryMap branch in output "
800  "metadata tree.\n";
801  }
802  b->Fill();
803 }
804 
805 void
806 art::RootDAQOutFile::writeFileCatalogMetadata(
807  FileStatsCollector const& stats,
808  FileCatalogMetadata::collection_type const& md,
809  FileCatalogMetadata::collection_type const& ssmd)
810 {
811  if (!dbEnabled_) {
812  return;
813  }
814  using namespace cet::sqlite;
815  Ntuple<std::string, std::string> fileCatalogMetadata{
816  rootFileDB_, "FileCatalog_metadata", {{"Name", "Value"}}, true};
817  Transaction txn{rootFileDB_};
818  for (auto const& kv : md) {
819  fileCatalogMetadata.insert(kv.first, kv.second);
820  }
821  // Add our own specific information: File format and friends.
822  fileCatalogMetadata.insert("file_format", "\"artroot\"");
823  fileCatalogMetadata.insert("file_format_era",
824  cet::canonical_string(getFileFormatEra()));
825  fileCatalogMetadata.insert("file_format_version",
826  to_string(getFileFormatVersion()));
827  // File start time.
828  namespace bpt = boost::posix_time;
829  auto formatted_time = [](auto const& t) {
830  return cet::canonical_string(bpt::to_iso_extended_string(t));
831  };
832  fileCatalogMetadata.insert("start_time",
833  formatted_time(stats.outputFileOpenTime()));
834  // File "end" time: now, since file is not actually closed yet.
835  fileCatalogMetadata.insert(
836  "end_time",
837  formatted_time(boost::posix_time::second_clock::universal_time()));
838  // Run/subRun information.
839  if (!stats.seenSubRuns().empty()) {
840  auto I = find_if(md.crbegin(), md.crend(), [](auto const& p) {
841  return p.first == "run_type";
842  });
843  if (I != md.crend()) {
844  ostringstream buf;
845  buf << "[ ";
846  for (auto const& srid : stats.seenSubRuns()) {
847  buf << "[ " << srid.run() << ", " << srid.subRun() << ", "
848  << cet::canonical_string(I->second) << " ], ";
849  }
850  // Rewind over last delimiter.
851  buf.seekp(-2, ios_base::cur);
852  buf << " ]";
853  fileCatalogMetadata.insert("runs", buf.str());
854  }
855  }
856  // Number of events.
857  fileCatalogMetadata.insert("event_count", to_string(stats.eventsThisFile()));
858  // first_event and last_event.
859  auto eidToTuple = [](EventID const& eid) -> string {
860  ostringstream eidStr;
861  eidStr << "[ " << eid.run() << ", " << eid.subRun() << ", " << eid.event()
862  << " ]";
863  return eidStr.str();
864  };
865  fileCatalogMetadata.insert("first_event", eidToTuple(stats.lowestEventID()));
866  fileCatalogMetadata.insert("last_event", eidToTuple(stats.highestEventID()));
867  // File parents.
868  if (!stats.parents().empty()) {
869  ostringstream pstring;
870  pstring << "[ ";
871  for (auto const& parent : stats.parents()) {
872  pstring << cet::canonical_string(parent) << ", ";
873  }
874  // Rewind over last delimiter.
875  pstring.seekp(-2, ios_base::cur);
876  pstring << " ]";
877  fileCatalogMetadata.insert("parents", pstring.str());
878  }
879  // Incoming stream-specific metadata overrides.
880  for (auto const& kv : ssmd) {
881  fileCatalogMetadata.insert(kv.first, kv.second);
882  }
883  txn.commit();
884 }
885 
886 void
887 art::RootDAQOutFile::writeParameterSetRegistry()
888 {
889  if (!dbEnabled_) {
890  return;
891  }
892  fhicl::ParameterSetRegistry::exportTo(rootFileDB_);
893 }
894 
895 void
896 art::RootDAQOutFile::writeProductDescriptionRegistry()
897 {
898  // Make a local copy of the MasterProductRegistry's ProductList,
899  // removing any transient or pruned products.
900  auto end = branchesWithStoredHistory_.end();
901  ProductRegistry reg;
902  for (auto const& pr : ProductMetaData::instance().productList()) {
903  if (branchesWithStoredHistory_.find(pr.second.productID()) == end) {
904  continue;
905  }
906  reg.productList_.emplace_hint(reg.productList_.end(), pr);
907  }
908  ProductRegistry const* regp = &reg;
909  TBranch* b = metaDataTree_->Branch(
910  metaBranchRootName<ProductRegistry>(), &regp, basketSize_, 0);
911  b->Fill();
912 }
913 
914 void
915 art::RootDAQOutFile::writeProductDependencies()
916 {
917  BranchChildren const* ppDeps = &om_->branchChildren();
918  TBranch* b = metaDataTree_->Branch(
919  metaBranchRootName<BranchChildren>(), &ppDeps, basketSize_, 0);
920  b->Fill();
921 }
922 
923 void
924 art::RootDAQOutFile::writeResults(ResultsPrincipal& resp)
925 {
926  pResultsAux_ = &resp.aux();
927  fillBranches<InResults>(resp, pResultsProductProvenanceVector_);
928 }
929 
930 void
931 art::RootDAQOutFile::writeTTrees()
932 {
933  TLOG(TLVL_TRACE) << "Start of RootDAQOutFile::writeTTrees";
934  RootOutputTree::writeTTree(metaDataTree_);
935  TLOG(TLVL_TRACE) << "RootDAQOutFile::writeTTrees after writing metaDataTree_";
936  RootOutputTree::writeTTree(fileIndexTree_);
937  TLOG(TLVL_TRACE) << "RootDAQOutFile::writeTTrees after writing fileIndexTree_";
938  RootOutputTree::writeTTree(parentageTree_);
939  TLOG(TLVL_TRACE) << "RootDAQOutFile::writeTTrees after writing parentageTree_";
940  // Write out the tree corresponding to each BranchType
941  for (int i = InEvent; i < NumBranchTypes; ++i) {
942  auto const branchType = static_cast<BranchType>(i);
943  treePointers_[branchType]->writeTree();
944  }
945  TLOG(TLVL_TRACE) << "End of RootDAQOutFile::writeTTrees";
946 }
947 
948 void
949 art::RootDAQOutFile::insertParents(std::set<ProductProvenance>& keptProvenance,
950  Principal const& principal,
951  vector<ProductID> const& parents)
952 {
953  for (auto const pid : parents) {
954  auto pp = principal.branchMapper().branchToProductProvenance(pid);
955  if (!pp) {
956  continue;
957  }
958  branchesWithStoredHistory_.insert(pid);
959  keptProvenance.insert(*pp);
960  if ((dropMetaData_ == DropMetaData::DropPrior) ||
961  (dropMetaData_ == DropMetaData::DropAll) ||
962  dropMetaDataForDroppedData_) {
963  continue;
964  }
965  auto const* pd = principal.getForOutput(pp->productID(), false).desc();
966  if (pd && pd->produced()) {
967  insertParents(keptProvenance, principal, pp->parentage().parents());
968  }
969  }
970 }
971 
972 template <art::BranchType BT>
973 void
974 art::RootDAQOutFile::fillBranches(Principal const& principal,
975  vector<ProductProvenance>* vpp)
976 {
977 TLOG(TLVL_TRACE) << "Start of RootDAQOutFile::fillBranches";
978  set<ProductProvenance> keptProvenance;
979  map<unsigned, unsigned> checksumToIndex;
980  for (auto const& val : selectedOutputItemList_[BT]) {
981  auto const* pd = val.branchDescription_;
982  branchesWithStoredHistory_.insert(pd->productID());
983  bool const readProductNow =
984  (pd->produced() || ((BT != InEvent) || !wasFastCloned_) ||
985  treePointers_[BT]->uncloned(pd->branchName()));
986  auto const& oh = principal.getForOutput(pd->productID(), readProductNow);
987  unique_ptr<ProductProvenance> prov;
988  if ((dropMetaData_ == DropMetaData::DropNone) ||
989  (pd->produced() && (dropMetaData_ == DropMetaData::DropPrior))) {
990  // We need provenance for the output file.
991  if (oh.productProvenance()) {
992  prov = std::make_unique<ProductProvenance>(
993  *keptProvenance.insert(*oh.productProvenance()).first);
994  if (parentageEnabled_ && (dropMetaData_ == DropMetaData::DropNone) &&
995  !dropMetaDataForDroppedData_) {
996  // Take extra effort to keep the provenance of our parents.
997  insertParents(keptProvenance,
998  principal,
999  oh.productProvenance()->parentage().parents());
1000  }
1001  } else {
1002  // No provenance, product was either not produced, or was
1003  // dropped, create provenance to remember that.
1004  prov = std::make_unique<ProductProvenance>(
1005  *keptProvenance
1006  .emplace(pd->productID(),
1007  pd->produced() ? productstatus::neverCreated() :
1008  productstatus::dropped())
1009  .first);
1010  }
1011  }
1012  if (readProductNow) {
1013  art::EDProduct const* product{nullptr};
1014  if (rangesEnabled_) {
1015  auto const& rs =
1016  getRangeSet<BT>(oh, principal.seenRanges(), pd->produced());
1017  if (RangeSetsSupported<BT>::value && !rs.is_valid()) {
1018  // Unfortunately, 'unknown' is the only viable product status
1019  // when this condition is triggered (due to the assert
1020  // statement in ProductStatus::setNotPresent). Whenever the
1021  // metadata revolution comes, this should be revised.
1022  {
1023  if (keptProvenance.erase(*prov) != 1ULL) {
1024  throw Exception(errors::LogicError,
1025  "art::RootDAQOutFile::fillBranches")
1026  << "Attempt to set product status for product whose provenance "
1027  "is not being recorded.";
1028  }
1029  keptProvenance.emplace(prov->productID(), productstatus::unknown());
1030  }
1031  }
1032  product = getProduct<BT>(oh, rs, pd->wrappedName());
1033  if (dbEnabled_) {
1034  setProductRangeSetID<BT>(
1035  rs, rootFileDB_, const_cast<EDProduct*>(product), checksumToIndex);
1036  }
1037  } else {
1038  product =
1039  getProduct<BT>(oh, art::RangeSet::invalid(), pd->wrappedName());
1040  }
1041  val.product_ = product;
1042  }
1043  }
1044  vpp->assign(keptProvenance.begin(), keptProvenance.end());
1045  TLOG(TLVL_TRACE) << "RootDAQOutFile::fillBranches before fillTree call";
1046  treePointers_[BT]->fillTree();
1047  TLOG(TLVL_TRACE) << "RootDAQOutFile::fillBranches after fillTree call";
1048  vpp->clear();
1049  vpp->shrink_to_fit();
1050  TLOG(TLVL_TRACE) << "End of RootDAQOutFile::fillBranches";
1051 }
1052 
1053 void
1054 art::RootDAQOutFile::setSubRunAuxiliaryRangeSetID(RangeSet const& ranges)
1055 {
1056  if (!rangesEnabled_) {
1057  return;
1058  }
1059  if (!dbEnabled_) {
1060  return;
1061  }
1062  subRunRSID_ = getNewRangeSetID(rootFileDB_, InSubRun, ranges.run());
1063  insertIntoEventRanges(rootFileDB_, ranges);
1064  auto const& eventRangesIDs = getExistingRangeSetIDs(rootFileDB_, ranges);
1065  insertIntoJoinTable(rootFileDB_, InSubRun, subRunRSID_, eventRangesIDs);
1066 }
1067 
1068 void
1069 art::RootDAQOutFile::setRunAuxiliaryRangeSetID(RangeSet const& ranges)
1070 {
1071  if (!rangesEnabled_) {
1072  return;
1073  }
1074  if (!dbEnabled_) {
1075  return;
1076  }
1077  runRSID_ = getNewRangeSetID(rootFileDB_, InRun, ranges.run());
1078  insertIntoEventRanges(rootFileDB_, ranges);
1079  auto const& eventRangesIDs = getExistingRangeSetIDs(rootFileDB_, ranges);
1080  insertIntoJoinTable(rootFileDB_, InRun, runRSID_, eventRangesIDs);
1081 }
1082 
1083 template <BranchType BT>
1084 std::enable_if_t<!RangeSetsSupported<BT>::value, art::EDProduct const*>
1085 art::RootDAQOutFile::getProduct(art::OutputHandle const& oh,
1086  art::RangeSet const& /*prunedProductRS*/,
1087  std::string const& wrappedName)
1088 {
1089  if (oh.isValid()) {
1090  return oh.wrapper();
1091  }
1092  return dummyProductCache_.product(wrappedName);
1093 }
1094 
1095 template <BranchType BT>
1096 std::enable_if_t<RangeSetsSupported<BT>::value, art::EDProduct const*>
1097 art::RootDAQOutFile::getProduct(art::OutputHandle const& oh,
1098  art::RangeSet const& prunedProductRS,
1099  std::string const& wrappedName)
1100 {
1101  if (rangesEnabled_) {
1102  if (oh.isValid() && prunedProductRS.is_valid()) {
1103  return oh.wrapper();
1104  }
1105  return dummyProductCache_.product(wrappedName);
1106  }
1107  if (oh.isValid()) {
1108  return oh.wrapper();
1109  }
1110  return dummyProductCache_.product(wrappedName);
1111 }