artdaq  v3_06_02
RootDAQOutFile.cc
1 #include "artdaq/ArtModules/RootDAQOutput-s85/RootDAQOutFile.h"
2 // vim: set sw=2:
3 
4 #include "art/Framework/IO/FileStatsCollector.h"
5 #include "art/Framework/IO/Root/DropMetaData.h"
6 #include "art/Framework/IO/Root/GetFileFormatEra.h"
7 #include "art/Framework/IO/Root/GetFileFormatVersion.h"
8 #include "art/Framework/IO/Root/RootDB/SQLErrMsg.h"
9 #include "art/Framework/IO/Root/RootDB/TKeyVFSOpenPolicy.h"
10 #include "art/Framework/IO/Root/RootFileBlock.h"
11 #include "art/Framework/IO/Root/checkDictionaries.h"
12 #include "art/Framework/IO/Root/detail/getObjectRequireDict.h"
13 #include "art/Framework/IO/Root/detail/readFileIndex.h"
14 #include "art/Framework/Principal/EventPrincipal.h"
15 #include "art/Framework/Principal/ResultsPrincipal.h"
16 #include "art/Framework/Principal/RunPrincipal.h"
17 #include "art/Framework/Principal/SubRunPrincipal.h"
18 #include "art/Framework/Services/System/DatabaseConnection.h"
19 #include "art/Persistency/Provenance/ProcessHistoryRegistry.h"
20 #include "art/Persistency/Provenance/ProductMetaData.h"
21 #include "art/Version/GetReleaseVersion.h"
22 #include "boost/date_time/posix_time/posix_time.hpp"
23 #include "canvas/Persistency/Provenance/BranchChildren.h"
24 #include "canvas/Persistency/Provenance/BranchType.h"
25 #include "canvas/Persistency/Provenance/EventAuxiliary.h"
26 #include "canvas/Persistency/Provenance/EventID.h"
27 #include "canvas/Persistency/Provenance/FileFormatVersion.h"
28 #include "canvas/Persistency/Provenance/History.h"
29 #include "canvas/Persistency/Provenance/ParameterSetBlob.h"
30 #include "canvas/Persistency/Provenance/Parentage.h"
31 #include "canvas/Persistency/Provenance/ParentageRegistry.h"
32 #include "canvas/Persistency/Provenance/ProcessHistoryID.h"
33 #include "canvas/Persistency/Provenance/ProductID.h"
34 #include "canvas/Persistency/Provenance/ProductProvenance.h"
35 #include "canvas/Persistency/Provenance/ProductStatus.h"
36 #include "canvas/Persistency/Provenance/ResultsAuxiliary.h"
37 #include "canvas/Persistency/Provenance/RunAuxiliary.h"
38 #include "canvas/Persistency/Provenance/SubRunAuxiliary.h"
39 #include "canvas/Persistency/Provenance/rootNames.h"
40 #include "canvas/Utilities/Exception.h"
41 #include "canvas_root_io/Utilities/DictionaryChecker.h"
42 #include "cetlib/canonical_string.h"
43 #include "cetlib/container_algorithms.h"
44 #include "cetlib/exempt_ptr.h"
45 #include "cetlib/sqlite/Ntuple.h"
46 #include "cetlib/sqlite/Transaction.h"
47 #include "cetlib/sqlite/create_table.h"
48 #include "cetlib/sqlite/exec.h"
49 #include "cetlib/sqlite/insert.h"
50 #include "fhiclcpp/ParameterSet.h"
51 #include "fhiclcpp/ParameterSetID.h"
52 #include "fhiclcpp/ParameterSetRegistry.h"
53 
54 #include "TFile.h"
55 #include "TTree.h"
56 #include "artdaq/DAQdata/Globals.hh"
57 #include "tracemf.h" // TLOG
58 #define TRACE_NAME (app_name + "_RootDAQOutFile").c_str()
59 
60 #include <fcntl.h> // posix_fadvise POSIX_FADV_DONTNEED
61 #include <sys/sysinfo.h> // sysinfo(sysinfo*)
62 #include <algorithm>
63 #include <map>
64 #include <set>
65 #include <utility>
66 #include <vector>
67 
68 using namespace cet;
69 using namespace std;
70 using art::BranchType;
72 using art::rootNames::metaBranchRootName;
73 
74 namespace {
75 
76 void create_table(sqlite3* const db,
77  std::string const& name,
78  std::vector<std::string> const& columns,
79  std::string const& suffix = {})
80 {
81  if (columns.empty())
82  throw art::Exception(art::errors::LogicError)
83  << "Number of sqlite columns specified for table: " << name << '\n'
84  << "is zero.\n";
85 
86  std::string ddl = "DROP TABLE IF EXISTS " + name +
87  "; "
88  "CREATE TABLE " +
89  name + "(" + columns.front();
90  std::for_each(columns.begin() + 1, columns.end(), [&ddl](auto const& col) {
91  ddl += "," + col;
92  });
93  ddl += ") ";
94  ddl += suffix;
95  ddl += ";";
96  sqlite::exec(db, ddl);
97 }
98 
99 void insert_eventRanges_row(sqlite3_stmt* stmt,
100  art::SubRunNumber_t const sr,
101  art::EventNumber_t const b,
102  art::EventNumber_t const e)
103 {
104  sqlite3_bind_int64(stmt, 1, sr);
105  sqlite3_bind_int64(stmt, 2, b);
106  sqlite3_bind_int64(stmt, 3, e);
107  sqlite3_step(stmt);
108  sqlite3_reset(stmt);
109 }
110 
111 void insert_rangeSets_eventSets_row(sqlite3_stmt* stmt,
112  unsigned const rsid,
113  unsigned const esid)
114 {
115  sqlite3_bind_int64(stmt, 1, rsid);
116  sqlite3_bind_int64(stmt, 2, esid);
117  sqlite3_step(stmt);
118  sqlite3_reset(stmt);
119 }
120 
121 unsigned
122 getNewRangeSetID(sqlite3* db,
123  art::BranchType const bt,
124  art::RunNumber_t const r)
125 {
126  sqlite::insert_into(db, art::BranchTypeToString(bt) + "RangeSets")
127  .values(r);
128  return sqlite3_last_insert_rowid(db);
129 }
130 
131 vector<unsigned>
132 getExistingRangeSetIDs(sqlite3* db, art::RangeSet const& rs)
133 {
134  vector<unsigned> rangeSetIDs;
135  cet::transform_all(
136  rs, std::back_inserter(rangeSetIDs), [db](auto const& range) {
137  sqlite::query_result<unsigned> r;
138  r << sqlite::select("ROWID")
139  .from(db, "EventRanges")
140  .where("SubRun=" + std::to_string(range.subRun()) +
141  " AND "
142  "begin=" +
143  std::to_string(range.begin()) +
144  " AND "
145  "end=" +
146  std::to_string(range.end()));
147  return unique_value(r);
148  });
149  return rangeSetIDs;
150 }
151 
152 void insertIntoEventRanges(sqlite3* db, art::RangeSet const& rs)
153 {
154  sqlite::Transaction txn{db};
155  sqlite3_stmt* stmt{nullptr};
156  std::string const ddl{
157  "INSERT INTO EventRanges(SubRun, begin, end) "
158  "VALUES(?, ?, ?);"};
159  sqlite3_prepare_v2(db, ddl.c_str(), -1, &stmt, nullptr);
160  for (auto const& range : rs)
161  {
162  insert_eventRanges_row(stmt, range.subRun(), range.begin(), range.end());
163  }
164  sqlite3_finalize(stmt);
165  txn.commit();
166 }
167 
168 void insertIntoJoinTable(sqlite3* db,
169  art::BranchType const bt,
170  unsigned const rsID,
171  vector<unsigned> const& eventRangesIDs)
172 {
173  sqlite::Transaction txn{db};
174  sqlite3_stmt* stmt{nullptr};
175  std::string const ddl{
176  "INSERT INTO " + art::BranchTypeToString(bt) +
177  "RangeSets_EventRanges(RangeSetsID, EventRangesID) Values(?,?);"};
178  sqlite3_prepare_v2(db, ddl.c_str(), -1, &stmt, nullptr);
179  cet::for_all(eventRangesIDs, [stmt, rsID](auto const eventRangeID) {
180  insert_rangeSets_eventSets_row(stmt, rsID, eventRangeID);
181  });
182  sqlite3_finalize(stmt);
183  txn.commit();
184 }
185 
186 void maybeInvalidateRangeSet(BranchType const bt,
187  art::RangeSet const& principalRS,
188  art::RangeSet& productRS)
189 {
190  assert(principalRS.is_sorted());
191  assert(productRS.is_sorted());
192 
193  if (!productRS.is_valid())
194  {
195  return;
196  }
197  if (bt == art::InRun && productRS.is_full_run())
198  {
199  return;
200  }
201  if (bt == art::InSubRun && productRS.is_full_subRun())
202  {
203  return;
204  }
205  if (productRS.ranges().empty())
206  {
207  return;
208  }
209  auto const r = productRS.run();
210  auto const& productFront = productRS.ranges().front();
211  if (!principalRS.contains(r, productFront.subRun(), productFront.begin()))
212  {
213  productRS = art::RangeSet::invalid();
214  }
215 }
216 
217 using art::detail::RangeSetsSupported;
218 
219 // The purpose of 'maybeInvalidateRangeSet' is to support the
220 // following situation. Suppose process 1 creates three files with
221 // one Run product each, all corresponding to the same Run. Let's
222 // call the individual Run product instances in the three separate
223 // files as A, B, and C. Now suppose that the three files serve as
224 // inputs to process 2, where a concatenation is being performed AND
225 // ALSO an output file switch. Process 2 results in two output
226 // files, and now, in process 3, we concatenate the outputs from
227 // process 2. The situation would look like this:
228 //
229 // Process 1: [A] [B] [C]
230 // \ / \ /
231 // Process 2: [A + B] [B + C]
232 // \ / \ /
233 // D=agg(A,B) | | E=agg(B,C)
234 // \ /
235 // Process 3: [D + E]
236 //
237 // Notice the complication in process 3: product 'B' will be
238 // aggregated twice: once with A, and once with C. Whenever the
239 // output from process 3 is read as input to another process, the
240 // fetched product will be equivalent to A+2B+C.
241 //
242 // To avoid this situation, we compare the RangeSet of the product
243 // with the RangeSet of the in-memory RunAuxiliary. If the
244 // beginning of B's RangeSet is not contained within the auxiliary's
245 // RangeSet, then a dummy product with an invalid RangeSet is
246 // written to disk. Instead of the diagram above, we have:
247 //
248 // Process 1: [A] [B] [C]
249 // \ / \ /
250 // Process 2: [A + B] [x + C]
251 // \ / \ /
252 // D=agg(A,B) | | E=agg(x,C)=C
253 // \ /
254 // Process 3: [D + E]
255 //
256 // where 'x' represent a dummy product. Upon aggregating D and E,
257 // we obtain the correctly formed A+B+C product.
258 template<BranchType BT>
259 std::enable_if_t<RangeSetsSupported<BT>::value, art::RangeSet>
260 getRangeSet(art::OutputHandle const& oh,
261  art::RangeSet const& principalRS,
262  bool const producedInThisProcess)
263 {
264  auto rs = oh.isValid() ? oh.rangeOfValidity() : art::RangeSet::invalid();
265  // Because a user can specify (e.g.):
266  // r.put(std::move(myProd), art::runFragment(myRangeSet));
267  // products that are produced in this process can have valid, yet
268  // arbitrary RangeSets. We therefore never invalidate a RangeSet
269  // that corresponds to a product produced in this process.
270  //
271  // It is possible for a user to specify a RangeSet which does not
272  // correspond AT ALL to the in-memory auxiliary RangeSet. In that
273  // case, users should not expect to be able to retrieve products
274  // for which no corresponding events or sub-runs were processed.
275  if (!producedInThisProcess)
276  {
277  maybeInvalidateRangeSet(BT, principalRS, rs);
278  }
279  return rs;
280 }
281 
282 template<BranchType BT>
283 std::enable_if_t<!RangeSetsSupported<BT>::value, art::RangeSet>
284 getRangeSet(art::OutputHandle const&,
285  art::RangeSet const& /*principalRS*/,
286  bool const /*producedInThisProcess*/)
287 {
288  return art::RangeSet::invalid();
289 }
290 
291 template<BranchType BT>
292 std::enable_if_t<!RangeSetsSupported<BT>::value>
293 setProductRangeSetID(art::RangeSet const& /*rs*/,
294  sqlite3*,
295  art::EDProduct*,
296  std::map<unsigned, unsigned>& /*checksumToIndexLookup*/)
297 {}
298 
299 template<BranchType BT>
300 std::enable_if_t<RangeSetsSupported<BT>::value>
301 setProductRangeSetID(art::RangeSet const& rs,
302  sqlite3* db,
303  art::EDProduct* product,
304  std::map<unsigned, unsigned>& checksumToIndexLookup)
305 {
306  if (!rs.is_valid())
307  {
308  // Invalid range-sets not written to DB
309  return;
310  }
311  // Set range sets for SubRun and Run products
312  auto it = checksumToIndexLookup.find(rs.checksum());
313  if (it != checksumToIndexLookup.cend())
314  {
315  product->setRangeSetID(it->second);
316  }
317  else
318  {
319  unsigned const rsID = getNewRangeSetID(db, BT, rs.run());
320  product->setRangeSetID(rsID);
321  checksumToIndexLookup.emplace(rs.checksum(), rsID);
322  insertIntoEventRanges(db, rs);
323  auto const& eventRangesIDs = getExistingRangeSetIDs(db, rs);
324  insertIntoJoinTable(db, BT, rsID, eventRangesIDs);
325  }
326 }
327 
328 } // unnamed namespace
329 
330 art::RootDAQOutFile::RootDAQOutFile(OutputModule* om,
331  string const& fileName,
332  ClosingCriteria const& fileSwitchCriteria,
333  int const compressionLevel,
334  unsigned freePercent,
335  unsigned freeMB,
336  int64_t const saveMemoryObjectThreshold,
337  int64_t const treeMaxVirtualSize,
338  int const splitLevel,
339  int const basketSize,
340  DropMetaData dropMetaData,
341  bool const dropMetaDataForDroppedData,
342  bool const fastCloningRequested,
343  bool const parentageEnabled,
344  bool const rangesEnabled,
345  bool const dbEnabled)
346  : om_{om}
347  , file_{fileName}
348  , fileSwitchCriteria_{fileSwitchCriteria}
349  , compressionLevel_{compressionLevel}
350  , freePercent_{freePercent}
351  , freeMB_{freeMB}
352  , saveMemoryObjectThreshold_{saveMemoryObjectThreshold}
353  , treeMaxVirtualSize_{treeMaxVirtualSize}
354  , splitLevel_{splitLevel}
355  , basketSize_{basketSize}
356  , dropMetaData_{dropMetaData}
357  , dropMetaDataForDroppedData_{dropMetaDataForDroppedData}
358  , fastCloningEnabledAtConstruction_{fastCloningRequested}
359  , filePtr_{TFile::Open(file_.c_str(), "recreate", "", compressionLevel)}
360  , treePointers_{{// Order (and number) must match BranchTypes.h!
361  std::make_unique<RootOutputTree>(
362  filePtr_.get(),
363  InEvent,
364  pEventAux_,
365  pEventProductProvenanceVector_,
366  basketSize,
367  splitLevel,
368  treeMaxVirtualSize,
369  saveMemoryObjectThreshold),
370  std::make_unique<RootOutputTree>(
371  filePtr_.get(),
372  InSubRun,
373  pSubRunAux_,
374  pSubRunProductProvenanceVector_,
375  basketSize,
376  splitLevel,
377  treeMaxVirtualSize,
378  saveMemoryObjectThreshold),
379  std::make_unique<RootOutputTree>(
380  filePtr_.get(),
381  InRun,
382  pRunAux_,
383  pRunProductProvenanceVector_,
384  basketSize,
385  splitLevel,
386  treeMaxVirtualSize,
387  saveMemoryObjectThreshold),
388  std::make_unique<RootOutputTree>(
389  filePtr_.get(),
390  InResults,
391  pResultsAux_,
392  pResultsProductProvenanceVector_,
393  basketSize,
394  splitLevel,
395  treeMaxVirtualSize,
396  saveMemoryObjectThreshold)}}
397  , parentageEnabled_{parentageEnabled}
398  , rangesEnabled_{rangesEnabled}
399  , dbEnabled_{dbEnabled}
400 {
401  if (dbEnabled_)
402  {
403  rootFileDB_ = ServiceHandle<DatabaseConnection> {}
404  ->get<TKeyVFSOpenPolicy>(
405  "RootFileDB", filePtr_.get(), SQLITE_OPEN_CREATE | SQLITE_OPEN_READWRITE);
406  }
407  // Don't split metadata tree or event description tree
408  metaDataTree_ =
409  RootOutputTree::makeTTree(filePtr_.get(), rootNames::metaDataTreeName(), 0);
410  fileIndexTree_ = RootOutputTree::makeTTree(
411  filePtr_.get(), rootNames::fileIndexTreeName(), 0);
412  {
413  FileIndex::Element elem;
414  auto const* findexElemPtr = &elem;
415  TBranch* b = fileIndexTree_->Branch(
416  metaBranchRootName<FileIndex::Element>(), &findexElemPtr, basketSize_, 0);
417  b->ResetAddress();
418  }
419  // Create the tree that will carry (event) History objects.
420  eventHistoryTree_ = RootOutputTree::makeTTree(
421  filePtr_.get(), rootNames::eventHistoryTreeName(), splitLevel);
422  if (!eventHistoryTree_)
423  {
424  throw art::Exception(art::errors::FatalRootError)
425  << "Failed to create the tree for History objects\n";
426  }
427  {
428  pHistory_ = new History;
429  TBranch* b = eventHistoryTree_->Branch(
430  rootNames::eventHistoryBranchName().c_str(), &pHistory_, basketSize, 0);
431  if (!b)
432  {
433  throw art::Exception(art::errors::FatalRootError)
434  << "Failed to create a branch for History in the output file\n";
435  }
436  delete pHistory_;
437  pHistory_ = nullptr;
438  }
439  // Check that dictionaries for the auxiliaries exist
440  root::DictionaryChecker checker{};
441  checker.checkDictionaries<EventAuxiliary>();
442  checker.checkDictionaries<SubRunAuxiliary>();
443  checker.checkDictionaries<RunAuxiliary>();
444  checker.checkDictionaries<ResultsAuxiliary>();
445  checker.reportMissingDictionaries();
446  if (dbEnabled_)
447  {
448  createDatabaseTables();
449  }
450  TLOG(3) << "RootDAQOutFile ctor complete";
451 }
452 
453 art::RootDAQOutFile::~RootDAQOutFile()
454 {
455  struct sysinfo info;
456  int sts = sysinfo(&info);
457  unsigned free_percent = (unsigned)(info.freeram * 100 / info.totalram);
458  unsigned free_MB = (unsigned)(info.freeram * info.mem_unit >> 20); // round down (1024.9 => 1024 MB)
459  TRACE(3, "~RootDAQOutFile free %%%u %.1fMB (%u) buffers=%fGB mem_unit=%u", free_percent, (float)info.freeram * info.mem_unit / (1024 * 1024), free_MB, (float)info.bufferram * info.mem_unit / (1024 * 1024 * 1024), info.mem_unit);
460  if (free_percent < freePercent_ || free_MB < freeMB_)
461  {
462  TLOG(3) << "RootDAQOutFile Flush/DONTNEED";
463  filePtr_->Flush();
464  sts = posix_fadvise(filePtr_->GetFd(), 0, 0 /*len,0=all*/, POSIX_FADV_DONTNEED);
465  }
466  TLOG(3) << "~RootDAQOutFile complete sts=" << sts;
467 }
468 
469 void art::RootDAQOutFile::createDatabaseTables()
470 {
471  if (!dbEnabled_)
472  {
473  return;
474  }
475  // Event ranges
476  create_table(rootFileDB_,
477  "EventRanges",
478  {"SubRun INTEGER",
479  "begin INTEGER",
480  "end INTEGER",
481  "UNIQUE (SubRun,begin,end) ON CONFLICT IGNORE"});
482  // SubRun range sets
483  using namespace cet::sqlite;
484  create_table(rootFileDB_, "SubRunRangeSets", column<int>{"Run"});
485  create_table(rootFileDB_,
486  "SubRunRangeSets_EventRanges",
487  {"RangeSetsID INTEGER",
488  "EventRangesID INTEGER",
489  "PRIMARY KEY(RangeSetsID,EventRangesID)"},
490  "WITHOUT ROWID");
491  // Run range sets
492  create_table(rootFileDB_, "RunRangeSets", column<int>{"Run"});
493  create_table(rootFileDB_,
494  "RunRangeSets_EventRanges",
495  {"RangeSetsID INTEGER",
496  "EventRangesID INTEGER",
497  "PRIMARY KEY(RangeSetsID,EventRangesID)"},
498  "WITHOUT ROWID");
499 }
500 
501 void art::RootDAQOutFile::selectProducts()
502 {
503  for (int i = InEvent; i < NumBranchTypes; ++i)
504  {
505  auto const bt = static_cast<BranchType>(i);
506  auto& items = selectedOutputItemList_[bt];
507  for (auto const& pd : om_->keptProducts()[bt])
508  {
509  // Persist Results products only if they have been produced by
510  // the current process.
511  if (bt == InResults && !pd->produced())
512  {
513  continue;
514  }
515  checkDictionaries(*pd);
516  // Although the transient flag is already checked when
517  // OutputModule::doSelectProducts is called, it can be flipped
518  // to 'true' after the BranchDescription transients have been
519  // fluffed, which happens during the checkDictionaries call.
520  if (pd->transient())
521  {
522  continue;
523  }
524  items.emplace(pd);
525  }
526  for (auto const& val : items)
527  {
528  treePointers_[bt]->addOutputBranch(*val.branchDescription_, val.product_);
529  }
530  }
531 }
532 
533 void art::RootDAQOutFile::beginInputFile(RootFileBlock const* rfb,
534  bool const fastCloneFromOutputModule)
535 {
536  // FIXME: the logic here is nasty.
537  bool shouldFastClone{fastCloningEnabledAtConstruction_ &&
538  fastCloneFromOutputModule && rfb};
539  // Create output branches, and then redo calculation to determine if
540  // fast cloning should be done.
541  selectProducts();
542  if (shouldFastClone &&
543  !treePointers_[InEvent]->checkSplitLevelAndBasketSize(rfb->tree()))
544  {
545  mf::LogWarning("FastCloning")
546  << "Fast cloning deactivated for this input file due to "
547  << "splitting level and/or basket size.";
548  shouldFastClone = false;
549  }
550  else if (rfb && rfb->tree() &&
551  rfb->tree()->GetCurrentFile()->GetVersion() < 60001)
552  {
553  mf::LogWarning("FastCloning")
554  << "Fast cloning deactivated for this input file due to "
555  << "ROOT version used to write it (< 6.00/01)\n"
556  "having a different splitting policy.";
557  shouldFastClone = false;
558  }
559  if (shouldFastClone && rfb->fileFormatVersion().value_ < 10)
560  {
561  mf::LogWarning("FastCloning")
562  << "Fast cloning deactivated for this input file due to "
563  << "reading in file that has a different ProductID schema.";
564  shouldFastClone = false;
565  }
566  if (shouldFastClone && !fastCloningEnabledAtConstruction_)
567  {
568  mf::LogWarning("FastCloning")
569  << "Fast cloning reactivated for this input file.";
570  }
571  treePointers_[InEvent]->beginInputFile(shouldFastClone);
572  auto tree = (rfb && rfb->tree()) ? rfb->tree() : nullptr;
573  wasFastCloned_ = treePointers_[InEvent]->fastCloneTree(tree);
574 }
575 
576 void art::RootDAQOutFile::incrementInputFileNumber()
577 {
578  fp_.update<Granularity::InputFile>();
579 }
580 
581 void art::RootDAQOutFile::respondToCloseInputFile(FileBlock const&)
582 {
583  cet::for_all(treePointers_, [](auto const& p) { p->setEntries(); });
584 }
585 
586 bool art::RootDAQOutFile::requestsToCloseFile()
587 {
588  using namespace std::chrono;
589  unsigned constexpr oneK{1024U};
590  if (fileSwitchCriteria_.fileProperties().size() !=
591  art::Defaults::size_max())
592  {
593  // Note: TFile::GetSize() is very, very slow, so we only call it
594  // if we actually need to.
595  fp_.updateSize(filePtr_->GetSize() / oneK);
596  }
597  fp_.updateAge(duration_cast<seconds>(steady_clock::now() - beginTime_));
598  return fileSwitchCriteria_.should_close(fp_);
599 }
600 
601 void art::RootDAQOutFile::writeOneEvent_EventPart(EventPrincipal const& e)
602 {
603  TLOG(TLVL_TRACE) << "Start of RootDAQOutFile::writeOne";
604  // Auxiliary branch.
605  // Note: pEventAux_ must be set before calling fillBranches
606  // since it gets written out in that routine.
607  pEventAux_ = &e.aux();
608  // Because getting the data may cause an exception to be
609  // thrown we want to do that first before writing anything
610  // to the file about this event.
611  fillBranches<InEvent>(e, pEventProductProvenanceVector_);
612 }
613 
614 void art::RootDAQOutFile::writeOneEvent_HistoryPart(EventPrincipal const& e)
615 {
616  // History branch.
617  History historyForOutput{e.history()};
618  historyForOutput.addEventSelectionEntry(om_->selectorConfig());
619  pHistory_ = &historyForOutput;
620  int sz = eventHistoryTree_->Fill();
621  pHistory_ = &e.history();
622  if (sz <= 0)
623  {
624  throw art::Exception(art::errors::FatalRootError)
625  << "Failed to fill the History tree for event: " << e.id()
626  << "\nTTree::Fill() returned " << sz << " bytes written." << endl;
627  }
628 }
629 
630 void art::RootDAQOutFile::writeOneEvent_FileIndexPart(EventPrincipal const& e)
631 {
632  // Add event to index
633  {
634  FileIndex::Element elem{e.aux().id(), fp_.eventEntryNumber()};
635  auto const* findexElemPtr = &elem;
636  fileIndexTree_->SetBranchAddress(metaBranchRootName<FileIndex::Element>(),
637  &findexElemPtr);
638  fileIndexTree_->Fill();
639  fileIndexTree_->ResetBranchAddress(
640  fileIndexTree_->GetBranch(metaBranchRootName<FileIndex::Element>()));
641  }
642 }
643 
644 void art::RootDAQOutFile::writeOne(EventPrincipal const& e)
645 {
646  writeOneEvent_EventPart(e);
647  writeOneEvent_HistoryPart(e);
648  // Add the dataType to the job report if it hasn't already been done
649  if (!dataTypeReported_)
650  {
651  string dataType{"MC"};
652  if (e.aux().isRealData())
653  {
654  dataType = "Data";
655  }
656  dataTypeReported_ = true;
657  }
658  writeOneEvent_FileIndexPart(e);
659  fp_.update<Granularity::Event>(status_);
660  TLOG(TLVL_TRACE) << "End of RootDAQOutFile::writeOne";
661 }
662 
663 void art::RootDAQOutFile::writeSubRun_FileIndexPart(SubRunPrincipal const& sr)
664 {
665  {
666  FileIndex::Element elem{EventID::invalidEvent(sr.aux().id()),
667  fp_.subRunEntryNumber()};
668  auto const* findexElemPtr = &elem;
669  fileIndexTree_->SetBranchAddress(metaBranchRootName<FileIndex::Element>(),
670  &findexElemPtr);
671  fileIndexTree_->Fill();
672  fileIndexTree_->ResetBranchAddress(
673  fileIndexTree_->GetBranch(metaBranchRootName<FileIndex::Element>()));
674  }
675 }
676 
677 void art::RootDAQOutFile::writeSubRun(SubRunPrincipal const& sr)
678 {
679  pSubRunAux_ = &sr.aux();
680  if (rangesEnabled_)
681  {
682  pSubRunAux_->setRangeSetID(subRunRSID_);
683  }
684  fillBranches<InSubRun>(sr, pSubRunProductProvenanceVector_);
685  writeSubRun_FileIndexPart(sr);
686  fp_.update<Granularity::SubRun>(status_);
687 }
688 
689 void art::RootDAQOutFile::writeRun_FileIndexPart(RunPrincipal const& r)
690 {
691  {
692  FileIndex::Element elem{EventID::invalidEvent(r.aux().id()),
693  fp_.runEntryNumber()};
694  auto const* findexElemPtr = &elem;
695  fileIndexTree_->SetBranchAddress(metaBranchRootName<FileIndex::Element>(),
696  &findexElemPtr);
697  fileIndexTree_->Fill();
698  fileIndexTree_->ResetBranchAddress(
699  fileIndexTree_->GetBranch(metaBranchRootName<FileIndex::Element>()));
700  }
701 }
702 
703 void art::RootDAQOutFile::writeRun(RunPrincipal const& r)
704 {
705  pRunAux_ = &r.aux();
706  if (rangesEnabled_)
707  {
708  pRunAux_->setRangeSetID(runRSID_);
709  }
710  fillBranches<InRun>(r, pRunProductProvenanceVector_);
711  writeRun_FileIndexPart(r);
712  fp_.update<Granularity::Run>(status_);
713 }
714 
715 void art::RootDAQOutFile::writeParentageRegistry()
716 {
717  auto pid = root::getObjectRequireDict<ParentageID>();
718  ParentageID const* hash = &pid;
719  TBranch* b = parentageTree_->Branch(
720  rootNames::parentageIDBranchName().c_str(), &hash, basketSize_, 0);
721  if (!b)
722  {
723  throw Exception(errors::FatalRootError)
724  << "Failed to create a branch for ParentageIDs in the output file";
725  }
726  b = nullptr;
727  hash = nullptr;
728  auto par = root::getObjectRequireDict<Parentage>();
729  Parentage const* desc = &par;
730  b = parentageTree_->Branch(
731  rootNames::parentageBranchName().c_str(), &desc, basketSize_, 0);
732  if (!b)
733  {
734  throw art::Exception(art::errors::FatalRootError)
735  << "Failed to create a branch for Parentages in the output file";
736  }
737  desc = nullptr;
738  for (auto const& pr : ParentageRegistry::get())
739  {
740  hash = &pr.first;
741  desc = &pr.second;
742  parentageTree_->Fill();
743  }
744  parentageTree_->SetBranchAddress(rootNames::parentageIDBranchName().c_str(),
745  reinterpret_cast<void*>(-1L));
746  parentageTree_->SetBranchAddress(rootNames::parentageBranchName().c_str(),
747  reinterpret_cast<void*>(-1L));
748 }
749 
750 void art::RootDAQOutFile::writeFileFormatVersion()
751 {
752  FileFormatVersion const ver{getFileFormatVersion(), getFileFormatEra()};
753  auto const* pver = &ver;
754  TBranch* b = metaDataTree_->Branch(
755  metaBranchRootName<FileFormatVersion>(), &pver, basketSize_, 0);
756  assert(b);
757  b->Fill();
758 }
759 
760 void art::RootDAQOutFile::writeFileIndex()
761 {
762  // Read file index.
763  auto findexPtr = &fileIndex_;
764  detail::readFileIndex(filePtr_.get(), fileIndexTree_, findexPtr);
765  // Sort it.
766  fileIndex_.sortBy_Run_SubRun_Event();
767  // Make replacement tree.
768  fileIndexTree_ = RootOutputTree::makeTTree(
769  filePtr_.get(), rootNames::fileIndexTreeName(), 0);
770  // Make FileIndex branch.
771  FileIndex::Element elem;
772  auto findexElemPtr = &elem;
773  auto b = fileIndexTree_->Branch(
774  metaBranchRootName<FileIndex::Element>(), &findexElemPtr, basketSize_, 0);
775  // Now write out the new sorted FileIndex.
776  for (auto& entry : fileIndex_)
777  {
778  findexElemPtr = &entry;
779  fileIndexTree_->Fill();
780  }
781  b->ResetAddress();
782 }
783 
784 void art::RootDAQOutFile::writeEventHistory()
785 {
786  RootOutputTree::writeTTree(eventHistoryTree_);
787 }
788 
789 void art::RootDAQOutFile::writeProcessConfigurationRegistry()
790 {
791  // We don't do this yet; currently we're storing a slightly
792  // bloated ProcessHistoryRegistry.
793 }
794 
795 void art::RootDAQOutFile::writeProcessHistoryRegistry()
796 {
797  ProcessHistoryMap pHistMap;
798  for (auto const& pr : ProcessHistoryRegistry::get())
799  {
800  pHistMap.emplace(pr);
801  }
802  auto const* p = &pHistMap;
803  TBranch* b = metaDataTree_->Branch(
804  metaBranchRootName<ProcessHistoryMap>(), &p, basketSize_, 0);
805  if (!b)
806  {
807  throw Exception(errors::LogicError)
808  << "Unable to locate required ProcessHistoryMap branch in output "
809  "metadata tree.\n";
810  }
811  b->Fill();
812 }
813 
814 void art::RootDAQOutFile::writeFileCatalogMetadata(
815  FileStatsCollector const& stats,
816  FileCatalogMetadata::collection_type const& md,
817  FileCatalogMetadata::collection_type const& ssmd)
818 {
819  if (!dbEnabled_)
820  {
821  return;
822  }
823  using namespace cet::sqlite;
824  Ntuple<std::string, std::string> fileCatalogMetadata{
825  rootFileDB_, "FileCatalog_metadata", {{"Name", "Value"}}, true};
826  Transaction txn{rootFileDB_};
827  for (auto const& kv : md)
828  {
829  fileCatalogMetadata.insert(kv.first, kv.second);
830  }
831  // Add our own specific information: File format and friends.
832  fileCatalogMetadata.insert("file_format", "\"artroot\"");
833  fileCatalogMetadata.insert("file_format_era",
834  cet::canonical_string(getFileFormatEra()));
835  fileCatalogMetadata.insert("file_format_version",
836  to_string(getFileFormatVersion()));
837  // File start time.
838  namespace bpt = boost::posix_time;
839  auto formatted_time = [](auto const& t) {
840  return cet::canonical_string(bpt::to_iso_extended_string(t));
841  };
842  fileCatalogMetadata.insert("start_time",
843  formatted_time(stats.outputFileOpenTime()));
844  // File "end" time: now, since file is not actually closed yet.
845  fileCatalogMetadata.insert(
846  "end_time",
847  formatted_time(boost::posix_time::second_clock::universal_time()));
848  // Run/subRun information.
849  if (!stats.seenSubRuns().empty())
850  {
851  auto I = find_if(md.crbegin(), md.crend(), [](auto const& p) {
852  return p.first == "run_type";
853  });
854  if (I != md.crend())
855  {
856  ostringstream buf;
857  buf << "[ ";
858  for (auto const& srid : stats.seenSubRuns())
859  {
860  buf << "[ " << srid.run() << ", " << srid.subRun() << ", "
861  << cet::canonical_string(I->second) << " ], ";
862  }
863  // Rewind over last delimiter.
864  buf.seekp(-2, ios_base::cur);
865  buf << " ]";
866  fileCatalogMetadata.insert("runs", buf.str());
867  }
868  }
869  // Number of events.
870  fileCatalogMetadata.insert("event_count", to_string(stats.eventsThisFile()));
871  // first_event and last_event.
872  auto eidToTuple = [](EventID const& eid) -> string {
873  ostringstream eidStr;
874  eidStr << "[ " << eid.run() << ", " << eid.subRun() << ", " << eid.event()
875  << " ]";
876  return eidStr.str();
877  };
878  fileCatalogMetadata.insert("first_event", eidToTuple(stats.lowestEventID()));
879  fileCatalogMetadata.insert("last_event", eidToTuple(stats.highestEventID()));
880  // File parents.
881  if (!stats.parents().empty())
882  {
883  ostringstream pstring;
884  pstring << "[ ";
885  for (auto const& parent : stats.parents())
886  {
887  pstring << cet::canonical_string(parent) << ", ";
888  }
889  // Rewind over last delimiter.
890  pstring.seekp(-2, ios_base::cur);
891  pstring << " ]";
892  fileCatalogMetadata.insert("parents", pstring.str());
893  }
894  // Incoming stream-specific metadata overrides.
895  for (auto const& kv : ssmd)
896  {
897  fileCatalogMetadata.insert(kv.first, kv.second);
898  }
899  txn.commit();
900 }
901 
902 void art::RootDAQOutFile::writeParameterSetRegistry()
903 {
904  if (!dbEnabled_)
905  {
906  return;
907  }
908  fhicl::ParameterSetRegistry::exportTo(rootFileDB_);
909 }
910 
911 void art::RootDAQOutFile::writeProductDescriptionRegistry()
912 {
913  // Make a local copy of the MasterProductRegistry's ProductList,
914  // removing any transient or pruned products.
915  auto end = branchesWithStoredHistory_.end();
916  ProductRegistry reg;
917  for (auto const& pr : ProductMetaData::instance().productList())
918  {
919  if (branchesWithStoredHistory_.find(pr.second.productID()) == end)
920  {
921  continue;
922  }
923  reg.productList_.emplace_hint(reg.productList_.end(), pr);
924  }
925  ProductRegistry const* regp = &reg;
926  TBranch* b = metaDataTree_->Branch(
927  metaBranchRootName<ProductRegistry>(), &regp, basketSize_, 0);
928  b->Fill();
929 }
930 
931 void art::RootDAQOutFile::writeProductDependencies()
932 {
933  BranchChildren const* ppDeps = &om_->branchChildren();
934  TBranch* b = metaDataTree_->Branch(
935  metaBranchRootName<BranchChildren>(), &ppDeps, basketSize_, 0);
936  b->Fill();
937 }
938 
939 void art::RootDAQOutFile::writeResults(ResultsPrincipal& resp)
940 {
941  pResultsAux_ = &resp.aux();
942  fillBranches<InResults>(resp, pResultsProductProvenanceVector_);
943 }
944 
945 void art::RootDAQOutFile::writeTTrees()
946 {
947  TLOG(TLVL_TRACE) << "Start of RootDAQOutFile::writeTTrees";
948  RootOutputTree::writeTTree(metaDataTree_);
949  TLOG(TLVL_TRACE) << "RootDAQOutFile::writeTTrees after writing metaDataTree_";
950  RootOutputTree::writeTTree(fileIndexTree_);
951  TLOG(TLVL_TRACE) << "RootDAQOutFile::writeTTrees after writing fileIndexTree_";
952  RootOutputTree::writeTTree(parentageTree_);
953  TLOG(TLVL_TRACE) << "RootDAQOutFile::writeTTrees after writing parentageTree_";
954  // Write out the tree corresponding to each BranchType
955  for (int i = InEvent; i < NumBranchTypes; ++i)
956  {
957  auto const branchType = static_cast<BranchType>(i);
958  treePointers_[branchType]->writeTree();
959  }
960  TLOG(TLVL_TRACE) << "End of RootDAQOutFile::writeTTrees";
961 }
962 
963 void art::RootDAQOutFile::insertParents(std::set<ProductProvenance>& keptProvenance,
964  Principal const& principal,
965  vector<ProductID> const& parents)
966 {
967  for (auto const pid : parents)
968  {
969  auto pp = principal.branchMapper().branchToProductProvenance(pid);
970  if (!pp)
971  {
972  continue;
973  }
974  branchesWithStoredHistory_.insert(pid);
975  keptProvenance.insert(*pp);
976  if ((dropMetaData_ == DropMetaData::DropPrior) ||
977  (dropMetaData_ == DropMetaData::DropAll) ||
978  dropMetaDataForDroppedData_)
979  {
980  continue;
981  }
982  auto const* pd = principal.getForOutput(pp->productID(), false).desc();
983  if (pd && pd->produced())
984  {
985  insertParents(keptProvenance, principal, pp->parentage().parents());
986  }
987  }
988 }
989 
990 template<art::BranchType BT>
991 void art::RootDAQOutFile::fillBranches(Principal const& principal,
992  vector<ProductProvenance>* vpp)
993 {
994  TLOG(TLVL_TRACE) << "Start of RootDAQOutFile::fillBranches";
995  set<ProductProvenance> keptProvenance;
996  map<unsigned, unsigned> checksumToIndex;
997  for (auto const& val : selectedOutputItemList_[BT])
998  {
999  auto const* pd = val.branchDescription_;
1000  branchesWithStoredHistory_.insert(pd->productID());
1001  bool const readProductNow =
1002  (pd->produced() || ((BT != InEvent) || !wasFastCloned_) ||
1003  treePointers_[BT]->uncloned(pd->branchName()));
1004  auto const& oh = principal.getForOutput(pd->productID(), readProductNow);
1005  unique_ptr<ProductProvenance> prov;
1006  if ((dropMetaData_ == DropMetaData::DropNone) ||
1007  (pd->produced() && (dropMetaData_ == DropMetaData::DropPrior)))
1008  {
1009  // We need provenance for the output file.
1010  if (oh.productProvenance())
1011  {
1012  prov = std::make_unique<ProductProvenance>(
1013  *keptProvenance.insert(*oh.productProvenance()).first);
1014  if (parentageEnabled_ && (dropMetaData_ == DropMetaData::DropNone) &&
1015  !dropMetaDataForDroppedData_)
1016  {
1017  // Take extra effort to keep the provenance of our parents.
1018  insertParents(keptProvenance,
1019  principal,
1020  oh.productProvenance()->parentage().parents());
1021  }
1022  }
1023  else
1024  {
1025  // No provenance, product was either not produced, or was
1026  // dropped, create provenance to remember that.
1027  prov = std::make_unique<ProductProvenance>(
1028  *keptProvenance
1029  .emplace(pd->productID(),
1030  pd->produced() ? productstatus::neverCreated() : productstatus::dropped())
1031  .first);
1032  }
1033  }
1034  if (readProductNow)
1035  {
1036  art::EDProduct const* product{nullptr};
1037  if (rangesEnabled_)
1038  {
1039  auto const& rs =
1040  getRangeSet<BT>(oh, principal.seenRanges(), pd->produced());
1041  if (RangeSetsSupported<BT>::value && !rs.is_valid())
1042  {
1043  // Unfortunately, 'unknown' is the only viable product status
1044  // when this condition is triggered (due to the assert
1045  // statement in ProductStatus::setNotPresent). Whenever the
1046  // metadata revolution comes, this should be revised.
1047  {
1048  if (keptProvenance.erase(*prov) != 1ULL)
1049  {
1050  throw Exception(errors::LogicError,
1051  "art::RootDAQOutFile::fillBranches")
1052  << "Attempt to set product status for product whose provenance "
1053  "is not being recorded.";
1054  }
1055  keptProvenance.emplace(prov->productID(), productstatus::unknown());
1056  }
1057  }
1058  product = getProduct<BT>(oh, rs, pd->wrappedName());
1059  if (dbEnabled_)
1060  {
1061  setProductRangeSetID<BT>(
1062  rs, rootFileDB_, const_cast<EDProduct*>(product), checksumToIndex);
1063  }
1064  }
1065  else
1066  {
1067  product =
1068  getProduct<BT>(oh, art::RangeSet::invalid(), pd->wrappedName());
1069  }
1070  val.product_ = product;
1071  }
1072  }
1073  vpp->assign(keptProvenance.begin(), keptProvenance.end());
1074  TLOG(TLVL_TRACE) << "RootDAQOutFile::fillBranches before fillTree call";
1075  treePointers_[BT]->fillTree();
1076  TLOG(TLVL_TRACE) << "RootDAQOutFile::fillBranches after fillTree call";
1077  vpp->clear();
1078  vpp->shrink_to_fit();
1079  TLOG(TLVL_TRACE) << "End of RootDAQOutFile::fillBranches";
1080 }
1081 
1082 void art::RootDAQOutFile::setSubRunAuxiliaryRangeSetID(RangeSet const& ranges)
1083 {
1084  if (!rangesEnabled_)
1085  {
1086  return;
1087  }
1088  if (!dbEnabled_)
1089  {
1090  return;
1091  }
1092  subRunRSID_ = getNewRangeSetID(rootFileDB_, InSubRun, ranges.run());
1093  insertIntoEventRanges(rootFileDB_, ranges);
1094  auto const& eventRangesIDs = getExistingRangeSetIDs(rootFileDB_, ranges);
1095  insertIntoJoinTable(rootFileDB_, InSubRun, subRunRSID_, eventRangesIDs);
1096 }
1097 
1098 void art::RootDAQOutFile::setRunAuxiliaryRangeSetID(RangeSet const& ranges)
1099 {
1100  if (!rangesEnabled_)
1101  {
1102  return;
1103  }
1104  if (!dbEnabled_)
1105  {
1106  return;
1107  }
1108  runRSID_ = getNewRangeSetID(rootFileDB_, InRun, ranges.run());
1109  insertIntoEventRanges(rootFileDB_, ranges);
1110  auto const& eventRangesIDs = getExistingRangeSetIDs(rootFileDB_, ranges);
1111  insertIntoJoinTable(rootFileDB_, InRun, runRSID_, eventRangesIDs);
1112 }
1113 
1114 template<BranchType BT>
1115 std::enable_if_t<!RangeSetsSupported<BT>::value, art::EDProduct const*>
1116 art::RootDAQOutFile::getProduct(art::OutputHandle const& oh,
1117  art::RangeSet const& /*prunedProductRS*/,
1118  std::string const& wrappedName)
1119 {
1120  if (oh.isValid())
1121  {
1122  return oh.wrapper();
1123  }
1124  return dummyProductCache_.product(wrappedName);
1125 }
1126 
1127 template<BranchType BT>
1128 std::enable_if_t<RangeSetsSupported<BT>::value, art::EDProduct const*>
1129 art::RootDAQOutFile::getProduct(art::OutputHandle const& oh,
1130  art::RangeSet const& prunedProductRS,
1131  std::string const& wrappedName)
1132 {
1133  if (rangesEnabled_)
1134  {
1135  if (oh.isValid() && prunedProductRS.is_valid())
1136  {
1137  return oh.wrapper();
1138  }
1139  return dummyProductCache_.product(wrappedName);
1140  }
1141  if (oh.isValid())
1142  {
1143  return oh.wrapper();
1144  }
1145  return dummyProductCache_.product(wrappedName);
1146 }