artdaq  v3_07_02
RootDAQOutFile.cc
1 #include "artdaq/ArtModules/RootDAQOutput/RootDAQOutFile.h"
2 // vim: set sw=2:
3 
4 #include "Rtypes.h"
5 #include "TBranchElement.h"
6 #include "TClass.h"
7 #include "TFile.h"
8 #include "TTree.h"
9 #include "art/Framework/IO/FileStatsCollector.h"
10 #include "art/Framework/IO/Root/DropMetaData.h"
11 #include "art/Framework/IO/Root/GetFileFormatEra.h"
12 #include "art/Framework/IO/Root/GetFileFormatVersion.h"
13 #include "art/Framework/IO/Root/RootDB/SQLErrMsg.h"
14 #include "art/Framework/IO/Root/RootDB/TKeyVFSOpenPolicy.h"
15 #include "art/Framework/IO/Root/RootFileBlock.h"
16 #include "art/Framework/IO/Root/checkDictionaries.h"
17 #include "art/Framework/IO/Root/detail/KeptProvenance.h"
18 #include "art/Framework/IO/Root/detail/getObjectRequireDict.h"
19 #include "art/Framework/Principal/EventPrincipal.h"
20 #include "art/Framework/Principal/ResultsPrincipal.h"
21 #include "art/Framework/Principal/RunPrincipal.h"
22 #include "art/Framework/Principal/SubRunPrincipal.h"
23 #include "art/Framework/Services/System/DatabaseConnection.h"
24 #include "art/Persistency/Provenance/ProcessHistoryRegistry.h"
25 #if ART_HEX_VERSION < 0x30000
26 #include "art/Persistency/Provenance/ProductMetaData.h"
27 #endif
28 #include "art/Version/GetReleaseVersion.h"
29 #include "artdaq/DAQdata/Globals.hh"
30 #include "boost/date_time/posix_time/posix_time.hpp"
31 #include "canvas/Persistency/Provenance/BranchChildren.h"
32 #include "canvas/Persistency/Provenance/BranchType.h"
33 #include "canvas/Persistency/Provenance/EventAuxiliary.h"
34 #include "canvas/Persistency/Provenance/EventID.h"
35 #include "canvas/Persistency/Provenance/FileFormatVersion.h"
36 #include "canvas/Persistency/Provenance/History.h"
37 #include "canvas/Persistency/Provenance/ParameterSetBlob.h"
38 #include "canvas/Persistency/Provenance/Parentage.h"
39 #include "canvas/Persistency/Provenance/ParentageRegistry.h"
40 #include "canvas/Persistency/Provenance/ProcessHistoryID.h"
41 #include "canvas/Persistency/Provenance/ProductStatus.h"
42 #include "canvas/Persistency/Provenance/ResultsAuxiliary.h"
43 #include "canvas/Persistency/Provenance/RunAuxiliary.h"
44 #include "canvas/Persistency/Provenance/SubRunAuxiliary.h"
45 #include "canvas/Persistency/Provenance/rootNames.h"
46 #include "canvas/Utilities/Exception.h"
47 #include "canvas_root_io/Utilities/DictionaryChecker.h"
48 #include "cetlib/canonical_string.h"
49 #include "cetlib/container_algorithms.h"
50 #include "cetlib/exempt_ptr.h"
51 #include "cetlib/sqlite/Ntuple.h"
52 #include "cetlib/sqlite/Transaction.h"
53 #include "cetlib/sqlite/create_table.h"
54 #include "cetlib/sqlite/exec.h"
55 #include "cetlib/sqlite/insert.h"
56 #include "fhiclcpp/ParameterSet.h"
57 #include "fhiclcpp/ParameterSetID.h"
58 #include "fhiclcpp/ParameterSetRegistry.h"
59 #include "tracemf.h" // TLOG
60 #define TRACE_NAME (app_name + "_RootDAQOutFile").c_str()
61 
62 #include <fcntl.h> // posix_fadvise POSIX_FADV_DONTNEED
63 #include <sys/sysinfo.h> // sysinfo(sysinfo*)
64 #include <algorithm>
65 #include <utility>
66 #include <vector>
67 
68 using namespace cet;
69 using namespace std;
70 using art::BranchType;
72 using art::rootNames::metaBranchRootName;
73 
74 namespace {
75 
76 void create_table(sqlite3* const db,
77  std::string const& name,
78  std::vector<std::string> const& columns,
79  std::string const& suffix = {})
80 {
81  if (columns.empty())
82  throw art::Exception(art::errors::LogicError)
83  << "Number of sqlite columns specified for table: " << name << '\n'
84  << "is zero.\n";
85 
86  std::string ddl = "DROP TABLE IF EXISTS " + name +
87  "; "
88  "CREATE TABLE " +
89  name + "(" + columns.front();
90  std::for_each(columns.begin() + 1, columns.end(), [&ddl](auto const& col) {
91  ddl += "," + col;
92  });
93  ddl += ") ";
94  ddl += suffix;
95  ddl += ";";
96  sqlite::exec(db, ddl);
97 }
98 
99 void insert_eventRanges_row(sqlite3_stmt* stmt,
100  art::SubRunNumber_t const sr,
101  art::EventNumber_t const b,
102  art::EventNumber_t const e)
103 {
104  sqlite3_bind_int64(stmt, 1, sr);
105  sqlite3_bind_int64(stmt, 2, b);
106  sqlite3_bind_int64(stmt, 3, e);
107  sqlite3_step(stmt);
108  sqlite3_reset(stmt);
109 }
110 
111 void insert_rangeSets_eventSets_row(sqlite3_stmt* stmt,
112  unsigned const rsid,
113  unsigned const esid)
114 {
115  sqlite3_bind_int64(stmt, 1, rsid);
116  sqlite3_bind_int64(stmt, 2, esid);
117  sqlite3_step(stmt);
118  sqlite3_reset(stmt);
119 }
120 
121 unsigned
122 getNewRangeSetID(sqlite3* db,
123  art::BranchType const bt,
124  art::RunNumber_t const r)
125 {
126  sqlite::insert_into(db, art::BranchTypeToString(bt) + "RangeSets")
127  .values(r);
128  return sqlite3_last_insert_rowid(db);
129 }
130 
131 vector<unsigned>
132 getExistingRangeSetIDs(sqlite3* db, art::RangeSet const& rs)
133 {
134  vector<unsigned> rangeSetIDs;
135  cet::transform_all(
136  rs, std::back_inserter(rangeSetIDs), [db](auto const& range) {
137  sqlite::query_result<unsigned> r;
138  r << sqlite::select("ROWID")
139  .from(db, "EventRanges")
140  .where("SubRun=" + std::to_string(range.subRun()) +
141  " AND "
142  "begin=" +
143  std::to_string(range.begin()) +
144  " AND "
145  "end=" +
146  std::to_string(range.end()));
147  return unique_value(r);
148  });
149  return rangeSetIDs;
150 }
151 
152 void insertIntoEventRanges(sqlite3* db, art::RangeSet const& rs)
153 {
154  sqlite::Transaction txn{db};
155  sqlite3_stmt* stmt{nullptr};
156  std::string const ddl{
157  "INSERT INTO EventRanges(SubRun, begin, end) "
158  "VALUES(?, ?, ?);"};
159  sqlite3_prepare_v2(db, ddl.c_str(), -1, &stmt, nullptr);
160  for (auto const& range : rs)
161  {
162  insert_eventRanges_row(stmt, range.subRun(), range.begin(), range.end());
163  }
164  sqlite3_finalize(stmt);
165  txn.commit();
166 }
167 
168 void insertIntoJoinTable(sqlite3* db,
169  art::BranchType const bt,
170  unsigned const rsID,
171  vector<unsigned> const& eventRangesIDs)
172 {
173  sqlite::Transaction txn{db};
174  sqlite3_stmt* stmt{nullptr};
175  std::string const ddl{
176  "INSERT INTO " + art::BranchTypeToString(bt) +
177  "RangeSets_EventRanges(RangeSetsID, EventRangesID) Values(?,?);"};
178  sqlite3_prepare_v2(db, ddl.c_str(), -1, &stmt, nullptr);
179  cet::for_all(eventRangesIDs, [stmt, rsID](auto const eventRangeID) {
180  insert_rangeSets_eventSets_row(stmt, rsID, eventRangeID);
181  });
182  sqlite3_finalize(stmt);
183  txn.commit();
184 }
185 
186 void maybeInvalidateRangeSet(BranchType const bt,
187  art::RangeSet const& principalRS,
188  art::RangeSet& productRS)
189 {
190  assert(principalRS.is_sorted());
191  assert(productRS.is_sorted());
192 
193  if (!productRS.is_valid())
194  return;
195  if (bt == art::InRun && productRS.is_full_run())
196  return;
197  if (bt == art::InSubRun && productRS.is_full_subRun())
198  return;
199  if (productRS.ranges().empty())
200  return;
201 
202  auto const r = productRS.run();
203  auto const& productFront = productRS.ranges().front();
204  if (!principalRS.contains(r, productFront.subRun(), productFront.begin()))
205  productRS = art::RangeSet::invalid();
206 }
207 
208 using art::detail::RangeSetsSupported;
209 
210 // The purpose of 'maybeInvalidateRangeSet' is to support the
211 // following situation. Suppose process 1 creates three files with
212 // one Run product each, all corresponding to the same Run. Let's
213 // call the individual Run product instances in the three separate
214 // files as A, B, and C. Now suppose that the three files serve as
215 // inputs to process 2, where a concatenation is being performed AND
216 // ALSO an output file switch. Process 2 results in two output
217 // files, and now, in process 3, we concatenate the outputs from
218 // process 2. The situation would look like this:
219 //
220 // Process 1: [A] [B] [C]
221 // \ / \ /
222 // Process 2: [A + B] [B + C]
223 // \ / \ /
224 // D=agg(A,B) | | E=agg(B,C)
225 // \ /
226 // Process 3: [D + E]
227 //
228 // Notice the complication in process 3: product 'B' will be
229 // aggregated twice: once with A, and once with C. Whenever the
230 // output from process 3 is read as input to another process, the
231 // fetched product will be equivalent to A+2B+C.
232 //
233 // To avoid this situation, we compare the RangeSet of the product
234 // with the RangeSet of the in-memory RunAuxiliary. If the
235 // beginning of B's RangeSet is not contained within the auxiliary's
236 // RangeSet, then a dummy product with an invalid RangeSet is
237 // written to disk. Instead of the diagram above, we have:
238 //
239 // Process 1: [A] [B] [C]
240 // \ / \ /
241 // Process 2: [A + B] [x + C]
242 // \ / \ /
243 // D=agg(A,B) | | E=agg(x,C)=C
244 // \ /
245 // Process 3: [D + E]
246 //
247 // where 'x' represent a dummy product. Upon aggregating D and E,
248 // we obtain the correctly formed A+B+C product.
249 template<BranchType BT>
250 std::enable_if_t<RangeSetsSupported<BT>::value, art::RangeSet>
251 getRangeSet(art::OutputHandle const& oh,
252  art::RangeSet const& principalRS,
253  bool const producedInThisProcess)
254 {
255  auto rs = oh.isValid() ? oh.rangeOfValidity() : art::RangeSet::invalid();
256  // Because a user can specify (e.g.):
257  // r.put(std::move(myProd), art::runFragment(myRangeSet));
258  // products that are produced in this process can have valid, yet
259  // arbitrary RangeSets. We therefore never invalidate a RangeSet
260  // that corresponds to a product produced in this process.
261  //
262  // It is possible for a user to specify a RangeSet which does not
263  // correspond AT ALL to the in-memory auxiliary RangeSet. In that
264  // case, users should not expect to be able to retrieve products
265  // for which no corresponding events or sub-runs were processed.
266  if (!producedInThisProcess)
267  {
268  maybeInvalidateRangeSet(BT, principalRS, rs);
269  }
270  return rs;
271 }
272 
273 template<BranchType BT>
274 std::enable_if_t<!RangeSetsSupported<BT>::value, art::RangeSet>
275 getRangeSet(art::OutputHandle const&,
276  art::RangeSet const& /*principalRS*/,
277  bool const /*producedInThisProcess*/)
278 {
279  return art::RangeSet::invalid();
280 }
281 
282 template<BranchType BT>
283 std::enable_if_t<!RangeSetsSupported<BT>::value>
284 setProductRangeSetID(art::RangeSet const& /*rs*/,
285  sqlite3*,
286  art::EDProduct*,
287  std::map<unsigned, unsigned>& /*checksumToIndexLookup*/)
288 {}
289 
290 template<BranchType BT>
291 std::enable_if_t<RangeSetsSupported<BT>::value>
292 setProductRangeSetID(art::RangeSet const& rs,
293  sqlite3* db,
294  art::EDProduct* product,
295  std::map<unsigned, unsigned>& checksumToIndexLookup)
296 {
297  if (!rs.is_valid()) // Invalid range-sets not written to DB
298  return;
299 
300  // Set range sets for SubRun and Run products
301  auto it = checksumToIndexLookup.find(rs.checksum());
302  if (it != checksumToIndexLookup.cend())
303  {
304  product->setRangeSetID(it->second);
305  }
306  else
307  {
308  unsigned const rsID = getNewRangeSetID(db, BT, rs.run());
309  product->setRangeSetID(rsID);
310  checksumToIndexLookup.emplace(rs.checksum(), rsID);
311  insertIntoEventRanges(db, rs);
312  auto const& eventRangesIDs = getExistingRangeSetIDs(db, rs);
313  insertIntoJoinTable(db, BT, rsID, eventRangesIDs);
314  }
315 }
316 
317 } // unnamed namespace
318 
319 art::RootDAQOutFile::RootDAQOutFile(OutputModule* om,
320  string const& fileName,
321  ClosingCriteria const& fileSwitchCriteria,
322  int const compressionLevel,
323  unsigned freePercent,
324  unsigned freeMB,
325  int64_t const saveMemoryObjectThreshold,
326  int64_t const treeMaxVirtualSize,
327  int const splitLevel,
328  int const basketSize,
329  DropMetaData dropMetaData,
330  bool const dropMetaDataForDroppedData,
331  bool const fastCloningRequested)
332  : om_{om}
333  , file_{fileName}
334  , fileSwitchCriteria_{fileSwitchCriteria}
335  , compressionLevel_{compressionLevel}
336  , freePercent_{freePercent}
337  , freeMB_{freeMB}
338  , saveMemoryObjectThreshold_{saveMemoryObjectThreshold}
339  , treeMaxVirtualSize_{treeMaxVirtualSize}
340  , splitLevel_{splitLevel}
341  , basketSize_{basketSize}
342  , dropMetaData_{dropMetaData}
343  , dropMetaDataForDroppedData_{dropMetaDataForDroppedData}
344  , fastCloningEnabledAtConstruction_{fastCloningRequested}
345  , filePtr_{TFile::Open(file_.c_str(), "recreate", "", compressionLevel)}
346  , treePointers_{{// Order (and number) must match BranchTypes.h!
347  std::make_unique<RootOutputTree>(
348  filePtr_.get(),
349  InEvent,
350  pEventAux_,
351  pEventProductProvenanceVector_,
352  basketSize,
353  splitLevel,
354  treeMaxVirtualSize,
355  saveMemoryObjectThreshold),
356  std::make_unique<RootOutputTree>(
357  filePtr_.get(),
358  InSubRun,
359  pSubRunAux_,
360  pSubRunProductProvenanceVector_,
361  basketSize,
362  splitLevel,
363  treeMaxVirtualSize,
364  saveMemoryObjectThreshold),
365  std::make_unique<RootOutputTree>(
366  filePtr_.get(),
367  InRun,
368  pRunAux_,
369  pRunProductProvenanceVector_,
370  basketSize,
371  splitLevel,
372  treeMaxVirtualSize,
373  saveMemoryObjectThreshold),
374  std::make_unique<RootOutputTree>(
375  filePtr_.get(),
376  InResults,
377  pResultsAux_,
378  pResultsProductProvenanceVector_,
379  basketSize,
380  splitLevel,
381  treeMaxVirtualSize,
382  saveMemoryObjectThreshold)}}
383  , rootFileDB_{ServiceHandle<DatabaseConnection>{}->get<TKeyVFSOpenPolicy>(
384  "RootFileDB",
385  filePtr_.get(),
386  SQLITE_OPEN_CREATE | SQLITE_OPEN_READWRITE)}
387 {
388  // Don't split metadata tree or event description tree
389  metaDataTree_ =
390  RootOutputTree::makeTTree(filePtr_.get(), rootNames::metaDataTreeName(), 0);
391  fileIndexTree_ = RootOutputTree::makeTTree(
392  filePtr_.get(), rootNames::fileIndexTreeName(), 0);
393  parentageTree_ = RootOutputTree::makeTTree(
394  filePtr_.get(), rootNames::parentageTreeName(), 0);
395  // Create the tree that will carry (event) History objects.
396  eventHistoryTree_ = RootOutputTree::makeTTree(
397  filePtr_.get(), rootNames::eventHistoryTreeName(), splitLevel);
398  if (!eventHistoryTree_)
399  {
400  throw art::Exception(art::errors::FatalRootError)
401  << "Failed to create the tree for History objects\n";
402  }
403 
404  pHistory_ = new History;
405  if (!eventHistoryTree_->Branch(rootNames::eventHistoryBranchName().c_str(),
406  &pHistory_,
407  basketSize,
408  0))
409  {
410  throw art::Exception(art::errors::FatalRootError)
411  << "Failed to create a branch for History in the output file\n";
412  }
413  delete pHistory_;
414  pHistory_ = nullptr;
415 
416  // Check that dictionaries for the auxiliaries exist
417  root::DictionaryChecker checker{};
418  checker.checkDictionaries<EventAuxiliary>();
419  checker.checkDictionaries<SubRunAuxiliary>();
420  checker.checkDictionaries<RunAuxiliary>();
421  checker.checkDictionaries<ResultsAuxiliary>();
422  checker.reportMissingDictionaries();
423 
424  createDatabaseTables();
425  TLOG(3) << "RootDAQOutFile ctor complete";
426 }
427 
428 art::RootDAQOutFile::~RootDAQOutFile()
429 {
430  struct sysinfo info;
431  int sts = sysinfo(&info);
432  unsigned free_percent = (unsigned)(info.freeram * 100 / info.totalram);
433  unsigned free_MB = (unsigned)(info.freeram * info.mem_unit >> 20); // round down (1024.9 => 1024 MB)
434  TRACE(3, "~RootDAQOutFile free %%%u %.1fMB (%u) buffers=%fGB mem_unit=%u", free_percent, (float)info.freeram * info.mem_unit / (1024 * 1024), free_MB, (float)info.bufferram * info.mem_unit / (1024 * 1024 * 1024), info.mem_unit);
435  if (free_percent < freePercent_ || free_MB < freeMB_)
436  {
437  TLOG(3) << "RootDAQOutFile Flush/DONTNEED";
438  filePtr_->Flush();
439  sts = posix_fadvise(filePtr_->GetFd(), 0, 0 /*len,0=all*/, POSIX_FADV_DONTNEED);
440  }
441  TLOG(3) << "~RootDAQOutFile complete sts=" << sts;
442 }
443 
444 void art::RootDAQOutFile::createDatabaseTables()
445 {
446  // Event ranges
447  create_table(rootFileDB_,
448  "EventRanges",
449  {"SubRun INTEGER",
450  "begin INTEGER",
451  "end INTEGER",
452  "UNIQUE (SubRun,begin,end) ON CONFLICT IGNORE"});
453 
454  // SubRun range sets
455  using namespace cet::sqlite;
456  create_table(rootFileDB_, "SubRunRangeSets", column<int>{"Run"});
457  create_table(rootFileDB_,
458  "SubRunRangeSets_EventRanges",
459  {"RangeSetsID INTEGER",
460  "EventRangesID INTEGER",
461  "PRIMARY KEY(RangeSetsID,EventRangesID)"},
462  "WITHOUT ROWID");
463 
464  // Run range sets
465  create_table(rootFileDB_, "RunRangeSets", column<int>{"Run"});
466  create_table(rootFileDB_,
467  "RunRangeSets_EventRanges",
468  {"RangeSetsID INTEGER",
469  "EventRangesID INTEGER",
470  "PRIMARY KEY(RangeSetsID,EventRangesID)"},
471  "WITHOUT ROWID");
472 }
473 
474 void art::RootDAQOutFile::selectProducts()
475 {
476  for (int i = InEvent; i < NumBranchTypes; ++i)
477  {
478  auto const bt = static_cast<BranchType>(i);
479  auto& items = selectedOutputItemList_[bt];
480 
481  for (auto const& pd : om_->keptProducts()[bt])
482  {
483  // Persist Results products only if they have been produced by
484  // the current process.
485  if (bt == InResults && !pd->produced())
486  continue;
487  checkDictionaries(*pd);
488 
489  // Although the transient flag is already checked when
490  // OutputModule::doSelectProducts is called, it can be flipped
491  // to 'true' after the BranchDescription transients have been
492  // fluffed, which happens during the checkDictionaries call.
493  if (pd->transient())
494  {
495  continue;
496  }
497 
498  items.emplace(pd);
499  }
500 
501  for (auto const& val : items)
502  {
503  treePointers_[bt]->addOutputBranch(*val.branchDescription_, val.product_);
504  }
505  }
506 }
507 
508 void art::RootDAQOutFile::beginInputFile(RootFileBlock const* rfb,
509  bool const fastCloneFromOutputModule)
510 {
511  // FIXME: the logic here is nasty.
512  bool shouldFastClone{fastCloningEnabledAtConstruction_ &&
513  fastCloneFromOutputModule && rfb};
514  // Create output branches, and then redo calculation to determine if
515  // fast cloning should be done.
516  selectProducts();
517  if (shouldFastClone &&
518  !treePointers_[InEvent]->checkSplitLevelAndBasketSize(rfb->tree()))
519  {
520  mf::LogWarning("FastCloning")
521  << "Fast cloning deactivated for this input file due to "
522  << "splitting level and/or basket size.";
523  shouldFastClone = false;
524  }
525  else if (rfb && rfb->tree() &&
526  rfb->tree()->GetCurrentFile()->GetVersion() < 60001)
527  {
528  mf::LogWarning("FastCloning")
529  << "Fast cloning deactivated for this input file due to "
530  << "ROOT version used to write it (< 6.00/01)\n"
531  "having a different splitting policy.";
532  shouldFastClone = false;
533  }
534 
535  if (shouldFastClone && rfb->fileFormatVersion().value_ < 10)
536  {
537  mf::LogWarning("FastCloning")
538  << "Fast cloning deactivated for this input file due to "
539  << "reading in file that has a different ProductID schema.";
540  shouldFastClone = false;
541  }
542 
543  if (shouldFastClone && !fastCloningEnabledAtConstruction_)
544  {
545  mf::LogWarning("FastCloning")
546  << "Fast cloning reactivated for this input file.";
547  }
548  treePointers_[InEvent]->beginInputFile(shouldFastClone);
549  auto tree = (rfb && rfb->tree()) ? rfb->tree() : nullptr;
550  wasFastCloned_ = treePointers_[InEvent]->fastCloneTree(tree);
551 }
552 
553 void art::RootDAQOutFile::incrementInputFileNumber()
554 {
555  fp_.update<Granularity::InputFile>();
556 }
557 
558 void art::RootDAQOutFile::respondToCloseInputFile(FileBlock const&)
559 {
560  cet::for_all(treePointers_, [](auto const& p) { p->setEntries(); });
561 }
562 
563 bool art::RootDAQOutFile::requestsToCloseFile()
564 {
565  using namespace std::chrono;
566  unsigned int constexpr oneK{1024u};
567  fp_.updateSize(filePtr_->GetSize() / oneK);
568  fp_.updateAge(duration_cast<seconds>(steady_clock::now() - beginTime_));
569  return fileSwitchCriteria_.should_close(fp_);
570 }
571 
572 void art::RootDAQOutFile::writeOne(EventPrincipal const& e)
573 {
574  TLOG(TLVL_TRACE) << "Start of RootDAQOutFile::writeOne";
575  // Auxiliary branch.
576  // Note: pEventAux_ must be set before calling fillBranches
577  // since it gets written out in that routine.
578  pEventAux_ = &e.aux();
579  // Because getting the data may cause an exception to be
580  // thrown we want to do that first before writing anything
581  // to the file about this event.
582  fillBranches<InEvent>(e, pEventProductProvenanceVector_);
583  // History branch.
584  History historyForOutput{e.history()};
585  historyForOutput.addEventSelectionEntry(om_->selectorConfig());
586  pHistory_ = &historyForOutput;
587  int sz = eventHistoryTree_->Fill();
588  if (sz <= 0)
589  {
590  throw art::Exception(art::errors::FatalRootError)
591  << "Failed to fill the History tree for event: " << e.id()
592  << "\nTTree::Fill() returned " << sz << " bytes written." << endl;
593  }
594  // Add the dataType to the job report if it hasn't already been done
595  if (!dataTypeReported_)
596  {
597  string dataType{"MC"};
598  if (pEventAux_->isRealData())
599  {
600  dataType = "Data";
601  }
602  dataTypeReported_ = true;
603  }
604  pHistory_ = &e.history();
605  // Add event to index
606  fileIndex_.addEntry(pEventAux_->id(), fp_.eventEntryNumber());
607  fp_.update<Granularity::Event>(status_);
608  TLOG(TLVL_TRACE) << "End of RootDAQOutFile::writeOne";
609 }
610 
611 void art::RootDAQOutFile::writeSubRun(SubRunPrincipal const& sr)
612 {
613  pSubRunAux_ = &sr.aux();
614  pSubRunAux_->setRangeSetID(subRunRSID_);
615  fillBranches<InSubRun>(sr, pSubRunProductProvenanceVector_);
616  fileIndex_.addEntry(EventID::invalidEvent(pSubRunAux_->id()),
617  fp_.subRunEntryNumber());
618  fp_.update<Granularity::SubRun>(status_);
619 }
620 
621 void art::RootDAQOutFile::writeRun(RunPrincipal const& r)
622 {
623  pRunAux_ = &r.aux();
624  pRunAux_->setRangeSetID(runRSID_);
625  fillBranches<InRun>(r, pRunProductProvenanceVector_);
626  fileIndex_.addEntry(EventID::invalidEvent(pRunAux_->id()),
627  fp_.runEntryNumber());
628  fp_.update<Granularity::Run>(status_);
629 }
630 
631 void art::RootDAQOutFile::writeParentageRegistry()
632 {
633  auto pid = root::getObjectRequireDict<ParentageID>();
634  ParentageID const* hash = &pid;
635  if (!parentageTree_->Branch(
636  rootNames::parentageIDBranchName().c_str(), &hash, basketSize_, 0))
637  {
638  throw Exception(errors::FatalRootError)
639  << "Failed to create a branch for ParentageIDs in the output file";
640  }
641  hash = nullptr;
642 
643  auto par = root::getObjectRequireDict<Parentage>();
644  Parentage const* desc = &par;
645  if (!parentageTree_->Branch(
646  rootNames::parentageBranchName().c_str(), &desc, basketSize_, 0))
647  {
648  throw art::Exception(art::errors::FatalRootError)
649  << "Failed to create a branch for Parentages in the output file";
650  }
651  desc = nullptr;
652 
653  for (auto const& pr : ParentageRegistry::get())
654  {
655  hash = &pr.first;
656  desc = &pr.second;
657  parentageTree_->Fill();
658  }
659  parentageTree_->SetBranchAddress(rootNames::parentageIDBranchName().c_str(),
660  nullptr);
661  parentageTree_->SetBranchAddress(rootNames::parentageBranchName().c_str(),
662  nullptr);
663 }
664 
665 void art::RootDAQOutFile::writeFileFormatVersion()
666 {
667  FileFormatVersion const ver{getFileFormatVersion(), getFileFormatEra()};
668  auto const* pver = &ver;
669  TBranch* b = metaDataTree_->Branch(
670  metaBranchRootName<FileFormatVersion>(), &pver, basketSize_, 0);
671  // FIXME: Turn this into a throw!
672  assert(b);
673  b->Fill();
674 }
675 
676 void art::RootDAQOutFile::writeFileIndex()
677 {
678  fileIndex_.sortBy_Run_SubRun_Event();
679  FileIndex::Element elem{};
680  auto const* findexElemPtr = &elem;
681  TBranch* b = fileIndexTree_->Branch(
682  metaBranchRootName<FileIndex::Element>(), &findexElemPtr, basketSize_, 0);
683  // FIXME: Turn this into a throw!
684  assert(b);
685  for (auto& entry : fileIndex_)
686  {
687  findexElemPtr = &entry;
688  b->Fill();
689  }
690  b->SetAddress(0);
691 }
692 
693 void art::RootDAQOutFile::writeEventHistory()
694 {
695  RootOutputTree::writeTTree(eventHistoryTree_);
696 }
697 
698 void art::RootDAQOutFile::writeProcessConfigurationRegistry()
699 {
700  // We don't do this yet; currently we're storing a slightly
701  // bloated ProcessHistoryRegistry.
702 }
703 
704 void art::RootDAQOutFile::writeProcessHistoryRegistry()
705 {
706  ProcessHistoryMap pHistMap;
707  for (auto const& pr : ProcessHistoryRegistry::get())
708  {
709  pHistMap.emplace(pr);
710  }
711  auto const* p = &pHistMap;
712  TBranch* b = metaDataTree_->Branch(
713  metaBranchRootName<ProcessHistoryMap>(), &p, basketSize_, 0);
714  if (b != nullptr)
715  {
716  b->Fill();
717  }
718  else
719  {
720  throw Exception(errors::LogicError) << "Unable to locate required "
721  "ProcessHistoryMap branch in output "
722  "metadata tree.\n";
723  }
724 }
725 
726 void art::RootDAQOutFile::writeFileCatalogMetadata(
727  FileStatsCollector const& stats,
728  FileCatalogMetadata::collection_type const& md,
729  FileCatalogMetadata::collection_type const& ssmd)
730 {
731  using namespace cet::sqlite;
732  Ntuple<std::string, std::string> fileCatalogMetadata{
733  rootFileDB_, "FileCatalog_metadata", {{"Name", "Value"}}, true};
734  Transaction txn{rootFileDB_};
735  for (auto const& kv : md)
736  {
737  fileCatalogMetadata.insert(kv.first, kv.second);
738  }
739  // Add our own specific information: File format and friends.
740  fileCatalogMetadata.insert("file_format", "\"artroot\"");
741  fileCatalogMetadata.insert("file_format_era",
742  cet::canonical_string(getFileFormatEra()));
743  fileCatalogMetadata.insert("file_format_version",
744  to_string(getFileFormatVersion()));
745 
746  // File start time.
747  namespace bpt = boost::posix_time;
748  auto formatted_time = [](auto const& t) {
749  return cet::canonical_string(bpt::to_iso_extended_string(t));
750  };
751  fileCatalogMetadata.insert("start_time",
752  formatted_time(stats.outputFileOpenTime()));
753  // File "end" time: now, since file is not actually closed yet.
754  fileCatalogMetadata.insert(
755  "end_time",
756  formatted_time(boost::posix_time::second_clock::universal_time()));
757 
758  // Run/subRun information.
759  if (!stats.seenSubRuns().empty())
760  {
761  auto I = find_if(md.crbegin(), md.crend(), [](auto const& p) {
762  return p.first == "run_type";
763  });
764 
765  if (I != md.crend())
766  {
767  ostringstream buf;
768  buf << "[ ";
769  for (auto const& srid : stats.seenSubRuns())
770  {
771  buf << "[ " << srid.run() << ", " << srid.subRun() << ", "
772  << cet::canonical_string(I->second) << " ], ";
773  }
774  // Rewind over last delimiter.
775  buf.seekp(-2, ios_base::cur);
776  buf << " ]";
777  fileCatalogMetadata.insert("runs", buf.str());
778  }
779  }
780  // Number of events.
781  fileCatalogMetadata.insert("event_count", to_string(stats.eventsThisFile()));
782  // first_event and last_event.
783  auto eidToTuple = [](EventID const& eid) -> string {
784  ostringstream eidStr;
785  eidStr << "[ " << eid.run() << ", " << eid.subRun() << ", " << eid.event()
786  << " ]";
787  return eidStr.str();
788  };
789  fileCatalogMetadata.insert("first_event", eidToTuple(stats.lowestEventID()));
790  fileCatalogMetadata.insert("last_event", eidToTuple(stats.highestEventID()));
791  // File parents.
792  if (!stats.parents().empty())
793  {
794  ostringstream pstring;
795  pstring << "[ ";
796  for (auto const& parent : stats.parents())
797  {
798  pstring << cet::canonical_string(parent) << ", ";
799  }
800  // Rewind over last delimiter.
801  pstring.seekp(-2, ios_base::cur);
802  pstring << " ]";
803  fileCatalogMetadata.insert("parents", pstring.str());
804  }
805  // Incoming stream-specific metadata overrides.
806  for (auto const& kv : ssmd)
807  {
808  fileCatalogMetadata.insert(kv.first, kv.second);
809  }
810  txn.commit();
811 }
812 
813 void art::RootDAQOutFile::writeParameterSetRegistry()
814 {
815  fhicl::ParameterSetRegistry::exportTo(rootFileDB_);
816 }
817 
818 void art::RootDAQOutFile::writeProductDescriptionRegistry()
819 {
820  // Make a local copy of the MasterProductRegistry's ProductList,
821  // removing any transient or pruned products.
822  auto end = branchesWithStoredHistory_.end();
823 
824  ProductRegistry reg;
825  for (auto const& pr : ProductMetaData::instance().productList())
826  {
827  if (branchesWithStoredHistory_.find(pr.second.productID()) == end)
828  {
829  continue;
830  }
831  reg.productList_.emplace_hint(reg.productList_.end(), pr);
832  }
833 
834  ProductRegistry const* regp = &reg;
835  TBranch* b = metaDataTree_->Branch(
836  metaBranchRootName<ProductRegistry>(), &regp, basketSize_, 0);
837  // FIXME: Turn this into a throw!
838  assert(b);
839  b->Fill();
840 }
841 
842 void art::RootDAQOutFile::writeProductDependencies()
843 {
844  BranchChildren const* ppDeps = &om_->branchChildren();
845  TBranch* b = metaDataTree_->Branch(
846  metaBranchRootName<BranchChildren>(), &ppDeps, basketSize_, 0);
847  // FIXME: Turn this into a throw!
848  assert(b);
849  b->Fill();
850 }
851 
852 void art::RootDAQOutFile::writeResults(ResultsPrincipal& resp)
853 {
854  pResultsAux_ = &resp.aux();
855  fillBranches<InResults>(resp, pResultsProductProvenanceVector_);
856 }
857 
858 void art::RootDAQOutFile::writeTTrees()
859 {
860  TLOG(TLVL_TRACE) << "Start of RootDAQOutFile::writeTTrees";
861  RootOutputTree::writeTTree(metaDataTree_);
862  TLOG(TLVL_TRACE) << "RootDAQOutFile::writeTTrees after writing metaDataTree_";
863  RootOutputTree::writeTTree(fileIndexTree_);
864  TLOG(TLVL_TRACE) << "RootDAQOutFile::writeTTrees after writing fileIndexTree_";
865  RootOutputTree::writeTTree(parentageTree_);
866  TLOG(TLVL_TRACE) << "RootDAQOutFile::writeTTrees after writing parentageTree_";
867  // Write out the tree corresponding to each BranchType
868  for (int i = InEvent; i < NumBranchTypes; ++i)
869  {
870  auto const branchType = static_cast<BranchType>(i);
871  treePointers_[branchType]->writeTree();
872  }
873  TLOG(TLVL_TRACE) << "End of RootDAQOutFile::writeTTrees";
874 }
875 
876 template<art::BranchType BT>
877 void art::RootDAQOutFile::fillBranches(Principal const& principal,
878  vector<ProductProvenance>* vpp)
879 {
880  TLOG(TLVL_TRACE) << "Start of RootDAQOutFile::fillBranches";
881  bool const fastCloning = (BT == InEvent) && wasFastCloned_;
882  detail::KeptProvenance keptProvenance{
883  dropMetaData_, dropMetaDataForDroppedData_, branchesWithStoredHistory_};
884  map<unsigned, unsigned> checksumToIndex;
885 
886  auto const& principalRS = principal.seenRanges();
887 
888  for (auto const& val : selectedOutputItemList_[BT])
889  {
890  auto const* pd = val.branchDescription_;
891  auto const pid = pd->productID();
892  branchesWithStoredHistory_.insert(pid);
893  bool const produced{pd->produced()};
894  bool const resolveProd = (produced || !fastCloning ||
895  treePointers_[BT]->uncloned(pd->branchName()));
896 
897  // Update the kept provenance
898  bool const keepProvenance =
899  (dropMetaData_ == DropMetaData::DropNone ||
900  (dropMetaData_ == DropMetaData::DropPrior && produced));
901  auto const& oh = principal.getForOutput(pid, resolveProd);
902 
903  unique_ptr<ProductProvenance> prov{nullptr};
904  if (keepProvenance)
905  {
906  if (oh.productProvenance())
907  {
908  prov = std::make_unique<ProductProvenance>(
909  keptProvenance.insert(*oh.productProvenance()));
910  keptProvenance.insertAncestors(*oh.productProvenance(), principal);
911  }
912  else
913  {
914  // No provenance, product was either not produced, or was
915  // dropped, create provenance to remember that.
916  auto const status =
917  produced ? productstatus::neverCreated() : productstatus::dropped();
918  prov = std::make_unique<ProductProvenance>(
919  keptProvenance.emplace(pid, status));
920  }
921  }
922 
923  // Resolve the product if necessary
924  if (resolveProd)
925  {
926  auto const& rs = getRangeSet<BT>(oh, principalRS, produced);
927  if (RangeSetsSupported<BT>::value && !rs.is_valid())
928  {
929  // Unfortunately, 'unknown' is the only viable product status
930  // when this condition is triggered (due to the assert
931  // statement in ProductStatus::setNotPresent). Whenever the
932  // metadata revolution comes, this should be revised.
933  keptProvenance.setStatus(*prov, productstatus::unknown());
934  }
935 
936  auto const* product = getProduct<BT>(oh, rs, pd->wrappedName());
937  setProductRangeSetID<BT>(
938  rs, rootFileDB_, const_cast<EDProduct*>(product), checksumToIndex);
939  val.product_ = product;
940  }
941  }
942  vpp->assign(keptProvenance.begin(), keptProvenance.end());
943  TLOG(TLVL_TRACE) << "RootDAQOutFile::fillBranches before fillTree call";
944  treePointers_[BT]->fillTree();
945  TLOG(TLVL_TRACE) << "RootDAQOutFile::fillBranches after fillTree call";
946  vpp->clear();
947  TLOG(TLVL_TRACE) << "End of RootDAQOutFile::fillBranches";
948 }
949 
950 void art::RootDAQOutFile::setSubRunAuxiliaryRangeSetID(RangeSet const& ranges)
951 {
952  subRunRSID_ = getNewRangeSetID(rootFileDB_, InSubRun, ranges.run());
953  insertIntoEventRanges(rootFileDB_, ranges);
954  auto const& eventRangesIDs = getExistingRangeSetIDs(rootFileDB_, ranges);
955  insertIntoJoinTable(rootFileDB_, InSubRun, subRunRSID_, eventRangesIDs);
956 }
957 
958 void art::RootDAQOutFile::setRunAuxiliaryRangeSetID(RangeSet const& ranges)
959 {
960  runRSID_ = getNewRangeSetID(rootFileDB_, InRun, ranges.run());
961  insertIntoEventRanges(rootFileDB_, ranges);
962  auto const& eventRangesIDs = getExistingRangeSetIDs(rootFileDB_, ranges);
963  insertIntoJoinTable(rootFileDB_, InRun, runRSID_, eventRangesIDs);
964 }
965 
966 template<BranchType BT>
967 std::enable_if_t<!RangeSetsSupported<BT>::value, art::EDProduct const*>
968 art::RootDAQOutFile::getProduct(art::OutputHandle const& oh,
969  art::RangeSet const& /*prunedProductRS*/,
970  std::string const& wrappedName)
971 {
972  return oh.isValid() ? oh.wrapper() : dummyProductCache_.product(wrappedName);
973 }
974 
975 template<BranchType BT>
976 std::enable_if_t<RangeSetsSupported<BT>::value, art::EDProduct const*>
977 art::RootDAQOutFile::getProduct(art::OutputHandle const& oh,
978  art::RangeSet const& prunedProductRS,
979  std::string const& wrappedName)
980 {
981  return (oh.isValid() && prunedProductRS.is_valid()) ? oh.wrapper() : dummyProductCache_.product(wrappedName);
982 }