artdaq  v3_02_00
RootDAQOutFile.cc
1 #include "artdaq/ArtModules/RootDAQOutput/RootDAQOutFile.h"
2 // vim: set sw=2:
3 
4 #include "Rtypes.h"
5 #include "TBranchElement.h"
6 #include "TClass.h"
7 #include "TFile.h"
8 #include "TTree.h"
9 #include "art/Framework/Core/FileBlock.h"
10 # include "art/Framework/Core/OutputFileGranularity.h"
11 # include "art/Framework/Services/System/DatabaseConnection.h"
12 # include "art/Persistency/RootDB/TKeyVFSOpenPolicy.h"
13 # include "cetlib/sqlite/Ntuple.h"
14 # include "cetlib/sqlite/Transaction.h"
15 # include "cetlib/sqlite/create_table.h"
16 # include "cetlib/sqlite/exec.h"
17 # include "cetlib/sqlite/insert.h"
18 #include "art/Framework/IO/FileStatsCollector.h"
19 #include "art/Framework/IO/Root/DropMetaData.h"
20 #include "art/Framework/IO/Root/GetFileFormatEra.h"
21 #include "art/Framework/IO/Root/GetFileFormatVersion.h"
22 #include "artdaq/ArtModules/RootDAQOutput/detail/KeptProvenance.h"
23 #include "artdaq/ArtModules/RootDAQOutput/RootDAQOutFile.h"
24 #include "art/Framework/Principal/EventPrincipal.h"
25 #include "art/Framework/Principal/ResultsPrincipal.h"
26 #include "art/Framework/Principal/RunPrincipal.h"
27 #include "art/Framework/Principal/SubRunPrincipal.h"
28 #include "art/Persistency/Provenance/BranchIDListRegistry.h"
29 #include "art/Persistency/Provenance/ProcessHistoryRegistry.h"
30 #include "art/Persistency/Provenance/ProductMetaData.h"
31 #include "art/Persistency/RootDB/SQLErrMsg.h"
32 #include "art/Version/GetReleaseVersion.h"
33 #include "boost/date_time/posix_time/posix_time.hpp"
34 #include "canvas/Persistency/Provenance/rootNames.h"
35 #include "canvas/Persistency/Provenance/BranchChildren.h"
36 #include "canvas/Persistency/Provenance/BranchID.h"
37 #include "canvas/Persistency/Provenance/BranchIDList.h"
38 #include "canvas/Persistency/Provenance/BranchType.h"
39 #include "canvas/Persistency/Provenance/EventAuxiliary.h"
40 #include "canvas/Persistency/Provenance/EventID.h"
41 #include "canvas/Persistency/Provenance/FileFormatVersion.h"
42 #include "canvas/Persistency/Provenance/History.h"
43 #include "canvas/Persistency/Provenance/ParameterSetBlob.h"
44 #include "canvas/Persistency/Provenance/Parentage.h"
45 #include "canvas/Persistency/Provenance/ParentageRegistry.h"
46 #include "canvas/Persistency/Provenance/ProcessHistoryID.h"
47 #include "canvas/Persistency/Provenance/ProductStatus.h"
48 #include "canvas/Persistency/Provenance/ResultsAuxiliary.h"
49 #include "canvas/Persistency/Provenance/RunAuxiliary.h"
50 #include "canvas/Persistency/Provenance/SubRunAuxiliary.h"
51 #include "canvas/Utilities/Exception.h"
52 #include "cetlib/canonical_string.h"
53 #include "cetlib/container_algorithms.h"
54 #include "cetlib/exempt_ptr.h"
55 #include "fhiclcpp/ParameterSet.h"
56 #include "fhiclcpp/ParameterSetID.h"
57 #include "fhiclcpp/ParameterSetRegistry.h"
58 #define TRACE_NAME "RootDAQOutFile"
59 #include "trace.h" // TRACE
60 
61 #include <algorithm>
62 #include <iomanip>
63 #include <sstream>
64 #include <utility>
65 #include <utility>
66 #include <vector>
67 
68 using namespace cet;
69 using namespace std;
70 using art::BranchType;
72 using art::rootNames::metaBranchRootName;
73 
74 namespace {
75 
76  void create_table(sqlite3* const db,
77  std::string const& name,
78  std::vector<std::string> const& columns,
79  std::string const& suffix = {})
80  {
81  if (columns.empty())
82  throw art::Exception(art::errors::LogicError)
83  << "Number of sqlite columns specified for table: "
84  << name << '\n'
85  << "is zero.\n";
86 
87  sqlite::Transaction txn{ db };
88  art::SQLErrMsg errMsg;
89  std::string ddl =
90  "DROP TABLE IF EXISTS " + name + "; "
91  "CREATE TABLE " + name +
92  "(" + columns.front();
93  std::for_each(columns.begin() + 1, columns.end(),
94  [&ddl](auto const& col)
95  {
96  ddl += "," + col;
97  });
98  ddl += ") ";
99  ddl += suffix;
100  ddl += ";";
101  sqlite3_exec(db, ddl.c_str(), nullptr, nullptr, errMsg);
102  errMsg.throwIfError();
103  txn.commit();
104  }
105 
106  void
107  insert_eventRanges_row(sqlite3_stmt* stmt,
108  art::SubRunNumber_t const sr,
109  art::EventNumber_t const b,
110  art::EventNumber_t const e)
111  {
112  sqlite3_bind_int64(stmt, 1, sr);
113  sqlite3_bind_int64(stmt, 2, b);
114  sqlite3_bind_int64(stmt, 3, e);
115  sqlite3_step(stmt);
116  sqlite3_reset(stmt);
117  }
118 
119  void
120  insert_rangeSets_row(sqlite3_stmt* stmt,
121  art::RunNumber_t const r)
122  {
123  sqlite3_bind_int64(stmt, 1, r);
124  sqlite3_step(stmt);
125  sqlite3_reset(stmt);
126  }
127 
128  void
129  insert_rangeSets_eventSets_row(sqlite3_stmt* stmt,
130  unsigned const rsid,
131  unsigned const esid)
132  {
133  sqlite3_bind_int64(stmt, 1, rsid);
134  sqlite3_bind_int64(stmt, 2, esid);
135  sqlite3_step(stmt);
136  sqlite3_reset(stmt);
137  }
138 
139  int
140  found_rowid(sqlite3_stmt* stmt)
141  {
142  return sqlite3_step(stmt) == SQLITE_ROW ?
143  sqlite3_column_int64(stmt, 0) :
144  throw art::Exception(art::errors::SQLExecutionError)
145  << "ROWID not found for EventRanges.\n"
146  << "Contact artists@fnal.gov.\n";
147  }
148 
149  unsigned
150  getNewRangeSetID(sqlite3* db,
151  art::BranchType const bt,
152  art::RunNumber_t const r)
153  {
154  sqlite::Transaction txn{ db };
155  sqlite3_stmt* stmt{ nullptr };
156  std::string const ddl{ "INSERT INTO " + art::BranchTypeToString(bt) + "RangeSets(Run) VALUES(?);" };
157  sqlite3_prepare_v2(db, ddl.c_str(), -1, &stmt, nullptr);
158  insert_rangeSets_row(stmt, r);
159  unsigned const rsID = sqlite3_last_insert_rowid(db);
160  sqlite3_finalize(stmt);
161  txn.commit();
162  return rsID;
163  }
164 
165  vector<unsigned>
166  getExistingRangeSetIDs(sqlite3* db, art::RangeSet const& rs)
167  {
168  vector<unsigned> rangeSetIDs;
169  for (auto const& range : rs)
170  {
171  sqlite::Transaction txn{ db };
172  sqlite3_stmt* stmt{ nullptr };
173  std::string const ddl{ "SELECT ROWID FROM EventRanges WHERE "
174  "SubRun=" + std::to_string(range.subRun()) + " AND "
175  "begin=" + std::to_string(range.begin()) + " AND "
176  "end=" + std::to_string(range.end()) + ";" };
177  sqlite3_prepare_v2(db, ddl.c_str(), -1, &stmt, nullptr);
178  rangeSetIDs.push_back(found_rowid(stmt));
179  sqlite3_finalize(stmt);
180  txn.commit();
181  }
182  return rangeSetIDs;
183  }
184 
185  void
186  insertIntoEventRanges(sqlite3* db, art::RangeSet const& rs)
187  {
188  sqlite::Transaction txn{ db };
189  sqlite3_stmt* stmt{ nullptr };
190  std::string const ddl{ "INSERT INTO EventRanges(SubRun, begin, end) "
191  "VALUES(?, ?, ?);" };
192  sqlite3_prepare_v2(db, ddl.c_str(), -1, &stmt, nullptr);
193  for (auto const& range : rs)
194  {
195  insert_eventRanges_row(stmt, range.subRun(), range.begin(), range.end());
196  }
197  sqlite3_finalize(stmt);
198  txn.commit();
199  }
200 
201  void
202  insertIntoJoinTable(sqlite3* db,
203  art::BranchType const bt,
204  unsigned const rsID,
205  vector<unsigned> const& eventRangesIDs)
206  {
207  sqlite::Transaction txn{ db };
208  sqlite3_stmt* stmt{ nullptr };
209  std::string const ddl{ "INSERT INTO " + art::BranchTypeToString(bt) +
210  "RangeSets_EventRanges(RangeSetsID, EventRangesID) Values(?,?);" };
211  sqlite3_prepare_v2(db, ddl.c_str(), -1, &stmt, nullptr);
212  cet::for_all(eventRangesIDs,
213  [stmt, rsID](auto const eventRangeID)
214  {
215  insert_rangeSets_eventSets_row(stmt, rsID, eventRangeID);
216  });
217  sqlite3_finalize(stmt);
218  txn.commit();
219  }
220 
221  void
222  maybeInvalidateRangeSet(BranchType const bt,
223  art::RangeSet const& principalRS,
224  art::RangeSet& productRS)
225  {
226  if (!productRS.is_valid())
227  return;
228 
229  assert(principalRS.is_sorted());
230  assert(productRS.is_sorted());
231 
232  if (bt == art::InRun && productRS.is_full_run()) return;
233  if (bt == art::InSubRun && productRS.is_full_subRun()) return;
234  assert(!productRS.ranges().empty());
235 
236  auto const r = productRS.run();
237  auto const& productFront = productRS.ranges().front();
238  if (!principalRS.contains(r, productFront.subRun(), productFront.begin()))
239  productRS = art::RangeSet::invalid();
240  }
241 
242  using art::detail::RangeSetsSupported;
243 
244  // The purpose of 'maybeInvalidateRangeSet' is to support the
245  // following situation. Suppose process 1 creates three files with
246  // one Run product each, all corresponding to the same Run. Let's
247  // call the individual Run product instances in the three separate
248  // files as A, B, and C. Now suppose that the three files serve as
249  // inputs to process 2, where a concatenation is being performed AND
250  // ALSO an output file switch. Process 2 results in two output
251  // files, and now, in process 3, we concatenate the outputs from
252  // process 2. The situation would look like this:
253  //
254  // Process 1: [A] [B] [C]
255  // \ / \ /
256  // Process 2: [A + B] [B + C]
257  // \ / \ /
258  // D=agg(A,B) | | E=agg(B,C)
259  // \ /
260  // Process 3: [D + E]
261  //
262  // Notice the complication in process 3: product 'B' will be
263  // aggregated twice: once with A, and once with C. Whenever the
264  // output from process 3 is read as input to another process, the
265  // fetched product will be equivalent to A+2B+C.
266  //
267  // To avoid this situation, we compare the RangeSet of the product
268  // with the RangeSet of the in-memory RunAuxiliary. If the
269  // beginning of B's RangeSet is not contained within the auxiliary's
270  // RangeSet, then a dummy product with an invalid RangeSet is
271  // written to disk. Instead of the diagram above, we have:
272  //
273  // Process 1: [A] [B] [C]
274  // \ / \ /
275  // Process 2: [A + B] [x + C]
276  // \ / \ /
277  // D=agg(A,B) | | E=agg(x,C)=C
278  // \ /
279  // Process 3: [D + E]
280  //
281  // where 'x' represent a dummy product. Upon aggregating D and E,
282  // we obtain the correctly formed A+B+C product.
283  template <BranchType BT>
284  std::enable_if_t<RangeSetsSupported<BT>::value, art::RangeSet>
285  getRangeSet(art::OutputHandle const& oh,
286  art::RangeSet const& principalRS,
287  bool const producedInThisProcess)
288  {
289  auto rs = oh.isValid() ? oh.rangeOfValidity() : art::RangeSet::invalid();
290  // Because a user can specify (e.g.):
291  // r.put(std::move(myProd), art::runFragment(myRangeSet));
292  // products that are produced in this process can have valid, yet
293  // arbitrary RangeSets. We therefore never invalidate a RangeSet
294  // that corresponds to a product produced in this process.
295  //
296  // It is possible for a user to specify a RangeSet which does not
297  // correspond AT ALL to the in-memory auxiliary RangeSet. In that
298  // case, users should not expect to be able to retrieve products
299  // for which no corresponding events or sub-runs were processed.
300  if (!producedInThisProcess)
301  {
302  maybeInvalidateRangeSet(BT, principalRS, rs);
303  }
304  return rs;
305  }
306 
307  template <BranchType BT>
308  std::enable_if_t<!RangeSetsSupported<BT>::value, art::RangeSet>
309  getRangeSet(art::OutputHandle const&,
310  art::RangeSet const& /*principalRS*/,
311  bool const /*producedInThisProcess*/)
312  {
313  return art::RangeSet::invalid();
314  }
315 
316  template <BranchType BT>
317  std::enable_if_t<!RangeSetsSupported<BT>::value>
318  setProductRangeSetID(art::RangeSet const& /*rs*/,
319  sqlite3*,
320  art::EDProduct*,
321  std::map<unsigned, unsigned>& /*checksumToIndexLookup*/)
322  {}
323 
324  template <BranchType BT>
325  std::enable_if_t<RangeSetsSupported<BT>::value>
326  setProductRangeSetID(art::RangeSet const& rs,
327  sqlite3* db,
328  art::EDProduct* product,
329  std::map<unsigned, unsigned>& checksumToIndexLookup)
330  {
331  if (!rs.is_valid()) // Invalid range-sets not written to DB
332  return;
333 
334  // Set range sets for SubRun and Run products
335  auto it = checksumToIndexLookup.find(rs.checksum());
336  if (it != checksumToIndexLookup.cend())
337  {
338  product->setRangeSetID(it->second);
339  }
340  else
341  {
342  unsigned const rsID = getNewRangeSetID(db, BT, rs.run());
343  product->setRangeSetID(rsID);
344  checksumToIndexLookup.emplace(rs.checksum(), rsID);
345  insertIntoEventRanges(db, rs);
346  auto const& eventRangesIDs = getExistingRangeSetIDs(db, rs);
347  insertIntoJoinTable(db, BT, rsID, eventRangesIDs);
348  }
349  }
350 
351 } // unnamed namespace
352 
353 art::
354 RootDAQOutFile::
355 RootDAQOutFile(OutputModule* om,
356  string const& fileName,
357  ClosingCriteria const& fileSwitchCriteria,
358  int const compressionLevel,
359  int64_t const saveMemoryObjectThreshold,
360  int64_t const treeMaxVirtualSize,
361  int const splitLevel,
362  int const basketSize,
363  DropMetaData dropMetaData,
364  bool const dropMetaDataForDroppedData,
365  bool const fastCloning)
366  : om_{ om }
367  , file_{ fileName }
368  , fileSwitchCriteria_{ fileSwitchCriteria }
369  , compressionLevel_{ compressionLevel }
370  , saveMemoryObjectThreshold_{ saveMemoryObjectThreshold }
371  , treeMaxVirtualSize_{ treeMaxVirtualSize }
372  , splitLevel_{ splitLevel }
373  , basketSize_{ basketSize }
374  , dropMetaData_{ dropMetaData }
375  , dropMetaDataForDroppedData_{ dropMetaDataForDroppedData }
376  , fastCloning_{ fastCloning }
377  , filePtr_{ TFile::Open(file_.c_str(), "recreate", "", compressionLevel) }
378  , treePointers_{ // Order (and number) must match BranchTypes.h!
379  std::make_unique<RootOutputTree>(
380  filePtr_.get(), InEvent, pEventAux_,
381  pEventProductProvenanceVector_, basketSize, splitLevel,
382  treeMaxVirtualSize, saveMemoryObjectThreshold),
383  std::make_unique<RootOutputTree>(
384  filePtr_.get(), InSubRun, pSubRunAux_,
385  pSubRunProductProvenanceVector_, basketSize, splitLevel,
386  treeMaxVirtualSize, saveMemoryObjectThreshold),
387  std::make_unique<RootOutputTree>(
388  filePtr_.get(), InRun, pRunAux_,
389  pRunProductProvenanceVector_, basketSize, splitLevel,
390  treeMaxVirtualSize, saveMemoryObjectThreshold),
391  std::make_unique<RootOutputTree>(
392  filePtr_.get(), InResults, pResultsAux_,
393  pResultsProductProvenanceVector_, basketSize, splitLevel,
394  treeMaxVirtualSize, saveMemoryObjectThreshold) }
395  , rootFileDB_{ ServiceHandle<DatabaseConnection>{}->get<TKeyVFSOpenPolicy>("RootFileDB",
396  filePtr_.get(),
397  SQLITE_OPEN_CREATE | SQLITE_OPEN_READWRITE) }
398 {
399  // Don't split metadata tree or event description tree
400  metaDataTree_ = RootOutputTree::makeTTree(filePtr_.get(), rootNames::metaDataTreeName(), 0);
401  fileIndexTree_ = RootOutputTree::makeTTree(filePtr_.get(), rootNames::fileIndexTreeName(), 0);
402  parentageTree_ = RootOutputTree::makeTTree(filePtr_.get(), rootNames::parentageTreeName(), 0);
403  // Create the tree that will carry (event) History objects.
404  eventHistoryTree_ = RootOutputTree::makeTTree(filePtr_.get(), rootNames::eventHistoryTreeName(), splitLevel);
405  if (!eventHistoryTree_)
406  {
407  throw art::Exception(art::errors::FatalRootError)
408  << "Failed to create the tree for History objects\n";
409  }
410 
411 
412  pHistory_ = new History;
413  if (!eventHistoryTree_->Branch(rootNames::eventHistoryBranchName().c_str(), &pHistory_, basketSize, 0))
414  {
415  throw art::Exception(art::errors::FatalRootError)
416  << "Failed to create a branch for History in the output file\n";
417  }
418  delete pHistory_;
419  pHistory_ = nullptr;
420 
421  createDatabaseTables();
422 }
423 
424 art::
425 RootDAQOutFile::OutputItem::Sorter::
426 Sorter(TTree* tree)
427 {
428  // Fill a map mapping branch names to an index specifying
429  // the order in the tree.
430  if (!tree)
431  {
432  return;
433  }
434  auto branches = tree->GetListOfBranches();
435  for (int i = 0, sz = branches->GetEntries(); i != sz; ++i)
436  {
437  auto br = reinterpret_cast<TBranchElement*>(branches->At(i));
438  treeMap_.emplace(string(br->GetName()), i);
439  }
440 }
441 
442 bool
443 art::
444 RootDAQOutFile::OutputItem::Sorter::
445 operator()(OutputItem const& lh, OutputItem const& rh) const
446 {
447  // Provides a comparison for sorting branches according
448  // to the index values in treeMap_. Branches not found
449  // are always put at the end, i.e. <not-found> is greater
450  // than <found>.
451  if (treeMap_.empty())
452  {
453  return lh < rh;
454  }
455  auto const& lname = lh.branchDescription_->branchName();
456  auto const& rname = rh.branchDescription_->branchName();
457  auto lit = treeMap_.find(lname);
458  auto rit = treeMap_.find(rname);
459  bool lfound = (lit != treeMap_.end());
460  bool rfound = (rit != treeMap_.end());
461  if (lfound && rfound)
462  {
463  return lit->second < rit->second;
464  }
465  if (lfound)
466  {
467  return true;
468  }
469  if (rfound)
470  {
471  return false;
472  }
473  return lh < rh;
474 }
475 
476 void
477 art::RootDAQOutFile::createDatabaseTables()
478 {
479 
480  // // FileCatalog metadata
481  // create_table(rootFileDB_, "FileCatalog_metadata",
482  // {"ID INTEGER PRIMARY KEY", "Name", "Value"});
483 
484  // Event ranges
485  create_table(rootFileDB_, "EventRanges",
486  { "SubRun INTEGER", "begin INTEGER", "end INTEGER", "UNIQUE (SubRun,begin,end) ON CONFLICT IGNORE" });
487 
488  // SubRun range sets
489  create_table(rootFileDB_, "SubRunRangeSets",
490  { "Run INTEGER" });
491  create_table(rootFileDB_, "SubRunRangeSets_EventRanges",
492  { "RangeSetsID INTEGER", "EventRangesID INTEGER", "PRIMARY KEY(RangeSetsID,EventRangesID)" },
493  "WITHOUT ROWID");
494 
495  // Run range sets
496  create_table(rootFileDB_, "RunRangeSets",
497  { "Run INTEGER" });
498  create_table(rootFileDB_, "RunRangeSets_EventRanges",
499  { "RangeSetsID INTEGER", "EventRangesID INTEGER", "PRIMARY KEY(RangeSetsID,EventRangesID)" },
500  "WITHOUT ROWID");
501 }
502 
503 void
504 art::
505 RootDAQOutFile::
506 selectProducts(FileBlock const& fb)
507 {
508  for (int i = InEvent; i < NumBranchTypes; ++i)
509  {
510  auto bt = static_cast<BranchType>(i);
511  auto& items = selectedOutputItemList_[bt];
512  for (auto const& val : items)
513  {
514  treePointers_[bt]->resetOutputBranchAddress(*val.branchDescription_);
515  }
516  items.clear();
517  for (auto const& bd : om_->keptProducts()[bt])
518  {
519  if ((bt < InResults) || bd->produced())
520  {
521  items.emplace_back(bd);
522  }
523  }
524  if ((bt == InEvent) && (fb.tree() != nullptr))
525  {
526  // Only care about sorting event tree because that's the only one
527  // we might fast clone.
528  sort(items.begin(), items.end(), OutputItem::Sorter(fb.tree()));
529  }
530  for (auto const& val : items)
531  {
532  treePointers_[bt]->addOutputBranch(*val.branchDescription_, val.product_);
533  }
534  }
535 }
536 
537 void
538 art::
539 RootDAQOutFile::
540 beginInputFile(FileBlock const& fb, bool fastClone)
541 {
542  selectProducts(fb);
543  auto const origCurrentlyFastCloning = currentlyFastCloning_;
544  currentlyFastCloning_ = fastCloning_ && fastClone;
545  if (currentlyFastCloning_ &&
546  !treePointers_[InEvent]->checkSplitLevelAndBasketSize(fb.tree()))
547  {
548  mf::LogWarning("FastCloning")
549  << "Fast cloning deactivated for this input file due to "
550  << "splitting level and/or basket size.";
551  currentlyFastCloning_ = false;
552  }
553  else if (fb.tree() && fb.tree()->GetCurrentFile()->GetVersion() < 60001)
554  {
555  mf::LogWarning("FastCloning")
556  << "Fast cloning deactivated for this input file due to "
557  << "ROOT version used to write it (< 6.00/01)\n"
558  "having a different splitting policy.";
559  currentlyFastCloning_ = false;
560  }
561 
562  if (currentlyFastCloning_ && fb.fileFormatVersion().value_ < 9)
563  {
564  mf::LogWarning("FastCloning")
565  << "Fast cloning deactivated for this input file due to "
566  << "reading in file that does not support RangeSets.";
567  currentlyFastCloning_ = false;
568  }
569 
570  if (currentlyFastCloning_ && !origCurrentlyFastCloning)
571  {
572  mf::LogWarning("FastCloning")
573  << "Fast cloning reactivated for this input file.";
574  }
575  treePointers_[InEvent]->beginInputFile(currentlyFastCloning_);
576  treePointers_[InEvent]->fastCloneTree(fb.tree());
577 }
578 
579 void
580 art::RootDAQOutFile::incrementInputFileNumber()
581 {
582  fp_.update<Granularity::InputFile>();
583 }
584 
585 void
586 art::
587 RootDAQOutFile::
588 respondToCloseInputFile(FileBlock const&)
589 {
590  cet::for_all(treePointers_, [](auto const& p) { p->setEntries(); });
591 }
592 
593 bool
594 art::
595 RootDAQOutFile::
596 requestsToCloseFile()
597 {
598  using namespace std::chrono;
599  unsigned int constexpr oneK{ 1024u };
600  TLOG(10) << "RootDAQOutFile::requestsToCloseFile start";
601  unsigned sz = filePtr_->GetSize();
602  TLOG(10) << "RootDAQOutFile::requestsToCloseFile after filePtr_->GetSize()";
603  fp_.updateSize(sz / oneK);
604  fp_.updateAge(duration_cast<seconds>(steady_clock::now() - beginTime_));
605  bool ret = fileSwitchCriteria_.should_close(fp_);
606  TLOG(10) << "RootDAQOutFile::requestsToCloseFile done/return";
607  return ret;
608 }
609 
610 void art::RootDAQOutFile::writeOne(EventPrincipal const& e)
611 {
612  TLOG(10) << "RootDAQOutFile::writeOne begin";
613  // Auxiliary branch.
614  // Note: pEventAux_ must be set before calling fillBranches
615  // since it gets written out in that routine.
616  pEventAux_ = &e.aux();
617  // Because getting the data may cause an exception to be
618  // thrown we want to do that first before writing anything
619  // to the file about this event.
620  fillBranches<InEvent>(e, pEventProductProvenanceVector_);
621  // History branch.
622  History historyForOutput{ e.history() };
623  historyForOutput.addEventSelectionEntry(om_->selectorConfig());
624  pHistory_ = &historyForOutput;
625  int sz = eventHistoryTree_->Fill();
626  if (sz <= 0)
627  {
628  throw art::Exception(art::errors::FatalRootError)
629  << "Failed to fill the History tree for event: "
630  << e.id()
631  << "\nTTree::Fill() returned "
632  << sz
633  << " bytes written."
634  << endl;
635  }
636  // Add the dataType to the job report if it hasn't already been done
637  if (!dataTypeReported_)
638  {
639  string dataType{ "MC" };
640  if (pEventAux_->isRealData())
641  {
642  dataType = "Data";
643  }
644  dataTypeReported_ = true;
645  }
646  pHistory_ = &e.history();
647  // Add event to index
648  fileIndex_.addEntry(pEventAux_->id(), fp_.eventEntryNumber());
649  fp_.update<Granularity::Event>(status_);
650  TLOG(10) << "RootDAQOutFile::writeOne done/return";
651 }
652 
653 void
654 art::
655 RootDAQOutFile::
656 writeSubRun(SubRunPrincipal const& sr)
657 {
658  pSubRunAux_ = &sr.aux();
659  pSubRunAux_->setRangeSetID(subRunRSID_);
660  fillBranches<InSubRun>(sr, pSubRunProductProvenanceVector_);
661  fileIndex_.addEntry(EventID::invalidEvent(pSubRunAux_->id()), fp_.subRunEntryNumber());
662  fp_.update<Granularity::SubRun>(status_);
663 }
664 
665 void
666 art::
667 RootDAQOutFile::
668 writeRun(RunPrincipal const& r)
669 {
670  pRunAux_ = &r.aux();
671  pRunAux_->setRangeSetID(runRSID_);
672  fillBranches<InRun>(r, pRunProductProvenanceVector_);
673  fileIndex_.addEntry(EventID::invalidEvent(pRunAux_->id()), fp_.runEntryNumber());
674  fp_.update<Granularity::Run>(status_);
675 }
676 
677 void
678 art::
679 RootDAQOutFile::
680 writeParentageRegistry()
681 {
682  ParentageID const* hash = new ParentageID;
683  if (!parentageTree_->Branch(rootNames::parentageIDBranchName().c_str(),
684  &hash, basketSize_, 0))
685  {
686  throw art::Exception(art::errors::FatalRootError)
687  << "Failed to create a branch for ParentageIDs in the output file";
688  }
689  delete hash;
690  hash = nullptr;
691  Parentage const* desc = new Parentage;
692  if (!parentageTree_->Branch(rootNames::parentageBranchName().c_str(), &desc,
693  basketSize_, 0))
694  {
695  throw art::Exception(art::errors::FatalRootError)
696  << "Failed to create a branch for Parentages in the output file";
697  }
698  delete desc;
699  desc = nullptr;
700  for (auto const& pr : ParentageRegistry::get())
701  {
702  hash = &pr.first;
703  desc = &pr.second;
704  parentageTree_->Fill();
705  }
706  parentageTree_->SetBranchAddress(rootNames::parentageIDBranchName().c_str(), nullptr);
707  parentageTree_->SetBranchAddress(rootNames::parentageBranchName().c_str(), nullptr);
708 }
709 
710 void
711 art::
712 RootDAQOutFile::
713 writeFileFormatVersion()
714 {
715  FileFormatVersion ver(getFileFormatVersion(), getFileFormatEra());
716  FileFormatVersion* pver = &ver;
717  TBranch* b = metaDataTree_->Branch(metaBranchRootName<FileFormatVersion>(),
718  &pver, basketSize_, 0);
719  // FIXME: Turn this into a throw!
720  assert(b);
721  b->Fill();
722 }
723 
724 void
725 art::
726 RootDAQOutFile::
727 writeFileIndex()
728 {
729  fileIndex_.sortBy_Run_SubRun_Event();
730  FileIndex::Element elem{};
731  auto findexElemPtr = &elem;
732  TBranch* b = fileIndexTree_->Branch(metaBranchRootName<FileIndex::Element>(),
733  &findexElemPtr, basketSize_, 0);
734  // FIXME: Turn this into a throw!
735  assert(b);
736  for (auto& entry : fileIndex_)
737  {
738  findexElemPtr = &entry;
739  b->Fill();
740  }
741  b->SetAddress(0);
742 }
743 
744 void
745 art::
746 RootDAQOutFile::
747 writeEventHistory()
748 {
749  RootOutputTree::writeTTree(eventHistoryTree_);
750 }
751 
752 void
753 art::
754 RootDAQOutFile::
755 writeProcessConfigurationRegistry()
756 {
757  // We don't do this yet; currently we're storing a slightly
758  // bloated ProcessHistoryRegistry.
759 }
760 
761 void
762 art::
763 RootDAQOutFile::
764 writeProcessHistoryRegistry()
765 {
766  ProcessHistoryMap pHistMap;
767  for (auto const& pr : ProcessHistoryRegistry::get())
768  {
769  pHistMap.emplace(pr);
770  }
771  auto const* p = &pHistMap;
772  TBranch* b = metaDataTree_->Branch(metaBranchRootName<ProcessHistoryMap>(),
773  &p, basketSize_, 0);
774  if (b != nullptr)
775  {
776  b->Fill();
777  }
778  else
779  {
780  throw Exception(errors::LogicError)
781  << "Unable to locate required ProcessHistoryMap branch in output metadata tree.\n";
782  }
783 }
784 
785 void
786 art::
787 RootDAQOutFile::
788 writeBranchIDListRegistry()
789 {
790  BranchIDLists const* p = &BranchIDListRegistry::instance().data();
791  TBranch* b = metaDataTree_->Branch(metaBranchRootName<BranchIDLists>(), &p,
792  basketSize_, 0);
793  // FIXME: Turn this into a throw!
794  assert(b);
795  b->Fill();
796 }
797 
798 void
799 art::
800 RootDAQOutFile::
801 writeFileCatalogMetadata(FileStatsCollector const& stats,
802  FileCatalogMetadata::collection_type const& md,
803  FileCatalogMetadata::collection_type const& ssmd)
804 {
805  using namespace cet::sqlite;
806  Ntuple<std::string, std::string> fileCatalogMetadata{ rootFileDB_, "FileCatalog_metadata", {"Name","Value"}, true };
807  Transaction txn{ rootFileDB_ };
808  for (auto const& kv : md)
809  {
810  fileCatalogMetadata.insert(kv.first, kv.second);
811  }
812  // Add our own specific information: File format and friends.
813  fileCatalogMetadata.insert("file_format", "\"artroot\"");
814  fileCatalogMetadata.insert("file_format_era", cet::canonical_string(getFileFormatEra()));
815  fileCatalogMetadata.insert("file_format_version", to_string(getFileFormatVersion()));
816 
817  // File start time.
818  namespace bpt = boost::posix_time;
819  auto formatted_time = [](auto const& t) { return cet::canonical_string(bpt::to_iso_extended_string(t)); };
820  fileCatalogMetadata.insert("start_time", formatted_time(stats.outputFileOpenTime()));
821  // File "end" time: now, since file is not actually closed yet.
822  fileCatalogMetadata.insert("end_time", formatted_time(boost::posix_time::second_clock::universal_time()));
823 
824  // Run/subRun information.
825  if (!stats.seenSubRuns().empty())
826  {
827 
828  auto I = find_if(md.crbegin(), md.crend(),
829  [](auto const& p) { return p.first == "run_type"; });
830 
831  if (I != md.crend())
832  {
833  ostringstream buf;
834  buf << "[ ";
835  for (auto const& srid : stats.seenSubRuns())
836  {
837  buf << "[ "
838  << srid.run()
839  << ", "
840  << srid.subRun()
841  << ", "
842  << cet::canonical_string(I->second)
843  << " ], ";
844  }
845  // Rewind over last delimiter.
846  buf.seekp(-2, ios_base::cur);
847  buf << " ]";
848  fileCatalogMetadata.insert("runs", buf.str());
849  }
850  }
851  // Number of events.
852  fileCatalogMetadata.insert("event_count", to_string(stats.eventsThisFile()));
853  // first_event and last_event.
854  auto eidToTuple = [](EventID const & eid)->string
855  {
856  ostringstream eidStr;
857  eidStr << "[ "
858  << eid.run()
859  << ", "
860  << eid.subRun()
861  << ", "
862  << eid.event()
863  << " ]";
864  return eidStr.str();
865  };
866  fileCatalogMetadata.insert("first_event", eidToTuple(stats.lowestEventID()));
867  fileCatalogMetadata.insert("last_event", eidToTuple(stats.highestEventID()));
868  // File parents.
869  if (!stats.parents().empty())
870  {
871  ostringstream pstring;
872  pstring << "[ ";
873  for (auto const& parent : stats.parents())
874  {
875  pstring << cet::canonical_string(parent) << ", ";
876  }
877  // Rewind over last delimiter.
878  pstring.seekp(-2, ios_base::cur);
879  pstring << " ]";
880  fileCatalogMetadata.insert("parents", pstring.str());
881  }
882  // Incoming stream-specific metadata overrides.
883  for (auto const& kv : ssmd)
884  {
885  fileCatalogMetadata.insert(kv.first, kv.second);
886  }
887  txn.commit();
888 }
889 
890 void
891 art::
892 RootDAQOutFile::
893 writeParameterSetRegistry()
894 {
895  fhicl::ParameterSetRegistry::exportTo(rootFileDB_);
896 }
897 
898 void
899 art::
900 RootDAQOutFile::
901 writeProductDescriptionRegistry()
902 {
903  // Make a local copy of the MasterProductRegistry's ProductList,
904  // removing any transient or pruned products.
905  auto end = branchesWithStoredHistory_.end();
906 
907  ProductRegistry reg;
908  for (auto const& pr : ProductMetaData::instance().productList())
909  {
910  if (branchesWithStoredHistory_.find(pr.second.branchID()) == end)
911  {
912  continue;
913  }
914  reg.productList_.emplace_hint(reg.productList_.end(), pr);
915  }
916 
917  auto* regp = &reg;
918  TBranch* b = metaDataTree_->Branch(metaBranchRootName<ProductRegistry>(),
919  &regp, basketSize_, 0);
920  // FIXME: Turn this into a throw!
921  assert(b);
922  b->Fill();
923 }
924 
925 void
926 art::
927 RootDAQOutFile::
928 writeProductDependencies()
929 {
930  BranchChildren& pDeps = const_cast<BranchChildren&>(om_->branchChildren());
931  BranchChildren* ppDeps = &pDeps;
932  TBranch* b = metaDataTree_->Branch(metaBranchRootName<BranchChildren>(),
933  &ppDeps, basketSize_, 0);
934  // FIXME: Turn this into a throw!
935  assert(b);
936  b->Fill();
937 }
938 
939 void
940 art::
941 RootDAQOutFile::
942 writeResults(ResultsPrincipal & resp)
943 {
944  pResultsAux_ = &resp.aux();
945  fillBranches<InResults>(resp, pResultsProductProvenanceVector_);
946 }
947 
948 void
949 art::RootDAQOutFile::writeTTrees()
950 {
951  RootOutputTree::writeTTree(metaDataTree_);
952  RootOutputTree::writeTTree(fileIndexTree_);
953  RootOutputTree::writeTTree(parentageTree_);
954  // Write out the tree corresponding to each BranchType
955  for (int i = InEvent; i < NumBranchTypes; ++i)
956  {
957  auto const branchType = static_cast<BranchType>(i);
958  treePointers_[branchType]->writeTree();
959  }
960 }
961 
962 template <art::BranchType BT>
963 void art::RootDAQOutFile::fillBranches(Principal const& principal,
964  vector<ProductProvenance>* vpp)
965 {
966  TLOG(11) << "RootDAQOutFile::fillBranches begin";
967  bool const fastCloning = (BT == InEvent) && currentlyFastCloning_;
968  detail::KeptProvenance keptProvenance{ dropMetaData_, dropMetaDataForDroppedData_, branchesWithStoredHistory_ };
969  map<unsigned, unsigned> checksumToIndex;
970 
971  auto const& principalRS = principal.seenRanges();
972 
973  for (auto const& val : selectedOutputItemList_[BT])
974  {
975  auto const* bd = val.branchDescription_;
976  auto const bid = bd->branchID();
977  branchesWithStoredHistory_.insert(bid);
978  bool const produced{ bd->produced() };
979  bool const resolveProd = (produced || !fastCloning ||
980  treePointers_[BT]->uncloned(bd->branchName()));
981 
982  // Update the kept provenance
983  bool const keepProvenance = (dropMetaData_ == DropMetaData::DropNone ||
984  (dropMetaData_ == DropMetaData::DropPrior &&
985  produced));
986  auto const& oh = principal.getForOutput(bid, resolveProd);
987 
988  unique_ptr<ProductProvenance> prov{ nullptr };
989  if (keepProvenance)
990  {
991  if (oh.productProvenance())
992  {
993  prov = std::make_unique<ProductProvenance>(keptProvenance.insert(*oh.productProvenance()));
994  keptProvenance.insertAncestors(*oh.productProvenance(), principal);
995  }
996  else
997  {
998  // No provenance, product was either not produced,
999  // or was dropped, create provenance to remember that.
1000  auto const status = produced ? productstatus::neverCreated() : productstatus::dropped();
1001  prov = std::make_unique<ProductProvenance>(keptProvenance.emplace(bid, status));
1002  }
1003  }
1004 
1005  // Resolve the product if necessary
1006  if (resolveProd)
1007  {
1008  auto const& rs = getRangeSet<BT>(oh, principalRS, produced);
1009  if (RangeSetsSupported<BT>::value && !rs.is_valid())
1010  {
1011  // Unfortunately, 'unknown' is the only viable product status
1012  // when this condition is triggered (due to the assert
1013  // statement in ProductStatus::setNotPresent). Whenever the
1014  // metadata revolution comes, this should be revised.
1015  keptProvenance.setStatus(*prov, productstatus::unknown());
1016  }
1017 
1018  auto const* product = getProduct<BT>(oh, rs, bd->wrappedName());
1019  setProductRangeSetID<BT>(rs,
1020  rootFileDB_,
1021  const_cast<EDProduct*>(product),
1022  checksumToIndex);
1023  val.product_ = product;
1024  }
1025  }
1026  vpp->assign(keptProvenance.begin(), keptProvenance.end());
1027  treePointers_[BT]->fillTree(); // write happes here
1028  vpp->clear();
1029  TLOG(11) << "RootDAQOutFile::fillBranches done/return";
1030 }
1031 
1032 void
1033 art::
1034 RootDAQOutFile::
1035 setSubRunAuxiliaryRangeSetID(RangeSet const& ranges)
1036 {
1037  subRunRSID_ = getNewRangeSetID(rootFileDB_, InSubRun, ranges.run());
1038  insertIntoEventRanges(rootFileDB_, ranges);
1039  auto const& eventRangesIDs = getExistingRangeSetIDs(rootFileDB_, ranges);
1040  insertIntoJoinTable(rootFileDB_, InSubRun, subRunRSID_, eventRangesIDs);
1041 }
1042 
1043 void
1044 art::
1045 RootDAQOutFile::
1046 setRunAuxiliaryRangeSetID(RangeSet const& ranges)
1047 {
1048  runRSID_ = getNewRangeSetID(rootFileDB_, InRun, ranges.run());
1049  insertIntoEventRanges(rootFileDB_, ranges);
1050  auto const& eventRangesIDs = getExistingRangeSetIDs(rootFileDB_, ranges);
1051  insertIntoJoinTable(rootFileDB_, InRun, runRSID_, eventRangesIDs);
1052 }
1053 
1054 template <BranchType BT>
1055 std::enable_if_t<!RangeSetsSupported<BT>::value, art::EDProduct const*>
1056 art::RootDAQOutFile::getProduct(art::OutputHandle const& oh,
1057  art::RangeSet const& /*prunedProductRS*/,
1058  std::string const& wrappedName)
1059 {
1060  if (oh.isValid())
1061  return oh.wrapper();
1062  else
1063  return dummyProductCache_.product(wrappedName);
1064 }
1065 
1066 template <BranchType BT>
1067 std::enable_if_t<RangeSetsSupported<BT>::value, art::EDProduct const*>
1068 art::RootDAQOutFile::getProduct(art::OutputHandle const& oh,
1069  art::RangeSet const& prunedProductRS,
1070  std::string const& wrappedName)
1071 {
1072  if (oh.isValid() && prunedProductRS.is_valid())
1073  return oh.wrapper();
1074  else
1075  return dummyProductCache_.product(wrappedName);
1076 }
An output module which allows specifying the output filename exactly. Used for testing (file -&gt; /dev/...