1 #include "artdaq/ArtModules/RootDAQOutput/RootDAQOutFile.h"
5 #include "TBranchElement.h"
9 #include "art/Framework/Core/FileBlock.h"
10 # include "art/Framework/Core/OutputFileGranularity.h"
11 # include "art/Framework/Services/System/DatabaseConnection.h"
12 # include "art/Persistency/RootDB/TKeyVFSOpenPolicy.h"
13 # include "cetlib/sqlite/Ntuple.h"
14 # include "cetlib/sqlite/Transaction.h"
15 # include "cetlib/sqlite/create_table.h"
16 # include "cetlib/sqlite/exec.h"
17 # include "cetlib/sqlite/insert.h"
18 #include "art/Framework/IO/FileStatsCollector.h"
19 #include "art/Framework/IO/Root/DropMetaData.h"
20 #include "art/Framework/IO/Root/GetFileFormatEra.h"
21 #include "art/Framework/IO/Root/GetFileFormatVersion.h"
22 #include "artdaq/ArtModules/RootDAQOutput/detail/KeptProvenance.h"
23 #include "artdaq/ArtModules/RootDAQOutput/RootDAQOutFile.h"
24 #include "art/Framework/Principal/EventPrincipal.h"
25 #include "art/Framework/Principal/ResultsPrincipal.h"
26 #include "art/Framework/Principal/RunPrincipal.h"
27 #include "art/Framework/Principal/SubRunPrincipal.h"
28 #include "art/Persistency/Provenance/BranchIDListRegistry.h"
29 #include "art/Persistency/Provenance/ProcessHistoryRegistry.h"
30 #include "art/Persistency/Provenance/ProductMetaData.h"
31 #include "art/Persistency/RootDB/SQLErrMsg.h"
32 #include "art/Version/GetReleaseVersion.h"
33 #include "boost/date_time/posix_time/posix_time.hpp"
34 #include "canvas/Persistency/Provenance/rootNames.h"
35 #include "canvas/Persistency/Provenance/BranchChildren.h"
36 #include "canvas/Persistency/Provenance/BranchID.h"
37 #include "canvas/Persistency/Provenance/BranchIDList.h"
38 #include "canvas/Persistency/Provenance/BranchType.h"
39 #include "canvas/Persistency/Provenance/EventAuxiliary.h"
40 #include "canvas/Persistency/Provenance/EventID.h"
41 #include "canvas/Persistency/Provenance/FileFormatVersion.h"
42 #include "canvas/Persistency/Provenance/History.h"
43 #include "canvas/Persistency/Provenance/ParameterSetBlob.h"
44 #include "canvas/Persistency/Provenance/Parentage.h"
45 #include "canvas/Persistency/Provenance/ParentageRegistry.h"
46 #include "canvas/Persistency/Provenance/ProcessHistoryID.h"
47 #include "canvas/Persistency/Provenance/ProductStatus.h"
48 #include "canvas/Persistency/Provenance/ResultsAuxiliary.h"
49 #include "canvas/Persistency/Provenance/RunAuxiliary.h"
50 #include "canvas/Persistency/Provenance/SubRunAuxiliary.h"
51 #include "canvas/Utilities/Exception.h"
52 #include "cetlib/canonical_string.h"
53 #include "cetlib/container_algorithms.h"
54 #include "cetlib/exempt_ptr.h"
55 #include "fhiclcpp/ParameterSet.h"
56 #include "fhiclcpp/ParameterSetID.h"
57 #include "fhiclcpp/ParameterSetRegistry.h"
58 #define TRACE_NAME "RootDAQOutFile"
70 using art::BranchType;
72 using art::rootNames::metaBranchRootName;
76 void create_table(sqlite3*
const db,
77 std::string
const& name,
78 std::vector<std::string>
const& columns,
79 std::string
const& suffix = {})
82 throw art::Exception(art::errors::LogicError)
83 <<
"Number of sqlite columns specified for table: "
87 sqlite::Transaction txn{ db };
88 art::SQLErrMsg errMsg;
90 "DROP TABLE IF EXISTS " + name +
"; "
91 "CREATE TABLE " + name +
92 "(" + columns.front();
93 std::for_each(columns.begin() + 1, columns.end(),
94 [&ddl](
auto const& col)
101 sqlite3_exec(db, ddl.c_str(),
nullptr,
nullptr, errMsg);
102 errMsg.throwIfError();
107 insert_eventRanges_row(sqlite3_stmt* stmt,
108 art::SubRunNumber_t
const sr,
109 art::EventNumber_t
const b,
110 art::EventNumber_t
const e)
112 sqlite3_bind_int64(stmt, 1, sr);
113 sqlite3_bind_int64(stmt, 2, b);
114 sqlite3_bind_int64(stmt, 3, e);
120 insert_rangeSets_row(sqlite3_stmt* stmt,
121 art::RunNumber_t
const r)
123 sqlite3_bind_int64(stmt, 1, r);
129 insert_rangeSets_eventSets_row(sqlite3_stmt* stmt,
133 sqlite3_bind_int64(stmt, 1, rsid);
134 sqlite3_bind_int64(stmt, 2, esid);
140 found_rowid(sqlite3_stmt* stmt)
142 return sqlite3_step(stmt) == SQLITE_ROW ?
143 sqlite3_column_int64(stmt, 0) :
144 throw art::Exception(art::errors::SQLExecutionError)
145 <<
"ROWID not found for EventRanges.\n"
146 <<
"Contact artists@fnal.gov.\n";
150 getNewRangeSetID(sqlite3* db,
151 art::BranchType
const bt,
152 art::RunNumber_t
const r)
154 sqlite::Transaction txn{ db };
155 sqlite3_stmt* stmt{
nullptr };
156 std::string
const ddl{
"INSERT INTO " + art::BranchTypeToString(bt) +
"RangeSets(Run) VALUES(?);" };
157 sqlite3_prepare_v2(db, ddl.c_str(), -1, &stmt,
nullptr);
158 insert_rangeSets_row(stmt, r);
159 unsigned const rsID = sqlite3_last_insert_rowid(db);
160 sqlite3_finalize(stmt);
166 getExistingRangeSetIDs(sqlite3* db, art::RangeSet
const& rs)
168 vector<unsigned> rangeSetIDs;
169 for (
auto const& range : rs)
171 sqlite::Transaction txn{ db };
172 sqlite3_stmt* stmt{
nullptr };
173 std::string
const ddl{
"SELECT ROWID FROM EventRanges WHERE "
174 "SubRun=" + std::to_string(range.subRun()) +
" AND "
175 "begin=" + std::to_string(range.begin()) +
" AND "
176 "end=" + std::to_string(range.end()) +
";" };
177 sqlite3_prepare_v2(db, ddl.c_str(), -1, &stmt,
nullptr);
178 rangeSetIDs.push_back(found_rowid(stmt));
179 sqlite3_finalize(stmt);
186 insertIntoEventRanges(sqlite3* db, art::RangeSet
const& rs)
188 sqlite::Transaction txn{ db };
189 sqlite3_stmt* stmt{
nullptr };
190 std::string
const ddl{
"INSERT INTO EventRanges(SubRun, begin, end) "
191 "VALUES(?, ?, ?);" };
192 sqlite3_prepare_v2(db, ddl.c_str(), -1, &stmt,
nullptr);
193 for (
auto const& range : rs)
195 insert_eventRanges_row(stmt, range.subRun(), range.begin(), range.end());
197 sqlite3_finalize(stmt);
202 insertIntoJoinTable(sqlite3* db,
203 art::BranchType
const bt,
205 vector<unsigned>
const& eventRangesIDs)
207 sqlite::Transaction txn{ db };
208 sqlite3_stmt* stmt{
nullptr };
209 std::string
const ddl{
"INSERT INTO " + art::BranchTypeToString(bt) +
210 "RangeSets_EventRanges(RangeSetsID, EventRangesID) Values(?,?);" };
211 sqlite3_prepare_v2(db, ddl.c_str(), -1, &stmt,
nullptr);
212 cet::for_all(eventRangesIDs,
213 [stmt, rsID](
auto const eventRangeID)
215 insert_rangeSets_eventSets_row(stmt, rsID, eventRangeID);
217 sqlite3_finalize(stmt);
222 maybeInvalidateRangeSet(BranchType
const bt,
223 art::RangeSet
const& principalRS,
224 art::RangeSet& productRS)
226 if (!productRS.is_valid())
229 assert(principalRS.is_sorted());
230 assert(productRS.is_sorted());
232 if (bt == art::InRun && productRS.is_full_run())
return;
233 if (bt == art::InSubRun && productRS.is_full_subRun())
return;
234 assert(!productRS.ranges().empty());
236 auto const r = productRS.run();
237 auto const& productFront = productRS.ranges().front();
238 if (!principalRS.contains(r, productFront.subRun(), productFront.begin()))
239 productRS = art::RangeSet::invalid();
242 using art::detail::RangeSetsSupported;
283 template <BranchType BT>
284 std::enable_if_t<RangeSetsSupported<BT>::value, art::RangeSet>
285 getRangeSet(art::OutputHandle
const& oh,
286 art::RangeSet
const& principalRS,
287 bool const producedInThisProcess)
289 auto rs = oh.isValid() ? oh.rangeOfValidity() : art::RangeSet::invalid();
300 if (!producedInThisProcess)
302 maybeInvalidateRangeSet(BT, principalRS, rs);
307 template <BranchType BT>
308 std::enable_if_t<!RangeSetsSupported<BT>::value, art::RangeSet>
309 getRangeSet(art::OutputHandle
const&,
310 art::RangeSet
const& ,
313 return art::RangeSet::invalid();
316 template <BranchType BT>
317 std::enable_if_t<!RangeSetsSupported<BT>::value>
318 setProductRangeSetID(art::RangeSet
const& ,
321 std::map<unsigned, unsigned>& )
324 template <BranchType BT>
325 std::enable_if_t<RangeSetsSupported<BT>::value>
326 setProductRangeSetID(art::RangeSet
const& rs,
328 art::EDProduct* product,
329 std::map<unsigned, unsigned>& checksumToIndexLookup)
335 auto it = checksumToIndexLookup.find(rs.checksum());
336 if (it != checksumToIndexLookup.cend())
338 product->setRangeSetID(it->second);
342 unsigned const rsID = getNewRangeSetID(db, BT, rs.run());
343 product->setRangeSetID(rsID);
344 checksumToIndexLookup.emplace(rs.checksum(), rsID);
345 insertIntoEventRanges(db, rs);
346 auto const& eventRangesIDs = getExistingRangeSetIDs(db, rs);
347 insertIntoJoinTable(db, BT, rsID, eventRangesIDs);
355 RootDAQOutFile(OutputModule* om,
356 string const& fileName,
357 ClosingCriteria
const& fileSwitchCriteria,
358 int const compressionLevel,
359 int64_t
const saveMemoryObjectThreshold,
360 int64_t
const treeMaxVirtualSize,
361 int const splitLevel,
362 int const basketSize,
363 DropMetaData dropMetaData,
364 bool const dropMetaDataForDroppedData,
365 bool const fastCloning)
368 , fileSwitchCriteria_{ fileSwitchCriteria }
369 , compressionLevel_{ compressionLevel }
370 , saveMemoryObjectThreshold_{ saveMemoryObjectThreshold }
371 , treeMaxVirtualSize_{ treeMaxVirtualSize }
372 , splitLevel_{ splitLevel }
373 , basketSize_{ basketSize }
374 , dropMetaData_{ dropMetaData }
375 , dropMetaDataForDroppedData_{ dropMetaDataForDroppedData }
376 , fastCloning_{ fastCloning }
377 , filePtr_{ TFile::Open(file_.c_str(),
"recreate",
"", compressionLevel) }
379 std::make_unique<RootOutputTree>(
380 filePtr_.get(), InEvent, pEventAux_,
381 pEventProductProvenanceVector_, basketSize, splitLevel,
382 treeMaxVirtualSize, saveMemoryObjectThreshold),
383 std::make_unique<RootOutputTree>(
384 filePtr_.get(), InSubRun, pSubRunAux_,
385 pSubRunProductProvenanceVector_, basketSize, splitLevel,
386 treeMaxVirtualSize, saveMemoryObjectThreshold),
387 std::make_unique<RootOutputTree>(
388 filePtr_.get(), InRun, pRunAux_,
389 pRunProductProvenanceVector_, basketSize, splitLevel,
390 treeMaxVirtualSize, saveMemoryObjectThreshold),
391 std::make_unique<RootOutputTree>(
392 filePtr_.get(), InResults, pResultsAux_,
393 pResultsProductProvenanceVector_, basketSize, splitLevel,
394 treeMaxVirtualSize, saveMemoryObjectThreshold) }
395 , rootFileDB_{ ServiceHandle<DatabaseConnection>{}->get<TKeyVFSOpenPolicy>(
"RootFileDB",
397 SQLITE_OPEN_CREATE | SQLITE_OPEN_READWRITE) }
400 metaDataTree_ = RootOutputTree::makeTTree(filePtr_.get(), rootNames::metaDataTreeName(), 0);
401 fileIndexTree_ = RootOutputTree::makeTTree(filePtr_.get(), rootNames::fileIndexTreeName(), 0);
402 parentageTree_ = RootOutputTree::makeTTree(filePtr_.get(), rootNames::parentageTreeName(), 0);
404 eventHistoryTree_ = RootOutputTree::makeTTree(filePtr_.get(), rootNames::eventHistoryTreeName(), splitLevel);
405 if (!eventHistoryTree_)
407 throw art::Exception(art::errors::FatalRootError)
408 <<
"Failed to create the tree for History objects\n";
412 pHistory_ =
new History;
413 if (!eventHistoryTree_->Branch(rootNames::eventHistoryBranchName().c_str(), &pHistory_, basketSize, 0))
415 throw art::Exception(art::errors::FatalRootError)
416 <<
"Failed to create a branch for History in the output file\n";
421 createDatabaseTables();
425 RootDAQOutFile::OutputItem::Sorter::
434 auto branches = tree->GetListOfBranches();
435 for (
int i = 0, sz = branches->GetEntries(); i != sz; ++i)
437 auto br =
reinterpret_cast<TBranchElement*
>(branches->At(i));
438 treeMap_.emplace(
string(br->GetName()), i);
444 RootDAQOutFile::OutputItem::Sorter::
445 operator()(OutputItem
const& lh, OutputItem
const& rh)
const
451 if (treeMap_.empty())
455 auto const& lname = lh.branchDescription_->branchName();
456 auto const& rname = rh.branchDescription_->branchName();
457 auto lit = treeMap_.find(lname);
458 auto rit = treeMap_.find(rname);
459 bool lfound = (lit != treeMap_.end());
460 bool rfound = (rit != treeMap_.end());
461 if (lfound && rfound)
463 return lit->second < rit->second;
477 art::RootDAQOutFile::createDatabaseTables()
485 create_table(rootFileDB_,
"EventRanges",
486 {
"SubRun INTEGER",
"begin INTEGER",
"end INTEGER",
"UNIQUE (SubRun,begin,end) ON CONFLICT IGNORE" });
489 create_table(rootFileDB_,
"SubRunRangeSets",
491 create_table(rootFileDB_,
"SubRunRangeSets_EventRanges",
492 {
"RangeSetsID INTEGER",
"EventRangesID INTEGER",
"PRIMARY KEY(RangeSetsID,EventRangesID)" },
496 create_table(rootFileDB_,
"RunRangeSets",
498 create_table(rootFileDB_,
"RunRangeSets_EventRanges",
499 {
"RangeSetsID INTEGER",
"EventRangesID INTEGER",
"PRIMARY KEY(RangeSetsID,EventRangesID)" },
506 selectProducts(FileBlock
const& fb)
508 for (
int i = InEvent; i < NumBranchTypes; ++i)
510 auto bt =
static_cast<BranchType
>(i);
511 auto& items = selectedOutputItemList_[bt];
512 for (
auto const& val : items)
514 treePointers_[bt]->resetOutputBranchAddress(*val.branchDescription_);
517 for (
auto const& bd : om_->keptProducts()[bt])
519 if ((bt < InResults) || bd->produced())
521 items.emplace_back(bd);
524 if ((bt == InEvent) && (fb.tree() !=
nullptr))
528 sort(items.begin(), items.end(), OutputItem::Sorter(fb.tree()));
530 for (
auto const& val : items)
532 treePointers_[bt]->addOutputBranch(*val.branchDescription_, val.product_);
540 beginInputFile(FileBlock
const& fb,
bool fastClone)
543 auto const origCurrentlyFastCloning = currentlyFastCloning_;
544 currentlyFastCloning_ = fastCloning_ && fastClone;
545 if (currentlyFastCloning_ &&
546 !treePointers_[InEvent]->checkSplitLevelAndBasketSize(fb.tree()))
548 mf::LogWarning(
"FastCloning")
549 <<
"Fast cloning deactivated for this input file due to "
550 <<
"splitting level and/or basket size.";
551 currentlyFastCloning_ =
false;
553 else if (fb.tree() && fb.tree()->GetCurrentFile()->GetVersion() < 60001)
555 mf::LogWarning(
"FastCloning")
556 <<
"Fast cloning deactivated for this input file due to "
557 <<
"ROOT version used to write it (< 6.00/01)\n"
558 "having a different splitting policy.";
559 currentlyFastCloning_ =
false;
562 if (currentlyFastCloning_ && fb.fileFormatVersion().value_ < 9)
564 mf::LogWarning(
"FastCloning")
565 <<
"Fast cloning deactivated for this input file due to "
566 <<
"reading in file that does not support RangeSets.";
567 currentlyFastCloning_ =
false;
570 if (currentlyFastCloning_ && !origCurrentlyFastCloning)
572 mf::LogWarning(
"FastCloning")
573 <<
"Fast cloning reactivated for this input file.";
575 treePointers_[InEvent]->beginInputFile(currentlyFastCloning_);
576 treePointers_[InEvent]->fastCloneTree(fb.tree());
580 art::RootDAQOutFile::incrementInputFileNumber()
582 fp_.update<Granularity::InputFile>();
588 respondToCloseInputFile(FileBlock
const&)
590 cet::for_all(treePointers_, [](
auto const& p) { p->setEntries(); });
596 requestsToCloseFile()
598 using namespace std::chrono;
599 unsigned int constexpr oneK{ 1024u };
600 TLOG(10) <<
"RootDAQOutFile::requestsToCloseFile start";
601 unsigned sz = filePtr_->GetSize();
602 TLOG(10) <<
"RootDAQOutFile::requestsToCloseFile after filePtr_->GetSize()";
603 fp_.updateSize(sz / oneK);
604 fp_.updateAge(duration_cast<seconds>(steady_clock::now() - beginTime_));
605 bool ret = fileSwitchCriteria_.should_close(fp_);
606 TLOG(10) <<
"RootDAQOutFile::requestsToCloseFile done/return";
610 void art::RootDAQOutFile::writeOne(EventPrincipal
const& e)
612 TLOG(10) <<
"RootDAQOutFile::writeOne begin";
616 pEventAux_ = &e.aux();
620 fillBranches<InEvent>(e, pEventProductProvenanceVector_);
622 History historyForOutput{ e.history() };
623 historyForOutput.addEventSelectionEntry(om_->selectorConfig());
624 pHistory_ = &historyForOutput;
625 int sz = eventHistoryTree_->Fill();
628 throw art::Exception(art::errors::FatalRootError)
629 <<
"Failed to fill the History tree for event: "
631 <<
"\nTTree::Fill() returned "
637 if (!dataTypeReported_)
639 string dataType{
"MC" };
640 if (pEventAux_->isRealData())
644 dataTypeReported_ =
true;
646 pHistory_ = &e.history();
648 fileIndex_.addEntry(pEventAux_->id(), fp_.eventEntryNumber());
649 fp_.update<Granularity::Event>(status_);
650 TLOG(10) <<
"RootDAQOutFile::writeOne done/return";
656 writeSubRun(SubRunPrincipal
const& sr)
658 pSubRunAux_ = &sr.aux();
659 pSubRunAux_->setRangeSetID(subRunRSID_);
660 fillBranches<InSubRun>(sr, pSubRunProductProvenanceVector_);
661 fileIndex_.addEntry(EventID::invalidEvent(pSubRunAux_->id()), fp_.subRunEntryNumber());
662 fp_.update<Granularity::SubRun>(status_);
668 writeRun(RunPrincipal
const& r)
671 pRunAux_->setRangeSetID(runRSID_);
672 fillBranches<InRun>(r, pRunProductProvenanceVector_);
673 fileIndex_.addEntry(EventID::invalidEvent(pRunAux_->id()), fp_.runEntryNumber());
674 fp_.update<Granularity::Run>(status_);
680 writeParentageRegistry()
682 ParentageID
const* hash =
new ParentageID;
683 if (!parentageTree_->Branch(rootNames::parentageIDBranchName().c_str(),
684 &hash, basketSize_, 0))
686 throw art::Exception(art::errors::FatalRootError)
687 <<
"Failed to create a branch for ParentageIDs in the output file";
691 Parentage
const* desc =
new Parentage;
692 if (!parentageTree_->Branch(rootNames::parentageBranchName().c_str(), &desc,
695 throw art::Exception(art::errors::FatalRootError)
696 <<
"Failed to create a branch for Parentages in the output file";
700 for (
auto const& pr : ParentageRegistry::get())
704 parentageTree_->Fill();
706 parentageTree_->SetBranchAddress(rootNames::parentageIDBranchName().c_str(),
nullptr);
707 parentageTree_->SetBranchAddress(rootNames::parentageBranchName().c_str(),
nullptr);
713 writeFileFormatVersion()
715 FileFormatVersion ver(getFileFormatVersion(), getFileFormatEra());
716 FileFormatVersion* pver = &ver;
717 TBranch* b = metaDataTree_->Branch(metaBranchRootName<FileFormatVersion>(),
718 &pver, basketSize_, 0);
729 fileIndex_.sortBy_Run_SubRun_Event();
730 FileIndex::Element elem{};
731 auto findexElemPtr = &elem;
732 TBranch* b = fileIndexTree_->Branch(metaBranchRootName<FileIndex::Element>(),
733 &findexElemPtr, basketSize_, 0);
736 for (
auto& entry : fileIndex_)
738 findexElemPtr = &entry;
749 RootOutputTree::writeTTree(eventHistoryTree_);
755 writeProcessConfigurationRegistry()
764 writeProcessHistoryRegistry()
766 ProcessHistoryMap pHistMap;
767 for (
auto const& pr : ProcessHistoryRegistry::get())
769 pHistMap.emplace(pr);
771 auto const* p = &pHistMap;
772 TBranch* b = metaDataTree_->Branch(metaBranchRootName<ProcessHistoryMap>(),
780 throw Exception(errors::LogicError)
781 <<
"Unable to locate required ProcessHistoryMap branch in output metadata tree.\n";
788 writeBranchIDListRegistry()
790 BranchIDLists
const* p = &BranchIDListRegistry::instance().data();
791 TBranch* b = metaDataTree_->Branch(metaBranchRootName<BranchIDLists>(), &p,
801 writeFileCatalogMetadata(FileStatsCollector
const& stats,
802 FileCatalogMetadata::collection_type
const& md,
803 FileCatalogMetadata::collection_type
const& ssmd)
805 using namespace cet::sqlite;
806 Ntuple<std::string, std::string> fileCatalogMetadata{ rootFileDB_,
"FileCatalog_metadata", {
"Name",
"Value"},
true };
807 Transaction txn{ rootFileDB_ };
808 for (
auto const& kv : md)
810 fileCatalogMetadata.insert(kv.first, kv.second);
813 fileCatalogMetadata.insert(
"file_format",
"\"artroot\"");
814 fileCatalogMetadata.insert(
"file_format_era", cet::canonical_string(getFileFormatEra()));
815 fileCatalogMetadata.insert(
"file_format_version", to_string(getFileFormatVersion()));
818 namespace bpt = boost::posix_time;
819 auto formatted_time = [](
auto const& t) {
return cet::canonical_string(bpt::to_iso_extended_string(t)); };
820 fileCatalogMetadata.insert(
"start_time", formatted_time(stats.outputFileOpenTime()));
822 fileCatalogMetadata.insert(
"end_time", formatted_time(boost::posix_time::second_clock::universal_time()));
825 if (!stats.seenSubRuns().empty())
828 auto I = find_if(md.crbegin(), md.crend(),
829 [](
auto const& p) {
return p.first ==
"run_type"; });
835 for (
auto const& srid : stats.seenSubRuns())
842 << cet::canonical_string(I->second)
846 buf.seekp(-2, ios_base::cur);
848 fileCatalogMetadata.insert(
"runs", buf.str());
852 fileCatalogMetadata.insert(
"event_count", to_string(stats.eventsThisFile()));
854 auto eidToTuple = [](EventID
const & eid)->
string
856 ostringstream eidStr;
866 fileCatalogMetadata.insert(
"first_event", eidToTuple(stats.lowestEventID()));
867 fileCatalogMetadata.insert(
"last_event", eidToTuple(stats.highestEventID()));
869 if (!stats.parents().empty())
871 ostringstream pstring;
873 for (
auto const& parent : stats.parents())
875 pstring << cet::canonical_string(parent) <<
", ";
878 pstring.seekp(-2, ios_base::cur);
880 fileCatalogMetadata.insert(
"parents", pstring.str());
883 for (
auto const& kv : ssmd)
885 fileCatalogMetadata.insert(kv.first, kv.second);
893 writeParameterSetRegistry()
895 fhicl::ParameterSetRegistry::exportTo(rootFileDB_);
901 writeProductDescriptionRegistry()
905 auto end = branchesWithStoredHistory_.end();
908 for (
auto const& pr : ProductMetaData::instance().productList())
910 if (branchesWithStoredHistory_.find(pr.second.branchID()) == end)
914 reg.productList_.emplace_hint(reg.productList_.end(), pr);
918 TBranch* b = metaDataTree_->Branch(metaBranchRootName<ProductRegistry>(),
919 ®p, basketSize_, 0);
928 writeProductDependencies()
930 BranchChildren& pDeps =
const_cast<BranchChildren&
>(om_->branchChildren());
931 BranchChildren* ppDeps = &pDeps;
932 TBranch* b = metaDataTree_->Branch(metaBranchRootName<BranchChildren>(),
933 &ppDeps, basketSize_, 0);
942 writeResults(ResultsPrincipal & resp)
944 pResultsAux_ = &resp.aux();
945 fillBranches<InResults>(resp, pResultsProductProvenanceVector_);
949 art::RootDAQOutFile::writeTTrees()
951 RootOutputTree::writeTTree(metaDataTree_);
952 RootOutputTree::writeTTree(fileIndexTree_);
953 RootOutputTree::writeTTree(parentageTree_);
955 for (
int i = InEvent; i < NumBranchTypes; ++i)
957 auto const branchType =
static_cast<BranchType
>(i);
958 treePointers_[branchType]->writeTree();
962 template <art::BranchType BT>
963 void art::RootDAQOutFile::fillBranches(Principal
const& principal,
964 vector<ProductProvenance>* vpp)
966 TLOG(11) <<
"RootDAQOutFile::fillBranches begin";
967 bool const fastCloning = (BT == InEvent) && currentlyFastCloning_;
968 detail::KeptProvenance keptProvenance{ dropMetaData_, dropMetaDataForDroppedData_, branchesWithStoredHistory_ };
969 map<unsigned, unsigned> checksumToIndex;
971 auto const& principalRS = principal.seenRanges();
973 for (
auto const& val : selectedOutputItemList_[BT])
975 auto const* bd = val.branchDescription_;
976 auto const bid = bd->branchID();
977 branchesWithStoredHistory_.insert(bid);
978 bool const produced{ bd->produced() };
979 bool const resolveProd = (produced || !fastCloning ||
980 treePointers_[BT]->uncloned(bd->branchName()));
983 bool const keepProvenance = (dropMetaData_ == DropMetaData::DropNone ||
984 (dropMetaData_ == DropMetaData::DropPrior &&
986 auto const& oh = principal.getForOutput(bid, resolveProd);
988 unique_ptr<ProductProvenance> prov{
nullptr };
991 if (oh.productProvenance())
993 prov = std::make_unique<ProductProvenance>(keptProvenance.insert(*oh.productProvenance()));
994 keptProvenance.insertAncestors(*oh.productProvenance(), principal);
1000 auto const status = produced ? productstatus::neverCreated() : productstatus::dropped();
1001 prov = std::make_unique<ProductProvenance>(keptProvenance.emplace(bid, status));
1008 auto const& rs = getRangeSet<BT>(oh, principalRS, produced);
1009 if (RangeSetsSupported<BT>::value && !rs.is_valid())
1015 keptProvenance.setStatus(*prov, productstatus::unknown());
1018 auto const* product = getProduct<BT>(oh, rs, bd->wrappedName());
1019 setProductRangeSetID<BT>(rs,
1021 const_cast<EDProduct*
>(product),
1023 val.product_ = product;
1026 vpp->assign(keptProvenance.begin(), keptProvenance.end());
1027 treePointers_[BT]->fillTree();
1029 TLOG(11) <<
"RootDAQOutFile::fillBranches done/return";
1035 setSubRunAuxiliaryRangeSetID(RangeSet
const& ranges)
1037 subRunRSID_ = getNewRangeSetID(rootFileDB_, InSubRun, ranges.run());
1038 insertIntoEventRanges(rootFileDB_, ranges);
1039 auto const& eventRangesIDs = getExistingRangeSetIDs(rootFileDB_, ranges);
1040 insertIntoJoinTable(rootFileDB_, InSubRun, subRunRSID_, eventRangesIDs);
1046 setRunAuxiliaryRangeSetID(RangeSet
const& ranges)
1048 runRSID_ = getNewRangeSetID(rootFileDB_, InRun, ranges.run());
1049 insertIntoEventRanges(rootFileDB_, ranges);
1050 auto const& eventRangesIDs = getExistingRangeSetIDs(rootFileDB_, ranges);
1051 insertIntoJoinTable(rootFileDB_, InRun, runRSID_, eventRangesIDs);
1054 template <BranchType BT>
1055 std::enable_if_t<!RangeSetsSupported<BT>::value, art::EDProduct
const*>
1056 art::RootDAQOutFile::getProduct(art::OutputHandle
const& oh,
1057 art::RangeSet
const& ,
1058 std::string
const& wrappedName)
1061 return oh.wrapper();
1063 return dummyProductCache_.product(wrappedName);
1066 template <BranchType BT>
1067 std::enable_if_t<RangeSetsSupported<BT>::value, art::EDProduct
const*>
1068 art::RootDAQOutFile::getProduct(art::OutputHandle
const& oh,
1069 art::RangeSet
const& prunedProductRS,
1070 std::string
const& wrappedName)
1072 if (oh.isValid() && prunedProductRS.is_valid())
1073 return oh.wrapper();
1075 return dummyProductCache_.product(wrappedName);
An output module which allows specifying the output filename exactly. Used for testing (file -> /dev/...