1 #include "artdaq/ArtModules/RootDAQOutFile.h"
5 #include "TBranchElement.h"
9 #include "art/Framework/Core/FileBlock.h"
10 #if ART_HEX_VERSION >= 0x20703
11 # include "art/Framework/Core/OutputFileGranularity.h"
12 # include "art/Framework/Services/System/DatabaseConnection.h"
13 # include "art/Persistency/RootDB/TKeyVFSOpenPolicy.h"
14 # include "cetlib/sqlite/Ntuple.h"
15 # include "cetlib/sqlite/Transaction.h"
16 # include "cetlib/sqlite/create_table.h"
17 # include "cetlib/sqlite/exec.h"
18 # include "cetlib/sqlite/insert.h"
20 # include "art/Framework/Services/Registry/ServiceHandle.h"
21 # include "cetlib/Ntuple/Transaction.h"
22 # include "art/Persistency/RootDB/SQLite3Wrapper.h"
24 #include "art/Framework/IO/FileStatsCollector.h"
25 #include "art/Framework/IO/Root/DropMetaData.h"
26 #include "art/Framework/IO/Root/GetFileFormatEra.h"
27 #include "art/Framework/IO/Root/GetFileFormatVersion.h"
28 #include "artdaq/ArtModules/detail/KeptProvenance.h"
29 #include "artdaq/ArtModules/RootDAQOutFile.h"
30 #include "art/Framework/Principal/EventPrincipal.h"
31 #include "art/Framework/Principal/ResultsPrincipal.h"
32 #include "art/Framework/Principal/RunPrincipal.h"
33 #include "art/Framework/Principal/SubRunPrincipal.h"
34 #include "art/Persistency/Provenance/BranchIDListRegistry.h"
35 #include "art/Persistency/Provenance/ProcessHistoryRegistry.h"
36 #include "art/Persistency/Provenance/ProductMetaData.h"
37 #include "art/Persistency/RootDB/SQLErrMsg.h"
38 #include "art/Version/GetReleaseVersion.h"
39 #include "boost/date_time/posix_time/posix_time.hpp"
40 #include "canvas/Persistency/Provenance/rootNames.h"
41 #include "canvas/Persistency/Provenance/BranchChildren.h"
42 #include "canvas/Persistency/Provenance/BranchID.h"
43 #include "canvas/Persistency/Provenance/BranchIDList.h"
44 #include "canvas/Persistency/Provenance/BranchType.h"
45 #include "canvas/Persistency/Provenance/EventAuxiliary.h"
46 #include "canvas/Persistency/Provenance/EventID.h"
47 #include "canvas/Persistency/Provenance/FileFormatVersion.h"
48 #include "canvas/Persistency/Provenance/History.h"
49 #include "canvas/Persistency/Provenance/ParameterSetBlob.h"
50 #include "canvas/Persistency/Provenance/Parentage.h"
51 #include "canvas/Persistency/Provenance/ParentageRegistry.h"
52 #include "canvas/Persistency/Provenance/ProcessHistoryID.h"
53 #include "canvas/Persistency/Provenance/ProductStatus.h"
54 #include "canvas/Persistency/Provenance/ResultsAuxiliary.h"
55 #include "canvas/Persistency/Provenance/RunAuxiliary.h"
56 #include "canvas/Persistency/Provenance/SubRunAuxiliary.h"
57 #include "canvas/Utilities/Exception.h"
58 #include "cetlib/canonical_string.h"
59 #include "cetlib/container_algorithms.h"
60 #include "cetlib/exempt_ptr.h"
61 #include "fhiclcpp/ParameterSet.h"
62 #include "fhiclcpp/ParameterSetID.h"
63 #include "fhiclcpp/ParameterSetRegistry.h"
64 #define TRACE_NAME "RootDAQOutFile.cc"
76 using art::BranchType;
78 using art::rootNames::metaBranchRootName;
82 void create_table(sqlite3*
const db,
83 std::string
const& name,
84 std::vector<std::string>
const& columns,
85 std::string
const& suffix = {})
88 throw art::Exception(art::errors::LogicError)
89 <<
"Number of sqlite columns specified for table: "
93 sqlite::Transaction txn {db};
94 art::SQLErrMsg errMsg;
96 "DROP TABLE IF EXISTS " + name +
"; "
97 "CREATE TABLE " + name +
99 std::for_each(columns.begin()+1, columns.end(),
100 [&ddl](
auto const& col) {
106 sqlite3_exec(db, ddl.c_str(),
nullptr,
nullptr, errMsg);
107 errMsg.throwIfError();
112 insert_eventRanges_row(sqlite3_stmt* stmt,
113 art::SubRunNumber_t
const sr,
114 art::EventNumber_t
const b,
115 art::EventNumber_t
const e)
117 sqlite3_bind_int64(stmt, 1, sr);
118 sqlite3_bind_int64(stmt, 2, b);
119 sqlite3_bind_int64(stmt, 3, e);
125 insert_rangeSets_row(sqlite3_stmt* stmt,
126 art::RunNumber_t
const r)
128 sqlite3_bind_int64(stmt, 1, r);
134 insert_rangeSets_eventSets_row(sqlite3_stmt* stmt,
138 sqlite3_bind_int64(stmt, 1, rsid);
139 sqlite3_bind_int64(stmt, 2, esid);
145 found_rowid(sqlite3_stmt* stmt)
147 return sqlite3_step(stmt) == SQLITE_ROW ?
148 sqlite3_column_int64(stmt,0) :
149 throw art::Exception(art::errors::SQLExecutionError)
150 <<
"ROWID not found for EventRanges.\n"
151 <<
"Contact artists@fnal.gov.\n";
155 getNewRangeSetID(sqlite3* db,
156 art::BranchType
const bt,
157 art::RunNumber_t
const r)
159 sqlite::Transaction txn {db};
160 sqlite3_stmt* stmt {
nullptr};
161 std::string
const ddl {
"INSERT INTO " + art::BranchTypeToString(bt) +
"RangeSets(Run) VALUES(?);"};
162 sqlite3_prepare_v2(db, ddl.c_str(), -1, &stmt,
nullptr);
163 insert_rangeSets_row(stmt, r);
164 unsigned const rsID = sqlite3_last_insert_rowid(db);
165 sqlite3_finalize(stmt);
171 getExistingRangeSetIDs(sqlite3* db, art::RangeSet
const& rs)
173 vector<unsigned> rangeSetIDs;
174 for (
auto const& range : rs) {
175 sqlite::Transaction txn {db};
176 sqlite3_stmt* stmt {
nullptr};
177 std::string
const ddl {
"SELECT ROWID FROM EventRanges WHERE "
178 "SubRun=" + std::to_string(range.subRun()) +
" AND "
179 "begin=" + std::to_string(range.begin()) +
" AND "
180 "end=" + std::to_string(range.end()) +
";"};
181 sqlite3_prepare_v2(db, ddl.c_str(), -1, &stmt,
nullptr);
182 rangeSetIDs.push_back(found_rowid(stmt));
183 sqlite3_finalize(stmt);
190 insertIntoEventRanges(sqlite3* db, art::RangeSet
const& rs)
192 sqlite::Transaction txn {db};
193 sqlite3_stmt* stmt {
nullptr};
194 std::string
const ddl {
"INSERT INTO EventRanges(SubRun, begin, end) "
196 sqlite3_prepare_v2(db, ddl.c_str(), -1, &stmt,
nullptr);
197 for (
auto const& range : rs) {
198 insert_eventRanges_row(stmt, range.subRun(), range.begin(), range.end());
200 sqlite3_finalize(stmt);
205 insertIntoJoinTable(sqlite3* db,
206 art::BranchType
const bt,
208 vector<unsigned>
const& eventRangesIDs)
210 sqlite::Transaction txn {db};
211 sqlite3_stmt* stmt {
nullptr};
212 std::string
const ddl {
"INSERT INTO "+art::BranchTypeToString(bt) +
213 "RangeSets_EventRanges(RangeSetsID, EventRangesID) Values(?,?);"};
214 sqlite3_prepare_v2(db, ddl.c_str(), -1, &stmt,
nullptr);
215 cet::for_all(eventRangesIDs,
216 [stmt,rsID](
auto const eventRangeID) {
217 insert_rangeSets_eventSets_row(stmt, rsID, eventRangeID);
219 sqlite3_finalize(stmt);
224 maybeInvalidateRangeSet(BranchType
const bt,
225 art::RangeSet
const& principalRS,
226 art::RangeSet& productRS)
228 if (!productRS.is_valid())
231 assert(principalRS.is_sorted());
232 assert(productRS.is_sorted());
234 if (bt == art::InRun && productRS.is_full_run())
return;
235 if (bt == art::InSubRun && productRS.is_full_subRun())
return;
236 assert(!productRS.ranges().empty());
238 auto const r = productRS.run();
239 auto const& productFront = productRS.ranges().front();
240 if (!principalRS.contains(r, productFront.subRun(), productFront.begin()))
241 productRS = art::RangeSet::invalid();
244 using art::detail::RangeSetsSupported;
285 template <BranchType BT>
286 std::enable_if_t<RangeSetsSupported<BT>::value, art::RangeSet>
287 getRangeSet(art::OutputHandle
const& oh,
288 art::RangeSet
const& principalRS,
289 bool const producedInThisProcess)
291 auto rs = oh.isValid() ? oh.rangeOfValidity() : art::RangeSet::invalid();
302 if (!producedInThisProcess) {
303 maybeInvalidateRangeSet(BT, principalRS, rs);
308 template <BranchType BT>
309 std::enable_if_t<!RangeSetsSupported<BT>::value, art::RangeSet>
310 getRangeSet(art::OutputHandle
const&,
311 art::RangeSet
const& ,
314 return art::RangeSet::invalid();
317 template <BranchType BT>
318 std::enable_if_t<!RangeSetsSupported<BT>::value>
319 setProductRangeSetID(art::RangeSet
const& ,
322 std::map<unsigned,unsigned>& )
325 template <BranchType BT>
326 std::enable_if_t<RangeSetsSupported<BT>::value>
327 setProductRangeSetID(art::RangeSet
const& rs,
329 art::EDProduct* product,
330 std::map<unsigned,unsigned>& checksumToIndexLookup)
336 auto it = checksumToIndexLookup.find(rs.checksum());
337 if (it != checksumToIndexLookup.cend()) {
338 product->setRangeSetID(it->second);
341 unsigned const rsID = getNewRangeSetID(db, BT, rs.run());
342 product->setRangeSetID(rsID);
343 checksumToIndexLookup.emplace(rs.checksum(), rsID);
344 insertIntoEventRanges(db, rs);
345 auto const& eventRangesIDs = getExistingRangeSetIDs(db, rs);
346 insertIntoJoinTable(db, BT, rsID, eventRangesIDs);
354 RootDAQOutFile(OutputModule* om,
355 string const& fileName,
356 ClosingCriteria
const& fileSwitchCriteria,
357 int const compressionLevel,
358 int64_t
const saveMemoryObjectThreshold,
359 int64_t
const treeMaxVirtualSize,
360 int const splitLevel,
361 int const basketSize,
362 DropMetaData dropMetaData,
363 bool const dropMetaDataForDroppedData,
364 bool const fastCloning)
367 , fileSwitchCriteria_{fileSwitchCriteria}
368 , compressionLevel_{compressionLevel}
369 , saveMemoryObjectThreshold_{saveMemoryObjectThreshold}
370 , treeMaxVirtualSize_{treeMaxVirtualSize}
371 , splitLevel_{splitLevel}
372 , basketSize_{basketSize}
373 , dropMetaData_{dropMetaData}
374 , dropMetaDataForDroppedData_{dropMetaDataForDroppedData}
375 , fastCloning_{fastCloning}
376 , filePtr_{TFile::Open(file_.c_str(),
"recreate",
"", compressionLevel)}
378 std::make_unique<RootOutputTree>(
379 # if ART_HEX_VERSION < 0x20703
380 static_cast<EventPrincipal*
>(
nullptr),
382 filePtr_.get(), InEvent, pEventAux_,
383 pEventProductProvenanceVector_, basketSize, splitLevel,
384 treeMaxVirtualSize, saveMemoryObjectThreshold),
385 std::make_unique<RootOutputTree>(
386 #
if ART_HEX_VERSION < 0x20703
387 static_cast<SubRunPrincipal*>(
nullptr),
389 filePtr_.get(), InSubRun, pSubRunAux_,
390 pSubRunProductProvenanceVector_, basketSize, splitLevel,
391 treeMaxVirtualSize, saveMemoryObjectThreshold),
392 std::make_unique<RootOutputTree>(
393 #
if ART_HEX_VERSION < 0x20703
394 static_cast<RunPrincipal*>(
nullptr),
396 filePtr_.get(), InRun, pRunAux_,
397 pRunProductProvenanceVector_, basketSize, splitLevel,
398 treeMaxVirtualSize, saveMemoryObjectThreshold),
399 std::make_unique<RootOutputTree>(
400 #
if ART_HEX_VERSION < 0x20703
401 static_cast<ResultsPrincipal*>(
nullptr),
403 filePtr_.get(), InResults, pResultsAux_,
404 pResultsProductProvenanceVector_, basketSize, splitLevel,
405 treeMaxVirtualSize, saveMemoryObjectThreshold) }
406 #
if ART_HEX_VERSION < 0x20703
407 , rootFileDB_{filePtr_.get(),
"RootFileDB", SQLITE_OPEN_CREATE | SQLITE_OPEN_READWRITE}
409 , rootFileDB_{ServiceHandle<DatabaseConnection>{}->get<TKeyVFSOpenPolicy>(
"RootFileDB",
411 SQLITE_OPEN_CREATE|SQLITE_OPEN_READWRITE)}
415 metaDataTree_ = RootOutputTree::makeTTree(filePtr_.get(), rootNames::metaDataTreeName(), 0);
416 fileIndexTree_ = RootOutputTree::makeTTree(filePtr_.get(), rootNames::fileIndexTreeName(), 0);
417 parentageTree_ = RootOutputTree::makeTTree(filePtr_.get(), rootNames::parentageTreeName(), 0);
419 eventHistoryTree_ = RootOutputTree::makeTTree(filePtr_.get(), rootNames::eventHistoryTreeName(), splitLevel);
420 if (!eventHistoryTree_) {
421 throw art::Exception(art::errors::FatalRootError)
422 <<
"Failed to create the tree for History objects\n";
426 pHistory_ =
new History;
427 if (!eventHistoryTree_->Branch(rootNames::eventHistoryBranchName().c_str(), &pHistory_, basketSize, 0)) {
428 throw art::Exception(art::errors::FatalRootError)
429 <<
"Failed to create a branch for History in the output file\n";
434 createDatabaseTables();
438 RootDAQOutFile::OutputItem::Sorter::
446 auto branches = tree->GetListOfBranches();
447 for (
int i = 0, sz = branches->GetEntries(); i != sz; ++i) {
448 auto br =
reinterpret_cast<TBranchElement*
>(branches->At(i));
449 treeMap_.emplace(
string(br->GetName()), i);
455 RootDAQOutFile::OutputItem::Sorter::
456 operator()(OutputItem
const& lh, OutputItem
const& rh)
const
462 if (treeMap_.empty()) {
465 auto const& lname = lh.branchDescription_->branchName();
466 auto const& rname = rh.branchDescription_->branchName();
467 auto lit = treeMap_.find(lname);
468 auto rit = treeMap_.find(rname);
469 bool lfound = (lit != treeMap_.end());
470 bool rfound = (rit != treeMap_.end());
471 if (lfound && rfound) {
472 return lit->second < rit->second;
484 art::RootDAQOutFile::createDatabaseTables()
492 create_table(rootFileDB_,
"EventRanges",
493 {
"SubRun INTEGER",
"begin INTEGER",
"end INTEGER",
"UNIQUE (SubRun,begin,end) ON CONFLICT IGNORE"});
496 create_table(rootFileDB_,
"SubRunRangeSets",
498 create_table(rootFileDB_,
"SubRunRangeSets_EventRanges",
499 {
"RangeSetsID INTEGER",
"EventRangesID INTEGER",
"PRIMARY KEY(RangeSetsID,EventRangesID)"},
503 create_table(rootFileDB_,
"RunRangeSets",
505 create_table(rootFileDB_,
"RunRangeSets_EventRanges",
506 {
"RangeSetsID INTEGER",
"EventRangesID INTEGER",
"PRIMARY KEY(RangeSetsID,EventRangesID)"},
513 selectProducts(FileBlock
const& fb)
515 for (
int i = InEvent; i < NumBranchTypes; ++i) {
516 auto bt =
static_cast<BranchType
>(i);
517 auto& items = selectedOutputItemList_[bt];
518 for (
auto const& val : items) {
519 treePointers_[bt]->resetOutputBranchAddress(*val.branchDescription_);
522 for (
auto const& bd : om_->keptProducts()[bt]) {
523 if ((bt < InResults) || bd->produced()) {
524 items.emplace_back(bd);
527 if ((bt == InEvent) && (fb.tree() !=
nullptr)) {
530 sort(items.begin(), items.end(), OutputItem::Sorter(fb.tree()));
532 for (
auto const& val : items) {
533 treePointers_[bt]->addOutputBranch(*val.branchDescription_, val.product_);
541 beginInputFile(FileBlock
const& fb,
bool fastClone)
544 auto const origCurrentlyFastCloning = currentlyFastCloning_;
545 currentlyFastCloning_ = fastCloning_ && fastClone;
546 if (currentlyFastCloning_ &&
547 !treePointers_[InEvent]->checkSplitLevelAndBasketSize(fb.tree())) {
548 mf::LogWarning(
"FastCloning")
549 <<
"Fast cloning deactivated for this input file due to "
550 <<
"splitting level and/or basket size.";
551 currentlyFastCloning_ =
false;
552 }
else if (fb.tree() && fb.tree()->GetCurrentFile()->GetVersion() < 60001) {
553 mf::LogWarning(
"FastCloning")
554 <<
"Fast cloning deactivated for this input file due to "
555 <<
"ROOT version used to write it (< 6.00/01)\n"
556 "having a different splitting policy.";
557 currentlyFastCloning_ =
false;
560 if (currentlyFastCloning_ && fb.fileFormatVersion().value_ < 9) {
561 mf::LogWarning(
"FastCloning")
562 <<
"Fast cloning deactivated for this input file due to "
563 <<
"reading in file that does not support RangeSets.";
564 currentlyFastCloning_ =
false;
567 if (currentlyFastCloning_ && !origCurrentlyFastCloning) {
568 mf::LogWarning(
"FastCloning")
569 <<
"Fast cloning reactivated for this input file.";
571 treePointers_[InEvent]->beginInputFile(currentlyFastCloning_);
572 treePointers_[InEvent]->fastCloneTree(fb.tree());
576 art::RootDAQOutFile::incrementInputFileNumber()
578 # if ART_HEX_VERSION >= 0x20703
579 fp_.update<Granularity::InputFile>();
581 fp_.update<Boundary::InputFile>();
588 respondToCloseInputFile(FileBlock
const&)
590 cet::for_all(treePointers_, [](
auto const& p){ p->setEntries(); });
596 requestsToCloseFile()
598 using namespace std::chrono;
599 unsigned int constexpr oneK {1024u};
600 TRACE( 10,
"RootDAQOutFile::requestsToCloseFile start" );
601 unsigned sz=filePtr_->GetSize();
602 TRACE( 10,
"RootDAQOutFile::requestsToCloseFile after filePtr_->GetSize()" );
603 fp_.updateSize(sz/oneK);
604 fp_.updateAge(duration_cast<seconds>(steady_clock::now() - beginTime_));
605 bool ret=fileSwitchCriteria_.should_close(fp_);
606 TRACE( 10,
"RootDAQOutFile::requestsToCloseFile done/return" );
610 void art::RootDAQOutFile::writeOne(EventPrincipal
const& e)
612 TRACE( 10,
"RootDAQOutFile::writeOne begin" );
616 pEventAux_ = &e.aux();
620 fillBranches<InEvent>(e, pEventProductProvenanceVector_);
622 History historyForOutput {e.history()};
623 historyForOutput.addEventSelectionEntry(om_->selectorConfig());
624 pHistory_ = &historyForOutput;
625 int sz = eventHistoryTree_->Fill();
627 throw art::Exception(art::errors::FatalRootError)
628 <<
"Failed to fill the History tree for event: "
630 <<
"\nTTree::Fill() returned "
636 if (!dataTypeReported_) {
637 string dataType {
"MC"};
638 if (pEventAux_->isRealData()) {
641 dataTypeReported_ =
true;
643 pHistory_ = &e.history();
645 fileIndex_.addEntry(pEventAux_->id(), fp_.eventEntryNumber());
646 # if ART_HEX_VERSION >= 0x20703
647 fp_.update<Granularity::Event>(status_);
649 fp_.update<Boundary::Event>(status_);
651 TRACE( 10,
"RootDAQOutFile::writeOne done/return" );
657 writeSubRun(SubRunPrincipal
const& sr)
659 pSubRunAux_ = &sr.aux();
660 pSubRunAux_->setRangeSetID(subRunRSID_);
661 fillBranches<InSubRun>(sr, pSubRunProductProvenanceVector_);
662 fileIndex_.addEntry(EventID::invalidEvent(pSubRunAux_->id()), fp_.subRunEntryNumber());
663 # if ART_HEX_VERSION >= 0x20703
664 fp_.update<Granularity::SubRun>(status_);
666 fp_.update<Boundary::SubRun>(status_);
673 writeRun(RunPrincipal
const& r)
676 pRunAux_->setRangeSetID(runRSID_);
677 fillBranches<InRun>(r, pRunProductProvenanceVector_);
678 fileIndex_.addEntry(EventID::invalidEvent(pRunAux_->id()), fp_.runEntryNumber());
679 # if ART_HEX_VERSION >= 0x20703
680 fp_.update<Granularity::Run>(status_);
682 fp_.update<Boundary::Run>(status_);
689 writeParentageRegistry()
691 ParentageID
const* hash =
new ParentageID;
692 if (!parentageTree_->Branch(rootNames::parentageIDBranchName().c_str(),
693 &hash, basketSize_, 0)) {
694 throw art::Exception(art::errors::FatalRootError)
695 <<
"Failed to create a branch for ParentageIDs in the output file";
699 Parentage
const* desc =
new Parentage;
700 if (!parentageTree_->Branch(rootNames::parentageBranchName().c_str(), &desc,
702 throw art::Exception(art::errors::FatalRootError)
703 <<
"Failed to create a branch for Parentages in the output file";
707 for (
auto const& pr : ParentageRegistry::get()) {
710 parentageTree_->Fill();
712 parentageTree_->SetBranchAddress(rootNames::parentageIDBranchName().c_str(),
nullptr);
713 parentageTree_->SetBranchAddress(rootNames::parentageBranchName().c_str(),
nullptr);
719 writeFileFormatVersion()
721 FileFormatVersion ver(getFileFormatVersion(), getFileFormatEra());
722 FileFormatVersion* pver = &ver;
723 TBranch* b = metaDataTree_->Branch(metaBranchRootName<FileFormatVersion>(),
724 &pver, basketSize_, 0);
735 fileIndex_.sortBy_Run_SubRun_Event();
736 FileIndex::Element elem {};
737 auto findexElemPtr = &elem;
738 TBranch* b = fileIndexTree_->Branch(metaBranchRootName<FileIndex::Element>(),
739 &findexElemPtr, basketSize_, 0);
742 for (
auto& entry: fileIndex_) {
743 findexElemPtr = &entry;
754 RootOutputTree::writeTTree(eventHistoryTree_);
760 writeProcessConfigurationRegistry()
769 writeProcessHistoryRegistry()
771 # if ART_HEX_VERSION >= 0x20703
772 ProcessHistoryMap pHistMap;
773 for (
auto const& pr : ProcessHistoryRegistry::get()) {
774 pHistMap.emplace(pr);
776 auto const* p = &pHistMap;
778 ProcessHistoryMap
const& r = ProcessHistoryRegistry::get();
779 ProcessHistoryMap* p = &
const_cast<ProcessHistoryMap&
>(r);
781 TBranch* b = metaDataTree_->Branch(metaBranchRootName<ProcessHistoryMap>(),
786 throw Exception(errors::LogicError)
787 <<
"Unable to locate required ProcessHistoryMap branch in output metadata tree.\n";
794 writeBranchIDListRegistry()
796 # if ART_HEX_VERSION >= 0x20703
797 BranchIDLists
const* p = &BranchIDListRegistry::instance().data();
799 BranchIDLists* p = &BranchIDListRegistry::instance()->data();
801 TBranch* b = metaDataTree_->Branch(metaBranchRootName<BranchIDLists>(), &p,
811 writeFileCatalogMetadata(FileStatsCollector
const& stats,
812 FileCatalogMetadata::collection_type
const& md,
813 FileCatalogMetadata::collection_type
const& ssmd)
815 # if ART_HEX_VERSION >= 0x20703
816 using namespace cet::sqlite;
817 Ntuple<std::string,std::string> fileCatalogMetadata {rootFileDB_,
"FileCatalog_metadata", {
"Name",
"Value"},
true};
818 Transaction txn {rootFileDB_};
819 for (
auto const& kv : md) {
820 fileCatalogMetadata.insert(kv.first, kv.second);
823 fileCatalogMetadata.insert(
"file_format",
"\"artroot\"");
824 fileCatalogMetadata.insert(
"file_format_era", cet::canonical_string(getFileFormatEra()));
825 fileCatalogMetadata.insert(
"file_format_version", to_string(getFileFormatVersion()));
827 ntuple::Ntuple<std::string,std::string> fileCatalogMetadata {rootFileDB_,
828 "FileCatalog_metadata",
831 using namespace sqlite;
832 Transaction txn {rootFileDB_};
833 for (
auto const& kv : md) {
834 insert_into(fileCatalogMetadata).values(kv.first, kv.second);
837 insert_into(fileCatalogMetadata).values(
"file_format",
"\"artroot\"");
838 insert_into(fileCatalogMetadata).values(
"file_format_era", cet::canonical_string(getFileFormatEra()));
839 insert_into(fileCatalogMetadata).values(
"file_format_version", to_string(getFileFormatVersion()));
843 namespace bpt = boost::posix_time;
844 auto formatted_time = [](
auto const& t){
return cet::canonical_string(bpt::to_iso_extended_string(t)); };
845 # if ART_HEX_VERSION >= 0x20703
846 fileCatalogMetadata.insert(
"start_time", formatted_time(stats.outputFileOpenTime()));
848 fileCatalogMetadata.insert(
"end_time", formatted_time(boost::posix_time::second_clock::universal_time()));
850 insert_into(fileCatalogMetadata).values(
"start_time", formatted_time(stats.outputFileOpenTime()));
852 insert_into(fileCatalogMetadata).values(
"end_time", formatted_time(boost::posix_time::second_clock::universal_time()));
856 if (!stats.seenSubRuns().empty()) {
858 auto I = find_if(md.crbegin(), md.crend(),
859 [](
auto const& p){
return p.first ==
"run_type"; } );
861 if (I != md.crend()) {
864 for (
auto const& srid : stats.seenSubRuns()) {
870 << cet::canonical_string(I->second)
874 buf.seekp(-2, ios_base::cur);
876 # if ART_HEX_VERSION >= 0x20703
877 fileCatalogMetadata.insert(
"runs", buf.str());
879 insert_into(fileCatalogMetadata).values(
"runs", buf.str());
884 # if ART_HEX_VERSION >= 0x20703
885 fileCatalogMetadata.insert(
"event_count", to_string(stats.eventsThisFile()));
887 insert_into(fileCatalogMetadata).values(
"event_count", to_string(stats.eventsThisFile()));
890 auto eidToTuple = [](EventID
const & eid)->
string {
891 ostringstream eidStr;
901 # if ART_HEX_VERSION >= 0x20703
902 fileCatalogMetadata.insert(
"first_event", eidToTuple(stats.lowestEventID()));
903 fileCatalogMetadata.insert(
"last_event", eidToTuple(stats.highestEventID()));
905 insert_into(fileCatalogMetadata).values(
"first_event", eidToTuple(stats.lowestEventID()));
906 insert_into(fileCatalogMetadata).values(
"last_event", eidToTuple(stats.highestEventID()));
909 if (!stats.parents().empty()) {
910 ostringstream pstring;
912 for (
auto const& parent : stats.parents()) {
913 pstring << cet::canonical_string(parent) <<
", ";
916 pstring.seekp(-2, ios_base::cur);
918 # if ART_HEX_VERSION >= 0x20703
919 fileCatalogMetadata.insert(
"parents", pstring.str());
921 insert_into(fileCatalogMetadata).values(
"parents", pstring.str());
925 for (
auto const& kv : ssmd) {
926 # if ART_HEX_VERSION >= 0x20703
927 fileCatalogMetadata.insert(kv.first, kv.second);
929 insert_into(fileCatalogMetadata).values(kv.first, kv.second);
938 writeParameterSetRegistry()
940 fhicl::ParameterSetRegistry::exportTo(rootFileDB_);
946 writeProductDescriptionRegistry()
950 auto end = branchesWithStoredHistory_.end();
953 for (
auto const& pr : ProductMetaData::instance().productList() ) {
954 if ( branchesWithStoredHistory_.find(pr.second.branchID()) == end ){
957 reg.productList_.emplace_hint(reg.productList_.end(),pr);
961 TBranch* b = metaDataTree_->Branch(metaBranchRootName<ProductRegistry>(),
962 ®p, basketSize_, 0);
971 writeProductDependencies()
973 BranchChildren& pDeps =
const_cast<BranchChildren&
>(om_->branchChildren());
974 BranchChildren* ppDeps = &pDeps;
975 TBranch* b = metaDataTree_->Branch(metaBranchRootName<BranchChildren>(),
976 &ppDeps, basketSize_, 0);
985 writeResults(ResultsPrincipal & resp)
987 pResultsAux_ = &resp.aux();
988 fillBranches<InResults>(resp, pResultsProductProvenanceVector_);
992 # if ART_HEX_VERSION >= 0x20703
993 art::RootDAQOutFile::writeTTrees()
999 # if ART_HEX_VERSION < 0x20703
1000 metaDataTree_->SetEntries(-1);
1002 RootOutputTree::writeTTree(metaDataTree_);
1003 RootOutputTree::writeTTree(fileIndexTree_);
1004 RootOutputTree::writeTTree(parentageTree_);
1006 for (
int i = InEvent; i < NumBranchTypes; ++i) {
1007 auto const branchType =
static_cast<BranchType
>(i);
1008 treePointers_[branchType]->writeTree();
1010 # if ART_HEX_VERSION < 0x20703
1014 rootFileDB_.reset();
1021 template <art::BranchType BT>
1022 void art::RootDAQOutFile::fillBranches(Principal
const& principal,
1023 vector<ProductProvenance>* vpp)
1025 TRACE( 11,
"RootDAQOutFile::fillBranches begin" );
1026 bool const fastCloning = (BT == InEvent) && currentlyFastCloning_;
1027 detail::KeptProvenance keptProvenance {dropMetaData_, dropMetaDataForDroppedData_, branchesWithStoredHistory_};
1028 map<unsigned,unsigned> checksumToIndex;
1030 auto const& principalRS = principal.seenRanges();
1032 for (
auto const& val : selectedOutputItemList_[BT]) {
1033 auto const* bd = val.branchDescription_;
1034 auto const bid = bd->branchID();
1035 branchesWithStoredHistory_.insert(bid);
1036 bool const produced {bd->produced()};
1037 bool const resolveProd = (produced || !fastCloning ||
1038 treePointers_[BT]->uncloned(bd->branchName()));
1041 bool const keepProvenance = (dropMetaData_ == DropMetaData::DropNone ||
1042 (dropMetaData_ == DropMetaData::DropPrior &&
1044 auto const& oh = principal.getForOutput(bid, resolveProd);
1046 unique_ptr<ProductProvenance> prov {
nullptr};
1047 if (keepProvenance) {
1048 if (oh.productProvenance()) {
1049 prov = std::make_unique<ProductProvenance>(keptProvenance.insert(*oh.productProvenance()));
1050 keptProvenance.insertAncestors(*oh.productProvenance(), principal);
1055 auto const status = produced ? productstatus::neverCreated() : productstatus::dropped();
1056 prov = std::make_unique<ProductProvenance>(keptProvenance.emplace(bid, status));
1062 auto const& rs = getRangeSet<BT>(oh, principalRS, produced);
1063 if (RangeSetsSupported<BT>::value && !rs.is_valid()) {
1068 keptProvenance.setStatus(*prov, productstatus::unknown());
1071 auto const* product = getProduct<BT>(oh, rs, bd->wrappedName());
1072 setProductRangeSetID<BT>(rs,
1074 const_cast<EDProduct*
>(product),
1076 val.product_ = product;
1079 vpp->assign(keptProvenance.begin(), keptProvenance.end());
1080 treePointers_[BT]->fillTree();
1082 TRACE( 11,
"RootDAQOutFile::fillBranches done/return" );
1088 setSubRunAuxiliaryRangeSetID(RangeSet
const& ranges)
1090 subRunRSID_ = getNewRangeSetID(rootFileDB_, InSubRun, ranges.run());
1091 insertIntoEventRanges(rootFileDB_, ranges);
1092 auto const& eventRangesIDs = getExistingRangeSetIDs(rootFileDB_, ranges);
1093 insertIntoJoinTable(rootFileDB_, InSubRun, subRunRSID_, eventRangesIDs);
1099 setRunAuxiliaryRangeSetID(RangeSet
const& ranges)
1101 runRSID_ = getNewRangeSetID(rootFileDB_, InRun, ranges.run());
1102 insertIntoEventRanges(rootFileDB_, ranges);
1103 auto const& eventRangesIDs = getExistingRangeSetIDs(rootFileDB_, ranges);
1104 insertIntoJoinTable(rootFileDB_, InRun, runRSID_, eventRangesIDs);
1107 template <BranchType BT>
1108 std::enable_if_t<!RangeSetsSupported<BT>::value, art::EDProduct
const*>
1109 art::RootDAQOutFile::getProduct(art::OutputHandle
const& oh,
1110 art::RangeSet
const& ,
1111 std::string
const& wrappedName)
1114 return oh.wrapper();
1116 return dummyProductCache_.product(wrappedName);
1119 template <BranchType BT>
1120 std::enable_if_t<RangeSetsSupported<BT>::value, art::EDProduct
const*>
1121 art::RootDAQOutFile::getProduct(art::OutputHandle
const& oh,
1122 art::RangeSet
const& prunedProductRS,
1123 std::string
const& wrappedName)
1125 if (oh.isValid() && prunedProductRS.is_valid())
1126 return oh.wrapper();
1128 return dummyProductCache_.product(wrappedName);