1 #include "artdaq/ArtModules/RootDAQOutput-s124/RootDAQOutFile.h"
2 #include "TRACE/tracemf.h"
5 #include "artdaq/DAQdata/Globals.hh"
7 #include "art/Framework/Core/OutputFileGranularity.h"
8 #include "art/Framework/Core/OutputModule.h"
9 #include "art/Framework/IO/ClosingCriteria.h"
10 #include "art/Framework/IO/FileStatsCollector.h"
11 #include "art/Framework/Principal/EventPrincipal.h"
12 #include "art/Framework/Principal/RangeSetsSupported.h"
13 #include "art/Framework/Principal/ResultsPrincipal.h"
14 #include "art/Framework/Principal/RunPrincipal.h"
15 #include "art/Framework/Principal/SubRunPrincipal.h"
16 #include "art/Framework/Services/Registry/ServiceHandle.h"
17 #include "art/Framework/Services/System/DatabaseConnection.h"
18 #include "art/Persistency/Provenance/ProcessHistoryRegistry.h"
19 #include "art_root_io/DropMetaData.h"
20 #include "art_root_io/GetFileFormatEra.h"
21 #include "art_root_io/GetFileFormatVersion.h"
22 #include "art_root_io/RootDB/TKeyVFSOpenPolicy.h"
23 #include "art_root_io/RootFileBlock.h"
24 #include "art_root_io/checkDictionaries.h"
25 #include "art_root_io/detail/getObjectRequireDict.h"
26 #include "boost/date_time/posix_time/posix_time.hpp"
27 #include "canvas/Persistency/Provenance/BranchChildren.h"
28 #include "canvas/Persistency/Provenance/BranchType.h"
29 #include "canvas/Persistency/Provenance/EventAuxiliary.h"
30 #include "canvas/Persistency/Provenance/EventID.h"
31 #include "canvas/Persistency/Provenance/FileFormatVersion.h"
32 #include "canvas/Persistency/Provenance/Parentage.h"
33 #include "canvas/Persistency/Provenance/ParentageRegistry.h"
34 #include "canvas/Persistency/Provenance/ProductStatus.h"
35 #include "canvas/Persistency/Provenance/ResultsAuxiliary.h"
36 #include "canvas/Persistency/Provenance/RunAuxiliary.h"
37 #include "canvas/Persistency/Provenance/SubRunAuxiliary.h"
38 #include "canvas/Persistency/Provenance/rootNames.h"
39 #include "canvas/Utilities/Exception.h"
40 #include "canvas_root_io/Utilities/DictionaryChecker.h"
41 #include "cetlib/canonical_string.h"
42 #include "cetlib/container_algorithms.h"
43 #include "cetlib/exempt_ptr.h"
44 #include "cetlib/sqlite/Ntuple.h"
45 #include "cetlib/sqlite/Transaction.h"
46 #include "cetlib/sqlite/create_table.h"
47 #include "cetlib/sqlite/exec.h"
48 #include "cetlib/sqlite/insert.h"
49 #include "fhiclcpp/ParameterSetRegistry.h"
50 #include "messagefacility/MessageLogger/MessageLogger.h"
51 #include "range/v3/view.hpp"
55 #define TRACE_NAME (app_name + "_RootDAQOutFile").c_str()
61 #include <sys/sysinfo.h>
67 using namespace hep::concurrency;
69 using art::BranchType;
71 using art::rootNames::metaBranchRootName;
79 void create_table(sqlite3*
const db,
81 vector<string>
const& columns,
82 string const& suffix = {})
86 throw art::Exception(art::errors::LogicError)
87 <<
"Number of sqlite columns specified for table: " << name <<
'\n'
90 string ddl =
"DROP TABLE IF EXISTS " + name +
93 name +
"(" + columns.front();
94 for_each(columns.begin() + 1, columns.end(), [&ddl](
auto const& col) {
100 sqlite::exec(db, ddl);
103 void insert_eventRanges_row(sqlite3_stmt* stmt,
104 art::SubRunNumber_t
const sr,
105 art::EventNumber_t
const b,
106 art::EventNumber_t
const e)
108 sqlite3_bind_int64(stmt, 1, sr);
109 sqlite3_bind_int64(stmt, 2, b);
110 sqlite3_bind_int64(stmt, 3, e);
115 void insert_rangeSets_eventSets_row(sqlite3_stmt* stmt,
119 sqlite3_bind_int64(stmt, 1, rsid);
120 sqlite3_bind_int64(stmt, 2, esid);
126 getNewRangeSetID(sqlite3* db,
127 art::BranchType
const bt,
128 art::RunNumber_t
const r)
130 sqlite::insert_into(db, art::BranchTypeToString(bt) +
"RangeSets")
132 return sqlite3_last_insert_rowid(db);
136 getExistingRangeSetIDs(sqlite3* db, art::RangeSet
const& rs)
139 vector<unsigned> rangeSetIDs;
140 cet::transform_all(rs, back_inserter(rangeSetIDs), [db](
auto const& range) {
141 sqlite::query_result<unsigned> r;
142 r << sqlite::select(
"ROWID")
143 .from(db,
"EventRanges")
144 .where(
"SubRun=" + to_string(range.subRun()) +
147 to_string(range.begin()) +
150 to_string(range.end()));
151 return unique_value(r);
156 void insertIntoEventRanges(sqlite3* db, art::RangeSet
const& rs)
158 sqlite::Transaction txn{db};
159 sqlite3_stmt* stmt{
nullptr};
161 "INSERT INTO EventRanges(SubRun, begin, end) "
163 sqlite3_prepare_v2(db, ddl.c_str(), -1, &stmt,
nullptr);
164 for (
auto const& range : rs)
166 insert_eventRanges_row(stmt, range.subRun(), range.begin(), range.end());
168 sqlite3_finalize(stmt);
172 void insertIntoJoinTable(sqlite3* db,
173 art::BranchType
const bt,
175 vector<unsigned>
const& eventRangesIDs)
177 sqlite::Transaction txn{db};
178 sqlite3_stmt* stmt{
nullptr};
180 "INSERT INTO " + art::BranchTypeToString(bt) +
181 "RangeSets_EventRanges(RangeSetsID, EventRangesID) Values(?,?);"};
182 sqlite3_prepare_v2(db, ddl.c_str(), -1, &stmt,
nullptr);
183 cet::for_all(eventRangesIDs, [stmt, rsID](
auto const eventRangeID) {
184 insert_rangeSets_eventSets_row(stmt, rsID, eventRangeID);
186 sqlite3_finalize(stmt);
190 void maybeInvalidateRangeSet(BranchType
const bt,
191 art::RangeSet
const& principalRS,
192 art::RangeSet& productRS)
194 assert(principalRS.is_sorted());
195 assert(productRS.is_sorted());
196 if (!productRS.is_valid())
200 if (bt == art::InRun && productRS.is_full_run())
204 if (bt == art::InSubRun && productRS.is_full_subRun())
208 if (productRS.ranges().empty())
212 auto const r = productRS.run();
213 auto const& productFront = productRS.ranges().front();
214 if (!principalRS.contains(r, productFront.subRun(), productFront.begin()))
216 productRS = art::RangeSet::invalid();
259 template<BranchType BT>
261 getRangeSet(art::OutputHandle
const& oh,
262 art::RangeSet
const& principalRS,
263 bool const producedInThisProcess)
265 if constexpr (!art::detail::range_sets_supported(BT))
267 return art::RangeSet::invalid();
270 auto rs = oh.isValid() ? oh.rangeOfValidity() : art::RangeSet::invalid();
281 if (!producedInThisProcess)
283 maybeInvalidateRangeSet(BT, principalRS, rs);
288 template<BranchType BT>
289 void setProductRangeSetID(art::RangeSet
const& rs,
291 art::EDProduct* product,
292 map<unsigned, unsigned>& checksumToIndexLookup)
294 if constexpr (!art::detail::range_sets_supported(BT))
304 auto it = checksumToIndexLookup.find(rs.checksum());
305 if (it != checksumToIndexLookup.cend())
307 product->setRangeSetID(it->second);
311 unsigned const rsID = getNewRangeSetID(db, BT, rs.run());
312 product->setRangeSetID(rsID);
313 checksumToIndexLookup.emplace(rs.checksum(), rsID);
314 insertIntoEventRanges(db, rs);
315 auto const& eventRangesIDs = getExistingRangeSetIDs(db, rs);
316 insertIntoJoinTable(db, BT, rsID, eventRangesIDs);
324 OutputItem::~OutputItem() =
default;
326 OutputItem::OutputItem(BranchDescription
const& bd)
327 : branchDescription{bd}, product{
nullptr}
330 RootDAQOutFile::RootDAQOutFile(OutputModule* om,
331 string const& fileName,
332 ClosingCriteria
const& fileSwitchCriteria,
333 int const compressionLevel,
334 unsigned freePercent,
336 int64_t
const saveMemoryObjectThreshold,
337 int64_t
const treeMaxVirtualSize,
338 int const splitLevel,
339 int const basketSize,
340 DropMetaData dropMetaData,
341 bool const dropMetaDataForDroppedData)
344 , fileSwitchCriteria_{fileSwitchCriteria}
345 , compressionLevel_{compressionLevel}
346 , freePercent_{freePercent}
348 , saveMemoryObjectThreshold_{saveMemoryObjectThreshold}
349 , treeMaxVirtualSize_{treeMaxVirtualSize}
350 , splitLevel_{splitLevel}
351 , basketSize_{basketSize}
352 , dropMetaData_{dropMetaData}
353 , dropMetaDataForDroppedData_{dropMetaDataForDroppedData}
354 , filePtr_{TFile::Open(file_.c_str(),
"recreate",
"", compressionLevel)}
356 using std::make_unique;
358 metaDataTree_ = RootOutputTree::makeTTree(
359 filePtr_.get(), rootNames::metaDataTreeName(), 0);
360 fileIndexTree_ = RootOutputTree::makeTTree(
361 filePtr_.get(), rootNames::fileIndexTreeName(), 0);
362 parentageTree_ = RootOutputTree::makeTTree(
363 filePtr_.get(), rootNames::parentageTreeName(), 0);
365 make_unique<RootOutputTree>(filePtr_.get(),
368 pEventProductProvenanceVector_,
372 saveMemoryObjectThreshold);
374 make_unique<RootOutputTree>(filePtr_.get(),
377 pSubRunProductProvenanceVector_,
381 saveMemoryObjectThreshold);
382 treePointers_[2] = make_unique<RootOutputTree>(filePtr_.get(),
385 pRunProductProvenanceVector_,
389 saveMemoryObjectThreshold);
391 make_unique<RootOutputTree>(filePtr_.get(),
394 pResultsProductProvenanceVector_,
398 saveMemoryObjectThreshold);
399 rootFileDB_.reset(ServiceHandle<DatabaseConnection> {
400 } -> get<TKeyVFSOpenPolicy>(
"RootFileDB",
402 SQLITE_OPEN_CREATE | SQLITE_OPEN_READWRITE));
403 beginTime_ = std::chrono::steady_clock::now();
405 root::DictionaryChecker checker;
406 checker.checkDictionaries<EventAuxiliary>();
407 checker.checkDictionaries<SubRunAuxiliary>();
408 checker.checkDictionaries<RunAuxiliary>();
409 checker.checkDictionaries<ResultsAuxiliary>();
410 checker.reportMissingDictionaries();
412 createDatabaseTables();
413 TLOG(TLVL_DEBUG + 32) <<
"RootDAQOutFile ctor complete";
416 art::RootDAQOutFile::~RootDAQOutFile()
419 int sts = sysinfo(&info);
420 auto free_percent =
static_cast<unsigned>(info.freeram * 100 / info.totalram);
421 auto free_MB =
static_cast<unsigned>(info.freeram * info.mem_unit >> 20);
422 TRACE(TLVL_DEBUG + 32,
"~RootDAQOutFile free %%%u %.1fMB (%u) buffers=%fGB mem_unit=%u",
423 free_percent, static_cast<float>(info.freeram * info.mem_unit / (1024 * 1024.0)),
424 free_MB, static_cast<float>(info.bufferram * info.mem_unit / (1024 * 1024 * 1024.0)), info.mem_unit);
425 if (free_percent < freePercent_ || free_MB < freeMB_)
427 TLOG(TLVL_DEBUG + 32) <<
"RootDAQOutFile Flush/DONTNEED";
429 sts = posix_fadvise(filePtr_->GetFd(), 0, 0 , POSIX_FADV_DONTNEED);
431 TLOG(TLVL_DEBUG + 32) <<
"~RootDAQOutFile complete sts=" << sts;
434 void art::RootDAQOutFile::createDatabaseTables()
437 create_table(*rootFileDB_,
442 "UNIQUE (SubRun,begin,end) ON CONFLICT IGNORE"});
444 using namespace cet::sqlite;
445 create_table(*rootFileDB_,
"SubRunRangeSets", column<int>{
"Run"});
446 create_table(*rootFileDB_,
447 "SubRunRangeSets_EventRanges",
448 {
"RangeSetsID INTEGER",
449 "EventRangesID INTEGER",
450 "PRIMARY KEY(RangeSetsID,EventRangesID)"},
453 create_table(*rootFileDB_,
"RunRangeSets", column<int>{
"Run"});
454 create_table(*rootFileDB_,
455 "RunRangeSets_EventRanges",
456 {
"RangeSetsID INTEGER",
457 "EventRangesID INTEGER",
458 "PRIMARY KEY(RangeSetsID,EventRangesID)"},
462 void RootDAQOutFile::setFileStatus(OutputFileStatus
const ofs)
464 std::lock_guard sentry{mutex_};
469 RootDAQOutFile::currentFileName()
const
471 std::lock_guard sentry{mutex_};
475 void RootDAQOutFile::selectProducts()
477 std::lock_guard sentry{mutex_};
478 auto selectProductsToWrite = [
this](BranchType
const bt) {
479 auto& items = selectedOutputItemList_[bt];
480 for (
auto const& pd : om_->keptProducts()[bt] | ranges::views::values)
484 if (bt == InResults && !pd.produced())
488 checkDictionaries(pd);
497 items.try_emplace(pd.productID(), pd);
499 for (
auto const& item : items | ranges::views::values)
501 treePointers_[bt]->addOutputBranch(item.branchDescription,
505 for_each_branch_type(selectProductsToWrite);
508 void RootDAQOutFile::beginInputFile(RootFileBlock
const* rfb,
509 FastCloningEnabled fastCloningEnabled)
511 std::lock_guard sentry{mutex_};
517 cet::exempt_ptr<TTree const> inputTree{
nullptr};
520 if (rfb->fileFormatVersion().value_ < 10)
522 fastCloningEnabled.disable(
523 "The input file has a different ProductID "
524 "schema than the in-memory schema.");
526 inputTree = rfb->tree();
529 if (!treePointers_[InEvent]->checkSplitLevelAndBasketSize(inputTree))
531 fastCloningEnabled.disable(
532 "The splitting level and/or basket size does not match between "
533 "input and output file.");
535 if (inputTree->GetCurrentFile()->GetVersion() < 60001)
537 fastCloningEnabled.disable(
538 "The ROOT version used to write the input file (< 6.00/01)\nhas a "
539 "different splitting policy.");
544 if (not fastCloningEnabled)
546 mf::LogWarning(
"FastCloning") << fastCloningEnabled.disabledBecause();
550 mf::LogInfo(
"FastCloning")
551 <<
"Fast cloning event data products from input file.";
552 wasFastCloned_ = treePointers_[InEvent]->fastCloneTree(inputTree);
555 void RootDAQOutFile::incrementInputFileNumber()
557 std::lock_guard sentry{mutex_};
558 fp_.update_inputFile();
561 void RootDAQOutFile::respondToCloseInputFile(FileBlock
const&)
563 std::lock_guard sentry{mutex_};
564 cet::for_all(treePointers_, [](
auto const& p) { p->setEntries(); });
567 bool RootDAQOutFile::requestsToCloseFile()
569 std::lock_guard sentry{mutex_};
570 using namespace std::chrono;
571 unsigned int constexpr oneK{1024u};
572 fp_.updateSize(filePtr_->GetSize() / oneK);
573 fp_.updateAge(duration_cast<seconds>(steady_clock::now() - beginTime_));
574 return fileSwitchCriteria_.should_close(fp_);
577 void RootDAQOutFile::writeOne(EventPrincipal
const& e)
579 std::lock_guard sentry{mutex_};
580 TLOG(TLVL_DEBUG + 33) <<
"Start of RootDAQOutFile::writeOne";
583 pEventAux_ = &e.eventAux();
587 fillBranches<InEvent>(e, pEventProductProvenanceVector_);
590 if (!dataTypeReported_)
592 string dataType{
"MC"};
593 if (pEventAux_->isRealData())
597 dataTypeReported_ =
true;
600 fileIndex_.addEntry(pEventAux_->eventID(), fp_.eventEntryNumber());
602 TLOG(TLVL_DEBUG + 33) <<
"End of RootDAQOutFile::writeOne";
605 void RootDAQOutFile::writeSubRun(SubRunPrincipal
const& sr)
607 std::lock_guard sentry{mutex_};
608 pSubRunAux_ = &sr.subRunAux();
609 pSubRunAux_->setRangeSetID(subRunRSID_);
610 fillBranches<InSubRun>(sr, pSubRunProductProvenanceVector_);
611 fileIndex_.addEntry(EventID::invalidEvent(pSubRunAux_->subRunID()),
612 fp_.subRunEntryNumber());
613 fp_.update_subRun(status_);
616 void RootDAQOutFile::writeRun(RunPrincipal
const& r)
618 std::lock_guard sentry{mutex_};
619 pRunAux_ = &r.runAux();
620 pRunAux_->setRangeSetID(runRSID_);
621 fillBranches<InRun>(r, pRunProductProvenanceVector_);
622 fileIndex_.addEntry(EventID::invalidEvent(pRunAux_->runID()),
623 fp_.runEntryNumber());
624 fp_.update_run(status_);
627 void RootDAQOutFile::writeParentageRegistry()
629 std::lock_guard sentry{mutex_};
630 auto pid = root::getObjectRequireDict<ParentageID>();
631 ParentageID
const* hash = &pid;
632 if (!parentageTree_->Branch(
633 rootNames::parentageIDBranchName().c_str(), &hash, basketSize_, 0))
635 throw Exception(errors::FatalRootError)
636 <<
"Failed to create a branch for ParentageIDs in the output file";
639 auto par = root::getObjectRequireDict<Parentage>();
640 Parentage
const* desc = ∥
641 if (!parentageTree_->Branch(
642 rootNames::parentageBranchName().c_str(), &desc, basketSize_, 0))
644 throw Exception(errors::FatalRootError)
645 <<
"Failed to create a branch for Parentages in the output file";
648 for (
auto const& pr : ParentageRegistry::get())
652 parentageTree_->Fill();
654 parentageTree_->SetBranchAddress(rootNames::parentageIDBranchName().c_str(),
656 parentageTree_->SetBranchAddress(rootNames::parentageBranchName().c_str(),
660 void RootDAQOutFile::writeFileFormatVersion()
662 std::lock_guard sentry{mutex_};
663 FileFormatVersion
const ver{getFileFormatVersion(), getFileFormatEra()};
664 auto const* pver = &ver;
665 TBranch* b = metaDataTree_->Branch(
666 metaBranchRootName<FileFormatVersion>(), &pver, basketSize_, 0);
672 void RootDAQOutFile::writeFileIndex()
674 std::lock_guard sentry{mutex_};
675 fileIndex_.sortBy_Run_SubRun_Event();
676 FileIndex::Element elem{};
677 auto const* findexElemPtr = &elem;
678 TBranch* b = fileIndexTree_->Branch(
679 metaBranchRootName<FileIndex::Element>(), &findexElemPtr, basketSize_, 0);
682 for (
auto& entry : fileIndex_)
684 findexElemPtr = &entry;
690 void RootDAQOutFile::writeProcessConfigurationRegistry()
696 void RootDAQOutFile::writeProcessHistoryRegistry()
698 std::lock_guard sentry{mutex_};
699 ProcessHistoryMap pHistMap;
700 for (
auto const& pr : ProcessHistoryRegistry::get())
702 pHistMap.emplace(pr);
704 auto const* p = &pHistMap;
705 TBranch* b = metaDataTree_->Branch(
706 metaBranchRootName<ProcessHistoryMap>(), &p, basketSize_, 0);
709 throw Exception(errors::LogicError)
710 <<
"Unable to locate required "
711 "ProcessHistoryMap branch in output "
717 void RootDAQOutFile::writeFileCatalogMetadata(
718 FileStatsCollector
const& stats,
719 FileCatalogMetadata::collection_type
const& md,
720 FileCatalogMetadata::collection_type
const& ssmd)
722 std::lock_guard sentry{mutex_};
723 using namespace cet::sqlite;
724 Ntuple<string, string> fileCatalogMetadata{
725 *rootFileDB_,
"FileCatalog_metadata", {{
"Name",
"Value"}},
true};
726 for (
auto const& [key, value] : md)
728 fileCatalogMetadata.insert(key, value);
732 fileCatalogMetadata.insert(
"file_format",
"\"artroot\"");
735 namespace bpt = boost::posix_time;
736 auto formatted_time = [](
auto const& t) {
737 return cet::canonical_string(bpt::to_iso_extended_string(t));
739 fileCatalogMetadata.insert(
"start_time",
740 formatted_time(stats.outputFileOpenTime()));
742 fileCatalogMetadata.insert(
744 formatted_time(boost::posix_time::second_clock::universal_time()));
746 if (!stats.seenSubRuns().empty())
748 auto I = find_if(md.crbegin(), md.crend(), [](
auto const& p) {
749 return p.first ==
"art.run_type";
753 std::ostringstream buf;
755 for (
auto const& srid : stats.seenSubRuns())
757 buf <<
"[ " << srid.run() <<
", " << srid.subRun() <<
", "
758 << cet::canonical_string(I->second) <<
" ], ";
761 buf.seekp(-2, std::ios_base::cur);
763 fileCatalogMetadata.insert(
"runs", buf.str());
767 fileCatalogMetadata.insert(
"event_count",
768 std::to_string(stats.eventsThisFile()));
769 fileCatalogMetadata.insert(
"first_event",
770 std::to_string(stats.lowestEventID().event()));
771 fileCatalogMetadata.insert(
"last_event",
772 std::to_string(stats.highestEventID().event()));
774 if (!stats.parents().empty())
776 std::ostringstream pstring;
778 for (
auto const& parent : stats.parents())
780 pstring << cet::canonical_string(parent) <<
", ";
783 pstring.seekp(-2, std::ios_base::cur);
785 fileCatalogMetadata.insert(
"parents", pstring.str());
790 auto eidToTuple = [](EventID
const& eid) ->
string {
791 std::ostringstream eidStr;
792 eidStr <<
"[ " << eid.run() <<
", " << eid.subRun() <<
", " << eid.event()
796 fileCatalogMetadata.insert(
"art.first_event",
797 eidToTuple(stats.lowestEventID()));
798 fileCatalogMetadata.insert(
"art.last_event",
799 eidToTuple(stats.highestEventID()));
800 fileCatalogMetadata.insert(
"art.file_format_era",
801 cet::canonical_string(getFileFormatEra()));
802 fileCatalogMetadata.insert(
"art.file_format_version",
803 std::to_string(getFileFormatVersion()));
806 for (
auto const& [key, value] : ssmd)
808 fileCatalogMetadata.insert(key, value);
812 void RootDAQOutFile::writeParameterSetRegistry()
814 std::lock_guard sentry{mutex_};
815 fhicl::ParameterSetRegistry::exportTo(*rootFileDB_);
818 void RootDAQOutFile::writeProductDescriptionRegistry()
820 std::lock_guard sentry{mutex_};
824 auto productDescriptionsToWrite = [
this, ®](BranchType
const bt) {
825 for (
auto const& desc :
826 descriptionsToPersist_[bt] | ranges::views::values)
828 reg.productList_.emplace(BranchKey{desc}, desc);
831 for_each_branch_type(productDescriptionsToWrite);
832 ProductRegistry
const* regp = ®
833 TBranch* b = metaDataTree_->Branch(
834 metaBranchRootName<ProductRegistry>(), ®p, basketSize_, 0);
840 void RootDAQOutFile::writeProductDependencies()
842 std::lock_guard sentry{mutex_};
843 BranchChildren
const* ppDeps = &om_->branchChildren();
844 TBranch* b = metaDataTree_->Branch(
845 metaBranchRootName<BranchChildren>(), &ppDeps, basketSize_, 0);
851 void RootDAQOutFile::writeResults(ResultsPrincipal& resp)
853 std::lock_guard sentry{mutex_};
854 pResultsAux_ = &resp.resultsAux();
855 fillBranches<InResults>(resp, pResultsProductProvenanceVector_);
858 void RootDAQOutFile::writeTTrees()
860 TLOG(TLVL_DEBUG + 33) <<
"Start of RootDAQOutFile::writeTTrees";
861 std::lock_guard sentry{mutex_};
862 RootOutputTree::writeTTree(metaDataTree_);
863 TLOG(TLVL_DEBUG + 33) <<
"RootDAQOutFile::writeTTrees after writing metaDataTree_";
864 RootOutputTree::writeTTree(fileIndexTree_);
865 TLOG(TLVL_DEBUG + 33) <<
"RootDAQOutFile::writeTTrees after writing fileIndexTree_";
866 RootOutputTree::writeTTree(parentageTree_);
867 TLOG(TLVL_DEBUG + 33) <<
"RootDAQOutFile::writeTTrees after writing parentageTree_";
868 for_each_branch_type(
869 [
this](BranchType
const bt) { treePointers_[bt]->writeTree(); });
870 TLOG(TLVL_DEBUG + 33) <<
"End of RootDAQOutFile::writeTTrees";
873 void RootDAQOutFile::setSubRunAuxiliaryRangeSetID(RangeSet
const& ranges)
875 std::lock_guard sentry{mutex_};
876 subRunRSID_ = getNewRangeSetID(*rootFileDB_, InSubRun, ranges.run());
877 insertIntoEventRanges(*rootFileDB_, ranges);
878 auto const& eventRangesIDs = getExistingRangeSetIDs(*rootFileDB_, ranges);
879 insertIntoJoinTable(*rootFileDB_, InSubRun, subRunRSID_, eventRangesIDs);
882 void RootDAQOutFile::setRunAuxiliaryRangeSetID(RangeSet
const& ranges)
884 std::lock_guard sentry{mutex_};
885 runRSID_ = getNewRangeSetID(*rootFileDB_, InRun, ranges.run());
886 insertIntoEventRanges(*rootFileDB_, ranges);
887 auto const& eventRangesIDs = getExistingRangeSetIDs(*rootFileDB_, ranges);
888 insertIntoJoinTable(*rootFileDB_, InRun, runRSID_, eventRangesIDs);
891 template<BranchType BT>
893 RootDAQOutFile::getProduct(OutputHandle
const& oh,
894 RangeSet
const& prunedProductRS,
895 string const& wrappedName)
897 std::lock_guard sentry{mutex_};
898 if constexpr (detail::range_sets_supported(BT))
900 if (!prunedProductRS.is_valid())
902 return dummyProductCache_.product(wrappedName);
905 return oh.isValid() ? oh.wrapper() : dummyProductCache_.product(wrappedName);
908 template<BranchType BT>
909 void RootDAQOutFile::fillBranches(Principal
const& principal,
910 vector<ProductProvenance>* vpp)
912 TLOG(TLVL_DEBUG + 33) <<
"Start of RootDAQOutFile::fillBranches";
913 std::lock_guard sentry{mutex_};
914 bool const fastCloning{BT == InEvent && wasFastCloned_};
915 map<unsigned, unsigned> checksumToIndex;
916 auto const& principalRS = principal.seenRanges();
920 bool const drop_no_metadata{dropMetaData_ == DropMetaData::DropNone};
921 bool const drop_prior_metadata{dropMetaData_ == DropMetaData::DropPrior};
922 bool const drop_all_metadata{dropMetaData_ == DropMetaData::DropAll};
924 std::set<ProductProvenance> keptprv;
925 for (
auto const& [pid, val] : selectedOutputItemList_[BT])
927 auto const& bd = val.branchDescription;
928 descriptionsToPersist_[BT].try_emplace(pid, bd);
929 bool const produced = bd.produced();
930 bool const resolveProd{produced || !fastCloning ||
931 treePointers_[BT]->uncloned(bd.branchName())};
933 bool const keepProvenance =
934 drop_no_metadata || (produced && drop_prior_metadata);
935 auto const& oh = principal.getForOutput(pid, resolveProd);
936 auto prov = keptprv.begin();
939 if (oh.productProvenance())
941 prov = keptprv.insert(*oh.productProvenance()).first;
942 if (!drop_all_metadata && !dropMetaDataForDroppedData_)
945 vector<ProductProvenance const*> stacked_pp;
946 stacked_pp.push_back(&*oh.productProvenance());
947 while (not empty(stacked_pp))
949 auto current_pp = stacked_pp.back();
950 stacked_pp.pop_back();
951 for (
auto const parent_bid :
952 current_pp->parentage().parents())
965 auto parent_bd = principal.getProductDescription(parent_bid);
971 descriptionsToPersist_[BT].try_emplace(parent_bid,
973 if (!parent_bd->produced())
979 principal.branchToProductProvenance(parent_bid);
980 if (!parent_pp || !drop_no_metadata)
984 if (!keptprv.insert(*parent_pp).second)
989 if (!drop_all_metadata && !dropMetaDataForDroppedData_)
991 stacked_pp.push_back(parent_pp.get());
1002 auto status = productstatus::dropped();
1005 status = productstatus::neverCreated();
1007 prov = keptprv.emplace(pid, status).first;
1017 auto const& rs = getRangeSet<BT>(oh, principalRS, produced);
1018 if (detail::range_sets_supported(BT) && !rs.is_valid())
1029 auto prov_bid = prov->productID();
1030 if (keptprv.erase(*prov) != 1ull)
1032 throw Exception(errors::LogicError,
"KeptProvenance::setStatus")
1033 <<
"Attempt to set product status for product whose provenance "
1034 "is not being recorded.\n";
1038 .emplace(prov_bid, productstatus::dummyToPreventDoubleCount())
1041 auto const* product = getProduct<BT>(oh, rs, bd.wrappedName());
1042 setProductRangeSetID<BT>(
1043 rs, *rootFileDB_,
const_cast<EDProduct*
>(product), checksumToIndex);
1044 val.product = product;
1047 vpp->assign(keptprv.begin(), keptprv.end());
1048 for (
auto const& val : *vpp)
1050 if (val.productStatus() == productstatus::uninitialized())
1052 throw Exception(errors::LogicError,
1053 "RootDAQOutFile::fillBranches(principal, vpp):")
1054 <<
"Attempt to write a product with uninitialized provenance!\n";
1058 TLOG(TLVL_DEBUG + 33) <<
"RootDAQOutFile::fillBranches before fillTree call";
1059 treePointers_[BT]->fillTree();
1060 TLOG(TLVL_DEBUG + 33) <<
"RootDAQOutFile::fillBranches after fillTree call";
1062 TLOG(TLVL_DEBUG + 33) <<
"End of RootDAQOutFile::fillBranches";