1 #include "artdaq/ArtModules/RootDAQOutput-s124/RootDAQOutFile.h"
2 #include "TRACE/tracemf.h"
5 #include "artdaq/DAQdata/Globals.hh"
7 #include "art/Framework/Core/OutputFileGranularity.h"
8 #include "art/Framework/Core/OutputModule.h"
9 #include "art/Framework/IO/ClosingCriteria.h"
10 #include "art/Framework/IO/FileStatsCollector.h"
11 #include "art/Framework/Principal/EventPrincipal.h"
12 #include "art/Framework/Principal/RangeSetsSupported.h"
13 #include "art/Framework/Principal/ResultsPrincipal.h"
14 #include "art/Framework/Principal/RunPrincipal.h"
15 #include "art/Framework/Principal/SubRunPrincipal.h"
16 #include "art/Framework/Services/Registry/ServiceHandle.h"
17 #include "art/Framework/Services/System/DatabaseConnection.h"
18 #include "art/Persistency/Provenance/ProcessHistoryRegistry.h"
19 #include "art_root_io/DropMetaData.h"
20 #include "art_root_io/GetFileFormatEra.h"
21 #include "art_root_io/GetFileFormatVersion.h"
22 #include "art_root_io/RootDB/TKeyVFSOpenPolicy.h"
23 #include "art_root_io/RootFileBlock.h"
24 #include "art_root_io/checkDictionaries.h"
25 #include "art_root_io/detail/getObjectRequireDict.h"
26 #include "boost/date_time/posix_time/posix_time.hpp"
27 #include "canvas/Persistency/Provenance/BranchChildren.h"
28 #include "canvas/Persistency/Provenance/BranchType.h"
29 #include "canvas/Persistency/Provenance/EventAuxiliary.h"
30 #include "canvas/Persistency/Provenance/EventID.h"
31 #include "canvas/Persistency/Provenance/FileFormatVersion.h"
32 #include "canvas/Persistency/Provenance/Parentage.h"
33 #include "canvas/Persistency/Provenance/ParentageRegistry.h"
34 #include "canvas/Persistency/Provenance/ProductStatus.h"
35 #include "canvas/Persistency/Provenance/ResultsAuxiliary.h"
36 #include "canvas/Persistency/Provenance/RunAuxiliary.h"
37 #include "canvas/Persistency/Provenance/SubRunAuxiliary.h"
38 #include "canvas/Persistency/Provenance/rootNames.h"
39 #include "canvas/Utilities/Exception.h"
40 #include "canvas_root_io/Utilities/DictionaryChecker.h"
41 #include "cetlib/canonical_string.h"
42 #include "cetlib/container_algorithms.h"
43 #include "cetlib/exempt_ptr.h"
44 #include "cetlib/sqlite/Ntuple.h"
45 #include "cetlib/sqlite/Transaction.h"
46 #include "cetlib/sqlite/create_table.h"
47 #include "cetlib/sqlite/exec.h"
48 #include "cetlib/sqlite/insert.h"
49 #include "fhiclcpp/ParameterSetRegistry.h"
50 #include "messagefacility/MessageLogger/MessageLogger.h"
51 #include "range/v3/view.hpp"
55 #define TRACE_NAME (app_name + "_RootDAQOutFile").c_str()
61 #include <sys/sysinfo.h>
67 using namespace hep::concurrency;
69 using art::BranchType;
71 using art::rootNames::metaBranchRootName;
79 void create_table(sqlite3*
const db,
81 vector<string>
const& columns,
82 string const& suffix = {})
86 throw art::Exception(art::errors::LogicError)
87 <<
"Number of sqlite columns specified for table: " << name <<
'\n'
90 string ddl =
"DROP TABLE IF EXISTS " + name +
93 name +
"(" + columns.front();
94 for_each(columns.begin() + 1, columns.end(), [&ddl](
auto const& col) {
100 sqlite::exec(db, ddl);
103 void insert_eventRanges_row(sqlite3_stmt* stmt,
104 art::SubRunNumber_t
const sr,
105 art::EventNumber_t
const b,
106 art::EventNumber_t
const e)
108 sqlite3_bind_int64(stmt, 1, sr);
109 sqlite3_bind_int64(stmt, 2, b);
110 sqlite3_bind_int64(stmt, 3, e);
115 void insert_rangeSets_eventSets_row(sqlite3_stmt* stmt,
119 sqlite3_bind_int64(stmt, 1, rsid);
120 sqlite3_bind_int64(stmt, 2, esid);
126 getNewRangeSetID(sqlite3* db,
127 art::BranchType
const bt,
128 art::RunNumber_t
const r)
130 sqlite::insert_into(db, art::BranchTypeToString(bt) +
"RangeSets")
132 return sqlite3_last_insert_rowid(db);
136 getExistingRangeSetIDs(sqlite3* db, art::RangeSet
const& rs)
139 vector<unsigned> rangeSetIDs;
140 cet::transform_all(rs, back_inserter(rangeSetIDs), [db](
auto const& range) {
141 sqlite::query_result<unsigned> r;
142 r << sqlite::select(
"ROWID")
143 .from(db,
"EventRanges")
144 .where(
"SubRun=" + to_string(range.subRun()) +
147 to_string(range.begin()) +
150 to_string(range.end()));
151 return unique_value(r);
156 void insertIntoEventRanges(sqlite3* db, art::RangeSet
const& rs)
158 sqlite::Transaction txn{db};
159 sqlite3_stmt* stmt{
nullptr};
161 "INSERT INTO EventRanges(SubRun, begin, end) "
163 sqlite3_prepare_v2(db, ddl.c_str(), -1, &stmt,
nullptr);
164 for (
auto const& range : rs)
166 insert_eventRanges_row(stmt, range.subRun(), range.begin(), range.end());
168 sqlite3_finalize(stmt);
172 void insertIntoJoinTable(sqlite3* db,
173 art::BranchType
const bt,
175 vector<unsigned>
const& eventRangesIDs)
177 sqlite::Transaction txn{db};
178 sqlite3_stmt* stmt{
nullptr};
180 "INSERT INTO " + art::BranchTypeToString(bt) +
181 "RangeSets_EventRanges(RangeSetsID, EventRangesID) Values(?,?);"};
182 sqlite3_prepare_v2(db, ddl.c_str(), -1, &stmt,
nullptr);
183 cet::for_all(eventRangesIDs, [stmt, rsID](
auto const eventRangeID) {
184 insert_rangeSets_eventSets_row(stmt, rsID, eventRangeID);
186 sqlite3_finalize(stmt);
190 void maybeInvalidateRangeSet(BranchType
const bt,
191 art::RangeSet
const& principalRS,
192 art::RangeSet& productRS)
194 assert(principalRS.is_sorted());
195 assert(productRS.is_sorted());
196 if (!productRS.is_valid())
200 if (bt == art::InRun && productRS.is_full_run())
204 if (bt == art::InSubRun && productRS.is_full_subRun())
208 if (productRS.ranges().empty())
212 auto const r = productRS.run();
213 auto const& productFront = productRS.ranges().front();
214 if (!principalRS.contains(r, productFront.subRun(), productFront.begin()))
216 productRS = art::RangeSet::invalid();
259 template<BranchType BT>
261 getRangeSet(art::OutputHandle
const& oh,
262 art::RangeSet
const& principalRS,
263 bool const producedInThisProcess)
265 if constexpr (!art::detail::range_sets_supported(BT))
267 return art::RangeSet::invalid();
270 auto rs = oh.isValid() ? oh.rangeOfValidity() : art::RangeSet::invalid();
281 if (!producedInThisProcess)
283 maybeInvalidateRangeSet(BT, principalRS, rs);
288 template<BranchType BT>
289 void setProductRangeSetID(art::RangeSet
const& rs,
291 art::EDProduct* product,
292 map<unsigned, unsigned>& checksumToIndexLookup)
294 if constexpr (!art::detail::range_sets_supported(BT))
304 auto it = checksumToIndexLookup.find(rs.checksum());
305 if (it != checksumToIndexLookup.cend())
307 product->setRangeSetID(it->second);
311 unsigned const rsID = getNewRangeSetID(db, BT, rs.run());
312 product->setRangeSetID(rsID);
313 checksumToIndexLookup.emplace(rs.checksum(), rsID);
314 insertIntoEventRanges(db, rs);
315 auto const& eventRangesIDs = getExistingRangeSetIDs(db, rs);
316 insertIntoJoinTable(db, BT, rsID, eventRangesIDs);
324 OutputItem::~OutputItem() =
default;
326 OutputItem::OutputItem(BranchDescription
const& bd)
327 : branchDescription{bd}, product{
nullptr}
330 RootDAQOutFile::RootDAQOutFile(OutputModule* om,
331 string const& fileName,
332 ClosingCriteria
const& fileSwitchCriteria,
333 int const compressionLevel,
334 unsigned freePercent,
336 int64_t
const saveMemoryObjectThreshold,
337 int64_t
const treeMaxVirtualSize,
338 int const splitLevel,
339 int const basketSize,
340 DropMetaData dropMetaData,
341 bool const dropMetaDataForDroppedData)
344 , fileSwitchCriteria_{fileSwitchCriteria}
345 , compressionLevel_{compressionLevel}
346 , freePercent_{freePercent}
348 , saveMemoryObjectThreshold_{saveMemoryObjectThreshold}
349 , treeMaxVirtualSize_{treeMaxVirtualSize}
350 , splitLevel_{splitLevel}
351 , basketSize_{basketSize}
352 , dropMetaData_{dropMetaData}
353 , dropMetaDataForDroppedData_{dropMetaDataForDroppedData}
354 , filePtr_{TFile::Open(file_.c_str(),
"recreate",
"", compressionLevel)}
356 using std::make_unique;
358 metaDataTree_ = RootOutputTree::makeTTree(
359 filePtr_.get(), rootNames::metaDataTreeName(), 0);
360 fileIndexTree_ = RootOutputTree::makeTTree(
361 filePtr_.get(), rootNames::fileIndexTreeName(), 0);
362 parentageTree_ = RootOutputTree::makeTTree(
363 filePtr_.get(), rootNames::parentageTreeName(), 0);
365 make_unique<RootOutputTree>(filePtr_.get(),
368 pEventProductProvenanceVector_,
372 saveMemoryObjectThreshold);
374 make_unique<RootOutputTree>(filePtr_.get(),
377 pSubRunProductProvenanceVector_,
381 saveMemoryObjectThreshold);
382 treePointers_[2] = make_unique<RootOutputTree>(filePtr_.get(),
385 pRunProductProvenanceVector_,
389 saveMemoryObjectThreshold);
391 make_unique<RootOutputTree>(filePtr_.get(),
394 pResultsProductProvenanceVector_,
398 saveMemoryObjectThreshold);
399 #if ART_HEX_VERSION > 0x31400
400 rootFileDB_ = ServiceHandle<DatabaseConnection> {
401 } -> get<TKeyVFSOpenPolicy>(
"RootFileDB",
403 SQLITE_OPEN_CREATE | SQLITE_OPEN_READWRITE);
405 rootFileDB_.reset(ServiceHandle<DatabaseConnection> {
406 } -> get<TKeyVFSOpenPolicy>(
"RootFileDB",
408 SQLITE_OPEN_CREATE | SQLITE_OPEN_READWRITE));
410 beginTime_ = std::chrono::steady_clock::now();
412 root::DictionaryChecker checker;
413 checker.checkDictionaries<EventAuxiliary>();
414 checker.checkDictionaries<SubRunAuxiliary>();
415 checker.checkDictionaries<RunAuxiliary>();
416 checker.checkDictionaries<ResultsAuxiliary>();
417 checker.reportMissingDictionaries();
419 createDatabaseTables();
420 TLOG(TLVL_DEBUG + 32) <<
"RootDAQOutFile ctor complete";
423 art::RootDAQOutFile::~RootDAQOutFile()
426 int sts = sysinfo(&info);
427 auto free_percent =
static_cast<unsigned>(info.freeram * 100 / info.totalram);
428 auto free_MB =
static_cast<unsigned>(info.freeram * info.mem_unit >> 20);
429 TRACE(TLVL_DEBUG + 32,
"~RootDAQOutFile free %%%u %.1fMB (%u) buffers=%fGB mem_unit=%u",
430 free_percent, static_cast<float>(info.freeram * info.mem_unit / (1024 * 1024.0)),
431 free_MB, static_cast<float>(info.bufferram * info.mem_unit / (1024 * 1024 * 1024.0)), info.mem_unit);
432 if (free_percent < freePercent_ || free_MB < freeMB_)
434 TLOG(TLVL_DEBUG + 32) <<
"RootDAQOutFile Flush/DONTNEED";
436 sts = posix_fadvise(filePtr_->GetFd(), 0, 0 , POSIX_FADV_DONTNEED);
438 TLOG(TLVL_DEBUG + 32) <<
"~RootDAQOutFile complete sts=" << sts;
441 void art::RootDAQOutFile::createDatabaseTables()
444 create_table(*rootFileDB_,
449 "UNIQUE (SubRun,begin,end) ON CONFLICT IGNORE"});
451 using namespace cet::sqlite;
452 create_table(*rootFileDB_,
"SubRunRangeSets", column<int>{
"Run"});
453 create_table(*rootFileDB_,
454 "SubRunRangeSets_EventRanges",
455 {
"RangeSetsID INTEGER",
456 "EventRangesID INTEGER",
457 "PRIMARY KEY(RangeSetsID,EventRangesID)"},
460 create_table(*rootFileDB_,
"RunRangeSets", column<int>{
"Run"});
461 create_table(*rootFileDB_,
462 "RunRangeSets_EventRanges",
463 {
"RangeSetsID INTEGER",
464 "EventRangesID INTEGER",
465 "PRIMARY KEY(RangeSetsID,EventRangesID)"},
469 void RootDAQOutFile::setFileStatus(OutputFileStatus
const ofs)
471 std::lock_guard sentry{mutex_};
476 RootDAQOutFile::currentFileName()
const
478 std::lock_guard sentry{mutex_};
482 void RootDAQOutFile::selectProducts()
484 std::lock_guard sentry{mutex_};
485 auto selectProductsToWrite = [
this](BranchType
const bt) {
486 auto& items = selectedOutputItemList_[bt];
487 for (
auto const& pd : om_->keptProducts()[bt] | ranges::views::values)
491 if (bt == InResults && !pd.produced())
495 checkDictionaries(pd);
504 items.try_emplace(pd.productID(), pd);
506 for (
auto const& item : items | ranges::views::values)
508 treePointers_[bt]->addOutputBranch(item.branchDescription,
512 for_each_branch_type(selectProductsToWrite);
515 void RootDAQOutFile::beginInputFile(RootFileBlock
const* rfb,
516 FastCloningEnabled fastCloningEnabled)
518 std::lock_guard sentry{mutex_};
524 cet::exempt_ptr<TTree const> inputTree{
nullptr};
527 if (rfb->fileFormatVersion().value_ < 10)
529 fastCloningEnabled.disable(
530 "The input file has a different ProductID "
531 "schema than the in-memory schema.");
533 inputTree = rfb->tree();
536 if (!treePointers_[InEvent]->checkSplitLevelAndBasketSize(inputTree))
538 fastCloningEnabled.disable(
539 "The splitting level and/or basket size does not match between "
540 "input and output file.");
542 if (inputTree->GetCurrentFile()->GetVersion() < 60001)
544 fastCloningEnabled.disable(
545 "The ROOT version used to write the input file (< 6.00/01)\nhas a "
546 "different splitting policy.");
551 if (not fastCloningEnabled)
553 mf::LogWarning(
"FastCloning") << fastCloningEnabled.disabledBecause();
557 mf::LogInfo(
"FastCloning")
558 <<
"Fast cloning event data products from input file.";
559 wasFastCloned_ = treePointers_[InEvent]->fastCloneTree(inputTree);
562 void RootDAQOutFile::incrementInputFileNumber()
564 std::lock_guard sentry{mutex_};
565 fp_.update_inputFile();
568 void RootDAQOutFile::respondToCloseInputFile(FileBlock
const&)
570 std::lock_guard sentry{mutex_};
571 cet::for_all(treePointers_, [](
auto const& p) { p->setEntries(); });
574 bool RootDAQOutFile::requestsToCloseFile()
576 std::lock_guard sentry{mutex_};
577 using namespace std::chrono;
578 unsigned int constexpr oneK{1024u};
579 fp_.updateSize(filePtr_->GetSize() / oneK);
580 fp_.updateAge(duration_cast<seconds>(steady_clock::now() - beginTime_));
581 return fileSwitchCriteria_.should_close(fp_);
584 void RootDAQOutFile::writeOne(EventPrincipal
const& e)
586 std::lock_guard sentry{mutex_};
587 TLOG(TLVL_DEBUG + 33) <<
"Start of RootDAQOutFile::writeOne";
590 pEventAux_ = &e.eventAux();
594 fillBranches<InEvent>(e, pEventProductProvenanceVector_);
597 if (!dataTypeReported_)
599 string dataType{
"MC"};
600 if (pEventAux_->isRealData())
604 dataTypeReported_ =
true;
607 fileIndex_.addEntry(pEventAux_->eventID(), fp_.eventEntryNumber());
609 TLOG(TLVL_DEBUG + 33) <<
"End of RootDAQOutFile::writeOne";
612 void RootDAQOutFile::writeSubRun(SubRunPrincipal
const& sr)
614 std::lock_guard sentry{mutex_};
615 pSubRunAux_ = &sr.subRunAux();
616 pSubRunAux_->setRangeSetID(subRunRSID_);
617 fillBranches<InSubRun>(sr, pSubRunProductProvenanceVector_);
618 fileIndex_.addEntry(EventID::invalidEvent(pSubRunAux_->subRunID()),
619 fp_.subRunEntryNumber());
620 fp_.update_subRun(status_);
623 void RootDAQOutFile::writeRun(RunPrincipal
const& r)
625 std::lock_guard sentry{mutex_};
626 pRunAux_ = &r.runAux();
627 pRunAux_->setRangeSetID(runRSID_);
628 fillBranches<InRun>(r, pRunProductProvenanceVector_);
629 fileIndex_.addEntry(EventID::invalidEvent(pRunAux_->runID()),
630 fp_.runEntryNumber());
631 fp_.update_run(status_);
634 void RootDAQOutFile::writeParentageRegistry()
636 std::lock_guard sentry{mutex_};
637 auto pid = root::getObjectRequireDict<ParentageID>();
638 ParentageID
const* hash = &pid;
639 if (!parentageTree_->Branch(
640 rootNames::parentageIDBranchName().c_str(), &hash, basketSize_, 0))
642 throw Exception(errors::FatalRootError)
643 <<
"Failed to create a branch for ParentageIDs in the output file";
646 auto par = root::getObjectRequireDict<Parentage>();
647 Parentage
const* desc = ∥
648 if (!parentageTree_->Branch(
649 rootNames::parentageBranchName().c_str(), &desc, basketSize_, 0))
651 throw Exception(errors::FatalRootError)
652 <<
"Failed to create a branch for Parentages in the output file";
655 for (
auto const& pr : ParentageRegistry::get())
659 parentageTree_->Fill();
661 parentageTree_->SetBranchAddress(rootNames::parentageIDBranchName().c_str(),
663 parentageTree_->SetBranchAddress(rootNames::parentageBranchName().c_str(),
667 void RootDAQOutFile::writeFileFormatVersion()
669 std::lock_guard sentry{mutex_};
670 FileFormatVersion
const ver{getFileFormatVersion(), getFileFormatEra()};
671 auto const* pver = &ver;
672 TBranch* b = metaDataTree_->Branch(
673 metaBranchRootName<FileFormatVersion>(), &pver, basketSize_, 0);
679 void RootDAQOutFile::writeFileIndex()
681 std::lock_guard sentry{mutex_};
682 fileIndex_.sortBy_Run_SubRun_Event();
683 FileIndex::Element elem{};
684 auto const* findexElemPtr = &elem;
685 TBranch* b = fileIndexTree_->Branch(
686 metaBranchRootName<FileIndex::Element>(), &findexElemPtr, basketSize_, 0);
689 for (
auto& entry : fileIndex_)
691 findexElemPtr = &entry;
697 void RootDAQOutFile::writeProcessConfigurationRegistry()
703 void RootDAQOutFile::writeProcessHistoryRegistry()
705 std::lock_guard sentry{mutex_};
706 ProcessHistoryMap pHistMap;
707 for (
auto const& pr : ProcessHistoryRegistry::get())
709 pHistMap.emplace(pr);
711 auto const* p = &pHistMap;
712 TBranch* b = metaDataTree_->Branch(
713 metaBranchRootName<ProcessHistoryMap>(), &p, basketSize_, 0);
716 throw Exception(errors::LogicError)
717 <<
"Unable to locate required "
718 "ProcessHistoryMap branch in output "
724 void RootDAQOutFile::writeFileCatalogMetadata(
725 FileStatsCollector
const& stats,
726 FileCatalogMetadata::collection_type
const& md,
727 FileCatalogMetadata::collection_type
const& ssmd)
729 std::lock_guard sentry{mutex_};
730 using namespace cet::sqlite;
731 Ntuple<string, string> fileCatalogMetadata{
732 *rootFileDB_,
"FileCatalog_metadata", {{
"Name",
"Value"}},
true};
733 for (
auto const& [key, value] : md)
735 fileCatalogMetadata.insert(key, value);
739 fileCatalogMetadata.insert(
"file_format",
"\"artroot\"");
742 namespace bpt = boost::posix_time;
743 auto formatted_time = [](
auto const& t) {
744 return cet::canonical_string(bpt::to_iso_extended_string(t));
746 fileCatalogMetadata.insert(
"start_time",
747 formatted_time(stats.outputFileOpenTime()));
749 fileCatalogMetadata.insert(
751 formatted_time(boost::posix_time::second_clock::universal_time()));
753 if (!stats.seenSubRuns().empty())
755 auto I = find_if(md.crbegin(), md.crend(), [](
auto const& p) {
756 return p.first ==
"art.run_type";
760 std::ostringstream buf;
762 for (
auto const& srid : stats.seenSubRuns())
764 buf <<
"[ " << srid.run() <<
", " << srid.subRun() <<
", "
765 << cet::canonical_string(I->second) <<
" ], ";
768 buf.seekp(-2, std::ios_base::cur);
770 fileCatalogMetadata.insert(
"runs", buf.str());
774 fileCatalogMetadata.insert(
"event_count",
775 std::to_string(stats.eventsThisFile()));
776 fileCatalogMetadata.insert(
"first_event",
777 std::to_string(stats.lowestEventID().event()));
778 fileCatalogMetadata.insert(
"last_event",
779 std::to_string(stats.highestEventID().event()));
781 if (!stats.parents().empty())
783 std::ostringstream pstring;
785 for (
auto const& parent : stats.parents())
787 pstring << cet::canonical_string(parent) <<
", ";
790 pstring.seekp(-2, std::ios_base::cur);
792 fileCatalogMetadata.insert(
"parents", pstring.str());
797 auto eidToTuple = [](EventID
const& eid) ->
string {
798 std::ostringstream eidStr;
799 eidStr <<
"[ " << eid.run() <<
", " << eid.subRun() <<
", " << eid.event()
803 fileCatalogMetadata.insert(
"art.first_event",
804 eidToTuple(stats.lowestEventID()));
805 fileCatalogMetadata.insert(
"art.last_event",
806 eidToTuple(stats.highestEventID()));
807 fileCatalogMetadata.insert(
"art.file_format_era",
808 cet::canonical_string(getFileFormatEra()));
809 fileCatalogMetadata.insert(
"art.file_format_version",
810 std::to_string(getFileFormatVersion()));
813 for (
auto const& [key, value] : ssmd)
815 fileCatalogMetadata.insert(key, value);
819 void RootDAQOutFile::writeParameterSetRegistry()
821 std::lock_guard sentry{mutex_};
822 fhicl::ParameterSetRegistry::exportTo(*rootFileDB_);
825 void RootDAQOutFile::writeProductDescriptionRegistry()
827 std::lock_guard sentry{mutex_};
831 auto productDescriptionsToWrite = [
this, ®](BranchType
const bt) {
832 for (
auto const& desc :
833 descriptionsToPersist_[bt] | ranges::views::values)
835 reg.productList_.emplace(BranchKey{desc}, desc);
838 for_each_branch_type(productDescriptionsToWrite);
839 ProductRegistry
const* regp = ®
840 TBranch* b = metaDataTree_->Branch(
841 metaBranchRootName<ProductRegistry>(), ®p, basketSize_, 0);
847 void RootDAQOutFile::writeProductDependencies()
849 std::lock_guard sentry{mutex_};
850 BranchChildren
const* ppDeps = &om_->branchChildren();
851 TBranch* b = metaDataTree_->Branch(
852 metaBranchRootName<BranchChildren>(), &ppDeps, basketSize_, 0);
858 void RootDAQOutFile::writeResults(ResultsPrincipal& resp)
860 std::lock_guard sentry{mutex_};
861 pResultsAux_ = &resp.resultsAux();
862 fillBranches<InResults>(resp, pResultsProductProvenanceVector_);
865 void RootDAQOutFile::writeTTrees()
867 TLOG(TLVL_DEBUG + 33) <<
"Start of RootDAQOutFile::writeTTrees";
868 std::lock_guard sentry{mutex_};
869 RootOutputTree::writeTTree(metaDataTree_);
870 TLOG(TLVL_DEBUG + 33) <<
"RootDAQOutFile::writeTTrees after writing metaDataTree_";
871 RootOutputTree::writeTTree(fileIndexTree_);
872 TLOG(TLVL_DEBUG + 33) <<
"RootDAQOutFile::writeTTrees after writing fileIndexTree_";
873 RootOutputTree::writeTTree(parentageTree_);
874 TLOG(TLVL_DEBUG + 33) <<
"RootDAQOutFile::writeTTrees after writing parentageTree_";
875 for_each_branch_type(
876 [
this](BranchType
const bt) { treePointers_[bt]->writeTree(); });
877 TLOG(TLVL_DEBUG + 33) <<
"End of RootDAQOutFile::writeTTrees";
880 void RootDAQOutFile::setSubRunAuxiliaryRangeSetID(RangeSet
const& ranges)
882 std::lock_guard sentry{mutex_};
883 subRunRSID_ = getNewRangeSetID(*rootFileDB_, InSubRun, ranges.run());
884 insertIntoEventRanges(*rootFileDB_, ranges);
885 auto const& eventRangesIDs = getExistingRangeSetIDs(*rootFileDB_, ranges);
886 insertIntoJoinTable(*rootFileDB_, InSubRun, subRunRSID_, eventRangesIDs);
889 void RootDAQOutFile::setRunAuxiliaryRangeSetID(RangeSet
const& ranges)
891 std::lock_guard sentry{mutex_};
892 runRSID_ = getNewRangeSetID(*rootFileDB_, InRun, ranges.run());
893 insertIntoEventRanges(*rootFileDB_, ranges);
894 auto const& eventRangesIDs = getExistingRangeSetIDs(*rootFileDB_, ranges);
895 insertIntoJoinTable(*rootFileDB_, InRun, runRSID_, eventRangesIDs);
898 template<BranchType BT>
900 RootDAQOutFile::getProduct(OutputHandle
const& oh,
901 RangeSet
const& prunedProductRS,
902 string const& wrappedName)
904 std::lock_guard sentry{mutex_};
905 if constexpr (detail::range_sets_supported(BT))
907 if (!prunedProductRS.is_valid())
909 return dummyProductCache_.product(wrappedName);
912 return oh.isValid() ? oh.wrapper() : dummyProductCache_.product(wrappedName);
915 template<BranchType BT>
916 void RootDAQOutFile::fillBranches(Principal
const& principal,
917 vector<ProductProvenance>* vpp)
919 TLOG(TLVL_DEBUG + 33) <<
"Start of RootDAQOutFile::fillBranches";
920 std::lock_guard sentry{mutex_};
921 bool const fastCloning{BT == InEvent && wasFastCloned_};
922 map<unsigned, unsigned> checksumToIndex;
923 auto const& principalRS = principal.seenRanges();
927 bool const drop_no_metadata{dropMetaData_ == DropMetaData::DropNone};
928 bool const drop_prior_metadata{dropMetaData_ == DropMetaData::DropPrior};
929 bool const drop_all_metadata{dropMetaData_ == DropMetaData::DropAll};
931 std::set<ProductProvenance> keptprv;
932 for (
auto const& [pid, val] : selectedOutputItemList_[BT])
934 auto const& bd = val.branchDescription;
935 descriptionsToPersist_[BT].try_emplace(pid, bd);
936 bool const produced = bd.produced();
937 bool const resolveProd{produced || !fastCloning ||
938 treePointers_[BT]->uncloned(bd.branchName())};
940 bool const keepProvenance =
941 drop_no_metadata || (produced && drop_prior_metadata);
942 auto const& oh = principal.getForOutput(pid, resolveProd);
943 auto prov = keptprv.begin();
946 if (oh.productProvenance())
948 prov = keptprv.insert(*oh.productProvenance()).first;
949 if (!drop_all_metadata && !dropMetaDataForDroppedData_)
952 vector<ProductProvenance const*> stacked_pp;
953 stacked_pp.push_back(&*oh.productProvenance());
954 while (not empty(stacked_pp))
956 auto current_pp = stacked_pp.back();
957 stacked_pp.pop_back();
958 for (
auto const parent_bid :
959 current_pp->parentage().parents())
972 auto parent_bd = principal.getProductDescription(parent_bid);
978 descriptionsToPersist_[BT].try_emplace(parent_bid,
980 if (!parent_bd->produced())
986 principal.branchToProductProvenance(parent_bid);
987 if (!parent_pp || !drop_no_metadata)
991 if (!keptprv.insert(*parent_pp).second)
996 if (!drop_all_metadata && !dropMetaDataForDroppedData_)
998 stacked_pp.push_back(parent_pp.get());
1009 auto status = productstatus::dropped();
1012 status = productstatus::neverCreated();
1014 prov = keptprv.emplace(pid, status).first;
1024 auto const& rs = getRangeSet<BT>(oh, principalRS, produced);
1025 if (detail::range_sets_supported(BT) && !rs.is_valid())
1036 auto prov_bid = prov->productID();
1037 if (keptprv.erase(*prov) != 1ull)
1039 throw Exception(errors::LogicError,
"KeptProvenance::setStatus")
1040 <<
"Attempt to set product status for product whose provenance "
1041 "is not being recorded.\n";
1045 .emplace(prov_bid, productstatus::dummyToPreventDoubleCount())
1048 auto const* product = getProduct<BT>(oh, rs, bd.wrappedName());
1049 setProductRangeSetID<BT>(
1050 rs, *rootFileDB_,
const_cast<EDProduct*
>(product), checksumToIndex);
1051 val.product = product;
1054 vpp->assign(keptprv.begin(), keptprv.end());
1055 for (
auto const& val : *vpp)
1057 if (val.productStatus() == productstatus::uninitialized())
1059 throw Exception(errors::LogicError,
1060 "RootDAQOutFile::fillBranches(principal, vpp):")
1061 <<
"Attempt to write a product with uninitialized provenance!\n";
1065 TLOG(TLVL_DEBUG + 33) <<
"RootDAQOutFile::fillBranches before fillTree call";
1066 treePointers_[BT]->fillTree();
1067 TLOG(TLVL_DEBUG + 33) <<
"RootDAQOutFile::fillBranches after fillTree call";
1069 TLOG(TLVL_DEBUG + 33) <<
"End of RootDAQOutFile::fillBranches";