1 #include "art/Framework/Core/FileBlock.h"
2 #include "art/Framework/Core/ModuleMacros.h"
3 #include "art/Framework/Core/OutputModule.h"
4 #include "art/Framework/Core/ResultsProducer.h"
5 #include "art/Framework/Core/RPManager.h"
6 #include "art/Framework/IO/FileStatsCollector.h"
7 #include "art/Framework/IO/PostCloseFileRenamer.h"
8 #include "art/Framework/IO/Root/DropMetaData.h"
9 #include "art/Framework/IO/Root/RootOutputClosingCriteria.h"
10 #include "art/Framework/Principal/EventPrincipal.h"
11 #include "art/Framework/Principal/Event.h"
12 #include "art/Framework/Principal/ResultsPrincipal.h"
13 #include "art/Framework/Principal/Results.h"
14 #include "art/Framework/Principal/RunPrincipal.h"
15 #include "art/Framework/Principal/Run.h"
16 #include "art/Framework/Principal/SubRunPrincipal.h"
17 #include "art/Framework/Principal/SubRun.h"
18 #include "art/Persistency/Provenance/ProductMetaData.h"
19 #include "art/Utilities/ConfigurationTable.h"
20 #include "art/Utilities/parent_path.h"
21 #include "art/Utilities/unique_filename.h"
23 #include "artdaq/ArtModules/RootDAQOutput/RootDAQOutFile.h"
24 #include "artdaq/ArtModules/RootDAQOutput/detail/rootOutputConfigurationTools.h"
25 #include "art/Framework/IO/detail/logFileAction.h"
27 #include "canvas/Persistency/Provenance/FileFormatVersion.h"
28 #include "canvas/Utilities/Exception.h"
29 #include "fhiclcpp/ParameterSet.h"
30 #include "fhiclcpp/types/Atom.h"
31 #include "fhiclcpp/types/OptionalAtom.h"
32 #include "fhiclcpp/types/Table.h"
33 #include "messagefacility/MessageLogger/MessageLogger.h"
34 #define TRACE_NAME "RootDAQOut"
37 #if ART_HEX_VERSION >= 0x20703
39 # include "TBranchElement.h"
61 static constexpr
char const* default_tmpDir {
"<parent-path-of-filename>"};
65 using Name = fhicl::Name;
66 using Comment = fhicl::Comment;
67 template <
typename T>
using Atom = fhicl::Atom<T>;
68 template <
typename T>
using OptionalAtom = fhicl::OptionalAtom<T>;
70 fhicl::TableFragment<art::OutputModule::Config> omConfig;
71 Atom<std::string> catalog { Name(
"catalog"),
"" };
72 OptionalAtom<bool> dropAllEvents { Name(
"dropAllEvents") };
73 Atom<bool> dropAllSubRuns { Name(
"dropAllSubRuns"),
false };
74 OptionalAtom<bool> fastCloning { Name(
"fastCloning") };
75 Atom<std::string> tmpDir { Name(
"tmpDir"), default_tmpDir };
76 Atom<int> compressionLevel { Name(
"compressionLevel"), 7 };
77 Atom<int64_t> saveMemoryObjectThreshold { Name(
"saveMemoryObjectThreshold"), -1l };
78 Atom<int64_t> treeMaxVirtualSize { Name(
"treeMaxVirtualSize"), -1 };
79 Atom<int> splitLevel { Name(
"splitLevel"), 99 };
80 Atom<int> basketSize { Name(
"basketSize"), 16384 };
81 Atom<bool> dropMetaDataForDroppedData { Name(
"dropMetaDataForDroppedData"),
false };
82 Atom<std::string> dropMetaData { Name(
"dropMetaData"),
"NONE" };
83 Atom<bool> writeParameterSets { Name(
"writeParameterSets"),
true };
84 fhicl::Table<ClosingCriteria::Config> fileProperties { Name(
"fileProperties") };
93 using namespace fhicl::detail;
94 ParameterBase* adjustFilename {
const_cast<fhicl::Atom<std::string>*
>(&omConfig().fileName)};
95 adjustFilename->set_value_type(fhicl::value_type::REQUIRED);
99 std::set<std::string> operator()()
const
101 std::set<std::string> keys {art::OutputModule::Config::KeysToIgnore::get()};
102 keys.insert(
"results");
109 using Parameters = art::WrappedTable<Config,Config::KeysToIgnore>;
113 void postSelectProducts(FileBlock
const&)
override;
115 void beginJob()
override;
116 void endJob()
override;
118 void event(EventPrincipal
const&)
override;
120 void beginSubRun(SubRunPrincipal
const&)
override;
121 void endSubRun(SubRunPrincipal
const&)
override;
123 void beginRun(RunPrincipal
const&)
override;
124 void endRun(RunPrincipal
const&)
override;
128 std::string
const& lastClosedFileName()
const override;
129 void openFile(FileBlock
const&)
override;
130 void respondToOpenInputFile(FileBlock
const&)
override;
131 void readResults(ResultsPrincipal
const& resp)
override;
132 void respondToCloseInputFile(FileBlock
const&)
override;
133 void incrementInputFileNumber()
override;
134 # if ART_HEX_VERSION < 0x20703
135 Boundary fileSwitchBoundary()
const override;
137 void write(EventPrincipal&)
override;
138 void writeSubRun(SubRunPrincipal&)
override;
139 void writeRun(RunPrincipal&)
override;
140 void setSubRunAuxiliaryRangeSetID(RangeSet
const&)
override;
141 void setRunAuxiliaryRangeSetID(RangeSet
const&)
override;
142 bool isFileOpen()
const override;
143 void setFileStatus(OutputFileStatus)
override;
144 bool requestsToCloseFile()
const override;
146 void startEndFile()
override;
147 void writeFileFormatVersion()
override;
148 void writeFileIndex()
override;
149 void writeEventHistory()
override;
150 void writeProcessConfigurationRegistry()
override;
151 void writeProcessHistoryRegistry()
override;
152 void writeParameterSetRegistry()
override;
153 void writeProductDescriptionRegistry()
override;
154 void writeParentageRegistry()
override;
155 void writeBranchIDListRegistry()
override;
157 doWriteFileCatalogMetadata(FileCatalogMetadata::collection_type
const& md,
158 FileCatalogMetadata::collection_type
const& ssmd)
160 void writeProductDependencies()
override;
161 void finishEndFile()
override;
162 void doRegisterProducts(MasterProductRegistry& mpr,
163 ModuleDescription
const& md)
override;
167 std::string
const catalog_;
168 bool dropAllEvents_ {
false};
169 bool dropAllSubRuns_;
170 std::string
const moduleLabel_;
171 int inputFileCount_ {0};
172 std::unique_ptr<RootDAQOutFile> rootOutputFile_ {
nullptr};
173 FileStatsCollector fstats_;
174 std::string
const filePattern_;
176 std::string lastClosedFileName_ {};
179 int const compressionLevel_;
180 int64_t
const saveMemoryObjectThreshold_;
181 int64_t
const treeMaxVirtualSize_;
182 int const splitLevel_;
183 int const basketSize_;
184 DropMetaData dropMetaData_;
185 bool dropMetaDataForDroppedData_;
189 bool fastCloning_ {
true};
194 bool writeParameterSets_;
195 ClosingCriteria fileProperties_;
202 RootDAQOut(Parameters
const& config)
203 : OutputModule{ config().omConfig, config.get_PSet() }
204 , catalog_{ config().catalog() }
205 , dropAllSubRuns_{ config().dropAllSubRuns() }
206 , moduleLabel_{ config.get_PSet().get<
string>(
"module_label") }
207 , fstats_{ moduleLabel_, processName() }
208 , filePattern_{ config().omConfig().fileName() }
209 , tmpDir_{ config().tmpDir() == default_tmpDir ? parent_path(filePattern_) : config().tmpDir() }
210 , compressionLevel_{ config().compressionLevel() }
211 , saveMemoryObjectThreshold_{ config().saveMemoryObjectThreshold() }
212 , treeMaxVirtualSize_{ config().treeMaxVirtualSize() }
213 , splitLevel_{ config().splitLevel() }
214 , basketSize_{ config().basketSize() }
215 , dropMetaData_{ config().dropMetaData() }
216 , dropMetaDataForDroppedData_{ config().dropMetaDataForDroppedData() }
217 , writeParameterSets_{ config().writeParameterSets() }
219 (detail::validateFileNamePattern(config.get_PSet().has_key(config().fileProperties.name()), filePattern_),
220 config().fileProperties()) }
221 , rpm_{ config.get_PSet() }
223 bool const dropAllEventsSet{ config().dropAllEvents(dropAllEvents_) };
224 dropAllEvents_ = detail::shouldDropEvents(dropAllEventsSet, dropAllEvents_, dropAllSubRuns_);
231 bool const fastCloningSet{ config().fastCloning(fastCloning_) };
232 fastCloning_ = detail::shouldFastClone(fastCloningSet, fastCloning_, wantAllEvents(), fileProperties_);
234 if (!writeParameterSets_) {
235 mf::LogWarning(
"PROVENANCE")
236 <<
"Output module " << moduleLabel_ <<
" has parameter writeParameterSets set to false.\n"
237 <<
"Parameter set provenance will not be available in subsequent jobs.\n"
238 <<
"Check your experiment's policy on this issue to avoid future problems\n"
239 <<
"with analysis reproducibility.\n";
245 openFile(FileBlock
const& fb)
253 respondToOpenInputFile(fb);
259 postSelectProducts(FileBlock
const& fb)
262 rootOutputFile_->selectProducts(fb);
268 respondToOpenInputFile(FileBlock
const& fb)
271 if (!isFileOpen())
return;
273 bool fastCloneThisOne = fastCloning_ && (fb.tree() !=
nullptr) &&
274 ((remainingEvents() < 0) ||
275 (remainingEvents() >= fb.tree()->GetEntries()));
276 if (fastCloning_ && !fastCloneThisOne) {
277 mf::LogWarning(
"FastCloning")
278 <<
"Fast cloning deactivated for this input file due to "
279 <<
"empty event tree and/or event limits.";
281 if (fastCloneThisOne && !fb.fastClonable()) {
282 mf::LogWarning(
"FastCloning")
283 <<
"Fast cloning deactivated for this input file due to "
284 <<
"information in FileBlock.";
285 fastCloneThisOne =
false;
287 rootOutputFile_->beginInputFile(fb, fastCloneThisOne && fastCloning_);
288 fstats_.recordInputFile(fb.fileName());
293 readResults(ResultsPrincipal
const& resp)
295 rpm_.for_each_RPWorker([&resp](RPWorker& w) {
296 Results
const res{ resp, w.moduleDescription() };
297 w.rp().doReadResults(res);
303 respondToCloseInputFile(FileBlock
const& fb)
306 rootOutputFile_->respondToCloseInputFile(fb);
310 #define STRINGIFY(x) #x
311 #define TOSTRING(x) STRINGIFY(x)
313 void art::RootDAQOut::write(EventPrincipal& ep)
315 TLOG(10) <<
"RootDAQOut::write begin - line " << __LINE__;
316 if (dropAllEvents_) {
319 if (hasNewlyDroppedBranch()[InEvent]) {
320 ep.addToProcessHistory();
322 rootOutputFile_->writeOne(ep);
323 fstats_.recordEvent(ep.id());
324 TLOG(10) <<
"RootDAQOut::write done/return";
329 setSubRunAuxiliaryRangeSetID(RangeSet
const& rs)
331 rootOutputFile_->setSubRunAuxiliaryRangeSetID(rs);
336 writeSubRun(SubRunPrincipal& sr)
338 if (dropAllSubRuns_) {
341 if (hasNewlyDroppedBranch()[InSubRun]) {
342 sr.addToProcessHistory();
344 rootOutputFile_->writeSubRun(sr);
345 fstats_.recordSubRun(sr.id());
350 setRunAuxiliaryRangeSetID(RangeSet
const& rs)
352 rootOutputFile_->setRunAuxiliaryRangeSetID(rs);
357 writeRun(RunPrincipal& r)
359 if (hasNewlyDroppedBranch()[InRun]) {
360 r.addToProcessHistory();
362 rootOutputFile_->writeRun(r);
363 fstats_.recordRun(r.id());
370 auto resp = std::make_unique<ResultsPrincipal>(ResultsAuxiliary{},
371 description().processConfiguration());
372 if (ProductMetaData::instance().productProduced(InResults) ||
373 hasNewlyDroppedBranch()[InResults]) {
374 resp->addToProcessHistory();
376 rpm_.for_each_RPWorker([&resp](RPWorker& w) {
377 Results res{ *resp, w.moduleDescription() };
378 w.rp().doWriteResults(*resp, res);
380 rootOutputFile_->writeResults(*resp);
385 writeFileFormatVersion()
387 rootOutputFile_->writeFileFormatVersion();
394 rootOutputFile_->writeFileIndex();
401 rootOutputFile_->writeEventHistory();
406 writeProcessConfigurationRegistry()
408 rootOutputFile_->writeProcessConfigurationRegistry();
413 writeProcessHistoryRegistry()
415 rootOutputFile_->writeProcessHistoryRegistry();
420 writeParameterSetRegistry()
422 if (writeParameterSets_) {
423 rootOutputFile_->writeParameterSetRegistry();
429 writeProductDescriptionRegistry()
431 rootOutputFile_->writeProductDescriptionRegistry();
436 writeParentageRegistry()
438 rootOutputFile_->writeParentageRegistry();
443 writeBranchIDListRegistry()
445 rootOutputFile_->writeBranchIDListRegistry();
450 doWriteFileCatalogMetadata(FileCatalogMetadata::collection_type
const& md,
451 FileCatalogMetadata::collection_type
const& ssmd)
453 rootOutputFile_->writeFileCatalogMetadata(fstats_, md, ssmd);
457 art::RootDAQOut::writeProductDependencies()
459 rootOutputFile_->writeProductDependencies();
463 art::RootDAQOut::finishEndFile()
465 # if ART_HEX_VERSION >= 0x20703
466 std::string
const currentFileName{ rootOutputFile_->currentFileName() };
467 rootOutputFile_->writeTTrees();
468 rootOutputFile_.reset();
469 fstats_.recordFileClose();
470 lastClosedFileName_ = PostCloseFileRenamer{ fstats_ }.maybeRenameFile(rootOutputFile_->currentFileName(), filePattern_);
472 rootOutputFile_->finishEndFile();
473 fstats_.recordFileClose();
474 lastClosedFileName_ = PostCloseFileRenamer{ fstats_ }.maybeRenameFile(rootOutputFile_->currentFileName(), filePattern_);
475 rootOutputFile_.reset();
477 detail::logFileAction(
"Closed output file ", lastClosedFileName_);
478 rpm_.invoke(&ResultsProducer::doClear);
483 doRegisterProducts(MasterProductRegistry& mpr,
484 ModuleDescription
const& md)
487 rpm_.for_each_RPWorker([&mpr, &md](RPWorker& w) {
488 auto const& params = w.params();
489 w.setModuleDescription(ModuleDescription{ params.rpPSetID,
491 md.moduleLabel() +
'#' + params.rpLabel,
492 md.processConfiguration() });
493 w.rp().registerProducts(mpr, w.moduleDescription());
498 art::RootDAQOut::setFileStatus(OutputFileStatus
const ofs)
501 rootOutputFile_->setFileStatus(ofs);
508 TLOG(10) <<
"RootDAQOut::isFileOpen start";
509 bool ret = rootOutputFile_.get() !=
nullptr;
510 TLOG(10) <<
"RootDAQOut::isFileOpen return " << ret;
515 art::RootDAQOut::incrementInputFileNumber()
518 rootOutputFile_->incrementInputFileNumber();
523 requestsToCloseFile()
const
527 ret = rootOutputFile_->requestsToCloseFile();
535 #if ART_HEX_VERSION < 0x20703
538 fileSwitchBoundary()
const
540 TLOG(10) <<
"RootDAQOut::fileSwitchBoundary start";
541 auto bb = fileProperties_.granularity();
542 TLOG(10) <<
"RootDAQOut::fileSwitchBoundary done/return";
551 if (inputFileCount_ == 0) {
552 throw art::Exception(art::errors::LogicError)
553 <<
"Attempt to open output file before input file. "
554 <<
"Please report this to the core framework developers.\n";
556 rootOutputFile_ = std::make_unique<RootDAQOutFile>(
this,
557 unique_filename(tmpDir_ +
"/RootDAQOut"),
560 saveMemoryObjectThreshold_,
565 dropMetaDataForDroppedData_,
567 fstats_.recordFileOpen();
568 detail::logFileAction(
"Opened output file with pattern ", filePattern_);
573 lastClosedFileName()
const
575 if (lastClosedFileName_.empty()) {
576 throw Exception(errors::LogicError,
"RootDAQOut::currentFileName(): ")
577 <<
"called before meaningful.\n";
579 return lastClosedFileName_;
586 rpm_.invoke(&ResultsProducer::doBeginJob);
593 rpm_.invoke(&ResultsProducer::doEndJob);
598 event(EventPrincipal
const& ep)
600 rpm_.for_each_RPWorker([&ep](RPWorker& w) {
601 Event
const e{ ep, w.moduleDescription() };
608 beginSubRun(art::SubRunPrincipal
const& srp)
610 rpm_.for_each_RPWorker([&srp](RPWorker& w) {
611 SubRun
const sr{ srp, w.moduleDescription() };
612 w.rp().doBeginSubRun(sr);
618 endSubRun(art::SubRunPrincipal
const& srp)
620 rpm_.for_each_RPWorker([&srp](RPWorker& w) {
621 SubRun
const sr{ srp, w.moduleDescription() };
622 w.rp().doEndSubRun(sr);
628 beginRun(art::RunPrincipal
const& rp)
630 rpm_.for_each_RPWorker([&rp](RPWorker& w) {
631 Run
const r{ rp, w.moduleDescription() };
632 w.rp().doBeginRun(r);
638 endRun(art::RunPrincipal
const& rp)
640 rpm_.for_each_RPWorker([&rp](RPWorker& w) {
641 Run
const r{ rp, w.moduleDescription() };