1 #include "art/Framework/Core/FileBlock.h"
2 #include "art/Framework/Core/ModuleMacros.h"
3 #include "art/Framework/Core/OutputModule.h"
4 #include "art/Framework/Core/ResultsProducer.h"
5 #include "art/Framework/Core/RPManager.h"
6 #include "art/Framework/IO/FileStatsCollector.h"
7 #include "art/Framework/IO/PostCloseFileRenamer.h"
8 #include "art/Framework/IO/Root/DropMetaData.h"
9 #include "art/Framework/IO/Root/RootOutputClosingCriteria.h"
10 #include "art/Framework/Principal/EventPrincipal.h"
11 #include "art/Framework/Principal/Event.h"
12 #include "art/Framework/Principal/ResultsPrincipal.h"
13 #include "art/Framework/Principal/Results.h"
14 #include "art/Framework/Principal/RunPrincipal.h"
15 #include "art/Framework/Principal/Run.h"
16 #include "art/Framework/Principal/SubRunPrincipal.h"
17 #include "art/Framework/Principal/SubRun.h"
18 #include "art/Persistency/Provenance/ProductMetaData.h"
19 #include "art/Utilities/ConfigurationTable.h"
20 #include "art/Utilities/parent_path.h"
21 #include "art/Utilities/unique_filename.h"
23 #include "artdaq/ArtModules/RootDAQOutput/RootDAQOutFile.h"
24 #include "artdaq/ArtModules/RootDAQOutput/detail/rootOutputConfigurationTools.h"
25 #include "art/Framework/IO/detail/logFileAction.h"
27 #include "canvas/Persistency/Provenance/FileFormatVersion.h"
28 #include "canvas/Utilities/Exception.h"
29 #include "fhiclcpp/ParameterSet.h"
30 #include "fhiclcpp/types/Atom.h"
31 #include "fhiclcpp/types/OptionalAtom.h"
32 #include "fhiclcpp/types/Table.h"
33 #include "messagefacility/MessageLogger/MessageLogger.h"
34 #define TRACE_NAME "RootDAQOut"
38 # include "TBranchElement.h"
59 static constexpr
char const* default_tmpDir {
"<parent-path-of-filename>"};
64 using Name = fhicl::Name;
65 using Comment = fhicl::Comment;
66 template <
typename T>
using Atom = fhicl::Atom<T>;
67 template <
typename T>
using OptionalAtom = fhicl::OptionalAtom<T>;
69 fhicl::TableFragment<art::OutputModule::Config> omConfig;
70 Atom<std::string> catalog { Name(
"catalog"),
"" };
71 OptionalAtom<bool> dropAllEvents { Name(
"dropAllEvents") };
72 Atom<bool> dropAllSubRuns { Name(
"dropAllSubRuns"),
false };
73 OptionalAtom<bool> fastCloning { Name(
"fastCloning") };
74 Atom<std::string> tmpDir { Name(
"tmpDir"), default_tmpDir };
75 Atom<int> compressionLevel { Name(
"compressionLevel"), 7 };
76 Atom<int64_t> saveMemoryObjectThreshold { Name(
"saveMemoryObjectThreshold"), -1l };
77 Atom<int64_t> treeMaxVirtualSize { Name(
"treeMaxVirtualSize"), -1 };
78 Atom<int> splitLevel { Name(
"splitLevel"), 99 };
79 Atom<int> basketSize { Name(
"basketSize"), 16384 };
80 Atom<bool> dropMetaDataForDroppedData { Name(
"dropMetaDataForDroppedData"),
false };
81 Atom<std::string> dropMetaData { Name(
"dropMetaData"),
"NONE" };
82 Atom<bool> writeParameterSets { Name(
"writeParameterSets"),
true };
83 fhicl::Table<ClosingCriteria::Config> fileProperties { Name(
"fileProperties") };
92 using namespace fhicl::detail;
93 ParameterBase* adjustFilename {
const_cast<fhicl::Atom<std::string>*
>(&omConfig().fileName)};
94 adjustFilename->set_value_type(fhicl::value_type::REQUIRED);
99 std::set<std::string> operator()()
const
101 std::set<std::string> keys {art::OutputModule::Config::KeysToIgnore::get()};
102 keys.insert(
"results");
109 using Parameters = art::WrappedTable<Config,Config::KeysToIgnore>;
113 void postSelectProducts(FileBlock
const&)
override;
115 void beginJob()
override;
116 void endJob()
override;
118 void event(EventPrincipal
const&)
override;
120 void beginSubRun(SubRunPrincipal
const&)
override;
121 void endSubRun(SubRunPrincipal
const&)
override;
123 void beginRun(RunPrincipal
const&)
override;
124 void endRun(RunPrincipal
const&)
override;
128 std::string
const& lastClosedFileName()
const override;
129 void openFile(FileBlock
const&)
override;
130 void respondToOpenInputFile(FileBlock
const&)
override;
131 void readResults(ResultsPrincipal
const& resp)
override;
132 void respondToCloseInputFile(FileBlock
const&)
override;
133 void incrementInputFileNumber()
override;
134 void write(EventPrincipal&)
override;
135 void writeSubRun(SubRunPrincipal&)
override;
136 void writeRun(RunPrincipal&)
override;
137 void setSubRunAuxiliaryRangeSetID(RangeSet
const&)
override;
138 void setRunAuxiliaryRangeSetID(RangeSet
const&)
override;
139 bool isFileOpen()
const override;
140 void setFileStatus(OutputFileStatus)
override;
141 bool requestsToCloseFile()
const override;
143 void startEndFile()
override;
144 void writeFileFormatVersion()
override;
145 void writeFileIndex()
override;
146 void writeEventHistory()
override;
147 void writeProcessConfigurationRegistry()
override;
148 void writeProcessHistoryRegistry()
override;
149 void writeParameterSetRegistry()
override;
150 void writeProductDescriptionRegistry()
override;
151 void writeParentageRegistry()
override;
152 void writeBranchIDListRegistry()
override;
154 doWriteFileCatalogMetadata(FileCatalogMetadata::collection_type
const& md,
155 FileCatalogMetadata::collection_type
const& ssmd)
157 void writeProductDependencies()
override;
158 void finishEndFile()
override;
159 void doRegisterProducts(MasterProductRegistry& mpr,
160 ModuleDescription
const& md)
override;
164 std::string
const catalog_;
165 bool dropAllEvents_ {
false};
166 bool dropAllSubRuns_;
167 std::string
const moduleLabel_;
168 int inputFileCount_ {0};
169 std::unique_ptr<RootDAQOutFile> rootOutputFile_ {
nullptr};
170 FileStatsCollector fstats_;
171 std::string
const filePattern_;
173 std::string lastClosedFileName_ {};
176 int const compressionLevel_;
177 int64_t
const saveMemoryObjectThreshold_;
178 int64_t
const treeMaxVirtualSize_;
179 int const splitLevel_;
180 int const basketSize_;
181 DropMetaData dropMetaData_;
182 bool dropMetaDataForDroppedData_;
186 bool fastCloning_ {
true};
191 bool writeParameterSets_;
192 ClosingCriteria fileProperties_;
199 RootDAQOut(Parameters
const& config)
200 : OutputModule{ config().omConfig, config.get_PSet() }
201 , catalog_{ config().catalog() }
202 , dropAllSubRuns_{ config().dropAllSubRuns() }
203 , moduleLabel_{ config.get_PSet().get<
string>(
"module_label") }
204 , fstats_{ moduleLabel_, processName() }
205 , filePattern_{ config().omConfig().fileName() }
206 , tmpDir_{ config().tmpDir() == default_tmpDir ? parent_path(filePattern_) : config().tmpDir() }
207 , compressionLevel_{ config().compressionLevel() }
208 , saveMemoryObjectThreshold_{ config().saveMemoryObjectThreshold() }
209 , treeMaxVirtualSize_{ config().treeMaxVirtualSize() }
210 , splitLevel_{ config().splitLevel() }
211 , basketSize_{ config().basketSize() }
212 , dropMetaData_{ config().dropMetaData() }
213 , dropMetaDataForDroppedData_{ config().dropMetaDataForDroppedData() }
214 , writeParameterSets_{ config().writeParameterSets() }
216 (detail::validateFileNamePattern(config.get_PSet().has_key(config().fileProperties.name()), filePattern_),
217 config().fileProperties()) }
218 , rpm_{ config.get_PSet() }
220 bool const dropAllEventsSet{ config().dropAllEvents(dropAllEvents_) };
221 dropAllEvents_ = detail::shouldDropEvents(dropAllEventsSet, dropAllEvents_, dropAllSubRuns_);
228 bool const fastCloningSet{ config().fastCloning(fastCloning_) };
229 fastCloning_ = detail::shouldFastClone(fastCloningSet, fastCloning_, wantAllEvents(), fileProperties_);
231 if (!writeParameterSets_)
233 mf::LogWarning(
"PROVENANCE")
234 <<
"Output module " << moduleLabel_ <<
" has parameter writeParameterSets set to false.\n"
235 <<
"Parameter set provenance will not be available in subsequent jobs.\n"
236 <<
"Check your experiment's policy on this issue to avoid future problems\n"
237 <<
"with analysis reproducibility.\n";
243 openFile(FileBlock
const& fb)
252 respondToOpenInputFile(fb);
258 postSelectProducts(FileBlock
const& fb)
262 rootOutputFile_->selectProducts(fb);
268 respondToOpenInputFile(FileBlock
const& fb)
271 if (!isFileOpen())
return;
273 bool fastCloneThisOne = fastCloning_ && (fb.tree() !=
nullptr) &&
274 ((remainingEvents() < 0) ||
275 (remainingEvents() >= fb.tree()->GetEntries()));
276 if (fastCloning_ && !fastCloneThisOne)
278 mf::LogWarning(
"FastCloning")
279 <<
"Fast cloning deactivated for this input file due to "
280 <<
"empty event tree and/or event limits.";
282 if (fastCloneThisOne && !fb.fastClonable())
284 mf::LogWarning(
"FastCloning")
285 <<
"Fast cloning deactivated for this input file due to "
286 <<
"information in FileBlock.";
287 fastCloneThisOne =
false;
289 rootOutputFile_->beginInputFile(fb, fastCloneThisOne && fastCloning_);
290 fstats_.recordInputFile(fb.fileName());
295 readResults(ResultsPrincipal
const& resp)
297 rpm_.for_each_RPWorker([&resp](RPWorker& w)
299 Results
const res{ resp, w.moduleDescription() };
300 w.rp().doReadResults(res);
306 respondToCloseInputFile(FileBlock
const& fb)
310 rootOutputFile_->respondToCloseInputFile(fb);
314 #define STRINGIFY(x) #x
315 #define TOSTRING(x) STRINGIFY(x)
317 void art::RootDAQOut::write(EventPrincipal& ep)
319 TLOG(10) <<
"RootDAQOut::write begin - line " << __LINE__;
324 if (hasNewlyDroppedBranch()[InEvent])
326 ep.addToProcessHistory();
328 rootOutputFile_->writeOne(ep);
329 fstats_.recordEvent(ep.id());
330 TLOG(10) <<
"RootDAQOut::write done/return";
335 setSubRunAuxiliaryRangeSetID(RangeSet
const& rs)
337 rootOutputFile_->setSubRunAuxiliaryRangeSetID(rs);
342 writeSubRun(SubRunPrincipal& sr)
348 if (hasNewlyDroppedBranch()[InSubRun])
350 sr.addToProcessHistory();
352 rootOutputFile_->writeSubRun(sr);
353 fstats_.recordSubRun(sr.id());
358 setRunAuxiliaryRangeSetID(RangeSet
const& rs)
360 rootOutputFile_->setRunAuxiliaryRangeSetID(rs);
365 writeRun(RunPrincipal& r)
367 if (hasNewlyDroppedBranch()[InRun])
369 r.addToProcessHistory();
371 rootOutputFile_->writeRun(r);
372 fstats_.recordRun(r.id());
379 auto resp = std::make_unique<ResultsPrincipal>(ResultsAuxiliary{},
380 description().processConfiguration());
381 if (ProductMetaData::instance().productProduced(InResults) ||
382 hasNewlyDroppedBranch()[InResults])
384 resp->addToProcessHistory();
386 rpm_.for_each_RPWorker([&resp](RPWorker& w)
388 Results res{ *resp, w.moduleDescription() };
389 w.rp().doWriteResults(*resp, res);
391 rootOutputFile_->writeResults(*resp);
396 writeFileFormatVersion()
398 rootOutputFile_->writeFileFormatVersion();
405 rootOutputFile_->writeFileIndex();
412 rootOutputFile_->writeEventHistory();
417 writeProcessConfigurationRegistry()
419 rootOutputFile_->writeProcessConfigurationRegistry();
424 writeProcessHistoryRegistry()
426 rootOutputFile_->writeProcessHistoryRegistry();
431 writeParameterSetRegistry()
433 if (writeParameterSets_)
435 rootOutputFile_->writeParameterSetRegistry();
441 writeProductDescriptionRegistry()
443 rootOutputFile_->writeProductDescriptionRegistry();
448 writeParentageRegistry()
450 rootOutputFile_->writeParentageRegistry();
455 writeBranchIDListRegistry()
457 rootOutputFile_->writeBranchIDListRegistry();
462 doWriteFileCatalogMetadata(FileCatalogMetadata::collection_type
const& md,
463 FileCatalogMetadata::collection_type
const& ssmd)
465 rootOutputFile_->writeFileCatalogMetadata(fstats_, md, ssmd);
469 art::RootDAQOut::writeProductDependencies()
471 rootOutputFile_->writeProductDependencies();
475 art::RootDAQOut::finishEndFile()
477 std::string
const currentFileName{ rootOutputFile_->currentFileName() };
478 rootOutputFile_->writeTTrees();
479 rootOutputFile_.reset();
480 fstats_.recordFileClose();
481 lastClosedFileName_ = PostCloseFileRenamer{ fstats_ }.maybeRenameFile(rootOutputFile_->currentFileName(), filePattern_);
482 detail::logFileAction(
"Closed output file ", lastClosedFileName_);
483 rpm_.invoke(&ResultsProducer::doClear);
488 doRegisterProducts(MasterProductRegistry& mpr,
489 ModuleDescription
const& md)
492 rpm_.for_each_RPWorker([&mpr, &md](RPWorker& w)
494 auto const& params = w.params();
495 w.setModuleDescription(ModuleDescription{ params.rpPSetID,
497 md.moduleLabel() +
'#' + params.rpLabel,
498 md.processConfiguration() });
499 w.rp().registerProducts(mpr, w.moduleDescription());
504 art::RootDAQOut::setFileStatus(OutputFileStatus
const ofs)
507 rootOutputFile_->setFileStatus(ofs);
514 TLOG(10) <<
"RootDAQOut::isFileOpen start";
515 bool ret = rootOutputFile_.get() !=
nullptr;
516 TLOG(10) <<
"RootDAQOut::isFileOpen return " << ret;
521 art::RootDAQOut::incrementInputFileNumber()
524 rootOutputFile_->incrementInputFileNumber();
529 requestsToCloseFile()
const
534 ret = rootOutputFile_->requestsToCloseFile();
547 if (inputFileCount_ == 0)
549 throw art::Exception(art::errors::LogicError)
550 <<
"Attempt to open output file before input file. "
551 <<
"Please report this to the core framework developers.\n";
553 rootOutputFile_ = std::make_unique<RootDAQOutFile>(
this,
554 unique_filename(tmpDir_ +
"/RootDAQOut"),
557 saveMemoryObjectThreshold_,
562 dropMetaDataForDroppedData_,
564 fstats_.recordFileOpen();
565 detail::logFileAction(
"Opened output file with pattern ", filePattern_);
570 lastClosedFileName()
const
572 if (lastClosedFileName_.empty())
574 throw Exception(errors::LogicError,
"RootDAQOut::currentFileName(): ")
575 <<
"called before meaningful.\n";
577 return lastClosedFileName_;
584 rpm_.invoke(&ResultsProducer::doBeginJob);
591 rpm_.invoke(&ResultsProducer::doEndJob);
596 event(EventPrincipal
const& ep)
598 rpm_.for_each_RPWorker([&ep](RPWorker& w)
600 Event
const e{ ep, w.moduleDescription() };
607 beginSubRun(art::SubRunPrincipal
const& srp)
609 rpm_.for_each_RPWorker([&srp](RPWorker& w)
611 SubRun
const sr{ srp, w.moduleDescription() };
612 w.rp().doBeginSubRun(sr);
618 endSubRun(art::SubRunPrincipal
const& srp)
620 rpm_.for_each_RPWorker([&srp](RPWorker& w)
622 SubRun
const sr{ srp, w.moduleDescription() };
623 w.rp().doEndSubRun(sr);
629 beginRun(art::RunPrincipal
const& rp)
631 rpm_.for_each_RPWorker([&rp](RPWorker& w)
633 Run
const r{ rp, w.moduleDescription() };
634 w.rp().doBeginRun(r);
640 endRun(art::RunPrincipal
const& rp)
642 rpm_.for_each_RPWorker([&rp](RPWorker& w)
644 Run
const r{ rp, w.moduleDescription() };