1 #include "art/Framework/Core/FileBlock.h"
2 #include "art/Framework/Core/ModuleMacros.h"
3 #include "art/Framework/Core/OutputModule.h"
4 #include "art/Framework/Core/ResultsProducer.h"
5 #include "art/Framework/Core/RPManager.h"
6 #include "art/Framework/IO/FileStatsCollector.h"
7 #include "art/Framework/IO/PostCloseFileRenamer.h"
8 #include "art/Framework/IO/Root/DropMetaData.h"
9 #include "art/Framework/IO/Root/RootOutputClosingCriteria.h"
10 #include "art/Framework/Principal/EventPrincipal.h"
11 #include "art/Framework/Principal/Event.h"
12 #include "art/Framework/Principal/ResultsPrincipal.h"
13 #include "art/Framework/Principal/Results.h"
14 #include "art/Framework/Principal/RunPrincipal.h"
15 #include "art/Framework/Principal/Run.h"
16 #include "art/Framework/Principal/SubRunPrincipal.h"
17 #include "art/Framework/Principal/SubRun.h"
18 #include "art/Persistency/Provenance/ProductMetaData.h"
19 #include "art/Utilities/ConfigurationTable.h"
20 #include "art/Utilities/parent_path.h"
21 #include "art/Utilities/unique_filename.h"
23 #include "artdaq/ArtModules/RootDAQOutput/RootDAQOutFile.h"
24 #include "artdaq/ArtModules/RootDAQOutput/detail/rootOutputConfigurationTools.h"
25 #include "art/Framework/IO/detail/logFileAction.h"
27 #include "canvas/Persistency/Provenance/FileFormatVersion.h"
28 #include "canvas/Utilities/Exception.h"
29 #include "fhiclcpp/ParameterSet.h"
30 #include "fhiclcpp/types/Atom.h"
31 #include "fhiclcpp/types/OptionalAtom.h"
32 #include "fhiclcpp/types/Table.h"
33 #include "messagefacility/MessageLogger/MessageLogger.h"
34 #define TRACE_NAME "RootDAQOut_module.cc"
37 #if ART_HEX_VERSION >= 0x20703
39 # include "TBranchElement.h"
61 static constexpr
char const* default_tmpDir {
"<parent-path-of-filename>"};
65 using Name = fhicl::Name;
66 using Comment = fhicl::Comment;
67 template <
typename T>
using Atom = fhicl::Atom<T>;
68 template <
typename T>
using OptionalAtom = fhicl::OptionalAtom<T>;
70 fhicl::TableFragment<art::OutputModule::Config> omConfig;
71 Atom<std::string> catalog { Name(
"catalog"),
"" };
72 OptionalAtom<bool> dropAllEvents { Name(
"dropAllEvents") };
73 Atom<bool> dropAllSubRuns { Name(
"dropAllSubRuns"),
false };
74 OptionalAtom<bool> fastCloning { Name(
"fastCloning") };
75 Atom<std::string> tmpDir { Name(
"tmpDir"), default_tmpDir };
76 Atom<int> compressionLevel { Name(
"compressionLevel"), 7 };
77 Atom<int64_t> saveMemoryObjectThreshold { Name(
"saveMemoryObjectThreshold"), -1l };
78 Atom<int64_t> treeMaxVirtualSize { Name(
"treeMaxVirtualSize"), -1 };
79 Atom<int> splitLevel { Name(
"splitLevel"), 99 };
80 Atom<int> basketSize { Name(
"basketSize"), 16384 };
81 Atom<bool> dropMetaDataForDroppedData { Name(
"dropMetaDataForDroppedData"),
false };
82 Atom<std::string> dropMetaData { Name(
"dropMetaData"),
"NONE" };
83 Atom<bool> writeParameterSets { Name(
"writeParameterSets"),
true };
84 fhicl::Table<ClosingCriteria::Config> fileProperties { Name(
"fileProperties") };
93 using namespace fhicl::detail;
94 ParameterBase* adjustFilename {
const_cast<fhicl::Atom<std::string>*
>(&omConfig().fileName)};
95 adjustFilename->set_value_type(fhicl::value_type::REQUIRED);
99 std::set<std::string> operator()()
const
101 std::set<std::string> keys {art::OutputModule::Config::KeysToIgnore::get()};
102 keys.insert(
"results");
109 using Parameters = art::WrappedTable<Config,Config::KeysToIgnore>;
113 void postSelectProducts(FileBlock
const&)
override;
115 void beginJob()
override;
116 void endJob()
override;
118 void event(EventPrincipal
const&)
override;
120 void beginSubRun(SubRunPrincipal
const&)
override;
121 void endSubRun(SubRunPrincipal
const&)
override;
123 void beginRun(RunPrincipal
const&)
override;
124 void endRun(RunPrincipal
const&)
override;
128 std::string
const& lastClosedFileName()
const override;
129 void openFile(FileBlock
const&)
override;
130 void respondToOpenInputFile(FileBlock
const&)
override;
131 void readResults(ResultsPrincipal
const& resp)
override;
132 void respondToCloseInputFile(FileBlock
const&)
override;
133 void incrementInputFileNumber()
override;
134 # if ART_HEX_VERSION < 0x20703
135 Boundary fileSwitchBoundary()
const override;
137 void write(EventPrincipal&)
override;
138 void writeSubRun(SubRunPrincipal&)
override;
139 void writeRun(RunPrincipal&)
override;
140 void setSubRunAuxiliaryRangeSetID(RangeSet
const&)
override;
141 void setRunAuxiliaryRangeSetID(RangeSet
const&)
override;
142 bool isFileOpen()
const override;
143 void setFileStatus(OutputFileStatus)
override;
144 bool requestsToCloseFile()
const override;
146 void startEndFile()
override;
147 void writeFileFormatVersion()
override;
148 void writeFileIndex()
override;
149 void writeEventHistory()
override;
150 void writeProcessConfigurationRegistry()
override;
151 void writeProcessHistoryRegistry()
override;
152 void writeParameterSetRegistry()
override;
153 void writeProductDescriptionRegistry()
override;
154 void writeParentageRegistry()
override;
155 void writeBranchIDListRegistry()
override;
157 doWriteFileCatalogMetadata(FileCatalogMetadata::collection_type
const& md,
158 FileCatalogMetadata::collection_type
const& ssmd)
160 void writeProductDependencies()
override;
161 void finishEndFile()
override;
162 void doRegisterProducts(MasterProductRegistry& mpr,
163 ModuleDescription
const& md)
override;
167 std::string
const catalog_;
168 bool dropAllEvents_ {
false};
169 bool dropAllSubRuns_;
170 std::string
const moduleLabel_;
171 int inputFileCount_ {0};
172 std::unique_ptr<RootDAQOutFile> rootOutputFile_ {
nullptr};
173 FileStatsCollector fstats_;
174 std::string
const filePattern_;
176 std::string lastClosedFileName_ {};
179 int const compressionLevel_;
180 int64_t
const saveMemoryObjectThreshold_;
181 int64_t
const treeMaxVirtualSize_;
182 int const splitLevel_;
183 int const basketSize_;
184 DropMetaData dropMetaData_;
185 bool dropMetaDataForDroppedData_;
189 bool fastCloning_ {
true};
194 bool writeParameterSets_;
195 ClosingCriteria fileProperties_;
202 RootDAQOut(Parameters
const& config)
203 : OutputModule{config().omConfig, config.get_PSet()}
204 , catalog_{config().catalog()}
205 , dropAllSubRuns_{config().dropAllSubRuns()}
206 , moduleLabel_{config.get_PSet().get<
string>(
"module_label")}
207 , fstats_{moduleLabel_, processName()}
208 , filePattern_{config().omConfig().fileName()}
209 , tmpDir_{config().tmpDir() == default_tmpDir ? parent_path(filePattern_) : config().tmpDir()}
210 , compressionLevel_{config().compressionLevel()}
211 , saveMemoryObjectThreshold_{config().saveMemoryObjectThreshold()}
212 , treeMaxVirtualSize_{config().treeMaxVirtualSize()}
213 , splitLevel_{config().splitLevel()}
214 , basketSize_{config().basketSize()}
215 , dropMetaData_{config().dropMetaData()}
216 , dropMetaDataForDroppedData_{config().dropMetaDataForDroppedData()}
217 , writeParameterSets_{config().writeParameterSets()}
219 (detail::validateFileNamePattern(config.get_PSet().has_key(config().fileProperties.name()), filePattern_),
220 config().fileProperties())}
221 , rpm_{config.get_PSet()}
223 bool const dropAllEventsSet {config().dropAllEvents(dropAllEvents_)};
224 dropAllEvents_ = detail::shouldDropEvents(dropAllEventsSet, dropAllEvents_, dropAllSubRuns_);
231 bool const fastCloningSet {config().fastCloning(fastCloning_)};
232 fastCloning_ = detail::shouldFastClone(fastCloningSet, fastCloning_, wantAllEvents(), fileProperties_);
234 if (!writeParameterSets_) {
235 mf::LogWarning(
"PROVENANCE")
236 <<
"Output module " << moduleLabel_ <<
" has parameter writeParameterSets set to false.\n"
237 <<
"Parameter set provenance will not be available in subsequent jobs.\n"
238 <<
"Check your experiment's policy on this issue to avoid future problems\n"
239 <<
"with analysis reproducibility.\n";
245 openFile(FileBlock
const& fb)
253 respondToOpenInputFile(fb);
259 postSelectProducts(FileBlock
const& fb)
262 rootOutputFile_->selectProducts(fb);
268 respondToOpenInputFile(FileBlock
const& fb)
271 if (!isFileOpen())
return;
273 bool fastCloneThisOne = fastCloning_ && (fb.tree() !=
nullptr) &&
274 ((remainingEvents() < 0) ||
275 (remainingEvents() >= fb.tree()->GetEntries()));
276 if (fastCloning_ && !fastCloneThisOne) {
277 mf::LogWarning(
"FastCloning")
278 <<
"Fast cloning deactivated for this input file due to "
279 <<
"empty event tree and/or event limits.";
281 if (fastCloneThisOne && !fb.fastClonable()) {
282 mf::LogWarning(
"FastCloning")
283 <<
"Fast cloning deactivated for this input file due to "
284 <<
"information in FileBlock.";
285 fastCloneThisOne =
false;
287 rootOutputFile_->beginInputFile(fb, fastCloneThisOne && fastCloning_);
288 fstats_.recordInputFile(fb.fileName());
293 readResults(ResultsPrincipal
const& resp)
295 rpm_.for_each_RPWorker([&resp](RPWorker& w) {
296 Results
const res {resp, w.moduleDescription()};
297 w.rp().doReadResults(res);
303 respondToCloseInputFile(FileBlock
const& fb)
306 rootOutputFile_->respondToCloseInputFile(fb);
310 #define STRINGIFY(x) #x
311 #define TOSTRING(x) STRINGIFY(x)
313 void art::RootDAQOut::write(EventPrincipal& ep)
315 TRACE( 10,
"RootDAQOut::write begin - line " TOSTRING(__LINE__) );
316 if (dropAllEvents_) {
319 if (hasNewlyDroppedBranch()[InEvent]) {
320 ep.addToProcessHistory();
322 rootOutputFile_->writeOne(ep);
323 fstats_.recordEvent(ep.id());
324 TRACE( 10,
"RootDAQOut::write done/return" );
329 setSubRunAuxiliaryRangeSetID(RangeSet
const& rs)
331 rootOutputFile_->setSubRunAuxiliaryRangeSetID(rs);
336 writeSubRun(SubRunPrincipal& sr)
338 if (dropAllSubRuns_) {
341 if (hasNewlyDroppedBranch()[InSubRun]) {
342 sr.addToProcessHistory();
344 rootOutputFile_->writeSubRun(sr);
345 fstats_.recordSubRun(sr.id());
350 setRunAuxiliaryRangeSetID(RangeSet
const& rs)
352 rootOutputFile_->setRunAuxiliaryRangeSetID(rs);
357 writeRun(RunPrincipal& r)
359 if (hasNewlyDroppedBranch()[InRun]) {
360 r.addToProcessHistory();
362 rootOutputFile_->writeRun(r);
363 fstats_.recordRun(r.id());
370 auto resp = std::make_unique<ResultsPrincipal>(ResultsAuxiliary{},
371 description().processConfiguration());
372 if (ProductMetaData::instance().productProduced(InResults) ||
373 hasNewlyDroppedBranch()[InResults]) {
374 resp->addToProcessHistory();
376 rpm_.for_each_RPWorker([&resp](RPWorker& w) {
377 Results res {*resp, w.moduleDescription()};
378 w.rp().doWriteResults(*resp, res);
380 rootOutputFile_->writeResults(*resp);
385 writeFileFormatVersion()
387 rootOutputFile_->writeFileFormatVersion();
394 rootOutputFile_->writeFileIndex();
401 rootOutputFile_->writeEventHistory();
406 writeProcessConfigurationRegistry()
408 rootOutputFile_->writeProcessConfigurationRegistry();
413 writeProcessHistoryRegistry()
415 rootOutputFile_->writeProcessHistoryRegistry();
420 writeParameterSetRegistry()
422 if (writeParameterSets_) {
423 rootOutputFile_->writeParameterSetRegistry();
429 writeProductDescriptionRegistry()
431 rootOutputFile_->writeProductDescriptionRegistry();
436 writeParentageRegistry()
438 rootOutputFile_->writeParentageRegistry();
443 writeBranchIDListRegistry()
445 rootOutputFile_->writeBranchIDListRegistry();
450 doWriteFileCatalogMetadata(FileCatalogMetadata::collection_type
const& md,
451 FileCatalogMetadata::collection_type
const& ssmd)
453 rootOutputFile_->writeFileCatalogMetadata(fstats_, md, ssmd);
457 art::RootDAQOut::writeProductDependencies()
459 rootOutputFile_->writeProductDependencies();
463 art::RootDAQOut::finishEndFile()
465 # if ART_HEX_VERSION >= 0x20703
466 std::string
const currentFileName {rootOutputFile_->currentFileName()};
467 rootOutputFile_->writeTTrees();
468 rootOutputFile_.reset();
469 fstats_.recordFileClose();
470 lastClosedFileName_ = PostCloseFileRenamer{fstats_}.maybeRenameFile(rootOutputFile_->currentFileName(), filePattern_);
472 rootOutputFile_->finishEndFile();
473 fstats_.recordFileClose();
474 lastClosedFileName_ = PostCloseFileRenamer{fstats_}.maybeRenameFile(rootOutputFile_->currentFileName(), filePattern_);
475 rootOutputFile_.reset();
477 detail::logFileAction(
"Closed output file ", lastClosedFileName_);
478 rpm_.invoke(&ResultsProducer::doClear);
483 doRegisterProducts(MasterProductRegistry& mpr,
484 ModuleDescription
const& md)
487 rpm_.for_each_RPWorker([&mpr, &md](RPWorker& w) {
488 auto const& params = w.params();
489 w.setModuleDescription(ModuleDescription{params.rpPSetID,
491 md.moduleLabel() +
'#' + params.rpLabel,
492 md.processConfiguration()});
493 w.rp().registerProducts(mpr, w.moduleDescription());
498 art::RootDAQOut::setFileStatus(OutputFileStatus
const ofs)
501 rootOutputFile_->setFileStatus(ofs);
508 TRACE( 10,
"RootDAQOut::isFileOpen start" );
509 bool ret=rootOutputFile_.get() !=
nullptr;
510 TRACE( 10,
"RootDAQOut::isFileOpen return %d", ret );
515 art::RootDAQOut::incrementInputFileNumber()
518 rootOutputFile_->incrementInputFileNumber();
523 requestsToCloseFile()
const
527 ret = rootOutputFile_->requestsToCloseFile();
534 #if ART_HEX_VERSION < 0x20703
537 fileSwitchBoundary()
const
539 TRACE( 10,
"RootDAQOut::fileSwitchBoundary start" );
540 auto bb=fileProperties_.granularity();
541 TRACE( 10,
"RootDAQOut::fileSwitchBoundary done/return" );
550 if (inputFileCount_ == 0) {
551 throw art::Exception(art::errors::LogicError)
552 <<
"Attempt to open output file before input file. "
553 <<
"Please report this to the core framework developers.\n";
555 rootOutputFile_ = std::make_unique<RootDAQOutFile>(
this,
556 unique_filename(tmpDir_ +
"/RootDAQOut"),
559 saveMemoryObjectThreshold_,
564 dropMetaDataForDroppedData_,
566 fstats_.recordFileOpen();
567 detail::logFileAction(
"Opened output file with pattern ", filePattern_);
572 lastClosedFileName()
const
574 if (lastClosedFileName_.empty()) {
575 throw Exception(errors::LogicError,
"RootDAQOut::currentFileName(): ")
576 <<
"called before meaningful.\n";
578 return lastClosedFileName_;
585 rpm_.invoke(&ResultsProducer::doBeginJob);
592 rpm_.invoke(&ResultsProducer::doEndJob);
597 event(EventPrincipal
const& ep)
599 rpm_.for_each_RPWorker([&ep](RPWorker& w) {
600 Event
const e {ep, w.moduleDescription()};
607 beginSubRun(art::SubRunPrincipal
const& srp)
609 rpm_.for_each_RPWorker([&srp](RPWorker& w) {
610 SubRun
const sr {srp, w.moduleDescription()};
611 w.rp().doBeginSubRun(sr);
617 endSubRun(art::SubRunPrincipal
const& srp)
619 rpm_.for_each_RPWorker([&srp](RPWorker& w) {
620 SubRun
const sr {srp, w.moduleDescription()};
621 w.rp().doEndSubRun(sr);
627 beginRun(art::RunPrincipal
const& rp)
629 rpm_.for_each_RPWorker([&rp](RPWorker& w) {
630 Run
const r {rp, w.moduleDescription()};
631 w.rp().doBeginRun(r);
637 endRun(art::RunPrincipal
const& rp)
639 rpm_.for_each_RPWorker([&rp](RPWorker& w) {
640 Run
const r {rp, w.moduleDescription()};