1 #include "art/Framework/Core/ModuleMacros.h"
2 #include "art/Framework/Core/OutputModule.h"
3 #include "art/Framework/Core/RPManager.h"
4 #include "art/Framework/Core/ResultsProducer.h"
5 #include "art/Framework/IO/ClosingCriteria.h"
6 #include "art/Framework/IO/FileStatsCollector.h"
7 #include "art/Framework/IO/PostCloseFileRenamer.h"
8 #include "art/Framework/IO/Root/DropMetaData.h"
9 #include "art/Framework/IO/Root/RootFileBlock.h"
10 #include "art/Framework/IO/Root/detail/rootOutputConfigurationTools.h"
11 #include "art/Framework/IO/detail/logFileAction.h"
12 #include "art/Framework/IO/detail/validateFileNamePattern.h"
13 #include "art/Framework/Principal/Event.h"
14 #include "art/Framework/Principal/EventPrincipal.h"
15 #include "art/Framework/Principal/Results.h"
16 #include "art/Framework/Principal/ResultsPrincipal.h"
17 #include "art/Framework/Principal/Run.h"
18 #include "art/Framework/Principal/RunPrincipal.h"
19 #include "art/Framework/Principal/SubRun.h"
20 #include "art/Framework/Principal/SubRunPrincipal.h"
21 #include "art/Persistency/Provenance/ProductMetaData.h"
22 #include "art/Utilities/parent_path.h"
23 #include "art/Utilities/unique_filename.h"
24 #include "artdaq/ArtModules/RootDAQOutput/RootDAQOutFile.h"
25 #include "artdaq/DAQdata/Globals.hh"
26 #include "canvas/Persistency/Provenance/FileFormatVersion.h"
27 #include "canvas/Persistency/Provenance/ProductTables.h"
28 #include "canvas/Utilities/Exception.h"
29 #include "fhiclcpp/ParameterSet.h"
30 #include "fhiclcpp/types/Atom.h"
31 #include "fhiclcpp/types/ConfigurationTable.h"
32 #include "fhiclcpp/types/OptionalAtom.h"
33 #include "fhiclcpp/types/OptionalSequence.h"
34 #include "fhiclcpp/types/Table.h"
35 #include "messagefacility/MessageLogger/MessageLogger.h"
37 #define TRACE_NAME (app_name + "_RootDAQOut").c_str()
48 std::string
const dev_null{
"/dev/null"};
59 static constexpr
char const* default_tmpDir{
"<parent-path-of-filename>"};
63 using Name = fhicl::Name;
64 using Comment = fhicl::Comment;
66 using Atom = fhicl::Atom<T>;
68 using OptionalAtom = fhicl::OptionalAtom<T>;
70 fhicl::TableFragment<art::OutputModule::Config> omConfig;
71 Atom<std::string> catalog{Name(
"catalog"),
""};
72 OptionalAtom<bool> dropAllEvents{Name(
"dropAllEvents")};
73 Atom<bool> dropAllSubRuns{Name(
"dropAllSubRuns"),
false};
74 OptionalAtom<bool> fastCloning{Name(
"fastCloning")};
75 Atom<std::string> tmpDir{Name(
"tmpDir"), default_tmpDir};
76 Atom<int> compressionLevel{Name(
"compressionLevel"), 7};
77 Atom<unsigned> freePercent{Name(
"freePercent"), 0};
78 Atom<unsigned> freeMB{Name(
"freeMB"), 0};
79 Atom<int64_t> saveMemoryObjectThreshold{Name(
"saveMemoryObjectThreshold"),
81 Atom<int64_t> treeMaxVirtualSize{Name(
"treeMaxVirtualSize"), -1};
82 Atom<int> splitLevel{Name(
"splitLevel"), 99};
83 Atom<int> basketSize{Name(
"basketSize"), 16384};
84 Atom<bool> dropMetaDataForDroppedData{Name(
"dropMetaDataForDroppedData"),
86 Atom<std::string> dropMetaData{Name(
"dropMetaData"),
"NONE"};
87 Atom<bool> writeParameterSets{Name(
"writeParameterSets"),
true};
88 fhicl::Table<ClosingCriteria::Config> fileProperties{Name(
"fileProperties")};
89 Atom<int> firstLoggerRank{Name(
"firstLoggerRank"), -1};
93 fhicl::Atom<string> appName{fhicl::Name(
"appName")};
94 fhicl::Atom<string> newString{fhicl::Name(
"newString")};
98 fhicl::Atom<string> targetString{fhicl::Name(
"targetString")};
99 fhicl::Sequence<fhicl::Table<NewSubStringForApp>> replacementList{fhicl::Name(
"replacementList")};
101 fhicl::OptionalSequence<fhicl::Table<FileNameSubstitution>> fileNameSubstitutions{Name(
"fileNameSubstitutions")};
110 using namespace fhicl::detail;
111 ParameterBase* adjustFilename{
112 const_cast<fhicl::Atom<std::string>*
>(&omConfig().fileName)};
113 adjustFilename->set_par_style(fhicl::par_style::REQUIRED);
118 std::set<std::string>
121 std::set<std::string> keys{
122 art::OutputModule::Config::KeysToIgnore::get()};
123 keys.insert(
"results");
129 using Parameters = fhicl::WrappedTable<Config, Config::KeysToIgnore>;
133 void postSelectProducts()
override;
135 void beginJob()
override;
136 void endJob()
override;
138 void event(EventPrincipal
const&)
override;
140 void beginSubRun(SubRunPrincipal
const&)
override;
141 void endSubRun(SubRunPrincipal
const&)
override;
143 void beginRun(RunPrincipal
const&)
override;
144 void endRun(RunPrincipal
const&)
override;
147 std::string fileNameAtOpen()
const;
148 std::string fileNameAtClose(std::string
const& currentFileName);
149 std::string
const& lastClosedFileName()
const override;
150 void openFile(FileBlock
const&)
override;
151 void respondToOpenInputFile(FileBlock
const&)
override;
152 void readResults(ResultsPrincipal
const& resp)
override;
153 void respondToCloseInputFile(FileBlock
const&)
override;
154 void incrementInputFileNumber()
override;
155 Granularity fileGranularity()
const override;
156 void write(EventPrincipal&)
override;
157 void writeSubRun(SubRunPrincipal&)
override;
158 void writeRun(RunPrincipal&)
override;
159 void setSubRunAuxiliaryRangeSetID(RangeSet
const&)
override;
160 void setRunAuxiliaryRangeSetID(RangeSet
const&)
override;
161 bool isFileOpen()
const override;
162 void setFileStatus(OutputFileStatus)
override;
163 bool requestsToCloseFile()
const override;
165 void startEndFile()
override;
166 void writeFileFormatVersion()
override;
167 void writeFileIndex()
override;
168 void writeEventHistory()
override;
169 void writeProcessConfigurationRegistry()
override;
170 void writeProcessHistoryRegistry()
override;
171 void writeParameterSetRegistry()
override;
172 void writeProductDescriptionRegistry()
override;
173 void writeParentageRegistry()
override;
174 void doWriteFileCatalogMetadata(
175 FileCatalogMetadata::collection_type
const& md,
176 FileCatalogMetadata::collection_type
const& ssmd)
override;
177 void writeProductDependencies()
override;
178 void finishEndFile()
override;
179 void doRegisterProducts(MasterProductRegistry& mpr,
180 ProductDescriptions& producedProducts,
181 ModuleDescription
const& md)
override;
182 std::string modifyFilePattern(std::string
const&,
Config const&);
185 std::string
const catalog_;
186 bool dropAllEvents_{
false};
187 bool dropAllSubRuns_;
188 std::string
const moduleLabel_;
189 int inputFileCount_{0};
190 std::unique_ptr<RootDAQOutFile> rootOutputFile_{
nullptr};
191 FileStatsCollector fstats_;
192 PostCloseFileRenamer fRenamer_{fstats_};
193 std::string
const filePattern_;
195 std::string lastClosedFileName_{};
198 int const compressionLevel_;
199 unsigned freePercent_;
201 int64_t
const saveMemoryObjectThreshold_;
202 int64_t
const treeMaxVirtualSize_;
203 int const splitLevel_;
204 int const basketSize_;
205 DropMetaData dropMetaData_;
206 bool dropMetaDataForDroppedData_;
210 bool fastCloningEnabled_{
true};
215 bool writeParameterSets_;
216 ClosingCriteria fileProperties_;
219 ProductTable producedResultsProducts_{};
223 art::RootDAQOut::RootDAQOut(Parameters
const& config)
224 : OutputModule{config().omConfig, config.get_PSet()}
225 , catalog_{config().catalog()}
226 , dropAllSubRuns_{config().dropAllSubRuns()}
227 , moduleLabel_{config.get_PSet().get<
string>(
"module_label")}
228 , fstats_{moduleLabel_, processName()}
229 , filePattern_{modifyFilePattern(config().omConfig().fileName(), config())}
230 , tmpDir_{config().tmpDir() == default_tmpDir ? parent_path(filePattern_) : config().tmpDir()}
231 , compressionLevel_{config().compressionLevel()}
232 , freePercent_{config().freePercent()}
233 , freeMB_{config().freeMB()}
234 , saveMemoryObjectThreshold_{config().saveMemoryObjectThreshold()}
235 , treeMaxVirtualSize_{config().treeMaxVirtualSize()}
236 , splitLevel_{config().splitLevel()}
237 , basketSize_{config().basketSize()}
238 , dropMetaData_{config().dropMetaData()}
239 , dropMetaDataForDroppedData_{config().dropMetaDataForDroppedData()}
240 , writeParameterSets_{config().writeParameterSets()}
242 detail::validateFileNamePattern(
243 config.get_PSet().has_key(config().fileProperties.name()),
245 config().fileProperties())}
246 , rpm_{config.get_PSet()}
248 TLOG(TLVL_INFO) <<
"RootDAQOut_module (s67 version) CONSTRUCTOR Start";
249 bool const dropAllEventsSet{config().dropAllEvents(dropAllEvents_)};
251 detail::shouldDropEvents(dropAllEventsSet, dropAllEvents_, dropAllSubRuns_);
258 bool const fastCloningSet{config().fastCloning(fastCloningEnabled_)};
259 fastCloningEnabled_ = detail::shouldFastClone(
260 fastCloningSet, fastCloningEnabled_, wantAllEvents(), fileProperties_);
262 if (!writeParameterSets_)
264 mf::LogWarning(
"PROVENANCE")
265 <<
"Output module " << moduleLabel_
266 <<
" has parameter writeParameterSets set to false.\n"
267 <<
"Parameter set provenance will not be available in subsequent jobs.\n"
268 <<
"Check your experiment's policy on this issue to avoid future "
270 <<
"with analysis reproducibility.\n";
274 void art::RootDAQOut::openFile(FileBlock
const& fb)
282 respondToOpenInputFile(fb);
286 void art::RootDAQOut::postSelectProducts()
290 rootOutputFile_->selectProducts();
294 void art::RootDAQOut::respondToOpenInputFile(FileBlock
const& fb)
300 auto const* rfb =
dynamic_cast<RootFileBlock const*
>(&fb);
302 bool fastCloneThisOne = fastCloningEnabled_ && rfb &&
303 (rfb->tree() !=
nullptr) &&
304 ((remainingEvents() < 0) ||
305 (remainingEvents() >= rfb->tree()->GetEntries()));
306 if (fastCloningEnabled_ && !fastCloneThisOne)
308 mf::LogWarning(
"FastCloning")
309 <<
"Fast cloning deactivated for this input file due to "
310 <<
"empty event tree and/or event limits.";
312 if (fastCloneThisOne && !rfb->fastClonable())
314 mf::LogWarning(
"FastCloning")
315 <<
"Fast cloning deactivated for this input file due to "
316 <<
"information in FileBlock.";
317 fastCloneThisOne =
false;
319 rootOutputFile_->beginInputFile(rfb, fastCloneThisOne);
320 fstats_.recordInputFile(fb.fileName());
323 void art::RootDAQOut::readResults(ResultsPrincipal
const& resp)
325 rpm_.for_each_RPWorker([&resp](RPWorker& w) { w.rp().doReadResults(resp); });
328 void art::RootDAQOut::respondToCloseInputFile(FileBlock
const& fb)
332 rootOutputFile_->respondToCloseInputFile(fb);
336 void art::RootDAQOut::write(EventPrincipal& ep)
338 TLOG(10) << __func__ <<
": enter; dropAllEvents_=" << dropAllEvents_;
343 if (hasNewlyDroppedBranch()[InEvent])
345 ep.addToProcessHistory();
347 rootOutputFile_->writeOne(ep);
348 fstats_.recordEvent(ep.id());
349 TLOG(9) << __func__ <<
": return";
352 void art::RootDAQOut::setSubRunAuxiliaryRangeSetID(RangeSet
const& rs)
354 rootOutputFile_->setSubRunAuxiliaryRangeSetID(rs);
357 void art::RootDAQOut::writeSubRun(SubRunPrincipal& sr)
363 if (hasNewlyDroppedBranch()[InSubRun])
365 sr.addToProcessHistory();
367 rootOutputFile_->writeSubRun(sr);
368 fstats_.recordSubRun(sr.id());
371 void art::RootDAQOut::setRunAuxiliaryRangeSetID(RangeSet
const& rs)
373 rootOutputFile_->setRunAuxiliaryRangeSetID(rs);
376 void art::RootDAQOut::writeRun(RunPrincipal& r)
378 if (hasNewlyDroppedBranch()[InRun])
380 r.addToProcessHistory();
382 rootOutputFile_->writeRun(r);
383 fstats_.recordRun(r.id());
386 void art::RootDAQOut::startEndFile()
388 auto resp = std::make_unique<ResultsPrincipal>(
389 ResultsAuxiliary{}, description().processConfiguration(),
nullptr);
390 resp->setProducedProducts(producedResultsProducts_);
391 if (ProductMetaData::instance().productProduced(InResults) ||
392 hasNewlyDroppedBranch()[InResults])
394 resp->addToProcessHistory();
396 rpm_.for_each_RPWorker(
397 [&resp](RPWorker& w) { w.rp().doWriteResults(*resp); });
398 rootOutputFile_->writeResults(*resp);
401 void art::RootDAQOut::writeFileFormatVersion()
403 rootOutputFile_->writeFileFormatVersion();
406 void art::RootDAQOut::writeFileIndex()
408 rootOutputFile_->writeFileIndex();
411 void art::RootDAQOut::writeEventHistory()
413 rootOutputFile_->writeEventHistory();
416 void art::RootDAQOut::writeProcessConfigurationRegistry()
418 rootOutputFile_->writeProcessConfigurationRegistry();
421 void art::RootDAQOut::writeProcessHistoryRegistry()
423 rootOutputFile_->writeProcessHistoryRegistry();
426 void art::RootDAQOut::writeParameterSetRegistry()
428 if (writeParameterSets_)
430 rootOutputFile_->writeParameterSetRegistry();
434 void art::RootDAQOut::writeProductDescriptionRegistry()
436 rootOutputFile_->writeProductDescriptionRegistry();
439 void art::RootDAQOut::writeParentageRegistry()
441 rootOutputFile_->writeParentageRegistry();
444 void art::RootDAQOut::doWriteFileCatalogMetadata(
445 FileCatalogMetadata::collection_type
const& md,
446 FileCatalogMetadata::collection_type
const& ssmd)
448 rootOutputFile_->writeFileCatalogMetadata(fstats_, md, ssmd);
451 void art::RootDAQOut::writeProductDependencies()
453 rootOutputFile_->writeProductDependencies();
456 void art::RootDAQOut::finishEndFile()
458 std::string
const currentFileName{rootOutputFile_->currentFileName()};
459 rootOutputFile_->writeTTrees();
460 rootOutputFile_.reset();
461 fstats_.recordFileClose();
462 lastClosedFileName_ =
463 fRenamer_.maybeRenameFile(currentFileName, filePattern_);
464 TLOG(TLVL_INFO) << __func__ <<
": Closed output file \"" << lastClosedFileName_ <<
"\"";
465 rpm_.invoke(&ResultsProducer::doClear);
468 void art::RootDAQOut::doRegisterProducts(MasterProductRegistry& mpr,
469 ProductDescriptions& producedProducts,
470 ModuleDescription
const& md)
473 rpm_.for_each_RPWorker([&mpr, &producedProducts, &md](RPWorker& w) {
474 auto const& params = w.params();
475 w.setModuleDescription(
476 ModuleDescription{params.rpPSetID,
478 md.moduleLabel() +
'#' + params.rpLabel,
479 md.processConfiguration(),
480 ModuleDescription::invalidID()});
481 w.rp().registerProducts(mpr, producedProducts, w.moduleDescription());
487 producedResultsProducts_ = ProductTable{producedProducts, InResults};
490 void art::RootDAQOut::setFileStatus(OutputFileStatus
const ofs)
493 rootOutputFile_->setFileStatus(ofs);
496 bool art::RootDAQOut::isFileOpen()
const
498 return rootOutputFile_.get() !=
nullptr;
501 void art::RootDAQOut::incrementInputFileNumber()
504 rootOutputFile_->incrementInputFileNumber();
507 bool art::RootDAQOut::requestsToCloseFile()
const
509 return isFileOpen() ? rootOutputFile_->requestsToCloseFile() :
false;
513 art::RootDAQOut::fileGranularity()
const
515 return fileProperties_.granularity();
518 void art::RootDAQOut::doOpenFile()
520 if (inputFileCount_ == 0)
522 throw art::Exception(art::errors::LogicError)
523 <<
"Attempt to open output file before input file. "
524 <<
"Please report this to the core framework developers.\n";
527 std::make_unique<RootDAQOutFile>(
this,
533 saveMemoryObjectThreshold_,
538 dropMetaDataForDroppedData_,
539 fastCloningEnabled_);
540 fstats_.recordFileOpen();
541 TLOG(TLVL_INFO) << __func__ <<
": Opened output file with pattern \"" << filePattern_ <<
"\"";
545 art::RootDAQOut::fileNameAtOpen()
const
547 return filePattern_ == dev_null ? dev_null : unique_filename(tmpDir_ +
"/RootOutput");
551 art::RootDAQOut::fileNameAtClose(std::string
const& currentFileName)
553 return filePattern_ == dev_null ? dev_null : fRenamer_.maybeRenameFile(currentFileName, filePattern_);
557 art::RootDAQOut::lastClosedFileName()
const
559 if (lastClosedFileName_.empty())
561 throw Exception(errors::LogicError,
"RootDAQOut::currentFileName(): ")
562 <<
"called before meaningful.\n";
564 return lastClosedFileName_;
567 void art::RootDAQOut::beginJob()
569 rpm_.invoke(&ResultsProducer::doBeginJob);
572 void art::RootDAQOut::endJob()
574 rpm_.invoke(&ResultsProducer::doEndJob);
575 showMissingConsumes();
578 void art::RootDAQOut::event(EventPrincipal
const& ep)
580 rpm_.for_each_RPWorker([&ep](RPWorker& w) { w.rp().doEvent(ep); });
583 void art::RootDAQOut::beginSubRun(art::SubRunPrincipal
const& srp)
585 rpm_.for_each_RPWorker([&srp](RPWorker& w) {
586 SubRun
const sr{srp, w.moduleDescription(), Consumer::non_module_context()};
587 w.rp().doBeginSubRun(srp);
591 void art::RootDAQOut::endSubRun(art::SubRunPrincipal
const& srp)
593 rpm_.for_each_RPWorker([&srp](RPWorker& w) { w.rp().doEndSubRun(srp); });
596 void art::RootDAQOut::beginRun(art::RunPrincipal
const& rp)
598 rpm_.for_each_RPWorker([&rp](RPWorker& w) { w.rp().doBeginRun(rp); });
601 void art::RootDAQOut::endRun(art::RunPrincipal
const& rp)
603 rpm_.for_each_RPWorker([&rp](RPWorker& w) { w.rp().doEndRun(rp); });
607 art::RootDAQOut::modifyFilePattern(std::string
const& inputPattern,
Config const& config)
609 TLOG(TLVL_DEBUG) << __func__ <<
": inputPattern=\"" << inputPattern <<
"\"";
613 int firstLoggerRank = config.firstLoggerRank();
614 std::vector<Config::FileNameSubstitution> subs;
615 config.fileNameSubstitutions(subs);
616 TLOG(TLVL_TRACE) << __func__ <<
": firstLoggerRank=" << firstLoggerRank
617 <<
", numberOfSubstitutionsProvided=" << subs.size();
620 std::string modifiedPattern = inputPattern;
621 std::string searchString;
622 size_t targetLocation;
623 int zeroBasedRelativeRank = my_rank;
624 int oneBasedRelativeRank = my_rank + 1;
625 if (firstLoggerRank >= 0)
627 zeroBasedRelativeRank -= firstLoggerRank;
628 oneBasedRelativeRank -= firstLoggerRank;
630 TLOG(TLVL_TRACE) << __func__ <<
": my_rank=" << my_rank <<
", zeroBasedRelativeRank=" << zeroBasedRelativeRank
631 <<
", oneBasedRelativeRank=" << oneBasedRelativeRank;
635 searchString =
"${ZeroBasedRelativeRank}";
636 targetLocation = modifiedPattern.find(searchString);
637 TLOG(TLVL_TRACE) << __func__ <<
":" << __LINE__ <<
" searchString=" << searchString <<
", targetLocation=" << targetLocation;
638 while (targetLocation != std::string::npos)
640 std::ostringstream oss;
641 oss << zeroBasedRelativeRank;
642 modifiedPattern.replace(targetLocation, searchString.length(), oss.str());
643 targetLocation = modifiedPattern.find(searchString);
644 TLOG(TLVL_TRACE) << __func__ <<
":" << __LINE__ <<
" searchString=" << searchString <<
", targetLocation=" << targetLocation;
649 searchString =
"${OneBasedRelativeRank}";
650 targetLocation = modifiedPattern.find(searchString);
651 TLOG(TLVL_TRACE) << __func__ <<
":" << __LINE__ <<
" searchString=" << searchString <<
", targetLocation=" << targetLocation;
652 while (targetLocation != std::string::npos)
654 std::ostringstream oss;
655 oss << oneBasedRelativeRank;
656 modifiedPattern.replace(targetLocation, searchString.length(), oss.str());
657 targetLocation = modifiedPattern.find(searchString);
658 TLOG(TLVL_TRACE) << __func__ <<
":" << __LINE__ <<
" searchString=" << searchString <<
", targetLocation=" << targetLocation;
663 searchString =
"${Rank}";
664 targetLocation = modifiedPattern.find(searchString);
665 TLOG(TLVL_TRACE) << __func__ <<
":" << __LINE__ <<
" searchString=" << searchString <<
", targetLocation=" << targetLocation;
666 while (targetLocation != std::string::npos)
668 std::ostringstream oss;
670 modifiedPattern.replace(targetLocation, searchString.length(), oss.str());
671 targetLocation = modifiedPattern.find(searchString);
672 TLOG(TLVL_TRACE) << __func__ <<
":" << __LINE__ <<
" searchString=" << searchString <<
", targetLocation=" << targetLocation;
676 for (uint32_t subIdx = 0; subIdx < subs.size(); ++subIdx)
679 const std::string BLAH =
"none_provided";
680 std::string newString = BLAH;
681 std::vector<Config::NewSubStringForApp> replacementList = subs[subIdx].replacementList();
682 for (uint32_t rdx = 0; rdx < replacementList.size(); ++rdx)
686 newString = replacementList[rdx].newString();
691 if (newString != BLAH)
694 searchString =
"${" + subs[subIdx].targetString() +
"}";
695 targetLocation = modifiedPattern.find(searchString);
696 TLOG(TLVL_TRACE) << __func__ <<
":" << __LINE__ <<
" searchString=" << searchString <<
", targetLocation=" << targetLocation;
697 while (targetLocation != std::string::npos)
699 modifiedPattern.replace(targetLocation, searchString.length(), newString);
700 targetLocation = modifiedPattern.find(searchString);
701 TLOG(TLVL_TRACE) << __func__ <<
":" << __LINE__ <<
" searchString=" << searchString <<
", targetLocation=" << targetLocation;
706 searchString = subs[subIdx].targetString();
707 targetLocation = modifiedPattern.find(searchString);
708 TLOG(TLVL_TRACE) << __func__ <<
":" << __LINE__ <<
" searchString=" << searchString <<
", targetLocation=" << targetLocation;
709 while (targetLocation != std::string::npos)
711 modifiedPattern.replace(targetLocation, searchString.length(), newString);
712 targetLocation = modifiedPattern.find(searchString);
713 TLOG(TLVL_TRACE) << __func__ <<
":" << __LINE__ <<
" searchString=" << searchString <<
", targetLocation=" << targetLocation;
718 TLOG(TLVL_DEBUG) << __func__ <<
": modifiedPattern = \"" << modifiedPattern <<
"\"";
719 return modifiedPattern;
static std::string app_name_
The name of the current application, to be used in logging and metrics.