00001 #include "art/Framework/Core/FileBlock.h"
00002 #include "art/Framework/Core/ModuleMacros.h"
00003 #include "art/Framework/Core/OutputModule.h"
00004 #include "art/Framework/Core/ResultsProducer.h"
00005 #include "art/Framework/Core/RPManager.h"
00006 #include "art/Framework/IO/FileStatsCollector.h"
00007 #include "art/Framework/IO/PostCloseFileRenamer.h"
00008 #include "art/Framework/IO/Root/DropMetaData.h"
00009 #include "art/Framework/IO/Root/RootOutputClosingCriteria.h"
00010 #include "art/Framework/Principal/EventPrincipal.h"
00011 #include "art/Framework/Principal/Event.h"
00012 #include "art/Framework/Principal/ResultsPrincipal.h"
00013 #include "art/Framework/Principal/Results.h"
00014 #include "art/Framework/Principal/RunPrincipal.h"
00015 #include "art/Framework/Principal/Run.h"
00016 #include "art/Framework/Principal/SubRunPrincipal.h"
00017 #include "art/Framework/Principal/SubRun.h"
00018 #include "art/Persistency/Provenance/ProductMetaData.h"
00019 #include "art/Utilities/ConfigurationTable.h"
00020 #include "art/Utilities/parent_path.h"
00021 #include "art/Utilities/unique_filename.h"
00022
00023 #include "artdaq/ArtModules/RootDAQOutput/RootDAQOutFile.h"
00024 #include "artdaq/ArtModules/RootDAQOutput/detail/rootOutputConfigurationTools.h"
00025 #include "art/Framework/IO/detail/logFileAction.h"
00026
00027 #include "canvas/Persistency/Provenance/FileFormatVersion.h"
00028 #include "canvas/Utilities/Exception.h"
00029 #include "fhiclcpp/ParameterSet.h"
00030 #include "fhiclcpp/types/Atom.h"
00031 #include "fhiclcpp/types/OptionalAtom.h"
00032 #include "fhiclcpp/types/Table.h"
00033 #include "messagefacility/MessageLogger/MessageLogger.h"
00034 #define TRACE_NAME "RootDAQOut"
00035 #include "trace.h"
00036
00037 # include "Rtypes.h"
00038 # include "TBranchElement.h"
00039 # include "TClass.h"
00040 # include "TFile.h"
00041 # include "TTree.h"
00042
00043 #include <iomanip>
00044 #include <memory>
00045 #include <sstream>
00046 #include <string>
00047 #include <utility>
00048
00049 using std::string;
00050
00051 namespace art {
00052 class RootDAQOut;
00053 class RootDAQOutFile;
00054 }
00055
00056 class art::RootDAQOut final : public OutputModule{
00057 public:
00058
00059 static constexpr char const* default_tmpDir {"<parent-path-of-filename>"};
00060
00061 struct Config
00062 {
00063
00064 using Name = fhicl::Name;
00065 using Comment = fhicl::Comment;
00066 template <typename T> using Atom = fhicl::Atom<T>;
00067 template <typename T> using OptionalAtom = fhicl::OptionalAtom<T>;
00068
00069 fhicl::TableFragment<art::OutputModule::Config> omConfig;
00070 Atom<std::string> catalog { Name("catalog"), "" };
00071 OptionalAtom<bool> dropAllEvents { Name("dropAllEvents") };
00072 Atom<bool> dropAllSubRuns { Name("dropAllSubRuns"), false };
00073 OptionalAtom<bool> fastCloning { Name("fastCloning") };
00074 Atom<std::string> tmpDir { Name("tmpDir"), default_tmpDir };
00075 Atom<int> compressionLevel { Name("compressionLevel"), 7 };
00076 Atom<int64_t> saveMemoryObjectThreshold { Name("saveMemoryObjectThreshold"), -1l };
00077 Atom<int64_t> treeMaxVirtualSize { Name("treeMaxVirtualSize"), -1 };
00078 Atom<int> splitLevel { Name("splitLevel"), 99 };
00079 Atom<int> basketSize { Name("basketSize"), 16384 };
00080 Atom<bool> dropMetaDataForDroppedData { Name("dropMetaDataForDroppedData"), false };
00081 Atom<std::string> dropMetaData { Name("dropMetaData"), "NONE" };
00082 Atom<bool> writeParameterSets { Name("writeParameterSets"), true };
00083 fhicl::Table<ClosingCriteria::Config> fileProperties { Name("fileProperties") };
00084
00085 Config()
00086 {
00087
00088
00089
00090
00091
00092 using namespace fhicl::detail;
00093 ParameterBase* adjustFilename {const_cast<fhicl::Atom<std::string>*>(&omConfig().fileName)};
00094 adjustFilename->set_value_type(fhicl::value_type::REQUIRED);
00095 }
00096
00097 struct KeysToIgnore
00098 {
00099 std::set<std::string> operator()() const
00100 {
00101 std::set<std::string> keys {art::OutputModule::Config::KeysToIgnore::get()};
00102 keys.insert("results");
00103 return keys;
00104 }
00105 };
00106
00107 };
00108
00109 using Parameters = art::WrappedTable<Config,Config::KeysToIgnore>;
00110
00111 explicit RootDAQOut(Parameters const&);
00112
00113 void postSelectProducts(FileBlock const&) override;
00114
00115 void beginJob() override;
00116 void endJob() override;
00117
00118 void event(EventPrincipal const&) override;
00119
00120 void beginSubRun(SubRunPrincipal const&) override;
00121 void endSubRun(SubRunPrincipal const&) override;
00122
00123 void beginRun(RunPrincipal const&) override;
00124 void endRun(RunPrincipal const&) override;
00125
00126 private:
00127
00128 std::string const& lastClosedFileName() const override;
00129 void openFile(FileBlock const&) override;
00130 void respondToOpenInputFile(FileBlock const&) override;
00131 void readResults(ResultsPrincipal const& resp) override;
00132 void respondToCloseInputFile(FileBlock const&) override;
00133 void incrementInputFileNumber() override;
00134 void write(EventPrincipal&) override;
00135 void writeSubRun(SubRunPrincipal&) override;
00136 void writeRun(RunPrincipal&) override;
00137 void setSubRunAuxiliaryRangeSetID(RangeSet const&) override;
00138 void setRunAuxiliaryRangeSetID(RangeSet const&) override;
00139 bool isFileOpen() const override;
00140 void setFileStatus(OutputFileStatus) override;
00141 bool requestsToCloseFile() const override;
00142 void doOpenFile();
00143 void startEndFile() override;
00144 void writeFileFormatVersion() override;
00145 void writeFileIndex() override;
00146 void writeEventHistory() override;
00147 void writeProcessConfigurationRegistry() override;
00148 void writeProcessHistoryRegistry() override;
00149 void writeParameterSetRegistry() override;
00150 void writeProductDescriptionRegistry() override;
00151 void writeParentageRegistry() override;
00152 void writeBranchIDListRegistry() override;
00153 void
00154 doWriteFileCatalogMetadata(FileCatalogMetadata::collection_type const& md,
00155 FileCatalogMetadata::collection_type const& ssmd)
00156 override;
00157 void writeProductDependencies() override;
00158 void finishEndFile() override;
00159 void doRegisterProducts(MasterProductRegistry& mpr,
00160 ModuleDescription const& md) override;
00161
00162 private:
00163
00164 std::string const catalog_;
00165 bool dropAllEvents_ {false};
00166 bool dropAllSubRuns_;
00167 std::string const moduleLabel_;
00168 int inputFileCount_ {0};
00169 std::unique_ptr<RootDAQOutFile> rootOutputFile_ {nullptr};
00170 FileStatsCollector fstats_;
00171 std::string const filePattern_;
00172 std::string tmpDir_;
00173 std::string lastClosedFileName_ {};
00174
00175
00176 int const compressionLevel_;
00177 int64_t const saveMemoryObjectThreshold_;
00178 int64_t const treeMaxVirtualSize_;
00179 int const splitLevel_;
00180 int const basketSize_;
00181 DropMetaData dropMetaData_;
00182 bool dropMetaDataForDroppedData_;
00183
00184
00185
00186 bool fastCloning_ {true};
00187
00188
00189
00190
00191 bool writeParameterSets_;
00192 ClosingCriteria fileProperties_;
00193
00194
00195 RPManager rpm_;
00196 };
00197
00198 art::RootDAQOut::
00199 RootDAQOut(Parameters const& config)
00200 : OutputModule{ config().omConfig, config.get_PSet() }
00201 , catalog_{ config().catalog() }
00202 , dropAllSubRuns_{ config().dropAllSubRuns() }
00203 , moduleLabel_{ config.get_PSet().get<string>("module_label") }
00204 , fstats_{ moduleLabel_, processName() }
00205 , filePattern_{ config().omConfig().fileName() }
00206 , tmpDir_{ config().tmpDir() == default_tmpDir ? parent_path(filePattern_) : config().tmpDir() }
00207 , compressionLevel_{ config().compressionLevel() }
00208 , saveMemoryObjectThreshold_{ config().saveMemoryObjectThreshold() }
00209 , treeMaxVirtualSize_{ config().treeMaxVirtualSize() }
00210 , splitLevel_{ config().splitLevel() }
00211 , basketSize_{ config().basketSize() }
00212 , dropMetaData_{ config().dropMetaData() }
00213 , dropMetaDataForDroppedData_{ config().dropMetaDataForDroppedData() }
00214 , writeParameterSets_{ config().writeParameterSets() }
00215 , fileProperties_{
00216 (detail::validateFileNamePattern(config.get_PSet().has_key(config().fileProperties.name()), filePattern_),
00217 config().fileProperties()) }
00218 , rpm_{ config.get_PSet() }
00219 {
00220 bool const dropAllEventsSet{ config().dropAllEvents(dropAllEvents_) };
00221 dropAllEvents_ = detail::shouldDropEvents(dropAllEventsSet, dropAllEvents_, dropAllSubRuns_);
00222
00223
00224
00225
00226
00227
00228 bool const fastCloningSet{ config().fastCloning(fastCloning_) };
00229 fastCloning_ = detail::shouldFastClone(fastCloningSet, fastCloning_, wantAllEvents(), fileProperties_);
00230
00231 if (!writeParameterSets_)
00232 {
00233 mf::LogWarning("PROVENANCE")
00234 << "Output module " << moduleLabel_ << " has parameter writeParameterSets set to false.\n"
00235 << "Parameter set provenance will not be available in subsequent jobs.\n"
00236 << "Check your experiment's policy on this issue to avoid future problems\n"
00237 << "with analysis reproducibility.\n";
00238 }
00239 }
00240
00241 void
00242 art::RootDAQOut::
00243 openFile(FileBlock const& fb)
00244 {
00245
00246
00247
00248
00249 if (!isFileOpen())
00250 {
00251 doOpenFile();
00252 respondToOpenInputFile(fb);
00253 }
00254 }
00255
00256 void
00257 art::RootDAQOut::
00258 postSelectProducts(FileBlock const& fb)
00259 {
00260 if (isFileOpen())
00261 {
00262 rootOutputFile_->selectProducts(fb);
00263 }
00264 }
00265
00266 void
00267 art::RootDAQOut::
00268 respondToOpenInputFile(FileBlock const& fb)
00269 {
00270 ++inputFileCount_;
00271 if (!isFileOpen()) return;
00272
00273 bool fastCloneThisOne = fastCloning_ && (fb.tree() != nullptr) &&
00274 ((remainingEvents() < 0) ||
00275 (remainingEvents() >= fb.tree()->GetEntries()));
00276 if (fastCloning_ && !fastCloneThisOne)
00277 {
00278 mf::LogWarning("FastCloning")
00279 << "Fast cloning deactivated for this input file due to "
00280 << "empty event tree and/or event limits.";
00281 }
00282 if (fastCloneThisOne && !fb.fastClonable())
00283 {
00284 mf::LogWarning("FastCloning")
00285 << "Fast cloning deactivated for this input file due to "
00286 << "information in FileBlock.";
00287 fastCloneThisOne = false;
00288 }
00289 rootOutputFile_->beginInputFile(fb, fastCloneThisOne && fastCloning_);
00290 fstats_.recordInputFile(fb.fileName());
00291 }
00292
00293 void
00294 art::RootDAQOut::
00295 readResults(ResultsPrincipal const& resp)
00296 {
00297 rpm_.for_each_RPWorker([&resp](RPWorker& w)
00298 {
00299 Results const res{ resp, w.moduleDescription() };
00300 w.rp().doReadResults(res);
00301 });
00302 }
00303
00304 void
00305 art::RootDAQOut::
00306 respondToCloseInputFile(FileBlock const& fb)
00307 {
00308 if (isFileOpen())
00309 {
00310 rootOutputFile_->respondToCloseInputFile(fb);
00311 }
00312 }
00313
00314 #define STRINGIFY(x) #x
00315 #define TOSTRING(x) STRINGIFY(x)
00316
00317 void art::RootDAQOut::write(EventPrincipal& ep)
00318 {
00319 TLOG(10) << "RootDAQOut::write begin - line " << __LINE__;
00320 if (dropAllEvents_)
00321 {
00322 return;
00323 }
00324 if (hasNewlyDroppedBranch()[InEvent])
00325 {
00326 ep.addToProcessHistory();
00327 }
00328 rootOutputFile_->writeOne(ep);
00329 fstats_.recordEvent(ep.id());
00330 TLOG(10) << "RootDAQOut::write done/return";
00331 }
00332
00333 void
00334 art::RootDAQOut::
00335 setSubRunAuxiliaryRangeSetID(RangeSet const& rs)
00336 {
00337 rootOutputFile_->setSubRunAuxiliaryRangeSetID(rs);
00338 }
00339
00340 void
00341 art::RootDAQOut::
00342 writeSubRun(SubRunPrincipal& sr)
00343 {
00344 if (dropAllSubRuns_)
00345 {
00346 return;
00347 }
00348 if (hasNewlyDroppedBranch()[InSubRun])
00349 {
00350 sr.addToProcessHistory();
00351 }
00352 rootOutputFile_->writeSubRun(sr);
00353 fstats_.recordSubRun(sr.id());
00354 }
00355
00356 void
00357 art::RootDAQOut::
00358 setRunAuxiliaryRangeSetID(RangeSet const& rs)
00359 {
00360 rootOutputFile_->setRunAuxiliaryRangeSetID(rs);
00361 }
00362
00363 void
00364 art::RootDAQOut::
00365 writeRun(RunPrincipal& r)
00366 {
00367 if (hasNewlyDroppedBranch()[InRun])
00368 {
00369 r.addToProcessHistory();
00370 }
00371 rootOutputFile_->writeRun(r);
00372 fstats_.recordRun(r.id());
00373 }
00374
00375 void
00376 art::RootDAQOut::
00377 startEndFile()
00378 {
00379 auto resp = std::make_unique<ResultsPrincipal>(ResultsAuxiliary{},
00380 description().processConfiguration());
00381 if (ProductMetaData::instance().productProduced(InResults) ||
00382 hasNewlyDroppedBranch()[InResults])
00383 {
00384 resp->addToProcessHistory();
00385 }
00386 rpm_.for_each_RPWorker([&resp](RPWorker& w)
00387 {
00388 Results res{ *resp, w.moduleDescription() };
00389 w.rp().doWriteResults(*resp, res);
00390 });
00391 rootOutputFile_->writeResults(*resp);
00392 }
00393
00394 void
00395 art::RootDAQOut::
00396 writeFileFormatVersion()
00397 {
00398 rootOutputFile_->writeFileFormatVersion();
00399 }
00400
00401 void
00402 art::RootDAQOut::
00403 writeFileIndex()
00404 {
00405 rootOutputFile_->writeFileIndex();
00406 }
00407
00408 void
00409 art::RootDAQOut::
00410 writeEventHistory()
00411 {
00412 rootOutputFile_->writeEventHistory();
00413 }
00414
00415 void
00416 art::RootDAQOut::
00417 writeProcessConfigurationRegistry()
00418 {
00419 rootOutputFile_->writeProcessConfigurationRegistry();
00420 }
00421
00422 void
00423 art::RootDAQOut::
00424 writeProcessHistoryRegistry()
00425 {
00426 rootOutputFile_->writeProcessHistoryRegistry();
00427 }
00428
00429 void
00430 art::RootDAQOut::
00431 writeParameterSetRegistry()
00432 {
00433 if (writeParameterSets_)
00434 {
00435 rootOutputFile_->writeParameterSetRegistry();
00436 }
00437 }
00438
00439 void
00440 art::RootDAQOut::
00441 writeProductDescriptionRegistry()
00442 {
00443 rootOutputFile_->writeProductDescriptionRegistry();
00444 }
00445
00446 void
00447 art::RootDAQOut::
00448 writeParentageRegistry()
00449 {
00450 rootOutputFile_->writeParentageRegistry();
00451 }
00452
00453 void
00454 art::RootDAQOut::
00455 writeBranchIDListRegistry()
00456 {
00457 rootOutputFile_->writeBranchIDListRegistry();
00458 }
00459
00460 void
00461 art::RootDAQOut::
00462 doWriteFileCatalogMetadata(FileCatalogMetadata::collection_type const& md,
00463 FileCatalogMetadata::collection_type const& ssmd)
00464 {
00465 rootOutputFile_->writeFileCatalogMetadata(fstats_, md, ssmd);
00466 }
00467
00468 void
00469 art::RootDAQOut::writeProductDependencies()
00470 {
00471 rootOutputFile_->writeProductDependencies();
00472 }
00473
00474 void
00475 art::RootDAQOut::finishEndFile()
00476 {
00477 std::string const currentFileName{ rootOutputFile_->currentFileName() };
00478 rootOutputFile_->writeTTrees();
00479 rootOutputFile_.reset();
00480 fstats_.recordFileClose();
00481 lastClosedFileName_ = PostCloseFileRenamer{ fstats_ }.maybeRenameFile(rootOutputFile_->currentFileName(), filePattern_);
00482 detail::logFileAction("Closed output file ", lastClosedFileName_);
00483 rpm_.invoke(&ResultsProducer::doClear);
00484 }
00485
00486 void
00487 art::RootDAQOut::
00488 doRegisterProducts(MasterProductRegistry& mpr,
00489 ModuleDescription const& md)
00490 {
00491
00492 rpm_.for_each_RPWorker([&mpr, &md](RPWorker& w)
00493 {
00494 auto const& params = w.params();
00495 w.setModuleDescription(ModuleDescription{ params.rpPSetID,
00496 params.rpPluginType,
00497 md.moduleLabel() + '#' + params.rpLabel,
00498 md.processConfiguration() });
00499 w.rp().registerProducts(mpr, w.moduleDescription());
00500 });
00501 }
00502
00503 void
00504 art::RootDAQOut::setFileStatus(OutputFileStatus const ofs)
00505 {
00506 if (isFileOpen())
00507 rootOutputFile_->setFileStatus(ofs);
00508 }
00509
00510 bool
00511 art::RootDAQOut::
00512 isFileOpen() const
00513 {
00514 TLOG(10) << "RootDAQOut::isFileOpen start";
00515 bool ret = rootOutputFile_.get() != nullptr;
00516 TLOG(10) << "RootDAQOut::isFileOpen return " << ret;
00517 return ret;
00518 }
00519
00520 void
00521 art::RootDAQOut::incrementInputFileNumber()
00522 {
00523 if (isFileOpen())
00524 rootOutputFile_->incrementInputFileNumber();
00525 }
00526
00527 bool
00528 art::RootDAQOut::
00529 requestsToCloseFile() const
00530 {
00531 bool ret;
00532 if (isFileOpen())
00533 {
00534 ret = rootOutputFile_->requestsToCloseFile();
00535 }
00536 else
00537 {
00538 ret = false;
00539 }
00540 return ret;
00541 }
00542
00543 void
00544 art::RootDAQOut::
00545 doOpenFile()
00546 {
00547 if (inputFileCount_ == 0)
00548 {
00549 throw art::Exception(art::errors::LogicError)
00550 << "Attempt to open output file before input file. "
00551 << "Please report this to the core framework developers.\n";
00552 }
00553 rootOutputFile_ = std::make_unique<RootDAQOutFile>(this,
00554 unique_filename(tmpDir_ + "/RootDAQOut"),
00555 fileProperties_,
00556 compressionLevel_,
00557 saveMemoryObjectThreshold_,
00558 treeMaxVirtualSize_,
00559 splitLevel_,
00560 basketSize_,
00561 dropMetaData_,
00562 dropMetaDataForDroppedData_,
00563 fastCloning_);
00564 fstats_.recordFileOpen();
00565 detail::logFileAction("Opened output file with pattern ", filePattern_);
00566 }
00567
00568 string const&
00569 art::RootDAQOut::
00570 lastClosedFileName() const
00571 {
00572 if (lastClosedFileName_.empty())
00573 {
00574 throw Exception(errors::LogicError, "RootDAQOut::currentFileName(): ")
00575 << "called before meaningful.\n";
00576 }
00577 return lastClosedFileName_;
00578 }
00579
00580 void
00581 art::RootDAQOut::
00582 beginJob()
00583 {
00584 rpm_.invoke(&ResultsProducer::doBeginJob);
00585 }
00586
00587 void
00588 art::RootDAQOut::
00589 endJob()
00590 {
00591 rpm_.invoke(&ResultsProducer::doEndJob);
00592 }
00593
00594 void
00595 art::RootDAQOut::
00596 event(EventPrincipal const& ep)
00597 {
00598 rpm_.for_each_RPWorker([&ep](RPWorker& w)
00599 {
00600 Event const e{ ep, w.moduleDescription() };
00601 w.rp().doEvent(e);
00602 });
00603 }
00604
00605 void
00606 art::RootDAQOut::
00607 beginSubRun(art::SubRunPrincipal const& srp)
00608 {
00609 rpm_.for_each_RPWorker([&srp](RPWorker& w)
00610 {
00611 SubRun const sr{ srp, w.moduleDescription() };
00612 w.rp().doBeginSubRun(sr);
00613 });
00614 }
00615
00616 void
00617 art::RootDAQOut::
00618 endSubRun(art::SubRunPrincipal const& srp)
00619 {
00620 rpm_.for_each_RPWorker([&srp](RPWorker& w)
00621 {
00622 SubRun const sr{ srp, w.moduleDescription() };
00623 w.rp().doEndSubRun(sr);
00624 });
00625 }
00626
00627 void
00628 art::RootDAQOut::
00629 beginRun(art::RunPrincipal const& rp)
00630 {
00631 rpm_.for_each_RPWorker([&rp](RPWorker& w)
00632 {
00633 Run const r{ rp, w.moduleDescription() };
00634 w.rp().doBeginRun(r);
00635 });
00636 }
00637
00638 void
00639 art::RootDAQOut::
00640 endRun(art::RunPrincipal const& rp)
00641 {
00642 rpm_.for_each_RPWorker([&rp](RPWorker& w)
00643 {
00644 Run const r{ rp, w.moduleDescription() };
00645 w.rp().doEndRun(r);
00646 });
00647 }
00648
00649 DEFINE_ART_MODULE(art::RootDAQOut)
00650
00651