00001 #include "art/Framework/Core/FileBlock.h"
00002 #include "art/Framework/Core/ModuleMacros.h"
00003 #include "art/Framework/Core/OutputModule.h"
00004 #include "art/Framework/Core/ResultsProducer.h"
00005 #include "art/Framework/Core/RPManager.h"
00006 #include "art/Framework/IO/FileStatsCollector.h"
00007 #include "art/Framework/IO/PostCloseFileRenamer.h"
00008 #include "art/Framework/IO/Root/DropMetaData.h"
00009 #include "art/Framework/IO/Root/RootOutputClosingCriteria.h"
00010 #include "art/Framework/Principal/EventPrincipal.h"
00011 #include "art/Framework/Principal/Event.h"
00012 #include "art/Framework/Principal/ResultsPrincipal.h"
00013 #include "art/Framework/Principal/Results.h"
00014 #include "art/Framework/Principal/RunPrincipal.h"
00015 #include "art/Framework/Principal/Run.h"
00016 #include "art/Framework/Principal/SubRunPrincipal.h"
00017 #include "art/Framework/Principal/SubRun.h"
00018 #include "art/Persistency/Provenance/ProductMetaData.h"
00019 #include "art/Utilities/ConfigurationTable.h"
00020 #include "art/Utilities/parent_path.h"
00021 #include "art/Utilities/unique_filename.h"
00022
00023 #include "artdaq/ArtModules/RootDAQOutput/RootDAQOutFile.h"
00024 #include "artdaq/ArtModules/RootDAQOutput/detail/rootOutputConfigurationTools.h"
00025 #include "art/Framework/IO/detail/logFileAction.h"
00026
00027 #include "canvas/Persistency/Provenance/FileFormatVersion.h"
00028 #include "canvas/Utilities/Exception.h"
00029 #include "fhiclcpp/ParameterSet.h"
00030 #include "fhiclcpp/types/Atom.h"
00031 #include "fhiclcpp/types/OptionalAtom.h"
00032 #include "fhiclcpp/types/Table.h"
00033 #include "messagefacility/MessageLogger/MessageLogger.h"
00034 #define TRACE_NAME "RootDAQOut_module.cc"
00035 #include "trace.h"
00036
00037 #if ART_HEX_VERSION >= 0x20703
00038 # include "Rtypes.h"
00039 # include "TBranchElement.h"
00040 # include "TClass.h"
00041 # include "TFile.h"
00042 # include "TTree.h"
00043 #endif
00044
00045 #include <iomanip>
00046 #include <memory>
00047 #include <sstream>
00048 #include <string>
00049 #include <utility>
00050
00051 using std::string;
00052
00053 namespace art {
00054 class RootDAQOut;
00055 class RootDAQOutFile;
00056 }
00057
00058 class art::RootDAQOut final : public OutputModule {
00059 public:
00060
00061 static constexpr char const* default_tmpDir {"<parent-path-of-filename>"};
00062
00063 struct Config {
00064
00065 using Name = fhicl::Name;
00066 using Comment = fhicl::Comment;
00067 template <typename T> using Atom = fhicl::Atom<T>;
00068 template <typename T> using OptionalAtom = fhicl::OptionalAtom<T>;
00069
00070 fhicl::TableFragment<art::OutputModule::Config> omConfig;
00071 Atom<std::string> catalog { Name("catalog"), "" };
00072 OptionalAtom<bool> dropAllEvents { Name("dropAllEvents") };
00073 Atom<bool> dropAllSubRuns { Name("dropAllSubRuns"), false };
00074 OptionalAtom<bool> fastCloning { Name("fastCloning") };
00075 Atom<std::string> tmpDir { Name("tmpDir"), default_tmpDir };
00076 Atom<int> compressionLevel { Name("compressionLevel"), 7 };
00077 Atom<int64_t> saveMemoryObjectThreshold { Name("saveMemoryObjectThreshold"), -1l };
00078 Atom<int64_t> treeMaxVirtualSize { Name("treeMaxVirtualSize"), -1 };
00079 Atom<int> splitLevel { Name("splitLevel"), 99 };
00080 Atom<int> basketSize { Name("basketSize"), 16384 };
00081 Atom<bool> dropMetaDataForDroppedData { Name("dropMetaDataForDroppedData"), false };
00082 Atom<std::string> dropMetaData { Name("dropMetaData"), "NONE" };
00083 Atom<bool> writeParameterSets { Name("writeParameterSets"), true };
00084 fhicl::Table<ClosingCriteria::Config> fileProperties { Name("fileProperties") };
00085
00086 Config()
00087 {
00088
00089
00090
00091
00092
00093 using namespace fhicl::detail;
00094 ParameterBase* adjustFilename {const_cast<fhicl::Atom<std::string>*>(&omConfig().fileName)};
00095 adjustFilename->set_value_type(fhicl::value_type::REQUIRED);
00096 }
00097
00098 struct KeysToIgnore {
00099 std::set<std::string> operator()() const
00100 {
00101 std::set<std::string> keys {art::OutputModule::Config::KeysToIgnore::get()};
00102 keys.insert("results");
00103 return keys;
00104 }
00105 };
00106
00107 };
00108
00109 using Parameters = art::WrappedTable<Config,Config::KeysToIgnore>;
00110
00111 explicit RootDAQOut(Parameters const&);
00112
00113 void postSelectProducts(FileBlock const&) override;
00114
00115 void beginJob() override;
00116 void endJob() override;
00117
00118 void event(EventPrincipal const&) override;
00119
00120 void beginSubRun(SubRunPrincipal const&) override;
00121 void endSubRun(SubRunPrincipal const&) override;
00122
00123 void beginRun(RunPrincipal const&) override;
00124 void endRun(RunPrincipal const&) override;
00125
00126 private:
00127
00128 std::string const& lastClosedFileName() const override;
00129 void openFile(FileBlock const&) override;
00130 void respondToOpenInputFile(FileBlock const&) override;
00131 void readResults(ResultsPrincipal const& resp) override;
00132 void respondToCloseInputFile(FileBlock const&) override;
00133 void incrementInputFileNumber() override;
00134 # if ART_HEX_VERSION < 0x20703
00135 Boundary fileSwitchBoundary() const override;
00136 # endif
00137 void write(EventPrincipal&) override;
00138 void writeSubRun(SubRunPrincipal&) override;
00139 void writeRun(RunPrincipal&) override;
00140 void setSubRunAuxiliaryRangeSetID(RangeSet const&) override;
00141 void setRunAuxiliaryRangeSetID(RangeSet const&) override;
00142 bool isFileOpen() const override;
00143 void setFileStatus(OutputFileStatus) override;
00144 bool requestsToCloseFile() const override;
00145 void doOpenFile();
00146 void startEndFile() override;
00147 void writeFileFormatVersion() override;
00148 void writeFileIndex() override;
00149 void writeEventHistory() override;
00150 void writeProcessConfigurationRegistry() override;
00151 void writeProcessHistoryRegistry() override;
00152 void writeParameterSetRegistry() override;
00153 void writeProductDescriptionRegistry() override;
00154 void writeParentageRegistry() override;
00155 void writeBranchIDListRegistry() override;
00156 void
00157 doWriteFileCatalogMetadata(FileCatalogMetadata::collection_type const& md,
00158 FileCatalogMetadata::collection_type const& ssmd)
00159 override;
00160 void writeProductDependencies() override;
00161 void finishEndFile() override;
00162 void doRegisterProducts(MasterProductRegistry& mpr,
00163 ModuleDescription const& md) override;
00164
00165 private:
00166
00167 std::string const catalog_;
00168 bool dropAllEvents_ {false};
00169 bool dropAllSubRuns_;
00170 std::string const moduleLabel_;
00171 int inputFileCount_ {0};
00172 std::unique_ptr<RootDAQOutFile> rootOutputFile_ {nullptr};
00173 FileStatsCollector fstats_;
00174 std::string const filePattern_;
00175 std::string tmpDir_;
00176 std::string lastClosedFileName_ {};
00177
00178
00179 int const compressionLevel_;
00180 int64_t const saveMemoryObjectThreshold_;
00181 int64_t const treeMaxVirtualSize_;
00182 int const splitLevel_;
00183 int const basketSize_;
00184 DropMetaData dropMetaData_;
00185 bool dropMetaDataForDroppedData_;
00186
00187
00188
00189 bool fastCloning_ {true};
00190
00191
00192
00193
00194 bool writeParameterSets_;
00195 ClosingCriteria fileProperties_;
00196
00197
00198 RPManager rpm_;
00199 };
00200
00201 art::RootDAQOut::
00202 RootDAQOut(Parameters const& config)
00203 : OutputModule{config().omConfig, config.get_PSet()}
00204 , catalog_{config().catalog()}
00205 , dropAllSubRuns_{config().dropAllSubRuns()}
00206 , moduleLabel_{config.get_PSet().get<string>("module_label")}
00207 , fstats_{moduleLabel_, processName()}
00208 , filePattern_{config().omConfig().fileName()}
00209 , tmpDir_{config().tmpDir() == default_tmpDir ? parent_path(filePattern_) : config().tmpDir()}
00210 , compressionLevel_{config().compressionLevel()}
00211 , saveMemoryObjectThreshold_{config().saveMemoryObjectThreshold()}
00212 , treeMaxVirtualSize_{config().treeMaxVirtualSize()}
00213 , splitLevel_{config().splitLevel()}
00214 , basketSize_{config().basketSize()}
00215 , dropMetaData_{config().dropMetaData()}
00216 , dropMetaDataForDroppedData_{config().dropMetaDataForDroppedData()}
00217 , writeParameterSets_{config().writeParameterSets()}
00218 , fileProperties_{
00219 (detail::validateFileNamePattern(config.get_PSet().has_key(config().fileProperties.name()), filePattern_),
00220 config().fileProperties())}
00221 , rpm_{config.get_PSet()}
00222 {
00223 bool const dropAllEventsSet {config().dropAllEvents(dropAllEvents_)};
00224 dropAllEvents_ = detail::shouldDropEvents(dropAllEventsSet, dropAllEvents_, dropAllSubRuns_);
00225
00226
00227
00228
00229
00230
00231 bool const fastCloningSet {config().fastCloning(fastCloning_)};
00232 fastCloning_ = detail::shouldFastClone(fastCloningSet, fastCloning_, wantAllEvents(), fileProperties_);
00233
00234 if (!writeParameterSets_) {
00235 mf::LogWarning("PROVENANCE")
00236 << "Output module " << moduleLabel_ << " has parameter writeParameterSets set to false.\n"
00237 << "Parameter set provenance will not be available in subsequent jobs.\n"
00238 << "Check your experiment's policy on this issue to avoid future problems\n"
00239 << "with analysis reproducibility.\n";
00240 }
00241 }
00242
00243 void
00244 art::RootDAQOut::
00245 openFile(FileBlock const& fb)
00246 {
00247
00248
00249
00250
00251 if (!isFileOpen()) {
00252 doOpenFile();
00253 respondToOpenInputFile(fb);
00254 }
00255 }
00256
00257 void
00258 art::RootDAQOut::
00259 postSelectProducts(FileBlock const& fb)
00260 {
00261 if (isFileOpen()) {
00262 rootOutputFile_->selectProducts(fb);
00263 }
00264 }
00265
00266 void
00267 art::RootDAQOut::
00268 respondToOpenInputFile(FileBlock const& fb)
00269 {
00270 ++inputFileCount_;
00271 if (!isFileOpen()) return;
00272
00273 bool fastCloneThisOne = fastCloning_ && (fb.tree() != nullptr) &&
00274 ((remainingEvents() < 0) ||
00275 (remainingEvents() >= fb.tree()->GetEntries()));
00276 if (fastCloning_ && !fastCloneThisOne) {
00277 mf::LogWarning("FastCloning")
00278 << "Fast cloning deactivated for this input file due to "
00279 << "empty event tree and/or event limits.";
00280 }
00281 if (fastCloneThisOne && !fb.fastClonable()) {
00282 mf::LogWarning("FastCloning")
00283 << "Fast cloning deactivated for this input file due to "
00284 << "information in FileBlock.";
00285 fastCloneThisOne = false;
00286 }
00287 rootOutputFile_->beginInputFile(fb, fastCloneThisOne && fastCloning_);
00288 fstats_.recordInputFile(fb.fileName());
00289 }
00290
00291 void
00292 art::RootDAQOut::
00293 readResults(ResultsPrincipal const& resp)
00294 {
00295 rpm_.for_each_RPWorker([&resp](RPWorker& w) {
00296 Results const res {resp, w.moduleDescription()};
00297 w.rp().doReadResults(res);
00298 } );
00299 }
00300
00301 void
00302 art::RootDAQOut::
00303 respondToCloseInputFile(FileBlock const& fb)
00304 {
00305 if (isFileOpen()) {
00306 rootOutputFile_->respondToCloseInputFile(fb);
00307 }
00308 }
00309
00310 #define STRINGIFY(x) #x
00311 #define TOSTRING(x) STRINGIFY(x)
00312
00313 void art::RootDAQOut::write(EventPrincipal& ep)
00314 {
00315 TRACE( 10, "RootDAQOut::write begin - line " TOSTRING(__LINE__) );
00316 if (dropAllEvents_) {
00317 return;
00318 }
00319 if (hasNewlyDroppedBranch()[InEvent]) {
00320 ep.addToProcessHistory();
00321 }
00322 rootOutputFile_->writeOne(ep);
00323 fstats_.recordEvent(ep.id());
00324 TRACE( 10, "RootDAQOut::write done/return" );
00325 }
00326
00327 void
00328 art::RootDAQOut::
00329 setSubRunAuxiliaryRangeSetID(RangeSet const& rs)
00330 {
00331 rootOutputFile_->setSubRunAuxiliaryRangeSetID(rs);
00332 }
00333
00334 void
00335 art::RootDAQOut::
00336 writeSubRun(SubRunPrincipal& sr)
00337 {
00338 if (dropAllSubRuns_) {
00339 return;
00340 }
00341 if (hasNewlyDroppedBranch()[InSubRun]) {
00342 sr.addToProcessHistory();
00343 }
00344 rootOutputFile_->writeSubRun(sr);
00345 fstats_.recordSubRun(sr.id());
00346 }
00347
00348 void
00349 art::RootDAQOut::
00350 setRunAuxiliaryRangeSetID(RangeSet const& rs)
00351 {
00352 rootOutputFile_->setRunAuxiliaryRangeSetID(rs);
00353 }
00354
00355 void
00356 art::RootDAQOut::
00357 writeRun(RunPrincipal& r)
00358 {
00359 if (hasNewlyDroppedBranch()[InRun]) {
00360 r.addToProcessHistory();
00361 }
00362 rootOutputFile_->writeRun(r);
00363 fstats_.recordRun(r.id());
00364 }
00365
00366 void
00367 art::RootDAQOut::
00368 startEndFile()
00369 {
00370 auto resp = std::make_unique<ResultsPrincipal>(ResultsAuxiliary{},
00371 description().processConfiguration());
00372 if (ProductMetaData::instance().productProduced(InResults) ||
00373 hasNewlyDroppedBranch()[InResults]) {
00374 resp->addToProcessHistory();
00375 }
00376 rpm_.for_each_RPWorker([&resp](RPWorker& w) {
00377 Results res {*resp, w.moduleDescription()};
00378 w.rp().doWriteResults(*resp, res);
00379 } );
00380 rootOutputFile_->writeResults(*resp);
00381 }
00382
00383 void
00384 art::RootDAQOut::
00385 writeFileFormatVersion()
00386 {
00387 rootOutputFile_->writeFileFormatVersion();
00388 }
00389
00390 void
00391 art::RootDAQOut::
00392 writeFileIndex()
00393 {
00394 rootOutputFile_->writeFileIndex();
00395 }
00396
00397 void
00398 art::RootDAQOut::
00399 writeEventHistory()
00400 {
00401 rootOutputFile_->writeEventHistory();
00402 }
00403
00404 void
00405 art::RootDAQOut::
00406 writeProcessConfigurationRegistry()
00407 {
00408 rootOutputFile_->writeProcessConfigurationRegistry();
00409 }
00410
00411 void
00412 art::RootDAQOut::
00413 writeProcessHistoryRegistry()
00414 {
00415 rootOutputFile_->writeProcessHistoryRegistry();
00416 }
00417
00418 void
00419 art::RootDAQOut::
00420 writeParameterSetRegistry()
00421 {
00422 if (writeParameterSets_) {
00423 rootOutputFile_->writeParameterSetRegistry();
00424 }
00425 }
00426
00427 void
00428 art::RootDAQOut::
00429 writeProductDescriptionRegistry()
00430 {
00431 rootOutputFile_->writeProductDescriptionRegistry();
00432 }
00433
00434 void
00435 art::RootDAQOut::
00436 writeParentageRegistry()
00437 {
00438 rootOutputFile_->writeParentageRegistry();
00439 }
00440
00441 void
00442 art::RootDAQOut::
00443 writeBranchIDListRegistry()
00444 {
00445 rootOutputFile_->writeBranchIDListRegistry();
00446 }
00447
00448 void
00449 art::RootDAQOut::
00450 doWriteFileCatalogMetadata(FileCatalogMetadata::collection_type const& md,
00451 FileCatalogMetadata::collection_type const& ssmd)
00452 {
00453 rootOutputFile_->writeFileCatalogMetadata(fstats_, md, ssmd);
00454 }
00455
00456 void
00457 art::RootDAQOut::writeProductDependencies()
00458 {
00459 rootOutputFile_->writeProductDependencies();
00460 }
00461
00462 void
00463 art::RootDAQOut::finishEndFile()
00464 {
00465 # if ART_HEX_VERSION >= 0x20703
00466 std::string const currentFileName {rootOutputFile_->currentFileName()};
00467 rootOutputFile_->writeTTrees();
00468 rootOutputFile_.reset();
00469 fstats_.recordFileClose();
00470 lastClosedFileName_ = PostCloseFileRenamer{fstats_}.maybeRenameFile(rootOutputFile_->currentFileName(), filePattern_);
00471 # else
00472 rootOutputFile_->finishEndFile();
00473 fstats_.recordFileClose();
00474 lastClosedFileName_ = PostCloseFileRenamer{fstats_}.maybeRenameFile(rootOutputFile_->currentFileName(), filePattern_);
00475 rootOutputFile_.reset();
00476 # endif
00477 detail::logFileAction("Closed output file ", lastClosedFileName_);
00478 rpm_.invoke(&ResultsProducer::doClear);
00479 }
00480
00481 void
00482 art::RootDAQOut::
00483 doRegisterProducts(MasterProductRegistry& mpr,
00484 ModuleDescription const& md)
00485 {
00486
00487 rpm_.for_each_RPWorker([&mpr, &md](RPWorker& w) {
00488 auto const& params = w.params();
00489 w.setModuleDescription(ModuleDescription{params.rpPSetID,
00490 params.rpPluginType,
00491 md.moduleLabel() + '#' + params.rpLabel,
00492 md.processConfiguration()});
00493 w.rp().registerProducts(mpr, w.moduleDescription());
00494 });
00495 }
00496
00497 void
00498 art::RootDAQOut::setFileStatus(OutputFileStatus const ofs)
00499 {
00500 if (isFileOpen())
00501 rootOutputFile_->setFileStatus(ofs);
00502 }
00503
00504 bool
00505 art::RootDAQOut::
00506 isFileOpen() const
00507 {
00508 TRACE( 10, "RootDAQOut::isFileOpen start" );
00509 bool ret=rootOutputFile_.get() != nullptr;
00510 TRACE( 10, "RootDAQOut::isFileOpen return %d", ret );
00511 return ret;
00512 }
00513
00514 void
00515 art::RootDAQOut::incrementInputFileNumber()
00516 {
00517 if (isFileOpen())
00518 rootOutputFile_->incrementInputFileNumber();
00519 }
00520
00521 bool
00522 art::RootDAQOut::
00523 requestsToCloseFile() const
00524 {
00525 bool ret;
00526 if (isFileOpen()) {
00527 ret = rootOutputFile_->requestsToCloseFile();
00528 } else {
00529 ret = false;
00530 }
00531 return ret;
00532 }
00533
00534 #if ART_HEX_VERSION < 0x20703
00535 art::Boundary
00536 art::RootDAQOut::
00537 fileSwitchBoundary() const
00538 {
00539 TRACE( 10, "RootDAQOut::fileSwitchBoundary start" );
00540 auto bb=fileProperties_.granularity();
00541 TRACE( 10, "RootDAQOut::fileSwitchBoundary done/return" );
00542 return bb;
00543 }
00544 #endif
00545
00546 void
00547 art::RootDAQOut::
00548 doOpenFile()
00549 {
00550 if (inputFileCount_ == 0) {
00551 throw art::Exception(art::errors::LogicError)
00552 << "Attempt to open output file before input file. "
00553 << "Please report this to the core framework developers.\n";
00554 }
00555 rootOutputFile_ = std::make_unique<RootDAQOutFile>(this,
00556 unique_filename(tmpDir_ + "/RootDAQOut"),
00557 fileProperties_,
00558 compressionLevel_,
00559 saveMemoryObjectThreshold_,
00560 treeMaxVirtualSize_,
00561 splitLevel_,
00562 basketSize_,
00563 dropMetaData_,
00564 dropMetaDataForDroppedData_,
00565 fastCloning_);
00566 fstats_.recordFileOpen();
00567 detail::logFileAction("Opened output file with pattern ", filePattern_);
00568 }
00569
00570 string const&
00571 art::RootDAQOut::
00572 lastClosedFileName() const
00573 {
00574 if (lastClosedFileName_.empty()) {
00575 throw Exception(errors::LogicError, "RootDAQOut::currentFileName(): ")
00576 << "called before meaningful.\n";
00577 }
00578 return lastClosedFileName_;
00579 }
00580
00581 void
00582 art::RootDAQOut::
00583 beginJob()
00584 {
00585 rpm_.invoke(&ResultsProducer::doBeginJob);
00586 }
00587
00588 void
00589 art::RootDAQOut::
00590 endJob()
00591 {
00592 rpm_.invoke(&ResultsProducer::doEndJob);
00593 }
00594
00595 void
00596 art::RootDAQOut::
00597 event(EventPrincipal const& ep)
00598 {
00599 rpm_.for_each_RPWorker([&ep](RPWorker& w) {
00600 Event const e {ep, w.moduleDescription()};
00601 w.rp().doEvent(e);
00602 });
00603 }
00604
00605 void
00606 art::RootDAQOut::
00607 beginSubRun(art::SubRunPrincipal const& srp)
00608 {
00609 rpm_.for_each_RPWorker([&srp](RPWorker& w) {
00610 SubRun const sr {srp, w.moduleDescription()};
00611 w.rp().doBeginSubRun(sr);
00612 });
00613 }
00614
00615 void
00616 art::RootDAQOut::
00617 endSubRun(art::SubRunPrincipal const& srp)
00618 {
00619 rpm_.for_each_RPWorker([&srp](RPWorker& w) {
00620 SubRun const sr {srp, w.moduleDescription()};
00621 w.rp().doEndSubRun(sr);
00622 });
00623 }
00624
00625 void
00626 art::RootDAQOut::
00627 beginRun(art::RunPrincipal const& rp)
00628 {
00629 rpm_.for_each_RPWorker([&rp](RPWorker& w) {
00630 Run const r {rp, w.moduleDescription()};
00631 w.rp().doBeginRun(r);
00632 });
00633 }
00634
00635 void
00636 art::RootDAQOut::
00637 endRun(art::RunPrincipal const& rp)
00638 {
00639 rpm_.for_each_RPWorker([&rp](RPWorker& w) {
00640 Run const r {rp, w.moduleDescription()};
00641 w.rp().doEndRun(r);
00642 });
00643 }
00644
00645 DEFINE_ART_MODULE(art::RootDAQOut)
00646
00647