1 #ifndef artdaq_ArtModules_detail_HDFFileReader_hh
2 #define artdaq_ArtModules_detail_HDFFileReader_hh
4 #include "artdaq/DAQdata/Globals.hh"
6 #include "artdaq-core/Data/ContainerFragment.hh"
7 #include "artdaq-core/Data/Fragment.hh"
8 #include "artdaq-core/Data/RawEvent.hh"
9 #include "artdaq-core/Utilities/ExceptionHandler.hh"
10 #include "artdaq-core/Utilities/TimeUtils.hh"
11 #include "artdaq-demo-hdf5/HDF5/MakeDatasetPlugin.hh"
12 #include "artdaq/ArtModules/ArtdaqFragmentNamingService.h"
14 #include "art/Framework/Core/Frameworkfwd.h"
16 #include "art/Framework/Core/FileBlock.h"
17 #include "art/Framework/Core/ProductRegistryHelper.h"
18 #include "art/Framework/IO/Sources/SourceHelper.h"
19 #include "art/Framework/IO/Sources/put_product_in_principal.h"
20 #include "art/Framework/Principal/EventPrincipal.h"
21 #include "art/Framework/Principal/RunPrincipal.h"
22 #include "art/Framework/Principal/SubRunPrincipal.h"
23 #include "art/Framework/Services/Registry/ServiceHandle.h"
24 #include "canvas/Persistency/Provenance/FileFormatVersion.h"
25 #include "fhiclcpp/ParameterSet.h"
66 std::unique_ptr<artdaq::hdf5::FragmentDataset>
inputFile_;
81 art::ProductRegistryHelper& help,
82 art::SourceHelper
const& pm)
91 volatile bool keep_looping =
true;
97 art::ServiceHandle<ArtdaqFragmentNamingServiceInterface> translator;
98 inputFile_ = artdaq::hdf5::MakeDatasetPlugin(ps,
"dataset");
100 help.reconstitutes<Fragments, art::InEvent>(
pretend_module_name, translator->GetUnidentifiedInstanceName());
103 help.reconstitutes<Fragments, art::InRun>(
pretend_module_name, translator->GetUnidentifiedInstanceName());
104 help.reconstitutes<Fragments, art::InSubRun>(
pretend_module_name, translator->GetUnidentifiedInstanceName());
110 metricMan->initialize(ps.get<fhicl::ParameterSet>(
"metrics", fhicl::ParameterSet()),
"art");
111 metricMan->do_start();
116 artdaq::ExceptionHandler(artdaq::ExceptionHandlerRethrow::no,
"Error loading metrics in HDFFileReader()");
119 std::set<std::string> instance_names = translator->GetAllProductInstanceNames();
120 for (
const auto& set_iter : instance_names)
125 TLOG_INFO(
"HDFFileReader") <<
"HDFFileReader initialized with ParameterSet: " << ps.to_string();
128 #if ART_HEX_VERSION < 0x30000
137 HDFFileReader(fhicl::ParameterSet
const& ps, art::ProductRegistryHelper& help, art::SourceHelper
const& pm,
138 art::MasterProductRegistry&)
156 void readFile(std::string
const&, art::FileBlock*& fb)
158 TLOG_ARB(5,
"HDFFileReader") <<
"readFile enter/start";
159 fb =
new art::FileBlock(art::FileFormatVersion(1,
"RawEvent2011"),
"nothing");
168 TLOG(TLVL_DEBUG,
"HDFFileReader") <<
"hasMoreData returning " << std::boolalpha << !
shutdownMsgReceived;
182 bool readNext(art::RunPrincipal*
const& inR, art::SubRunPrincipal*
const& inSR, art::RunPrincipal*& outR,
183 art::SubRunPrincipal*& outSR, art::EventPrincipal*& outE)
185 TLOG_DEBUG(
"HDFFileReader") <<
"readNext BEGIN";
186 art::ServiceHandle<ArtdaqFragmentNamingServiceInterface> translator;
200 TLOG_INFO(
"HDFFileReader") <<
"Shutdown Message received, returning false (should exit art)";
204 auto read_start_time = std::chrono::steady_clock::now();
206 std::unordered_map<artdaq::Fragment::type_t, std::unique_ptr<artdaq::Fragments>> eventMap =
inputFile_->readNextEvent();
207 if (eventMap.empty())
209 TLOG_ERROR(
"HDFFileReader") <<
"No data received, either because of incompatible plugin or end of file. Returning false (should exit art)";
214 auto got_event_time = std::chrono::steady_clock::now();
215 auto firstFragmentType = eventMap.begin()->first;
216 TLOG_DEBUG(
"HDFFileReader") <<
"First Fragment type is " <<
static_cast<int>(firstFragmentType) <<
" ("
217 << translator->GetInstanceNameForType(firstFragmentType) <<
")";
223 if (firstFragmentType == Fragment::EndOfDataFragmentType)
225 TLOG_DEBUG(
"HDFFileReader") <<
"Received shutdown message, returning false";
230 auto evtHeader =
inputFile_->getEventHeader(eventMap.begin()->second->at(0).sequenceID());
231 if (evtHeader ==
nullptr)
233 TLOG_DEBUG(
"HDFFileReader") <<
"Did not receive Event Header for sequence ID " << eventMap.begin()->second->at(0).sequenceID() <<
", skipping event";
238 TLOG_TRACE(
"HDFFileReader") <<
"EventHeader has Run " << evtHeader->run_id <<
", SubRun " << evtHeader->subrun_id <<
", Event " << evtHeader->event_id <<
", SeqID " << evtHeader->sequence_id <<
", IsComplete " << evtHeader->is_complete;
243 art::Timestamp currentTime = 0;
245 art::TimeValue_t lo_res_time = time(0);
246 TLOG_ARB(15,
"HDFFileReader") <<
"lo_res_time = " << lo_res_time;
247 currentTime = ((lo_res_time & 0xffffffff) << 32);
249 timespec hi_res_time;
250 int retcode = clock_gettime(CLOCK_REALTIME, &hi_res_time);
251 TLOG_ARB(15,
"HDFFileReader") <<
"hi_res_time tv_sec = " << hi_res_time.tv_sec
252 <<
" tv_nsec = " << hi_res_time.tv_nsec <<
" (retcode = " << retcode <<
")";
255 currentTime = ((hi_res_time.tv_sec & 0xffffffff) << 32) | (hi_res_time.tv_nsec & 0xffffffff);
259 TLOG_ERROR(
"HDFFileReader")
260 <<
"Unable to fetch a high-resolution time with clock_gettime for art::Event Timestamp. "
261 <<
"The art::Event Timestamp will be zero for event " << evtHeader->event_id;
265 if (inR ==
nullptr || inR->run() != evtHeader->run_id)
267 outR =
pmaker.makeRunPrincipal(evtHeader->run_id, currentTime);
270 if (firstFragmentType == Fragment::EndOfRunFragmentType)
272 art::EventID
const evid(art::EventID::flushEvent());
273 outR =
pmaker.makeRunPrincipal(evid.runID(), currentTime);
274 outSR =
pmaker.makeSubRunPrincipal(evid.subRunID(), currentTime);
275 outE =
pmaker.makeEventPrincipal(evid, currentTime);
278 else if (firstFragmentType == Fragment::EndOfSubrunFragmentType)
281 if (inR ==
nullptr || inR->run() != evtHeader->run_id)
283 outSR =
pmaker.makeSubRunPrincipal(evtHeader->run_id, evtHeader->subrun_id, currentTime);
284 #if ART_HEX_VERSION > 0x30000
285 art::EventID
const evid(art::EventID::flushEvent(outSR->subRunID()));
287 art::EventID
const evid(art::EventID::flushEvent(outSR->id()));
289 outE =
pmaker.makeEventPrincipal(evid, currentTime);
297 #if ART_HEX_VERSION > 0x30000
298 if (inSR !=
nullptr && !inSR->subRunID().isFlush() && inSR->subRun() == evtHeader->subrun_id)
300 if (inSR !=
nullptr && !inSR->id().isFlush() && inSR->subRun() == evtHeader->subrun_id)
303 #if ART_HEX_VERSION > 0x30000
304 art::EventID
const evid(art::EventID::flushEvent(inR->runID()));
306 art::EventID
const evid(art::EventID::flushEvent(inR->id()));
308 outSR =
pmaker.makeSubRunPrincipal(evid.subRunID(), currentTime);
309 outE =
pmaker.makeEventPrincipal(evid, currentTime);
316 outSR =
pmaker.makeSubRunPrincipal(evtHeader->run_id, evtHeader->subrun_id, currentTime);
317 #if ART_HEX_VERSION > 0x30000
318 art::EventID
const evid(art::EventID::flushEvent(outSR->subRunID()));
320 art::EventID
const evid(art::EventID::flushEvent(outSR->id()));
322 outE =
pmaker.makeEventPrincipal(evid, currentTime);
333 art::SubRunID subrun_check(evtHeader->run_id, evtHeader->subrun_id);
334 #if ART_HEX_VERSION > 0x30000
335 if (inSR ==
nullptr || subrun_check != inSR->subRunID())
338 if (inSR ==
nullptr || subrun_check != inSR->id())
341 outSR =
pmaker.makeSubRunPrincipal(evtHeader->run_id, evtHeader->subrun_id, currentTime);
343 outE =
pmaker.makeEventPrincipal(evtHeader->run_id, evtHeader->subrun_id, evtHeader->event_id, currentTime);
346 for (
auto& fragmentTypePair : eventMap)
348 auto type_code = fragmentTypePair.first;
349 TLOG_TRACE(
"HDFFileReader") <<
"Before GetFragmentsByType call, type is " <<
static_cast<int>(type_code);
350 TLOG_TRACE(
"HDFFileReader") <<
"After GetFragmentsByType call, number of fragments is " << fragmentTypePair.second->size();
352 std::unordered_map<std::string, std::unique_ptr<Fragments>> derived_fragments;
353 for (
auto& frag : *fragmentTypePair.second)
357 std::pair<bool, std::string> instance_name_result =
358 translator->GetInstanceNameForFragment(frag);
359 std::string label = instance_name_result.second;
360 if (!instance_name_result.first)
362 TLOG_WARNING(
"HDFFileReader")
363 <<
"UnknownFragmentType: The product instance name mapping for fragment type \"" <<
static_cast<int>(type_code)
364 <<
"\" is not known. Fragments of this "
365 <<
"type will be stored in the event with an instance name of \"" << label <<
"\".";
367 if (!derived_fragments.count(label))
369 derived_fragments[label] = std::make_unique<Fragments>();
371 auto fragSize = frag.size();
372 TLOG_TRACE(
"HDFFileReader") <<
"Moving Fragment with size " << fragSize <<
" from type map (type=" << type_code <<
") to derived map label=" << label;
373 derived_fragments[label]->emplace_back(std::move(frag));
375 TLOG_TRACE(
"HDFFileReader") <<
"Placing derived fragments in outE";
376 for (
auto& type : derived_fragments)
378 put_product_in_principal(std::move(type.second),
384 TLOG_TRACE(
"HDFFileReader") <<
"After putting fragments in event";
386 auto read_finish_time = std::chrono::steady_clock::now();
387 TLOG_ARB(10,
"HDFFileReader") <<
"readNext: bytesRead=" <<
bytesRead <<
" metricMan=" <<
static_cast<void*
>(metricMan.get());
390 metricMan->sendMetric(
"Avg Processing Time", artdaq::TimeUtils::GetElapsedTime(
last_read_time, read_start_time),
391 "s", 2, MetricMode::Average);
392 metricMan->sendMetric(
"Avg Input Wait Time", artdaq::TimeUtils::GetElapsedTime(read_start_time, got_event_time),
393 "s", 3, MetricMode::Average);
394 metricMan->sendMetric(
"Avg Read Time", artdaq::TimeUtils::GetElapsedTime(got_event_time, read_finish_time),
"s",
395 3, MetricMode::Average);
396 metricMan->sendMetric(
"bytesRead",
bytesRead,
"B", 3, MetricMode::LastPoint);
399 TLOG_TRACE(
"HDFFileReader") <<
"Returning from readNext";
The HDFFileReader is a class which implements the methods needed by art::Source.
art::SourceHelper const & pmaker
An art::SourceHelper instance.
bool readNext(art::RunPrincipal *const &inR, art::SubRunPrincipal *const &inSR, art::RunPrincipal *&outR, art::SubRunPrincipal *&outSR, art::EventPrincipal *&outE)
Dequeue a RawEvent and declare its Fragment contents to art, creating Run, SubRun, and EventPrincipal objects as necessary.
HDFFileReader(HDFFileReader const &)=delete
Copy Constructor is deleted.
std::string pretend_module_name
The module name to store data under.
HDFFileReader(fhicl::ParameterSet const &ps, art::ProductRegistryHelper &help, art::SourceHelper const &pm)
HDFFileReader Constructor.
HDFFileReader & operator=(HDFFileReader const &)=delete
Copy Assignment operator is deleted.
bool shutdownMsgReceived
Whether a shutdown message has been received.
void readFile(std::string const &, art::FileBlock *&fb)
Emulate opening a file.
std::unique_ptr< artdaq::hdf5::FragmentDataset > inputFile_
The Dataset plugin which this input source will be reading from.
HDFFileReader(fhicl::ParameterSet const &ps, art::ProductRegistryHelper &help, art::SourceHelper const &pm, art::MasterProductRegistry &)
HDFFileReader Constructor.
virtual ~HDFFileReader()
HDFFileReader destructor.
size_t bytesRead
running total of number of bytes received
unsigned readNext_calls_
The number of times readNext has been called.
void closeCurrentFile()
Emulate closing a file. No-Op.
std::chrono::steady_clock::time_point last_read_time
Time last read was completed.
bool hasMoreData() const
Whether more data is expected from the HDFFileReader.