1 #ifndef artdaq_ArtModules_detail_SharedMemoryReader_hh
2 #define artdaq_ArtModules_detail_SharedMemoryReader_hh
4 #include "art/Framework/Core/Frameworkfwd.h"
5 #include "artdaq-core/Utilities/ExceptionHandler.hh"
6 #include "artdaq/DAQdata/Globals.hh"
9 #include "art/Framework/Core/FileBlock.h"
10 #include "art/Framework/Core/ProductRegistryHelper.h"
11 #include "art/Framework/IO/Sources/SourceHelper.h"
12 #include "art/Framework/IO/Sources/put_product_in_principal.h"
13 #include "art/Framework/Principal/EventPrincipal.h"
14 #include "art/Framework/Principal/RunPrincipal.h"
15 #include "art/Framework/Principal/SubRunPrincipal.h"
16 #include "artdaq-core/Core/SharedMemoryEventReceiver.hh"
17 #include "artdaq-core/Core/SharedMemoryManager.hh"
18 #include "artdaq-core/Data/ContainerFragment.hh"
19 #include "artdaq-core/Data/Fragment.hh"
20 #include "artdaq-core/Utilities/TimeUtils.hh"
21 #include "canvas/Persistency/Provenance/FileFormatVersion.h"
22 #include "fhiclcpp/ParameterSet.h"
26 #include "artdaq-core/Data/RawEvent.hh"
33 template<std::map<artdaq::Fragment::type_t, std::
string> getDefaultTypes() = artdaq::Fragment::MakeSystemTypeMap>
75 SharedMemoryReader(fhicl::ParameterSet
const& ps, art::ProductRegistryHelper& help, art::SourceHelper
const& pm)
76 :
pmaker(pm),
waiting_time(ps.get<double>(
"waiting_time", 86400.0)),
resume_after_timeout(ps.get<bool>(
"resume_after_timeout", true)),
pretend_module_name(ps.get<std::string>(
"raw_data_label",
"daq")),
unidentified_instance_name(
"unidentified"),
shutdownMsgReceived(false),
outputFileCloseNeeded(false),
bytesRead(0),
last_read_time(std::chrono::steady_clock::now()),
fragment_type_map_(getDefaultTypes()),
readNext_calls_(0)
89 new SharedMemoryEventReceiver(ps.get<uint32_t>(
"shared_memory_key", 0xBEE70000 + getppid()),
90 ps.get<uint32_t>(
"broadcast_shared_memory_key", 0xCEE70000 + getppid())));
92 TLOG(TLVL_INFO,
"SharedMemoryReader") <<
"Rank set to " << my_rank;
94 char const* artapp_env = getenv(
"ARTDAQ_APPLICATION_NAME");
95 std::string artapp_str =
"";
96 if (artapp_env != NULL)
98 artapp_str = std::string(artapp_env) +
"_";
101 app_name = artapp_str +
"art" + std::to_string(
incoming_events->GetMyId());
103 artapp_env = getenv(
"ARTDAQ_RANK");
104 if (artapp_env != NULL && my_rank < 0)
106 my_rank = std::atoi(artapp_env);
108 TLOG(TLVL_INFO) <<
"app_name is " << app_name <<
", rank " << my_rank;
114 metricMan->initialize(ps.get<fhicl::ParameterSet>(
"metrics", fhicl::ParameterSet()), app_name);
115 metricMan->do_start();
120 ExceptionHandler(ExceptionHandlerRethrow::no,
"Error loading metrics in SharedMemoryReader()");
127 help.reconstitutes<Fragments, art::InEvent>(
pretend_module_name,
"Container" + it->second);
129 auto extraTypes = ps.get<std::vector<std::pair<Fragment::type_t, std::string>>>(
130 "fragment_type_map", std::vector<std::pair<Fragment::type_t, std::string>>());
131 for (
auto it = extraTypes.begin(); it != extraTypes.end(); ++it)
135 help.reconstitutes<Fragments, art::InEvent>(
pretend_module_name,
"Container" + it->second);
137 TLOG_INFO(
"SharedMemoryReader") <<
"SharedMemoryReader initialized with ParameterSet: " << ps.to_string();
144 #if ART_HEX_VERSION < 0x30000
153 SharedMemoryReader(fhicl::ParameterSet
const& ps, art::ProductRegistryHelper& help, art::SourceHelper
const& pm,
154 art::MasterProductRegistry&)
172 void readFile(std::string
const&, art::FileBlock*& fb)
174 TLOG_ARB(5,
"SharedMemoryReader") <<
"readFile enter/start";
175 fb =
new art::FileBlock(art::FileFormatVersion(1,
"RawEvent2011"),
"nothing");
194 bool readNext(art::RunPrincipal*
const& inR, art::SubRunPrincipal*
const& inSR, art::RunPrincipal*& outR,
195 art::SubRunPrincipal*& outSR, art::EventPrincipal*& outE)
197 TLOG_DEBUG(
"SharedMemoryReader") <<
"readNext BEGIN";
214 TLOG_INFO(
"SharedMemoryReader") <<
"Shutdown Message received, returning false (should exit art)";
218 auto read_start_time = std::chrono::steady_clock::now();
220 bool keep_looping =
true;
221 bool got_event =
false;
223 if (sleepTimeUsec > 100000) sleepTimeUsec = 100000;
226 TLOG_TRACE(
"SharedMemoryReader") <<
"ReadyForRead loops BEGIN";
227 keep_looping =
false;
228 auto start_time = std::chrono::steady_clock::now();
229 while (!got_event && TimeUtils::GetElapsedTimeMicroseconds(start_time) < 1000)
234 TLOG_TRACE(
"SharedMemoryReader") <<
"ReadyForRead spin end, poll begin";
235 while (!got_event && TimeUtils::GetElapsedTime(start_time) <
waiting_time)
240 usleep(sleepTimeUsec);
245 TLOG_TRACE(
"SharedMemoryReader") <<
"ReadyForRead loops END";
248 TLOG_INFO(
"SharedMemoryReader") <<
"InputFailure: Reading timed out in SharedMemoryReader::readNext()";
255 TLOG_INFO(
"SharedMemoryReader") <<
"Did not receive an event from Shared Memory, returning false";
259 TLOG_DEBUG(
"SharedMemoryReader") <<
"Got Event!";
260 auto got_event_time = std::chrono::steady_clock::now();
262 auto errflag =
false;
264 if (errflag)
goto start;
266 if (errflag)
goto start;
267 if (fragmentTypes.size() == 0)
269 TLOG_ERROR(
"SharedMemoryReader") <<
"Event has no Fragments! Aborting!";
273 auto firstFragmentType = *fragmentTypes.begin();
274 TLOG_DEBUG(
"SharedMemoryReader") <<
"First Fragment type is " << (int)firstFragmentType <<
" ("
282 if (firstFragmentType == Fragment::EndOfDataFragmentType)
284 TLOG_DEBUG(
"SharedMemoryReader") <<
"Received shutdown message, returning false";
295 art::Timestamp currentTime = 0;
297 art::TimeValue_t lo_res_time = time(0);
298 TLOG_ARB(15,
"SharedMemoryReader") <<
"lo_res_time = " << lo_res_time;
299 currentTime = ((lo_res_time & 0xffffffff) << 32);
301 timespec hi_res_time;
302 int retcode = clock_gettime(CLOCK_REALTIME, &hi_res_time);
303 TLOG_ARB(15,
"SharedMemoryReader") <<
"hi_res_time tv_sec = " << hi_res_time.tv_sec
304 <<
" tv_nsec = " << hi_res_time.tv_nsec <<
" (retcode = " << retcode <<
")";
307 currentTime = ((hi_res_time.tv_sec & 0xffffffff) << 32) | (hi_res_time.tv_nsec & 0xffffffff);
311 TLOG_ERROR(
"SharedMemoryReader")
312 <<
"Unable to fetch a high-resolution time with clock_gettime for art::Event Timestamp. "
313 <<
"The art::Event Timestamp will be zero for event " << evtHeader->event_id;
317 if (inR == 0 || inR->run() != evtHeader->run_id)
319 outR =
pmaker.makeRunPrincipal(evtHeader->run_id, currentTime);
322 if (firstFragmentType == Fragment::EndOfRunFragmentType)
324 art::EventID
const evid(art::EventID::flushEvent());
325 outR =
pmaker.makeRunPrincipal(evid.runID(), currentTime);
326 outSR =
pmaker.makeSubRunPrincipal(evid.subRunID(), currentTime);
327 outE =
pmaker.makeEventPrincipal(evid, currentTime);
331 else if (firstFragmentType == Fragment::EndOfSubrunFragmentType)
334 if (inR == 0 || inR->run() != evtHeader->run_id)
336 outSR =
pmaker.makeSubRunPrincipal(evtHeader->run_id, evtHeader->subrun_id, currentTime);
337 #if ART_HEX_VERSION > 0x30000
338 art::EventID
const evid(art::EventID::flushEvent(outSR->subRunID()));
340 art::EventID
const evid(art::EventID::flushEvent(outSR->id()));
342 outE =
pmaker.makeEventPrincipal(evid, currentTime);
350 #if ART_HEX_VERSION > 0x30000
351 if (inSR != 0 && !inSR->subRunID().isFlush() && inSR->subRun() == evtHeader->subrun_id)
353 if (inSR != 0 && !inSR->id().isFlush() && inSR->subRun() == evtHeader->subrun_id)
356 #if ART_HEX_VERSION > 0x30000
357 art::EventID
const evid(art::EventID::flushEvent(inR->runID()));
359 art::EventID
const evid(art::EventID::flushEvent(inR->id()));
361 outSR =
pmaker.makeSubRunPrincipal(evid.subRunID(), currentTime);
362 outE =
pmaker.makeEventPrincipal(evid, currentTime);
369 outSR =
pmaker.makeSubRunPrincipal(evtHeader->run_id, evtHeader->subrun_id, currentTime);
370 #if ART_HEX_VERSION > 0x30000
371 art::EventID
const evid(art::EventID::flushEvent(outSR->subRunID()));
373 art::EventID
const evid(art::EventID::flushEvent(outSR->id()));
375 outE =
pmaker.makeEventPrincipal(evid, currentTime);
387 art::SubRunID subrun_check(evtHeader->run_id, evtHeader->subrun_id);
388 #if ART_HEX_VERSION > 0x30000
389 if (inSR == 0 || subrun_check != inSR->subRunID())
392 if (inSR == 0 || subrun_check != inSR->id())
395 outSR =
pmaker.makeSubRunPrincipal(evtHeader->run_id, evtHeader->subrun_id, currentTime);
397 outE =
pmaker.makeEventPrincipal(evtHeader->run_id, evtHeader->subrun_id, evtHeader->event_id, currentTime);
400 std::map<Fragment::type_t, std::string>::const_iterator iter_end =
fragment_type_map_.end();
401 for (
auto& type_code : fragmentTypes)
403 std::map<Fragment::type_t, std::string>::const_iterator iter =
fragment_type_map_.find(type_code);
404 TLOG_TRACE(
"SharedMemoryReader") <<
"Before GetFragmentsByType call, type is " << (int)type_code;
405 auto product =
incoming_events->GetFragmentsByType(errflag, type_code);
406 TLOG_TRACE(
"SharedMemoryReader") <<
"After GetFragmentsByType call";
407 if (errflag)
goto start;
408 for (
auto& frag : *product)
bytesRead += frag.sizeBytes();
409 if (iter != iter_end)
411 if (type_code == artdaq::Fragment::ContainerFragmentType)
413 std::unordered_map<std::string, std::unique_ptr<Fragments>> derived_fragments;
414 derived_fragments[iter->second] = std::make_unique<Fragments>();
416 for (
size_t ii = 0; ii < product->size(); ++ii)
418 ContainerFragment cf(product->at(ii));
420 if (contained_type != iter_end)
422 auto label = iter->second + contained_type->second;
423 if (!derived_fragments.count(label))
425 derived_fragments[label] = std::make_unique<Fragments>();
427 derived_fragments[label]->emplace_back(std::move(product->at(ii)));
431 derived_fragments[iter->second]->emplace_back(std::move(product->at(ii)));
435 for (
auto& type : derived_fragments)
448 TLOG_WARNING(
"SharedMemoryReader")
449 <<
"UnknownFragmentType: The product instance name mapping for fragment type \"" << ((int)type_code)
450 <<
"\" is not known. Fragments of this "
454 TLOG_TRACE(
"SharedMemoryReader") <<
"After putting fragments in event";
456 auto read_finish_time = std::chrono::steady_clock::now();
459 TLOG_ARB(10,
"SharedMemoryReader") <<
"readNext: bytesRead=" <<
bytesRead <<
" qsize=" << qsize <<
" cap=" << qcap
460 <<
" metricMan=" << (
void*)metricMan.get();
463 metricMan->sendMetric(
"Avg Processing Time", artdaq::TimeUtils::GetElapsedTime(
last_read_time, read_start_time),
464 "s", 2, MetricMode::Average);
465 metricMan->sendMetric(
"Avg Input Wait Time", artdaq::TimeUtils::GetElapsedTime(read_start_time, got_event_time),
466 "s", 3, MetricMode::Average);
467 metricMan->sendMetric(
"Avg Read Time", artdaq::TimeUtils::GetElapsedTime(got_event_time, read_finish_time),
"s",
468 3, MetricMode::Average);
469 metricMan->sendMetric(
"bytesRead",
bytesRead,
"B", 3, MetricMode::LastPoint);
471 metricMan->sendMetric(
"queue%Used", static_cast<unsigned long int>(qsize * 100 / qcap),
"%", 5,
472 MetricMode::LastPoint);
475 TLOG_TRACE(
"SharedMemoryReader") <<
"Returning from readNext";
480 std::map<Fragment::type_t, std::string>
The SharedMemoryReader is a class which implements the methods needed by art::Source.
unsigned readNext_calls_
The number of times readNext has been called.
SharedMemoryReader(fhicl::ParameterSet const &ps, art::ProductRegistryHelper &help, art::SourceHelper const &pm)
SharedMemoryReader Constructor.
SharedMemoryReader(fhicl::ParameterSet const &ps, art::ProductRegistryHelper &help, art::SourceHelper const &pm, art::MasterProductRegistry &)
SharedMemoryReader Constructor.
static void CleanUpGlobals()
Clean up statically-allocated Manager class instances.
art::SourceHelper const & pmaker
An art::SourceHelper instance.
SharedMemoryReader & operator=(SharedMemoryReader const &)=delete
Copy Assignment operator is deleted.
double waiting_time
The amount of time to wait for an event from the queue.
bool resume_after_timeout
Whether to resume if the dequeue action times out.
std::string unidentified_instance_name
The name to use for unknown Fragment types.
bool outputFileCloseNeeded
If an explicit output file close message is needed.
std::unique_ptr< SharedMemoryEventReceiver > incoming_events
The events from the EventStore.
virtual ~SharedMemoryReader()
SharedMemoryReader destructor.
size_t bytesRead
running total of number of bytes received
void closeCurrentFile()
Emulate closing a file. No-Op.
std::chrono::steady_clock::time_point last_read_time
Time last read was completed.
std::string pretend_module_name
The module name to store data under.
bool readNext(art::RunPrincipal *const &inR, art::SubRunPrincipal *const &inSR, art::RunPrincipal *&outR, art::SubRunPrincipal *&outSR, art::EventPrincipal *&outE)
Dequeue a RawEvent and declare its Fragment contents to art, creating Run, SubRun, and EventPrincipal objects as necessary.
std::map< Fragment::type_t, std::string > fragment_type_map_
The Fragment type names that this SharedMemoryReader knows about.
bool hasMoreData() const
Whether more data is expected from the SharedMemoryReader.
void readFile(std::string const &, art::FileBlock *&fb)
Emulate opening a file.
SharedMemoryReader(SharedMemoryReader const &)=delete
Copy Constructor is deleted.
bool shutdownMsgReceived
Whether a shutdown message has been received.