artdaq  v3_05_00
SharedMemoryReader.hh
1 #ifndef artdaq_ArtModules_detail_SharedMemoryReader_hh
2 #define artdaq_ArtModules_detail_SharedMemoryReader_hh
3 
4 #include "art/Framework/Core/Frameworkfwd.h"
5 #include "artdaq-core/Utilities/ExceptionHandler.hh"
6 #include "artdaq/DAQdata/Globals.hh"
7 
8 #include <sys/time.h>
9 #include "art/Framework/Core/FileBlock.h"
10 #include "art/Framework/Core/ProductRegistryHelper.h"
11 #include "art/Framework/IO/Sources/SourceHelper.h"
12 #include "art/Framework/IO/Sources/put_product_in_principal.h"
13 #include "art/Framework/Principal/EventPrincipal.h"
14 #include "art/Framework/Principal/RunPrincipal.h"
15 #include "art/Framework/Principal/SubRunPrincipal.h"
16 #include "artdaq-core/Core/SharedMemoryEventReceiver.hh"
17 #include "artdaq-core/Core/SharedMemoryManager.hh"
18 #include "artdaq-core/Data/ContainerFragment.hh"
19 #include "artdaq-core/Data/Fragment.hh"
20 #include "artdaq-core/Utilities/TimeUtils.hh"
21 #include "canvas/Persistency/Provenance/FileFormatVersion.h"
22 #include "fhiclcpp/ParameterSet.h"
23 
24 #include <map>
25 #include <string>
26 #include "artdaq-core/Data/RawEvent.hh"
27 
28 namespace artdaq {
29 namespace detail {
33 template<std::map<artdaq::Fragment::type_t, std::string> getDefaultTypes() = artdaq::Fragment::MakeSystemTypeMap>
35 {
39  SharedMemoryReader(SharedMemoryReader const&) = delete;
40 
46 
47  art::SourceHelper const& pmaker;
48  std::unique_ptr<SharedMemoryEventReceiver> incoming_events;
49  double waiting_time;
51  std::string pretend_module_name;
55  size_t bytesRead;
56  std::chrono::steady_clock::time_point last_read_time;
57  // std::unique_ptr<SharedMemoryManager> data_shm; ///< SharedMemoryManager containing data
58  // std::unique_ptr<SharedMemoryManager> broadcast_shm; ///< SharedMemoryManager containing broadcasts (control
59  // Fragments)
60 
75  SharedMemoryReader(fhicl::ParameterSet const& ps, art::ProductRegistryHelper& help, art::SourceHelper const& pm)
76  : pmaker(pm), waiting_time(ps.get<double>("waiting_time", 86400.0)), resume_after_timeout(ps.get<bool>("resume_after_timeout", true)), pretend_module_name(ps.get<std::string>("raw_data_label", "daq")), unidentified_instance_name("unidentified"), shutdownMsgReceived(false), outputFileCloseNeeded(false), bytesRead(0), last_read_time(std::chrono::steady_clock::now()), fragment_type_map_(getDefaultTypes()), readNext_calls_(0)
77  {
78  // For testing
79  // if (ps.has_key("buffer_count") && (ps.has_key("max_event_size_bytes") ||
80  // (ps.has_key("expected_fragments_per_event") && ps.has_key("max_fragment_size_bytes"))))
81  //{
82  // data_shm.reset(new SharedMemoryManager(ps.get<uint32_t>("shared_memory_key", 0xBEE70000 + getppid()),
83  // ps.get<int>("buffer_count"), ps.has_key("max_event_size_bytes") ? ps.get<size_t>("max_event_size_bytes") :
84  // ps.get<size_t>("expected_fragments_per_event") * ps.get<size_t>("max_fragment_size_bytes")));
85  // broadcast_shm.reset(new SharedMemoryManager(ps.get<uint32_t>("broadcast_shared_memory_key", 0xCEE70000 +
86  // getppid()), ps.get<int>("broadcast_buffer_count", 5), ps.get<size_t>("broadcast_buffer_size", 0x100000)));
87  //}
88  incoming_events.reset(
89  new SharedMemoryEventReceiver(ps.get<uint32_t>("shared_memory_key", 0xBEE70000 + getppid()),
90  ps.get<uint32_t>("broadcast_shared_memory_key", 0xCEE70000 + getppid())));
91  my_rank = incoming_events->GetRank();
92  TLOG(TLVL_INFO, "SharedMemoryReader") << "Rank set to " << my_rank;
93 
94  char const* artapp_env = getenv("ARTDAQ_APPLICATION_NAME");
95  std::string artapp_str = "";
96  if (artapp_env != NULL)
97  {
98  artapp_str = std::string(artapp_env) + "_";
99  }
100 
101  app_name = artapp_str + "art" + std::to_string(incoming_events->GetMyId());
102 
103  artapp_env = getenv("ARTDAQ_RANK");
104  if (artapp_env != NULL && my_rank < 0)
105  {
106  my_rank = std::atoi(artapp_env);
107  }
108  TLOG(TLVL_INFO) << "app_name is " << app_name << ", rank " << my_rank;
109 
110  try
111  {
112  if (metricMan)
113  {
114  metricMan->initialize(ps.get<fhicl::ParameterSet>("metrics", fhicl::ParameterSet()), app_name);
115  metricMan->do_start();
116  }
117  }
118  catch (...)
119  {
120  ExceptionHandler(ExceptionHandlerRethrow::no, "Error loading metrics in SharedMemoryReader()");
121  }
122 
123  help.reconstitutes<Fragments, art::InEvent>(pretend_module_name, unidentified_instance_name);
124  for (auto it = fragment_type_map_.begin(); it != fragment_type_map_.end(); ++it)
125  {
126  help.reconstitutes<Fragments, art::InEvent>(pretend_module_name, it->second);
127  help.reconstitutes<Fragments, art::InEvent>(pretend_module_name, "Container" + it->second);
128  }
129  auto extraTypes = ps.get<std::vector<std::pair<Fragment::type_t, std::string>>>(
130  "fragment_type_map", std::vector<std::pair<Fragment::type_t, std::string>>());
131  for (auto it = extraTypes.begin(); it != extraTypes.end(); ++it)
132  {
133  fragment_type_map_[it->first] = it->second;
134  help.reconstitutes<Fragments, art::InEvent>(pretend_module_name, it->second);
135  help.reconstitutes<Fragments, art::InEvent>(pretend_module_name, "Container" + it->second);
136  }
137  TLOG_INFO("SharedMemoryReader") << "SharedMemoryReader initialized with ParameterSet: " << ps.to_string();
138  // for(auto& type : fragment_type_map_)
139  //{
140  // TLOG_INFO("SharedMemoryReader") << "Fragment Type " << type.second << " has typeid " << type.first ;
141  //}
142  }
143 
144 #if ART_HEX_VERSION < 0x30000
145 
153  SharedMemoryReader(fhicl::ParameterSet const& ps, art::ProductRegistryHelper& help, art::SourceHelper const& pm,
154  art::MasterProductRegistry&)
155  : SharedMemoryReader(ps, help, pm) {}
156 #endif
157 
162 
167 
172  void readFile(std::string const&, art::FileBlock*& fb)
173  {
174  TLOG_ARB(5, "SharedMemoryReader") << "readFile enter/start";
175  fb = new art::FileBlock(art::FileFormatVersion(1, "RawEvent2011"), "nothing");
176  }
177 
182  bool hasMoreData() const { return (!shutdownMsgReceived); }
183 
194  bool readNext(art::RunPrincipal* const& inR, art::SubRunPrincipal* const& inSR, art::RunPrincipal*& outR,
195  art::SubRunPrincipal*& outSR, art::EventPrincipal*& outE)
196  {
197  TLOG_DEBUG("SharedMemoryReader") << "readNext BEGIN";
198  /*if (outputFileCloseNeeded) {
199  outputFileCloseNeeded = false;
200  return false;
201  }*/
202  // Establish default 'results'
203  outR = 0;
204  outSR = 0;
205  outE = 0;
206  // Try to get an event from the queue. We'll continuously loop, either until:
207  // 1) we have read a RawEvent off the queue, or
208  // 2) we have timed out, AND we are told the when we timeout we
209  // should stop.
210  // In any case, if we time out, we emit an informational message.
211 
213  {
214  TLOG_INFO("SharedMemoryReader") << "Shutdown Message received, returning false (should exit art)";
215  return false;
216  }
217 
218  auto read_start_time = std::chrono::steady_clock::now();
219  start:
220  bool keep_looping = true;
221  bool got_event = false;
222  auto sleepTimeUsec = waiting_time * 1000; // waiting_time * 1000000 us/s / 1000 reps = us/rep
223  if (sleepTimeUsec > 100000) sleepTimeUsec = 100000; // Don't wait longer than 1/10th of a second
224  while (keep_looping)
225  {
226  TLOG_TRACE("SharedMemoryReader") << "ReadyForRead loops BEGIN";
227  keep_looping = false;
228  auto start_time = std::chrono::steady_clock::now();
229  while (!got_event && TimeUtils::GetElapsedTimeMicroseconds(start_time) < 1000)
230  {
231  // BURN CPU for 1 ms!
232  got_event = incoming_events->ReadyForRead();
233  }
234  TLOG_TRACE("SharedMemoryReader") << "ReadyForRead spin end, poll begin";
235  while (!got_event && TimeUtils::GetElapsedTime(start_time) < waiting_time)
236  {
237  got_event = incoming_events->ReadyForRead();
238  if (!got_event)
239  {
240  usleep(sleepTimeUsec);
241  // TLOG_INFO("SharedMemoryReader") << "Waited " << TimeUtils::GetElapsedTime(start_time) << " of " <<
242  // waiting_time ;
243  }
244  }
245  TLOG_TRACE("SharedMemoryReader") << "ReadyForRead loops END";
246  if (!got_event)
247  {
248  TLOG_INFO("SharedMemoryReader") << "InputFailure: Reading timed out in SharedMemoryReader::readNext()";
249  keep_looping = resume_after_timeout;
250  }
251  }
252 
253  if (!got_event)
254  {
255  TLOG_INFO("SharedMemoryReader") << "Did not receive an event from Shared Memory, returning false";
256  shutdownMsgReceived = true;
257  return false;
258  }
259  TLOG_DEBUG("SharedMemoryReader") << "Got Event!";
260  auto got_event_time = std::chrono::steady_clock::now();
261 
262  auto errflag = false;
263  auto evtHeader = incoming_events->ReadHeader(errflag);
264  if (errflag) goto start; // Buffer was changed out from under reader!
265  auto fragmentTypes = incoming_events->GetFragmentTypes(errflag);
266  if (errflag) goto start; // Buffer was changed out from under reader!
267  if (fragmentTypes.size() == 0)
268  {
269  TLOG_ERROR("SharedMemoryReader") << "Event has no Fragments! Aborting!";
270  incoming_events->ReleaseBuffer();
271  return false;
272  }
273  auto firstFragmentType = *fragmentTypes.begin();
274  TLOG_DEBUG("SharedMemoryReader") << "First Fragment type is " << (int)firstFragmentType << " ("
275  << fragment_type_map_[firstFragmentType] << ")";
276 
277  // We return false, indicating we're done reading, if:
278  // 1) we did not obtain an event, because we timed out and were
279  // configured NOT to keep trying after a timeout, or
280  // 2) the event we read was the end-of-data marker: a null
281  // pointer
282  if (firstFragmentType == Fragment::EndOfDataFragmentType)
283  {
284  TLOG_DEBUG("SharedMemoryReader") << "Received shutdown message, returning false";
285  shutdownMsgReceived = true;
286  incoming_events->ReleaseBuffer();
287  return false;
288  }
289 
290  size_t qsize = incoming_events->ReadReadyCount(); // save the qsize at this point
291 
292  // Check the number of fragments in the RawEvent. If we have a single
293  // fragment and that fragment is marked as EndRun or EndSubrun we'll create
294  // the special principals for that.
295  art::Timestamp currentTime = 0;
296 #if 0
297  art::TimeValue_t lo_res_time = time(0);
298  TLOG_ARB(15, "SharedMemoryReader") << "lo_res_time = " << lo_res_time;
299  currentTime = ((lo_res_time & 0xffffffff) << 32);
300 #endif
301  timespec hi_res_time;
302  int retcode = clock_gettime(CLOCK_REALTIME, &hi_res_time);
303  TLOG_ARB(15, "SharedMemoryReader") << "hi_res_time tv_sec = " << hi_res_time.tv_sec
304  << " tv_nsec = " << hi_res_time.tv_nsec << " (retcode = " << retcode << ")";
305  if (retcode == 0)
306  {
307  currentTime = ((hi_res_time.tv_sec & 0xffffffff) << 32) | (hi_res_time.tv_nsec & 0xffffffff);
308  }
309  else
310  {
311  TLOG_ERROR("SharedMemoryReader")
312  << "Unable to fetch a high-resolution time with clock_gettime for art::Event Timestamp. "
313  << "The art::Event Timestamp will be zero for event " << evtHeader->event_id;
314  }
315 
316  // make new run if inR is 0 or if the run has changed
317  if (inR == 0 || inR->run() != evtHeader->run_id)
318  {
319  outR = pmaker.makeRunPrincipal(evtHeader->run_id, currentTime);
320  }
321 
322  if (firstFragmentType == Fragment::EndOfRunFragmentType)
323  {
324  art::EventID const evid(art::EventID::flushEvent());
325  outR = pmaker.makeRunPrincipal(evid.runID(), currentTime);
326  outSR = pmaker.makeSubRunPrincipal(evid.subRunID(), currentTime);
327  outE = pmaker.makeEventPrincipal(evid, currentTime);
328  incoming_events->ReleaseBuffer();
329  return true;
330  }
331  else if (firstFragmentType == Fragment::EndOfSubrunFragmentType)
332  {
333  // Check if inR == 0 or is a new run
334  if (inR == 0 || inR->run() != evtHeader->run_id)
335  {
336  outSR = pmaker.makeSubRunPrincipal(evtHeader->run_id, evtHeader->subrun_id, currentTime);
337 #if ART_HEX_VERSION > 0x30000
338  art::EventID const evid(art::EventID::flushEvent(outSR->subRunID()));
339 #else
340  art::EventID const evid(art::EventID::flushEvent(outSR->id()));
341 #endif
342  outE = pmaker.makeEventPrincipal(evid, currentTime);
343  }
344  else
345  {
346  // If the previous subrun was neither 0 nor flush and was identical with the current
347  // subrun, then it must have been associated with a data event. In that case, we need
348  // to generate a flush event with a valid run but flush subrun and event number in order
349  // to end the subrun.
350 #if ART_HEX_VERSION > 0x30000
351  if (inSR != 0 && !inSR->subRunID().isFlush() && inSR->subRun() == evtHeader->subrun_id)
352 #else
353  if (inSR != 0 && !inSR->id().isFlush() && inSR->subRun() == evtHeader->subrun_id)
354 #endif
355  {
356 #if ART_HEX_VERSION > 0x30000
357  art::EventID const evid(art::EventID::flushEvent(inR->runID()));
358 #else
359  art::EventID const evid(art::EventID::flushEvent(inR->id()));
360 #endif
361  outSR = pmaker.makeSubRunPrincipal(evid.subRunID(), currentTime);
362  outE = pmaker.makeEventPrincipal(evid, currentTime);
363  // If this is either a new or another empty subrun, then generate a flush event with
364  // valid run and subrun numbers but flush event number
365  //} else if(inSR==0 || inSR->id().isFlush()){
366  }
367  else
368  {
369  outSR = pmaker.makeSubRunPrincipal(evtHeader->run_id, evtHeader->subrun_id, currentTime);
370 #if ART_HEX_VERSION > 0x30000
371  art::EventID const evid(art::EventID::flushEvent(outSR->subRunID()));
372 #else
373  art::EventID const evid(art::EventID::flushEvent(outSR->id()));
374 #endif
375  outE = pmaker.makeEventPrincipal(evid, currentTime);
376  // Possible error condition
377  //} else {
378  }
379  outR = 0;
380  }
381  // outputFileCloseNeeded = true;
382  incoming_events->ReleaseBuffer();
383  return true;
384  }
385 
386  // make new subrun if inSR is 0 or if the subrun has changed
387  art::SubRunID subrun_check(evtHeader->run_id, evtHeader->subrun_id);
388 #if ART_HEX_VERSION > 0x30000
389  if (inSR == 0 || subrun_check != inSR->subRunID())
390  {
391 #else
392  if (inSR == 0 || subrun_check != inSR->id())
393  {
394 #endif
395  outSR = pmaker.makeSubRunPrincipal(evtHeader->run_id, evtHeader->subrun_id, currentTime);
396  }
397  outE = pmaker.makeEventPrincipal(evtHeader->run_id, evtHeader->subrun_id, evtHeader->event_id, currentTime);
398 
399  // insert the Fragments of each type into the EventPrincipal
400  std::map<Fragment::type_t, std::string>::const_iterator iter_end = fragment_type_map_.end();
401  for (auto& type_code : fragmentTypes)
402  {
403  std::map<Fragment::type_t, std::string>::const_iterator iter = fragment_type_map_.find(type_code);
404  TLOG_TRACE("SharedMemoryReader") << "Before GetFragmentsByType call, type is " << (int)type_code;
405  auto product = incoming_events->GetFragmentsByType(errflag, type_code);
406  TLOG_TRACE("SharedMemoryReader") << "After GetFragmentsByType call";
407  if (errflag) goto start; // Buffer was changed out from under reader!
408  for (auto& frag : *product) bytesRead += frag.sizeBytes();
409  if (iter != iter_end)
410  {
411  if (type_code == artdaq::Fragment::ContainerFragmentType)
412  {
413  std::unordered_map<std::string, std::unique_ptr<Fragments>> derived_fragments;
414  derived_fragments[iter->second] = std::make_unique<Fragments>();
415 
416  for (size_t ii = 0; ii < product->size(); ++ii)
417  {
418  ContainerFragment cf(product->at(ii));
419  auto contained_type = fragment_type_map_.find(cf.fragment_type());
420  if (contained_type != iter_end)
421  {
422  auto label = iter->second + contained_type->second;
423  if (!derived_fragments.count(label))
424  {
425  derived_fragments[label] = std::make_unique<Fragments>();
426  }
427  derived_fragments[label]->emplace_back(std::move(product->at(ii)));
428  }
429  else
430  {
431  derived_fragments[iter->second]->emplace_back(std::move(product->at(ii)));
432  }
433  }
434 
435  for (auto& type : derived_fragments)
436  {
437  put_product_in_principal(std::move(type.second), *outE, pretend_module_name, type.first);
438  }
439  }
440  else
441  {
442  put_product_in_principal(std::move(product), *outE, pretend_module_name, iter->second);
443  }
444  }
445  else
446  {
447  put_product_in_principal(std::move(product), *outE, pretend_module_name, unidentified_instance_name);
448  TLOG_WARNING("SharedMemoryReader")
449  << "UnknownFragmentType: The product instance name mapping for fragment type \"" << ((int)type_code)
450  << "\" is not known. Fragments of this "
451  << "type will be stored in the event with an instance name of \"" << unidentified_instance_name << "\".";
452  }
453  }
454  TLOG_TRACE("SharedMemoryReader") << "After putting fragments in event";
455 
456  auto read_finish_time = std::chrono::steady_clock::now();
457  incoming_events->ReleaseBuffer();
458  auto qcap = incoming_events->size();
459  TLOG_ARB(10, "SharedMemoryReader") << "readNext: bytesRead=" << bytesRead << " qsize=" << qsize << " cap=" << qcap
460  << " metricMan=" << (void*)metricMan.get();
461  if (metricMan)
462  {
463  metricMan->sendMetric("Avg Processing Time", artdaq::TimeUtils::GetElapsedTime(last_read_time, read_start_time),
464  "s", 2, MetricMode::Average);
465  metricMan->sendMetric("Avg Input Wait Time", artdaq::TimeUtils::GetElapsedTime(read_start_time, got_event_time),
466  "s", 3, MetricMode::Average);
467  metricMan->sendMetric("Avg Read Time", artdaq::TimeUtils::GetElapsedTime(got_event_time, read_finish_time), "s",
468  3, MetricMode::Average);
469  metricMan->sendMetric("bytesRead", bytesRead, "B", 3, MetricMode::LastPoint);
470  if (qcap > 0)
471  metricMan->sendMetric("queue%Used", static_cast<unsigned long int>(qsize * 100 / qcap), "%", 5,
472  MetricMode::LastPoint);
473  }
474 
475  TLOG_TRACE("SharedMemoryReader") << "Returning from readNext";
476  last_read_time = std::chrono::steady_clock::now();
477  return true;
478  }
479 
480  std::map<Fragment::type_t, std::string>
482  unsigned readNext_calls_;
483 };
484 } // namespace detail
485 } // namespace artdaq
486 
487 #endif /* artdaq_ArtModules_detail_SharedMemoryReader_hh */
The SharedMemoryReader is a class which implements the methods needed by art::Source.
unsigned readNext_calls_
The number of times readNext has been called.
SharedMemoryReader(fhicl::ParameterSet const &ps, art::ProductRegistryHelper &help, art::SourceHelper const &pm)
SharedMemoryReader Constructor.
SharedMemoryReader(fhicl::ParameterSet const &ps, art::ProductRegistryHelper &help, art::SourceHelper const &pm, art::MasterProductRegistry &)
SharedMemoryReader Constructor.
static void CleanUpGlobals()
Clean up statically-allocated Manager class instances.
Definition: Globals.hh:150
art::SourceHelper const & pmaker
An art::SourceHelper instance.
SharedMemoryReader & operator=(SharedMemoryReader const &)=delete
Copy Assignment operator is deleted.
double waiting_time
The amount of time to wait for an event from the queue.
bool resume_after_timeout
Whether to resume if the dequeue action times out.
std::string unidentified_instance_name
The name to use for unknown Fragment types.
bool outputFileCloseNeeded
If an explicit output file close message is needed.
std::unique_ptr< SharedMemoryEventReceiver > incoming_events
The events from the EventStore.
virtual ~SharedMemoryReader()
SharedMemoryReader destructor.
size_t bytesRead
running total of number of bytes received
void closeCurrentFile()
Emulate closing a file. No-Op.
std::chrono::steady_clock::time_point last_read_time
Time last read was completed.
std::string pretend_module_name
The module name to store data under.
bool readNext(art::RunPrincipal *const &inR, art::SubRunPrincipal *const &inSR, art::RunPrincipal *&outR, art::SubRunPrincipal *&outSR, art::EventPrincipal *&outE)
Dequeue a RawEvent and declare its Fragment contents to art, creating Run, SubRun, and EventPrincipal objects as necessary.
std::map< Fragment::type_t, std::string > fragment_type_map_
The Fragment type names that this SharedMemoryReader knows about.
bool hasMoreData() const
Whether more data is expected from the SharedMemoryReader.
void readFile(std::string const &, art::FileBlock *&fb)
Emulate opening a file.
SharedMemoryReader(SharedMemoryReader const &)=delete
Copy Constructor is deleted.
bool shutdownMsgReceived
Whether a shutdown message has been received.