artdaq  v3_07_02
SharedMemoryReader.hh
1 #ifndef artdaq_ArtModules_detail_SharedMemoryReader_hh
2 #define artdaq_ArtModules_detail_SharedMemoryReader_hh
3 
4 #include "art/Framework/Core/Frameworkfwd.h"
5 #include "artdaq-core/Utilities/ExceptionHandler.hh"
6 #include "artdaq/DAQdata/Globals.hh"
7 
8 #include <sys/time.h>
9 #include "art/Framework/Core/FileBlock.h"
10 #include "art/Framework/Core/ProductRegistryHelper.h"
11 #include "art/Framework/IO/Sources/SourceHelper.h"
12 #include "art/Framework/IO/Sources/put_product_in_principal.h"
13 #include "art/Framework/Principal/EventPrincipal.h"
14 #include "art/Framework/Principal/RunPrincipal.h"
15 #include "art/Framework/Principal/SubRunPrincipal.h"
16 #include "art/Framework/Services/Registry/ServiceHandle.h"
17 #include "artdaq-core/Core/SharedMemoryEventReceiver.hh"
18 #include "artdaq-core/Core/SharedMemoryManager.hh"
19 #include "artdaq-core/Data/ContainerFragment.hh"
20 #include "artdaq-core/Data/Fragment.hh"
21 #include "artdaq-core/Utilities/TimeUtils.hh"
22 #include "artdaq/ArtModules/ArtdaqSharedMemoryService.h"
23 #include "canvas/Persistency/Provenance/FileFormatVersion.h"
24 #include "fhiclcpp/ParameterSet.h"
25 
26 #include <map>
27 #include <string>
28 #include "artdaq-core/Data/RawEvent.hh"
29 
30 #define build_key(seed) seed + ((GetPartitionNumber() + 1) << 16) + (getppid() & 0xFFFF)
31 
32 namespace artdaq {
33 namespace detail {
39 {
40 public:
42  : type_map_() {}
43  virtual ~DefaultFragmentTypeTranslator() = default;
44 
48  virtual void SetBasicTypes(std::map<Fragment::type_t, std::string> const& type_map)
49  {
50  type_map_ = type_map;
51  }
52 
56  virtual void AddExtraType(artdaq::Fragment::type_t type_id, std::string type_name)
57  {
58  type_map_[type_id] = type_name;
59  }
60 
65  virtual std::string GetInstanceNameForType(artdaq::Fragment::type_t type_id, std::string unidentified_instance_name)
66  {
67  if (type_map_.count(type_id) > 0) { return type_map_[type_id]; }
68  return unidentified_instance_name;
69  }
70 
76  virtual std::set<std::string> GetAllProductInstanceNames()
77  {
78  std::set<std::string> output;
79  for (const auto& map_iter : type_map_)
80  {
81  std::string instance_name = map_iter.second;
82  if (!output.count(instance_name))
83  {
84  output.insert(instance_name);
85  TLOG_TRACE("DefaultFragmentTypeTranslator") << "Adding product instance name \"" << map_iter.second
86  << "\" to list of expected names";
87  }
88  }
89 
90  auto container_type = type_map_.find(Fragment::type_t(artdaq::Fragment::ContainerFragmentType));
91  if (container_type != type_map_.end())
92  {
93  std::string container_type_name = container_type->second;
94  std::set<std::string> tmp_copy = output;
95  for (const auto& set_iter : tmp_copy)
96  {
97  output.insert(container_type_name + set_iter);
98  }
99  }
100 
101  return output;
102  }
103 
110  virtual std::pair<bool, std::string>
111  GetInstanceNameForFragment(artdaq::Fragment const& fragment, std::string unidentified_instance_name)
112  {
113  auto type_map_end = type_map_.end();
114  bool success_code = true;
115  std::string instance_name;
116 
117  auto primary_type = type_map_.find(fragment.type());
118  if (primary_type != type_map_end)
119  {
120  instance_name = primary_type->second;
121  if (fragment.type() == artdaq::Fragment::ContainerFragmentType)
122  {
123  artdaq::ContainerFragment cf(fragment);
124  auto contained_type = type_map_.find(cf.fragment_type());
125  if (contained_type != type_map_end)
126  {
127  instance_name += contained_type->second;
128  }
129  }
130  }
131  else
132  {
133  instance_name = unidentified_instance_name;
134  success_code = false;
135  }
136 
137  return std::make_pair(success_code, instance_name);
138  }
139 
140 protected:
141  std::map<Fragment::type_t, std::string> type_map_;
142 };
143 
147 template<std::map<artdaq::Fragment::type_t, std::string> getDefaultTypes() = artdaq::Fragment::MakeSystemTypeMap,
148  class FTT = artdaq::detail::DefaultFragmentTypeTranslator>
150 {
154  SharedMemoryReader(SharedMemoryReader const&) = delete;
155 
161 
162  art::SourceHelper const& pmaker;
163  double waiting_time;
165  std::string pretend_module_name;
169  size_t bytesRead;
170  std::chrono::steady_clock::time_point last_read_time;
171  // std::unique_ptr<SharedMemoryManager> data_shm; ///< SharedMemoryManager containing data
172  // std::unique_ptr<SharedMemoryManager> broadcast_shm; ///< SharedMemoryManager containing broadcasts (control
173  // Fragments)
174 
189  SharedMemoryReader(fhicl::ParameterSet const& ps,
190  art::ProductRegistryHelper& help,
191  art::SourceHelper const& pm)
192  : pmaker(pm)
193  , waiting_time(ps.get<double>("waiting_time", 86400.0))
194  , resume_after_timeout(ps.get<bool>("resume_after_timeout", true))
195  , pretend_module_name(ps.get<std::string>("raw_data_label", "daq"))
196  , unidentified_instance_name("unidentified")
197  , shutdownMsgReceived(false)
198  , outputFileCloseNeeded(false)
199  , bytesRead(0)
200  , last_read_time(std::chrono::steady_clock::now())
201  , readNext_calls_(0)
202  {
203 #if 0
204  volatile bool keep_looping = true;
205  while (keep_looping)
206  {
207  usleep(10000);
208  }
209 #endif
210 
211  // Make sure the ArtdaqSharedMemoryService is available
212  art::ServiceHandle<ArtdaqSharedMemoryService> shm;
213 
214  help.reconstitutes<Fragments, art::InEvent>(pretend_module_name, unidentified_instance_name);
215 
216  // Workaround for #22979
217  help.reconstitutes<Fragments, art::InRun>(pretend_module_name, unidentified_instance_name);
218  help.reconstitutes<Fragments, art::InSubRun>(pretend_module_name, unidentified_instance_name);
219 
220  translator_.SetBasicTypes(getDefaultTypes());
221  auto extraTypes = ps.get<std::vector<std::pair<Fragment::type_t, std::string>>>("fragment_type_map", std::vector<std::pair<Fragment::type_t, std::string>>());
222  for (auto it = extraTypes.begin(); it != extraTypes.end(); ++it)
223  {
224  translator_.AddExtraType(it->first, it->second);
225  }
226  std::set<std::string> instance_names = translator_.GetAllProductInstanceNames();
227  for (const auto& set_iter : instance_names)
228  {
229  help.reconstitutes<Fragments, art::InEvent>(pretend_module_name, set_iter);
230  }
231 
232  TLOG_INFO("SharedMemoryReader") << "SharedMemoryReader initialized with ParameterSet: " << ps.to_string();
233  }
234 
235 #if ART_HEX_VERSION < 0x30000
236 
244  SharedMemoryReader(fhicl::ParameterSet const& ps, art::ProductRegistryHelper& help, art::SourceHelper const& pm,
245  art::MasterProductRegistry&)
246  : SharedMemoryReader(ps, help, pm) {}
247 #endif
248 
252  virtual ~SharedMemoryReader() {}
253 
258 
263  void readFile(std::string const&, art::FileBlock*& fb)
264  {
265  TLOG_ARB(5, "SharedMemoryReader") << "readFile enter/start";
266  fb = new art::FileBlock(art::FileFormatVersion(1, "RawEvent2011"), "nothing");
267  }
268 
273  bool hasMoreData() const
274  {
275  TLOG(TLVL_DEBUG, "SharedMemoryReader") << "hasMoreData returning " << std::boolalpha << !shutdownMsgReceived;
276  return (!shutdownMsgReceived);
277  }
278 
289  bool readNext(art::RunPrincipal* const& inR, art::SubRunPrincipal* const& inSR, art::RunPrincipal*& outR,
290  art::SubRunPrincipal*& outSR, art::EventPrincipal*& outE)
291  {
292  TLOG_DEBUG("SharedMemoryReader") << "readNext BEGIN";
293  art::ServiceHandle<ArtdaqSharedMemoryService> shm;
294 
295  /*if (outputFileCloseNeeded) {
296  outputFileCloseNeeded = false;
297  return false;
298  }*/
299  // Establish default 'results'
300  outR = 0;
301  outSR = 0;
302  outE = 0;
303  // Try to get an event from the queue. We'll continuously loop, either until:
304  // 1) we have read a RawEvent off the queue, or
305  // 2) we have timed out, AND we are told the when we timeout we
306  // should stop.
307  // In any case, if we time out, we emit an informational message.
308 
310  {
311  TLOG_INFO("SharedMemoryReader") << "Shutdown Message received, returning false (should exit art)";
312  return false;
313  }
314 
315  auto read_start_time = std::chrono::steady_clock::now();
316 
317  std::unordered_map<artdaq::Fragment::type_t, std::unique_ptr<artdaq::Fragments>> eventMap;
318  while (eventMap.size() == 0 && artdaq::TimeUtils::GetElapsedTime(read_start_time) < waiting_time)
319  {
320  eventMap = shm->ReceiveEvent(false);
321  }
322  if (eventMap.size() == 0)
323  {
324  TLOG_ERROR("SharedMemoryReader") << "No data received after " << waiting_time << " seconds. Returning false (should exit art)";
325  shutdownMsgReceived = true;
326  return false;
327  }
328 
329  auto got_event_time = std::chrono::steady_clock::now();
330  auto firstFragmentType = eventMap.begin()->first;
331  TLOG_DEBUG("SharedMemoryReader") << "First Fragment type is " << (int)firstFragmentType << " ("
332  << translator_.GetInstanceNameForType(firstFragmentType, unidentified_instance_name) << ")";
333  // We return false, indicating we're done reading, if:
334  // 1) we did not obtain an event, because we timed out and were
335  // configured NOT to keep trying after a timeout, or
336  // 2) the event we read was the end-of-data marker: a null
337  // pointer
338  if (firstFragmentType == Fragment::EndOfDataFragmentType)
339  {
340  TLOG_DEBUG("SharedMemoryReader") << "Received shutdown message, returning false";
341  shutdownMsgReceived = true;
342  return false;
343  }
344 
345  auto evtHeader = shm->GetEventHeader();
346  size_t qsize = shm->GetQueueSize(); // save the qsize at this point
347 
348  // Check the number of fragments in the RawEvent. If we have a single
349  // fragment and that fragment is marked as EndRun or EndSubrun we'll create
350  // the special principals for that.
351  art::Timestamp currentTime = 0;
352 #if 0
353  art::TimeValue_t lo_res_time = time(0);
354  TLOG_ARB(15, "SharedMemoryReader") << "lo_res_time = " << lo_res_time;
355  currentTime = ((lo_res_time & 0xffffffff) << 32);
356 #endif
357  timespec hi_res_time;
358  int retcode = clock_gettime(CLOCK_REALTIME, &hi_res_time);
359  TLOG_ARB(15, "SharedMemoryReader") << "hi_res_time tv_sec = " << hi_res_time.tv_sec
360  << " tv_nsec = " << hi_res_time.tv_nsec << " (retcode = " << retcode << ")";
361  if (retcode == 0)
362  {
363  currentTime = ((hi_res_time.tv_sec & 0xffffffff) << 32) | (hi_res_time.tv_nsec & 0xffffffff);
364  }
365  else
366  {
367  TLOG_ERROR("SharedMemoryReader")
368  << "Unable to fetch a high-resolution time with clock_gettime for art::Event Timestamp. "
369  << "The art::Event Timestamp will be zero for event " << evtHeader->event_id;
370  }
371 
372  // make new run if inR is 0 or if the run has changed
373  if (inR == 0 || inR->run() != evtHeader->run_id)
374  {
375  outR = pmaker.makeRunPrincipal(evtHeader->run_id, currentTime);
376  }
377 
378  if (firstFragmentType == Fragment::EndOfRunFragmentType)
379  {
380  art::EventID const evid(art::EventID::flushEvent());
381  outR = pmaker.makeRunPrincipal(evid.runID(), currentTime);
382  outSR = pmaker.makeSubRunPrincipal(evid.subRunID(), currentTime);
383  outE = pmaker.makeEventPrincipal(evid, currentTime);
384  return true;
385  }
386  else if (firstFragmentType == Fragment::EndOfSubrunFragmentType)
387  {
388  // Check if inR == 0 or is a new run
389  if (inR == 0 || inR->run() != evtHeader->run_id)
390  {
391  outSR = pmaker.makeSubRunPrincipal(evtHeader->run_id, evtHeader->subrun_id, currentTime);
392 #if ART_HEX_VERSION > 0x30000
393  art::EventID const evid(art::EventID::flushEvent(outSR->subRunID()));
394 #else
395  art::EventID const evid(art::EventID::flushEvent(outSR->id()));
396 #endif
397  outE = pmaker.makeEventPrincipal(evid, currentTime);
398  }
399  else
400  {
401  // If the previous subrun was neither 0 nor flush and was identical with the current
402  // subrun, then it must have been associated with a data event. In that case, we need
403  // to generate a flush event with a valid run but flush subrun and event number in order
404  // to end the subrun.
405 #if ART_HEX_VERSION > 0x30000
406  if (inSR != 0 && !inSR->subRunID().isFlush() && inSR->subRun() == evtHeader->subrun_id)
407 #else
408  if (inSR != 0 && !inSR->id().isFlush() && inSR->subRun() == evtHeader->subrun_id)
409 #endif
410  {
411 #if ART_HEX_VERSION > 0x30000
412  art::EventID const evid(art::EventID::flushEvent(inR->runID()));
413 #else
414  art::EventID const evid(art::EventID::flushEvent(inR->id()));
415 #endif
416  outSR = pmaker.makeSubRunPrincipal(evid.subRunID(), currentTime);
417  outE = pmaker.makeEventPrincipal(evid, currentTime);
418  // If this is either a new or another empty subrun, then generate a flush event with
419  // valid run and subrun numbers but flush event number
420  //} else if(inSR==0 || inSR->id().isFlush()){
421  }
422  else
423  {
424  outSR = pmaker.makeSubRunPrincipal(evtHeader->run_id, evtHeader->subrun_id, currentTime);
425 #if ART_HEX_VERSION > 0x30000
426  art::EventID const evid(art::EventID::flushEvent(outSR->subRunID()));
427 #else
428  art::EventID const evid(art::EventID::flushEvent(outSR->id()));
429 #endif
430  outE = pmaker.makeEventPrincipal(evid, currentTime);
431  // Possible error condition
432  //} else {
433  }
434  outR = 0;
435  }
436  // outputFileCloseNeeded = true;
437  return true;
438  }
439 
440  // make new subrun if inSR is 0 or if the subrun has changed
441  art::SubRunID subrun_check(evtHeader->run_id, evtHeader->subrun_id);
442 #if ART_HEX_VERSION > 0x30000
443  if (inSR == 0 || subrun_check != inSR->subRunID())
444  {
445 #else
446  if (inSR == 0 || subrun_check != inSR->id())
447  {
448 #endif
449  outSR = pmaker.makeSubRunPrincipal(evtHeader->run_id, evtHeader->subrun_id, currentTime);
450  }
451  outE = pmaker.makeEventPrincipal(evtHeader->run_id, evtHeader->subrun_id, evtHeader->event_id, currentTime);
452 
453  double fragmentLatency = 0;
454  double fragmentLatencyMax = 0.0;
455  size_t fragmentCount = 0;
456 
457  // insert the Fragments of each type into the EventPrincipal
458  for (auto& fragmentTypePair : eventMap)
459  {
460  auto type_code = fragmentTypePair.first;
461  TLOG_TRACE("SharedMemoryReader") << "Before GetFragmentsByType call, type is " << (int)type_code;
462  TLOG_TRACE("SharedMemoryReader") << "After GetFragmentsByType call, number of fragments is " << fragmentTypePair.second->size();
463 
464  std::unordered_map<std::string, std::unique_ptr<Fragments>> derived_fragments;
465  for (auto& frag : *fragmentTypePair.second)
466  {
467  bytesRead += frag.sizeBytes();
468  auto latency_s = frag.getLatency(true);
469  double latency = latency_s.tv_sec + (latency_s.tv_nsec / 1000000000.0);
470 
471  fragmentLatency += latency;
472  fragmentCount++;
473  if (latency > fragmentLatencyMax) fragmentLatencyMax = latency;
474 
475  std::pair<bool, std::string> instance_name_result =
476  translator_.GetInstanceNameForFragment(frag, unidentified_instance_name);
477  std::string label = instance_name_result.second;
478  if (!instance_name_result.first)
479  {
480  TLOG_WARNING("SharedMemoryReader")
481  << "UnknownFragmentType: The product instance name mapping for fragment type \"" << ((int)type_code)
482  << "\" is not known. Fragments of this "
483  << "type will be stored in the event with an instance name of \"" << unidentified_instance_name << "\".";
484  }
485  if (!derived_fragments.count(label))
486  {
487  derived_fragments[label] = std::make_unique<Fragments>();
488  }
489  derived_fragments[label]->emplace_back(std::move(frag));
490  }
491  for (auto& type : derived_fragments)
492  {
493  put_product_in_principal(std::move(type.second),
494  *outE,
496  type.first);
497  }
498  }
499  TLOG_TRACE("SharedMemoryReader") << "After putting fragments in event";
500 
501  auto read_finish_time = std::chrono::steady_clock::now();
502  auto qcap = shm->GetQueueCapacity();
503  TLOG_ARB(10, "SharedMemoryReader") << "readNext: bytesRead=" << bytesRead << " qsize=" << qsize << " cap=" << qcap
504  << " metricMan=" << (void*)metricMan.get();
505  if (metricMan)
506  {
507  metricMan->sendMetric("Avg Processing Time", artdaq::TimeUtils::GetElapsedTime(last_read_time, read_start_time),
508  "s", 2, MetricMode::Average);
509  metricMan->sendMetric("Avg Input Wait Time", artdaq::TimeUtils::GetElapsedTime(read_start_time, got_event_time),
510  "s", 3, MetricMode::Average);
511  metricMan->sendMetric("Avg Read Time", artdaq::TimeUtils::GetElapsedTime(got_event_time, read_finish_time), "s",
512  3, MetricMode::Average);
513  metricMan->sendMetric("bytesRead", bytesRead, "B", 3, MetricMode::LastPoint);
514  if (qcap > 0)
515  metricMan->sendMetric("queue%Used", static_cast<unsigned long int>(qsize * 100 / qcap), "%", 5,
516  MetricMode::LastPoint);
517 
518  metricMan->sendMetric("SharedMemoryReader Latency", fragmentLatency / fragmentCount, "s", 4, MetricMode::Average);
519  metricMan->sendMetric("SharedMemoryReader Maximum Latency", fragmentLatencyMax, "s", 4, MetricMode::Maximum);
520  }
521 
522  TLOG_TRACE("SharedMemoryReader") << "Returning from readNext";
523  last_read_time = std::chrono::steady_clock::now();
524  return true;
525  }
526 
527  unsigned readNext_calls_;
529 };
530 } // namespace detail
531 } // namespace artdaq
532 
533 #endif /* artdaq_ArtModules_detail_SharedMemoryReader_hh */
bool hasMoreData() const
Whether more data is expected from the SharedMemoryReader.
void closeCurrentFile()
Emulate closing a file. No-Op.
bool readNext(art::RunPrincipal *const &inR, art::SubRunPrincipal *const &inSR, art::RunPrincipal *&outR, art::SubRunPrincipal *&outSR, art::EventPrincipal *&outE)
Dequeue a RawEvent and declare its Fragment contents to art, creating Run, SubRun, and EventPrincipal objects as necessary.
size_t bytesRead
running total of number of bytes received
The SharedMemoryReader is a class which implements the methods needed by art::Source.
FTT translator_
An instance of the template parameter FragmentTypeTranslator that translates Fragment Type IDs to str...
SharedMemoryReader(SharedMemoryReader const &)=delete
Copy Constructor is deleted.
std::string pretend_module_name
The module name to store data under.
bool shutdownMsgReceived
Whether a shutdown message has been received.
bool resume_after_timeout
Whether to resume if the dequeue action times out.
double waiting_time
The amount of time to wait for an event from the queue.
SharedMemoryReader(fhicl::ParameterSet const &ps, art::ProductRegistryHelper &help, art::SourceHelper const &pm)
SharedMemoryReader Constructor.
unsigned readNext_calls_
The number of times readNext has been called.
virtual void SetBasicTypes(std::map< Fragment::type_t, std::string > const &type_map)
Sets the basic types to be translated. (Should not include &quot;container&quot; types.)
void readFile(std::string const &, art::FileBlock *&fb)
Emulate opening a file.
bool outputFileCloseNeeded
If an explicit output file close message is needed.
virtual void AddExtraType(artdaq::Fragment::type_t type_id, std::string type_name)
Adds an additional type to be translated.
SharedMemoryReader & operator=(SharedMemoryReader const &)=delete
Copy Assignment operator is deleted.
virtual std::string GetInstanceNameForType(artdaq::Fragment::type_t type_id, std::string unidentified_instance_name)
Returns the basic translation for the specified type. Defaults to the specified unidentified_instance...
art::SourceHelper const & pmaker
An art::SourceHelper instance.
virtual std::pair< bool, std::string > GetInstanceNameForFragment(artdaq::Fragment const &fragment, std::string unidentified_instance_name)
Returns the product instance name for the specified fragment, based on the types that have been speci...
std::string unidentified_instance_name
The name to use for unknown Fragment types.
std::map< Fragment::type_t, std::string > type_map_
Map relating Fragment Type to strings.
virtual std::set< std::string > GetAllProductInstanceNames()
Returns the full set of product instance names which may be present in the data, based on the types t...
SharedMemoryReader(fhicl::ParameterSet const &ps, art::ProductRegistryHelper &help, art::SourceHelper const &pm, art::MasterProductRegistry &)
SharedMemoryReader Constructor.
The DefaultFragmentTypeTranslator class provides default behavior for experiment-specific customizati...
std::chrono::steady_clock::time_point last_read_time
Time last read was completed.
virtual ~SharedMemoryReader()
SharedMemoryReader destructor.