artdaq  v3_03_00
SharedMemoryReader.hh
1 #ifndef artdaq_ArtModules_detail_SharedMemoryReader_hh
2 #define artdaq_ArtModules_detail_SharedMemoryReader_hh
3 
4 #include "artdaq/DAQdata/Globals.hh"
5 #include "artdaq-core/Utilities/ExceptionHandler.hh"
6 #include "art/Framework/Core/Frameworkfwd.h"
7 
8 #include "art/Framework/Core/FileBlock.h"
9 #include "art/Framework/Core/ProductRegistryHelper.h"
10 #include "art/Framework/IO/Sources/SourceHelper.h"
11 #include "art/Framework/Principal/EventPrincipal.h"
12 #include "art/Framework/Principal/RunPrincipal.h"
13 #include "art/Framework/Principal/SubRunPrincipal.h"
14 #include "artdaq-core/Core/SharedMemoryManager.hh"
15 #include "artdaq-core/Utilities/TimeUtils.hh"
16 #include "fhiclcpp/ParameterSet.h"
17 #include "artdaq-core/Data/Fragment.hh"
18 #include "artdaq-core/Data/ContainerFragment.hh"
19 #include "artdaq-core/Core/SharedMemoryEventReceiver.hh"
20 #include "art/Framework/IO/Sources/put_product_in_principal.h"
21 #include "canvas/Persistency/Provenance/FileFormatVersion.h"
22 #include <sys/time.h>
23 
24 
25 #include <string>
26 #include <map>
27 #include "artdaq-core/Data/RawEvent.hh"
28 
29 namespace artdaq
30 {
31  namespace detail
32  {
36  template<std::map<artdaq::Fragment::type_t, std::string> getDefaultTypes() = artdaq::Fragment::MakeSystemTypeMap >
38  {
42  SharedMemoryReader(SharedMemoryReader const&) = delete;
43 
49 
50  art::SourceHelper const& pmaker;
51  std::unique_ptr<SharedMemoryEventReceiver> incoming_events;
52  double waiting_time;
54  std::string pretend_module_name;
58  size_t bytesRead;
59  //std::unique_ptr<SharedMemoryManager> data_shm; ///< SharedMemoryManager containing data
60  //std::unique_ptr<SharedMemoryManager> broadcast_shm; ///< SharedMemoryManager containing broadcasts (control Fragments)
61 
76  SharedMemoryReader(fhicl::ParameterSet const& ps,
77  art::ProductRegistryHelper& help,
78  art::SourceHelper const& pm)
79  : pmaker(pm)
80  , waiting_time(ps.get<double>("waiting_time", 86400.0))
81  , resume_after_timeout(ps.get<bool>("resume_after_timeout", true))
82  , pretend_module_name(ps.get<std::string>("raw_data_label", "daq"))
83  , unidentified_instance_name("unidentified")
84  , shutdownMsgReceived(false)
85  , outputFileCloseNeeded(false)
86  , bytesRead(0)
87  , fragment_type_map_(getDefaultTypes())
88  , readNext_calls_(0)
89  {
90  try {
91  if (metricMan)
92  {
93  metricMan->initialize(ps.get<fhicl::ParameterSet>("metrics", fhicl::ParameterSet()), "artdaqart");
94  metricMan->do_start();
95  }
96  }
97  catch (...)
98  {
99  ExceptionHandler(ExceptionHandlerRethrow::no, "Error loading metrics in SharedMemoryReader()");
100  }
101 
102  // For testing
103  //if (ps.has_key("buffer_count") && (ps.has_key("max_event_size_bytes") || (ps.has_key("expected_fragments_per_event") && ps.has_key("max_fragment_size_bytes"))))
104  //{
105  // data_shm.reset(new SharedMemoryManager(ps.get<uint32_t>("shared_memory_key", 0xBEE70000 + getppid()), ps.get<int>("buffer_count"), ps.has_key("max_event_size_bytes") ? ps.get<size_t>("max_event_size_bytes") : ps.get<size_t>("expected_fragments_per_event") * ps.get<size_t>("max_fragment_size_bytes")));
106  // broadcast_shm.reset(new SharedMemoryManager(ps.get<uint32_t>("broadcast_shared_memory_key", 0xCEE70000 + getppid()), ps.get<int>("broadcast_buffer_count", 5), ps.get<size_t>("broadcast_buffer_size", 0x100000)));
107  //}
108  incoming_events.reset(new SharedMemoryEventReceiver(ps.get<uint32_t>("shared_memory_key", 0xBEE70000 + getppid()), ps.get<uint32_t>("broadcast_shared_memory_key", 0xCEE70000 + getppid())));
109  my_rank = incoming_events->GetRank();
110 
111  help.reconstitutes<Fragments, art::InEvent>(pretend_module_name, unidentified_instance_name);
112  for (auto it = fragment_type_map_.begin(); it != fragment_type_map_.end(); ++it)
113  {
114  help.reconstitutes<Fragments, art::InEvent>(pretend_module_name, it->second);
115  help.reconstitutes<Fragments, art::InEvent>(pretend_module_name, "Container" + it->second);
116  }
117  auto extraTypes = ps.get<std::vector<std::pair<Fragment::type_t, std::string>>>("fragment_type_map", std::vector<std::pair<Fragment::type_t, std::string>>());
118  for (auto it = extraTypes.begin(); it != extraTypes.end(); ++it)
119  {
120  fragment_type_map_[it->first] = it->second;
121  help.reconstitutes<Fragments, art::InEvent>(pretend_module_name, it->second);
122  help.reconstitutes<Fragments, art::InEvent>(pretend_module_name, "Container" + it->second);
123  }
124  TLOG_INFO("SharedMemoryReader") << "SharedMemoryReader initialized with ParameterSet: " << ps.to_string();
125  //for(auto& type : fragment_type_map_)
126  //{
127  // TLOG_INFO("SharedMemoryReader") << "Fragment Type " << type.second << " has typeid " << type.first ;
128  //}
129  }
130 
139  SharedMemoryReader(fhicl::ParameterSet const& ps,
140  art::ProductRegistryHelper& help,
141  art::SourceHelper const& pm,
142  art::MasterProductRegistry&) : SharedMemoryReader(ps, help, pm)
143  {}
144 
149  artdaq::Globals::CleanUpGlobals();
150  }
151 
156 
161  void readFile(std::string const&, art::FileBlock*& fb)
162  {
163  TLOG_ARB(5, "SharedMemoryReader") << "readFile enter/start";
164  fb = new art::FileBlock(art::FileFormatVersion(1, "RawEvent2011"), "nothing");
165  }
166 
171  bool hasMoreData() const { return (!shutdownMsgReceived); }
172 
183  bool readNext(art::RunPrincipal* const & inR,
184  art::SubRunPrincipal* const & inSR,
185  art::RunPrincipal*& outR,
186  art::SubRunPrincipal*& outSR,
187  art::EventPrincipal*& outE)
188  {
189  TLOG_DEBUG("SharedMemoryReader") << "readNext BEGIN";
190  /*if (outputFileCloseNeeded) {
191  outputFileCloseNeeded = false;
192  return false;
193  }*/
194  // Establish default 'results'
195  outR = 0;
196  outSR = 0;
197  outE = 0;
198  // Try to get an event from the queue. We'll continuously loop, either until:
199  // 1) we have read a RawEvent off the queue, or
200  // 2) we have timed out, AND we are told the when we timeout we
201  // should stop.
202  // In any case, if we time out, we emit an informational message.
203 
204  if (shutdownMsgReceived)
205  {
206  TLOG_INFO("SharedMemoryReader") << "Shutdown Message received, returning false (should exit art)";
207  return false;
208  }
209 
210  start:
211  bool keep_looping = true;
212  bool got_event = false;
213  auto sleepTimeUsec = waiting_time * 1000; // waiting_time * 1000000 us/s / 1000 reps = us/rep
214  if (sleepTimeUsec > 100000) sleepTimeUsec = 100000; // Don't wait longer than 1/10th of a second
215  while (keep_looping)
216  {
217  TLOG_TRACE("SharedMemoryReader") << "ReadyForRead loops BEGIN";
218  keep_looping = false;
219  auto start_time = std::chrono::steady_clock::now();
220  while (!got_event && TimeUtils::GetElapsedTimeMicroseconds(start_time) < 1000)
221  {
222  // BURN CPU for 1 ms!
223  got_event = incoming_events->ReadyForRead();
224  }
225  TLOG_TRACE("SharedMemoryReader") << "ReadyForRead spin end, poll begin";
226  while (!got_event && TimeUtils::GetElapsedTime(start_time) < waiting_time)
227  {
228  got_event = incoming_events->ReadyForRead();
229  if (!got_event)
230  {
231  usleep(sleepTimeUsec);
232  //TLOG_INFO("SharedMemoryReader") << "Waited " << TimeUtils::GetElapsedTime(start_time) << " of " << waiting_time ;
233  }
234  }
235  TLOG_TRACE("SharedMemoryReader") << "ReadyForRead loops END";
236  if (!got_event)
237  {
238  TLOG_INFO("SharedMemoryReader")
239  << "InputFailure: Reading timed out in SharedMemoryReader::readNext()";
240  keep_looping = resume_after_timeout;
241  }
242  }
243 
244  if (!got_event)
245  {
246  TLOG_INFO("SharedMemoryReader") << "Did not receive an event from Shared Memory, returning false";
247  shutdownMsgReceived = true;
248  return false;
249  }
250  TLOG_DEBUG("SharedMemoryReader") << "Got Event!";
251 
252  auto errflag = false;
253  auto evtHeader = incoming_events->ReadHeader(errflag);
254  if (errflag) goto start; // Buffer was changed out from under reader!
255  auto fragmentTypes = incoming_events->GetFragmentTypes(errflag);
256  if (errflag) goto start; // Buffer was changed out from under reader!
257  if (fragmentTypes.size() == 0)
258  {
259  TLOG_ERROR("SharedMemoryReader") << "Event has no Fragments! Aborting!";
260  incoming_events->ReleaseBuffer();
261  return false;
262  }
263  auto firstFragmentType = *fragmentTypes.begin();
264  TLOG_DEBUG("SharedMemoryReader") << "First Fragment type is " << (int)firstFragmentType << " (" << fragment_type_map_[firstFragmentType] << ")";
265 
266  // We return false, indicating we're done reading, if:
267  // 1) we did not obtain an event, because we timed out and were
268  // configured NOT to keep trying after a timeout, or
269  // 2) the event we read was the end-of-data marker: a null
270  // pointer
271  if (firstFragmentType == Fragment::EndOfDataFragmentType)
272  {
273  TLOG_DEBUG("SharedMemoryReader") << "Received shutdown message, returning false";
274  shutdownMsgReceived = true;
275  incoming_events->ReleaseBuffer();
276  return false;
277  }
278 
279  size_t qsize = incoming_events->ReadReadyCount(); // save the qsize at this point
280 
281  // Check the number of fragments in the RawEvent. If we have a single
282  // fragment and that fragment is marked as EndRun or EndSubrun we'll create
283  // the special principals for that.
284  art::Timestamp currentTime = 0;
285 #if 0
286  art::TimeValue_t lo_res_time = time(0);
287  TLOG_ARB(15, "SharedMemoryReader") << "lo_res_time = " << lo_res_time;
288  currentTime = ((lo_res_time & 0xffffffff) << 32);
289 #endif
290  timespec hi_res_time;
291  int retcode = clock_gettime(CLOCK_REALTIME, &hi_res_time);
292  TLOG_ARB(15, "SharedMemoryReader") << "hi_res_time tv_sec = " << hi_res_time.tv_sec
293  << " tv_nsec = " << hi_res_time.tv_nsec
294  << " (retcode = " << retcode << ")";
295  if (retcode == 0)
296  {
297  currentTime = ((hi_res_time.tv_sec & 0xffffffff) << 32) |
298  (hi_res_time.tv_nsec & 0xffffffff);
299  }
300  else
301  {
302  TLOG_ERROR("SharedMemoryReader") << "Unable to fetch a high-resolution time with clock_gettime for art::Event Timestamp. "
303  << "The art::Event Timestamp will be zero for event " << evtHeader->event_id;
304  }
305 
306  // make new run if inR is 0 or if the run has changed
307  if (inR == 0 || inR->run() != evtHeader->run_id)
308  {
309  outR = pmaker.makeRunPrincipal(evtHeader->run_id,
310  currentTime);
311  }
312 
313  if (firstFragmentType == Fragment::EndOfRunFragmentType)
314  {
315  art::EventID const evid(art::EventID::flushEvent());
316  outR = pmaker.makeRunPrincipal(evid.runID(), currentTime);
317  outSR = pmaker.makeSubRunPrincipal(evid.subRunID(), currentTime);
318  outE = pmaker.makeEventPrincipal(evid, currentTime);
319  incoming_events->ReleaseBuffer();
320  return true;
321  }
322  else if (firstFragmentType == Fragment::EndOfSubrunFragmentType)
323  {
324  // Check if inR == 0 or is a new run
325  if (inR == 0 || inR->run() != evtHeader->run_id)
326  {
327  outSR = pmaker.makeSubRunPrincipal(evtHeader->run_id,
328  evtHeader->subrun_id,
329  currentTime);
330  art::EventID const evid(art::EventID::flushEvent(outSR->id()));
331  outE = pmaker.makeEventPrincipal(evid, currentTime);
332  }
333  else
334  {
335  // If the previous subrun was neither 0 nor flush and was identical with the current
336  // subrun, then it must have been associated with a data event. In that case, we need
337  // to generate a flush event with a valid run but flush subrun and event number in order
338  // to end the subrun.
339  if (inSR != 0 && !inSR->id().isFlush() && inSR->subRun() == evtHeader->subrun_id)
340  {
341  art::EventID const evid(art::EventID::flushEvent(inR->id()));
342  outSR = pmaker.makeSubRunPrincipal(evid.subRunID(), currentTime);
343  outE = pmaker.makeEventPrincipal(evid, currentTime);
344  // If this is either a new or another empty subrun, then generate a flush event with
345  // valid run and subrun numbers but flush event number
346  //} else if(inSR==0 || inSR->id().isFlush()){
347  }
348  else
349  {
350  outSR = pmaker.makeSubRunPrincipal(evtHeader->run_id,
351  evtHeader->subrun_id,
352  currentTime);
353  art::EventID const evid(art::EventID::flushEvent(outSR->id()));
354  outE = pmaker.makeEventPrincipal(evid, currentTime);
355  // Possible error condition
356  //} else {
357  }
358  outR = 0;
359  }
360  //outputFileCloseNeeded = true;
361  incoming_events->ReleaseBuffer();
362  return true;
363  }
364 
365  // make new subrun if inSR is 0 or if the subrun has changed
366  art::SubRunID subrun_check(evtHeader->run_id, evtHeader->subrun_id);
367  if (inSR == 0 || subrun_check != inSR->id())
368  {
369  outSR = pmaker.makeSubRunPrincipal(evtHeader->run_id,
370  evtHeader->subrun_id,
371  currentTime);
372  }
373  outE = pmaker.makeEventPrincipal(evtHeader->run_id,
374  evtHeader->subrun_id,
375  evtHeader->event_id,
376  currentTime);
377 
378  // insert the Fragments of each type into the EventPrincipal
379  std::map<Fragment::type_t, std::string>::const_iterator iter_end =
380  fragment_type_map_.end();
381  for (auto& type_code : fragmentTypes)
382  {
383  std::map<Fragment::type_t, std::string>::const_iterator iter =
384  fragment_type_map_.find(type_code);
385  auto product = incoming_events->GetFragmentsByType(errflag, type_code);
386  if (errflag) goto start; // Buffer was changed out from under reader!
387  for (auto &frag : *product)
388  bytesRead += frag.sizeBytes();
389  if (iter != iter_end)
390  {
391  if (type_code == artdaq::Fragment::ContainerFragmentType)
392  {
393  std::unordered_map<std::string, std::unique_ptr<Fragments>> derived_fragments;
394  derived_fragments[iter->second] = std::make_unique<Fragments>();
395 
396  for (size_t ii = 0; ii < product->size(); ++ii)
397  {
398  ContainerFragment cf(product->at(ii));
399  auto contained_type = fragment_type_map_.find(cf.fragment_type());
400  if (contained_type != iter_end)
401  {
402  auto label = iter->second + contained_type->second;
403  if (!derived_fragments.count(label))
404  {
405  derived_fragments[label] = std::make_unique<Fragments>();
406  }
407  derived_fragments[label]->emplace_back(std::move(product->at(ii)));
408  }
409  else
410  {
411  derived_fragments[iter->second]->emplace_back(std::move(product->at(ii)));
412  }
413  }
414 
415  for (auto& type : derived_fragments)
416  {
417  put_product_in_principal(std::move(type.second),
418  *outE,
420  type.first);
421  }
422 
423  }
424  else
425  {
426  put_product_in_principal(std::move(product),
427  *outE,
429  iter->second);
430  }
431  }
432  else
433  {
434  put_product_in_principal(std::move(product),
435  *outE,
438  TLOG_WARNING("SharedMemoryReader")
439  << "UnknownFragmentType: The product instance name mapping for fragment type \""
440  << ((int)type_code) << "\" is not known. Fragments of this "
441  << "type will be stored in the event with an instance name of \""
442  << unidentified_instance_name << "\".";
443  }
444  }
445  incoming_events->ReleaseBuffer();
446  TLOG_ARB(10, "SharedMemoryReader") << "readNext: bytesRead=" << bytesRead << " qsize=" << qsize << " cap=" << incoming_events->size() << " metricMan=" << (void*)metricMan.get();
447  if (metricMan)
448  {
449  metricMan->sendMetric("bytesRead", bytesRead, "B", 5, MetricMode::Accumulate, "", true);
450  metricMan->sendMetric("queue%Used", static_cast<unsigned long int>(qsize * 100 / incoming_events->size()), "%", 5, MetricMode::LastPoint, "", true);
451  }
452 
453  return true;
454  }
455 
456  std::map<Fragment::type_t, std::string> fragment_type_map_;
457  unsigned readNext_calls_;
458  };
459  } // detail
460 } // artdaq
461 
462 
463 #endif /* artdaq_ArtModules_detail_SharedMemoryReader_hh */
The SharedMemoryReader is a class which implements the methods needed by art::Source.
unsigned readNext_calls_
The number of times readNext has been called.
SharedMemoryReader(fhicl::ParameterSet const &ps, art::ProductRegistryHelper &help, art::SourceHelper const &pm)
SharedMemoryReader Constructor.
SharedMemoryReader(fhicl::ParameterSet const &ps, art::ProductRegistryHelper &help, art::SourceHelper const &pm, art::MasterProductRegistry &)
SharedMemoryReader Constructor.
art::SourceHelper const & pmaker
An art::SourceHelper instance.
SharedMemoryReader & operator=(SharedMemoryReader const &)=delete
Copy Assignment operator is deleted.
double waiting_time
The amount of time to wait for an event from the queue.
bool resume_after_timeout
Whether to resume if the dequeue action times out.
std::string unidentified_instance_name
The name to use for unknown Fragment types.
bool outputFileCloseNeeded
If an explicit output file close message is needed.
std::unique_ptr< SharedMemoryEventReceiver > incoming_events
The events from the EventStore.
virtual ~SharedMemoryReader()
SharedMemoryReader destructor.
size_t bytesRead
running total of number of bytes received
void closeCurrentFile()
Emulate closing a file. No-Op.
std::string pretend_module_name
The module name to store data under.
bool readNext(art::RunPrincipal *const &inR, art::SubRunPrincipal *const &inSR, art::RunPrincipal *&outR, art::SubRunPrincipal *&outSR, art::EventPrincipal *&outE)
Dequeue a RawEvent and declare its Fragment contents to art, creating Run, SubRun, and EventPrincipal objects as necessary.
std::map< Fragment::type_t, std::string > fragment_type_map_
The Fragment type names that this SharedMemoryReader knows about.
bool hasMoreData() const
Whether more data is expected from the SharedMemoryReader.
void readFile(std::string const &, art::FileBlock *&fb)
Emulate opening a file.
SharedMemoryReader(SharedMemoryReader const &)=delete
Copy Constructor is deleted.
bool shutdownMsgReceived
Whether a shutdown message has been received.