artdaq  v3_02_01
SharedMemoryReader.hh
1 #ifndef artdaq_ArtModules_detail_SharedMemoryReader_hh
2 #define artdaq_ArtModules_detail_SharedMemoryReader_hh
3 
4 #include "artdaq/DAQdata/Globals.hh"
5 #include "art/Framework/Core/Frameworkfwd.h"
6 
7 #include "art/Framework/Core/FileBlock.h"
8 #include "art/Framework/Core/ProductRegistryHelper.h"
9 #include "art/Framework/IO/Sources/SourceHelper.h"
10 #include "art/Framework/Principal/EventPrincipal.h"
11 #include "art/Framework/Principal/RunPrincipal.h"
12 #include "art/Framework/Principal/SubRunPrincipal.h"
13 #include "artdaq-core/Core/SharedMemoryManager.hh"
14 #include "artdaq-core/Utilities/TimeUtils.hh"
15 #include "fhiclcpp/ParameterSet.h"
16 #include "artdaq-core/Data/Fragment.hh"
17 #include "artdaq-core/Data/ContainerFragment.hh"
18 #include "artdaq-core/Core/SharedMemoryEventReceiver.hh"
19 #include "art/Framework/IO/Sources/put_product_in_principal.h"
20 #include "canvas/Persistency/Provenance/FileFormatVersion.h"
21 #include <sys/time.h>
22 
23 
24 #include <string>
25 #include <map>
26 #include "artdaq-core/Data/RawEvent.hh"
27 
28 namespace artdaq
29 {
30  namespace detail
31  {
35  template<std::map<artdaq::Fragment::type_t, std::string> getDefaultTypes() = artdaq::Fragment::MakeSystemTypeMap >
37  {
41  SharedMemoryReader(SharedMemoryReader const&) = delete;
42 
48 
49  art::SourceHelper const& pmaker;
50  std::unique_ptr<SharedMemoryEventReceiver> incoming_events;
51  double waiting_time;
53  std::string pretend_module_name;
57  size_t bytesRead;
58  //std::unique_ptr<SharedMemoryManager> data_shm; ///< SharedMemoryManager containing data
59  //std::unique_ptr<SharedMemoryManager> broadcast_shm; ///< SharedMemoryManager containing broadcasts (control Fragments)
60 
75  SharedMemoryReader(fhicl::ParameterSet const& ps,
76  art::ProductRegistryHelper& help,
77  art::SourceHelper const& pm)
78  : pmaker(pm)
79  , waiting_time(ps.get<double>("waiting_time", 86400.0))
80  , resume_after_timeout(ps.get<bool>("resume_after_timeout", true))
81  , pretend_module_name(ps.get<std::string>("raw_data_label", "daq"))
82  , unidentified_instance_name("unidentified")
83  , shutdownMsgReceived(false)
84  , outputFileCloseNeeded(false)
85  , bytesRead(0)
86  , fragment_type_map_(getDefaultTypes())
87  , readNext_calls_(0)
88  {
89  // For testing
90  //if (ps.has_key("buffer_count") && (ps.has_key("max_event_size_bytes") || (ps.has_key("expected_fragments_per_event") && ps.has_key("max_fragment_size_bytes"))))
91  //{
92  // data_shm.reset(new SharedMemoryManager(ps.get<uint32_t>("shared_memory_key", 0xBEE70000 + getppid()), ps.get<int>("buffer_count"), ps.has_key("max_event_size_bytes") ? ps.get<size_t>("max_event_size_bytes") : ps.get<size_t>("expected_fragments_per_event") * ps.get<size_t>("max_fragment_size_bytes")));
93  // broadcast_shm.reset(new SharedMemoryManager(ps.get<uint32_t>("broadcast_shared_memory_key", 0xCEE70000 + getppid()), ps.get<int>("broadcast_buffer_count", 5), ps.get<size_t>("broadcast_buffer_size", 0x100000)));
94  //}
95  incoming_events.reset(new SharedMemoryEventReceiver(ps.get<uint32_t>("shared_memory_key", 0xBEE70000 + getppid()), ps.get<uint32_t>("broadcast_shared_memory_key", 0xCEE70000 + getppid())));
96 
97  help.reconstitutes<Fragments, art::InEvent>(pretend_module_name, unidentified_instance_name);
98  for (auto it = fragment_type_map_.begin(); it != fragment_type_map_.end(); ++it)
99  {
100  help.reconstitutes<Fragments, art::InEvent>(pretend_module_name, it->second);
101  help.reconstitutes<Fragments, art::InEvent>(pretend_module_name, "Container" + it->second);
102  }
103  auto extraTypes = ps.get<std::vector<std::pair<Fragment::type_t, std::string>>>("fragment_type_map", std::vector<std::pair<Fragment::type_t, std::string>>());
104  for (auto it = extraTypes.begin(); it != extraTypes.end(); ++it)
105  {
106  fragment_type_map_[it->first] = it->second;
107  help.reconstitutes<Fragments, art::InEvent>(pretend_module_name, it->second);
108  help.reconstitutes<Fragments, art::InEvent>(pretend_module_name, "Container" + it->second);
109  }
110  TLOG_INFO("SharedMemoryReader") << "SharedMemoryReader initialized with ParameterSet: " << ps.to_string() ;
111  //for(auto& type : fragment_type_map_)
112  //{
113  // TLOG_INFO("SharedMemoryReader") << "Fragment Type " << type.second << " has typeid " << type.first ;
114  //}
115  }
116 
125  SharedMemoryReader(fhicl::ParameterSet const& ps,
126  art::ProductRegistryHelper& help,
127  art::SourceHelper const& pm,
128  art::MasterProductRegistry&) : SharedMemoryReader(ps, help, pm)
129  {}
130 
134  virtual ~SharedMemoryReader() = default;
135 
140 
145  void readFile(std::string const&, art::FileBlock*& fb)
146  {
147  TLOG_ARB(5, "SharedMemoryReader") <<"readFile enter/start";
148  fb = new art::FileBlock(art::FileFormatVersion(1, "RawEvent2011"), "nothing");
149  }
150 
155  bool hasMoreData() const { return (!shutdownMsgReceived); }
156 
167  bool readNext(art::RunPrincipal* const & inR,
168  art::SubRunPrincipal* const & inSR,
169  art::RunPrincipal*& outR,
170  art::SubRunPrincipal*& outSR,
171  art::EventPrincipal*& outE)
172  {
173  TLOG_DEBUG("SharedMemoryReader") << "readNext BEGIN" ;
174  /*if (outputFileCloseNeeded) {
175  outputFileCloseNeeded = false;
176  return false;
177  }*/
178  // Establish default 'results'
179  outR = 0;
180  outSR = 0;
181  outE = 0;
182  // Try to get an event from the queue. We'll continuously loop, either until:
183  // 1) we have read a RawEvent off the queue, or
184  // 2) we have timed out, AND we are told the when we timeout we
185  // should stop.
186  // In any case, if we time out, we emit an informational message.
187 
188  if (shutdownMsgReceived) return false;
189 
190  start:
191  bool keep_looping = true;
192  bool got_event = false;
193  auto sleepTimeUsec = waiting_time * 1000; // waiting_time * 1000000 us/s / 1000 reps = us/rep
194  if (sleepTimeUsec > 100000) sleepTimeUsec = 100000; // Don't wait longer than 1/10th of a second
195  while (keep_looping)
196  {
197  keep_looping = false;
198  auto start_time = std::chrono::steady_clock::now();
199  while (!got_event && TimeUtils::GetElapsedTimeMicroseconds(start_time) < 1000)
200  {
201  // BURN CPU for 1 ms!
202  got_event = incoming_events->ReadyForRead();
203  }
204  while (!got_event && TimeUtils::GetElapsedTime(start_time) < waiting_time)
205  {
206  got_event = incoming_events->ReadyForRead();
207  if (!got_event)
208  {
209  usleep(sleepTimeUsec);
210  //TLOG_INFO("SharedMemoryReader") << "Waited " << TimeUtils::GetElapsedTime(start_time) << " of " << waiting_time ;
211  }
212  }
213  if (!got_event)
214  {
215  TLOG_INFO("SharedMemoryReader")
216  << "InputFailure: Reading timed out in SharedMemoryReader::readNext()" ;
217  keep_looping = resume_after_timeout;
218  }
219  }
220 
221  if (!got_event)
222  {
223  TLOG_INFO("SharedMemoryReader") << "Did not receive an event from Shared Memory, returning false" ;
224  shutdownMsgReceived = true;
225  return false;
226  }
227  TLOG_DEBUG("SharedMemoryReader") << "Got Event!" ;
228 
229  auto errflag = false;
230  auto evtHeader = incoming_events->ReadHeader(errflag);
231  if (errflag) goto start; // Buffer was changed out from under reader!
232  auto fragmentTypes = incoming_events->GetFragmentTypes(errflag);
233  if (errflag) goto start; // Buffer was changed out from under reader!
234  if (fragmentTypes.size() == 0)
235  {
236  TLOG_ERROR("SharedMemoryReader") << "Event has no Fragments! Aborting!" ;
237  incoming_events->ReleaseBuffer();
238  return false;
239  }
240  auto firstFragmentType = *fragmentTypes.begin();
241  TLOG_DEBUG("SharedMemoryReader") << "First Fragment type is " << (int)firstFragmentType << " (" << fragment_type_map_[firstFragmentType] << ")" ;
242 
243  // We return false, indicating we're done reading, if:
244  // 1) we did not obtain an event, because we timed out and were
245  // configured NOT to keep trying after a timeout, or
246  // 2) the event we read was the end-of-data marker: a null
247  // pointer
248  if (firstFragmentType == Fragment::EndOfDataFragmentType)
249  {
250  TLOG_DEBUG("SharedMemoryReader") << "Received shutdown message, returning false" ;
251  shutdownMsgReceived = true;
252  incoming_events->ReleaseBuffer();
253  return false;
254  }
255 
256  size_t qsize = incoming_events->ReadReadyCount(); // save the qsize at this point
257 
258  // Check the number of fragments in the RawEvent. If we have a single
259  // fragment and that fragment is marked as EndRun or EndSubrun we'll create
260  // the special principals for that.
261  art::Timestamp currentTime = time(0);
262 
263  // make new run if inR is 0 or if the run has changed
264  if (inR == 0 || inR->run() != evtHeader->run_id)
265  {
266  outR = pmaker.makeRunPrincipal(evtHeader->run_id,
267  currentTime);
268  }
269 
270  if (firstFragmentType == Fragment::EndOfRunFragmentType)
271  {
272  art::EventID const evid(art::EventID::flushEvent());
273  outR = pmaker.makeRunPrincipal(evid.runID(), currentTime);
274  outSR = pmaker.makeSubRunPrincipal(evid.subRunID(), currentTime);
275  outE = pmaker.makeEventPrincipal(evid, currentTime);
276  incoming_events->ReleaseBuffer();
277  return true;
278  }
279  else if (firstFragmentType == Fragment::EndOfSubrunFragmentType)
280  {
281  // Check if inR == 0 or is a new run
282  if (inR == 0 || inR->run() != evtHeader->run_id)
283  {
284  outSR = pmaker.makeSubRunPrincipal(evtHeader->run_id,
285  evtHeader->subrun_id,
286  currentTime);
287  art::EventID const evid(art::EventID::flushEvent(outSR->id()));
288  outE = pmaker.makeEventPrincipal(evid, currentTime);
289  }
290  else
291  {
292  // If the previous subrun was neither 0 nor flush and was identical with the current
293  // subrun, then it must have been associated with a data event. In that case, we need
294  // to generate a flush event with a valid run but flush subrun and event number in order
295  // to end the subrun.
296  if (inSR != 0 && !inSR->id().isFlush() && inSR->subRun() == evtHeader->subrun_id)
297  {
298  art::EventID const evid(art::EventID::flushEvent(inR->id()));
299  outSR = pmaker.makeSubRunPrincipal(evid.subRunID(), currentTime);
300  outE = pmaker.makeEventPrincipal(evid, currentTime);
301  // If this is either a new or another empty subrun, then generate a flush event with
302  // valid run and subrun numbers but flush event number
303  //} else if(inSR==0 || inSR->id().isFlush()){
304  }
305  else
306  {
307  outSR = pmaker.makeSubRunPrincipal(evtHeader->run_id,
308  evtHeader->subrun_id,
309  currentTime);
310  art::EventID const evid(art::EventID::flushEvent(outSR->id()));
311  outE = pmaker.makeEventPrincipal(evid, currentTime);
312  // Possible error condition
313  //} else {
314  }
315  outR = 0;
316  }
317  //outputFileCloseNeeded = true;
318  incoming_events->ReleaseBuffer();
319  return true;
320  }
321 
322  // make new subrun if inSR is 0 or if the subrun has changed
323  art::SubRunID subrun_check(evtHeader->run_id, evtHeader->subrun_id);
324  if (inSR == 0 || subrun_check != inSR->id())
325  {
326  outSR = pmaker.makeSubRunPrincipal(evtHeader->run_id,
327  evtHeader->subrun_id,
328  currentTime);
329  }
330  outE = pmaker.makeEventPrincipal(evtHeader->run_id,
331  evtHeader->subrun_id,
332  evtHeader->event_id,
333  currentTime);
334 
335  // insert the Fragments of each type into the EventPrincipal
336  std::map<Fragment::type_t, std::string>::const_iterator iter_end =
337  fragment_type_map_.end();
338  for (auto& type_code : fragmentTypes)
339  {
340  std::map<Fragment::type_t, std::string>::const_iterator iter =
341  fragment_type_map_.find(type_code);
342  auto product = incoming_events->GetFragmentsByType(errflag, type_code);
343  if (errflag) goto start; // Buffer was changed out from under reader!
344  for (auto &frag : *product)
345  bytesRead += frag.sizeBytes();
346  if (iter != iter_end)
347  {
348  if (type_code == artdaq::Fragment::ContainerFragmentType)
349  {
350  std::unordered_map<std::string, std::unique_ptr<Fragments>> derived_fragments;
351  derived_fragments[iter->second] = std::make_unique<Fragments>();
352 
353  for (size_t ii = 0; ii < product->size(); ++ii)
354  {
355  ContainerFragment cf(product->at(ii));
356  auto contained_type = fragment_type_map_.find(cf.fragment_type());
357  if (contained_type != iter_end)
358  {
359  auto label = iter->second + contained_type->second;
360  if (!derived_fragments.count(label))
361  {
362  derived_fragments[label] = std::make_unique<Fragments>();
363  }
364  derived_fragments[label]->emplace_back(std::move(product->at(ii)));
365  }
366  else
367  {
368  derived_fragments[iter->second]->emplace_back(std::move(product->at(ii)));
369  }
370  }
371 
372  for (auto& type : derived_fragments)
373  {
374  put_product_in_principal(std::move(type.second),
375  *outE,
377  type.first);
378  }
379 
380  }
381  else
382  {
383  put_product_in_principal(std::move(product),
384  *outE,
386  iter->second);
387  }
388  }
389  else
390  {
391  put_product_in_principal(std::move(product),
392  *outE,
395  TLOG_WARNING("SharedMemoryReader")
396  << "UnknownFragmentType: The product instance name mapping for fragment type \""
397  << ((int)type_code) << "\" is not known. Fragments of this "
398  << "type will be stored in the event with an instance name of \""
399  << unidentified_instance_name << "\"." ;
400  }
401  }
402  incoming_events->ReleaseBuffer();
403  TLOG_ARB(10, "SharedMemoryReader") << "readNext: bytesRead=" << bytesRead << " qsize=" << qsize << " cap=" << incoming_events->size() << " metricMan=" << (void*)metricMan ;
404  if (metricMan)
405  {
406  metricMan->sendMetric("bytesRead", bytesRead, "B", 5, MetricMode::Accumulate, "", true);
407  metricMan->sendMetric("queue%Used", static_cast<unsigned long int>(qsize * 100 / incoming_events->size()), "%", 5, MetricMode::LastPoint, "", true);
408  }
409 
410  return true;
411  }
412 
413  std::map<Fragment::type_t, std::string> fragment_type_map_;
414  unsigned readNext_calls_;
415  };
416  } // detail
417 } // artdaq
418 
419 
420 #endif /* artdaq_ArtModules_detail_SharedMemoryReader_hh */
The SharedMemoryReader is a class which implements the methods needed by art::Source.
unsigned readNext_calls_
The number of times readNext has been called.
SharedMemoryReader(fhicl::ParameterSet const &ps, art::ProductRegistryHelper &help, art::SourceHelper const &pm)
SharedMemoryReader Constructor.
SharedMemoryReader(fhicl::ParameterSet const &ps, art::ProductRegistryHelper &help, art::SourceHelper const &pm, art::MasterProductRegistry &)
SharedMemoryReader Constructor.
virtual ~SharedMemoryReader()=default
SharedMemoryReader destructor.
art::SourceHelper const & pmaker
An art::SourceHelper instance.
SharedMemoryReader & operator=(SharedMemoryReader const &)=delete
Copy Assignment operator is deleted.
double waiting_time
The amount of time to wait for an event from the queue.
bool resume_after_timeout
Whether to resume if the dequeue action times out.
std::string unidentified_instance_name
The name to use for unknown Fragment types.
bool outputFileCloseNeeded
If an explicit output file close message is needed.
std::unique_ptr< SharedMemoryEventReceiver > incoming_events
The events from the EventStore.
size_t bytesRead
running total of number of bytes received
void closeCurrentFile()
Emulate closing a file. No-Op.
std::string pretend_module_name
The module name to store data under.
bool readNext(art::RunPrincipal *const &inR, art::SubRunPrincipal *const &inSR, art::RunPrincipal *&outR, art::SubRunPrincipal *&outSR, art::EventPrincipal *&outE)
Dequeue a RawEvent and declare its Fragment contents to art, creating Run, SubRun, and EventPrincipal objects as necessary.
std::map< Fragment::type_t, std::string > fragment_type_map_
The Fragment type names that this SharedMemoryReader knows about.
bool hasMoreData() const
Whether more data is expected from the SharedMemoryReader.
void readFile(std::string const &, art::FileBlock *&fb)
Emulate opening a file.
SharedMemoryReader(SharedMemoryReader const &)=delete
Copy Constructor is deleted.
bool shutdownMsgReceived
Whether a shutdown message has been received.