artdaq  v2_03_03
 All Classes Namespaces Files Functions Variables Typedefs Enumerations Enumerator Pages
EventStore.cc
1 #include "artdaq/DAQdata/Globals.hh" // Before trace.h gets included in ConcurrentQueue (from GlobalQueue)
2 #include "artdaq/DAQrate/EventStore.hh"
3 #include <utility>
4 #include <cstring>
5 #include <dlfcn.h>
6 #include <iomanip>
7 #include <fstream>
8 #include <sstream>
9 #include <thread>
10 #include <chrono>
11 
12 #include "cetlib/exception.h"
13 #include "artdaq-core/Core/StatisticsCollection.hh"
14 #include "artdaq-core/Core/SimpleQueueReader.hh"
15 #include "artdaq/Application/Routing/RoutingPacket.hh"
17 
18 using namespace std;
19 
20 namespace artdaq
21 {
22  const std::string EventStore::EVENT_RATE_STAT_KEY("EventStoreEventRate");
23  const std::string EventStore::INCOMPLETE_EVENT_STAT_KEY("EventStoreIncompleteEvents");
24 
25  EventStore::EventStore(const fhicl::ParameterSet& pset, size_t num_fragments_per_event, run_id_t run,
26  size_t event_queue_depth, size_t max_incomplete_event_count)
27  : num_fragments_per_event_(num_fragments_per_event)
28  , max_queue_size_(pset.get<size_t>("event_queue_depth", event_queue_depth))
29  , max_incomplete_count_(pset.get<size_t>("max_incomplete_events", max_incomplete_event_count))
30  , run_id_(run)
31  , subrun_id_(0)
32  , events_()
33  , queue_(getGlobalQueue(max_queue_size_))
34  , reader_thread_launch_time_(std::chrono::steady_clock::now())
35  , send_requests_(pset.get<bool>("send_requests", false))
36  , active_requests_()
37  , request_port_(pset.get<int>("request_port", 3001))
38  , request_delay_(pset.get<size_t>("request_delay_ms", 10))
39  , multicast_out_addr_(pset.get<std::string>("output_address", "localhost"))
40  , request_mode_(detail::RequestMessageMode::Normal)
41  , seqIDModulus_(1)
42  , lastFlushedSeqID_(0)
43  , highestSeqIDSeen_(0)
44  , enq_timeout_(pset.get<double>("event_queue_wait_time", 5.0))
45  , enq_check_count_(pset.get<size_t>("event_queue_check_count", 5000))
46  , printSummaryStats_(pset.get<bool>("print_event_store_stats", false))
47  , incomplete_event_report_interval_ms_(pset.get<int>("incomplete_event_report_interval_ms", -1))
48  , last_incomplete_event_report_time_(std::chrono::steady_clock::now())
49  , token_socket_(-1)
50  , art_thread_wait_ms_(pset.get<int>("art_thread_wait_ms", 4000))
51  {
52  TLOG_DEBUG("EventStore") << "EventStore CONSTRUCTOR" << TLOG_ENDL;
53  initStatistics_();
54  setup_requests_(pset.get<std::string>("request_address", "227.128.12.26"));
55 
56  auto rmConfig = pset.get<fhicl::ParameterSet>("routing_token_config", fhicl::ParameterSet());
57  send_routing_tokens_ = rmConfig.get<bool>("use_routing_master", false);
58  token_port_ = rmConfig.get<int>("routing_token_port", 35555);
59  token_address_ = rmConfig.get<std::string>("routing_master_hostname", "localhost");
60  setup_tokens_();
61  TRACE(12, "artdaq::EventStore::EventStore ctor - reader_thread_ initialized");
62  }
63 
64  EventStore::EventStore(const fhicl::ParameterSet& pset,
65  size_t num_fragments_per_event,
66  run_id_t run,
67  int argc,
68  char* argv[],
69  ART_CMDLINE_FCN* reader)
70  : EventStore(pset, num_fragments_per_event, run, 50, 50)
71  {
72  reader_thread_ = (std::async(std::launch::async, reader, argc, argv));
73  }
74 
75  EventStore::EventStore(const fhicl::ParameterSet& pset,
76  size_t num_fragments_per_event,
77  run_id_t run,
78  const std::string& configString,
79  ART_CFGSTRING_FCN* reader)
80  : EventStore(pset, num_fragments_per_event, run, 20, 20)
81  {
82  reader_thread_ = (std::async(std::launch::async, reader, configString));
83  }
84 
86  {
87  TLOG_DEBUG("EventStore") << "Shutting down EventStore" << TLOG_ENDL;
88  if (printSummaryStats_)
89  {
90  reportStatistics_();
91  }
92  shutdown(request_socket_, 2);
93  close(request_socket_);
94  shutdown(token_socket_, 2);
95  close(token_socket_);
96  }
97 
98  void EventStore::insert(FragmentPtr pfrag,
99  bool printWarningWhenFragmentIsDropped)
100  {
101  // We should never get a null pointer, nor should we get a
102  // Fragment without a good fragment ID.
103  assert(pfrag != nullptr);
104  assert(pfrag->fragmentID() != Fragment::InvalidFragmentID);
105 
106  // find the event being built and put the fragment into it,
107  // start new event if not already present
108  // if the event is complete, delete it and report timing
109 
110  // The sequenceID is expected to be correct in the incoming fragment.
111  // The EventStore will divide it by the seqIDModulus to support the use case
112  // of the aggregator which needs to bunch groups of serialized events with
113  // continuous sequence IDs together.
114  if (pfrag->sequenceID() > highestSeqIDSeen_)
115  {
116  highestSeqIDSeen_ = pfrag->sequenceID();
117 
118  // Get the timestamp of this fragment, in experiment-defined clocks
119  Fragment::timestamp_t timestamp = pfrag->timestamp();
120 
121  // Send a request to the board readers!
122  if (send_requests_)
123  {
124  std::lock_guard<std::mutex> lk(request_mutex_);
125  active_requests_[highestSeqIDSeen_] = timestamp;
126  send_request_();
127  }
128  }
129 
130  // When we're in the "EndOfRun" condition, send requests for EVERY fragment inserted!
131  // This helps to make sure that all BoardReaders get the EndOfRun request and send all their data.
132  if (send_requests_ && request_mode_ == detail::RequestMessageMode::EndOfRun)
133  {
134  std::lock_guard<std::mutex> lk(request_mutex_);
135  send_request_();
136  }
137  Fragment::sequence_id_t sequence_id = ((pfrag->sequenceID() - (1 + lastFlushedSeqID_)) / seqIDModulus_) + 1;
138  TRACE(13, "EventStore::insert seq=%lu fragID=%d id=%d lastFlushed=%lu seqIDMod=%d seq=%lu"
139  , pfrag->sequenceID(), pfrag->fragmentID(), my_rank, lastFlushedSeqID_, seqIDModulus_, sequence_id);
140 
141 
142  // Find if the right event id is already known to events_ and, if so, where
143  // it is.
144  EventMap::iterator loc = events_.lower_bound(sequence_id);
145 
146  if (loc == events_.end() || events_.key_comp()(sequence_id, loc->first))
147  {
148  // We don't have an event with this id; create one and insert it at loc,
149  // and ajust loc to point to the newly inserted event.
150  RawEvent_ptr newevent(new RawEvent(run_id_, subrun_id_, pfrag->sequenceID()));
151  loc =
152  events_.insert(loc, EventMap::value_type(sequence_id, newevent));
153  }
154 
155  // Now insert the fragment into the event we have located.
156  loc->second->insertFragment(std::move(pfrag));
157  if (loc->second->numFragments() == num_fragments_per_event_)
158  {
159  // This RawEvent is complete; capture it, remove it from the
160  // map, report on statistics, and put the shared pointer onto
161  // the event queue.
162  RawEvent_ptr complete_event(loc->second);
163  complete_event->markComplete();
164 
165  events_.erase(loc);
166 
167  if (send_requests_)
168  {
169  std::lock_guard<std::mutex> lk(request_mutex_);
170  active_requests_.erase(sequence_id);
171  }
172  // 13-Dec-2012, KAB - this monitoring needs to come before
173  // the enqueueing of the event lest it be empty by the
174  // time that we ask for the word count.
175  MonitoredQuantityPtr mqPtr = StatisticsCollection::getInstance().
176  getMonitoredQuantity(EVENT_RATE_STAT_KEY);
177  if (mqPtr.get() != 0)
178  {
179  mqPtr->addSample(complete_event->wordCount());
180  }
181  TRACE(14, "EventStore::insert seq=%lu enqTimedWait start", sequence_id);
182  bool enqSuccess = queue_.enqTimedWait(complete_event, enq_timeout_);
183  TRACE(enqSuccess ? 14 : 0, "EventStore::insert seq=%lu enqTimedWait complete", sequence_id);
184  if (metricMan)
185  {
186  metricMan->sendMetric("Current Event Number", sequence_id, "id", 2, MetricMode::LastPoint);
187  }
188  if (!enqSuccess)
189  {
190  //TRACE_CNTL( "modeM", 0 );
191  if (printWarningWhenFragmentIsDropped)
192  {
193  TLOG_WARNING("EventStore") << "Enqueueing event " << sequence_id
194  << " FAILED, queue size = "
195  << queue_.size() <<
196  "; apparently no events were removed from this process's queue during the " << std::to_string(enq_timeout_.count())
197  << "-second timeout period" << TLOG_ENDL;
198  }
199  else
200  {
201  TLOG_DEBUG("EventStore") << "Enqueueing event " << sequence_id
202  << " FAILED, queue size = "
203  << queue_.size() <<
204  "; apparently no events were removed from this process's queue during the " << std::to_string(enq_timeout_.count())
205  << "-second timeout period" << TLOG_ENDL;
206  }
207  }
208  else
209  {
210  send_routing_token_(1);
211  }
212  }
213  MonitoredQuantityPtr mqPtr = StatisticsCollection::getInstance().
214  getMonitoredQuantity(INCOMPLETE_EVENT_STAT_KEY);
215  if (mqPtr.get() != 0)
216  {
217  mqPtr->addSample(events_.size());
218  }
219  }
220 
221  EventStore::EventStoreInsertResult EventStore::insert(FragmentPtr pfrag, FragmentPtr& rejectedFragment)
222  {
223  // Test whether this fragment can be safely accepted. If we accept
224  // it, and it completes an event, then we want to be sure that it
225  // can be pushed onto the event queue. If not, we return it and
226  // let the caller know that we didn't accept it.
227  TRACE(12, "EventStore: Testing if queue is full");
228  if (queue_.full())
229  {
230  size_t sleepTime = 1000000 * (enq_timeout_.count() / enq_check_count_);
231  TRACE(12, "EventStore: sleepTime is %lu.", sleepTime);
232  size_t loopCount = 0;
233  while (loopCount < enq_check_count_ && queue_.full())
234  {
235  ++loopCount;
236  usleep(sleepTime);
237  }
238  if (queue_.full())
239  {
240  rejectedFragment = std::move(pfrag);
242  }
243  }
244  TRACE(12, "EventStore: Testing if there's room in the EventStore");
245  auto incomplete_full = events_.size() >= max_incomplete_count_;
246  if (incomplete_full)
247  {
248  EventMap::iterator loc = events_.lower_bound(pfrag->sequenceID());
249 
250  if (loc == events_.end() || events_.key_comp()(pfrag->sequenceID(), loc->first))
251  {
252  rejectedFragment = std::move(pfrag);
254  }
255  }
256 
257  TRACE(12, "EventStore: Performing insert");
258  insert(std::move(pfrag));
260  }
261 
262  bool
263  EventStore::endOfData(int& readerReturnValue)
264  {
265  TLOG_DEBUG("EventStore") << "EventStore::endOfData" << TLOG_ENDL;
266  RawEvent_ptr end_of_data(nullptr);
267  TRACE(4, "EventStore::endOfData: Enqueuing end_of_data event");
268  bool enqSuccess = queue_.enqTimedWait(end_of_data, enq_timeout_);
269  if (!enqSuccess)
270  {
271  return false;
272  }
273  TRACE(4, "EventStore::endOfData: Getting return code from art thread");
274  readerReturnValue = reader_thread_.get();
275  return true;
276  }
277 
278  void EventStore::setSeqIDModulus(unsigned int seqIDModulus)
279  {
280  seqIDModulus_ = seqIDModulus;
281  }
282 
284  {
285  bool enqSuccess;
286  size_t initialStoreSize = events_.size();
287  TLOG_DEBUG("EventStore") << "Flushing " << initialStoreSize
288  << " stale events from the EventStore." << TLOG_ENDL;
289  EventMap::iterator loc;
290  std::vector<sequence_id_t> flushList;
291  for (loc = events_.begin(); loc != events_.end(); ++loc)
292  {
293  RawEvent_ptr complete_event(loc->second);
294  MonitoredQuantityPtr mqPtr = StatisticsCollection::getInstance().
295  getMonitoredQuantity(EVENT_RATE_STAT_KEY);
296  if (mqPtr.get() != 0)
297  {
298  mqPtr->addSample(complete_event->wordCount());
299  }
300  enqSuccess = queue_.enqTimedWait(complete_event, enq_timeout_);
301  if (!enqSuccess)
302  {
303  break;
304  }
305  else
306  {
307  flushList.push_back(loc->first);
308  }
309  }
310  for (size_t idx = 0; idx < flushList.size(); ++idx)
311  {
312  events_.erase(flushList[idx]);
313  }
314  TLOG_DEBUG("EventStore") << "Done flushing " << flushList.size()
315  << " stale events from the EventStore." << TLOG_ENDL;
316 
317  lastFlushedSeqID_ = highestSeqIDSeen_;
318  return (flushList.size() >= initialStoreSize);
319  }
320 
322  {
323  if (!queue_.queueReaderIsReady())
324  {
325  TLOG_WARNING("EventStore") << "Run start requested, but the art thread is not yet ready, waiting up to " << art_thread_wait_ms_ << " msec..." << TLOG_ENDL;
326  while (!queue_.queueReaderIsReady() && std::chrono::duration_cast<std::chrono::milliseconds>(std::chrono::steady_clock::now() - reader_thread_launch_time_).count() < art_thread_wait_ms_)
327  {
328  usleep(1000); // wait 1 ms
329  }
330  if (queue_.queueReaderIsReady())
331  {
332  auto dur = std::chrono::duration_cast<std::chrono::milliseconds>(queue_.getReadyTime() - reader_thread_launch_time_).count();
333  TLOG_INFO("EventStore") << "art initialization took (roughly) " << std::setw(4) << std::to_string(dur) << " ms." << TLOG_ENDL;
334  }
335  else
336  {
337  auto dur = std::chrono::duration_cast<std::chrono::milliseconds>(std::chrono::steady_clock::now() - reader_thread_launch_time_).count();
338  TLOG_ERROR("EventStore") << "art thread still not ready after " << dur << " ms. Continuing to start..." << TLOG_ENDL;
339  }
340  }
341  run_id_ = runID;
342  subrun_id_ = 1;
343  lastFlushedSeqID_ = 0;
344  highestSeqIDSeen_ = 0;
345  send_routing_token_(max_queue_size_);
346  TLOG_DEBUG("EventStore") << "Starting run " << run_id_
347  << ", max queue size = "
348  << max_queue_size_
349  << ", queue capacity = "
350  << queue_.capacity()
351  << ", queue size = "
352  << queue_.size() << TLOG_ENDL;
353  if (metricMan)
354  {
355  double runSubrun = run_id_ + ((double)subrun_id_ / 10000);
356  metricMan->sendMetric("Run Number", runSubrun, "Run:Subrun", 1, MetricMode::LastPoint);
357  }
358  }
359 
361  {
362  ++subrun_id_;
363  if (metricMan)
364  {
365  double runSubrun = run_id_ + ((double)subrun_id_ / 10000);
366  metricMan->sendMetric("Run Number", runSubrun, "Run:Subrun", 1, MetricMode::LastPoint);
367  }
368  }
369 
371  {
372  RawEvent_ptr endOfRunEvent(new RawEvent(run_id_, subrun_id_, 0));
373  std::unique_ptr<artdaq::Fragment>
374  endOfRunFrag(new
375  Fragment(static_cast<size_t>
376  (ceil(sizeof(my_rank) /
377  static_cast<double>(sizeof(Fragment::value_type))))));
378 
379  endOfRunFrag->setSystemType(Fragment::EndOfRunFragmentType);
380  *endOfRunFrag->dataBegin() = my_rank;
381  endOfRunEvent->insertFragment(std::move(endOfRunFrag));
382 
383  return queue_.enqTimedWait(endOfRunEvent, enq_timeout_);
384  }
385 
387  {
388  RawEvent_ptr endOfSubrunEvent(new RawEvent(run_id_, subrun_id_, 0));
389  std::unique_ptr<artdaq::Fragment>
390  endOfSubrunFrag(new
391  Fragment(static_cast<size_t>
392  (ceil(sizeof(my_rank) /
393  static_cast<double>(sizeof(Fragment::value_type))))));
394 
395  endOfSubrunFrag->setSystemType(Fragment::EndOfSubrunFragmentType);
396  *endOfSubrunFrag->dataBegin() = my_rank;
397  endOfSubrunEvent->insertFragment(std::move(endOfSubrunFrag));
398 
399  return queue_.enqTimedWait(endOfSubrunEvent, enq_timeout_);
400  }
401 
402  void EventStore::setRequestMode(detail::RequestMessageMode mode)
403  {
404  request_mode_ = mode;
405  if (send_requests_ && request_mode_ == detail::RequestMessageMode::EndOfRun)
406  {
407  send_request_();
408  }
409  }
410 
411  void
412  EventStore::initStatistics_()
413  {
414  MonitoredQuantityPtr mqPtr = StatisticsCollection::getInstance().
415  getMonitoredQuantity(EVENT_RATE_STAT_KEY);
416  if (mqPtr.get() == 0)
417  {
418  mqPtr.reset(new MonitoredQuantity(3.0, 300.0));
419  StatisticsCollection::getInstance().
420  addMonitoredQuantity(EVENT_RATE_STAT_KEY, mqPtr);
421  }
422  mqPtr->reset();
423 
424  mqPtr = StatisticsCollection::getInstance().
425  getMonitoredQuantity(INCOMPLETE_EVENT_STAT_KEY);
426  if (mqPtr.get() == 0)
427  {
428  mqPtr.reset(new MonitoredQuantity(3.0, 300.0));
429  StatisticsCollection::getInstance().
430  addMonitoredQuantity(INCOMPLETE_EVENT_STAT_KEY, mqPtr);
431  }
432  mqPtr->reset();
433  }
434 
435  void
436  EventStore::reportStatistics_()
437  {
438  MonitoredQuantityPtr mqPtr = StatisticsCollection::getInstance().
439  getMonitoredQuantity(EVENT_RATE_STAT_KEY);
440  if (mqPtr.get() != 0)
441  {
442  ostringstream oss;
443  oss << EVENT_RATE_STAT_KEY << "_" << setfill('0') << setw(4) << run_id_
444  << "_" << setfill('0') << setw(4) << my_rank << ".txt";
445  std::string filename = oss.str();
446  ofstream outStream(filename.c_str());
447  mqPtr->waitUntilAccumulatorsHaveBeenFlushed(3.0);
448  artdaq::MonitoredQuantityStats stats;
449  mqPtr->getStats(stats);
450  outStream << "EventStore rank " << my_rank << ": events processed = "
451  << stats.fullSampleCount << " at " << stats.fullSampleRate
452  << " events/sec, data rate = "
453  << (stats.fullValueRate * sizeof(RawDataType)
454  / 1024.0 / 1024.0) << " MB/sec, duration = "
455  << stats.fullDuration << " sec" << std::endl
456  << " minimum event size = "
457  << (stats.fullValueMin * sizeof(RawDataType)
458  / 1024.0 / 1024.0)
459  << " MB, maximum event size = "
460  << (stats.fullValueMax * sizeof(RawDataType)
461  / 1024.0 / 1024.0)
462  << " MB" << std::endl;
463  bool foundTheStart = false;
464  for (int idx = 0; idx < (int)stats.recentBinnedDurations.size(); ++idx)
465  {
466  if (stats.recentBinnedDurations[idx] > 0.0)
467  {
468  foundTheStart = true;
469  }
470  if (foundTheStart)
471  {
472  outStream << " " << std::fixed << std::setprecision(3)
473  << stats.recentBinnedEndTimes[idx]
474  << ": " << stats.recentBinnedSampleCounts[idx]
475  << " events at "
476  << (stats.recentBinnedSampleCounts[idx] /
477  stats.recentBinnedDurations[idx])
478  << " events/sec, data rate = "
479  << (stats.recentBinnedValueSums[idx] *
480  sizeof(RawDataType) / 1024.0 / 1024.0 /
481  stats.recentBinnedDurations[idx])
482  << " MB/sec, bin size = "
483  << stats.recentBinnedDurations[idx]
484  << " sec" << std::endl;
485  }
486  }
487  outStream.close();
488  }
489 
490  mqPtr = StatisticsCollection::getInstance().
491  getMonitoredQuantity(INCOMPLETE_EVENT_STAT_KEY);
492  if (mqPtr.get() != 0)
493  {
494  ostringstream oss;
495  oss << INCOMPLETE_EVENT_STAT_KEY << "_" << setfill('0')
496  << setw(4) << run_id_
497  << "_" << setfill('0') << setw(4) << my_rank << ".txt";
498  std::string filename = oss.str();
499  ofstream outStream(filename.c_str());
500  mqPtr->waitUntilAccumulatorsHaveBeenFlushed(3.0);
501  artdaq::MonitoredQuantityStats stats;
502  mqPtr->getStats(stats);
503  outStream << "EventStore rank " << my_rank << ": fragments processed = "
504  << stats.fullSampleCount << " at " << stats.fullSampleRate
505  << " fragments/sec, average incomplete event count = "
506  << stats.fullValueAverage << " duration = "
507  << stats.fullDuration << " sec" << std::endl
508  << " minimum incomplete event count = "
509  << stats.fullValueMin << ", maximum incomplete event count = "
510  << stats.fullValueMax << std::endl;
511  bool foundTheStart = false;
512  for (int idx = 0; idx < (int)stats.recentBinnedDurations.size(); ++idx)
513  {
514  if (stats.recentBinnedDurations[idx] > 0.0)
515  {
516  foundTheStart = true;
517  }
518  if (foundTheStart && stats.recentBinnedSampleCounts[idx] > 0.0)
519  {
520  outStream << " " << std::fixed << std::setprecision(3)
521  << stats.recentBinnedEndTimes[idx]
522  << ": " << stats.recentBinnedSampleCounts[idx]
523  << " fragments at "
524  << (stats.recentBinnedSampleCounts[idx] /
525  stats.recentBinnedDurations[idx])
526  << " fragments/sec, average incomplete event count = "
527  << (stats.recentBinnedValueSums[idx] /
528  stats.recentBinnedSampleCounts[idx])
529  << ", bin size = "
530  << stats.recentBinnedDurations[idx]
531  << " sec" << std::endl;
532  }
533  }
534  outStream << "Incomplete count now = " << events_.size() << std::endl;
535  outStream.close();
536  }
537  }
538 
539  void
540  EventStore::setup_requests_(std::string request_address)
541  {
542  if (send_requests_)
543  {
544  request_socket_ = socket(AF_INET, SOCK_DGRAM, IPPROTO_UDP);
545  if (!request_socket_)
546  {
547  TLOG_ERROR("EventStore") << "I failed to create the socket for sending Data Requests!" << TLOG_ENDL;
548  exit(1);
549  }
550  int sts = ResolveHost(request_address.c_str(), request_port_, request_addr_);
551  if (sts == -1)
552  {
553  TLOG_ERROR("EventStore") << "Unable to resolve Data Request address" << TLOG_ENDL;
554  exit(1);
555  }
556 
557  if (multicast_out_addr_ != "localhost")
558  {
559  struct in_addr addr;
560  int sts = ResolveHost(multicast_out_addr_.c_str(), addr);
561  if (sts == -1)
562  {
563  TLOG_ERROR("EventStore") << "Unable to resolve multicast interface address" << TLOG_ENDL;
564  exit(1);
565  }
566 
567  int yes = 1;
568  if (setsockopt(request_socket_, SOL_SOCKET, SO_REUSEADDR, &yes, sizeof(yes)) < 0)
569  {
570  TLOG_ERROR("EventStore") << "Unable to enable port reuse on request socket" << TLOG_ENDL;
571  exit(1);
572  }
573  if (setsockopt(request_socket_, IPPROTO_IP, IP_MULTICAST_IF, &addr, sizeof(addr)) == -1)
574  {
575  TLOG_ERROR("EventStore") << "Cannot set outgoing interface." << TLOG_ENDL;
576  exit(1);
577  }
578  }
579  int yes = 1;
580  if (setsockopt(request_socket_, SOL_SOCKET, SO_BROADCAST, (void*)&yes, sizeof(int)) == -1)
581  {
582  TLOG_ERROR("EventStore") << "Cannot set request socket to broadcast." << TLOG_ENDL;
583  exit(1);
584  }
585  }
586  }
587 
588  void
589  EventStore::setup_tokens_()
590  {
591  if (send_routing_tokens_)
592  {
593  TLOG_DEBUG("EventStore") << "Creating Routing Token sending socket" << TLOG_ENDL;
594  token_socket_ = TCPConnect(token_address_.c_str(), token_port_);
595  if (!token_socket_)
596  {
597  TLOG_ERROR("EventStore") << "I failed to create the socket for sending Routing Tokens!" << TLOG_ENDL;
598  exit(1);
599  }
600  }
601  }
602 
603  void EventStore::do_send_request_()
604  {
605  std::this_thread::sleep_for(std::chrono::microseconds(request_delay_));
606 
607  detail::RequestMessage message;
608  {
609  std::lock_guard<std::mutex> lk(request_mutex_);
610  for (auto& req : active_requests_)
611  {
612  message.addRequest(req.first, req.second);
613  }
614  }
615  message.header()->mode = request_mode_;
616  char str[INET_ADDRSTRLEN];
617  inet_ntop(AF_INET, &(request_addr_.sin_addr), str, INET_ADDRSTRLEN);
618  TLOG_DEBUG("EventStore") << "Sending request for " << std::to_string(message.size()) << " events to multicast group " << str << TLOG_ENDL;
619  if (sendto(request_socket_, message.header(), sizeof(detail::RequestHeader), 0, (struct sockaddr *)&request_addr_, sizeof(request_addr_)) < 0)
620  {
621  TLOG_ERROR("EventStore") << "Error sending request message header" << TLOG_ENDL;
622  }
623  if (sendto(request_socket_, message.buffer(), sizeof(detail::RequestPacket) * message.size(), 0, (struct sockaddr *)&request_addr_, sizeof(request_addr_)) < 0)
624  {
625  TLOG_ERROR("EventStore") << "Error sending request message data" << TLOG_ENDL;
626  }
627  }
628 
629  void EventStore::send_routing_token_(int nSlots)
630  {
631  TLOG_DEBUG("EventStore") << "send_routing_token_ called, send_routing_tokens_=" << std::boolalpha << send_routing_tokens_ << TLOG_ENDL;
632  if (!send_routing_tokens_) return;
633  if (token_socket_ == -1) setup_tokens_();
634  detail::RoutingToken token;
635  token.header = TOKEN_MAGIC;
636  token.rank = my_rank;
637  token.new_slots_free = nSlots;
638 
639  TLOG_DEBUG("EventStore") << "Sending RoutingToken to " << token_address_ << ":" << token_port_ << TLOG_ENDL;
640  size_t sts = 0;
641  while (sts < sizeof(detail::RoutingToken))
642  {
643  auto res = send(token_socket_, reinterpret_cast<uint8_t*>(&token) + sts, sizeof(detail::RoutingToken) - sts, 0);
644  if (res == -1)
645  {
646  usleep(1000);
647  continue;
648  }
649  sts += res;
650  }
651  TLOG_DEBUG("EventStore") << "Done sending RoutingToken to " << token_address_ << ":" << token_port_ << TLOG_ENDL;
652  }
653 
654  void
655  EventStore::send_request_()
656  {
657  std::thread request([=] { do_send_request_(); });
658  request.detach();
659  }
660 
661  void
663  {
664  if (metricMan)
665  {
666  metricMan->sendMetric("Incomplete Event Count", events_.size(), "events", 1, MetricMode::LastPoint);
667 
668  MonitoredQuantityPtr mqPtr = StatisticsCollection::getInstance().
669  getMonitoredQuantity(EVENT_RATE_STAT_KEY);
670  if (mqPtr.get() != 0)
671  {
672  artdaq::MonitoredQuantityStats stats;
673  mqPtr->getStats(stats);
674 
675  metricMan->sendMetric("Event Count", static_cast<unsigned long>(stats.fullSampleCount), "events", 1, MetricMode::Accumulate);
676  metricMan->sendMetric("Event Rate", stats.recentSampleRate, "events/sec", 1, MetricMode::Average);
677  metricMan->sendMetric("Average Event Size", (stats.recentValueAverage * sizeof(artdaq::RawDataType)), "bytes/fragment", 2, MetricMode::Average);
678  metricMan->sendMetric("Data Rate", (stats.recentValueRate * sizeof(artdaq::RawDataType)), "bytes/sec", 2, MetricMode::Average);
679  }
680  }
681  if (incomplete_event_report_interval_ms_ > 0 && events_.size())
682  {
683  if (std::chrono::duration_cast<std::chrono::milliseconds>(std::chrono::steady_clock::now() - last_incomplete_event_report_time_).count() < incomplete_event_report_interval_ms_) return;
684  last_incomplete_event_report_time_ = std::chrono::steady_clock::now();
685  std::ostringstream oss;
686  oss << "Incomplete Events (" << num_fragments_per_event_ << "): ";
687  for (auto& ev : events_)
688  {
689  oss << ev.first << " (" << ev.second->numFragments() << "), ";
690  }
691  TLOG_DEBUG("EventStore") << oss.str() << TLOG_ENDL;
692  }
693  }
694 }
void insert(FragmentPtr pfrag, bool printWarningWhenFragmentIsDropped=true)
Give ownership of the Fragment to the EventStore.
Definition: EventStore.cc:98
static const std::string EVENT_RATE_STAT_KEY
Key for the Event Rate MonitoredQuantity.
Definition: EventStore.hh:69
void setRequestMode(detail::RequestMessageMode mode)
Set the mode for RequestMessages. Used to indicate when EventStore should enter &quot;EndOfRun&quot; mode...
Definition: EventStore.cc:402
int ResolveHost(char const *host_in, in_addr &addr)
Convert a string hostname to a in_addr suitable for socket communication.
Definition: TCPConnect.cc:29
int TCPConnect(char const *host_in, int dflt_port, long flags=0, int sndbufsiz=0)
Connect to a host on a given port.
Definition: TCPConnect.cc:122
bool endOfData(int &readerReturnValue)
Indicate that the end of input has been reached to the art thread.
Definition: EventStore.cc:263
EventStoreInsertResult
This enumeration contains possible status codes of insertion attempts.
Definition: EventStore.hh:75
void startRun(run_id_t runID)
Start a Run.
Definition: EventStore.cc:321
EventStore()=delete
Default Constructor is deleted.
virtual ~EventStore()
EventStore Destructor.
Definition: EventStore.cc:85
The Fragment was successfully inserted.
void startSubrun()
Start a new Subrun, incrementing the subrun number.
Definition: EventStore.cc:360
The EventStore class collects Fragment objects, until it receives a complete event, at which point the event is handed over to the art thread.
Definition: EventStore.hh:49
static const std::string INCOMPLETE_EVENT_STAT_KEY
Key for the Incomplete Events MonitoredQuantity.
Definition: EventStore.hh:70
The EventStore is full, but the Fragment was accepted as it is for an already-open event...
void setSeqIDModulus(unsigned int seqIDModulus)
Set the parameter that will be used to determine which sequence IDs get grouped together into events...
Definition: EventStore.cc:278
void sendMetrics()
Send metrics to the MetricManager, if one has been instantiated in the application.
Definition: EventStore.cc:662
bool flushData()
Push any incomplete events onto the queue.
Definition: EventStore.cc:283
RawEvent::run_id_t run_id_t
Copy RawEvent::run_id_t into local scope.
Definition: EventStore.hh:64
The EventStore is full, and the Fragment was rejected.
The Fragment was rejected, because the RawEventQueue is full.
bool endRun()
Send an EndOfRunFragment to the art thread.
Definition: EventStore.cc:370
bool endSubrun()
Send an EndOfSubRunFragment to the art thread.
Definition: EventStore.cc:386