artdaq  v2_03_00
 All Classes Namespaces Files Functions Variables Typedefs Enumerations Enumerator Pages
EventStore.cc
1 #include "artdaq/DAQdata/Globals.hh" // Before trace.h gets included in ConcurrentQueue (from GlobalQueue)
2 #include "artdaq/DAQrate/EventStore.hh"
3 #include <utility>
4 #include <cstring>
5 #include <dlfcn.h>
6 #include <iomanip>
7 #include <fstream>
8 #include <sstream>
9 #include <thread>
10 #include <chrono>
11 
12 #include "cetlib/exception.h"
13 #include "artdaq-core/Core/StatisticsCollection.hh"
14 #include "artdaq-core/Core/SimpleQueueReader.hh"
15 #include "artdaq/DAQrate/detail/RequestMessage.hh"
16 #include "artdaq/Application/Routing/RoutingPacket.hh"
18 
19 using namespace std;
20 
21 namespace artdaq
22 {
23  const std::string EventStore::EVENT_RATE_STAT_KEY("EventStoreEventRate");
24  const std::string EventStore::INCOMPLETE_EVENT_STAT_KEY("EventStoreIncompleteEvents");
25 
26  EventStore::EventStore(const fhicl::ParameterSet& pset, size_t num_fragments_per_event, run_id_t run,
27  size_t event_queue_depth, size_t max_incomplete_event_count)
28  : num_fragments_per_event_(num_fragments_per_event)
29  , max_queue_size_(pset.get<size_t>("event_queue_depth", event_queue_depth))
30  , max_incomplete_count_(pset.get<size_t>("max_incomplete_events", max_incomplete_event_count))
31  , run_id_(run)
32  , subrun_id_(0)
33  , events_()
34  , queue_(getGlobalQueue(max_queue_size_))
35  , reader_thread_launch_time_(std::chrono::steady_clock::now())
36  , send_requests_(pset.get<bool>("send_requests", false))
37  , active_requests_()
38  , request_port_(pset.get<int>("request_port", 3001))
39  , request_delay_(pset.get<size_t>("request_delay_ms", 10))
40  , multicast_out_addr_(pset.get<std::string>("output_address", "localhost"))
41  , seqIDModulus_(1)
42  , lastFlushedSeqID_(0)
43  , highestSeqIDSeen_(0)
44  , enq_timeout_(pset.get<double>("event_queue_wait_time", 5.0))
45  , enq_check_count_(pset.get<size_t>("event_queue_check_count", 5000))
46  , printSummaryStats_(pset.get<bool>("print_event_store_stats", false))
47  , incomplete_event_report_interval_ms_(pset.get<int>("incomplete_event_report_interval_ms", -1))
48  , last_incomplete_event_report_time_(std::chrono::steady_clock::now())
49  , token_socket_(-1)
50  , art_thread_wait_ms_(pset.get<int>("art_thread_wait_ms", 4000))
51  {
52  TLOG_DEBUG("EventStore") << "EventStore CONSTRUCTOR" << TLOG_ENDL;
53  initStatistics_();
54  setup_requests_(pset.get<std::string>("request_address", "227.128.12.26"));
55 
56  auto rmConfig = pset.get<fhicl::ParameterSet>("routing_token_config", fhicl::ParameterSet());
57  send_routing_tokens_ = rmConfig.get<bool>("use_routing_master", false);
58  token_port_ = rmConfig.get<int>("routing_token_port", 35555);
59  token_address_ = rmConfig.get<std::string>("routing_master_hostname", "localhost");
60  setup_tokens_();
61  TRACE(12, "artdaq::EventStore::EventStore ctor - reader_thread_ initialized");
62  }
63 
64  EventStore::EventStore(const fhicl::ParameterSet& pset,
65  size_t num_fragments_per_event,
66  run_id_t run,
67  int argc,
68  char* argv[],
69  ART_CMDLINE_FCN* reader)
70  : EventStore(pset, num_fragments_per_event, run, 50, 50)
71  { reader_thread_ = (std::async(std::launch::async, reader, argc, argv)); }
72 
73  EventStore::EventStore(const fhicl::ParameterSet& pset,
74  size_t num_fragments_per_event,
75  run_id_t run,
76  const std::string& configString,
77  ART_CFGSTRING_FCN* reader)
78  : EventStore(pset, num_fragments_per_event, run, 20, 20)
79  { reader_thread_ = (std::async(std::launch::async, reader, configString)); }
80 
82  {
83  TLOG_DEBUG("EventStore") << "Shutting down EventStore" << TLOG_ENDL;
84  if (printSummaryStats_)
85  {
86  reportStatistics_();
87  }
88  shutdown(request_socket_, 2);
89  close(request_socket_);
90  shutdown(token_socket_, 2);
91  close(token_socket_);
92  }
93 
94  void EventStore::insert(FragmentPtr pfrag,
95  bool printWarningWhenFragmentIsDropped)
96  {
97  // We should never get a null pointer, nor should we get a
98  // Fragment without a good fragment ID.
99  assert(pfrag != nullptr);
100  assert(pfrag->fragmentID() != Fragment::InvalidFragmentID);
101 
102  // find the event being built and put the fragment into it,
103  // start new event if not already present
104  // if the event is complete, delete it and report timing
105 
106  // The sequenceID is expected to be correct in the incoming fragment.
107  // The EventStore will divide it by the seqIDModulus to support the use case
108  // of the aggregator which needs to bunch groups of serialized events with
109  // continuous sequence IDs together.
110  if (pfrag->sequenceID() > highestSeqIDSeen_)
111  {
112  highestSeqIDSeen_ = pfrag->sequenceID();
113 
114  // Get the timestamp of this fragment, in experiment-defined clocks
115  Fragment::timestamp_t timestamp = pfrag->timestamp();
116 
117  // Send a request to the board readers!
118  if (send_requests_)
119  {
120  std::lock_guard<std::mutex> lk(request_mutex_);
121  active_requests_[highestSeqIDSeen_] = timestamp;
122  send_request_();
123  }
124  }
125  Fragment::sequence_id_t sequence_id = ((pfrag->sequenceID() - (1 + lastFlushedSeqID_)) / seqIDModulus_) + 1;
126  TRACE(13, "EventStore::insert seq=%lu fragID=%d id=%d lastFlushed=%lu seqIDMod=%d seq=%lu"
127  , pfrag->sequenceID(), pfrag->fragmentID(), my_rank, lastFlushedSeqID_, seqIDModulus_, sequence_id);
128 
129 
130  // Find if the right event id is already known to events_ and, if so, where
131  // it is.
132  EventMap::iterator loc = events_.lower_bound(sequence_id);
133 
134  if (loc == events_.end() || events_.key_comp()(sequence_id, loc->first))
135  {
136  // We don't have an event with this id; create one and insert it at loc,
137  // and ajust loc to point to the newly inserted event.
138  RawEvent_ptr newevent(new RawEvent(run_id_, subrun_id_, pfrag->sequenceID()));
139  loc =
140  events_.insert(loc, EventMap::value_type(sequence_id, newevent));
141  }
142 
143  // Now insert the fragment into the event we have located.
144  loc->second->insertFragment(std::move(pfrag));
145  if (loc->second->numFragments() == num_fragments_per_event_)
146  {
147  // This RawEvent is complete; capture it, remove it from the
148  // map, report on statistics, and put the shared pointer onto
149  // the event queue.
150  RawEvent_ptr complete_event(loc->second);
151  complete_event->markComplete();
152 
153  events_.erase(loc);
154 
155  if (send_requests_)
156  {
157  std::lock_guard<std::mutex> lk(request_mutex_);
158  active_requests_.erase(sequence_id);
159  }
160  // 13-Dec-2012, KAB - this monitoring needs to come before
161  // the enqueueing of the event lest it be empty by the
162  // time that we ask for the word count.
163  MonitoredQuantityPtr mqPtr = StatisticsCollection::getInstance().
164  getMonitoredQuantity(EVENT_RATE_STAT_KEY);
165  if (mqPtr.get() != 0)
166  {
167  mqPtr->addSample(complete_event->wordCount());
168  }
169  TRACE(14, "EventStore::insert seq=%lu enqTimedWait start", sequence_id);
170  bool enqSuccess = queue_.enqTimedWait(complete_event, enq_timeout_);
171  TRACE(enqSuccess ? 14 : 0, "EventStore::insert seq=%lu enqTimedWait complete", sequence_id);
172  if (!enqSuccess)
173  {
174  //TRACE_CNTL( "modeM", 0 );
175  if (printWarningWhenFragmentIsDropped)
176  {
177  TLOG_WARNING("EventStore") << "Enqueueing event " << sequence_id
178  << " FAILED, queue size = "
179  << queue_.size() <<
180  "; apparently no events were removed from this process's queue during the " << std::to_string(enq_timeout_.count())
181  << "-second timeout period" << TLOG_ENDL;
182  }
183  else
184  {
185  TLOG_DEBUG("EventStore") << "Enqueueing event " << sequence_id
186  << " FAILED, queue size = "
187  << queue_.size() <<
188  "; apparently no events were removed from this process's queue during the " << std::to_string(enq_timeout_.count())
189  << "-second timeout period" << TLOG_ENDL;
190  }
191  }
192  else
193  {
194  send_routing_token_(1);
195  }
196  }
197  MonitoredQuantityPtr mqPtr = StatisticsCollection::getInstance().
198  getMonitoredQuantity(INCOMPLETE_EVENT_STAT_KEY);
199  if (mqPtr.get() != 0)
200  {
201  mqPtr->addSample(events_.size());
202  }
203  }
204 
205  EventStore::EventStoreInsertResult EventStore::insert(FragmentPtr pfrag, FragmentPtr& rejectedFragment)
206  {
207  // Test whether this fragment can be safely accepted. If we accept
208  // it, and it completes an event, then we want to be sure that it
209  // can be pushed onto the event queue. If not, we return it and
210  // let the caller know that we didn't accept it.
211  TRACE(12, "EventStore: Testing if queue is full");
212  if (queue_.full())
213  {
214  size_t sleepTime = 1000000 * (enq_timeout_.count() / enq_check_count_);
215  TRACE(12, "EventStore: sleepTime is %lu.", sleepTime);
216  size_t loopCount = 0;
217  while (loopCount < enq_check_count_ && queue_.full())
218  {
219  ++loopCount;
220  usleep(sleepTime);
221  }
222  if (queue_.full())
223  {
224  rejectedFragment = std::move(pfrag);
226  }
227  }
228  TRACE(12, "EventStore: Testing if there's room in the EventStore");
229  auto incomplete_full = events_.size() >= max_incomplete_count_;
230  if (incomplete_full)
231  {
232  EventMap::iterator loc = events_.lower_bound(pfrag->sequenceID());
233 
234  if (loc == events_.end() || events_.key_comp()(pfrag->sequenceID(), loc->first))
235  {
236  rejectedFragment = std::move(pfrag);
238  }
239  }
240 
241  TRACE(12, "EventStore: Performing insert");
242  insert(std::move(pfrag));
244  }
245 
246  bool
247  EventStore::endOfData(int& readerReturnValue)
248  {
249  TLOG_DEBUG("EventStore") << "EventStore::endOfData" << TLOG_ENDL;
250  RawEvent_ptr end_of_data(nullptr);
251  TRACE(4, "EventStore::endOfData: Enqueuing end_of_data event");
252  bool enqSuccess = queue_.enqTimedWait(end_of_data, enq_timeout_);
253  if (!enqSuccess)
254  {
255  return false;
256  }
257  TRACE(4, "EventStore::endOfData: Getting return code from art thread");
258  readerReturnValue = reader_thread_.get();
259  return true;
260  }
261 
262  void EventStore::setSeqIDModulus(unsigned int seqIDModulus)
263  {
264  seqIDModulus_ = seqIDModulus;
265  }
266 
268  {
269  bool enqSuccess;
270  size_t initialStoreSize = events_.size();
271  TLOG_DEBUG("EventStore") << "Flushing " << initialStoreSize
272  << " stale events from the EventStore." << TLOG_ENDL;
273  EventMap::iterator loc;
274  std::vector<sequence_id_t> flushList;
275  for (loc = events_.begin(); loc != events_.end(); ++loc)
276  {
277  RawEvent_ptr complete_event(loc->second);
278  MonitoredQuantityPtr mqPtr = StatisticsCollection::getInstance().
279  getMonitoredQuantity(EVENT_RATE_STAT_KEY);
280  if (mqPtr.get() != 0)
281  {
282  mqPtr->addSample(complete_event->wordCount());
283  }
284  enqSuccess = queue_.enqTimedWait(complete_event, enq_timeout_);
285  if (!enqSuccess)
286  {
287  break;
288  }
289  else
290  {
291  flushList.push_back(loc->first);
292  }
293  }
294  for (size_t idx = 0; idx < flushList.size(); ++idx)
295  {
296  events_.erase(flushList[idx]);
297  }
298  TLOG_DEBUG("EventStore") << "Done flushing " << flushList.size()
299  << " stale events from the EventStore." << TLOG_ENDL;
300 
301  lastFlushedSeqID_ = highestSeqIDSeen_;
302  return (flushList.size() >= initialStoreSize);
303  }
304 
306  {
307  if (!queue_.queueReaderIsReady())
308  {
309  TLOG_WARNING("EventStore") << "Run start requested, but the art thread is not yet ready, waiting up to " << art_thread_wait_ms_ << " msec..." << TLOG_ENDL;
310  while (!queue_.queueReaderIsReady() && std::chrono::duration_cast<std::chrono::milliseconds>(std::chrono::steady_clock::now() - reader_thread_launch_time_).count() < art_thread_wait_ms_)
311  {
312  usleep(1000); // wait 1 ms
313  }
314  if (queue_.queueReaderIsReady())
315  {
316  auto dur = std::chrono::duration_cast<std::chrono::milliseconds>(queue_.getReadyTime() - reader_thread_launch_time_).count();
317  TLOG_INFO("EventStore") << "art initialization took (roughly) " << std::setw(4) << std::to_string(dur) << " ms." << TLOG_ENDL;
318  }
319  else
320  {
321  auto dur = std::chrono::duration_cast<std::chrono::milliseconds>(std::chrono::steady_clock::now() - reader_thread_launch_time_).count();
322  TLOG_ERROR("EventStore") << "art thread still not ready after " << dur << " ms. Continuing to start..." << TLOG_ENDL;
323  }
324  }
325  run_id_ = runID;
326  subrun_id_ = 1;
327  lastFlushedSeqID_ = 0;
328  highestSeqIDSeen_ = 0;
329  send_routing_token_(max_queue_size_);
330  TLOG_DEBUG("EventStore") << "Starting run " << run_id_
331  << ", max queue size = "
332  << max_queue_size_
333  << ", queue capacity = "
334  << queue_.capacity()
335  << ", queue size = "
336  << queue_.size() << TLOG_ENDL;
337  if (metricMan)
338  {
339  double runSubrun = run_id_ + ((double)subrun_id_ / 10000);
340  metricMan->sendMetric("Run Number", runSubrun, "Run:Subrun", 1, false);
341  }
342  }
343 
345  {
346  ++subrun_id_;
347  if (metricMan)
348  {
349  double runSubrun = run_id_ + ((double)subrun_id_ / 10000);
350  metricMan->sendMetric("Run Number", runSubrun, "Run:Subrun", 1, false);
351  }
352  }
353 
355  {
356  RawEvent_ptr endOfRunEvent(new RawEvent(run_id_, subrun_id_, 0));
357  std::unique_ptr<artdaq::Fragment>
358  endOfRunFrag(new
359  Fragment(static_cast<size_t>
360  (ceil(sizeof(my_rank) /
361  static_cast<double>(sizeof(Fragment::value_type))))));
362 
363  endOfRunFrag->setSystemType(Fragment::EndOfRunFragmentType);
364  *endOfRunFrag->dataBegin() = my_rank;
365  endOfRunEvent->insertFragment(std::move(endOfRunFrag));
366 
367  return queue_.enqTimedWait(endOfRunEvent, enq_timeout_);
368  }
369 
371  {
372  RawEvent_ptr endOfSubrunEvent(new RawEvent(run_id_, subrun_id_, 0));
373  std::unique_ptr<artdaq::Fragment>
374  endOfSubrunFrag(new
375  Fragment(static_cast<size_t>
376  (ceil(sizeof(my_rank) /
377  static_cast<double>(sizeof(Fragment::value_type))))));
378 
379  endOfSubrunFrag->setSystemType(Fragment::EndOfSubrunFragmentType);
380  *endOfSubrunFrag->dataBegin() = my_rank;
381  endOfSubrunEvent->insertFragment(std::move(endOfSubrunFrag));
382 
383  return queue_.enqTimedWait(endOfSubrunEvent, enq_timeout_);
384  }
385 
386  void
387  EventStore::initStatistics_()
388  {
389  MonitoredQuantityPtr mqPtr = StatisticsCollection::getInstance().
390  getMonitoredQuantity(EVENT_RATE_STAT_KEY);
391  if (mqPtr.get() == 0)
392  {
393  mqPtr.reset(new MonitoredQuantity(3.0, 300.0));
394  StatisticsCollection::getInstance().
395  addMonitoredQuantity(EVENT_RATE_STAT_KEY, mqPtr);
396  }
397  mqPtr->reset();
398 
399  mqPtr = StatisticsCollection::getInstance().
400  getMonitoredQuantity(INCOMPLETE_EVENT_STAT_KEY);
401  if (mqPtr.get() == 0)
402  {
403  mqPtr.reset(new MonitoredQuantity(3.0, 300.0));
404  StatisticsCollection::getInstance().
405  addMonitoredQuantity(INCOMPLETE_EVENT_STAT_KEY, mqPtr);
406  }
407  mqPtr->reset();
408  }
409 
410  void
411  EventStore::reportStatistics_()
412  {
413  MonitoredQuantityPtr mqPtr = StatisticsCollection::getInstance().
414  getMonitoredQuantity(EVENT_RATE_STAT_KEY);
415  if (mqPtr.get() != 0)
416  {
417  ostringstream oss;
418  oss << EVENT_RATE_STAT_KEY << "_" << setfill('0') << setw(4) << run_id_
419  << "_" << setfill('0') << setw(4) << my_rank << ".txt";
420  std::string filename = oss.str();
421  ofstream outStream(filename.c_str());
422  mqPtr->waitUntilAccumulatorsHaveBeenFlushed(3.0);
423  artdaq::MonitoredQuantityStats stats;
424  mqPtr->getStats(stats);
425  outStream << "EventStore rank " << my_rank << ": events processed = "
426  << stats.fullSampleCount << " at " << stats.fullSampleRate
427  << " events/sec, data rate = "
428  << (stats.fullValueRate * sizeof(RawDataType)
429  / 1024.0 / 1024.0) << " MB/sec, duration = "
430  << stats.fullDuration << " sec" << std::endl
431  << " minimum event size = "
432  << (stats.fullValueMin * sizeof(RawDataType)
433  / 1024.0 / 1024.0)
434  << " MB, maximum event size = "
435  << (stats.fullValueMax * sizeof(RawDataType)
436  / 1024.0 / 1024.0)
437  << " MB" << std::endl;
438  bool foundTheStart = false;
439  for (int idx = 0; idx < (int)stats.recentBinnedDurations.size(); ++idx)
440  {
441  if (stats.recentBinnedDurations[idx] > 0.0)
442  {
443  foundTheStart = true;
444  }
445  if (foundTheStart)
446  {
447  outStream << " " << std::fixed << std::setprecision(3)
448  << stats.recentBinnedEndTimes[idx]
449  << ": " << stats.recentBinnedSampleCounts[idx]
450  << " events at "
451  << (stats.recentBinnedSampleCounts[idx] /
452  stats.recentBinnedDurations[idx])
453  << " events/sec, data rate = "
454  << (stats.recentBinnedValueSums[idx] *
455  sizeof(RawDataType) / 1024.0 / 1024.0 /
456  stats.recentBinnedDurations[idx])
457  << " MB/sec, bin size = "
458  << stats.recentBinnedDurations[idx]
459  << " sec" << std::endl;
460  }
461  }
462  outStream.close();
463  }
464 
465  mqPtr = StatisticsCollection::getInstance().
466  getMonitoredQuantity(INCOMPLETE_EVENT_STAT_KEY);
467  if (mqPtr.get() != 0)
468  {
469  ostringstream oss;
470  oss << INCOMPLETE_EVENT_STAT_KEY << "_" << setfill('0')
471  << setw(4) << run_id_
472  << "_" << setfill('0') << setw(4) << my_rank << ".txt";
473  std::string filename = oss.str();
474  ofstream outStream(filename.c_str());
475  mqPtr->waitUntilAccumulatorsHaveBeenFlushed(3.0);
476  artdaq::MonitoredQuantityStats stats;
477  mqPtr->getStats(stats);
478  outStream << "EventStore rank " << my_rank << ": fragments processed = "
479  << stats.fullSampleCount << " at " << stats.fullSampleRate
480  << " fragments/sec, average incomplete event count = "
481  << stats.fullValueAverage << " duration = "
482  << stats.fullDuration << " sec" << std::endl
483  << " minimum incomplete event count = "
484  << stats.fullValueMin << ", maximum incomplete event count = "
485  << stats.fullValueMax << std::endl;
486  bool foundTheStart = false;
487  for (int idx = 0; idx < (int)stats.recentBinnedDurations.size(); ++idx)
488  {
489  if (stats.recentBinnedDurations[idx] > 0.0)
490  {
491  foundTheStart = true;
492  }
493  if (foundTheStart && stats.recentBinnedSampleCounts[idx] > 0.0)
494  {
495  outStream << " " << std::fixed << std::setprecision(3)
496  << stats.recentBinnedEndTimes[idx]
497  << ": " << stats.recentBinnedSampleCounts[idx]
498  << " fragments at "
499  << (stats.recentBinnedSampleCounts[idx] /
500  stats.recentBinnedDurations[idx])
501  << " fragments/sec, average incomplete event count = "
502  << (stats.recentBinnedValueSums[idx] /
503  stats.recentBinnedSampleCounts[idx])
504  << ", bin size = "
505  << stats.recentBinnedDurations[idx]
506  << " sec" << std::endl;
507  }
508  }
509  outStream << "Incomplete count now = " << events_.size() << std::endl;
510  outStream.close();
511  }
512  }
513 
514  void
515  EventStore::setup_requests_(std::string request_address)
516  {
517  if (send_requests_)
518  {
519  request_socket_ = socket(AF_INET, SOCK_DGRAM, IPPROTO_UDP);
520  if (!request_socket_)
521  {
522  TLOG_ERROR("EventStore") << "I failed to create the socket for sending Data Requests!" << TLOG_ENDL;
523  exit(1);
524  }
525  int sts = ResolveHost(request_address.c_str(), request_port_, request_addr_);
526  if (sts == -1)
527  {
528  TLOG_ERROR("EventStore") << "Unable to resolve Data Request address" << TLOG_ENDL;
529  exit(1);
530  }
531 
532  if (multicast_out_addr_ != "localhost") {
533  struct in_addr addr;
534  int sts = ResolveHost(multicast_out_addr_.c_str(), addr);
535  if (sts == -1)
536  {
537  TLOG_ERROR("EventStore") << "Unable to resolve multicast interface address" << TLOG_ENDL;
538  exit(1);
539  }
540 
541  int yes = 1;
542  if (setsockopt(request_socket_, SOL_SOCKET, SO_REUSEADDR, &yes, sizeof(yes)) < 0)
543  {
544  TLOG_ERROR("EventStore") << "Unable to enable port reuse on request socket" << TLOG_ENDL;
545  exit(1);
546  }
547  if (setsockopt(request_socket_, IPPROTO_IP, IP_MULTICAST_IF, &addr, sizeof(addr)) == -1)
548  {
549  TLOG_ERROR("EventStore") << "Cannot set outgoing interface." << TLOG_ENDL;
550  exit(1);
551  }
552  }
553  int yes = 1;
554  if (setsockopt(request_socket_, SOL_SOCKET, SO_BROADCAST, (void*)&yes, sizeof(int)) == -1)
555  {
556  TLOG_ERROR("EventStore") << "Cannot set request socket to broadcast." << TLOG_ENDL;
557  exit(1);
558  }
559  }
560  }
561 
562  void
563  EventStore::setup_tokens_()
564  {
565  if (send_routing_tokens_)
566  {
567  TLOG_DEBUG("EventStore") << "Creating Routing Token sending socket" << TLOG_ENDL;
568  token_socket_ = TCPConnect(token_address_.c_str(), token_port_);
569  if (!token_socket_)
570  {
571  TLOG_ERROR("EventStore") << "I failed to create the socket for sending Routing Tokens!" << TLOG_ENDL;
572  exit(1);
573  }
574  }
575  }
576 
577  void EventStore::do_send_request_()
578  {
579  std::this_thread::sleep_for(std::chrono::microseconds(request_delay_));
580 
581  detail::RequestMessage message;
582  {
583  std::lock_guard<std::mutex> lk(request_mutex_);
584  for (auto& req : active_requests_)
585  {
586  message.addRequest(req.first, req.second);
587  }
588  }
589  char str[INET_ADDRSTRLEN];
590  inet_ntop(AF_INET, &(request_addr_.sin_addr), str, INET_ADDRSTRLEN);
591  TLOG_DEBUG("EventStore") << "Sending request for " << std::to_string(message.size()) << " events to multicast group " << str << TLOG_ENDL;
592  if (sendto(request_socket_, message.header(), sizeof(detail::RequestHeader), 0, (struct sockaddr *)&request_addr_, sizeof(request_addr_)) < 0)
593  {
594  TLOG_ERROR("EventStore") << "Error sending request message header" << TLOG_ENDL;
595  }
596  if (sendto(request_socket_, message.buffer(), sizeof(detail::RequestPacket) * message.size(), 0, (struct sockaddr *)&request_addr_, sizeof(request_addr_)) < 0)
597  {
598  TLOG_ERROR("EventStore") << "Error sending request message data" << TLOG_ENDL;
599  }
600  }
601 
602  void EventStore::send_routing_token_(int nSlots)
603  {
604  TLOG_DEBUG("EventStore") << "send_routing_token_ called, send_routing_tokens_=" << std::boolalpha << send_routing_tokens_ << TLOG_ENDL;
605  if (!send_routing_tokens_) return;
606  if (token_socket_ == -1) setup_tokens_();
607  detail::RoutingToken token;
608  token.header = TOKEN_MAGIC;
609  token.rank = my_rank;
610  token.new_slots_free = nSlots;
611 
612  TLOG_DEBUG("EventStore") << "Sending RoutingToken to " << token_address_ << ":" << token_port_ << TLOG_ENDL;
613  size_t sts = 0;
614  while (sts < sizeof(detail::RoutingToken)) {
615  auto res = send(token_socket_, reinterpret_cast<uint8_t*>(&token) + sts, sizeof(detail::RoutingToken) - sts, 0);
616  if (res == -1) {
617  usleep(1000);
618  continue;
619  }
620  sts += res;
621  }
622  TLOG_DEBUG("EventStore") << "Done sending RoutingToken to " << token_address_ << ":" << token_port_ << TLOG_ENDL;
623  }
624 
625  void
626  EventStore::send_request_()
627  {
628  std::thread request([=] { do_send_request_(); });
629  request.detach();
630  }
631 
632  void
634  {
635  if (metricMan)
636  {
637  metricMan->sendMetric("Incomplete Event Count", events_.size(),
638  "events", 1);
639  }
640  if (incomplete_event_report_interval_ms_ > 0 && events_.size())
641  {
642  if (std::chrono::duration_cast<std::chrono::milliseconds>(std::chrono::steady_clock::now() - last_incomplete_event_report_time_).count() < incomplete_event_report_interval_ms_) return;
643  last_incomplete_event_report_time_ = std::chrono::steady_clock::now();
644  std::ostringstream oss;
645  oss << "Incomplete Events (" << num_fragments_per_event_ << "): ";
646  for (auto& ev : events_)
647  {
648  oss << ev.first << " (" << ev.second->numFragments() << "), ";
649  }
650  TLOG_DEBUG("EventStore") << oss.str() << TLOG_ENDL;
651  }
652  }
653 }
void insert(FragmentPtr pfrag, bool printWarningWhenFragmentIsDropped=true)
Give ownership of the Fragment to the EventStore.
Definition: EventStore.cc:94
static const std::string EVENT_RATE_STAT_KEY
Key for the Event Rate MonitoredQuantity.
Definition: EventStore.hh:68
int ResolveHost(char const *host_in, in_addr &addr)
Convert a string hostname to a in_addr suitable for socket communication.
Definition: TCPConnect.cc:29
int TCPConnect(char const *host_in, int dflt_port, long flags=0, int sndbufsiz=0)
Connect to a host on a given port.
Definition: TCPConnect.cc:122
bool endOfData(int &readerReturnValue)
Indicate that the end of input has been reached to the art thread.
Definition: EventStore.cc:247
EventStoreInsertResult
This enumeration contains possible status codes of insertion attempts.
Definition: EventStore.hh:74
void startRun(run_id_t runID)
Start a Run.
Definition: EventStore.cc:305
EventStore()=delete
Default Constructor is deleted.
virtual ~EventStore()
EventStore Destructor.
Definition: EventStore.cc:81
The Fragment was successfully inserted.
void startSubrun()
Start a new Subrun, incrementing the subrun number.
Definition: EventStore.cc:344
The EventStore class collects Fragment objects, until it receives a complete event, at which point the event is handed over to the art thread.
Definition: EventStore.hh:48
static const std::string INCOMPLETE_EVENT_STAT_KEY
Key for the Incomplete Events MonitoredQuantity.
Definition: EventStore.hh:69
The EventStore is full, but the Fragment was accepted as it is for an already-open event...
void setSeqIDModulus(unsigned int seqIDModulus)
Set the parameter that will be used to determine which sequence IDs get grouped together into events...
Definition: EventStore.cc:262
void sendMetrics()
Send metrics to the MetricManager, if one has been instantiated in the application.
Definition: EventStore.cc:633
bool flushData()
Push any incomplete events onto the queue.
Definition: EventStore.cc:267
RawEvent::run_id_t run_id_t
Copy RawEvent::run_id_t into local scope.
Definition: EventStore.hh:63
The EventStore is full, and the Fragment was rejected.
The Fragment was rejected, because the RawEventQueue is full.
bool endRun()
Send an EndOfRunFragment to the art thread.
Definition: EventStore.cc:354
bool endSubrun()
Send an EndOfSubRunFragment to the art thread.
Definition: EventStore.cc:370