artdaq  v3_09_01
BoardReaderCore.cc
1 
2 #include "artdaq/DAQdata/Globals.hh" // include these 2 first -
3 #define TRACE_NAME (app_name + "_BoardReaderCore").c_str()
4 
5 #include "artdaq-core/Data/Fragment.hh"
6 #include "artdaq-core/Utilities/ExceptionHandler.hh"
7 #include "artdaq/Application/BoardReaderCore.hh"
8 #include "artdaq/Application/TaskType.hh"
9 #include "artdaq/Generators/makeCommandableFragmentGenerator.hh"
10 
11 #include <pthread.h>
12 #include <sched.h>
13 #include <algorithm>
14 #include <thread>
15 #include <memory>
16 #include "canvas/Utilities/Exception.h"
17 #include "cetlib_except/exception.h"
18 
19 const std::string artdaq::BoardReaderCore::
20  FRAGMENTS_PROCESSED_STAT_KEY("BoardReaderCoreFragmentsProcessed");
21 const std::string artdaq::BoardReaderCore::
22  INPUT_WAIT_STAT_KEY("BoardReaderCoreInputWaitTime");
23 const std::string artdaq::BoardReaderCore::BUFFER_WAIT_STAT_KEY("BoardReaderCoreBufferWaitTime");
24 const std::string artdaq::BoardReaderCore::REQUEST_WAIT_STAT_KEY("BoardReaderCoreRequestWaitTime");
25  const std::string artdaq::BoardReaderCore::
26  BRSYNC_WAIT_STAT_KEY("BoardReaderCoreBRSyncWaitTime");
27 const std::string artdaq::BoardReaderCore::
28  OUTPUT_WAIT_STAT_KEY("BoardReaderCoreOutputWaitTime");
29 const std::string artdaq::BoardReaderCore::
30  FRAGMENTS_PER_READ_STAT_KEY("BoardReaderCoreFragmentsPerRead");
31 
32 std::unique_ptr<artdaq::DataSenderManager> artdaq::BoardReaderCore::sender_ptr_ = nullptr;
33 
35  : parent_application_(parent_application)
36  /*, local_group_comm_(local_group_comm)*/
37  , generator_ptr_(nullptr)
38  , run_id_(art::RunID::flushRun())
39  , fragment_count_(0)
40  , stop_requested_(false)
41  , pause_requested_(false)
42 {
43  TLOG(TLVL_DEBUG) << "Constructor";
46  statsHelper_.addMonitoredQuantityName(BUFFER_WAIT_STAT_KEY);
47  statsHelper_.addMonitoredQuantityName(REQUEST_WAIT_STAT_KEY);
51 }
52 
54 {
55  TLOG(TLVL_DEBUG) << "Destructor";
56  TLOG(TLVL_DEBUG) << "Stopping Request Receiver BEGIN";
57  request_receiver_ptr_.reset(nullptr);
58  TLOG(TLVL_DEBUG) << "Stopping Request Receiver END";
59 }
60 
61 bool artdaq::BoardReaderCore::initialize(fhicl::ParameterSet const& pset, uint64_t /*unused*/, uint64_t /*unused*/)
62 {
63  TLOG(TLVL_DEBUG) << "initialize method called with "
64  << "ParameterSet = \"" << pset.to_string() << "\".";
65 
66  // pull out the relevant parts of the ParameterSet
67  fhicl::ParameterSet daq_pset;
68  try
69  {
70  daq_pset = pset.get<fhicl::ParameterSet>("daq");
71  }
72  catch (...)
73  {
74  TLOG(TLVL_ERROR)
75  << "Unable to find the DAQ parameters in the initialization "
76  << "ParameterSet: \"" + pset.to_string() + "\".";
77  return false;
78  }
79  fhicl::ParameterSet fr_pset;
80  try
81  {
82  fr_pset = daq_pset.get<fhicl::ParameterSet>("fragment_receiver");
83  data_pset_ = fr_pset;
84  }
85  catch (...)
86  {
87  TLOG(TLVL_ERROR)
88  << "Unable to find the fragment_receiver parameters in the DAQ "
89  << "initialization ParameterSet: \"" + daq_pset.to_string() + "\".";
90  return false;
91  }
92 
93  // pull out the Metric part of the ParameterSet
94  fhicl::ParameterSet metric_pset;
95  try
96  {
97  metric_pset = daq_pset.get<fhicl::ParameterSet>("metrics");
98  }
99  catch (...)
100  {} // OK if there's no metrics table defined in the FHiCL
101 
102  if (metric_pset.is_empty())
103  {
104  TLOG(TLVL_INFO) << "No metric plugins appear to be defined";
105  }
106  try
107  {
108  metricMan->initialize(metric_pset, app_name);
109  }
110  catch (...)
111  {
112  ExceptionHandler(ExceptionHandlerRethrow::no,
113  "Error loading metrics in BoardReaderCore::initialize()");
114  }
115 
116  if (daq_pset.has_key("rank"))
117  {
118  if (my_rank >= 0 && daq_pset.get<int>("rank") != my_rank)
119  {
120  TLOG(TLVL_WARNING) << "BoardReader rank specified at startup is different than rank specified at configure! Using rank received at configure!";
121  }
122  my_rank = daq_pset.get<int>("rank");
123  }
124  if (my_rank == -1)
125  {
126  TLOG(TLVL_ERROR) << "BoardReader rank not specified at startup or in configuration! Aborting";
127  throw cet::exception("RankNotSpecifiedError") << "BoardReader rank not specified at startup or in configuration! Aborting";
128  }
129 
130  // create the requested CommandableFragmentGenerator
131  auto frag_gen_name = fr_pset.get<std::string>("generator", "");
132  if (frag_gen_name.length() == 0)
133  {
134  TLOG(TLVL_ERROR)
135  << "No fragment generator (parameter name = \"generator\") was "
136  << "specified in the fragment_receiver ParameterSet. The "
137  << "DAQ initialization PSet was \"" << daq_pset.to_string() << "\".";
138  return false;
139  }
140 
141  try
142  {
143  generator_ptr_ = artdaq::makeCommandableFragmentGenerator(frag_gen_name, fr_pset);
144  }
145  catch (...)
146  {
147  std::stringstream exception_string;
148  exception_string << "Exception thrown during initialization of fragment generator of type \""
149  << frag_gen_name << "\"";
150 
151  ExceptionHandler(ExceptionHandlerRethrow::no, exception_string.str());
152 
153  TLOG(TLVL_DEBUG) << "FHiCL parameter set used to initialize the fragment generator which threw an exception: " << fr_pset.to_string();
154 
155  return false;
156  }
157 
158  try
159  {
160  fragment_buffer_ptr_.reset(new FragmentBuffer(fr_pset));
161  }
162  catch (...)
163  {
164  std::stringstream exception_string;
165  exception_string << "Exception thrown during initialization of Fragment Buffer";
166 
167  ExceptionHandler(ExceptionHandlerRethrow::no, exception_string.str());
168 
169  TLOG(TLVL_DEBUG) << "FHiCL parameter set used to initialize the fragment buffer which threw an exception: " << fr_pset.to_string();
170 
171  return false;
172  }
173 
174  std::shared_ptr<RequestBuffer> request_buffer = std::make_shared<RequestBuffer>(fr_pset.get<artdaq::Fragment::sequence_id_t>("request_increment", 1));
175 
176  try
177  {
178  request_receiver_ptr_.reset(new RequestReceiver(fr_pset, request_buffer));
179  generator_ptr_->SetRequestBuffer(request_buffer);
180  fragment_buffer_ptr_->SetRequestBuffer(request_buffer);
181  }
182  catch (...)
183  {
184  ExceptionHandler(ExceptionHandlerRethrow::no, "Exception thrown during initialization of request receiver");
185 
186  TLOG(TLVL_DEBUG) << "FHiCL parameter set used to initialize the request receiver which threw an exception: " << fr_pset.to_string();
187 
188  return false;
189 
190  }
191  metricMan->setPrefix(generator_ptr_->metricsReportingInstanceName());
192 
193  rt_priority_ = fr_pset.get<int>("rt_priority", 0);
194 
195  // fetch the monitoring parameters and create the MonitoredQuantity instances
196  statsHelper_.createCollectors(fr_pset, 100, 30.0, 60.0, FRAGMENTS_PROCESSED_STAT_KEY);
197 
198  // check if we should skip the sequence ID test...
199  skip_seqId_test_ = (generator_ptr_->fragmentIDs().size() > 1 || fragment_buffer_ptr_->request_mode() != RequestMode::Ignored);
200 
201  verbose_ = fr_pset.get<bool>("verbose", true);
202 
203  return true;
204 }
205 
206 bool artdaq::BoardReaderCore::start(art::RunID id, uint64_t timeout, uint64_t timestamp)
207 {
208  TLOG((verbose_ ? TLVL_INFO : TLVL_DEBUG)) << "Starting run " << id.run();
209  stop_requested_.store(false);
210  pause_requested_.store(false);
211 
212  fragment_count_ = 0;
213  prev_seq_id_ = 0;
214  statsHelper_.resetStatistics();
215 
216  fragment_buffer_ptr_->Reset(false);
217 
218  metricMan->do_start();
219  generator_ptr_->StartCmd(id.run(), timeout, timestamp);
220  run_id_ = id;
221 
222  request_receiver_ptr_->SetRunNumber(static_cast<uint32_t>(id.run()));
223  request_receiver_ptr_->startRequestReception();
224 
225  TLOG((verbose_ ? TLVL_INFO : TLVL_DEBUG)) << "Completed the Start transition (Started run) for run " << run_id_.run()
226  << ", timeout = " << timeout << ", timestamp = " << timestamp;
227  return true;
228 }
229 
230 bool artdaq::BoardReaderCore::stop(uint64_t timeout, uint64_t timestamp)
231 {
232  TLOG((verbose_ ? TLVL_INFO : TLVL_DEBUG)) << "Stopping run " << run_id_.run() << " after " << fragment_count_ << " fragments.";
233  stop_requested_.store(true);
234 
235  TLOG(TLVL_DEBUG) << "Stopping Request reception BEGIN";
236  request_receiver_ptr_->stopRequestReception();
237  TLOG(TLVL_DEBUG) << "Stopping Request reception END";
238 
239  TLOG(TLVL_DEBUG) << "Stopping CommandableFragmentGenerator BEGIN";
240  generator_ptr_->StopCmd(timeout, timestamp);
241  TLOG(TLVL_DEBUG) << "Stopping CommandableFragmentGenerator END";
242 
243  TLOG(TLVL_DEBUG) << "Stopping FragmentBuffer";
244  fragment_buffer_ptr_->Stop();
245 
246  TLOG(TLVL_DEBUG) << "Stopping DataSenderManager";
247  if (sender_ptr_)
248  {
249  sender_ptr_->StopSender();
250  }
251 
252  TLOG((verbose_ ? TLVL_INFO : TLVL_DEBUG)) << "Completed the Stop transition for run " << run_id_.run();
253  return true;
254 }
255 
256 bool artdaq::BoardReaderCore::pause(uint64_t timeout, uint64_t timestamp)
257 {
258  TLOG((verbose_ ? TLVL_INFO : TLVL_DEBUG)) << "Pausing run " << run_id_.run() << " after " << fragment_count_ << " fragments.";
259  pause_requested_.store(true);
260  generator_ptr_->PauseCmd(timeout, timestamp);
261  TLOG((verbose_ ? TLVL_INFO : TLVL_DEBUG)) << "Completed the Pause transition for run " << run_id_.run();
262  return true;
263 }
264 
265 bool artdaq::BoardReaderCore::resume(uint64_t timeout, uint64_t timestamp)
266 {
267  TLOG((verbose_ ? TLVL_INFO : TLVL_DEBUG)) << "Resuming run " << run_id_.run();
268  pause_requested_.store(false);
269  metricMan->do_start();
270  generator_ptr_->ResumeCmd(timeout, timestamp);
271  TLOG((verbose_ ? TLVL_INFO : TLVL_DEBUG)) << "Completed the Resume transition for run " << run_id_.run();
272  return true;
273 }
274 
275 bool artdaq::BoardReaderCore::shutdown(uint64_t /*unused*/)
276 {
277  TLOG((verbose_ ? TLVL_INFO : TLVL_DEBUG)) << "Starting Shutdown transition";
278  generator_ptr_->joinThreads(); // Cleanly shut down the CommandableFragmentGenerator
279  generator_ptr_.reset(nullptr);
280  metricMan->shutdown();
281  TLOG((verbose_ ? TLVL_INFO : TLVL_DEBUG)) << "Completed Shutdown transition";
282  return true;
283 }
284 
285 bool artdaq::BoardReaderCore::soft_initialize(fhicl::ParameterSet const& pset, uint64_t timeout, uint64_t timestamp)
286 {
287  TLOG(TLVL_DEBUG) << "soft_initialize method called with "
288  << "ParameterSet = \"" << pset.to_string()
289  << "\". Forwarding to initialize.";
290  return initialize(pset, timeout, timestamp);
291 }
292 
293 bool artdaq::BoardReaderCore::reinitialize(fhicl::ParameterSet const& pset, uint64_t timeout, uint64_t timestamp)
294 {
295  TLOG(TLVL_DEBUG) << "reinitialize method called with "
296  << "ParameterSet = \"" << pset.to_string()
297  << "\". Forwarding to initalize.";
298  return initialize(pset, timeout, timestamp);
299 }
300 
302 {
303  if (rt_priority_ > 0)
304  {
305 #pragma GCC diagnostic push
306 #pragma GCC diagnostic ignored "-Wmissing-field-initializers"
307  sched_param s_param = {};
308  s_param.sched_priority = rt_priority_;
309  if (pthread_setschedparam(pthread_self(), SCHED_RR, &s_param))
310  TLOG(TLVL_WARNING) << "setting realtime priority failed";
311 #pragma GCC diagnostic pop
312  }
313 
314  // try-catch block here?
315 
316  // how to turn RT PRI off?
317  if (rt_priority_ > 0)
318  {
319 #pragma GCC diagnostic push
320 #pragma GCC diagnostic ignored "-Wmissing-field-initializers"
321  sched_param s_param = {};
322  s_param.sched_priority = rt_priority_;
323  int status = pthread_setschedparam(pthread_self(), SCHED_RR, &s_param);
324  if (status != 0)
325  {
326  TLOG(TLVL_ERROR)
327  << "Failed to set realtime priority to " << rt_priority_
328  << ", return code = " << status;
329  }
330 #pragma GCC diagnostic pop
331  }
332 
333  TLOG(TLVL_DEBUG) << "Waiting for first fragment.";
334  artdaq::MonitoredQuantityStats::TIME_POINT_T startTime, after_input, after_buffer;
335  artdaq::FragmentPtrs frags;
336 
337  bool active = true;
338 
339  while (active)
340  {
341  startTime = artdaq::MonitoredQuantity::getCurrentTime();
342 
343  TLOG(18) << "receive_fragments getNext start";
344  active = generator_ptr_->getNext(frags);
345  TLOG(18) << "receive_fragments getNext done (active=" << active << ")";
346  // 08-May-2015, KAB & JCF: if the generator getNext() method returns false
347  // (which indicates that the data flow has stopped) *and* the reason that
348  // it has stopped is because there was an exception that wasn't handled by
349  // the experiment-specific FragmentGenerator class, we move to the
350  // InRunError state so that external observers (e.g. RunControl or
351  // DAQInterface) can see that there was a problem.
352  if (!active && generator_ptr_ && generator_ptr_->exception())
353  {
354  parent_application_.in_run_failure();
355  }
356 
357  after_input = artdaq::MonitoredQuantity::getCurrentTime();
358 
359 
360  if (!active) { break; }
361  statsHelper_.addSample(FRAGMENTS_PER_READ_STAT_KEY, frags.size());
362 
363  if (frags.size() > 0)
364  {
365  TLOG(18) << "receive_fragments AddFragmentsToBuffer start";
366  fragment_buffer_ptr_->AddFragmentsToBuffer(std::move(frags));
367  TLOG(18) << "receive_fragments AddFragmentsToBuffer done";
368  }
369 
370  after_buffer = artdaq::MonitoredQuantity::getCurrentTime();
371  TLOG(16) << "receive_fragments INPUT_WAIT=" << (after_input - startTime) << ", BUFFER_WAIT=" << (after_buffer - after_input);
372  statsHelper_.addSample(INPUT_WAIT_STAT_KEY, after_input - startTime);
373  statsHelper_.addSample(BUFFER_WAIT_STAT_KEY, after_buffer - after_input);
374  if (statsHelper_.statsRollingWindowHasMoved()) { sendMetrics_(); }
375  frags.clear();
376  }
377 
378  // 11-May-2015, KAB: call MetricManager::do_stop whenever we exit the
379  // processing fragments loop so that metrics correctly go to zero when
380  // there is no data flowing
381  metricMan->do_stop();
382 
383  TLOG(TLVL_DEBUG) << "receive_fragments loop end";
384 }
386 {
387  if (rt_priority_ > 0)
388  {
389 #pragma GCC diagnostic push
390 #pragma GCC diagnostic ignored "-Wmissing-field-initializers"
391  sched_param s_param = {};
392  s_param.sched_priority = rt_priority_;
393  if (pthread_setschedparam(pthread_self(), SCHED_RR, &s_param) != 0)
394  {
395  TLOG(TLVL_WARNING) << "setting realtime priority failed";
396  }
397 #pragma GCC diagnostic pop
398  }
399 
400  // try-catch block here?
401 
402  // how to turn RT PRI off?
403  if (rt_priority_ > 0)
404  {
405 #pragma GCC diagnostic push
406 #pragma GCC diagnostic ignored "-Wmissing-field-initializers"
407  sched_param s_param = {};
408  s_param.sched_priority = rt_priority_;
409  int status = pthread_setschedparam(pthread_self(), SCHED_RR, &s_param);
410  if (status != 0)
411  {
412  TLOG(TLVL_ERROR)
413  << "Failed to set realtime priority to " << rt_priority_
414  << ", return code = " << status;
415  }
416 #pragma GCC diagnostic pop
417  }
418 
419  TLOG(TLVL_DEBUG) << "Initializing DataSenderManager. my_rank=" << my_rank;
420  sender_ptr_ = std::make_unique<artdaq::DataSenderManager>(data_pset_);
421 
422  TLOG(TLVL_DEBUG) << "Waiting for first fragment.";
423  artdaq::MonitoredQuantityStats::TIME_POINT_T startTime;
424  double delta_time;
425  artdaq::FragmentPtrs frags;
426  auto targetFragCount = generator_ptr_->fragmentIDs().size();
427 
428  bool active = true;
429 
430  while (active)
431  {
432  startTime = artdaq::MonitoredQuantity::getCurrentTime();
433 
434  TLOG(18) << "send_fragments applyRequests start";
435  active = fragment_buffer_ptr_->applyRequests(frags);
436  TLOG(18) << "send_fragments applyRequests done (active=" << active << ")";
437  // 08-May-2015, KAB & JCF: if the generator getNext() method returns false
438  // (which indicates that the data flow has stopped) *and* the reason that
439  // it has stopped is because there was an exception that wasn't handled by
440  // the experiment-specific FragmentGenerator class, we move to the
441  // InRunError state so that external observers (e.g. RunControl or
442  // DAQInterface) can see that there was a problem.
443  if (!active && generator_ptr_ && generator_ptr_->exception())
444  {
445  parent_application_.in_run_failure();
446  }
447 
448  delta_time = artdaq::MonitoredQuantity::getCurrentTime() - startTime;
449 
450  TLOG(16) << "send_fragments REQUEST_WAIT=" << delta_time;
451  statsHelper_.addSample(REQUEST_WAIT_STAT_KEY, delta_time);
452 
453  if (!active) { break; }
454 
455  for (auto& fragPtr : frags)
456  {
457  if (fragPtr == nullptr)
458  {
459  TLOG(TLVL_WARNING) << "Encountered a bad fragment pointer in fragment " << fragment_count_ << ". "
460  << "This is most likely caused by a problem with the Fragment Generator!";
461  continue;
462  }
463  if (fragment_count_ == 0)
464  {
465  TLOG(TLVL_DEBUG) << "Received first Fragment from Fragment Generator, sequence ID " << fragPtr->sequenceID() << ", size = " << fragPtr->sizeBytes() << " bytes.";
466  }
467 
468  if (fragPtr->type() == Fragment::EndOfRunFragmentType || fragPtr->type() == Fragment::EndOfSubrunFragmentType || fragPtr->type() == Fragment::InitFragmentType)
469  {
470  // Just broadcast any system Fragments in the output
471  artdaq::Fragment::sequence_id_t sequence_id = fragPtr->sequenceID();
472  statsHelper_.addSample(FRAGMENTS_PROCESSED_STAT_KEY, fragPtr->sizeBytes());
473 
474  startTime = artdaq::MonitoredQuantity::getCurrentTime();
475  TLOG(17) << "send_fragments seq=" << sequence_id << " sendFragment start";
476  auto res = sender_ptr_->sendFragment(std::move(*fragPtr));
477  TLOG(17) << "send_fragments seq=" << sequence_id << " sendFragment done (dest=" << res.first << ", sts=" << TransferInterface::CopyStatusToString(res.second) << ")";
478  ++fragment_count_;
479  statsHelper_.addSample(OUTPUT_WAIT_STAT_KEY,
480  artdaq::MonitoredQuantity::getCurrentTime() - startTime);
481  continue;
482  }
483 
484  artdaq::Fragment::sequence_id_t sequence_id = fragPtr->sequenceID();
485  SetMFIteration("Sequence ID " + std::to_string(sequence_id));
486  statsHelper_.addSample(FRAGMENTS_PROCESSED_STAT_KEY, fragPtr->sizeBytes());
487 
488  /*if ((fragment_count_ % 250) == 0)
489  {
490  TLOG(TLVL_DEBUG)
491  << "Sending fragment " << fragment_count_
492  << " with sequence id " << sequence_id << ".";
493  }*/
494 
495  // check for continous sequence IDs
496  if (!skip_seqId_test_ && abs(static_cast<int64_t>(sequence_id) - static_cast<int64_t>(prev_seq_id_)) > 1)
497  {
498  TLOG(TLVL_WARNING)
499  << "Missing sequence IDs: current sequence ID = "
500  << sequence_id << ", previous sequence ID = "
501  << prev_seq_id_ << ".";
502  }
503  prev_seq_id_ = sequence_id;
504 
505  startTime = artdaq::MonitoredQuantity::getCurrentTime();
506  TLOG(17) << "send_fragments seq=" << sequence_id << " sendFragment start";
507  auto res = sender_ptr_->sendFragment(std::move(*fragPtr));
508  if (sender_ptr_->GetSentSequenceIDCount(sequence_id) == targetFragCount)
509  {
510  sender_ptr_->RemoveRoutingTableEntry(sequence_id);
511  }
512  TLOG(17) << "send_fragments seq=" << sequence_id << " sendFragment done (dest=" << res.first << ", sts=" << TransferInterface::CopyStatusToString(res.second) << ")";
513  ++fragment_count_;
514  statsHelper_.addSample(OUTPUT_WAIT_STAT_KEY,
515  artdaq::MonitoredQuantity::getCurrentTime() - startTime);
516 
517  bool readyToReport = statsHelper_.readyToReport();
518  if (readyToReport)
519  {
520  TLOG(TLVL_INFO) << buildStatisticsString_();
521  }
522 
523  // Turn on lvls (mem and/or slow) 3,13,14 to log every send.
524  TLOG(((fragment_count_ == 1) ? TLVL_DEBUG
525  : (((fragment_count_ % 250) == 0 || readyToReport) ? 13 : 14)))
526  << ((fragment_count_ == 1)
527  ? "Sent first Fragment"
528  : "Sending fragment " + std::to_string(fragment_count_))
529  << " with SeqID " << sequence_id << ".";
530  }
531  if (statsHelper_.statsRollingWindowHasMoved()) { sendMetrics_(); }
532  frags.clear();
533  std::this_thread::yield();
534  }
535 
536  sender_ptr_.reset(nullptr);
537 
538  // 11-May-2015, KAB: call MetricManager::do_stop whenever we exit the
539  // processing fragments loop so that metrics correctly go to zero when
540  // there is no data flowing
541  metricMan->do_stop();
542 
543  TLOG(TLVL_DEBUG) << "send_fragments loop end";
544 }
545 
546 std::string artdaq::BoardReaderCore::report(std::string const& which) const
547 {
548  std::string resultString;
549 
550  // pass the request to the FragmentGenerator instance, if it's available
551  if (generator_ptr_ != nullptr && which != "core")
552  {
553  resultString = generator_ptr_->ReportCmd(which);
554  if (resultString.length() > 0) { return resultString; }
555  }
556 
557  // handle the request at this level, if we can
558  // --> nothing here yet
559 
560  // if we haven't been able to come up with any report so far, say so
561  std::string tmpString = app_name + " run number = ";
562  tmpString.append(boost::lexical_cast<std::string>(run_id_.run()));
563 
564  tmpString.append(", Sent Fragment count = ");
565  tmpString.append(boost::lexical_cast<std::string>(fragment_count_));
566 
567  if (!which.empty() && which != "core")
568  {
569  tmpString.append(". Command=\"" + which + "\" is not currently supported.");
570  }
571  return tmpString;
572 }
573 
574 bool artdaq::BoardReaderCore::metaCommand(std::string const& command, std::string const& arg)
575 {
576  TLOG(TLVL_DEBUG) << "metaCommand method called with "
577  << "command = \"" << command << "\""
578  << ", arg = \"" << arg << "\""
579  << ".";
580 
581  if (generator_ptr_)
582  {
583  return generator_ptr_->metaCommand(command, arg);
584  }
585 
586  return true;
587 }
588 
589 std::string artdaq::BoardReaderCore::buildStatisticsString_()
590 {
591  std::ostringstream oss;
592  double fragmentsGeneratedCount = 1.0;
593  double fragmentsOutputCount = 1.0;
594  oss << app_name << " statistics:" << std::endl;
595 
596  oss << " Fragments read: ";
597  artdaq::MonitoredQuantityPtr mqPtr = artdaq::StatisticsCollection::getInstance().getMonitoredQuantity(FRAGMENTS_PER_READ_STAT_KEY);
598  if (mqPtr.get() != nullptr)
599  {
600  artdaq::MonitoredQuantityStats stats;
601  mqPtr->getStats(stats);
602  oss << stats.recentSampleCount << " fragments generated at "
603  << stats.recentSampleRate << " reads/sec, fragment rate = "
604  << stats.recentValueRate << " fragments/sec, monitor window = "
605  << stats.recentDuration << " sec, min::max read size = "
606  << stats.recentValueMin
607  << "::"
608  << stats.recentValueMax
609  << " fragments";
610  fragmentsGeneratedCount = std::max(double(stats.recentSampleCount), 1.0);
611  oss << " Average times per fragment: ";
612  if (stats.recentSampleRate > 0.0)
613  {
614  oss << " elapsed time = "
615  << (1.0 / stats.recentSampleRate) << " sec";
616  }
617  }
618 
619  oss << std::endl;
620  mqPtr = artdaq::StatisticsCollection::getInstance().getMonitoredQuantity(FRAGMENTS_PROCESSED_STAT_KEY);
621  if (mqPtr.get() != nullptr)
622  {
623  artdaq::MonitoredQuantityStats stats;
624  mqPtr->getStats(stats);
625  oss << " Fragment output statistics: "
626  << stats.recentSampleCount << " fragments sent at "
627  << stats.recentSampleRate << " fragments/sec, effective data rate = "
628  << (stats.recentValueRate * sizeof(artdaq::RawDataType) / 1024.0 / 1024.0) << " MB/sec, monitor window = "
629  << stats.recentDuration << " sec, min::max event size = "
630  << (stats.recentValueMin * sizeof(artdaq::RawDataType) / 1024.0 / 1024.0)
631  << "::"
632  << (stats.recentValueMax * sizeof(artdaq::RawDataType) / 1024.0 / 1024.0)
633  << " MB" << std::endl;
634  fragmentsOutputCount = std::max(double(stats.recentSampleCount), 1.0);
635  }
636 
637  // 31-Dec-2014, KAB - Just a reminder that using "fragmentCount" in the
638  // denominator of the calculations below is important because the way that
639  // the accumulation of these statistics is done is not fragment-by-fragment
640  // but read-by-read (where each read can contain multiple fragments).
641  // 29-Aug-2016, KAB - BRSYNC_WAIT and OUTPUT_WAIT are now done fragment-by-
642  // fragment, but we'll leave the calculation the same. (The alternative
643  // would be to use recentValueAverage().)
644 
645  mqPtr = artdaq::StatisticsCollection::getInstance().getMonitoredQuantity(INPUT_WAIT_STAT_KEY);
646  if (mqPtr.get() != nullptr)
647  {
648  oss << " Input wait time = "
649  << (mqPtr->getRecentValueSum() / fragmentsGeneratedCount) << " s/fragment";
650  }
651  mqPtr = artdaq::StatisticsCollection::getInstance().getMonitoredQuantity(BUFFER_WAIT_STAT_KEY);
652  if (mqPtr.get() != 0)
653  {
654  oss << ", buffer wait time = "
655  << (mqPtr->getRecentValueSum() / fragmentsGeneratedCount) << " s/fragment";
656  }
657  mqPtr = artdaq::StatisticsCollection::getInstance().getMonitoredQuantity(REQUEST_WAIT_STAT_KEY);
658  if (mqPtr.get() != 0)
659  {
660  oss << ", request wait time = "
661  << (mqPtr->getRecentValueSum() / fragmentsOutputCount) << " s/fragment";
662  }
663 
664  mqPtr = artdaq::StatisticsCollection::getInstance().getMonitoredQuantity(BRSYNC_WAIT_STAT_KEY);
665  if (mqPtr.get() != nullptr)
666  {
667  oss << ", BRsync wait time = "
668  << (mqPtr->getRecentValueSum() / fragmentsOutputCount) << " s/fragment";
669  }
670 
671  mqPtr = artdaq::StatisticsCollection::getInstance().getMonitoredQuantity(OUTPUT_WAIT_STAT_KEY);
672  if (mqPtr.get() != nullptr)
673  {
674  oss << ", output wait time = "
675  << (mqPtr->getRecentValueSum() / fragmentsOutputCount) << " s/fragment";
676  }
677 
678 
679  return oss.str();
680 }
681 
682 void artdaq::BoardReaderCore::sendMetrics_()
683 {
684  //TLOG(TLVL_DEBUG) << "Sending metrics " << __LINE__ ;
685  double fragmentCount = 1.0;
686  artdaq::MonitoredQuantityPtr mqPtr = artdaq::StatisticsCollection::getInstance().getMonitoredQuantity(FRAGMENTS_PROCESSED_STAT_KEY);
687  if (mqPtr.get() != nullptr)
688  {
689  artdaq::MonitoredQuantityStats stats;
690  mqPtr->getStats(stats);
691  fragmentCount = std::max(double(stats.recentSampleCount), 1.0);
692  metricMan->sendMetric("Fragment Count", stats.fullSampleCount, "fragments", 1, MetricMode::LastPoint);
693  metricMan->sendMetric("Fragment Rate", stats.recentSampleRate, "fragments/sec", 1, MetricMode::Average);
694  metricMan->sendMetric("Average Fragment Size", (stats.recentValueAverage * sizeof(artdaq::RawDataType)), "bytes/fragment", 2, MetricMode::Average);
695  metricMan->sendMetric("Data Rate", (stats.recentValueRate * sizeof(artdaq::RawDataType)), "bytes/sec", 2, MetricMode::Average);
696  }
697 
698  // 31-Dec-2014, KAB - Just a reminder that using "fragmentCount" in the
699  // denominator of the calculations below is important because the way that
700  // the accumulation of these statistics is done is not fragment-by-fragment
701  // but read-by-read (where each read can contain multiple fragments).
702  // 29-Aug-2016, KAB - BRSYNC_WAIT and OUTPUT_WAIT are now done fragment-by-
703  // fragment, but we'll leave the calculation the same. (The alternative
704  // would be to use recentValueAverage().)
705 
706  mqPtr = artdaq::StatisticsCollection::getInstance().getMonitoredQuantity(INPUT_WAIT_STAT_KEY);
707  if (mqPtr.get() != nullptr)
708  {
709  metricMan->sendMetric("Avg Input Wait Time", (mqPtr->getRecentValueSum() / fragmentCount), "seconds/fragment", 3, MetricMode::Average);
710  }
711 
712  mqPtr = artdaq::StatisticsCollection::getInstance().getMonitoredQuantity(BUFFER_WAIT_STAT_KEY);
713  if (mqPtr.get() != 0)
714  {
715  metricMan->sendMetric("Avg Buffer Wait Time", (mqPtr->getRecentValueSum() / fragmentCount), "seconds/fragment", 3, MetricMode::Average);
716  }
717  mqPtr = artdaq::StatisticsCollection::getInstance().getMonitoredQuantity(REQUEST_WAIT_STAT_KEY);
718  if (mqPtr.get() != 0)
719  {
720  metricMan->sendMetric("Avg Request Response Wait Time", (mqPtr->getRecentValueSum() / fragmentCount), "seconds/fragment", 3, MetricMode::Average);
721  }
722  mqPtr = artdaq::StatisticsCollection::getInstance().getMonitoredQuantity(BRSYNC_WAIT_STAT_KEY);
723  if (mqPtr.get() != nullptr)
724  {
725  metricMan->sendMetric("Avg BoardReader Sync Wait Time", (mqPtr->getRecentValueSum() / fragmentCount), "seconds/fragment", 3, MetricMode::Average);
726  }
727 
728  mqPtr = artdaq::StatisticsCollection::getInstance().getMonitoredQuantity(OUTPUT_WAIT_STAT_KEY);
729  if (mqPtr.get() != nullptr)
730  {
731  metricMan->sendMetric("Avg Output Wait Time", (mqPtr->getRecentValueSum() / fragmentCount), "seconds/fragment", 3, MetricMode::Average);
732  }
733 
734  mqPtr = artdaq::StatisticsCollection::getInstance().getMonitoredQuantity(FRAGMENTS_PER_READ_STAT_KEY);
735  if (mqPtr.get() != nullptr)
736  {
737  metricMan->sendMetric("Avg Frags Per Read", mqPtr->getRecentValueAverage(), "fragments/read", 4, MetricMode::Average);
738  }
739 }
void addMonitoredQuantityName(std::string const &statKey)
Add a MonitoredQuantity name to the list.
Commandable is the base class for all artdaq components which implement the artdaq state machine...
Definition: Commandable.hh:20
bool initialize(fhicl::ParameterSet const &pset, uint64_t timeout, uint64_t timestamp)
Initialize the BoardReaderCore.
static const std::string FRAGMENTS_PROCESSED_STAT_KEY
Key for the Fragments Processed MonitoredQuantity.
bool reinitialize(fhicl::ParameterSet const &pset, uint64_t timeout, uint64_t timestamp)
Reinitialize the BoardReader. No-Op.
static const std::string INPUT_WAIT_STAT_KEY
Key for the Input Wait MonitoredQuantity.
bool stop(uint64_t timeout, uint64_t timestamp)
Stop the BoardReader, and the CommandableFragmentGenerator.
virtual ~BoardReaderCore()
BoardReaderCore Destructor.
static std::string CopyStatusToString(CopyStatus in)
Convert a CopyStatus variable to its string represenatation
BoardReaderCore(Commandable &parent_application)
BoardReaderCore Constructor.
std::unique_ptr< CommandableFragmentGenerator > makeCommandableFragmentGenerator(std::string const &generator_plugin_spec, fhicl::ParameterSet const &ps)
Load a CommandableFragmentGenerator plugin.
FragmentBuffer is a FragmentGenerator-derived abstract class that defines the interface for a Fragmen...
Receive data requests and make them available to CommandableFragmentGenerator or other interested par...
static const std::string BRSYNC_WAIT_STAT_KEY
Key for the Sync Wait MonitoredQuantity.
static const std::string FRAGMENTS_PER_READ_STAT_KEY
Key for the Fragments Per Read MonitoredQuantity.
static const std::string OUTPUT_WAIT_STAT_KEY
Key for the Output Wait MonitoredQuantity.
void send_fragments()
Main working loop of the BoardReaderCore, pt. 2.
bool soft_initialize(fhicl::ParameterSet const &pset, uint64_t timeout, uint64_t timestamp)
Soft-Initialize the BoardReader. No-Op.
std::string report(std::string const &which) const
Send a report on a given run-time quantity.
bool shutdown(uint64_t timeout)
Shutdown the BoardReader, and the CommandableFragmentGenerator.
bool start(art::RunID id, uint64_t timeout, uint64_t timestamp)
Start the BoardReader, and the CommandableFragmentGenerator.
bool resume(uint64_t timeout, uint64_t timestamp)
Resume the BoardReader, and the CommandableFragmentGenerator.
void receive_fragments()
Main working loop of the BoardReaderCore.
bool pause(uint64_t timeout, uint64_t timestamp)
Pause the BoardReader, and the CommandableFragmentGenerator.
bool metaCommand(std::string const &command, std::string const &arg)
Run a user-defined command on the CommandableFragmentGenerator.