artdaq  v3_09_04
BoardReaderCore.cc
1 
2 #include "artdaq/DAQdata/Globals.hh" // include these 2 first -
3 #define TRACE_NAME (app_name + "_BoardReaderCore").c_str()
4 
5 #include "artdaq-core/Data/Fragment.hh"
6 #include "artdaq-core/Utilities/ExceptionHandler.hh"
7 #include "artdaq/Application/BoardReaderCore.hh"
8 #include "artdaq/Application/TaskType.hh"
9 #include "artdaq/Generators/makeCommandableFragmentGenerator.hh"
10 
11 #include <pthread.h>
12 #include <sched.h>
13 #include <algorithm>
14 #include <memory>
15 #include <thread>
16 #include "canvas/Utilities/Exception.h"
17 #include "cetlib_except/exception.h"
18 
19 const std::string artdaq::BoardReaderCore::
20  FRAGMENTS_PROCESSED_STAT_KEY("BoardReaderCoreFragmentsProcessed");
21 const std::string artdaq::BoardReaderCore::
22  INPUT_WAIT_STAT_KEY("BoardReaderCoreInputWaitTime");
23 const std::string artdaq::BoardReaderCore::BUFFER_WAIT_STAT_KEY("BoardReaderCoreBufferWaitTime");
24 const std::string artdaq::BoardReaderCore::REQUEST_WAIT_STAT_KEY("BoardReaderCoreRequestWaitTime");
25 const std::string artdaq::BoardReaderCore::
26  BRSYNC_WAIT_STAT_KEY("BoardReaderCoreBRSyncWaitTime");
27 const std::string artdaq::BoardReaderCore::
28  OUTPUT_WAIT_STAT_KEY("BoardReaderCoreOutputWaitTime");
29 const std::string artdaq::BoardReaderCore::
30  FRAGMENTS_PER_READ_STAT_KEY("BoardReaderCoreFragmentsPerRead");
31 
32 std::unique_ptr<artdaq::DataSenderManager> artdaq::BoardReaderCore::sender_ptr_ = nullptr;
33 
35  : parent_application_(parent_application)
36  /*, local_group_comm_(local_group_comm)*/
37  , generator_ptr_(nullptr)
38  , run_id_(art::RunID::flushRun())
39  , fragment_count_(0)
40  , stop_requested_(false)
41  , pause_requested_(false)
42 {
43  TLOG(TLVL_DEBUG) << "Constructor";
51 }
52 
54 {
55  TLOG(TLVL_DEBUG) << "Destructor";
56  TLOG(TLVL_DEBUG) << "Stopping Request Receiver BEGIN";
57  request_receiver_ptr_.reset(nullptr);
58  TLOG(TLVL_DEBUG) << "Stopping Request Receiver END";
59 }
60 
61 bool artdaq::BoardReaderCore::initialize(fhicl::ParameterSet const& pset, uint64_t /*unused*/, uint64_t /*unused*/)
62 {
63  TLOG(TLVL_DEBUG) << "initialize method called with "
64  << "ParameterSet = \"" << pset.to_string() << "\".";
65 
66  // pull out the relevant parts of the ParameterSet
67  fhicl::ParameterSet daq_pset;
68  try
69  {
70  daq_pset = pset.get<fhicl::ParameterSet>("daq");
71  }
72  catch (...)
73  {
74  TLOG(TLVL_ERROR)
75  << "Unable to find the DAQ parameters in the initialization "
76  << "ParameterSet: \"" + pset.to_string() + "\".";
77  return false;
78  }
79  fhicl::ParameterSet fr_pset;
80  try
81  {
82  fr_pset = daq_pset.get<fhicl::ParameterSet>("fragment_receiver");
83  data_pset_ = fr_pset;
84  }
85  catch (...)
86  {
87  TLOG(TLVL_ERROR)
88  << "Unable to find the fragment_receiver parameters in the DAQ "
89  << "initialization ParameterSet: \"" + daq_pset.to_string() + "\".";
90  return false;
91  }
92 
93  // pull out the Metric part of the ParameterSet
94  fhicl::ParameterSet metric_pset;
95  try
96  {
97  metric_pset = daq_pset.get<fhicl::ParameterSet>("metrics");
98  }
99  catch (...)
100  {} // OK if there's no metrics table defined in the FHiCL
101 
102  if (metric_pset.is_empty())
103  {
104  TLOG(TLVL_INFO) << "No metric plugins appear to be defined";
105  }
106  try
107  {
108  metricMan->initialize(metric_pset, app_name);
109  }
110  catch (...)
111  {
112  ExceptionHandler(ExceptionHandlerRethrow::no,
113  "Error loading metrics in BoardReaderCore::initialize()");
114  }
115 
116  if (daq_pset.has_key("rank"))
117  {
118  if (my_rank >= 0 && daq_pset.get<int>("rank") != my_rank)
119  {
120  TLOG(TLVL_WARNING) << "BoardReader rank specified at startup is different than rank specified at configure! Using rank received at configure!";
121  }
122  my_rank = daq_pset.get<int>("rank");
123  }
124  if (my_rank == -1)
125  {
126  TLOG(TLVL_ERROR) << "BoardReader rank not specified at startup or in configuration! Aborting";
127  throw cet::exception("RankNotSpecifiedError") << "BoardReader rank not specified at startup or in configuration! Aborting";
128  }
129 
130  // create the requested CommandableFragmentGenerator
131  auto frag_gen_name = fr_pset.get<std::string>("generator", "");
132  if (frag_gen_name.length() == 0)
133  {
134  TLOG(TLVL_ERROR)
135  << "No fragment generator (parameter name = \"generator\") was "
136  << "specified in the fragment_receiver ParameterSet. The "
137  << "DAQ initialization PSet was \"" << daq_pset.to_string() << "\".";
138  return false;
139  }
140 
141  try
142  {
143  generator_ptr_ = artdaq::makeCommandableFragmentGenerator(frag_gen_name, fr_pset);
144  }
145  catch (...)
146  {
147  std::stringstream exception_string;
148  exception_string << "Exception thrown during initialization of fragment generator of type \""
149  << frag_gen_name << "\"";
150 
151  ExceptionHandler(ExceptionHandlerRethrow::no, exception_string.str());
152 
153  TLOG(TLVL_DEBUG) << "FHiCL parameter set used to initialize the fragment generator which threw an exception: " << fr_pset.to_string();
154 
155  return false;
156  }
157 
158  try
159  {
160  fragment_buffer_ptr_.reset(new FragmentBuffer(fr_pset));
161  }
162  catch (...)
163  {
164  std::stringstream exception_string;
165  exception_string << "Exception thrown during initialization of Fragment Buffer";
166 
167  ExceptionHandler(ExceptionHandlerRethrow::no, exception_string.str());
168 
169  TLOG(TLVL_DEBUG) << "FHiCL parameter set used to initialize the fragment buffer which threw an exception: " << fr_pset.to_string();
170 
171  return false;
172  }
173 
174  std::shared_ptr<RequestBuffer> request_buffer = std::make_shared<RequestBuffer>(fr_pset.get<artdaq::Fragment::sequence_id_t>("request_increment", 1));
175 
176  try
177  {
178  request_receiver_ptr_.reset(new RequestReceiver(fr_pset, request_buffer));
179  generator_ptr_->SetRequestBuffer(request_buffer);
180  fragment_buffer_ptr_->SetRequestBuffer(request_buffer);
181  }
182  catch (...)
183  {
184  ExceptionHandler(ExceptionHandlerRethrow::no, "Exception thrown during initialization of request receiver");
185 
186  TLOG(TLVL_DEBUG) << "FHiCL parameter set used to initialize the request receiver which threw an exception: " << fr_pset.to_string();
187 
188  return false;
189  }
190  metricMan->setPrefix(generator_ptr_->metricsReportingInstanceName());
191 
192  rt_priority_ = fr_pset.get<int>("rt_priority", 0);
193 
194  // fetch the monitoring parameters and create the MonitoredQuantity instances
195  statsHelper_.createCollectors(fr_pset, 100, 30.0, 60.0, FRAGMENTS_PROCESSED_STAT_KEY);
196 
197  // check if we should skip the sequence ID test...
198  skip_seqId_test_ = (generator_ptr_->fragmentIDs().size() > 1 || fragment_buffer_ptr_->request_mode() != RequestMode::Ignored);
199 
200  verbose_ = fr_pset.get<bool>("verbose", true);
201 
202  return true;
203 }
204 
205 bool artdaq::BoardReaderCore::start(art::RunID id, uint64_t timeout, uint64_t timestamp)
206 {
207  TLOG((verbose_ ? TLVL_INFO : TLVL_DEBUG)) << "Starting run " << id.run();
208  stop_requested_.store(false);
209  pause_requested_.store(false);
210 
211  fragment_count_ = 0;
212  prev_seq_id_ = 0;
213  statsHelper_.resetStatistics();
214 
215  fragment_buffer_ptr_->Reset(false);
216 
217  metricMan->do_start();
218  generator_ptr_->StartCmd(id.run(), timeout, timestamp);
219  run_id_ = id;
220 
221  request_receiver_ptr_->SetRunNumber(static_cast<uint32_t>(id.run()));
222  request_receiver_ptr_->startRequestReception();
223 
224  running_ = true;
225  TLOG((verbose_ ? TLVL_INFO : TLVL_DEBUG)) << "Completed the Start transition (Started run) for run " << run_id_.run()
226  << ", timeout = " << timeout << ", timestamp = " << timestamp;
227  return true;
228 }
229 
230 bool artdaq::BoardReaderCore::stop(uint64_t timeout, uint64_t timestamp)
231 {
232  TLOG((verbose_ ? TLVL_INFO : TLVL_DEBUG)) << "Stopping run " << run_id_.run() << " after " << fragment_count_ << " fragments.";
233  stop_requested_.store(true);
234 
235  TLOG(TLVL_DEBUG) << "Stopping Request reception BEGIN";
236  request_receiver_ptr_->stopRequestReception();
237  TLOG(TLVL_DEBUG) << "Stopping Request reception END";
238 
239  TLOG(TLVL_DEBUG) << "Stopping CommandableFragmentGenerator BEGIN";
240  generator_ptr_->StopCmd(timeout, timestamp);
241  TLOG(TLVL_DEBUG) << "Stopping CommandableFragmentGenerator END";
242 
243  TLOG(TLVL_DEBUG) << "Stopping FragmentBuffer";
244  fragment_buffer_ptr_->Stop();
245 
246  TLOG(TLVL_DEBUG) << "Stopping DataSenderManager";
247  if (sender_ptr_)
248  {
249  sender_ptr_->StopSender();
250  }
251 
252  running_ = false;
253  TLOG((verbose_ ? TLVL_INFO : TLVL_DEBUG)) << "Completed the Stop transition for run " << run_id_.run();
254  return true;
255 }
256 
257 bool artdaq::BoardReaderCore::pause(uint64_t timeout, uint64_t timestamp)
258 {
259  TLOG((verbose_ ? TLVL_INFO : TLVL_DEBUG)) << "Pausing run " << run_id_.run() << " after " << fragment_count_ << " fragments.";
260  pause_requested_.store(true);
261  generator_ptr_->PauseCmd(timeout, timestamp);
262  TLOG((verbose_ ? TLVL_INFO : TLVL_DEBUG)) << "Completed the Pause transition for run " << run_id_.run();
263  return true;
264 }
265 
266 bool artdaq::BoardReaderCore::resume(uint64_t timeout, uint64_t timestamp)
267 {
268  TLOG((verbose_ ? TLVL_INFO : TLVL_DEBUG)) << "Resuming run " << run_id_.run();
269  pause_requested_.store(false);
270  metricMan->do_start();
271  generator_ptr_->ResumeCmd(timeout, timestamp);
272  TLOG((verbose_ ? TLVL_INFO : TLVL_DEBUG)) << "Completed the Resume transition for run " << run_id_.run();
273  return true;
274 }
275 
276 bool artdaq::BoardReaderCore::shutdown(uint64_t /*unused*/)
277 {
278  TLOG((verbose_ ? TLVL_INFO : TLVL_DEBUG)) << "Starting Shutdown transition";
279  generator_ptr_->joinThreads(); // Cleanly shut down the CommandableFragmentGenerator
280  generator_ptr_.reset(nullptr);
281  metricMan->shutdown();
282  TLOG((verbose_ ? TLVL_INFO : TLVL_DEBUG)) << "Completed Shutdown transition";
283  return true;
284 }
285 
286 bool artdaq::BoardReaderCore::soft_initialize(fhicl::ParameterSet const& pset, uint64_t timeout, uint64_t timestamp)
287 {
288  TLOG(TLVL_DEBUG) << "soft_initialize method called with "
289  << "ParameterSet = \"" << pset.to_string()
290  << "\". Forwarding to initialize.";
291  return initialize(pset, timeout, timestamp);
292 }
293 
294 bool artdaq::BoardReaderCore::reinitialize(fhicl::ParameterSet const& pset, uint64_t timeout, uint64_t timestamp)
295 {
296  TLOG(TLVL_DEBUG) << "reinitialize method called with "
297  << "ParameterSet = \"" << pset.to_string()
298  << "\". Forwarding to initalize.";
299  return initialize(pset, timeout, timestamp);
300 }
301 
303 {
304  if (rt_priority_ > 0)
305  {
306 #pragma GCC diagnostic push
307 #pragma GCC diagnostic ignored "-Wmissing-field-initializers"
308  sched_param s_param = {};
309  s_param.sched_priority = rt_priority_;
310  if (pthread_setschedparam(pthread_self(), SCHED_RR, &s_param))
311  TLOG(TLVL_WARNING) << "setting realtime priority failed";
312 #pragma GCC diagnostic pop
313  }
314 
315  // try-catch block here?
316 
317  // how to turn RT PRI off?
318  if (rt_priority_ > 0)
319  {
320 #pragma GCC diagnostic push
321 #pragma GCC diagnostic ignored "-Wmissing-field-initializers"
322  sched_param s_param = {};
323  s_param.sched_priority = rt_priority_;
324  int status = pthread_setschedparam(pthread_self(), SCHED_RR, &s_param);
325  if (status != 0)
326  {
327  TLOG(TLVL_ERROR)
328  << "Failed to set realtime priority to " << rt_priority_
329  << ", return code = " << status;
330  }
331 #pragma GCC diagnostic pop
332  }
333 
334  TLOG(TLVL_DEBUG) << "Waiting for first fragment.";
335  artdaq::MonitoredQuantityStats::TIME_POINT_T startTime, after_input, after_buffer;
336  artdaq::FragmentPtrs frags;
337 
338  receiver_thread_active_ = true;
339 
340  auto wait_start = std::chrono::steady_clock::now();
341  while (!running_ && TimeUtils::GetElapsedTime(wait_start) < start_transition_timeout_)
342  {
343  usleep(10000);
344  }
345  if (!running_)
346  {
347  TLOG(TLVL_ERROR) << "Timeout (" << start_transition_timeout_ << " s) while waiting for Start after receive_fragments thread started!";
348  receiver_thread_active_ = false;
349  }
350 
351  while (receiver_thread_active_)
352  {
353  startTime = artdaq::MonitoredQuantity::getCurrentTime();
354 
355  TLOG(18) << "receive_fragments getNext start";
356  receiver_thread_active_ = generator_ptr_->getNext(frags);
357  TLOG(18) << "receive_fragments getNext done (receiver_thread_active_=" << receiver_thread_active_ << ")";
358 
359  // 08-May-2015, KAB & JCF: if the generator getNext() method returns false
360  // (which indicates that the data flow has stopped) *and* the reason that
361  // it has stopped is because there was an exception that wasn't handled by
362  // the experiment-specific FragmentGenerator class, we move to the
363  // InRunError state so that external observers (e.g. RunControl or
364  // DAQInterface) can see that there was a problem.
365  if (!receiver_thread_active_ && generator_ptr_ && generator_ptr_->exception())
366  {
367  parent_application_.in_run_failure();
368  }
369 
370  after_input = artdaq::MonitoredQuantity::getCurrentTime();
371 
372  if (!receiver_thread_active_) { break; }
373  statsHelper_.addSample(FRAGMENTS_PER_READ_STAT_KEY, frags.size());
374 
375  if (frags.size() > 0)
376  {
377  TLOG(18) << "receive_fragments AddFragmentsToBuffer start";
378  fragment_buffer_ptr_->AddFragmentsToBuffer(std::move(frags));
379  TLOG(18) << "receive_fragments AddFragmentsToBuffer done";
380  }
381 
382  after_buffer = artdaq::MonitoredQuantity::getCurrentTime();
383  TLOG(16) << "receive_fragments INPUT_WAIT=" << (after_input - startTime) << ", BUFFER_WAIT=" << (after_buffer - after_input);
384  statsHelper_.addSample(INPUT_WAIT_STAT_KEY, after_input - startTime);
385  statsHelper_.addSample(BUFFER_WAIT_STAT_KEY, after_buffer - after_input);
386  if (statsHelper_.statsRollingWindowHasMoved()) { sendMetrics_(); }
387  frags.clear();
388  }
389 
390  // 11-May-2015, KAB: call MetricManager::do_stop whenever we exit the
391  // processing fragments loop so that metrics correctly go to zero when
392  // there is no data flowing
393  metricMan->do_stop();
394 
395  TLOG(TLVL_DEBUG) << "receive_fragments loop end";
396 }
398 {
399  if (rt_priority_ > 0)
400  {
401 #pragma GCC diagnostic push
402 #pragma GCC diagnostic ignored "-Wmissing-field-initializers"
403  sched_param s_param = {};
404  s_param.sched_priority = rt_priority_;
405  if (pthread_setschedparam(pthread_self(), SCHED_RR, &s_param) != 0)
406  {
407  TLOG(TLVL_WARNING) << "setting realtime priority failed";
408  }
409 #pragma GCC diagnostic pop
410  }
411 
412  // try-catch block here?
413 
414  // how to turn RT PRI off?
415  if (rt_priority_ > 0)
416  {
417 #pragma GCC diagnostic push
418 #pragma GCC diagnostic ignored "-Wmissing-field-initializers"
419  sched_param s_param = {};
420  s_param.sched_priority = rt_priority_;
421  int status = pthread_setschedparam(pthread_self(), SCHED_RR, &s_param);
422  if (status != 0)
423  {
424  TLOG(TLVL_ERROR)
425  << "Failed to set realtime priority to " << rt_priority_
426  << ", return code = " << status;
427  }
428 #pragma GCC diagnostic pop
429  }
430 
431  TLOG(TLVL_DEBUG) << "Initializing DataSenderManager. my_rank=" << my_rank;
432  sender_ptr_ = std::make_unique<artdaq::DataSenderManager>(data_pset_);
433 
434  TLOG(TLVL_DEBUG) << "Waiting for first fragment.";
435  artdaq::MonitoredQuantityStats::TIME_POINT_T startTime;
436  double delta_time;
437  artdaq::FragmentPtrs frags;
438  auto targetFragCount = generator_ptr_->fragmentIDs().size();
439 
440  sender_thread_active_ = true;
441 
442  auto wait_start = std::chrono::steady_clock::now();
443  while (!running_ && TimeUtils::GetElapsedTime(wait_start) < start_transition_timeout_)
444  {
445  usleep(10000);
446  }
447  if (!running_)
448  {
449  TLOG(TLVL_ERROR) << "Timeout (" << start_transition_timeout_ << " s) while waiting for Start after send_fragments thread started!";
450  sender_thread_active_ = false;
451  }
452 
453  while (sender_thread_active_)
454  {
455  startTime = artdaq::MonitoredQuantity::getCurrentTime();
456 
457  TLOG(18) << "send_fragments applyRequests start";
458  sender_thread_active_ = fragment_buffer_ptr_->applyRequests(frags);
459  TLOG(18) << "send_fragments applyRequests done (sender_thread_active_=" << sender_thread_active_ << ")";
460  // 08-May-2015, KAB & JCF: if the generator getNext() method returns false
461  // (which indicates that the data flow has stopped) *and* the reason that
462  // it has stopped is because there was an exception that wasn't handled by
463  // the experiment-specific FragmentGenerator class, we move to the
464  // InRunError state so that external observers (e.g. RunControl or
465  // DAQInterface) can see that there was a problem.
466  if (!sender_thread_active_ && generator_ptr_ && generator_ptr_->exception())
467  {
468  parent_application_.in_run_failure();
469  }
470 
471  delta_time = artdaq::MonitoredQuantity::getCurrentTime() - startTime;
472 
473  TLOG(16) << "send_fragments REQUEST_WAIT=" << delta_time;
474  statsHelper_.addSample(REQUEST_WAIT_STAT_KEY, delta_time);
475 
476  if (!sender_thread_active_) { break; }
477 
478  for (auto& fragPtr : frags)
479  {
480  if (fragPtr == nullptr)
481  {
482  TLOG(TLVL_WARNING) << "Encountered a bad fragment pointer in fragment " << fragment_count_ << ". "
483  << "This is most likely caused by a problem with the Fragment Generator!";
484  continue;
485  }
486  if (fragment_count_ == 0)
487  {
488  TLOG(TLVL_DEBUG) << "Received first Fragment from Fragment Generator, sequence ID " << fragPtr->sequenceID() << ", size = " << fragPtr->sizeBytes() << " bytes.";
489  }
490 
491  if (fragPtr->type() == Fragment::EndOfRunFragmentType || fragPtr->type() == Fragment::EndOfSubrunFragmentType || fragPtr->type() == Fragment::InitFragmentType)
492  {
493  // Just broadcast any system Fragments in the output
494  artdaq::Fragment::sequence_id_t sequence_id = fragPtr->sequenceID();
495  statsHelper_.addSample(FRAGMENTS_PROCESSED_STAT_KEY, fragPtr->sizeBytes());
496 
497  startTime = artdaq::MonitoredQuantity::getCurrentTime();
498  TLOG(17) << "send_fragments seq=" << sequence_id << " sendFragment start";
499  auto res = sender_ptr_->sendFragment(std::move(*fragPtr));
500  TLOG(17) << "send_fragments seq=" << sequence_id << " sendFragment done (dest=" << res.first << ", sts=" << TransferInterface::CopyStatusToString(res.second) << ")";
501  ++fragment_count_;
502  statsHelper_.addSample(OUTPUT_WAIT_STAT_KEY,
503  artdaq::MonitoredQuantity::getCurrentTime() - startTime);
504  continue;
505  }
506 
507  artdaq::Fragment::sequence_id_t sequence_id = fragPtr->sequenceID();
508  SetMFIteration("Sequence ID " + std::to_string(sequence_id));
509  statsHelper_.addSample(FRAGMENTS_PROCESSED_STAT_KEY, fragPtr->sizeBytes());
510 
511  /*if ((fragment_count_ % 250) == 0)
512  {
513  TLOG(TLVL_DEBUG)
514  << "Sending fragment " << fragment_count_
515  << " with sequence id " << sequence_id << ".";
516  }*/
517 
518  // check for continous sequence IDs
519  if (!skip_seqId_test_ && abs(static_cast<int64_t>(sequence_id) - static_cast<int64_t>(prev_seq_id_)) > 1)
520  {
521  TLOG(TLVL_WARNING)
522  << "Missing sequence IDs: current sequence ID = "
523  << sequence_id << ", previous sequence ID = "
524  << prev_seq_id_ << ".";
525  }
526  prev_seq_id_ = sequence_id;
527 
528  startTime = artdaq::MonitoredQuantity::getCurrentTime();
529  TLOG(17) << "send_fragments seq=" << sequence_id << " sendFragment start";
530  auto res = sender_ptr_->sendFragment(std::move(*fragPtr));
531  if (sender_ptr_->GetSentSequenceIDCount(sequence_id) == targetFragCount)
532  {
533  sender_ptr_->RemoveRoutingTableEntry(sequence_id);
534  }
535  TLOG(17) << "send_fragments seq=" << sequence_id << " sendFragment done (dest=" << res.first << ", sts=" << TransferInterface::CopyStatusToString(res.second) << ")";
536  ++fragment_count_;
537  statsHelper_.addSample(OUTPUT_WAIT_STAT_KEY,
538  artdaq::MonitoredQuantity::getCurrentTime() - startTime);
539 
540  bool readyToReport = statsHelper_.readyToReport();
541  if (readyToReport)
542  {
543  TLOG(TLVL_INFO) << buildStatisticsString_();
544  }
545 
546  // Turn on lvls (mem and/or slow) 3,13,14 to log every send.
547  TLOG(((fragment_count_ == 1) ? TLVL_DEBUG
548  : (((fragment_count_ % 250) == 0 || readyToReport) ? 13 : 14)))
549  << ((fragment_count_ == 1)
550  ? "Sent first Fragment"
551  : "Sending fragment " + std::to_string(fragment_count_))
552  << " with SeqID " << sequence_id << ".";
553  }
554  if (statsHelper_.statsRollingWindowHasMoved()) { sendMetrics_(); }
555  frags.clear();
556  std::this_thread::yield();
557  }
558 
559  sender_ptr_.reset(nullptr);
560 
561  // 11-May-2015, KAB: call MetricManager::do_stop whenever we exit the
562  // processing fragments loop so that metrics correctly go to zero when
563  // there is no data flowing
564  metricMan->do_stop();
565 
566  TLOG(TLVL_DEBUG) << "send_fragments loop end";
567 }
568 
569 std::string artdaq::BoardReaderCore::report(std::string const& which) const
570 {
571  std::string resultString;
572 
573  // pass the request to the FragmentGenerator instance, if it's available
574  if (generator_ptr_ != nullptr && which != "core")
575  {
576  resultString = generator_ptr_->ReportCmd(which);
577  if (resultString.length() > 0) { return resultString; }
578  }
579 
580  // handle the request at this level, if we can
581  // --> nothing here yet
582 
583  // if we haven't been able to come up with any report so far, say so
584  std::string tmpString = app_name + " run number = ";
585  tmpString.append(boost::lexical_cast<std::string>(run_id_.run()));
586 
587  tmpString.append(", Sent Fragment count = ");
588  tmpString.append(boost::lexical_cast<std::string>(fragment_count_));
589 
590  if (!which.empty() && which != "core")
591  {
592  tmpString.append(". Command=\"" + which + "\" is not currently supported.");
593  }
594  return tmpString;
595 }
596 
597 bool artdaq::BoardReaderCore::metaCommand(std::string const& command, std::string const& arg)
598 {
599  TLOG(TLVL_DEBUG) << "metaCommand method called with "
600  << "command = \"" << command << "\""
601  << ", arg = \"" << arg << "\""
602  << ".";
603 
604  if (generator_ptr_)
605  {
606  return generator_ptr_->metaCommand(command, arg);
607  }
608 
609  return true;
610 }
611 
612 std::string artdaq::BoardReaderCore::buildStatisticsString_()
613 {
614  std::ostringstream oss;
615  double fragmentsGeneratedCount = 1.0;
616  double fragmentsOutputCount = 1.0;
617  oss << app_name << " statistics:" << std::endl;
618 
619  oss << " Fragments read: ";
620  artdaq::MonitoredQuantityPtr mqPtr = artdaq::StatisticsCollection::getInstance().getMonitoredQuantity(FRAGMENTS_PER_READ_STAT_KEY);
621  if (mqPtr.get() != nullptr)
622  {
623  artdaq::MonitoredQuantityStats stats;
624  mqPtr->getStats(stats);
625  oss << stats.recentSampleCount << " fragments generated at "
626  << stats.recentSampleRate << " reads/sec, fragment rate = "
627  << stats.recentValueRate << " fragments/sec, monitor window = "
628  << stats.recentDuration << " sec, min::max read size = "
629  << stats.recentValueMin
630  << "::"
631  << stats.recentValueMax
632  << " fragments";
633  fragmentsGeneratedCount = std::max(double(stats.recentSampleCount), 1.0);
634  oss << " Average times per fragment: ";
635  if (stats.recentSampleRate > 0.0)
636  {
637  oss << " elapsed time = "
638  << (1.0 / stats.recentSampleRate) << " sec";
639  }
640  }
641 
642  oss << std::endl;
643  mqPtr = artdaq::StatisticsCollection::getInstance().getMonitoredQuantity(FRAGMENTS_PROCESSED_STAT_KEY);
644  if (mqPtr.get() != nullptr)
645  {
646  artdaq::MonitoredQuantityStats stats;
647  mqPtr->getStats(stats);
648  oss << " Fragment output statistics: "
649  << stats.recentSampleCount << " fragments sent at "
650  << stats.recentSampleRate << " fragments/sec, effective data rate = "
651  << (stats.recentValueRate * sizeof(artdaq::RawDataType) / 1024.0 / 1024.0) << " MB/sec, monitor window = "
652  << stats.recentDuration << " sec, min::max event size = "
653  << (stats.recentValueMin * sizeof(artdaq::RawDataType) / 1024.0 / 1024.0)
654  << "::"
655  << (stats.recentValueMax * sizeof(artdaq::RawDataType) / 1024.0 / 1024.0)
656  << " MB" << std::endl;
657  fragmentsOutputCount = std::max(double(stats.recentSampleCount), 1.0);
658  }
659 
660  // 31-Dec-2014, KAB - Just a reminder that using "fragmentCount" in the
661  // denominator of the calculations below is important because the way that
662  // the accumulation of these statistics is done is not fragment-by-fragment
663  // but read-by-read (where each read can contain multiple fragments).
664  // 29-Aug-2016, KAB - BRSYNC_WAIT and OUTPUT_WAIT are now done fragment-by-
665  // fragment, but we'll leave the calculation the same. (The alternative
666  // would be to use recentValueAverage().)
667 
668  mqPtr = artdaq::StatisticsCollection::getInstance().getMonitoredQuantity(INPUT_WAIT_STAT_KEY);
669  if (mqPtr.get() != nullptr)
670  {
671  oss << " Input wait time = "
672  << (mqPtr->getRecentValueSum() / fragmentsGeneratedCount) << " s/fragment";
673  }
674  mqPtr = artdaq::StatisticsCollection::getInstance().getMonitoredQuantity(BUFFER_WAIT_STAT_KEY);
675  if (mqPtr.get() != 0)
676  {
677  oss << ", buffer wait time = "
678  << (mqPtr->getRecentValueSum() / fragmentsGeneratedCount) << " s/fragment";
679  }
680  mqPtr = artdaq::StatisticsCollection::getInstance().getMonitoredQuantity(REQUEST_WAIT_STAT_KEY);
681  if (mqPtr.get() != 0)
682  {
683  oss << ", request wait time = "
684  << (mqPtr->getRecentValueSum() / fragmentsOutputCount) << " s/fragment";
685  }
686 
687  mqPtr = artdaq::StatisticsCollection::getInstance().getMonitoredQuantity(BRSYNC_WAIT_STAT_KEY);
688  if (mqPtr.get() != nullptr)
689  {
690  oss << ", BRsync wait time = "
691  << (mqPtr->getRecentValueSum() / fragmentsOutputCount) << " s/fragment";
692  }
693 
694  mqPtr = artdaq::StatisticsCollection::getInstance().getMonitoredQuantity(OUTPUT_WAIT_STAT_KEY);
695  if (mqPtr.get() != nullptr)
696  {
697  oss << ", output wait time = "
698  << (mqPtr->getRecentValueSum() / fragmentsOutputCount) << " s/fragment";
699  }
700 
701  return oss.str();
702 }
703 
704 void artdaq::BoardReaderCore::sendMetrics_()
705 {
706  //TLOG(TLVL_DEBUG) << "Sending metrics " << __LINE__ ;
707  double fragmentCount = 1.0;
708  artdaq::MonitoredQuantityPtr mqPtr = artdaq::StatisticsCollection::getInstance().getMonitoredQuantity(FRAGMENTS_PROCESSED_STAT_KEY);
709  if (mqPtr.get() != nullptr)
710  {
711  artdaq::MonitoredQuantityStats stats;
712  mqPtr->getStats(stats);
713  fragmentCount = std::max(double(stats.recentSampleCount), 1.0);
714  metricMan->sendMetric("Fragment Count", stats.fullSampleCount, "fragments", 1, MetricMode::LastPoint);
715  metricMan->sendMetric("Fragment Rate", stats.recentSampleRate, "fragments/sec", 1, MetricMode::Average);
716  metricMan->sendMetric("Average Fragment Size", (stats.recentValueAverage * sizeof(artdaq::RawDataType)), "bytes/fragment", 2, MetricMode::Average);
717  metricMan->sendMetric("Data Rate", (stats.recentValueRate * sizeof(artdaq::RawDataType)), "bytes/sec", 2, MetricMode::Average);
718  }
719 
720  // 31-Dec-2014, KAB - Just a reminder that using "fragmentCount" in the
721  // denominator of the calculations below is important because the way that
722  // the accumulation of these statistics is done is not fragment-by-fragment
723  // but read-by-read (where each read can contain multiple fragments).
724  // 29-Aug-2016, KAB - BRSYNC_WAIT and OUTPUT_WAIT are now done fragment-by-
725  // fragment, but we'll leave the calculation the same. (The alternative
726  // would be to use recentValueAverage().)
727 
728  mqPtr = artdaq::StatisticsCollection::getInstance().getMonitoredQuantity(INPUT_WAIT_STAT_KEY);
729  if (mqPtr.get() != nullptr)
730  {
731  metricMan->sendMetric("Avg Input Wait Time", (mqPtr->getRecentValueSum() / fragmentCount), "seconds/fragment", 3, MetricMode::Average);
732  }
733 
734  mqPtr = artdaq::StatisticsCollection::getInstance().getMonitoredQuantity(BUFFER_WAIT_STAT_KEY);
735  if (mqPtr.get() != 0)
736  {
737  metricMan->sendMetric("Avg Buffer Wait Time", (mqPtr->getRecentValueSum() / fragmentCount), "seconds/fragment", 3, MetricMode::Average);
738  }
739  mqPtr = artdaq::StatisticsCollection::getInstance().getMonitoredQuantity(REQUEST_WAIT_STAT_KEY);
740  if (mqPtr.get() != 0)
741  {
742  metricMan->sendMetric("Avg Request Response Wait Time", (mqPtr->getRecentValueSum() / fragmentCount), "seconds/fragment", 3, MetricMode::Average);
743  }
744  mqPtr = artdaq::StatisticsCollection::getInstance().getMonitoredQuantity(BRSYNC_WAIT_STAT_KEY);
745  if (mqPtr.get() != nullptr)
746  {
747  metricMan->sendMetric("Avg BoardReader Sync Wait Time", (mqPtr->getRecentValueSum() / fragmentCount), "seconds/fragment", 3, MetricMode::Average);
748  }
749 
750  mqPtr = artdaq::StatisticsCollection::getInstance().getMonitoredQuantity(OUTPUT_WAIT_STAT_KEY);
751  if (mqPtr.get() != nullptr)
752  {
753  metricMan->sendMetric("Avg Output Wait Time", (mqPtr->getRecentValueSum() / fragmentCount), "seconds/fragment", 3, MetricMode::Average);
754  }
755 
756  mqPtr = artdaq::StatisticsCollection::getInstance().getMonitoredQuantity(FRAGMENTS_PER_READ_STAT_KEY);
757  if (mqPtr.get() != nullptr)
758  {
759  metricMan->sendMetric("Avg Frags Per Read", mqPtr->getRecentValueAverage(), "fragments/read", 4, MetricMode::Average);
760  }
761 }
static const std::string BUFFER_WAIT_STAT_KEY
Key for the Fragment Buffer Wait MonitoredQuantity.
void addMonitoredQuantityName(std::string const &statKey)
Add a MonitoredQuantity name to the list.
Commandable is the base class for all artdaq components which implement the artdaq state machine...
Definition: Commandable.hh:20
bool initialize(fhicl::ParameterSet const &pset, uint64_t timeout, uint64_t timestamp)
Initialize the BoardReaderCore.
static const std::string FRAGMENTS_PROCESSED_STAT_KEY
Key for the Fragments Processed MonitoredQuantity.
bool reinitialize(fhicl::ParameterSet const &pset, uint64_t timeout, uint64_t timestamp)
Reinitialize the BoardReader. No-Op.
static const std::string INPUT_WAIT_STAT_KEY
Key for the Input Wait MonitoredQuantity.
bool stop(uint64_t timeout, uint64_t timestamp)
Stop the BoardReader, and the CommandableFragmentGenerator.
virtual ~BoardReaderCore()
BoardReaderCore Destructor.
static std::string CopyStatusToString(CopyStatus in)
Convert a CopyStatus variable to its string represenatation
BoardReaderCore(Commandable &parent_application)
BoardReaderCore Constructor.
std::unique_ptr< CommandableFragmentGenerator > makeCommandableFragmentGenerator(std::string const &generator_plugin_spec, fhicl::ParameterSet const &ps)
Load a CommandableFragmentGenerator plugin.
FragmentBuffer is a FragmentGenerator-derived abstract class that defines the interface for a Fragmen...
Receive data requests and make them available to CommandableFragmentGenerator or other interested par...
static const std::string BRSYNC_WAIT_STAT_KEY
Key for the Sync Wait MonitoredQuantity.
static const std::string FRAGMENTS_PER_READ_STAT_KEY
Key for the Fragments Per Read MonitoredQuantity.
static const std::string REQUEST_WAIT_STAT_KEY
Key for the Request Buffer Wait MonitoredQuantity.
static const std::string OUTPUT_WAIT_STAT_KEY
Key for the Output Wait MonitoredQuantity.
void send_fragments()
Main working loop of the BoardReaderCore, pt. 2.
bool soft_initialize(fhicl::ParameterSet const &pset, uint64_t timeout, uint64_t timestamp)
Soft-Initialize the BoardReader. No-Op.
std::string report(std::string const &which) const
Send a report on a given run-time quantity.
bool shutdown(uint64_t timeout)
Shutdown the BoardReader, and the CommandableFragmentGenerator.
bool start(art::RunID id, uint64_t timeout, uint64_t timestamp)
Start the BoardReader, and the CommandableFragmentGenerator.
bool resume(uint64_t timeout, uint64_t timestamp)
Resume the BoardReader, and the CommandableFragmentGenerator.
void receive_fragments()
Main working loop of the BoardReaderCore.
bool pause(uint64_t timeout, uint64_t timestamp)
Pause the BoardReader, and the CommandableFragmentGenerator.
bool metaCommand(std::string const &command, std::string const &arg)
Run a user-defined command on the CommandableFragmentGenerator.