artdaq  v3_11_01
BoardReaderCore.cc
1 
2 #include "artdaq/DAQdata/Globals.hh" // include these 2 first -
3 #define TRACE_NAME (app_name + "_BoardReaderCore").c_str()
4 
5 #include "artdaq-core/Data/Fragment.hh"
6 #include "artdaq-core/Utilities/ExceptionHandler.hh"
7 #include "artdaq/Application/BoardReaderCore.hh"
8 #include "artdaq/Application/TaskType.hh"
9 #include "artdaq/Generators/makeCommandableFragmentGenerator.hh"
10 
11 #include <pthread.h>
12 #include <sched.h>
13 #include <algorithm>
14 #include <memory>
15 #include <thread>
16 #include "canvas/Utilities/Exception.h"
17 #include "cetlib_except/exception.h"
18 
19 const std::string artdaq::BoardReaderCore::
20  FRAGMENTS_PROCESSED_STAT_KEY("BoardReaderCoreFragmentsProcessed");
21 const std::string artdaq::BoardReaderCore::
22  INPUT_WAIT_STAT_KEY("BoardReaderCoreInputWaitTime");
23 const std::string artdaq::BoardReaderCore::BUFFER_WAIT_STAT_KEY("BoardReaderCoreBufferWaitTime");
24 const std::string artdaq::BoardReaderCore::REQUEST_WAIT_STAT_KEY("BoardReaderCoreRequestWaitTime");
25 const std::string artdaq::BoardReaderCore::
26  OUTPUT_WAIT_STAT_KEY("BoardReaderCoreOutputWaitTime");
27 const std::string artdaq::BoardReaderCore::
28  FRAGMENTS_PER_READ_STAT_KEY("BoardReaderCoreFragmentsPerRead");
29 
30 std::unique_ptr<artdaq::DataSenderManager> artdaq::BoardReaderCore::sender_ptr_ = nullptr;
31 
33  : parent_application_(parent_application)
34  /*, local_group_comm_(local_group_comm)*/
35  , generator_ptr_(nullptr)
36  , run_id_(art::RunID::flushRun())
37  , fragment_count_(0)
38  , stop_requested_(false)
39  , pause_requested_(false)
40 {
41  TLOG(TLVL_DEBUG) << "Constructor";
48 }
49 
51 {
52  TLOG(TLVL_DEBUG) << "Destructor";
53  TLOG(TLVL_DEBUG) << "Stopping Request Receiver BEGIN";
54  request_receiver_ptr_.reset(nullptr);
55  TLOG(TLVL_DEBUG) << "Stopping Request Receiver END";
56 }
57 
58 bool artdaq::BoardReaderCore::initialize(fhicl::ParameterSet const& pset, uint64_t /*unused*/, uint64_t /*unused*/)
59 {
60  TLOG(TLVL_DEBUG) << "initialize method called with "
61  << "ParameterSet = \"" << pset.to_string() << "\".";
62 
63  // pull out the relevant parts of the ParameterSet
64  fhicl::ParameterSet daq_pset;
65  try
66  {
67  daq_pset = pset.get<fhicl::ParameterSet>("daq");
68  }
69  catch (...)
70  {
71  TLOG(TLVL_ERROR)
72  << "Unable to find the DAQ parameters in the initialization "
73  << "ParameterSet: \"" + pset.to_string() + "\".";
74  return false;
75  }
76  fhicl::ParameterSet fr_pset;
77  try
78  {
79  fr_pset = daq_pset.get<fhicl::ParameterSet>("fragment_receiver");
80  data_pset_ = fr_pset;
81  }
82  catch (...)
83  {
84  TLOG(TLVL_ERROR)
85  << "Unable to find the fragment_receiver parameters in the DAQ "
86  << "initialization ParameterSet: \"" + daq_pset.to_string() + "\".";
87  return false;
88  }
89 
90  // pull out the Metric part of the ParameterSet
91  fhicl::ParameterSet metric_pset;
92  try
93  {
94  metric_pset = daq_pset.get<fhicl::ParameterSet>("metrics");
95  }
96  catch (...)
97  {} // OK if there's no metrics table defined in the FHiCL
98 
99  if (metric_pset.is_empty())
100  {
101  TLOG(TLVL_INFO) << "No metric plugins appear to be defined";
102  }
103  try
104  {
105  metricMan->initialize(metric_pset, app_name);
106  }
107  catch (...)
108  {
109  ExceptionHandler(ExceptionHandlerRethrow::no,
110  "Error loading metrics in BoardReaderCore::initialize()");
111  }
112 
113  if (daq_pset.has_key("rank"))
114  {
115  if (my_rank >= 0 && daq_pset.get<int>("rank") != my_rank)
116  {
117  TLOG(TLVL_WARNING) << "BoardReader rank specified at startup is different than rank specified at configure! Using rank received at configure!";
118  }
119  my_rank = daq_pset.get<int>("rank");
120  }
121  if (my_rank == -1)
122  {
123  TLOG(TLVL_ERROR) << "BoardReader rank not specified at startup or in configuration! Aborting";
124  throw cet::exception("RankNotSpecifiedError") << "BoardReader rank not specified at startup or in configuration! Aborting";
125  }
126 
127  // create the requested CommandableFragmentGenerator
128  auto frag_gen_name = fr_pset.get<std::string>("generator", "");
129  if (frag_gen_name.length() == 0)
130  {
131  TLOG(TLVL_ERROR)
132  << "No fragment generator (parameter name = \"generator\") was "
133  << "specified in the fragment_receiver ParameterSet. The "
134  << "DAQ initialization PSet was \"" << daq_pset.to_string() << "\".";
135  return false;
136  }
137 
138  try
139  {
140  generator_ptr_ = artdaq::makeCommandableFragmentGenerator(frag_gen_name, fr_pset);
141  }
142  catch (...)
143  {
144  std::stringstream exception_string;
145  exception_string << "Exception thrown during initialization of fragment generator of type \""
146  << frag_gen_name << "\"";
147 
148  ExceptionHandler(ExceptionHandlerRethrow::no, exception_string.str());
149 
150  TLOG(TLVL_DEBUG) << "FHiCL parameter set used to initialize the fragment generator which threw an exception: " << fr_pset.to_string();
151 
152  return false;
153  }
154 
155  try
156  {
157  fragment_buffer_ptr_ = std::make_shared<FragmentBuffer>(fr_pset);
158  }
159  catch (...)
160  {
161  std::stringstream exception_string;
162  exception_string << "Exception thrown during initialization of Fragment Buffer";
163 
164  ExceptionHandler(ExceptionHandlerRethrow::no, exception_string.str());
165 
166  TLOG(TLVL_DEBUG) << "FHiCL parameter set used to initialize the fragment buffer which threw an exception: " << fr_pset.to_string();
167 
168  return false;
169  }
170 
171  std::shared_ptr<RequestBuffer> request_buffer = std::make_shared<RequestBuffer>(fr_pset.get<artdaq::Fragment::sequence_id_t>("request_increment", 1));
172 
173  try
174  {
175  request_receiver_ptr_.reset(new RequestReceiver(fr_pset, request_buffer));
176  generator_ptr_->SetRequestBuffer(request_buffer);
177  generator_ptr_->SetFragmentBuffer(fragment_buffer_ptr_);
178  fragment_buffer_ptr_->SetRequestBuffer(request_buffer);
179  }
180  catch (...)
181  {
182  ExceptionHandler(ExceptionHandlerRethrow::no, "Exception thrown during initialization of request receiver");
183 
184  TLOG(TLVL_DEBUG) << "FHiCL parameter set used to initialize the request receiver which threw an exception: " << fr_pset.to_string();
185 
186  return false;
187  }
188  metricMan->setPrefix(generator_ptr_->metricsReportingInstanceName());
189 
190  rt_priority_ = fr_pset.get<int>("rt_priority", 0);
191 
192  // fetch the monitoring parameters and create the MonitoredQuantity instances
193  statsHelper_.createCollectors(fr_pset, 100, 30.0, 60.0, FRAGMENTS_PROCESSED_STAT_KEY);
194 
195  // check if we should skip the sequence ID test...
196  skip_seqId_test_ = (fr_pset.get<bool>("skip_seqID_test", false) || generator_ptr_->fragmentIDs().size() > 1 || fragment_buffer_ptr_->request_mode() != RequestMode::Ignored);
197 
198  verbose_ = fr_pset.get<bool>("verbose", true);
199 
200  return true;
201 }
202 
203 bool artdaq::BoardReaderCore::start(art::RunID id, uint64_t timeout, uint64_t timestamp)
204 {
205  TLOG((verbose_ ? TLVL_INFO : TLVL_DEBUG)) << "Starting run " << id.run();
206  stop_requested_.store(false);
207  pause_requested_.store(false);
208 
209  fragment_count_ = 0;
210  prev_seq_id_ = 0;
211  statsHelper_.resetStatistics();
212 
213  fragment_buffer_ptr_->Reset(false);
214 
215  metricMan->do_start();
216  generator_ptr_->StartCmd(id.run(), timeout, timestamp);
217  run_id_ = id;
218 
219  request_receiver_ptr_->SetRunNumber(static_cast<uint32_t>(id.run()));
220  request_receiver_ptr_->startRequestReception();
221 
222  running_ = true;
223  TLOG((verbose_ ? TLVL_INFO : TLVL_DEBUG)) << "Completed the Start transition (Started run) for run " << run_id_.run()
224  << ", timeout = " << timeout << ", timestamp = " << timestamp;
225  return true;
226 }
227 
228 bool artdaq::BoardReaderCore::stop(uint64_t timeout, uint64_t timestamp)
229 {
230  TLOG((verbose_ ? TLVL_INFO : TLVL_DEBUG)) << "Stopping run " << run_id_.run() << " after " << fragment_count_ << " fragments.";
231  stop_requested_.store(true);
232 
233  TLOG(TLVL_DEBUG) << "Stopping Request reception BEGIN";
234  request_receiver_ptr_->stopRequestReception();
235  TLOG(TLVL_DEBUG) << "Stopping Request reception END";
236 
237  TLOG(TLVL_DEBUG) << "Stopping CommandableFragmentGenerator BEGIN";
238  generator_ptr_->StopCmd(timeout, timestamp);
239  TLOG(TLVL_DEBUG) << "Stopping CommandableFragmentGenerator END";
240 
241  TLOG(TLVL_DEBUG) << "Stopping FragmentBuffer";
242  fragment_buffer_ptr_->Stop();
243 
244  TLOG(TLVL_DEBUG) << "Stopping DataSenderManager";
245  if (sender_ptr_)
246  {
247  sender_ptr_->StopSender();
248  }
249 
250  running_ = false;
251  TLOG((verbose_ ? TLVL_INFO : TLVL_DEBUG)) << "Completed the Stop transition for run " << run_id_.run();
252  return true;
253 }
254 
255 bool artdaq::BoardReaderCore::pause(uint64_t timeout, uint64_t timestamp)
256 {
257  TLOG((verbose_ ? TLVL_INFO : TLVL_DEBUG)) << "Pausing run " << run_id_.run() << " after " << fragment_count_ << " fragments.";
258  pause_requested_.store(true);
259  generator_ptr_->PauseCmd(timeout, timestamp);
260  TLOG((verbose_ ? TLVL_INFO : TLVL_DEBUG)) << "Completed the Pause transition for run " << run_id_.run();
261  return true;
262 }
263 
264 bool artdaq::BoardReaderCore::resume(uint64_t timeout, uint64_t timestamp)
265 {
266  TLOG((verbose_ ? TLVL_INFO : TLVL_DEBUG)) << "Resuming run " << run_id_.run();
267  pause_requested_.store(false);
268  metricMan->do_start();
269  generator_ptr_->ResumeCmd(timeout, timestamp);
270  TLOG((verbose_ ? TLVL_INFO : TLVL_DEBUG)) << "Completed the Resume transition for run " << run_id_.run();
271  return true;
272 }
273 
274 bool artdaq::BoardReaderCore::shutdown(uint64_t /*unused*/)
275 {
276  TLOG((verbose_ ? TLVL_INFO : TLVL_DEBUG)) << "Starting Shutdown transition";
277  generator_ptr_->joinThreads(); // Cleanly shut down the CommandableFragmentGenerator
278  generator_ptr_.reset(nullptr);
279  metricMan->shutdown();
280  TLOG((verbose_ ? TLVL_INFO : TLVL_DEBUG)) << "Completed Shutdown transition";
281  return true;
282 }
283 
284 bool artdaq::BoardReaderCore::soft_initialize(fhicl::ParameterSet const& pset, uint64_t timeout, uint64_t timestamp)
285 {
286  TLOG(TLVL_DEBUG) << "soft_initialize method called with "
287  << "ParameterSet = \"" << pset.to_string()
288  << "\". Forwarding to initialize.";
289  return initialize(pset, timeout, timestamp);
290 }
291 
292 bool artdaq::BoardReaderCore::reinitialize(fhicl::ParameterSet const& pset, uint64_t timeout, uint64_t timestamp)
293 {
294  TLOG(TLVL_DEBUG) << "reinitialize method called with "
295  << "ParameterSet = \"" << pset.to_string()
296  << "\". Forwarding to initalize.";
297  return initialize(pset, timeout, timestamp);
298 }
299 
301 {
302  if (rt_priority_ > 0)
303  {
304 #pragma GCC diagnostic push
305 #pragma GCC diagnostic ignored "-Wmissing-field-initializers"
306  sched_param s_param = {};
307  s_param.sched_priority = rt_priority_;
308  if (pthread_setschedparam(pthread_self(), SCHED_RR, &s_param))
309  TLOG(TLVL_WARNING) << "setting realtime priority failed";
310 #pragma GCC diagnostic pop
311  }
312 
313  // try-catch block here?
314 
315  // how to turn RT PRI off?
316  if (rt_priority_ > 0)
317  {
318 #pragma GCC diagnostic push
319 #pragma GCC diagnostic ignored "-Wmissing-field-initializers"
320  sched_param s_param = {};
321  s_param.sched_priority = rt_priority_;
322  int status = pthread_setschedparam(pthread_self(), SCHED_RR, &s_param);
323  if (status != 0)
324  {
325  TLOG(TLVL_ERROR)
326  << "Failed to set realtime priority to " << rt_priority_
327  << ", return code = " << status;
328  }
329 #pragma GCC diagnostic pop
330  }
331 
332  TLOG(TLVL_DEBUG) << "Waiting for first fragment.";
333  artdaq::MonitoredQuantityStats::TIME_POINT_T startTime, after_input, after_buffer;
334  artdaq::FragmentPtrs frags;
335 
336  receiver_thread_active_ = true;
337 
338  auto wait_start = std::chrono::steady_clock::now();
339  while (!running_ && TimeUtils::GetElapsedTime(wait_start) < start_transition_timeout_)
340  {
341  usleep(10000);
342  }
343  if (!running_)
344  {
345  TLOG(TLVL_ERROR) << "Timeout (" << start_transition_timeout_ << " s) while waiting for Start after receive_fragments thread started!";
346  receiver_thread_active_ = false;
347  }
348 
349  while (receiver_thread_active_)
350  {
351  startTime = artdaq::MonitoredQuantity::getCurrentTime();
352 
353  TLOG(18) << "receive_fragments getNext start";
354  receiver_thread_active_ = generator_ptr_->getNext(frags);
355  TLOG(18) << "receive_fragments getNext done (receiver_thread_active_=" << receiver_thread_active_ << ")";
356 
357  // 08-May-2015, KAB & JCF: if the generator getNext() method returns false
358  // (which indicates that the data flow has stopped) *and* the reason that
359  // it has stopped is because there was an exception that wasn't handled by
360  // the experiment-specific FragmentGenerator class, we move to the
361  // InRunError state so that external observers (e.g. RunControl or
362  // DAQInterface) can see that there was a problem.
363  if (!receiver_thread_active_ && generator_ptr_ && generator_ptr_->exception())
364  {
365  parent_application_.in_run_failure();
366  }
367 
368  after_input = artdaq::MonitoredQuantity::getCurrentTime();
369 
370  if (!receiver_thread_active_) { break; }
371  statsHelper_.addSample(FRAGMENTS_PER_READ_STAT_KEY, frags.size());
372 
373  if (frags.size() > 0)
374  {
375  TLOG(18) << "receive_fragments AddFragmentsToBuffer start";
376  fragment_buffer_ptr_->AddFragmentsToBuffer(std::move(frags));
377  TLOG(18) << "receive_fragments AddFragmentsToBuffer done";
378  }
379 
380  after_buffer = artdaq::MonitoredQuantity::getCurrentTime();
381  TLOG(16) << "receive_fragments INPUT_WAIT=" << (after_input - startTime) << ", BUFFER_WAIT=" << (after_buffer - after_input);
382  statsHelper_.addSample(INPUT_WAIT_STAT_KEY, after_input - startTime);
383  statsHelper_.addSample(BUFFER_WAIT_STAT_KEY, after_buffer - after_input);
384  if (statsHelper_.statsRollingWindowHasMoved()) { sendMetrics_(); }
385  frags.clear();
386  }
387 
388  // 11-May-2015, KAB: call MetricManager::do_stop whenever we exit the
389  // processing fragments loop so that metrics correctly go to zero when
390  // there is no data flowing
391  metricMan->do_stop();
392 
393  TLOG(TLVL_DEBUG) << "receive_fragments loop end";
394 }
396 {
397  if (rt_priority_ > 0)
398  {
399 #pragma GCC diagnostic push
400 #pragma GCC diagnostic ignored "-Wmissing-field-initializers"
401  sched_param s_param = {};
402  s_param.sched_priority = rt_priority_;
403  if (pthread_setschedparam(pthread_self(), SCHED_RR, &s_param) != 0)
404  {
405  TLOG(TLVL_WARNING) << "setting realtime priority failed";
406  }
407 #pragma GCC diagnostic pop
408  }
409 
410  // try-catch block here?
411 
412  // how to turn RT PRI off?
413  if (rt_priority_ > 0)
414  {
415 #pragma GCC diagnostic push
416 #pragma GCC diagnostic ignored "-Wmissing-field-initializers"
417  sched_param s_param = {};
418  s_param.sched_priority = rt_priority_;
419  int status = pthread_setschedparam(pthread_self(), SCHED_RR, &s_param);
420  if (status != 0)
421  {
422  TLOG(TLVL_ERROR)
423  << "Failed to set realtime priority to " << rt_priority_
424  << ", return code = " << status;
425  }
426 #pragma GCC diagnostic pop
427  }
428 
429  TLOG(TLVL_DEBUG) << "Initializing DataSenderManager. my_rank=" << my_rank;
430  sender_ptr_ = std::make_unique<artdaq::DataSenderManager>(data_pset_);
431 
432  TLOG(TLVL_DEBUG) << "Waiting for first fragment.";
433  artdaq::MonitoredQuantityStats::TIME_POINT_T startTime;
434  double delta_time;
435  artdaq::FragmentPtrs frags;
436  auto targetFragCount = generator_ptr_->fragmentIDs().size();
437 
438  sender_thread_active_ = true;
439 
440  auto wait_start = std::chrono::steady_clock::now();
441  while (!running_ && TimeUtils::GetElapsedTime(wait_start) < start_transition_timeout_)
442  {
443  usleep(10000);
444  }
445  if (!running_)
446  {
447  TLOG(TLVL_ERROR) << "Timeout (" << start_transition_timeout_ << " s) while waiting for Start after send_fragments thread started!";
448  sender_thread_active_ = false;
449  }
450 
451  while (sender_thread_active_)
452  {
453  startTime = artdaq::MonitoredQuantity::getCurrentTime();
454 
455  TLOG(18) << "send_fragments applyRequests start";
456  sender_thread_active_ = fragment_buffer_ptr_->applyRequests(frags);
457  TLOG(18) << "send_fragments applyRequests done (sender_thread_active_=" << sender_thread_active_ << ")";
458  // 08-May-2015, KAB & JCF: if the generator getNext() method returns false
459  // (which indicates that the data flow has stopped) *and* the reason that
460  // it has stopped is because there was an exception that wasn't handled by
461  // the experiment-specific FragmentGenerator class, we move to the
462  // InRunError state so that external observers (e.g. RunControl or
463  // DAQInterface) can see that there was a problem.
464  if (!sender_thread_active_ && generator_ptr_ && generator_ptr_->exception())
465  {
466  parent_application_.in_run_failure();
467  }
468 
469  delta_time = artdaq::MonitoredQuantity::getCurrentTime() - startTime;
470 
471  TLOG(16) << "send_fragments REQUEST_WAIT=" << delta_time;
472  statsHelper_.addSample(REQUEST_WAIT_STAT_KEY, delta_time);
473 
474  if (!sender_thread_active_) { break; }
475 
476  for (auto& fragPtr : frags)
477  {
478  if (fragPtr == nullptr)
479  {
480  TLOG(TLVL_WARNING) << "Encountered a bad fragment pointer in fragment " << fragment_count_ << ". "
481  << "This is most likely caused by a problem with the Fragment Generator!";
482  continue;
483  }
484  if (fragment_count_ == 0)
485  {
486  TLOG(TLVL_DEBUG) << "Received first Fragment from Fragment Generator, sequence ID " << fragPtr->sequenceID() << ", size = " << fragPtr->sizeBytes() << " bytes.";
487  }
488 
489  if (fragPtr->type() == Fragment::EndOfRunFragmentType || fragPtr->type() == Fragment::EndOfSubrunFragmentType || fragPtr->type() == Fragment::InitFragmentType)
490  {
491  // Just broadcast any system Fragments in the output
492  artdaq::Fragment::sequence_id_t sequence_id = fragPtr->sequenceID();
493  statsHelper_.addSample(FRAGMENTS_PROCESSED_STAT_KEY, fragPtr->sizeBytes());
494 
495  startTime = artdaq::MonitoredQuantity::getCurrentTime();
496  TLOG(17) << "send_fragments seq=" << sequence_id << " sendFragment start";
497  auto res = sender_ptr_->sendFragment(std::move(*fragPtr));
498  TLOG(17) << "send_fragments seq=" << sequence_id << " sendFragment done (dest=" << res.first << ", sts=" << TransferInterface::CopyStatusToString(res.second) << ")";
499  ++fragment_count_;
500  statsHelper_.addSample(OUTPUT_WAIT_STAT_KEY,
501  artdaq::MonitoredQuantity::getCurrentTime() - startTime);
502  continue;
503  }
504 
505  artdaq::Fragment::sequence_id_t sequence_id = fragPtr->sequenceID();
506  SetMFIteration("Sequence ID " + std::to_string(sequence_id));
507  statsHelper_.addSample(FRAGMENTS_PROCESSED_STAT_KEY, fragPtr->sizeBytes());
508 
509  /*if ((fragment_count_ % 250) == 0)
510  {
511  TLOG(TLVL_DEBUG)
512  << "Sending fragment " << fragment_count_
513  << " with sequence id " << sequence_id << ".";
514  }*/
515 
516  // check for continous sequence IDs
517  if (!skip_seqId_test_ && abs(static_cast<int64_t>(sequence_id) - static_cast<int64_t>(prev_seq_id_)) > 1)
518  {
519  TLOG(TLVL_WARNING)
520  << "Missing sequence IDs: current sequence ID = "
521  << sequence_id << ", previous sequence ID = "
522  << prev_seq_id_ << ".";
523  }
524  prev_seq_id_ = sequence_id;
525 
526  startTime = artdaq::MonitoredQuantity::getCurrentTime();
527  TLOG(17) << "send_fragments seq=" << sequence_id << " sendFragment start";
528  auto res = sender_ptr_->sendFragment(std::move(*fragPtr));
529  if (sender_ptr_->GetSentSequenceIDCount(sequence_id) == targetFragCount)
530  {
531  sender_ptr_->RemoveRoutingTableEntry(sequence_id);
532  }
533  TLOG(17) << "send_fragments seq=" << sequence_id << " sendFragment done (dest=" << res.first << ", sts=" << TransferInterface::CopyStatusToString(res.second) << ")";
534  ++fragment_count_;
535  statsHelper_.addSample(OUTPUT_WAIT_STAT_KEY,
536  artdaq::MonitoredQuantity::getCurrentTime() - startTime);
537 
538  bool readyToReport = statsHelper_.readyToReport();
539  if (readyToReport)
540  {
541  TLOG(TLVL_INFO) << buildStatisticsString_();
542  }
543 
544  // Turn on lvls (mem and/or slow) 3,13,14 to log every send.
545  TLOG(((fragment_count_ == 1) ? TLVL_DEBUG
546  : (((fragment_count_ % 250) == 0 || readyToReport) ? 13 : 14)))
547  << ((fragment_count_ == 1)
548  ? "Sent first Fragment"
549  : "Sending fragment " + std::to_string(fragment_count_))
550  << " with SeqID " << sequence_id << ".";
551  }
552  if (statsHelper_.statsRollingWindowHasMoved()) { sendMetrics_(); }
553  frags.clear();
554  std::this_thread::yield();
555  }
556 
557  sender_ptr_.reset(nullptr);
558 
559  // 11-May-2015, KAB: call MetricManager::do_stop whenever we exit the
560  // processing fragments loop so that metrics correctly go to zero when
561  // there is no data flowing
562  metricMan->do_stop();
563 
564  TLOG(TLVL_DEBUG) << "send_fragments loop end";
565 }
566 
567 std::string artdaq::BoardReaderCore::report(std::string const& which) const
568 {
569  std::string resultString;
570 
571  // pass the request to the FragmentGenerator instance, if it's available
572  if (generator_ptr_ != nullptr && which != "core")
573  {
574  resultString = generator_ptr_->ReportCmd(which);
575  if (resultString.length() > 0) { return resultString; }
576  }
577 
578  // handle the request at this level, if we can
579  // --> nothing here yet
580 
581  // if we haven't been able to come up with any report so far, say so
582  std::string tmpString = app_name + " run number = ";
583  tmpString.append(boost::lexical_cast<std::string>(run_id_.run()));
584 
585  tmpString.append(", Sent Fragment count = ");
586  tmpString.append(boost::lexical_cast<std::string>(fragment_count_));
587 
588  if (!which.empty() && which != "core")
589  {
590  tmpString.append(". Command=\"" + which + "\" is not currently supported.");
591  }
592  return tmpString;
593 }
594 
595 bool artdaq::BoardReaderCore::metaCommand(std::string const& command, std::string const& arg)
596 {
597  TLOG(TLVL_DEBUG) << "metaCommand method called with "
598  << "command = \"" << command << "\""
599  << ", arg = \"" << arg << "\""
600  << ".";
601 
602  if (generator_ptr_)
603  {
604  return generator_ptr_->metaCommand(command, arg);
605  }
606 
607  return true;
608 }
609 
610 std::string artdaq::BoardReaderCore::buildStatisticsString_()
611 {
612  std::ostringstream oss;
613  double fragmentsGeneratedCount = 1.0;
614  double fragmentsOutputCount = 1.0;
615  oss << app_name << " statistics:" << std::endl;
616 
617  oss << " Fragments read: ";
618  artdaq::MonitoredQuantityPtr mqPtr = artdaq::StatisticsCollection::getInstance().getMonitoredQuantity(FRAGMENTS_PER_READ_STAT_KEY);
619  if (mqPtr.get() != nullptr)
620  {
621  artdaq::MonitoredQuantityStats stats;
622  mqPtr->getStats(stats);
623  oss << stats.recentSampleCount << " fragments generated at "
624  << stats.recentSampleRate << " reads/sec, fragment rate = "
625  << stats.recentValueRate << " fragments/sec, monitor window = "
626  << stats.recentDuration << " sec, min::max read size = "
627  << stats.recentValueMin
628  << "::"
629  << stats.recentValueMax
630  << " fragments";
631  fragmentsGeneratedCount = std::max(double(stats.recentSampleCount), 1.0);
632  oss << " Average times per fragment: ";
633  if (stats.recentSampleRate > 0.0)
634  {
635  oss << " elapsed time = "
636  << (1.0 / stats.recentSampleRate) << " sec";
637  }
638  }
639 
640  oss << std::endl;
641  mqPtr = artdaq::StatisticsCollection::getInstance().getMonitoredQuantity(FRAGMENTS_PROCESSED_STAT_KEY);
642  if (mqPtr.get() != nullptr)
643  {
644  artdaq::MonitoredQuantityStats stats;
645  mqPtr->getStats(stats);
646  oss << " Fragment output statistics: "
647  << stats.recentSampleCount << " fragments sent at "
648  << stats.recentSampleRate << " fragments/sec, effective data rate = "
649  << (stats.recentValueRate * sizeof(artdaq::RawDataType) / 1024.0 / 1024.0) << " MB/sec, monitor window = "
650  << stats.recentDuration << " sec, min::max event size = "
651  << (stats.recentValueMin * sizeof(artdaq::RawDataType) / 1024.0 / 1024.0)
652  << "::"
653  << (stats.recentValueMax * sizeof(artdaq::RawDataType) / 1024.0 / 1024.0)
654  << " MB" << std::endl;
655  fragmentsOutputCount = std::max(double(stats.recentSampleCount), 1.0);
656  }
657 
658  // 31-Dec-2014, KAB - Just a reminder that using "fragmentCount" in the
659  // denominator of the calculations below is important because the way that
660  // the accumulation of these statistics is done is not fragment-by-fragment
661  // but read-by-read (where each read can contain multiple fragments).
662  // 29-Aug-2016, KAB - BRSYNC_WAIT and OUTPUT_WAIT are now done fragment-by-
663  // fragment, but we'll leave the calculation the same. (The alternative
664  // would be to use recentValueAverage().)
665 
666  mqPtr = artdaq::StatisticsCollection::getInstance().getMonitoredQuantity(INPUT_WAIT_STAT_KEY);
667  if (mqPtr.get() != nullptr)
668  {
669  oss << " Input wait time = "
670  << (mqPtr->getRecentValueSum() / fragmentsGeneratedCount) << " s/fragment";
671  }
672  mqPtr = artdaq::StatisticsCollection::getInstance().getMonitoredQuantity(BUFFER_WAIT_STAT_KEY);
673  if (mqPtr.get() != 0)
674  {
675  oss << ", buffer wait time = "
676  << (mqPtr->getRecentValueSum() / fragmentsGeneratedCount) << " s/fragment";
677  }
678  mqPtr = artdaq::StatisticsCollection::getInstance().getMonitoredQuantity(REQUEST_WAIT_STAT_KEY);
679  if (mqPtr.get() != 0)
680  {
681  oss << ", request wait time = "
682  << (mqPtr->getRecentValueSum() / fragmentsOutputCount) << " s/fragment";
683  }
684 
685  mqPtr = artdaq::StatisticsCollection::getInstance().getMonitoredQuantity(OUTPUT_WAIT_STAT_KEY);
686  if (mqPtr.get() != nullptr)
687  {
688  oss << ", output wait time = "
689  << (mqPtr->getRecentValueSum() / fragmentsOutputCount) << " s/fragment";
690  }
691 
692  return oss.str();
693 }
694 
695 void artdaq::BoardReaderCore::sendMetrics_()
696 {
697  //TLOG(TLVL_DEBUG) << "Sending metrics " << __LINE__ ;
698  double fragmentCount = 1.0;
699  artdaq::MonitoredQuantityPtr mqPtr = artdaq::StatisticsCollection::getInstance().getMonitoredQuantity(FRAGMENTS_PROCESSED_STAT_KEY);
700  if (mqPtr.get() != nullptr)
701  {
702  artdaq::MonitoredQuantityStats stats;
703  mqPtr->getStats(stats);
704  fragmentCount = std::max(double(stats.recentSampleCount), 1.0);
705  metricMan->sendMetric("Fragment Count", stats.fullSampleCount, "fragments", 1, MetricMode::LastPoint);
706  metricMan->sendMetric("Fragment Rate", stats.recentSampleRate, "fragments/sec", 1, MetricMode::Average);
707  metricMan->sendMetric("Average Fragment Size", (stats.recentValueAverage * sizeof(artdaq::RawDataType)), "bytes/fragment", 2, MetricMode::Average);
708  metricMan->sendMetric("Data Rate", (stats.recentValueRate * sizeof(artdaq::RawDataType)), "bytes/sec", 2, MetricMode::Average);
709  }
710 
711  // 31-Dec-2014, KAB - Just a reminder that using "fragmentCount" in the
712  // denominator of the calculations below is important because the way that
713  // the accumulation of these statistics is done is not fragment-by-fragment
714  // but read-by-read (where each read can contain multiple fragments).
715  // 29-Aug-2016, KAB - BRSYNC_WAIT and OUTPUT_WAIT are now done fragment-by-
716  // fragment, but we'll leave the calculation the same. (The alternative
717  // would be to use recentValueAverage().)
718 
719  mqPtr = artdaq::StatisticsCollection::getInstance().getMonitoredQuantity(INPUT_WAIT_STAT_KEY);
720  if (mqPtr.get() != nullptr)
721  {
722  metricMan->sendMetric("Avg Input Wait Time", (mqPtr->getRecentValueSum() / fragmentCount), "seconds/fragment", 3, MetricMode::Average);
723  }
724 
725  mqPtr = artdaq::StatisticsCollection::getInstance().getMonitoredQuantity(BUFFER_WAIT_STAT_KEY);
726  if (mqPtr.get() != 0)
727  {
728  metricMan->sendMetric("Avg Buffer Wait Time", (mqPtr->getRecentValueSum() / fragmentCount), "seconds/fragment", 3, MetricMode::Average);
729  }
730  mqPtr = artdaq::StatisticsCollection::getInstance().getMonitoredQuantity(REQUEST_WAIT_STAT_KEY);
731  if (mqPtr.get() != 0)
732  {
733  metricMan->sendMetric("Avg Request Response Wait Time", (mqPtr->getRecentValueSum() / fragmentCount), "seconds/fragment", 3, MetricMode::Average);
734  }
735  mqPtr = artdaq::StatisticsCollection::getInstance().getMonitoredQuantity(OUTPUT_WAIT_STAT_KEY);
736  if (mqPtr.get() != nullptr)
737  {
738  metricMan->sendMetric("Avg Output Wait Time", (mqPtr->getRecentValueSum() / fragmentCount), "seconds/fragment", 3, MetricMode::Average);
739  }
740 
741  mqPtr = artdaq::StatisticsCollection::getInstance().getMonitoredQuantity(FRAGMENTS_PER_READ_STAT_KEY);
742  if (mqPtr.get() != nullptr)
743  {
744  metricMan->sendMetric("Avg Frags Per Read", mqPtr->getRecentValueAverage(), "fragments/read", 4, MetricMode::Average);
745  }
746 }
static const std::string BUFFER_WAIT_STAT_KEY
Key for the Fragment Buffer Wait MonitoredQuantity.
void addMonitoredQuantityName(std::string const &statKey)
Add a MonitoredQuantity name to the list.
Commandable is the base class for all artdaq components which implement the artdaq state machine...
Definition: Commandable.hh:20
bool initialize(fhicl::ParameterSet const &pset, uint64_t timeout, uint64_t timestamp)
Initialize the BoardReaderCore.
static const std::string FRAGMENTS_PROCESSED_STAT_KEY
Key for the Fragments Processed MonitoredQuantity.
bool reinitialize(fhicl::ParameterSet const &pset, uint64_t timeout, uint64_t timestamp)
Reinitialize the BoardReader. No-Op.
static const std::string INPUT_WAIT_STAT_KEY
Key for the Input Wait MonitoredQuantity.
bool stop(uint64_t timeout, uint64_t timestamp)
Stop the BoardReader, and the CommandableFragmentGenerator.
virtual ~BoardReaderCore()
BoardReaderCore Destructor.
static std::string CopyStatusToString(CopyStatus in)
Convert a CopyStatus variable to its string represenatation
BoardReaderCore(Commandable &parent_application)
BoardReaderCore Constructor.
std::unique_ptr< CommandableFragmentGenerator > makeCommandableFragmentGenerator(std::string const &generator_plugin_spec, fhicl::ParameterSet const &ps)
Load a CommandableFragmentGenerator plugin.
Receive data requests and make them available to CommandableFragmentGenerator or other interested par...
static const std::string FRAGMENTS_PER_READ_STAT_KEY
Key for the Fragments Per Read MonitoredQuantity.
static const std::string REQUEST_WAIT_STAT_KEY
Key for the Request Buffer Wait MonitoredQuantity.
static const std::string OUTPUT_WAIT_STAT_KEY
Key for the Output Wait MonitoredQuantity.
void send_fragments()
Main working loop of the BoardReaderCore, pt. 2.
bool soft_initialize(fhicl::ParameterSet const &pset, uint64_t timeout, uint64_t timestamp)
Soft-Initialize the BoardReader. No-Op.
std::string report(std::string const &which) const
Send a report on a given run-time quantity.
bool shutdown(uint64_t timeout)
Shutdown the BoardReader, and the CommandableFragmentGenerator.
bool start(art::RunID id, uint64_t timeout, uint64_t timestamp)
Start the BoardReader, and the CommandableFragmentGenerator.
bool resume(uint64_t timeout, uint64_t timestamp)
Resume the BoardReader, and the CommandableFragmentGenerator.
void receive_fragments()
Main working loop of the BoardReaderCore.
bool pause(uint64_t timeout, uint64_t timestamp)
Pause the BoardReader, and the CommandableFragmentGenerator.
bool metaCommand(std::string const &command, std::string const &arg)
Run a user-defined command on the CommandableFragmentGenerator.