1 #define TRACE_NAME "BoardReaderCore"
3 #include "artdaq/Application/TaskType.hh"
4 #include "artdaq/Application/BoardReaderCore.hh"
5 #include "artdaq-core/Data/Fragment.hh"
6 #include "artdaq-core/Utilities/ExceptionHandler.hh"
7 #include "artdaq/Application/makeCommandableFragmentGenerator.hh"
8 #include "canvas/Utilities/Exception.h"
9 #include "cetlib/exception.h"
25 std::unique_ptr<artdaq::DataSenderManager> artdaq::BoardReaderCore::sender_ptr_ =
nullptr;
28 parent_application_(parent_application)
30 , generator_ptr_(nullptr)
31 , stop_requested_(false)
32 , pause_requested_(false)
34 TLOG_DEBUG(app_name) <<
"Constructor" << TLOG_ENDL;
40 metricMan = &metricMan_;
45 TLOG_DEBUG(app_name) <<
"Destructor" << TLOG_ENDL;
50 TLOG_DEBUG(app_name) <<
"initialize method called with " <<
"ParameterSet = \"" << pset.to_string() <<
"\"." << TLOG_ENDL;
53 fhicl::ParameterSet daq_pset;
56 daq_pset = pset.get<fhicl::ParameterSet>(
"daq");
61 <<
"Unable to find the DAQ parameters in the initialization "
62 <<
"ParameterSet: \"" + pset.to_string() +
"\"." << TLOG_ENDL;
65 fhicl::ParameterSet fr_pset;
68 fr_pset = daq_pset.get<fhicl::ParameterSet>(
"fragment_receiver");
74 <<
"Unable to find the fragment_receiver parameters in the DAQ "
75 <<
"initialization ParameterSet: \"" + daq_pset.to_string() +
"\"." << TLOG_ENDL;
80 fhicl::ParameterSet metric_pset;
83 metric_pset = daq_pset.get<fhicl::ParameterSet>(
"metrics");
87 if (metric_pset.is_empty())
89 TLOG_INFO(app_name) <<
"No metric plugins appear to be defined" << TLOG_ENDL;
93 metricMan_.initialize(metric_pset, app_name);
97 ExceptionHandler(ExceptionHandlerRethrow::no,
98 "Error loading metrics in BoardReaderCore::initialize()");
102 std::string frag_gen_name = fr_pset.get<std::string>(
"generator",
"");
103 if (frag_gen_name.length() == 0)
106 <<
"No fragment generator (parameter name = \"generator\") was "
107 <<
"specified in the fragment_receiver ParameterSet. The "
108 <<
"DAQ initialization PSet was \"" << daq_pset.to_string() <<
"\"." << TLOG_ENDL;
118 std::stringstream exception_string;
119 exception_string <<
"Exception thrown during initialization of fragment generator of type \""
120 << frag_gen_name <<
"\"";
122 ExceptionHandler(ExceptionHandlerRethrow::no, exception_string.str());
124 TLOG_DEBUG(app_name) <<
"FHiCL parameter set used to initialize the fragment generator which threw an exception: " << fr_pset.to_string() << TLOG_ENDL;
128 metricMan_.setPrefix(generator_ptr_->metricsReportingInstanceName());
130 rt_priority_ = fr_pset.get<
int>(
"rt_priority", 0);
173 statsHelper_.createCollectors(fr_pset, 100, 30.0, 60.0, FRAGMENTS_PROCESSED_STAT_KEY);
176 skip_seqId_test_ = (generator_ptr_->fragmentIDs().size() > 1);
183 stop_requested_.store(
false);
184 pause_requested_.store(
false);
188 statsHelper_.resetStatistics();
190 metricMan_.do_start();
191 generator_ptr_->StartCmd(
id.run(), timeout, timestamp);
194 TLOG_DEBUG(app_name) <<
"Started run " << run_id_.run() <<
195 ", timeout = " << timeout <<
", timestamp = " << timestamp << TLOG_ENDL;
201 TLOG_DEBUG(app_name) <<
"Stopping run " << run_id_.run()
202 <<
" after " << fragment_count_
203 <<
" fragments." << TLOG_ENDL;
204 stop_requested_.store(
true);
205 generator_ptr_->StopCmd(timeout, timestamp);
211 TLOG_DEBUG(app_name) <<
"Pausing run " << run_id_.run()
212 <<
" after " << fragment_count_
213 <<
" fragments." << TLOG_ENDL;
214 pause_requested_.store(
true);
215 generator_ptr_->PauseCmd(timeout, timestamp);
221 TLOG_DEBUG(app_name) <<
"Resuming run " << run_id_.run() << TLOG_ENDL;
222 pause_requested_.store(
false);
223 metricMan_.do_start();
224 generator_ptr_->ResumeCmd(timeout, timestamp);
230 generator_ptr_->joinThreads();
231 generator_ptr_.reset(
nullptr);
232 metricMan_.shutdown();
238 TLOG_DEBUG(app_name) <<
"soft_initialize method called with "
239 <<
"ParameterSet = \"" << pset.to_string()
240 <<
"\"." << TLOG_ENDL;
246 TLOG_DEBUG(app_name) <<
"reinitialize method called with "
247 <<
"ParameterSet = \"" << pset.to_string()
248 <<
"\"." << TLOG_ENDL;
254 if (rt_priority_ > 0)
256 #pragma GCC diagnostic push
257 #pragma GCC diagnostic ignored "-Wmissing-field-initializers"
258 sched_param s_param = {};
259 s_param.sched_priority = rt_priority_;
260 if (pthread_setschedparam(pthread_self(), SCHED_RR, &s_param))
261 TLOG_WARNING(app_name) <<
"setting realtime priority failed" << TLOG_ENDL;
262 #pragma GCC diagnostic pop
268 if (rt_priority_ > 0)
270 #pragma GCC diagnostic push
271 #pragma GCC diagnostic ignored "-Wmissing-field-initializers"
272 sched_param s_param = {};
273 s_param.sched_priority = rt_priority_;
274 int status = pthread_setschedparam(pthread_self(), SCHED_RR, &s_param);
278 <<
"Failed to set realtime priority to " << rt_priority_
279 <<
", return code = " << status << TLOG_ENDL;
281 #pragma GCC diagnostic pop
284 TLOG_DEBUG(app_name) <<
"Initializing DataSenderManager. my_rank=" << my_rank << TLOG_ENDL;
289 TLOG_DEBUG(app_name) <<
"Waiting for first fragment." << TLOG_ENDL;
290 artdaq::MonitoredQuantityStats::TIME_POINT_T startTime;
292 artdaq::FragmentPtrs frags;
298 startTime = artdaq::MonitoredQuantity::getCurrentTime();
300 TRACE(18, app_name +
"::process_fragments getNext start");
301 active = generator_ptr_->getNext(frags);
302 TRACE(18, app_name +
"::process_fragments getNext done (active=%i)", active);
309 if (!active && generator_ptr_->exception())
311 parent_application_.in_run_failure();
314 delta_time = artdaq::MonitoredQuantity::getCurrentTime() - startTime;
315 statsHelper_.addSample(INPUT_WAIT_STAT_KEY, delta_time);
317 TLOG_ARB(16,app_name) <<
"process_fragments INPUT_WAIT="<<std::to_string( delta_time) << TLOG_ENDL;
319 if (!active) {
break; }
320 statsHelper_.addSample(FRAGMENTS_PER_READ_STAT_KEY, frags.size());
322 for (
auto& fragPtr : frags)
326 TLOG_WARNING(app_name) <<
"Encountered a bad fragment pointer in fragment " << fragment_count_ <<
". "
327 <<
"This is most likely caused by a problem with the Fragment Generator!" << TLOG_ENDL;
330 artdaq::Fragment::sequence_id_t sequence_id = fragPtr->sequenceID();
331 statsHelper_.addSample(FRAGMENTS_PROCESSED_STAT_KEY, fragPtr->size());
333 if ((fragment_count_ % 250) == 0)
336 <<
"Sending fragment " << fragment_count_
337 <<
" (%250) with sequence id " << sequence_id <<
"." << TLOG_ENDL;
437 if (!skip_seqId_test_ && abs(sequence_id - prev_seq_id_) > 1)
439 TLOG_WARNING(app_name)
440 <<
"Missing sequence IDs: current sequence ID = "
441 << sequence_id <<
", previous sequence ID = "
442 << prev_seq_id_ <<
"." << TLOG_ENDL;
444 prev_seq_id_ = sequence_id;
446 startTime = artdaq::MonitoredQuantity::getCurrentTime();
447 TLOG_ARB(17,app_name) <<
"process_fragments seq="<< std::to_string(sequence_id) <<
" sendFragment start" << TLOG_ENDL;
448 auto res = sender_ptr_->sendFragment(std::move(*fragPtr));
449 TLOG_ARB(17, app_name) <<
"process_fragments seq=" << std::to_string(sequence_id) <<
" sendFragment done (res="<< res<<
")"<<TLOG_ENDL;
451 statsHelper_.addSample(OUTPUT_WAIT_STAT_KEY,
452 artdaq::MonitoredQuantity::getCurrentTime() - startTime);
454 bool readyToReport = statsHelper_.readyToReport(fragment_count_);
457 std::string statString = buildStatisticsString_();
458 TLOG_DEBUG(app_name) << statString << TLOG_ENDL;
460 if (fragment_count_ == 1 || readyToReport)
463 <<
"Sending fragment " << fragment_count_
464 <<
" with sequence id " << sequence_id <<
"." << TLOG_ENDL;
467 if (statsHelper_.statsRollingWindowHasMoved()) { sendMetrics_(); }
479 metricMan_.do_stop();
481 sender_ptr_.reset(
nullptr);
486 std::string resultString;
489 if (generator_ptr_.get() != 0)
491 resultString = generator_ptr_->ReportCmd(which);
492 if (resultString.length() > 0) {
return resultString; }
499 std::string tmpString = app_name +
" run number = ";
500 tmpString.append(boost::lexical_cast<std::string>(run_id_.run()));
501 tmpString.append(
". Command=\"" + which +
"\" is not currently supported.");
505 std::string artdaq::BoardReaderCore::buildStatisticsString_()
507 std::ostringstream oss;
508 oss << app_name <<
" statistics:" << std::endl;
510 double fragmentCount = 1.0;
511 artdaq::MonitoredQuantityPtr mqPtr = artdaq::StatisticsCollection::getInstance().
512 getMonitoredQuantity(FRAGMENTS_PROCESSED_STAT_KEY);
513 if (mqPtr.get() != 0)
515 artdaq::MonitoredQuantityStats stats;
516 mqPtr->getStats(stats);
517 oss <<
" Fragment statistics: "
518 << stats.recentSampleCount <<
" fragments received at "
519 << stats.recentSampleRate <<
" fragments/sec, effective data rate = "
520 << (stats.recentValueRate *
sizeof(artdaq::RawDataType)
521 / 1024.0 / 1024.0) <<
" MB/sec, monitor window = "
522 << stats.recentDuration <<
" sec, min::max event size = "
523 << (stats.recentValueMin *
sizeof(artdaq::RawDataType)
526 << (stats.recentValueMax *
sizeof(artdaq::RawDataType)
528 <<
" MB" << std::endl;
529 fragmentCount = std::max(
double(stats.recentSampleCount), 1.0);
530 oss <<
" Average times per fragment: ";
531 if (stats.recentSampleRate > 0.0)
533 oss <<
" elapsed time = "
534 << (1.0 / stats.recentSampleRate) <<
" sec";
546 mqPtr = artdaq::StatisticsCollection::getInstance().
547 getMonitoredQuantity(INPUT_WAIT_STAT_KEY);
548 if (mqPtr.get() != 0)
550 oss <<
", input wait time = "
551 << (mqPtr->getRecentValueSum() / fragmentCount) <<
" sec";
554 mqPtr = artdaq::StatisticsCollection::getInstance().
555 getMonitoredQuantity(BRSYNC_WAIT_STAT_KEY);
556 if (mqPtr.get() != 0)
558 oss <<
", BRsync wait time = "
559 << (mqPtr->getRecentValueSum() / fragmentCount) <<
" sec";
562 mqPtr = artdaq::StatisticsCollection::getInstance().
563 getMonitoredQuantity(OUTPUT_WAIT_STAT_KEY);
564 if (mqPtr.get() != 0)
566 oss <<
", output wait time = "
567 << (mqPtr->getRecentValueSum() / fragmentCount) <<
" sec";
570 oss << std::endl <<
" Fragments per read: ";
571 mqPtr = artdaq::StatisticsCollection::getInstance().
572 getMonitoredQuantity(FRAGMENTS_PER_READ_STAT_KEY);
573 if (mqPtr.get() != 0)
575 artdaq::MonitoredQuantityStats stats;
576 mqPtr->getStats(stats);
578 << stats.recentValueAverage
580 << stats.recentValueMin
582 << stats.recentValueMax;
588 void artdaq::BoardReaderCore::sendMetrics_()
591 double fragmentCount = 1.0;
592 artdaq::MonitoredQuantityPtr mqPtr = artdaq::StatisticsCollection::getInstance().
593 getMonitoredQuantity(FRAGMENTS_PROCESSED_STAT_KEY);
594 if (mqPtr.get() != 0)
596 artdaq::MonitoredQuantityStats stats;
597 mqPtr->getStats(stats);
598 fragmentCount = std::max(
double(stats.recentSampleCount), 1.0);
599 metricMan_.sendMetric(
"Fragment Count", static_cast<unsigned long>(stats.fullSampleCount),
"fragments", 1, MetricMode::Accumulate);
600 metricMan_.sendMetric(
"Fragment Rate", stats.recentSampleRate,
"fragments/sec", 1, MetricMode::Average);
601 metricMan_.sendMetric(
"Average Fragment Size", (stats.recentValueAverage *
sizeof(artdaq::RawDataType)),
"bytes/fragment", 2, MetricMode::Average);
602 metricMan_.sendMetric(
"Data Rate", (stats.recentValueRate *
sizeof(artdaq::RawDataType)),
"bytes/sec", 2, MetricMode::Average);
613 mqPtr = artdaq::StatisticsCollection::getInstance().
614 getMonitoredQuantity(INPUT_WAIT_STAT_KEY);
615 if (mqPtr.get() != 0)
617 metricMan_.sendMetric(
"Avg Input Wait Time", (mqPtr->getRecentValueSum() / fragmentCount),
"seconds/fragment", 3, MetricMode::Average);
620 mqPtr = artdaq::StatisticsCollection::getInstance().
621 getMonitoredQuantity(BRSYNC_WAIT_STAT_KEY);
622 if (mqPtr.get() != 0)
624 metricMan_.sendMetric(
"Avg BoardReader Sync Wait Time", (mqPtr->getRecentValueSum() / fragmentCount),
"seconds/fragment", 3, MetricMode::Average);
627 mqPtr = artdaq::StatisticsCollection::getInstance().
628 getMonitoredQuantity(OUTPUT_WAIT_STAT_KEY);
629 if (mqPtr.get() != 0)
631 metricMan_.sendMetric(
"Avg Output Wait Time", (mqPtr->getRecentValueSum() / fragmentCount),
"seconds/fragment", 3, MetricMode::Average);
634 mqPtr = artdaq::StatisticsCollection::getInstance().
635 getMonitoredQuantity(FRAGMENTS_PER_READ_STAT_KEY);
636 if (mqPtr.get() != 0)
638 metricMan_.sendMetric(
"Avg Frags Per Read", mqPtr->getRecentValueAverage(),
"fragments/read", 4, MetricMode::Average);
void addMonitoredQuantityName(std::string const &statKey)
Add a MonitoredQuantity name to the list.
Commandable is the base class for all artdaq components which implement the artdaq state machine...
Sends Fragment objects using TransferInterface plugins. Uses Routing Tables if confgiured, otherwise will Round-Robin Fragments to the destinations.
static const std::string FRAGMENTS_PROCESSED_STAT_KEY
Key for the Fragments Processed MonitoredQuantity.
static const std::string INPUT_WAIT_STAT_KEY
Key for the Input Wait MonitoredQuantity.
bool stop(uint64_t timeout, uint64_t timestamp)
Stop the BoardReader, and the CommandableFragmentGenerator.
virtual ~BoardReaderCore()
BoardReaderCore Destructor.
BoardReaderCore(Commandable &parent_application)
BoardReaderCore Constructor.
std::unique_ptr< CommandableFragmentGenerator > makeCommandableFragmentGenerator(std::string const &generator_plugin_spec, fhicl::ParameterSet const &ps)
Load a CommandableFragmentGenerator plugin.
bool reinitialize(fhicl::ParameterSet const &pset, uint64_t, uint64_t)
Reinitialize the BoardReader. No-Op.
bool soft_initialize(fhicl::ParameterSet const &pset, uint64_t, uint64_t)
Soft-Initialize the BoardReader. No-Op.
static const std::string BRSYNC_WAIT_STAT_KEY
Key for the Sync Wait MonitoredQuantity.
static const std::string FRAGMENTS_PER_READ_STAT_KEY
Key for the Fragments Per Read MonitoredQuantity.
static const std::string OUTPUT_WAIT_STAT_KEY
Key for the Output Wait MonitoredQuantity.
bool initialize(fhicl::ParameterSet const &pset, uint64_t, uint64_t)
Initialize the BoardReaderCore.
std::string report(std::string const &which) const
Send a report on a given run-time quantity.
void process_fragments()
Main working loop of the BoardReaderCore.
bool start(art::RunID id, uint64_t timeout, uint64_t timestamp)
Start the BoardReader, and the CommandableFragmentGenerator.
bool resume(uint64_t timeout, uint64_t timestamp)
Resume the BoardReader, and the CommandableFragmentGenerator.
bool pause(uint64_t timeout, uint64_t timestamp)
Pause the BoardReader, and the CommandableFragmentGenerator.
bool shutdown(uint64_t)
Shutdown the BoardReader, and the CommandableFragmentGenerator.