1 #include "artdaq/Application/TaskType.hh"
2 #include "artdaq/Application/BoardReaderCore.hh"
3 #include "artdaq-core/Data/Fragment.hh"
4 #include "artdaq-core/Utilities/ExceptionHandler.hh"
5 #include "artdaq/Application/makeCommandableFragmentGenerator.hh"
6 #include "canvas/Utilities/Exception.h"
7 #include "cetlib/exception.h"
12 #define TRACE_NAME "BoardReaderCore"
25 std::unique_ptr<artdaq::DataSenderManager> artdaq::BoardReaderCore::sender_ptr_ =
nullptr;
28 int rank, std::string name) :
29 parent_application_(parent_application)
31 , generator_ptr_(nullptr)
33 , stop_requested_(false)
34 , pause_requested_(false)
36 TLOG_DEBUG(name_) <<
"Constructor" << TLOG_ENDL;
42 metricMan = &metricMan_;
48 TLOG_DEBUG(name_) <<
"Destructor" << TLOG_ENDL;
53 TLOG_DEBUG(name_) <<
"initialize method called with "
54 <<
"ParameterSet = \"" << pset.to_string()
55 <<
"\"." << TLOG_ENDL;
58 fhicl::ParameterSet daq_pset;
61 daq_pset = pset.get<fhicl::ParameterSet>(
"daq");
66 <<
"Unable to find the DAQ parameters in the initialization "
67 <<
"ParameterSet: \"" + pset.to_string() +
"\"." << TLOG_ENDL;
70 fhicl::ParameterSet fr_pset;
73 fr_pset = daq_pset.get<fhicl::ParameterSet>(
"fragment_receiver");
79 <<
"Unable to find the fragment_receiver parameters in the DAQ "
80 <<
"initialization ParameterSet: \"" + daq_pset.to_string() +
"\"." << TLOG_ENDL;
85 fhicl::ParameterSet metric_pset;
88 metric_pset = daq_pset.get<fhicl::ParameterSet>(
"metrics");
92 if (metric_pset.is_empty())
94 TLOG_INFO(name_) <<
"No metric plugins appear to be defined" << TLOG_ENDL;
98 metricMan_.initialize(metric_pset, name_);
102 ExceptionHandler(ExceptionHandlerRethrow::no,
103 "Error loading metrics in BoardReaderCore::initialize()");
107 std::string frag_gen_name = fr_pset.get<std::string>(
"generator",
"");
108 if (frag_gen_name.length() == 0)
111 <<
"No fragment generator (parameter name = \"generator\") was "
112 <<
"specified in the fragment_receiver ParameterSet. The "
113 <<
"DAQ initialization PSet was \"" << daq_pset.to_string() <<
"\"." << TLOG_ENDL;
123 std::stringstream exception_string;
124 exception_string <<
"Exception thrown during initialization of fragment generator of type \""
125 << frag_gen_name <<
"\"";
127 ExceptionHandler(ExceptionHandlerRethrow::no, exception_string.str());
129 TLOG_DEBUG(name_) <<
"FHiCL parameter set used to initialize the fragment generator which threw an exception: " << fr_pset.to_string() << TLOG_ENDL;
133 metricMan_.setPrefix(generator_ptr_->metricsReportingInstanceName());
135 rt_priority_ = fr_pset.get<
int>(
"rt_priority", 0);
178 statsHelper_.createCollectors(fr_pset, 100, 30.0, 60.0, FRAGMENTS_PROCESSED_STAT_KEY);
181 skip_seqId_test_ = (generator_ptr_->fragmentIDs().size() > 1);
188 stop_requested_.store(
false);
189 pause_requested_.store(
false);
193 statsHelper_.resetStatistics();
195 metricMan_.do_start();
196 generator_ptr_->StartCmd(
id.run(), timeout, timestamp);
199 TLOG_DEBUG(name_) <<
"Started run " << run_id_.run() <<
200 ", timeout = " << timeout <<
", timestamp = " << timestamp << TLOG_ENDL;
206 TLOG_DEBUG(name_) <<
"Stopping run " << run_id_.run()
207 <<
" after " << fragment_count_
208 <<
" fragments." << TLOG_ENDL;
209 stop_requested_.store(
true);
210 generator_ptr_->StopCmd(timeout, timestamp);
216 TLOG_DEBUG(name_) <<
"Pausing run " << run_id_.run()
217 <<
" after " << fragment_count_
218 <<
" fragments." << TLOG_ENDL;
219 pause_requested_.store(
true);
220 generator_ptr_->PauseCmd(timeout, timestamp);
226 TLOG_DEBUG(name_) <<
"Resuming run " << run_id_.run() << TLOG_ENDL;
227 pause_requested_.store(
false);
228 metricMan_.do_start();
229 generator_ptr_->ResumeCmd(timeout, timestamp);
235 generator_ptr_.reset(
nullptr);
236 metricMan_.shutdown();
242 TLOG_DEBUG(name_) <<
"soft_initialize method called with "
243 <<
"ParameterSet = \"" << pset.to_string()
244 <<
"\"." << TLOG_ENDL;
250 TLOG_DEBUG(name_) <<
"reinitialize method called with "
251 <<
"ParameterSet = \"" << pset.to_string()
252 <<
"\"." << TLOG_ENDL;
258 if (rt_priority_ > 0)
260 #pragma GCC diagnostic push
261 #pragma GCC diagnostic ignored "-Wmissing-field-initializers"
262 sched_param s_param = {};
263 s_param.sched_priority = rt_priority_;
264 if (pthread_setschedparam(pthread_self(), SCHED_RR, &s_param))
265 TLOG_WARNING(name_) <<
"setting realtime priority failed" << TLOG_ENDL;
266 #pragma GCC diagnostic pop
272 if (rt_priority_ > 0)
274 #pragma GCC diagnostic push
275 #pragma GCC diagnostic ignored "-Wmissing-field-initializers"
276 sched_param s_param = {};
277 s_param.sched_priority = rt_priority_;
278 int status = pthread_setschedparam(pthread_self(), SCHED_RR, &s_param);
282 <<
"Failed to set realtime priority to " << rt_priority_
283 <<
", return code = " << status << TLOG_ENDL;
285 #pragma GCC diagnostic pop
288 TLOG_DEBUG(name_) <<
"Initializing DataSenderManager. my_rank=" << my_rank << TLOG_ENDL;
293 TLOG_DEBUG(name_) <<
"Waiting for first fragment." << TLOG_ENDL;
294 artdaq::MonitoredQuantityStats::TIME_POINT_T startTime;
296 artdaq::FragmentPtrs frags;
302 startTime = artdaq::MonitoredQuantity::getCurrentTime();
304 TRACE(18, name_ +
"::process_fragments getNext start");
305 active = generator_ptr_->getNext(frags);
306 TRACE(18, name_ +
"::process_fragments getNext done (active=%i)", active);
313 if (!active && generator_ptr_->exception())
315 parent_application_.in_run_failure();
318 delta_time = artdaq::MonitoredQuantity::getCurrentTime() - startTime;
319 statsHelper_.addSample(INPUT_WAIT_STAT_KEY, delta_time);
321 TRACE(16, name_ +
"::process_fragments INPUT_WAIT=%f", delta_time);
323 if (!active) {
break; }
324 statsHelper_.addSample(FRAGMENTS_PER_READ_STAT_KEY, frags.size());
326 for (
auto& fragPtr : frags)
330 TLOG_WARNING(name_) <<
"Encountered a bad fragment pointer in fragment " << fragment_count_ <<
". "
331 <<
"This is most likely caused by a problem with the Fragment Generator!" << TLOG_ENDL;
334 artdaq::Fragment::sequence_id_t sequence_id = fragPtr->sequenceID();
335 statsHelper_.addSample(FRAGMENTS_PROCESSED_STAT_KEY, fragPtr->size());
337 if ((fragment_count_ % 250) == 0)
340 <<
"Sending fragment " << fragment_count_
341 <<
" with sequence id " << sequence_id <<
"." << TLOG_ENDL;
441 if (!skip_seqId_test_ && abs(sequence_id - prev_seq_id_) > 1)
444 <<
"Missing sequence IDs: current sequence ID = "
445 << sequence_id <<
", previous sequence ID = "
446 << prev_seq_id_ <<
"." << TLOG_ENDL;
448 prev_seq_id_ = sequence_id;
450 startTime = artdaq::MonitoredQuantity::getCurrentTime();
451 TRACE(17, name_ +
"::process_fragments seq=%lu sendFragment start", sequence_id);
452 auto res = sender_ptr_->sendFragment(std::move(*fragPtr));
453 TRACE(17, name_ +
"::process_fragments seq=%lu sendFragment done (res=%i)", sequence_id, res);
455 statsHelper_.addSample(OUTPUT_WAIT_STAT_KEY,
456 artdaq::MonitoredQuantity::getCurrentTime() - startTime);
458 bool readyToReport = statsHelper_.readyToReport(fragment_count_);
461 std::string statString = buildStatisticsString_();
462 TLOG_DEBUG(name_) << statString << TLOG_ENDL;
464 if (fragment_count_ == 1 || readyToReport)
467 <<
"Sending fragment " << fragment_count_
468 <<
" with sequence id " << sequence_id <<
"." << TLOG_ENDL;
471 if (statsHelper_.statsRollingWindowHasMoved()) { sendMetrics_(); }
483 metricMan_.do_stop();
485 sender_ptr_.reset(
nullptr);
486 return fragment_count_;
491 std::string resultString;
494 if (generator_ptr_.get() != 0)
496 resultString = generator_ptr_->ReportCmd(which);
497 if (resultString.length() > 0) {
return resultString; }
504 std::string tmpString = name_ +
" run number = ";
505 tmpString.append(boost::lexical_cast<std::string>(run_id_.run()));
506 tmpString.append(
". Command=\"" + which +
"\" is not currently supported.");
510 std::string artdaq::BoardReaderCore::buildStatisticsString_()
512 std::ostringstream oss;
513 oss << name_ <<
" statistics:" << std::endl;
515 double fragmentCount = 1.0;
516 artdaq::MonitoredQuantityPtr mqPtr = artdaq::StatisticsCollection::getInstance().
517 getMonitoredQuantity(FRAGMENTS_PROCESSED_STAT_KEY);
518 if (mqPtr.get() != 0)
520 artdaq::MonitoredQuantityStats stats;
521 mqPtr->getStats(stats);
522 oss <<
" Fragment statistics: "
523 << stats.recentSampleCount <<
" fragments received at "
524 << stats.recentSampleRate <<
" fragments/sec, effective data rate = "
525 << (stats.recentValueRate *
sizeof(artdaq::RawDataType)
526 / 1024.0 / 1024.0) <<
" MB/sec, monitor window = "
527 << stats.recentDuration <<
" sec, min::max event size = "
528 << (stats.recentValueMin *
sizeof(artdaq::RawDataType)
531 << (stats.recentValueMax *
sizeof(artdaq::RawDataType)
533 <<
" MB" << std::endl;
534 fragmentCount = std::max(
double(stats.recentSampleCount), 1.0);
535 oss <<
" Average times per fragment: ";
536 if (stats.recentSampleRate > 0.0)
538 oss <<
" elapsed time = "
539 << (1.0 / stats.recentSampleRate) <<
" sec";
551 mqPtr = artdaq::StatisticsCollection::getInstance().
552 getMonitoredQuantity(INPUT_WAIT_STAT_KEY);
553 if (mqPtr.get() != 0)
555 oss <<
", input wait time = "
556 << (mqPtr->getRecentValueSum() / fragmentCount) <<
" sec";
559 mqPtr = artdaq::StatisticsCollection::getInstance().
560 getMonitoredQuantity(BRSYNC_WAIT_STAT_KEY);
561 if (mqPtr.get() != 0)
563 oss <<
", BRsync wait time = "
564 << (mqPtr->getRecentValueSum() / fragmentCount) <<
" sec";
567 mqPtr = artdaq::StatisticsCollection::getInstance().
568 getMonitoredQuantity(OUTPUT_WAIT_STAT_KEY);
569 if (mqPtr.get() != 0)
571 oss <<
", output wait time = "
572 << (mqPtr->getRecentValueSum() / fragmentCount) <<
" sec";
575 oss << std::endl <<
" Fragments per read: ";
576 mqPtr = artdaq::StatisticsCollection::getInstance().
577 getMonitoredQuantity(FRAGMENTS_PER_READ_STAT_KEY);
578 if (mqPtr.get() != 0)
580 artdaq::MonitoredQuantityStats stats;
581 mqPtr->getStats(stats);
583 << stats.recentValueAverage
585 << stats.recentValueMin
587 << stats.recentValueMax;
593 void artdaq::BoardReaderCore::sendMetrics_()
596 double fragmentCount = 1.0;
597 artdaq::MonitoredQuantityPtr mqPtr = artdaq::StatisticsCollection::getInstance().
598 getMonitoredQuantity(FRAGMENTS_PROCESSED_STAT_KEY);
599 if (mqPtr.get() != 0)
601 artdaq::MonitoredQuantityStats stats;
602 mqPtr->getStats(stats);
603 fragmentCount = std::max(
double(stats.recentSampleCount), 1.0);
604 metricMan_.sendMetric(
"Fragment Count", static_cast<unsigned long>(stats.fullSampleCount),
"fragments", 1, MetricMode::Accumulate);
605 metricMan_.sendMetric(
"Fragment Rate", stats.recentSampleRate,
"fragments/sec", 1, MetricMode::Average);
606 metricMan_.sendMetric(
"Average Fragment Size", (stats.recentValueAverage *
sizeof(artdaq::RawDataType)),
"bytes/fragment", 2, MetricMode::Average);
607 metricMan_.sendMetric(
"Data Rate", (stats.recentValueRate *
sizeof(artdaq::RawDataType)),
"bytes/sec", 2, MetricMode::Average);
618 mqPtr = artdaq::StatisticsCollection::getInstance().
619 getMonitoredQuantity(INPUT_WAIT_STAT_KEY);
620 if (mqPtr.get() != 0)
622 metricMan_.sendMetric(
"Avg Input Wait Time", (mqPtr->getRecentValueSum() / fragmentCount),
"seconds/fragment", 3, MetricMode::Average);
625 mqPtr = artdaq::StatisticsCollection::getInstance().
626 getMonitoredQuantity(BRSYNC_WAIT_STAT_KEY);
627 if (mqPtr.get() != 0)
629 metricMan_.sendMetric(
"Avg BoardReader Sync Wait Time", (mqPtr->getRecentValueSum() / fragmentCount),
"seconds/fragment", 3, MetricMode::Average);
632 mqPtr = artdaq::StatisticsCollection::getInstance().
633 getMonitoredQuantity(OUTPUT_WAIT_STAT_KEY);
634 if (mqPtr.get() != 0)
636 metricMan_.sendMetric(
"Avg Output Wait Time", (mqPtr->getRecentValueSum() / fragmentCount),
"seconds/fragment", 3, MetricMode::Average);
639 mqPtr = artdaq::StatisticsCollection::getInstance().
640 getMonitoredQuantity(FRAGMENTS_PER_READ_STAT_KEY);
641 if (mqPtr.get() != 0)
643 metricMan_.sendMetric(
"Avg Frags Per Read", mqPtr->getRecentValueAverage(),
"fragments/read", 4, MetricMode::Average);
BoardReaderCore(Commandable &parent_application, int rank, std::string name)
BoardReaderCore Constructor.
void addMonitoredQuantityName(std::string const &statKey)
Add a MonitoredQuantity name to the list.
Commandable is the base class for all artdaq components which implement the artdaq state machine...
Sends Fragment objects using TransferInterface plugins. Uses Routing Tables if confgiured, otherwise will Round-Robin Fragments to the destinations.
static const std::string FRAGMENTS_PROCESSED_STAT_KEY
Key for the Fragments Processed MonitoredQuantity.
static const std::string INPUT_WAIT_STAT_KEY
Key for the Input Wait MonitoredQuantity.
bool stop(uint64_t timeout, uint64_t timestamp)
Stop the BoardReader, and the CommandableFragmentGenerator.
virtual ~BoardReaderCore()
BoardReaderCore Destructor.
std::unique_ptr< CommandableFragmentGenerator > makeCommandableFragmentGenerator(std::string const &generator_plugin_spec, fhicl::ParameterSet const &ps)
Load a CommandableFragmentGenerator plugin.
bool reinitialize(fhicl::ParameterSet const &pset, uint64_t, uint64_t)
Reinitialize the BoardReader. No-Op.
bool soft_initialize(fhicl::ParameterSet const &pset, uint64_t, uint64_t)
Soft-Initialize the BoardReader. No-Op.
static const std::string BRSYNC_WAIT_STAT_KEY
Key for the Sync Wait MonitoredQuantity.
static const std::string FRAGMENTS_PER_READ_STAT_KEY
Key for the Fragments Per Read MonitoredQuantity.
static const std::string OUTPUT_WAIT_STAT_KEY
Key for the Output Wait MonitoredQuantity.
bool initialize(fhicl::ParameterSet const &pset, uint64_t, uint64_t)
Initialize the BoardReaderCore.
std::string report(std::string const &which) const
Send a report on a given run-time quantity.
bool start(art::RunID id, uint64_t timeout, uint64_t timestamp)
Start the BoardReader, and the CommandableFragmentGenerator.
bool resume(uint64_t timeout, uint64_t timestamp)
Resume the BoardReader, and the CommandableFragmentGenerator.
bool pause(uint64_t timeout, uint64_t timestamp)
Pause the BoardReader, and the CommandableFragmentGenerator.
bool shutdown(uint64_t)
Shutdown the BoardReader, and the CommandableFragmentGenerator.
size_t process_fragments()
Main working loop of the BoardReaderCore.