1 #define TRACE_NAME "BoardReaderCore"
3 #include "artdaq/Application/TaskType.hh"
4 #include "artdaq/Application/BoardReaderCore.hh"
5 #include "artdaq-core/Data/Fragment.hh"
6 #include "artdaq-core/Utilities/ExceptionHandler.hh"
7 #include "artdaq/Application/makeCommandableFragmentGenerator.hh"
8 #include "canvas/Utilities/Exception.h"
9 #include "cetlib/exception.h"
25 std::unique_ptr<artdaq::DataSenderManager> artdaq::BoardReaderCore::sender_ptr_ =
nullptr;
28 int rank, std::string name) :
29 parent_application_(parent_application)
31 , generator_ptr_(nullptr)
33 , stop_requested_(false)
34 , pause_requested_(false)
36 TLOG_DEBUG(name_) <<
"Constructor" << TLOG_ENDL;
42 metricMan = &metricMan_;
48 TLOG_DEBUG(name_) <<
"Destructor" << TLOG_ENDL;
53 TLOG_DEBUG(name_) <<
"initialize method called with " <<
"ParameterSet = \"" << pset.to_string() <<
"\"." << TLOG_ENDL;
56 fhicl::ParameterSet daq_pset;
59 daq_pset = pset.get<fhicl::ParameterSet>(
"daq");
64 <<
"Unable to find the DAQ parameters in the initialization "
65 <<
"ParameterSet: \"" + pset.to_string() +
"\"." << TLOG_ENDL;
68 fhicl::ParameterSet fr_pset;
71 fr_pset = daq_pset.get<fhicl::ParameterSet>(
"fragment_receiver");
77 <<
"Unable to find the fragment_receiver parameters in the DAQ "
78 <<
"initialization ParameterSet: \"" + daq_pset.to_string() +
"\"." << TLOG_ENDL;
83 fhicl::ParameterSet metric_pset;
86 metric_pset = daq_pset.get<fhicl::ParameterSet>(
"metrics");
90 if (metric_pset.is_empty())
92 TLOG_INFO(name_) <<
"No metric plugins appear to be defined" << TLOG_ENDL;
96 metricMan_.initialize(metric_pset, name_);
100 ExceptionHandler(ExceptionHandlerRethrow::no,
101 "Error loading metrics in BoardReaderCore::initialize()");
105 std::string frag_gen_name = fr_pset.get<std::string>(
"generator",
"");
106 if (frag_gen_name.length() == 0)
109 <<
"No fragment generator (parameter name = \"generator\") was "
110 <<
"specified in the fragment_receiver ParameterSet. The "
111 <<
"DAQ initialization PSet was \"" << daq_pset.to_string() <<
"\"." << TLOG_ENDL;
121 std::stringstream exception_string;
122 exception_string <<
"Exception thrown during initialization of fragment generator of type \""
123 << frag_gen_name <<
"\"";
125 ExceptionHandler(ExceptionHandlerRethrow::no, exception_string.str());
127 TLOG_DEBUG(name_) <<
"FHiCL parameter set used to initialize the fragment generator which threw an exception: " << fr_pset.to_string() << TLOG_ENDL;
131 metricMan_.setPrefix(generator_ptr_->metricsReportingInstanceName());
133 rt_priority_ = fr_pset.get<
int>(
"rt_priority", 0);
176 statsHelper_.createCollectors(fr_pset, 100, 30.0, 60.0, FRAGMENTS_PROCESSED_STAT_KEY);
179 skip_seqId_test_ = (generator_ptr_->fragmentIDs().size() > 1);
186 stop_requested_.store(
false);
187 pause_requested_.store(
false);
191 statsHelper_.resetStatistics();
193 metricMan_.do_start();
194 generator_ptr_->StartCmd(
id.run(), timeout, timestamp);
197 TLOG_DEBUG(name_) <<
"Started run " << run_id_.run() <<
198 ", timeout = " << timeout <<
", timestamp = " << timestamp << TLOG_ENDL;
204 TLOG_DEBUG(name_) <<
"Stopping run " << run_id_.run()
205 <<
" after " << fragment_count_
206 <<
" fragments." << TLOG_ENDL;
207 stop_requested_.store(
true);
208 generator_ptr_->StopCmd(timeout, timestamp);
214 TLOG_DEBUG(name_) <<
"Pausing run " << run_id_.run()
215 <<
" after " << fragment_count_
216 <<
" fragments." << TLOG_ENDL;
217 pause_requested_.store(
true);
218 generator_ptr_->PauseCmd(timeout, timestamp);
224 TLOG_DEBUG(name_) <<
"Resuming run " << run_id_.run() << TLOG_ENDL;
225 pause_requested_.store(
false);
226 metricMan_.do_start();
227 generator_ptr_->ResumeCmd(timeout, timestamp);
233 generator_ptr_->joinThreads();
234 generator_ptr_.reset(
nullptr);
235 metricMan_.shutdown();
241 TLOG_DEBUG(name_) <<
"soft_initialize method called with "
242 <<
"ParameterSet = \"" << pset.to_string()
243 <<
"\"." << TLOG_ENDL;
249 TLOG_DEBUG(name_) <<
"reinitialize method called with "
250 <<
"ParameterSet = \"" << pset.to_string()
251 <<
"\"." << TLOG_ENDL;
257 if (rt_priority_ > 0)
259 #pragma GCC diagnostic push
260 #pragma GCC diagnostic ignored "-Wmissing-field-initializers"
261 sched_param s_param = {};
262 s_param.sched_priority = rt_priority_;
263 if (pthread_setschedparam(pthread_self(), SCHED_RR, &s_param))
264 TLOG_WARNING(name_) <<
"setting realtime priority failed" << TLOG_ENDL;
265 #pragma GCC diagnostic pop
271 if (rt_priority_ > 0)
273 #pragma GCC diagnostic push
274 #pragma GCC diagnostic ignored "-Wmissing-field-initializers"
275 sched_param s_param = {};
276 s_param.sched_priority = rt_priority_;
277 int status = pthread_setschedparam(pthread_self(), SCHED_RR, &s_param);
281 <<
"Failed to set realtime priority to " << rt_priority_
282 <<
", return code = " << status << TLOG_ENDL;
284 #pragma GCC diagnostic pop
287 TLOG_DEBUG(name_) <<
"Initializing DataSenderManager. my_rank=" << my_rank << TLOG_ENDL;
292 TLOG_DEBUG(name_) <<
"Waiting for first fragment." << TLOG_ENDL;
293 artdaq::MonitoredQuantityStats::TIME_POINT_T startTime;
295 artdaq::FragmentPtrs frags;
301 startTime = artdaq::MonitoredQuantity::getCurrentTime();
303 TRACE(18, name_ +
"::process_fragments getNext start");
304 active = generator_ptr_->getNext(frags);
305 TRACE(18, name_ +
"::process_fragments getNext done (active=%i)", active);
312 if (!active && generator_ptr_->exception())
314 parent_application_.in_run_failure();
317 delta_time = artdaq::MonitoredQuantity::getCurrentTime() - startTime;
318 statsHelper_.addSample(INPUT_WAIT_STAT_KEY, delta_time);
320 TLOG_ARB(16,name_) <<
"process_fragments INPUT_WAIT="<<std::to_string( delta_time) << TLOG_ENDL;
322 if (!active) {
break; }
323 statsHelper_.addSample(FRAGMENTS_PER_READ_STAT_KEY, frags.size());
325 for (
auto& fragPtr : frags)
329 TLOG_WARNING(name_) <<
"Encountered a bad fragment pointer in fragment " << fragment_count_ <<
". "
330 <<
"This is most likely caused by a problem with the Fragment Generator!" << TLOG_ENDL;
333 artdaq::Fragment::sequence_id_t sequence_id = fragPtr->sequenceID();
334 statsHelper_.addSample(FRAGMENTS_PROCESSED_STAT_KEY, fragPtr->size());
336 if ((fragment_count_ % 250) == 0)
339 <<
"Sending fragment " << fragment_count_
340 <<
" (%250) with sequence id " << sequence_id <<
"." << TLOG_ENDL;
440 if (!skip_seqId_test_ && abs(sequence_id - prev_seq_id_) > 1)
443 <<
"Missing sequence IDs: current sequence ID = "
444 << sequence_id <<
", previous sequence ID = "
445 << prev_seq_id_ <<
"." << TLOG_ENDL;
447 prev_seq_id_ = sequence_id;
449 startTime = artdaq::MonitoredQuantity::getCurrentTime();
450 TLOG_ARB(17,name_) <<
"process_fragments seq="<< std::to_string(sequence_id) <<
" sendFragment start" << TLOG_ENDL;
451 auto res = sender_ptr_->sendFragment(std::move(*fragPtr));
452 TLOG_ARB(17, name_) <<
"process_fragments seq=" << std::to_string(sequence_id) <<
" sendFragment done (res="<< res<<
")"<<TLOG_ENDL;
454 statsHelper_.addSample(OUTPUT_WAIT_STAT_KEY,
455 artdaq::MonitoredQuantity::getCurrentTime() - startTime);
457 bool readyToReport = statsHelper_.readyToReport(fragment_count_);
460 std::string statString = buildStatisticsString_();
461 TLOG_DEBUG(name_) << statString << TLOG_ENDL;
463 if (fragment_count_ == 1 || readyToReport)
466 <<
"Sending fragment " << fragment_count_
467 <<
" with sequence id " << sequence_id <<
"." << TLOG_ENDL;
470 if (statsHelper_.statsRollingWindowHasMoved()) { sendMetrics_(); }
482 metricMan_.do_stop();
484 sender_ptr_.reset(
nullptr);
489 std::string resultString;
492 if (generator_ptr_.get() != 0)
494 resultString = generator_ptr_->ReportCmd(which);
495 if (resultString.length() > 0) {
return resultString; }
502 std::string tmpString = name_ +
" run number = ";
503 tmpString.append(boost::lexical_cast<std::string>(run_id_.run()));
504 tmpString.append(
". Command=\"" + which +
"\" is not currently supported.");
508 std::string artdaq::BoardReaderCore::buildStatisticsString_()
510 std::ostringstream oss;
511 oss << name_ <<
" statistics:" << std::endl;
513 double fragmentCount = 1.0;
514 artdaq::MonitoredQuantityPtr mqPtr = artdaq::StatisticsCollection::getInstance().
515 getMonitoredQuantity(FRAGMENTS_PROCESSED_STAT_KEY);
516 if (mqPtr.get() != 0)
518 artdaq::MonitoredQuantityStats stats;
519 mqPtr->getStats(stats);
520 oss <<
" Fragment statistics: "
521 << stats.recentSampleCount <<
" fragments received at "
522 << stats.recentSampleRate <<
" fragments/sec, effective data rate = "
523 << (stats.recentValueRate *
sizeof(artdaq::RawDataType)
524 / 1024.0 / 1024.0) <<
" MB/sec, monitor window = "
525 << stats.recentDuration <<
" sec, min::max event size = "
526 << (stats.recentValueMin *
sizeof(artdaq::RawDataType)
529 << (stats.recentValueMax *
sizeof(artdaq::RawDataType)
531 <<
" MB" << std::endl;
532 fragmentCount = std::max(
double(stats.recentSampleCount), 1.0);
533 oss <<
" Average times per fragment: ";
534 if (stats.recentSampleRate > 0.0)
536 oss <<
" elapsed time = "
537 << (1.0 / stats.recentSampleRate) <<
" sec";
549 mqPtr = artdaq::StatisticsCollection::getInstance().
550 getMonitoredQuantity(INPUT_WAIT_STAT_KEY);
551 if (mqPtr.get() != 0)
553 oss <<
", input wait time = "
554 << (mqPtr->getRecentValueSum() / fragmentCount) <<
" sec";
557 mqPtr = artdaq::StatisticsCollection::getInstance().
558 getMonitoredQuantity(BRSYNC_WAIT_STAT_KEY);
559 if (mqPtr.get() != 0)
561 oss <<
", BRsync wait time = "
562 << (mqPtr->getRecentValueSum() / fragmentCount) <<
" sec";
565 mqPtr = artdaq::StatisticsCollection::getInstance().
566 getMonitoredQuantity(OUTPUT_WAIT_STAT_KEY);
567 if (mqPtr.get() != 0)
569 oss <<
", output wait time = "
570 << (mqPtr->getRecentValueSum() / fragmentCount) <<
" sec";
573 oss << std::endl <<
" Fragments per read: ";
574 mqPtr = artdaq::StatisticsCollection::getInstance().
575 getMonitoredQuantity(FRAGMENTS_PER_READ_STAT_KEY);
576 if (mqPtr.get() != 0)
578 artdaq::MonitoredQuantityStats stats;
579 mqPtr->getStats(stats);
581 << stats.recentValueAverage
583 << stats.recentValueMin
585 << stats.recentValueMax;
591 void artdaq::BoardReaderCore::sendMetrics_()
594 double fragmentCount = 1.0;
595 artdaq::MonitoredQuantityPtr mqPtr = artdaq::StatisticsCollection::getInstance().
596 getMonitoredQuantity(FRAGMENTS_PROCESSED_STAT_KEY);
597 if (mqPtr.get() != 0)
599 artdaq::MonitoredQuantityStats stats;
600 mqPtr->getStats(stats);
601 fragmentCount = std::max(
double(stats.recentSampleCount), 1.0);
602 metricMan_.sendMetric(
"Fragment Count", static_cast<unsigned long>(stats.fullSampleCount),
"fragments", 1, MetricMode::Accumulate);
603 metricMan_.sendMetric(
"Fragment Rate", stats.recentSampleRate,
"fragments/sec", 1, MetricMode::Average);
604 metricMan_.sendMetric(
"Average Fragment Size", (stats.recentValueAverage *
sizeof(artdaq::RawDataType)),
"bytes/fragment", 2, MetricMode::Average);
605 metricMan_.sendMetric(
"Data Rate", (stats.recentValueRate *
sizeof(artdaq::RawDataType)),
"bytes/sec", 2, MetricMode::Average);
616 mqPtr = artdaq::StatisticsCollection::getInstance().
617 getMonitoredQuantity(INPUT_WAIT_STAT_KEY);
618 if (mqPtr.get() != 0)
620 metricMan_.sendMetric(
"Avg Input Wait Time", (mqPtr->getRecentValueSum() / fragmentCount),
"seconds/fragment", 3, MetricMode::Average);
623 mqPtr = artdaq::StatisticsCollection::getInstance().
624 getMonitoredQuantity(BRSYNC_WAIT_STAT_KEY);
625 if (mqPtr.get() != 0)
627 metricMan_.sendMetric(
"Avg BoardReader Sync Wait Time", (mqPtr->getRecentValueSum() / fragmentCount),
"seconds/fragment", 3, MetricMode::Average);
630 mqPtr = artdaq::StatisticsCollection::getInstance().
631 getMonitoredQuantity(OUTPUT_WAIT_STAT_KEY);
632 if (mqPtr.get() != 0)
634 metricMan_.sendMetric(
"Avg Output Wait Time", (mqPtr->getRecentValueSum() / fragmentCount),
"seconds/fragment", 3, MetricMode::Average);
637 mqPtr = artdaq::StatisticsCollection::getInstance().
638 getMonitoredQuantity(FRAGMENTS_PER_READ_STAT_KEY);
639 if (mqPtr.get() != 0)
641 metricMan_.sendMetric(
"Avg Frags Per Read", mqPtr->getRecentValueAverage(),
"fragments/read", 4, MetricMode::Average);
BoardReaderCore(Commandable &parent_application, int rank, std::string name)
BoardReaderCore Constructor.
void addMonitoredQuantityName(std::string const &statKey)
Add a MonitoredQuantity name to the list.
Commandable is the base class for all artdaq components which implement the artdaq state machine...
Sends Fragment objects using TransferInterface plugins. Uses Routing Tables if confgiured, otherwise will Round-Robin Fragments to the destinations.
static const std::string FRAGMENTS_PROCESSED_STAT_KEY
Key for the Fragments Processed MonitoredQuantity.
static const std::string INPUT_WAIT_STAT_KEY
Key for the Input Wait MonitoredQuantity.
bool stop(uint64_t timeout, uint64_t timestamp)
Stop the BoardReader, and the CommandableFragmentGenerator.
virtual ~BoardReaderCore()
BoardReaderCore Destructor.
std::unique_ptr< CommandableFragmentGenerator > makeCommandableFragmentGenerator(std::string const &generator_plugin_spec, fhicl::ParameterSet const &ps)
Load a CommandableFragmentGenerator plugin.
bool reinitialize(fhicl::ParameterSet const &pset, uint64_t, uint64_t)
Reinitialize the BoardReader. No-Op.
bool soft_initialize(fhicl::ParameterSet const &pset, uint64_t, uint64_t)
Soft-Initialize the BoardReader. No-Op.
static const std::string BRSYNC_WAIT_STAT_KEY
Key for the Sync Wait MonitoredQuantity.
static const std::string FRAGMENTS_PER_READ_STAT_KEY
Key for the Fragments Per Read MonitoredQuantity.
static const std::string OUTPUT_WAIT_STAT_KEY
Key for the Output Wait MonitoredQuantity.
bool initialize(fhicl::ParameterSet const &pset, uint64_t, uint64_t)
Initialize the BoardReaderCore.
std::string report(std::string const &which) const
Send a report on a given run-time quantity.
void process_fragments()
Main working loop of the BoardReaderCore.
bool start(art::RunID id, uint64_t timeout, uint64_t timestamp)
Start the BoardReader, and the CommandableFragmentGenerator.
bool resume(uint64_t timeout, uint64_t timestamp)
Resume the BoardReader, and the CommandableFragmentGenerator.
bool pause(uint64_t timeout, uint64_t timestamp)
Pause the BoardReader, and the CommandableFragmentGenerator.
bool shutdown(uint64_t)
Shutdown the BoardReader, and the CommandableFragmentGenerator.