1 #include "artdaq/Application/TaskType.hh"
2 #include "artdaq/Application/BoardReaderCore.hh"
3 #include "artdaq-core/Data/Fragment.hh"
4 #include "artdaq-core/Utilities/ExceptionHandler.hh"
5 #include "artdaq/Application/makeCommandableFragmentGenerator.hh"
6 #include "canvas/Utilities/Exception.h"
7 #include "cetlib/exception.h"
12 #define TRACE_NAME "BoardReaderCore"
25 std::unique_ptr<artdaq::DataSenderManager> artdaq::BoardReaderCore::sender_ptr_ =
nullptr;
28 int rank, std::string name) :
29 parent_application_(parent_application)
31 , generator_ptr_(nullptr)
33 , stop_requested_(false)
34 , pause_requested_(false)
36 TLOG_DEBUG(name_) <<
"Constructor" << TLOG_ENDL;
42 metricMan = &metricMan_;
48 TLOG_DEBUG(name_) <<
"Destructor" << TLOG_ENDL;
53 TLOG_DEBUG(name_) <<
"initialize method called with "
54 <<
"ParameterSet = \"" << pset.to_string()
55 <<
"\"." << TLOG_ENDL;
58 fhicl::ParameterSet daq_pset;
61 daq_pset = pset.get<fhicl::ParameterSet>(
"daq");
66 <<
"Unable to find the DAQ parameters in the initialization "
67 <<
"ParameterSet: \"" + pset.to_string() +
"\"." << TLOG_ENDL;
70 fhicl::ParameterSet fr_pset;
73 fr_pset = daq_pset.get<fhicl::ParameterSet>(
"fragment_receiver");
79 <<
"Unable to find the fragment_receiver parameters in the DAQ "
80 <<
"initialization ParameterSet: \"" + daq_pset.to_string() +
"\"." << TLOG_ENDL;
85 fhicl::ParameterSet metric_pset;
88 metric_pset = daq_pset.get<fhicl::ParameterSet>(
"metrics");
92 if (metric_pset.is_empty())
94 TLOG_INFO(name_) <<
"No metric plugins appear to be defined" << TLOG_ENDL;
98 metricMan_.initialize(metric_pset, name_);
102 ExceptionHandler(ExceptionHandlerRethrow::no,
103 "Error loading metrics in BoardReaderCore::initialize()");
107 std::string frag_gen_name = fr_pset.get<std::string>(
"generator",
"");
108 if (frag_gen_name.length() == 0)
111 <<
"No fragment generator (parameter name = \"generator\") was "
112 <<
"specified in the fragment_receiver ParameterSet. The "
113 <<
"DAQ initialization PSet was \"" << daq_pset.to_string() <<
"\"." << TLOG_ENDL;
123 std::stringstream exception_string;
124 exception_string <<
"Exception thrown during initialization of fragment generator of type \""
125 << frag_gen_name <<
"\"";
127 ExceptionHandler(ExceptionHandlerRethrow::no, exception_string.str());
129 TLOG_DEBUG(name_) <<
"FHiCL parameter set used to initialize the fragment generator which threw an exception: " << fr_pset.to_string() << TLOG_ENDL;
133 metricMan_.setPrefix(generator_ptr_->metricsReportingInstanceName());
135 rt_priority_ = fr_pset.get<
int>(
"rt_priority", 0);
178 statsHelper_.createCollectors(fr_pset, 100, 30.0, 60.0, FRAGMENTS_PROCESSED_STAT_KEY);
181 skip_seqId_test_ = (generator_ptr_->fragmentIDs().size() > 1);
188 stop_requested_.store(
false);
189 pause_requested_.store(
false);
193 statsHelper_.resetStatistics();
195 metricMan_.do_start();
196 generator_ptr_->StartCmd(
id.run(), timeout, timestamp);
199 TLOG_DEBUG(name_) <<
"Started run " << run_id_.run() <<
200 ", timeout = " << timeout <<
", timestamp = " << timestamp << TLOG_ENDL;
206 TLOG_DEBUG(name_) <<
"Stopping run " << run_id_.run()
207 <<
" after " << fragment_count_
208 <<
" fragments." << TLOG_ENDL;
209 stop_requested_.store(
true);
210 generator_ptr_->StopCmd(timeout, timestamp);
216 TLOG_DEBUG(name_) <<
"Pausing run " << run_id_.run()
217 <<
" after " << fragment_count_
218 <<
" fragments." << TLOG_ENDL;
219 pause_requested_.store(
true);
220 generator_ptr_->PauseCmd(timeout, timestamp);
226 TLOG_DEBUG(name_) <<
"Resuming run " << run_id_.run() << TLOG_ENDL;
227 pause_requested_.store(
false);
228 metricMan_.do_start();
229 generator_ptr_->ResumeCmd(timeout, timestamp);
235 generator_ptr_.reset(
nullptr);
236 metricMan_.shutdown();
242 TLOG_DEBUG(name_) <<
"soft_initialize method called with "
243 <<
"ParameterSet = \"" << pset.to_string()
244 <<
"\"." << TLOG_ENDL;
250 TLOG_DEBUG(name_) <<
"reinitialize method called with "
251 <<
"ParameterSet = \"" << pset.to_string()
252 <<
"\"." << TLOG_ENDL;
258 if (rt_priority_ > 0)
260 #pragma GCC diagnostic push
261 #pragma GCC diagnostic ignored "-Wmissing-field-initializers"
262 sched_param s_param = {};
263 s_param.sched_priority = rt_priority_;
264 if (pthread_setschedparam(pthread_self(), SCHED_RR, &s_param))
265 TLOG_WARNING(name_) <<
"setting realtime priority failed" << TLOG_ENDL;
266 #pragma GCC diagnostic pop
272 if (rt_priority_ > 0)
274 #pragma GCC diagnostic push
275 #pragma GCC diagnostic ignored "-Wmissing-field-initializers"
276 sched_param s_param = {};
277 s_param.sched_priority = rt_priority_;
278 int status = pthread_setschedparam(pthread_self(), SCHED_RR, &s_param);
282 <<
"Failed to set realtime priority to " << rt_priority_
283 <<
", return code = " << status << TLOG_ENDL;
285 #pragma GCC diagnostic pop
288 TLOG_DEBUG(name_) <<
"Initializing DataSenderManager. my_rank=" << my_rank << TLOG_ENDL;
293 TLOG_DEBUG(name_) <<
"Waiting for first fragment." << TLOG_ENDL;
294 artdaq::MonitoredQuantityStats::TIME_POINT_T startTime;
296 artdaq::FragmentPtrs frags;
302 startTime = artdaq::MonitoredQuantity::getCurrentTime();
304 active = generator_ptr_->getNext(frags);
311 if (!active && generator_ptr_->exception())
313 parent_application_.in_run_failure();
316 delta_time = artdaq::MonitoredQuantity::getCurrentTime() - startTime;
317 statsHelper_.addSample(INPUT_WAIT_STAT_KEY, delta_time);
319 TRACE(16,
"%s::process_fragments INPUT_WAIT=%f", name_.c_str(), delta_time);
321 if (!active) {
break; }
322 statsHelper_.addSample(FRAGMENTS_PER_READ_STAT_KEY, frags.size());
324 for (
auto& fragPtr : frags)
328 TLOG_WARNING(name_) <<
"Encountered a bad fragment pointer in fragment " << fragment_count_ <<
". "
329 <<
"This is most likely caused by a problem with the Fragment Generator!" << TLOG_ENDL;
332 artdaq::Fragment::sequence_id_t sequence_id = fragPtr->sequenceID();
333 statsHelper_.addSample(FRAGMENTS_PROCESSED_STAT_KEY, fragPtr->size());
335 if ((fragment_count_ % 250) == 0)
338 <<
"Sending fragment " << fragment_count_
339 <<
" with sequence id " << sequence_id <<
"." << TLOG_ENDL;
439 if (!skip_seqId_test_ && abs(sequence_id - prev_seq_id_) > 1)
442 <<
"Missing sequence IDs: current sequence ID = "
443 << sequence_id <<
", previous sequence ID = "
444 << prev_seq_id_ <<
"." << TLOG_ENDL;
446 prev_seq_id_ = sequence_id;
448 startTime = artdaq::MonitoredQuantity::getCurrentTime();
449 TRACE(17,
"%s::process_fragments seq=%lu sendFragment start", name_.c_str(), sequence_id);
450 auto res = sender_ptr_->sendFragment(std::move(*fragPtr));
451 TRACE(17,
"%s::process_fragments seq=%lu sendFragment done (res=%i)", name_.c_str(), sequence_id,res);
453 statsHelper_.addSample(OUTPUT_WAIT_STAT_KEY,
454 artdaq::MonitoredQuantity::getCurrentTime() - startTime);
456 bool readyToReport = statsHelper_.readyToReport(fragment_count_);
459 std::string statString = buildStatisticsString_();
460 TLOG_DEBUG(name_) << statString << TLOG_ENDL;
462 if (fragment_count_ == 1 || readyToReport)
465 <<
"Sending fragment " << fragment_count_
466 <<
" with sequence id " << sequence_id <<
"." << TLOG_ENDL;
469 if (statsHelper_.statsRollingWindowHasMoved()) { sendMetrics_(); }
481 metricMan_.do_stop();
483 sender_ptr_.reset(
nullptr);
484 return fragment_count_;
489 std::string resultString;
492 if (generator_ptr_.get() != 0)
494 resultString = generator_ptr_->ReportCmd(which);
495 if (resultString.length() > 0) {
return resultString; }
502 std::string tmpString = name_ +
" run number = ";
503 tmpString.append(boost::lexical_cast<std::string>(run_id_.run()));
504 tmpString.append(
". Command=\"" + which +
"\" is not currently supported.");
508 std::string artdaq::BoardReaderCore::buildStatisticsString_()
510 std::ostringstream oss;
511 oss << name_ <<
" statistics:" << std::endl;
513 double fragmentCount = 1.0;
514 artdaq::MonitoredQuantityPtr mqPtr = artdaq::StatisticsCollection::getInstance().
515 getMonitoredQuantity(FRAGMENTS_PROCESSED_STAT_KEY);
516 if (mqPtr.get() != 0)
518 artdaq::MonitoredQuantityStats stats;
519 mqPtr->getStats(stats);
520 oss <<
" Fragment statistics: "
521 << stats.recentSampleCount <<
" fragments received at "
522 << stats.recentSampleRate <<
" fragments/sec, effective data rate = "
523 << (stats.recentValueRate *
sizeof(artdaq::RawDataType)
524 / 1024.0 / 1024.0) <<
" MB/sec, monitor window = "
525 << stats.recentDuration <<
" sec, min::max event size = "
526 << (stats.recentValueMin *
sizeof(artdaq::RawDataType)
529 << (stats.recentValueMax *
sizeof(artdaq::RawDataType)
531 <<
" MB" << std::endl;
532 fragmentCount = std::max(
double(stats.recentSampleCount), 1.0);
533 oss <<
" Average times per fragment: ";
534 if (stats.recentSampleRate > 0.0)
536 oss <<
" elapsed time = "
537 << (1.0 / stats.recentSampleRate) <<
" sec";
549 mqPtr = artdaq::StatisticsCollection::getInstance().
550 getMonitoredQuantity(INPUT_WAIT_STAT_KEY);
551 if (mqPtr.get() != 0)
553 oss <<
", input wait time = "
554 << (mqPtr->getRecentValueSum() / fragmentCount) <<
" sec";
557 mqPtr = artdaq::StatisticsCollection::getInstance().
558 getMonitoredQuantity(BRSYNC_WAIT_STAT_KEY);
559 if (mqPtr.get() != 0)
561 oss <<
", BRsync wait time = "
562 << (mqPtr->getRecentValueSum() / fragmentCount) <<
" sec";
565 mqPtr = artdaq::StatisticsCollection::getInstance().
566 getMonitoredQuantity(OUTPUT_WAIT_STAT_KEY);
567 if (mqPtr.get() != 0)
569 oss <<
", output wait time = "
570 << (mqPtr->getRecentValueSum() / fragmentCount) <<
" sec";
573 oss << std::endl <<
" Fragments per read: ";
574 mqPtr = artdaq::StatisticsCollection::getInstance().
575 getMonitoredQuantity(FRAGMENTS_PER_READ_STAT_KEY);
576 if (mqPtr.get() != 0)
578 artdaq::MonitoredQuantityStats stats;
579 mqPtr->getStats(stats);
581 << stats.recentValueAverage
583 << stats.recentValueMin
585 << stats.recentValueMax;
591 void artdaq::BoardReaderCore::sendMetrics_()
594 double fragmentCount = 1.0;
595 artdaq::MonitoredQuantityPtr mqPtr = artdaq::StatisticsCollection::getInstance().
596 getMonitoredQuantity(FRAGMENTS_PROCESSED_STAT_KEY);
597 if (mqPtr.get() != 0)
599 artdaq::MonitoredQuantityStats stats;
600 mqPtr->getStats(stats);
601 fragmentCount = std::max(
double(stats.recentSampleCount), 1.0);
602 metricMan_.sendMetric(
"Fragment Count",
603 static_cast<unsigned long>(stats.fullSampleCount),
605 metricMan_.sendMetric(
"Fragment Rate",
606 stats.recentSampleRate,
"fragments/sec", 1);
607 metricMan_.sendMetric(
"Average Fragment Size",
608 (stats.recentValueAverage *
sizeof(artdaq::RawDataType)
609 ),
"bytes/fragment", 2);
610 metricMan_.sendMetric(
"Data Rate",
611 (stats.recentValueRate *
sizeof(artdaq::RawDataType)
623 mqPtr = artdaq::StatisticsCollection::getInstance().
624 getMonitoredQuantity(INPUT_WAIT_STAT_KEY);
625 if (mqPtr.get() != 0)
627 metricMan_.sendMetric(
"Avg Input Wait Time",
628 (mqPtr->getRecentValueSum() / fragmentCount),
629 "seconds/fragment", 3,
false);
632 mqPtr = artdaq::StatisticsCollection::getInstance().
633 getMonitoredQuantity(BRSYNC_WAIT_STAT_KEY);
634 if (mqPtr.get() != 0)
636 metricMan_.sendMetric(
"Avg BoardReader Sync Wait Time",
637 (mqPtr->getRecentValueSum() / fragmentCount),
638 "seconds/fragment", 3,
false);
641 mqPtr = artdaq::StatisticsCollection::getInstance().
642 getMonitoredQuantity(OUTPUT_WAIT_STAT_KEY);
643 if (mqPtr.get() != 0)
645 metricMan_.sendMetric(
"Avg Output Wait Time",
646 (mqPtr->getRecentValueSum() / fragmentCount),
647 "seconds/fragment", 3,
false);
650 mqPtr = artdaq::StatisticsCollection::getInstance().
651 getMonitoredQuantity(FRAGMENTS_PER_READ_STAT_KEY);
652 if (mqPtr.get() != 0)
654 metricMan_.sendMetric(
"Avg Frags Per Read",
655 mqPtr->getRecentValueAverage(),
"fragments/read", 4,
false);
BoardReaderCore(Commandable &parent_application, int rank, std::string name)
BoardReaderCore Constructor.
void addMonitoredQuantityName(std::string const &statKey)
Add a MonitoredQuantity name to the list.
Commandable is the base class for all artdaq components which implement the artdaq state machine...
Sends Fragment objects using TransferInterface plugins. Uses Routing Tables if confgiured, otherwise will Round-Robin Fragments to the destinations.
static const std::string FRAGMENTS_PROCESSED_STAT_KEY
Key for the Fragments Processed MonitoredQuantity.
static const std::string INPUT_WAIT_STAT_KEY
Key for the Input Wait MonitoredQuantity.
bool stop(uint64_t timeout, uint64_t timestamp)
Stop the BoardReader, and the CommandableFragmentGenerator.
virtual ~BoardReaderCore()
BoardReaderCore Destructor.
std::unique_ptr< CommandableFragmentGenerator > makeCommandableFragmentGenerator(std::string const &generator_plugin_spec, fhicl::ParameterSet const &ps)
Load a CommandableFragmentGenerator plugin.
bool reinitialize(fhicl::ParameterSet const &pset, uint64_t, uint64_t)
Reinitialize the BoardReader. No-Op.
bool soft_initialize(fhicl::ParameterSet const &pset, uint64_t, uint64_t)
Soft-Initialize the BoardReader. No-Op.
static const std::string BRSYNC_WAIT_STAT_KEY
Key for the Sync Wait MonitoredQuantity.
static const std::string FRAGMENTS_PER_READ_STAT_KEY
Key for the Fragments Per Read MonitoredQuantity.
static const std::string OUTPUT_WAIT_STAT_KEY
Key for the Output Wait MonitoredQuantity.
bool initialize(fhicl::ParameterSet const &pset, uint64_t, uint64_t)
Initialize the BoardReaderCore.
std::string report(std::string const &which) const
Send a report on a given run-time quantity.
bool start(art::RunID id, uint64_t timeout, uint64_t timestamp)
Start the BoardReader, and the CommandableFragmentGenerator.
bool resume(uint64_t timeout, uint64_t timestamp)
Resume the BoardReader, and the CommandableFragmentGenerator.
bool pause(uint64_t timeout, uint64_t timestamp)
Pause the BoardReader, and the CommandableFragmentGenerator.
bool shutdown(uint64_t)
Shutdown the BoardReader, and the CommandableFragmentGenerator.
size_t process_fragments()
Main working loop of the BoardReaderCore.