1 #define TRACE_NAME "BoardReaderCore"
3 #include "artdaq/Application/TaskType.hh"
4 #include "artdaq/Application/BoardReaderCore.hh"
5 #include "artdaq-core/Data/Fragment.hh"
6 #include "artdaq-core/Utilities/ExceptionHandler.hh"
7 #include "artdaq/Application/makeCommandableFragmentGenerator.hh"
8 #include "canvas/Utilities/Exception.h"
9 #include "cetlib/exception.h"
25 std::unique_ptr<artdaq::DataSenderManager> artdaq::BoardReaderCore::sender_ptr_ =
nullptr;
28 parent_application_(parent_application)
30 , generator_ptr_(nullptr)
31 , stop_requested_(false)
32 , pause_requested_(false)
34 TLOG_DEBUG(app_name) <<
"Constructor" << TLOG_ENDL;
40 metricMan = &metricMan_;
45 TLOG_DEBUG(app_name) <<
"Destructor" << TLOG_ENDL;
50 TLOG_DEBUG(app_name) <<
"initialize method called with " <<
"ParameterSet = \"" << pset.to_string() <<
"\"." << TLOG_ENDL;
53 fhicl::ParameterSet daq_pset;
56 daq_pset = pset.get<fhicl::ParameterSet>(
"daq");
61 <<
"Unable to find the DAQ parameters in the initialization "
62 <<
"ParameterSet: \"" + pset.to_string() +
"\"." << TLOG_ENDL;
65 fhicl::ParameterSet fr_pset;
68 fr_pset = daq_pset.get<fhicl::ParameterSet>(
"fragment_receiver");
74 <<
"Unable to find the fragment_receiver parameters in the DAQ "
75 <<
"initialization ParameterSet: \"" + daq_pset.to_string() +
"\"." << TLOG_ENDL;
80 fhicl::ParameterSet metric_pset;
83 metric_pset = daq_pset.get<fhicl::ParameterSet>(
"metrics");
87 if (metric_pset.is_empty())
89 TLOG_INFO(app_name) <<
"No metric plugins appear to be defined" << TLOG_ENDL;
93 metricMan_.initialize(metric_pset, app_name);
97 ExceptionHandler(ExceptionHandlerRethrow::no,
98 "Error loading metrics in BoardReaderCore::initialize()");
102 std::string frag_gen_name = fr_pset.get<std::string>(
"generator",
"");
103 if (frag_gen_name.length() == 0)
106 <<
"No fragment generator (parameter name = \"generator\") was "
107 <<
"specified in the fragment_receiver ParameterSet. The "
108 <<
"DAQ initialization PSet was \"" << daq_pset.to_string() <<
"\"." << TLOG_ENDL;
118 std::stringstream exception_string;
119 exception_string <<
"Exception thrown during initialization of fragment generator of type \""
120 << frag_gen_name <<
"\"";
122 ExceptionHandler(ExceptionHandlerRethrow::no, exception_string.str());
124 TLOG_DEBUG(app_name) <<
"FHiCL parameter set used to initialize the fragment generator which threw an exception: " << fr_pset.to_string() << TLOG_ENDL;
128 metricMan_.setPrefix(generator_ptr_->metricsReportingInstanceName());
130 rt_priority_ = fr_pset.get<
int>(
"rt_priority", 0);
173 statsHelper_.createCollectors(fr_pset, 100, 30.0, 60.0, FRAGMENTS_PROCESSED_STAT_KEY);
176 skip_seqId_test_ = (generator_ptr_->fragmentIDs().size() > 1);
183 stop_requested_.store(
false);
184 pause_requested_.store(
false);
188 statsHelper_.resetStatistics();
190 metricMan_.do_start();
191 generator_ptr_->StartCmd(
id.run(), timeout, timestamp);
194 TLOG_DEBUG(app_name) <<
"Started run " << run_id_.run() <<
195 ", timeout = " << timeout <<
", timestamp = " << timestamp << TLOG_ENDL;
201 TLOG_DEBUG(app_name) <<
"Stopping run " << run_id_.run()
202 <<
" after " << fragment_count_
203 <<
" fragments." << TLOG_ENDL;
204 stop_requested_.store(
true);
205 generator_ptr_->StopCmd(timeout, timestamp);
211 TLOG_DEBUG(app_name) <<
"Pausing run " << run_id_.run()
212 <<
" after " << fragment_count_
213 <<
" fragments." << TLOG_ENDL;
214 pause_requested_.store(
true);
215 generator_ptr_->PauseCmd(timeout, timestamp);
221 TLOG_DEBUG(app_name) <<
"Resuming run " << run_id_.run() << TLOG_ENDL;
222 pause_requested_.store(
false);
223 metricMan_.do_start();
224 generator_ptr_->ResumeCmd(timeout, timestamp);
230 generator_ptr_->joinThreads();
231 generator_ptr_.reset(
nullptr);
232 metricMan_.shutdown();
238 TLOG_DEBUG(app_name) <<
"soft_initialize method called with "
239 <<
"ParameterSet = \"" << pset.to_string()
240 <<
"\"." << TLOG_ENDL;
246 TLOG_DEBUG(app_name) <<
"reinitialize method called with "
247 <<
"ParameterSet = \"" << pset.to_string()
248 <<
"\"." << TLOG_ENDL;
254 if (rt_priority_ > 0)
256 #pragma GCC diagnostic push
257 #pragma GCC diagnostic ignored "-Wmissing-field-initializers"
258 sched_param s_param = {};
259 s_param.sched_priority = rt_priority_;
260 if (pthread_setschedparam(pthread_self(), SCHED_RR, &s_param))
261 TLOG_WARNING(app_name) <<
"setting realtime priority failed" << TLOG_ENDL;
262 #pragma GCC diagnostic pop
268 if (rt_priority_ > 0)
270 #pragma GCC diagnostic push
271 #pragma GCC diagnostic ignored "-Wmissing-field-initializers"
272 sched_param s_param = {};
273 s_param.sched_priority = rt_priority_;
274 int status = pthread_setschedparam(pthread_self(), SCHED_RR, &s_param);
278 <<
"Failed to set realtime priority to " << rt_priority_
279 <<
", return code = " << status << TLOG_ENDL;
281 #pragma GCC diagnostic pop
284 TLOG_DEBUG(app_name) <<
"Initializing DataSenderManager. my_rank=" << my_rank << TLOG_ENDL;
287 TLOG_DEBUG(app_name) <<
"Waiting for first fragment." << TLOG_ENDL;
288 artdaq::MonitoredQuantityStats::TIME_POINT_T startTime;
290 artdaq::FragmentPtrs frags;
295 startTime = artdaq::MonitoredQuantity::getCurrentTime();
297 TRACE(18, app_name +
"::process_fragments getNext start");
298 active = generator_ptr_->getNext(frags);
299 TRACE(18, app_name +
"::process_fragments getNext done (active=%i)", active);
306 if (!active && generator_ptr_ && generator_ptr_->exception())
308 parent_application_.in_run_failure();
311 delta_time = artdaq::MonitoredQuantity::getCurrentTime() - startTime;
312 statsHelper_.addSample(INPUT_WAIT_STAT_KEY, delta_time);
314 TLOG_ARB(16,app_name) <<
"process_fragments INPUT_WAIT="<<std::to_string( delta_time) << TLOG_ENDL;
316 if (!active) {
break; }
317 statsHelper_.addSample(FRAGMENTS_PER_READ_STAT_KEY, frags.size());
319 for (
auto& fragPtr : frags)
323 TLOG_WARNING(app_name) <<
"Encountered a bad fragment pointer in fragment " << fragment_count_ <<
". "
324 <<
"This is most likely caused by a problem with the Fragment Generator!" << TLOG_ENDL;
327 artdaq::Fragment::sequence_id_t sequence_id = fragPtr->sequenceID();
328 statsHelper_.addSample(FRAGMENTS_PROCESSED_STAT_KEY, fragPtr->size());
330 if ((fragment_count_ % 250) == 0)
333 <<
"Sending fragment " << fragment_count_
334 <<
" (%250) with sequence id " << sequence_id <<
"." << TLOG_ENDL;
338 if (!skip_seqId_test_ && abs(sequence_id - prev_seq_id_) > 1)
340 TLOG_WARNING(app_name)
341 <<
"Missing sequence IDs: current sequence ID = "
342 << sequence_id <<
", previous sequence ID = "
343 << prev_seq_id_ <<
"." << TLOG_ENDL;
345 prev_seq_id_ = sequence_id;
347 startTime = artdaq::MonitoredQuantity::getCurrentTime();
348 TLOG_ARB(17,app_name) <<
"process_fragments seq="<< std::to_string(sequence_id) <<
" sendFragment start" << TLOG_ENDL;
349 auto res = sender_ptr_->sendFragment(std::move(*fragPtr));
350 TLOG_ARB(17, app_name) <<
"process_fragments seq=" << std::to_string(sequence_id) <<
" sendFragment done (res="<< res<<
")"<<TLOG_ENDL;
352 statsHelper_.addSample(OUTPUT_WAIT_STAT_KEY,
353 artdaq::MonitoredQuantity::getCurrentTime() - startTime);
355 bool readyToReport = statsHelper_.readyToReport(fragment_count_);
358 std::string statString = buildStatisticsString_();
359 TLOG_DEBUG(app_name) << statString << TLOG_ENDL;
361 if (fragment_count_ == 1 || readyToReport)
364 <<
"Sending fragment " << fragment_count_
365 <<
" with sequence id " << sequence_id <<
"." << TLOG_ENDL;
368 if (statsHelper_.statsRollingWindowHasMoved()) { sendMetrics_(); }
375 metricMan_.do_stop();
377 sender_ptr_.reset(
nullptr);
382 std::string resultString;
385 if (generator_ptr_.get() != 0)
387 resultString = generator_ptr_->ReportCmd(which);
388 if (resultString.length() > 0) {
return resultString; }
395 std::string tmpString = app_name +
" run number = ";
396 tmpString.append(boost::lexical_cast<std::string>(run_id_.run()));
397 tmpString.append(
". Command=\"" + which +
"\" is not currently supported.");
403 TLOG_DEBUG(app_name) <<
"metaCommand method called with "
404 <<
"command = \"" << command <<
"\""
405 <<
", arg = \"" << arg <<
"\""
408 if (generator_ptr_)
return generator_ptr_->metaCommand(command, arg);
413 std::string artdaq::BoardReaderCore::buildStatisticsString_()
415 std::ostringstream oss;
416 oss << app_name <<
" statistics:" << std::endl;
418 double fragmentCount = 1.0;
419 artdaq::MonitoredQuantityPtr mqPtr = artdaq::StatisticsCollection::getInstance().
420 getMonitoredQuantity(FRAGMENTS_PROCESSED_STAT_KEY);
421 if (mqPtr.get() != 0)
423 artdaq::MonitoredQuantityStats stats;
424 mqPtr->getStats(stats);
425 oss <<
" Fragment statistics: "
426 << stats.recentSampleCount <<
" fragments received at "
427 << stats.recentSampleRate <<
" fragments/sec, effective data rate = "
428 << (stats.recentValueRate *
sizeof(artdaq::RawDataType)
429 / 1024.0 / 1024.0) <<
" MB/sec, monitor window = "
430 << stats.recentDuration <<
" sec, min::max event size = "
431 << (stats.recentValueMin *
sizeof(artdaq::RawDataType)
434 << (stats.recentValueMax *
sizeof(artdaq::RawDataType)
436 <<
" MB" << std::endl;
437 fragmentCount = std::max(
double(stats.recentSampleCount), 1.0);
438 oss <<
" Average times per fragment: ";
439 if (stats.recentSampleRate > 0.0)
441 oss <<
" elapsed time = "
442 << (1.0 / stats.recentSampleRate) <<
" sec";
454 mqPtr = artdaq::StatisticsCollection::getInstance().
455 getMonitoredQuantity(INPUT_WAIT_STAT_KEY);
456 if (mqPtr.get() != 0)
458 oss <<
", input wait time = "
459 << (mqPtr->getRecentValueSum() / fragmentCount) <<
" sec";
462 mqPtr = artdaq::StatisticsCollection::getInstance().
463 getMonitoredQuantity(BRSYNC_WAIT_STAT_KEY);
464 if (mqPtr.get() != 0)
466 oss <<
", BRsync wait time = "
467 << (mqPtr->getRecentValueSum() / fragmentCount) <<
" sec";
470 mqPtr = artdaq::StatisticsCollection::getInstance().
471 getMonitoredQuantity(OUTPUT_WAIT_STAT_KEY);
472 if (mqPtr.get() != 0)
474 oss <<
", output wait time = "
475 << (mqPtr->getRecentValueSum() / fragmentCount) <<
" sec";
478 oss << std::endl <<
" Fragments per read: ";
479 mqPtr = artdaq::StatisticsCollection::getInstance().
480 getMonitoredQuantity(FRAGMENTS_PER_READ_STAT_KEY);
481 if (mqPtr.get() != 0)
483 artdaq::MonitoredQuantityStats stats;
484 mqPtr->getStats(stats);
486 << stats.recentValueAverage
488 << stats.recentValueMin
490 << stats.recentValueMax;
496 void artdaq::BoardReaderCore::sendMetrics_()
499 double fragmentCount = 1.0;
500 artdaq::MonitoredQuantityPtr mqPtr = artdaq::StatisticsCollection::getInstance().
501 getMonitoredQuantity(FRAGMENTS_PROCESSED_STAT_KEY);
502 if (mqPtr.get() != 0)
504 artdaq::MonitoredQuantityStats stats;
505 mqPtr->getStats(stats);
506 fragmentCount = std::max(
double(stats.recentSampleCount), 1.0);
507 metricMan_.sendMetric(
"Fragment Count", static_cast<unsigned long>(stats.fullSampleCount),
"fragments", 1, MetricMode::Accumulate);
508 metricMan_.sendMetric(
"Fragment Rate", stats.recentSampleRate,
"fragments/sec", 1, MetricMode::Average);
509 metricMan_.sendMetric(
"Average Fragment Size", (stats.recentValueAverage *
sizeof(artdaq::RawDataType)),
"bytes/fragment", 2, MetricMode::Average);
510 metricMan_.sendMetric(
"Data Rate", (stats.recentValueRate *
sizeof(artdaq::RawDataType)),
"bytes/sec", 2, MetricMode::Average);
521 mqPtr = artdaq::StatisticsCollection::getInstance().
522 getMonitoredQuantity(INPUT_WAIT_STAT_KEY);
523 if (mqPtr.get() != 0)
525 metricMan_.sendMetric(
"Avg Input Wait Time", (mqPtr->getRecentValueSum() / fragmentCount),
"seconds/fragment", 3, MetricMode::Average);
528 mqPtr = artdaq::StatisticsCollection::getInstance().
529 getMonitoredQuantity(BRSYNC_WAIT_STAT_KEY);
530 if (mqPtr.get() != 0)
532 metricMan_.sendMetric(
"Avg BoardReader Sync Wait Time", (mqPtr->getRecentValueSum() / fragmentCount),
"seconds/fragment", 3, MetricMode::Average);
535 mqPtr = artdaq::StatisticsCollection::getInstance().
536 getMonitoredQuantity(OUTPUT_WAIT_STAT_KEY);
537 if (mqPtr.get() != 0)
539 metricMan_.sendMetric(
"Avg Output Wait Time", (mqPtr->getRecentValueSum() / fragmentCount),
"seconds/fragment", 3, MetricMode::Average);
542 mqPtr = artdaq::StatisticsCollection::getInstance().
543 getMonitoredQuantity(FRAGMENTS_PER_READ_STAT_KEY);
544 if (mqPtr.get() != 0)
546 metricMan_.sendMetric(
"Avg Frags Per Read", mqPtr->getRecentValueAverage(),
"fragments/read", 4, MetricMode::Average);
void addMonitoredQuantityName(std::string const &statKey)
Add a MonitoredQuantity name to the list.
Commandable is the base class for all artdaq components which implement the artdaq state machine...
Sends Fragment objects using TransferInterface plugins. Uses Routing Tables if confgiured, otherwise will Round-Robin Fragments to the destinations.
static const std::string FRAGMENTS_PROCESSED_STAT_KEY
Key for the Fragments Processed MonitoredQuantity.
static const std::string INPUT_WAIT_STAT_KEY
Key for the Input Wait MonitoredQuantity.
bool stop(uint64_t timeout, uint64_t timestamp)
Stop the BoardReader, and the CommandableFragmentGenerator.
virtual ~BoardReaderCore()
BoardReaderCore Destructor.
BoardReaderCore(Commandable &parent_application)
BoardReaderCore Constructor.
std::unique_ptr< CommandableFragmentGenerator > makeCommandableFragmentGenerator(std::string const &generator_plugin_spec, fhicl::ParameterSet const &ps)
Load a CommandableFragmentGenerator plugin.
bool reinitialize(fhicl::ParameterSet const &pset, uint64_t, uint64_t)
Reinitialize the BoardReader. No-Op.
bool soft_initialize(fhicl::ParameterSet const &pset, uint64_t, uint64_t)
Soft-Initialize the BoardReader. No-Op.
static const std::string BRSYNC_WAIT_STAT_KEY
Key for the Sync Wait MonitoredQuantity.
static const std::string FRAGMENTS_PER_READ_STAT_KEY
Key for the Fragments Per Read MonitoredQuantity.
static const std::string OUTPUT_WAIT_STAT_KEY
Key for the Output Wait MonitoredQuantity.
bool initialize(fhicl::ParameterSet const &pset, uint64_t, uint64_t)
Initialize the BoardReaderCore.
std::string report(std::string const &which) const
Send a report on a given run-time quantity.
void process_fragments()
Main working loop of the BoardReaderCore.
bool start(art::RunID id, uint64_t timeout, uint64_t timestamp)
Start the BoardReader, and the CommandableFragmentGenerator.
bool resume(uint64_t timeout, uint64_t timestamp)
Resume the BoardReader, and the CommandableFragmentGenerator.
bool pause(uint64_t timeout, uint64_t timestamp)
Pause the BoardReader, and the CommandableFragmentGenerator.
bool shutdown(uint64_t)
Shutdown the BoardReader, and the CommandableFragmentGenerator.
bool metaCommand(std::string const &command, std::string const &arg)
Run a user-defined command on the CommandableFragmentGenerator.