2 #define TRACE_NAME (app_name + "_BoardReaderCore").c_str() // include these 2 first -
3 #include "artdaq/DAQdata/Globals.hh"
4 #include "artdaq/Application/TaskType.hh"
5 #include "artdaq/Application/BoardReaderCore.hh"
6 #include "artdaq-core/Data/Fragment.hh"
7 #include "artdaq-core/Utilities/ExceptionHandler.hh"
8 #include "artdaq/Application/makeCommandableFragmentGenerator.hh"
9 #include "canvas/Utilities/Exception.h"
10 #include "cetlib_except/exception.h"
26 std::unique_ptr<artdaq::DataSenderManager> artdaq::BoardReaderCore::sender_ptr_ =
nullptr;
29 parent_application_(parent_application)
31 , generator_ptr_(nullptr)
32 , stop_requested_(false)
33 , pause_requested_(false)
35 TLOG(TLVL_DEBUG) <<
"Constructor";
41 metricMan = &metricMan_;
46 TLOG(TLVL_DEBUG) <<
"Destructor";
51 TLOG(TLVL_DEBUG) <<
"initialize method called with " <<
"ParameterSet = \"" << pset.to_string() <<
"\".";
54 fhicl::ParameterSet daq_pset;
57 daq_pset = pset.get<fhicl::ParameterSet>(
"daq");
62 <<
"Unable to find the DAQ parameters in the initialization "
63 <<
"ParameterSet: \"" + pset.to_string() +
"\".";
66 fhicl::ParameterSet fr_pset;
69 fr_pset = daq_pset.get<fhicl::ParameterSet>(
"fragment_receiver");
75 <<
"Unable to find the fragment_receiver parameters in the DAQ "
76 <<
"initialization ParameterSet: \"" + daq_pset.to_string() +
"\".";
81 fhicl::ParameterSet metric_pset;
84 metric_pset = daq_pset.get<fhicl::ParameterSet>(
"metrics");
88 if (metric_pset.is_empty())
90 TLOG(TLVL_INFO) <<
"No metric plugins appear to be defined";
94 metricMan_.initialize(metric_pset, app_name);
98 ExceptionHandler(ExceptionHandlerRethrow::no,
99 "Error loading metrics in BoardReaderCore::initialize()");
102 if (daq_pset.has_key(
"rank"))
104 if (my_rank >= 0 && daq_pset.get<
int>(
"rank") != my_rank) {
105 TLOG(TLVL_WARNING) <<
"BoardReader rank specified at startup is different than rank specified at configure! Using rank received at configure!";
107 my_rank = daq_pset.get<
int>(
"rank");
111 TLOG(TLVL_ERROR) <<
"BoardReader rank not specified at startup or in configuration! Aborting";
117 std::string frag_gen_name = fr_pset.get<std::string>(
"generator",
"");
118 if (frag_gen_name.length() == 0)
121 <<
"No fragment generator (parameter name = \"generator\") was "
122 <<
"specified in the fragment_receiver ParameterSet. The "
123 <<
"DAQ initialization PSet was \"" << daq_pset.to_string() <<
"\".";
133 std::stringstream exception_string;
134 exception_string <<
"Exception thrown during initialization of fragment generator of type \""
135 << frag_gen_name <<
"\"";
137 ExceptionHandler(ExceptionHandlerRethrow::no, exception_string.str());
139 TLOG(TLVL_DEBUG) <<
"FHiCL parameter set used to initialize the fragment generator which threw an exception: " << fr_pset.to_string();
143 metricMan_.setPrefix(generator_ptr_->metricsReportingInstanceName());
145 rt_priority_ = fr_pset.get<
int>(
"rt_priority", 0);
148 statsHelper_.createCollectors(fr_pset, 100, 30.0, 60.0, FRAGMENTS_PROCESSED_STAT_KEY);
151 skip_seqId_test_ = (generator_ptr_->fragmentIDs().size() > 1 || generator_ptr_->request_mode() != RequestMode::Ignored);
153 verbose_ = fr_pset.get<
bool>(
"verbose",
true);
160 logMessage_(
"Starting run " + boost::lexical_cast<std::string>(
id.run()));
161 stop_requested_.store(
false);
162 pause_requested_.store(
false);
166 statsHelper_.resetStatistics();
168 metricMan_.do_start();
169 generator_ptr_->StartCmd(
id.run(), timeout, timestamp);
172 logMessage_(
"Completed the Start transition (Started run) for run " +
173 boost::lexical_cast<std::string>(run_id_.run()) +
174 ", timeout = " + boost::lexical_cast<std::string>(timeout) +
175 ", timestamp = " + boost::lexical_cast<std::string>(timestamp));
181 logMessage_(
"Stopping run " + boost::lexical_cast<std::string>(run_id_.run()) +
182 " after " + boost::lexical_cast<std::string>(fragment_count_) +
" fragments.");
183 stop_requested_.store(
true);
184 generator_ptr_->StopCmd(timeout, timestamp);
185 logMessage_(
"Completed the Stop transition for run " + boost::lexical_cast<std::string>(run_id_.run()));
191 logMessage_(
"Pausing run " + boost::lexical_cast<std::string>(run_id_.run()) +
192 " after " + boost::lexical_cast<std::string>(fragment_count_) +
" fragments.");
193 pause_requested_.store(
true);
194 generator_ptr_->PauseCmd(timeout, timestamp);
195 logMessage_(
"Completed the Pause transition for run " + boost::lexical_cast<std::string>(run_id_.run()));
201 logMessage_(
"Resuming run " + boost::lexical_cast<std::string>(run_id_.run()));
202 pause_requested_.store(
false);
203 metricMan_.do_start();
204 generator_ptr_->ResumeCmd(timeout, timestamp);
205 logMessage_(
"Completed the Resume transition for run " + boost::lexical_cast<std::string>(run_id_.run()));
211 logMessage_(
"Starting Shutdown transition");
212 generator_ptr_->joinThreads();
213 generator_ptr_.reset(
nullptr);
214 metricMan_.shutdown();
215 logMessage_(
"Completed Shutdown transition");
221 TLOG(TLVL_DEBUG) <<
"soft_initialize method called with "
222 <<
"ParameterSet = \"" << pset.to_string()
229 TLOG(TLVL_DEBUG) <<
"reinitialize method called with "
230 <<
"ParameterSet = \"" << pset.to_string()
237 if (rt_priority_ > 0)
239 #pragma GCC diagnostic push
240 #pragma GCC diagnostic ignored "-Wmissing-field-initializers"
241 sched_param s_param = {};
242 s_param.sched_priority = rt_priority_;
243 if (pthread_setschedparam(pthread_self(), SCHED_RR, &s_param))
244 TLOG(TLVL_WARNING) <<
"setting realtime priority failed";
245 #pragma GCC diagnostic pop
251 if (rt_priority_ > 0)
253 #pragma GCC diagnostic push
254 #pragma GCC diagnostic ignored "-Wmissing-field-initializers"
255 sched_param s_param = {};
256 s_param.sched_priority = rt_priority_;
257 int status = pthread_setschedparam(pthread_self(), SCHED_RR, &s_param);
261 <<
"Failed to set realtime priority to " << rt_priority_
262 <<
", return code = " << status;
264 #pragma GCC diagnostic pop
267 TLOG(TLVL_DEBUG) <<
"Initializing DataSenderManager. my_rank=" << my_rank;
270 TLOG(TLVL_DEBUG) <<
"Waiting for first fragment.";
271 artdaq::MonitoredQuantityStats::TIME_POINT_T startTime;
273 artdaq::FragmentPtrs frags;
278 startTime = artdaq::MonitoredQuantity::getCurrentTime();
280 TLOG(18) <<
"process_fragments getNext start";
281 active = generator_ptr_->getNext(frags);
282 TLOG(18) <<
"process_fragments getNext done (active=" << active <<
")";
289 if (!active && generator_ptr_ && generator_ptr_->exception())
291 parent_application_.in_run_failure();
294 delta_time = artdaq::MonitoredQuantity::getCurrentTime() - startTime;
295 statsHelper_.addSample(INPUT_WAIT_STAT_KEY, delta_time);
297 TLOG(16) <<
"process_fragments INPUT_WAIT=" << delta_time;
299 if (!active) {
break; }
300 statsHelper_.addSample(FRAGMENTS_PER_READ_STAT_KEY, frags.size());
302 for (
auto& fragPtr : frags)
306 TLOG(TLVL_WARNING) <<
"Encountered a bad fragment pointer in fragment " << fragment_count_ <<
". "
307 <<
"This is most likely caused by a problem with the Fragment Generator!";
310 artdaq::Fragment::sequence_id_t sequence_id = fragPtr->sequenceID();
311 SetMFIteration(
"Sequence ID " + std::to_string(sequence_id));
312 statsHelper_.addSample(FRAGMENTS_PROCESSED_STAT_KEY, fragPtr->size());
314 if ((fragment_count_ % 250) == 0)
317 <<
"Sending fragment " << fragment_count_
318 <<
" with sequence id " << sequence_id <<
".";
322 if (!skip_seqId_test_ && abs(static_cast<int64_t>(sequence_id) - static_cast<int64_t>(prev_seq_id_)) > 1)
325 <<
"Missing sequence IDs: current sequence ID = "
326 << sequence_id <<
", previous sequence ID = "
327 << prev_seq_id_ <<
".";
329 prev_seq_id_ = sequence_id;
331 startTime = artdaq::MonitoredQuantity::getCurrentTime();
332 TLOG(17) <<
"process_fragments seq=" << sequence_id <<
" sendFragment start";
333 auto res = sender_ptr_->sendFragment(std::move(*fragPtr));
336 statsHelper_.addSample(OUTPUT_WAIT_STAT_KEY,
337 artdaq::MonitoredQuantity::getCurrentTime() - startTime);
339 bool readyToReport = statsHelper_.readyToReport(fragment_count_);
342 std::string statString = buildStatisticsString_();
343 TLOG(TLVL_DEBUG) << statString;
345 if (fragment_count_ == 1 || readyToReport)
348 <<
"Sending fragment " << fragment_count_
349 <<
" with sequence id " << sequence_id <<
".";
352 if (statsHelper_.statsRollingWindowHasMoved()) { sendMetrics_(); }
359 metricMan_.do_stop();
361 sender_ptr_.reset(
nullptr);
366 std::string resultString;
369 if (generator_ptr_.get() != 0)
371 resultString = generator_ptr_->ReportCmd(which);
372 if (resultString.length() > 0) {
return resultString; }
379 std::string tmpString = app_name +
" run number = ";
380 tmpString.append(boost::lexical_cast<std::string>(run_id_.run()));
381 tmpString.append(
". Command=\"" + which +
"\" is not currently supported.");
387 TLOG(TLVL_DEBUG) <<
"metaCommand method called with "
388 <<
"command = \"" << command <<
"\""
389 <<
", arg = \"" << arg <<
"\""
392 if (generator_ptr_)
return generator_ptr_->metaCommand(command, arg);
397 std::string artdaq::BoardReaderCore::buildStatisticsString_()
399 std::ostringstream oss;
400 oss << app_name <<
" statistics:" << std::endl;
402 double fragmentCount = 1.0;
403 artdaq::MonitoredQuantityPtr mqPtr = artdaq::StatisticsCollection::getInstance().
404 getMonitoredQuantity(FRAGMENTS_PROCESSED_STAT_KEY);
405 if (mqPtr.get() != 0)
407 artdaq::MonitoredQuantityStats stats;
408 mqPtr->getStats(stats);
409 oss <<
" Fragment statistics: "
410 << stats.recentSampleCount <<
" fragments received at "
411 << stats.recentSampleRate <<
" fragments/sec, effective data rate = "
412 << (stats.recentValueRate *
sizeof(artdaq::RawDataType)
413 / 1024.0 / 1024.0) <<
" MB/sec, monitor window = "
414 << stats.recentDuration <<
" sec, min::max event size = "
415 << (stats.recentValueMin *
sizeof(artdaq::RawDataType)
418 << (stats.recentValueMax *
sizeof(artdaq::RawDataType)
420 <<
" MB" << std::endl;
421 fragmentCount = std::max(
double(stats.recentSampleCount), 1.0);
422 oss <<
" Average times per fragment: ";
423 if (stats.recentSampleRate > 0.0)
425 oss <<
" elapsed time = "
426 << (1.0 / stats.recentSampleRate) <<
" sec";
438 mqPtr = artdaq::StatisticsCollection::getInstance().
439 getMonitoredQuantity(INPUT_WAIT_STAT_KEY);
440 if (mqPtr.get() != 0)
442 oss <<
", input wait time = "
443 << (mqPtr->getRecentValueSum() / fragmentCount) <<
" sec";
446 mqPtr = artdaq::StatisticsCollection::getInstance().
447 getMonitoredQuantity(BRSYNC_WAIT_STAT_KEY);
448 if (mqPtr.get() != 0)
450 oss <<
", BRsync wait time = "
451 << (mqPtr->getRecentValueSum() / fragmentCount) <<
" sec";
454 mqPtr = artdaq::StatisticsCollection::getInstance().
455 getMonitoredQuantity(OUTPUT_WAIT_STAT_KEY);
456 if (mqPtr.get() != 0)
458 oss <<
", output wait time = "
459 << (mqPtr->getRecentValueSum() / fragmentCount) <<
" sec";
462 oss << std::endl <<
" Fragments per read: ";
463 mqPtr = artdaq::StatisticsCollection::getInstance().
464 getMonitoredQuantity(FRAGMENTS_PER_READ_STAT_KEY);
465 if (mqPtr.get() != 0)
467 artdaq::MonitoredQuantityStats stats;
468 mqPtr->getStats(stats);
470 << stats.recentValueAverage
472 << stats.recentValueMin
474 << stats.recentValueMax;
480 void artdaq::BoardReaderCore::sendMetrics_()
483 double fragmentCount = 1.0;
484 artdaq::MonitoredQuantityPtr mqPtr = artdaq::StatisticsCollection::getInstance().
485 getMonitoredQuantity(FRAGMENTS_PROCESSED_STAT_KEY);
486 if (mqPtr.get() != 0)
488 artdaq::MonitoredQuantityStats stats;
489 mqPtr->getStats(stats);
490 fragmentCount = std::max(
double(stats.recentSampleCount), 1.0);
491 metricMan_.sendMetric(
"Fragment Count", static_cast<unsigned long>(stats.fullSampleCount),
"fragments", 1, MetricMode::LastPoint);
492 metricMan_.sendMetric(
"Fragment Rate", stats.recentSampleRate,
"fragments/sec", 1, MetricMode::Average);
493 metricMan_.sendMetric(
"Average Fragment Size", (stats.recentValueAverage *
sizeof(artdaq::RawDataType)),
"bytes/fragment", 2, MetricMode::Average);
494 metricMan_.sendMetric(
"Data Rate", (stats.recentValueRate *
sizeof(artdaq::RawDataType)),
"bytes/sec", 2, MetricMode::Average);
505 mqPtr = artdaq::StatisticsCollection::getInstance().
506 getMonitoredQuantity(INPUT_WAIT_STAT_KEY);
507 if (mqPtr.get() != 0)
509 metricMan_.sendMetric(
"Avg Input Wait Time", (mqPtr->getRecentValueSum() / fragmentCount),
"seconds/fragment", 3, MetricMode::Average);
512 mqPtr = artdaq::StatisticsCollection::getInstance().
513 getMonitoredQuantity(BRSYNC_WAIT_STAT_KEY);
514 if (mqPtr.get() != 0)
516 metricMan_.sendMetric(
"Avg BoardReader Sync Wait Time", (mqPtr->getRecentValueSum() / fragmentCount),
"seconds/fragment", 3, MetricMode::Average);
519 mqPtr = artdaq::StatisticsCollection::getInstance().
520 getMonitoredQuantity(OUTPUT_WAIT_STAT_KEY);
521 if (mqPtr.get() != 0)
523 metricMan_.sendMetric(
"Avg Output Wait Time", (mqPtr->getRecentValueSum() / fragmentCount),
"seconds/fragment", 3, MetricMode::Average);
526 mqPtr = artdaq::StatisticsCollection::getInstance().
527 getMonitoredQuantity(FRAGMENTS_PER_READ_STAT_KEY);
528 if (mqPtr.get() != 0)
530 metricMan_.sendMetric(
"Avg Frags Per Read", mqPtr->getRecentValueAverage(),
"fragments/read", 4, MetricMode::Average);
534 void artdaq::BoardReaderCore::logMessage_(std::string
const& text)
538 TLOG(TLVL_INFO) << text;
542 TLOG(TLVL_DEBUG) << text;
void addMonitoredQuantityName(std::string const &statKey)
Add a MonitoredQuantity name to the list.
Commandable is the base class for all artdaq components which implement the artdaq state machine...
Sends Fragment objects using TransferInterface plugins. Uses Routing Tables if confgiured, otherwise will Round-Robin Fragments to the destinations.
static const std::string FRAGMENTS_PROCESSED_STAT_KEY
Key for the Fragments Processed MonitoredQuantity.
static const std::string INPUT_WAIT_STAT_KEY
Key for the Input Wait MonitoredQuantity.
bool stop(uint64_t timeout, uint64_t timestamp)
Stop the BoardReader, and the CommandableFragmentGenerator.
virtual ~BoardReaderCore()
BoardReaderCore Destructor.
static std::string CopyStatusToString(CopyStatus in)
Convert a CopyStatus variable to its string represenatation
BoardReaderCore(Commandable &parent_application)
BoardReaderCore Constructor.
std::unique_ptr< CommandableFragmentGenerator > makeCommandableFragmentGenerator(std::string const &generator_plugin_spec, fhicl::ParameterSet const &ps)
Load a CommandableFragmentGenerator plugin.
bool reinitialize(fhicl::ParameterSet const &pset, uint64_t, uint64_t)
Reinitialize the BoardReader. No-Op.
bool soft_initialize(fhicl::ParameterSet const &pset, uint64_t, uint64_t)
Soft-Initialize the BoardReader. No-Op.
static const std::string BRSYNC_WAIT_STAT_KEY
Key for the Sync Wait MonitoredQuantity.
static const std::string FRAGMENTS_PER_READ_STAT_KEY
Key for the Fragments Per Read MonitoredQuantity.
static const std::string OUTPUT_WAIT_STAT_KEY
Key for the Output Wait MonitoredQuantity.
bool initialize(fhicl::ParameterSet const &pset, uint64_t, uint64_t)
Initialize the BoardReaderCore.
std::string report(std::string const &which) const
Send a report on a given run-time quantity.
void process_fragments()
Main working loop of the BoardReaderCore.
bool start(art::RunID id, uint64_t timeout, uint64_t timestamp)
Start the BoardReader, and the CommandableFragmentGenerator.
bool resume(uint64_t timeout, uint64_t timestamp)
Resume the BoardReader, and the CommandableFragmentGenerator.
bool pause(uint64_t timeout, uint64_t timestamp)
Pause the BoardReader, and the CommandableFragmentGenerator.
bool shutdown(uint64_t)
Shutdown the BoardReader, and the CommandableFragmentGenerator.
bool metaCommand(std::string const &command, std::string const &arg)
Run a user-defined command on the CommandableFragmentGenerator.