2 #define TRACE_NAME (app_name + "_BoardReaderCore").c_str() // include these 2 first -
3 #include "artdaq/DAQdata/Globals.hh"
4 #include "artdaq/Application/TaskType.hh"
5 #include "artdaq/Application/BoardReaderCore.hh"
6 #include "artdaq-core/Data/Fragment.hh"
7 #include "artdaq-core/Utilities/ExceptionHandler.hh"
8 #include "artdaq/Application/makeCommandableFragmentGenerator.hh"
9 #include "canvas/Utilities/Exception.h"
10 #include "cetlib_except/exception.h"
26 std::unique_ptr<artdaq::DataSenderManager> artdaq::BoardReaderCore::sender_ptr_ =
nullptr;
29 parent_application_(parent_application)
31 , generator_ptr_(nullptr)
32 , stop_requested_(false)
33 , pause_requested_(false)
35 TLOG(TLVL_DEBUG) <<
"Constructor";
45 TLOG(TLVL_DEBUG) <<
"Destructor";
50 TLOG(TLVL_DEBUG) <<
"initialize method called with " <<
"ParameterSet = \"" << pset.to_string() <<
"\".";
53 fhicl::ParameterSet daq_pset;
56 daq_pset = pset.get<fhicl::ParameterSet>(
"daq");
61 <<
"Unable to find the DAQ parameters in the initialization "
62 <<
"ParameterSet: \"" + pset.to_string() +
"\".";
65 fhicl::ParameterSet fr_pset;
68 fr_pset = daq_pset.get<fhicl::ParameterSet>(
"fragment_receiver");
74 <<
"Unable to find the fragment_receiver parameters in the DAQ "
75 <<
"initialization ParameterSet: \"" + daq_pset.to_string() +
"\".";
80 fhicl::ParameterSet metric_pset;
83 metric_pset = daq_pset.get<fhicl::ParameterSet>(
"metrics");
87 if (metric_pset.is_empty())
89 TLOG(TLVL_INFO) <<
"No metric plugins appear to be defined";
93 metricMan->initialize(metric_pset, app_name);
97 ExceptionHandler(ExceptionHandlerRethrow::no,
98 "Error loading metrics in BoardReaderCore::initialize()");
101 if (daq_pset.has_key(
"rank"))
103 if (my_rank >= 0 && daq_pset.get<
int>(
"rank") != my_rank) {
104 TLOG(TLVL_WARNING) <<
"BoardReader rank specified at startup is different than rank specified at configure! Using rank received at configure!";
106 my_rank = daq_pset.get<
int>(
"rank");
110 TLOG(TLVL_ERROR) <<
"BoardReader rank not specified at startup or in configuration! Aborting";
116 std::string frag_gen_name = fr_pset.get<std::string>(
"generator",
"");
117 if (frag_gen_name.length() == 0)
120 <<
"No fragment generator (parameter name = \"generator\") was "
121 <<
"specified in the fragment_receiver ParameterSet. The "
122 <<
"DAQ initialization PSet was \"" << daq_pset.to_string() <<
"\".";
132 std::stringstream exception_string;
133 exception_string <<
"Exception thrown during initialization of fragment generator of type \""
134 << frag_gen_name <<
"\"";
136 ExceptionHandler(ExceptionHandlerRethrow::no, exception_string.str());
138 TLOG(TLVL_DEBUG) <<
"FHiCL parameter set used to initialize the fragment generator which threw an exception: " << fr_pset.to_string();
142 metricMan->setPrefix(generator_ptr_->metricsReportingInstanceName());
144 rt_priority_ = fr_pset.get<
int>(
"rt_priority", 0);
147 statsHelper_.createCollectors(fr_pset, 100, 30.0, 60.0, FRAGMENTS_PROCESSED_STAT_KEY);
150 skip_seqId_test_ = (generator_ptr_->fragmentIDs().size() > 1 || generator_ptr_->request_mode() != RequestMode::Ignored);
152 verbose_ = fr_pset.get<
bool>(
"verbose",
true);
159 logMessage_(
"Starting run " + boost::lexical_cast<std::string>(
id.run()));
160 stop_requested_.store(
false);
161 pause_requested_.store(
false);
165 statsHelper_.resetStatistics();
167 metricMan->do_start();
168 generator_ptr_->StartCmd(
id.run(), timeout, timestamp);
171 logMessage_(
"Completed the Start transition (Started run) for run " +
172 boost::lexical_cast<std::string>(run_id_.run()) +
173 ", timeout = " + boost::lexical_cast<std::string>(timeout) +
174 ", timestamp = " + boost::lexical_cast<std::string>(timestamp));
180 logMessage_(
"Stopping run " + boost::lexical_cast<std::string>(run_id_.run()) +
181 " after " + boost::lexical_cast<std::string>(fragment_count_) +
" fragments.");
182 stop_requested_.store(
true);
184 TLOG(TLVL_DEBUG) <<
"Stopping CommandableFragmentGenerator BEGIN";
185 generator_ptr_->StopCmd(timeout, timestamp);
186 TLOG(TLVL_DEBUG) <<
"Stopping CommandableFragmentGenerator END";
188 TLOG(TLVL_DEBUG) <<
"Stopping DataSenderManager";
189 if(sender_ptr_) sender_ptr_->StopSender();
191 logMessage_(
"Completed the Stop transition for run " + boost::lexical_cast<std::string>(run_id_.run()));
197 logMessage_(
"Pausing run " + boost::lexical_cast<std::string>(run_id_.run()) +
198 " after " + boost::lexical_cast<std::string>(fragment_count_) +
" fragments.");
199 pause_requested_.store(
true);
200 generator_ptr_->PauseCmd(timeout, timestamp);
201 logMessage_(
"Completed the Pause transition for run " + boost::lexical_cast<std::string>(run_id_.run()));
207 logMessage_(
"Resuming run " + boost::lexical_cast<std::string>(run_id_.run()));
208 pause_requested_.store(
false);
209 metricMan->do_start();
210 generator_ptr_->ResumeCmd(timeout, timestamp);
211 logMessage_(
"Completed the Resume transition for run " + boost::lexical_cast<std::string>(run_id_.run()));
217 logMessage_(
"Starting Shutdown transition");
218 generator_ptr_->joinThreads();
219 generator_ptr_.reset(
nullptr);
220 metricMan->shutdown();
221 logMessage_(
"Completed Shutdown transition");
227 TLOG(TLVL_DEBUG) <<
"soft_initialize method called with "
228 <<
"ParameterSet = \"" << pset.to_string()
235 TLOG(TLVL_DEBUG) <<
"reinitialize method called with "
236 <<
"ParameterSet = \"" << pset.to_string()
243 if (rt_priority_ > 0)
245 #pragma GCC diagnostic push
246 #pragma GCC diagnostic ignored "-Wmissing-field-initializers"
247 sched_param s_param = {};
248 s_param.sched_priority = rt_priority_;
249 if (pthread_setschedparam(pthread_self(), SCHED_RR, &s_param))
250 TLOG(TLVL_WARNING) <<
"setting realtime priority failed";
251 #pragma GCC diagnostic pop
257 if (rt_priority_ > 0)
259 #pragma GCC diagnostic push
260 #pragma GCC diagnostic ignored "-Wmissing-field-initializers"
261 sched_param s_param = {};
262 s_param.sched_priority = rt_priority_;
263 int status = pthread_setschedparam(pthread_self(), SCHED_RR, &s_param);
267 <<
"Failed to set realtime priority to " << rt_priority_
268 <<
", return code = " << status;
270 #pragma GCC diagnostic pop
273 TLOG(TLVL_DEBUG) <<
"Initializing DataSenderManager. my_rank=" << my_rank;
276 TLOG(TLVL_DEBUG) <<
"Waiting for first fragment.";
277 artdaq::MonitoredQuantityStats::TIME_POINT_T startTime;
279 artdaq::FragmentPtrs frags;
284 startTime = artdaq::MonitoredQuantity::getCurrentTime();
286 TLOG(18) <<
"process_fragments getNext start";
287 active = generator_ptr_->getNext(frags);
288 TLOG(18) <<
"process_fragments getNext done (active=" << active <<
")";
295 if (!active && generator_ptr_ && generator_ptr_->exception())
297 parent_application_.in_run_failure();
300 delta_time = artdaq::MonitoredQuantity::getCurrentTime() - startTime;
301 statsHelper_.addSample(INPUT_WAIT_STAT_KEY, delta_time);
303 TLOG(16) <<
"process_fragments INPUT_WAIT=" << delta_time;
305 if (!active) {
break; }
306 statsHelper_.addSample(FRAGMENTS_PER_READ_STAT_KEY, frags.size());
308 for (
auto& fragPtr : frags)
312 TLOG(TLVL_WARNING) <<
"Encountered a bad fragment pointer in fragment " << fragment_count_ <<
". "
313 <<
"This is most likely caused by a problem with the Fragment Generator!";
316 artdaq::Fragment::sequence_id_t sequence_id = fragPtr->sequenceID();
317 SetMFIteration(
"Sequence ID " + std::to_string(sequence_id));
318 statsHelper_.addSample(FRAGMENTS_PROCESSED_STAT_KEY, fragPtr->size());
328 if (!skip_seqId_test_ && abs(static_cast<int64_t>(sequence_id) - static_cast<int64_t>(prev_seq_id_)) > 1)
331 <<
"Missing sequence IDs: current sequence ID = "
332 << sequence_id <<
", previous sequence ID = "
333 << prev_seq_id_ <<
".";
335 prev_seq_id_ = sequence_id;
337 startTime = artdaq::MonitoredQuantity::getCurrentTime();
338 TLOG(17) <<
"process_fragments seq=" << sequence_id <<
" sendFragment start";
339 auto res = sender_ptr_->sendFragment(std::move(*fragPtr));
342 statsHelper_.addSample(OUTPUT_WAIT_STAT_KEY,
343 artdaq::MonitoredQuantity::getCurrentTime() - startTime);
345 bool readyToReport = statsHelper_.readyToReport(fragment_count_);
348 std::string statString = buildStatisticsString_();
349 TLOG(TLVL_INFO) << statString;
351 if (fragment_count_ % 250 == 1 || readyToReport)
354 <<
"Sending fragment " << fragment_count_
355 <<
" with SeqID " << sequence_id <<
".";
358 if (statsHelper_.statsRollingWindowHasMoved()) { sendMetrics_(); }
362 sender_ptr_.reset(
nullptr);
367 metricMan->do_stop();
369 TLOG(TLVL_DEBUG) <<
"process_fragments loop end";
374 std::string resultString;
377 if (generator_ptr_.get() != 0)
379 resultString = generator_ptr_->ReportCmd(which);
380 if (resultString.length() > 0) {
return resultString; }
387 std::string tmpString = app_name +
" run number = ";
388 tmpString.append(boost::lexical_cast<std::string>(run_id_.run()));
389 tmpString.append(
". Command=\"" + which +
"\" is not currently supported.");
395 TLOG(TLVL_DEBUG) <<
"metaCommand method called with "
396 <<
"command = \"" << command <<
"\""
397 <<
", arg = \"" << arg <<
"\""
400 if (generator_ptr_)
return generator_ptr_->metaCommand(command, arg);
405 std::string artdaq::BoardReaderCore::buildStatisticsString_()
407 std::ostringstream oss;
408 oss << app_name <<
" statistics:" << std::endl;
410 double fragmentCount = 1.0;
411 artdaq::MonitoredQuantityPtr mqPtr = artdaq::StatisticsCollection::getInstance().
412 getMonitoredQuantity(FRAGMENTS_PROCESSED_STAT_KEY);
413 if (mqPtr.get() != 0)
415 artdaq::MonitoredQuantityStats stats;
416 mqPtr->getStats(stats);
417 oss <<
" Fragment statistics: "
418 << stats.recentSampleCount <<
" fragments received at "
419 << stats.recentSampleRate <<
" fragments/sec, effective data rate = "
420 << (stats.recentValueRate *
sizeof(artdaq::RawDataType)
421 / 1024.0 / 1024.0) <<
" MB/sec, monitor window = "
422 << stats.recentDuration <<
" sec, min::max event size = "
423 << (stats.recentValueMin *
sizeof(artdaq::RawDataType)
426 << (stats.recentValueMax *
sizeof(artdaq::RawDataType)
428 <<
" MB" << std::endl;
429 fragmentCount = std::max(
double(stats.recentSampleCount), 1.0);
430 oss <<
" Average times per fragment: ";
431 if (stats.recentSampleRate > 0.0)
433 oss <<
" elapsed time = "
434 << (1.0 / stats.recentSampleRate) <<
" sec";
446 mqPtr = artdaq::StatisticsCollection::getInstance().
447 getMonitoredQuantity(INPUT_WAIT_STAT_KEY);
448 if (mqPtr.get() != 0)
450 oss <<
", input wait time = "
451 << (mqPtr->getRecentValueSum() / fragmentCount) <<
" sec";
454 mqPtr = artdaq::StatisticsCollection::getInstance().
455 getMonitoredQuantity(BRSYNC_WAIT_STAT_KEY);
456 if (mqPtr.get() != 0)
458 oss <<
", BRsync wait time = "
459 << (mqPtr->getRecentValueSum() / fragmentCount) <<
" sec";
462 mqPtr = artdaq::StatisticsCollection::getInstance().
463 getMonitoredQuantity(OUTPUT_WAIT_STAT_KEY);
464 if (mqPtr.get() != 0)
466 oss <<
", output wait time = "
467 << (mqPtr->getRecentValueSum() / fragmentCount) <<
" sec";
470 oss << std::endl <<
" Fragments per read: ";
471 mqPtr = artdaq::StatisticsCollection::getInstance().
472 getMonitoredQuantity(FRAGMENTS_PER_READ_STAT_KEY);
473 if (mqPtr.get() != 0)
475 artdaq::MonitoredQuantityStats stats;
476 mqPtr->getStats(stats);
478 << stats.recentValueAverage
480 << stats.recentValueMin
482 << stats.recentValueMax;
488 void artdaq::BoardReaderCore::sendMetrics_()
491 double fragmentCount = 1.0;
492 artdaq::MonitoredQuantityPtr mqPtr = artdaq::StatisticsCollection::getInstance().
493 getMonitoredQuantity(FRAGMENTS_PROCESSED_STAT_KEY);
494 if (mqPtr.get() != 0)
496 artdaq::MonitoredQuantityStats stats;
497 mqPtr->getStats(stats);
498 fragmentCount = std::max(
double(stats.recentSampleCount), 1.0);
499 metricMan->sendMetric(
"Fragment Count", static_cast<unsigned long>(stats.fullSampleCount),
"fragments", 1, MetricMode::LastPoint);
500 metricMan->sendMetric(
"Fragment Rate", stats.recentSampleRate,
"fragments/sec", 1, MetricMode::Average);
501 metricMan->sendMetric(
"Average Fragment Size", (stats.recentValueAverage *
sizeof(artdaq::RawDataType)),
"bytes/fragment", 2, MetricMode::Average);
502 metricMan->sendMetric(
"Data Rate", (stats.recentValueRate *
sizeof(artdaq::RawDataType)),
"bytes/sec", 2, MetricMode::Average);
513 mqPtr = artdaq::StatisticsCollection::getInstance().
514 getMonitoredQuantity(INPUT_WAIT_STAT_KEY);
515 if (mqPtr.get() != 0)
517 metricMan->sendMetric(
"Avg Input Wait Time", (mqPtr->getRecentValueSum() / fragmentCount),
"seconds/fragment", 3, MetricMode::Average);
520 mqPtr = artdaq::StatisticsCollection::getInstance().
521 getMonitoredQuantity(BRSYNC_WAIT_STAT_KEY);
522 if (mqPtr.get() != 0)
524 metricMan->sendMetric(
"Avg BoardReader Sync Wait Time", (mqPtr->getRecentValueSum() / fragmentCount),
"seconds/fragment", 3, MetricMode::Average);
527 mqPtr = artdaq::StatisticsCollection::getInstance().
528 getMonitoredQuantity(OUTPUT_WAIT_STAT_KEY);
529 if (mqPtr.get() != 0)
531 metricMan->sendMetric(
"Avg Output Wait Time", (mqPtr->getRecentValueSum() / fragmentCount),
"seconds/fragment", 3, MetricMode::Average);
534 mqPtr = artdaq::StatisticsCollection::getInstance().
535 getMonitoredQuantity(FRAGMENTS_PER_READ_STAT_KEY);
536 if (mqPtr.get() != 0)
538 metricMan->sendMetric(
"Avg Frags Per Read", mqPtr->getRecentValueAverage(),
"fragments/read", 4, MetricMode::Average);
542 void artdaq::BoardReaderCore::logMessage_(std::string
const& text)
546 TLOG(TLVL_INFO) << text;
550 TLOG(TLVL_DEBUG) << text;
void addMonitoredQuantityName(std::string const &statKey)
Add a MonitoredQuantity name to the list.
Commandable is the base class for all artdaq components which implement the artdaq state machine...
Sends Fragment objects using TransferInterface plugins. Uses Routing Tables if confgiured, otherwise will Round-Robin Fragments to the destinations.
static const std::string FRAGMENTS_PROCESSED_STAT_KEY
Key for the Fragments Processed MonitoredQuantity.
static const std::string INPUT_WAIT_STAT_KEY
Key for the Input Wait MonitoredQuantity.
bool stop(uint64_t timeout, uint64_t timestamp)
Stop the BoardReader, and the CommandableFragmentGenerator.
virtual ~BoardReaderCore()
BoardReaderCore Destructor.
static std::string CopyStatusToString(CopyStatus in)
Convert a CopyStatus variable to its string represenatation
BoardReaderCore(Commandable &parent_application)
BoardReaderCore Constructor.
std::unique_ptr< CommandableFragmentGenerator > makeCommandableFragmentGenerator(std::string const &generator_plugin_spec, fhicl::ParameterSet const &ps)
Load a CommandableFragmentGenerator plugin.
bool reinitialize(fhicl::ParameterSet const &pset, uint64_t, uint64_t)
Reinitialize the BoardReader. No-Op.
bool soft_initialize(fhicl::ParameterSet const &pset, uint64_t, uint64_t)
Soft-Initialize the BoardReader. No-Op.
static const std::string BRSYNC_WAIT_STAT_KEY
Key for the Sync Wait MonitoredQuantity.
static const std::string FRAGMENTS_PER_READ_STAT_KEY
Key for the Fragments Per Read MonitoredQuantity.
static const std::string OUTPUT_WAIT_STAT_KEY
Key for the Output Wait MonitoredQuantity.
bool initialize(fhicl::ParameterSet const &pset, uint64_t, uint64_t)
Initialize the BoardReaderCore.
std::string report(std::string const &which) const
Send a report on a given run-time quantity.
void process_fragments()
Main working loop of the BoardReaderCore.
bool start(art::RunID id, uint64_t timeout, uint64_t timestamp)
Start the BoardReader, and the CommandableFragmentGenerator.
bool resume(uint64_t timeout, uint64_t timestamp)
Resume the BoardReader, and the CommandableFragmentGenerator.
bool pause(uint64_t timeout, uint64_t timestamp)
Pause the BoardReader, and the CommandableFragmentGenerator.
bool shutdown(uint64_t)
Shutdown the BoardReader, and the CommandableFragmentGenerator.
bool metaCommand(std::string const &command, std::string const &arg)
Run a user-defined command on the CommandableFragmentGenerator.