2 #define TRACE_NAME (app_name + "_BoardReaderCore").c_str() // include these 2 first -
3 #include "artdaq/DAQdata/Globals.hh"
4 #include "artdaq/Application/TaskType.hh"
5 #include "artdaq/Application/BoardReaderCore.hh"
6 #include "artdaq-core/Data/Fragment.hh"
7 #include "artdaq-core/Utilities/ExceptionHandler.hh"
8 #include "artdaq/Application/makeCommandableFragmentGenerator.hh"
9 #include "canvas/Utilities/Exception.h"
10 #include "cetlib_except/exception.h"
26 std::unique_ptr<artdaq::DataSenderManager> artdaq::BoardReaderCore::sender_ptr_ =
nullptr;
29 parent_application_(parent_application)
31 , generator_ptr_(nullptr)
32 , stop_requested_(false)
33 , pause_requested_(false)
35 TLOG(TLVL_DEBUG) <<
"Constructor";
41 metricMan = &metricMan_;
46 TLOG(TLVL_DEBUG) <<
"Destructor";
51 TLOG(TLVL_DEBUG) <<
"initialize method called with " <<
"ParameterSet = \"" << pset.to_string() <<
"\".";
54 fhicl::ParameterSet daq_pset;
57 daq_pset = pset.get<fhicl::ParameterSet>(
"daq");
62 <<
"Unable to find the DAQ parameters in the initialization "
63 <<
"ParameterSet: \"" + pset.to_string() +
"\".";
66 fhicl::ParameterSet fr_pset;
69 fr_pset = daq_pset.get<fhicl::ParameterSet>(
"fragment_receiver");
75 <<
"Unable to find the fragment_receiver parameters in the DAQ "
76 <<
"initialization ParameterSet: \"" + daq_pset.to_string() +
"\".";
81 fhicl::ParameterSet metric_pset;
84 metric_pset = daq_pset.get<fhicl::ParameterSet>(
"metrics");
88 if (metric_pset.is_empty())
90 TLOG(TLVL_INFO) <<
"No metric plugins appear to be defined";
94 metricMan_.initialize(metric_pset, app_name);
98 ExceptionHandler(ExceptionHandlerRethrow::no,
99 "Error loading metrics in BoardReaderCore::initialize()");
102 if (daq_pset.has_key(
"rank"))
104 if (my_rank >= 0 && daq_pset.get<
int>(
"rank") != my_rank) {
105 TLOG(TLVL_WARNING) <<
"BoardReader rank specified at startup is different than rank specified at configure! Using rank received at configure!";
107 my_rank = daq_pset.get<
int>(
"rank");
111 TLOG(TLVL_ERROR) <<
"BoardReader rank not specified at startup or in configuration! Aborting";
117 std::string frag_gen_name = fr_pset.get<std::string>(
"generator",
"");
118 if (frag_gen_name.length() == 0)
121 <<
"No fragment generator (parameter name = \"generator\") was "
122 <<
"specified in the fragment_receiver ParameterSet. The "
123 <<
"DAQ initialization PSet was \"" << daq_pset.to_string() <<
"\".";
133 std::stringstream exception_string;
134 exception_string <<
"Exception thrown during initialization of fragment generator of type \""
135 << frag_gen_name <<
"\"";
137 ExceptionHandler(ExceptionHandlerRethrow::no, exception_string.str());
139 TLOG(TLVL_DEBUG) <<
"FHiCL parameter set used to initialize the fragment generator which threw an exception: " << fr_pset.to_string();
143 metricMan_.setPrefix(generator_ptr_->metricsReportingInstanceName());
145 rt_priority_ = fr_pset.get<
int>(
"rt_priority", 0);
148 statsHelper_.createCollectors(fr_pset, 100, 30.0, 60.0, FRAGMENTS_PROCESSED_STAT_KEY);
151 skip_seqId_test_ = (generator_ptr_->fragmentIDs().size() > 1);
153 verbose_ = fr_pset.get<
bool>(
"verbose",
true);
160 logMessage_(
"Starting run " + boost::lexical_cast<std::string>(
id.run()));
161 stop_requested_.store(
false);
162 pause_requested_.store(
false);
166 statsHelper_.resetStatistics();
168 metricMan_.do_start();
169 generator_ptr_->StartCmd(
id.run(), timeout, timestamp);
172 logMessage_(
"Completed the Start transition (Started run) for run " +
173 boost::lexical_cast<std::string>(run_id_.run()) +
174 ", timeout = " + boost::lexical_cast<std::string>(timeout) +
175 ", timestamp = " + boost::lexical_cast<std::string>(timestamp));
181 logMessage_(
"Stopping run " + boost::lexical_cast<std::string>(run_id_.run()) +
182 " after " + boost::lexical_cast<std::string>(fragment_count_) +
" fragments.");
183 stop_requested_.store(
true);
184 generator_ptr_->StopCmd(timeout, timestamp);
185 logMessage_(
"Completed the Stop transition for run " + boost::lexical_cast<std::string>(run_id_.run()));
191 logMessage_(
"Pausing run " + boost::lexical_cast<std::string>(run_id_.run()) +
192 " after " + boost::lexical_cast<std::string>(fragment_count_) +
" fragments.");
193 pause_requested_.store(
true);
194 generator_ptr_->PauseCmd(timeout, timestamp);
195 logMessage_(
"Completed the Pause transition for run " + boost::lexical_cast<std::string>(run_id_.run()));
201 logMessage_(
"Resuming run " + boost::lexical_cast<std::string>(run_id_.run()));
202 pause_requested_.store(
false);
203 metricMan_.do_start();
204 generator_ptr_->ResumeCmd(timeout, timestamp);
205 logMessage_(
"Completed the Resume transition for run " + boost::lexical_cast<std::string>(run_id_.run()));
211 logMessage_(
"Starting Shutdown transition");
212 generator_ptr_->joinThreads();
213 generator_ptr_.reset(
nullptr);
214 metricMan_.shutdown();
215 logMessage_(
"Completed Shutdown transition");
221 TLOG(TLVL_DEBUG) <<
"soft_initialize method called with "
222 <<
"ParameterSet = \"" << pset.to_string()
229 TLOG(TLVL_DEBUG) <<
"reinitialize method called with "
230 <<
"ParameterSet = \"" << pset.to_string()
237 if (rt_priority_ > 0)
239 #pragma GCC diagnostic push
240 #pragma GCC diagnostic ignored "-Wmissing-field-initializers"
241 sched_param s_param = {};
242 s_param.sched_priority = rt_priority_;
243 if (pthread_setschedparam(pthread_self(), SCHED_RR, &s_param))
244 TLOG(TLVL_WARNING) <<
"setting realtime priority failed";
245 #pragma GCC diagnostic pop
251 if (rt_priority_ > 0)
253 #pragma GCC diagnostic push
254 #pragma GCC diagnostic ignored "-Wmissing-field-initializers"
255 sched_param s_param = {};
256 s_param.sched_priority = rt_priority_;
257 int status = pthread_setschedparam(pthread_self(), SCHED_RR, &s_param);
261 <<
"Failed to set realtime priority to " << rt_priority_
262 <<
", return code = " << status;
264 #pragma GCC diagnostic pop
267 TLOG(TLVL_DEBUG) <<
"Initializing DataSenderManager. my_rank=" << my_rank;
270 TLOG(TLVL_DEBUG) <<
"Waiting for first fragment.";
271 artdaq::MonitoredQuantityStats::TIME_POINT_T startTime;
273 artdaq::FragmentPtrs frags;
278 startTime = artdaq::MonitoredQuantity::getCurrentTime();
280 TLOG(18) <<
"process_fragments getNext start";
281 active = generator_ptr_->getNext(frags);
282 TLOG(18) <<
"process_fragments getNext done (active=" << active <<
")";
289 if (!active && generator_ptr_ && generator_ptr_->exception())
291 parent_application_.in_run_failure();
294 delta_time = artdaq::MonitoredQuantity::getCurrentTime() - startTime;
295 statsHelper_.addSample(INPUT_WAIT_STAT_KEY, delta_time);
297 TLOG(16) <<
"process_fragments INPUT_WAIT=" << std::to_string(delta_time);
299 if (!active) {
break; }
300 statsHelper_.addSample(FRAGMENTS_PER_READ_STAT_KEY, frags.size());
302 for (
auto& fragPtr : frags)
306 TLOG(TLVL_WARNING) <<
"Encountered a bad fragment pointer in fragment " << fragment_count_ <<
". "
307 <<
"This is most likely caused by a problem with the Fragment Generator!";
310 artdaq::Fragment::sequence_id_t sequence_id = fragPtr->sequenceID();
311 #if ART_HEX_VERSION >=0x21100
312 SetMFIteration(
"Sequence ID " + std::to_string(sequence_id));
314 statsHelper_.addSample(FRAGMENTS_PROCESSED_STAT_KEY, fragPtr->size());
316 if ((fragment_count_ % 250) == 0)
319 <<
"Sending fragment " << fragment_count_
320 <<
" with sequence id " << sequence_id <<
".";
324 if (!skip_seqId_test_ && abs(static_cast<int64_t>(sequence_id) - static_cast<int64_t>(prev_seq_id_)) > 1)
327 <<
"Missing sequence IDs: current sequence ID = "
328 << sequence_id <<
", previous sequence ID = "
329 << prev_seq_id_ <<
".";
331 prev_seq_id_ = sequence_id;
333 startTime = artdaq::MonitoredQuantity::getCurrentTime();
334 TLOG(17) <<
"process_fragments seq=" << std::to_string(sequence_id) <<
" sendFragment start";
335 auto res = sender_ptr_->sendFragment(std::move(*fragPtr));
336 TLOG(17) <<
"process_fragments seq=" << std::to_string(sequence_id) <<
" sendFragment done (dest=" << res.first <<
", sts=" << TransferInterface::CopyStatusToString(res.second) <<
")";
338 statsHelper_.addSample(OUTPUT_WAIT_STAT_KEY,
339 artdaq::MonitoredQuantity::getCurrentTime() - startTime);
341 bool readyToReport = statsHelper_.readyToReport(fragment_count_);
344 std::string statString = buildStatisticsString_();
345 TLOG(TLVL_DEBUG) << statString;
347 if (fragment_count_ == 1 || readyToReport)
350 <<
"Sending fragment " << fragment_count_
351 <<
" with sequence id " << sequence_id <<
".";
354 if (statsHelper_.statsRollingWindowHasMoved()) { sendMetrics_(); }
361 metricMan_.do_stop();
363 sender_ptr_.reset(
nullptr);
368 std::string resultString;
371 if (generator_ptr_.get() != 0)
373 resultString = generator_ptr_->ReportCmd(which);
374 if (resultString.length() > 0) {
return resultString; }
381 std::string tmpString = app_name +
" run number = ";
382 tmpString.append(boost::lexical_cast<std::string>(run_id_.run()));
383 tmpString.append(
". Command=\"" + which +
"\" is not currently supported.");
389 TLOG(TLVL_DEBUG) <<
"metaCommand method called with "
390 <<
"command = \"" << command <<
"\""
391 <<
", arg = \"" << arg <<
"\""
394 if (generator_ptr_)
return generator_ptr_->metaCommand(command, arg);
399 std::string artdaq::BoardReaderCore::buildStatisticsString_()
401 std::ostringstream oss;
402 oss << app_name <<
" statistics:" << std::endl;
404 double fragmentCount = 1.0;
405 artdaq::MonitoredQuantityPtr mqPtr = artdaq::StatisticsCollection::getInstance().
406 getMonitoredQuantity(FRAGMENTS_PROCESSED_STAT_KEY);
407 if (mqPtr.get() != 0)
409 artdaq::MonitoredQuantityStats stats;
410 mqPtr->getStats(stats);
411 oss <<
" Fragment statistics: "
412 << stats.recentSampleCount <<
" fragments received at "
413 << stats.recentSampleRate <<
" fragments/sec, effective data rate = "
414 << (stats.recentValueRate *
sizeof(artdaq::RawDataType)
415 / 1024.0 / 1024.0) <<
" MB/sec, monitor window = "
416 << stats.recentDuration <<
" sec, min::max event size = "
417 << (stats.recentValueMin *
sizeof(artdaq::RawDataType)
420 << (stats.recentValueMax *
sizeof(artdaq::RawDataType)
422 <<
" MB" << std::endl;
423 fragmentCount = std::max(
double(stats.recentSampleCount), 1.0);
424 oss <<
" Average times per fragment: ";
425 if (stats.recentSampleRate > 0.0)
427 oss <<
" elapsed time = "
428 << (1.0 / stats.recentSampleRate) <<
" sec";
440 mqPtr = artdaq::StatisticsCollection::getInstance().
441 getMonitoredQuantity(INPUT_WAIT_STAT_KEY);
442 if (mqPtr.get() != 0)
444 oss <<
", input wait time = "
445 << (mqPtr->getRecentValueSum() / fragmentCount) <<
" sec";
448 mqPtr = artdaq::StatisticsCollection::getInstance().
449 getMonitoredQuantity(BRSYNC_WAIT_STAT_KEY);
450 if (mqPtr.get() != 0)
452 oss <<
", BRsync wait time = "
453 << (mqPtr->getRecentValueSum() / fragmentCount) <<
" sec";
456 mqPtr = artdaq::StatisticsCollection::getInstance().
457 getMonitoredQuantity(OUTPUT_WAIT_STAT_KEY);
458 if (mqPtr.get() != 0)
460 oss <<
", output wait time = "
461 << (mqPtr->getRecentValueSum() / fragmentCount) <<
" sec";
464 oss << std::endl <<
" Fragments per read: ";
465 mqPtr = artdaq::StatisticsCollection::getInstance().
466 getMonitoredQuantity(FRAGMENTS_PER_READ_STAT_KEY);
467 if (mqPtr.get() != 0)
469 artdaq::MonitoredQuantityStats stats;
470 mqPtr->getStats(stats);
472 << stats.recentValueAverage
474 << stats.recentValueMin
476 << stats.recentValueMax;
482 void artdaq::BoardReaderCore::sendMetrics_()
485 double fragmentCount = 1.0;
486 artdaq::MonitoredQuantityPtr mqPtr = artdaq::StatisticsCollection::getInstance().
487 getMonitoredQuantity(FRAGMENTS_PROCESSED_STAT_KEY);
488 if (mqPtr.get() != 0)
490 artdaq::MonitoredQuantityStats stats;
491 mqPtr->getStats(stats);
492 fragmentCount = std::max(
double(stats.recentSampleCount), 1.0);
493 metricMan_.sendMetric(
"Fragment Count", static_cast<unsigned long>(stats.fullSampleCount),
"fragments", 1, MetricMode::LastPoint);
494 metricMan_.sendMetric(
"Fragment Rate", stats.recentSampleRate,
"fragments/sec", 1, MetricMode::Average);
495 metricMan_.sendMetric(
"Average Fragment Size", (stats.recentValueAverage *
sizeof(artdaq::RawDataType)),
"bytes/fragment", 2, MetricMode::Average);
496 metricMan_.sendMetric(
"Data Rate", (stats.recentValueRate *
sizeof(artdaq::RawDataType)),
"bytes/sec", 2, MetricMode::Average);
507 mqPtr = artdaq::StatisticsCollection::getInstance().
508 getMonitoredQuantity(INPUT_WAIT_STAT_KEY);
509 if (mqPtr.get() != 0)
511 metricMan_.sendMetric(
"Avg Input Wait Time", (mqPtr->getRecentValueSum() / fragmentCount),
"seconds/fragment", 3, MetricMode::Average);
514 mqPtr = artdaq::StatisticsCollection::getInstance().
515 getMonitoredQuantity(BRSYNC_WAIT_STAT_KEY);
516 if (mqPtr.get() != 0)
518 metricMan_.sendMetric(
"Avg BoardReader Sync Wait Time", (mqPtr->getRecentValueSum() / fragmentCount),
"seconds/fragment", 3, MetricMode::Average);
521 mqPtr = artdaq::StatisticsCollection::getInstance().
522 getMonitoredQuantity(OUTPUT_WAIT_STAT_KEY);
523 if (mqPtr.get() != 0)
525 metricMan_.sendMetric(
"Avg Output Wait Time", (mqPtr->getRecentValueSum() / fragmentCount),
"seconds/fragment", 3, MetricMode::Average);
528 mqPtr = artdaq::StatisticsCollection::getInstance().
529 getMonitoredQuantity(FRAGMENTS_PER_READ_STAT_KEY);
530 if (mqPtr.get() != 0)
532 metricMan_.sendMetric(
"Avg Frags Per Read", mqPtr->getRecentValueAverage(),
"fragments/read", 4, MetricMode::Average);
536 void artdaq::BoardReaderCore::logMessage_(std::string
const& text)
540 TLOG(TLVL_INFO) << text;
544 TLOG(TLVL_DEBUG) << text;
void addMonitoredQuantityName(std::string const &statKey)
Add a MonitoredQuantity name to the list.
Commandable is the base class for all artdaq components which implement the artdaq state machine...
Sends Fragment objects using TransferInterface plugins. Uses Routing Tables if confgiured, otherwise will Round-Robin Fragments to the destinations.
static const std::string FRAGMENTS_PROCESSED_STAT_KEY
Key for the Fragments Processed MonitoredQuantity.
static const std::string INPUT_WAIT_STAT_KEY
Key for the Input Wait MonitoredQuantity.
bool stop(uint64_t timeout, uint64_t timestamp)
Stop the BoardReader, and the CommandableFragmentGenerator.
virtual ~BoardReaderCore()
BoardReaderCore Destructor.
BoardReaderCore(Commandable &parent_application)
BoardReaderCore Constructor.
std::unique_ptr< CommandableFragmentGenerator > makeCommandableFragmentGenerator(std::string const &generator_plugin_spec, fhicl::ParameterSet const &ps)
Load a CommandableFragmentGenerator plugin.
bool reinitialize(fhicl::ParameterSet const &pset, uint64_t, uint64_t)
Reinitialize the BoardReader. No-Op.
bool soft_initialize(fhicl::ParameterSet const &pset, uint64_t, uint64_t)
Soft-Initialize the BoardReader. No-Op.
static const std::string BRSYNC_WAIT_STAT_KEY
Key for the Sync Wait MonitoredQuantity.
static const std::string FRAGMENTS_PER_READ_STAT_KEY
Key for the Fragments Per Read MonitoredQuantity.
static const std::string OUTPUT_WAIT_STAT_KEY
Key for the Output Wait MonitoredQuantity.
bool initialize(fhicl::ParameterSet const &pset, uint64_t, uint64_t)
Initialize the BoardReaderCore.
std::string report(std::string const &which) const
Send a report on a given run-time quantity.
void process_fragments()
Main working loop of the BoardReaderCore.
bool start(art::RunID id, uint64_t timeout, uint64_t timestamp)
Start the BoardReader, and the CommandableFragmentGenerator.
bool resume(uint64_t timeout, uint64_t timestamp)
Resume the BoardReader, and the CommandableFragmentGenerator.
bool pause(uint64_t timeout, uint64_t timestamp)
Pause the BoardReader, and the CommandableFragmentGenerator.
bool shutdown(uint64_t)
Shutdown the BoardReader, and the CommandableFragmentGenerator.
bool metaCommand(std::string const &command, std::string const &arg)
Run a user-defined command on the CommandableFragmentGenerator.