2 #include "artdaq/DAQdata/Globals.hh"
3 #define TRACE_NAME (app_name + "_BoardReaderCore").c_str()
5 #include "artdaq-core/Data/Fragment.hh"
6 #include "artdaq-core/Utilities/ExceptionHandler.hh"
7 #include "artdaq/Application/BoardReaderCore.hh"
8 #include "artdaq/Application/TaskType.hh"
9 #include "artdaq/Generators/makeCommandableFragmentGenerator.hh"
14 #include "canvas/Utilities/Exception.h"
15 #include "cetlib_except/exception.h"
28 std::unique_ptr<artdaq::DataSenderManager> artdaq::BoardReaderCore::sender_ptr_ =
nullptr;
31 : parent_application_(parent_application)
33 , generator_ptr_(nullptr)
34 , run_id_(art::RunID::flushRun())
36 , stop_requested_(false)
37 , pause_requested_(false)
39 TLOG(TLVL_DEBUG) <<
"Constructor";
49 TLOG(TLVL_DEBUG) <<
"Destructor";
54 TLOG(TLVL_DEBUG) <<
"initialize method called with "
55 <<
"ParameterSet = \"" << pset.to_string() <<
"\".";
58 fhicl::ParameterSet daq_pset;
61 daq_pset = pset.get<fhicl::ParameterSet>(
"daq");
66 <<
"Unable to find the DAQ parameters in the initialization "
67 <<
"ParameterSet: \"" + pset.to_string() +
"\".";
70 fhicl::ParameterSet fr_pset;
73 fr_pset = daq_pset.get<fhicl::ParameterSet>(
"fragment_receiver");
79 <<
"Unable to find the fragment_receiver parameters in the DAQ "
80 <<
"initialization ParameterSet: \"" + daq_pset.to_string() +
"\".";
85 fhicl::ParameterSet metric_pset;
88 metric_pset = daq_pset.get<fhicl::ParameterSet>(
"metrics");
93 if (metric_pset.is_empty())
95 TLOG(TLVL_INFO) <<
"No metric plugins appear to be defined";
99 metricMan->initialize(metric_pset, app_name);
103 ExceptionHandler(ExceptionHandlerRethrow::no,
104 "Error loading metrics in BoardReaderCore::initialize()");
107 if (daq_pset.has_key(
"rank"))
109 if (my_rank >= 0 && daq_pset.get<
int>(
"rank") != my_rank)
111 TLOG(TLVL_WARNING) <<
"BoardReader rank specified at startup is different than rank specified at configure! Using rank received at configure!";
113 my_rank = daq_pset.get<
int>(
"rank");
117 TLOG(TLVL_ERROR) <<
"BoardReader rank not specified at startup or in configuration! Aborting";
122 std::string frag_gen_name = fr_pset.get<std::string>(
"generator",
"");
123 if (frag_gen_name.length() == 0)
126 <<
"No fragment generator (parameter name = \"generator\") was "
127 <<
"specified in the fragment_receiver ParameterSet. The "
128 <<
"DAQ initialization PSet was \"" << daq_pset.to_string() <<
"\".";
138 std::stringstream exception_string;
139 exception_string <<
"Exception thrown during initialization of fragment generator of type \""
140 << frag_gen_name <<
"\"";
142 ExceptionHandler(ExceptionHandlerRethrow::no, exception_string.str());
144 TLOG(TLVL_DEBUG) <<
"FHiCL parameter set used to initialize the fragment generator which threw an exception: " << fr_pset.to_string();
148 metricMan->setPrefix(generator_ptr_->metricsReportingInstanceName());
150 rt_priority_ = fr_pset.get<
int>(
"rt_priority", 0);
153 statsHelper_.createCollectors(fr_pset, 100, 30.0, 60.0, FRAGMENTS_PROCESSED_STAT_KEY);
156 skip_seqId_test_ = (generator_ptr_->fragmentIDs().size() > 1 || generator_ptr_->request_mode() != RequestMode::Ignored);
158 verbose_ = fr_pset.get<
bool>(
"verbose",
true);
165 logMessage_(
"Starting run " + boost::lexical_cast<std::string>(
id.run()));
166 stop_requested_.store(
false);
167 pause_requested_.store(
false);
171 statsHelper_.resetStatistics();
173 metricMan->do_start();
174 generator_ptr_->StartCmd(
id.run(), timeout, timestamp);
177 logMessage_(
"Completed the Start transition (Started run) for run " +
178 boost::lexical_cast<std::string>(run_id_.run()) +
179 ", timeout = " + boost::lexical_cast<std::string>(timeout) +
180 ", timestamp = " + boost::lexical_cast<std::string>(timestamp));
186 logMessage_(
"Stopping run " + boost::lexical_cast<std::string>(run_id_.run()) +
187 " after " + boost::lexical_cast<std::string>(fragment_count_) +
" fragments.");
188 stop_requested_.store(
true);
190 TLOG(TLVL_DEBUG) <<
"Stopping CommandableFragmentGenerator BEGIN";
191 generator_ptr_->StopCmd(timeout, timestamp);
192 TLOG(TLVL_DEBUG) <<
"Stopping CommandableFragmentGenerator END";
194 TLOG(TLVL_DEBUG) <<
"Stopping DataSenderManager";
195 if (sender_ptr_) sender_ptr_->StopSender();
197 logMessage_(
"Completed the Stop transition for run " + boost::lexical_cast<std::string>(run_id_.run()));
203 logMessage_(
"Pausing run " + boost::lexical_cast<std::string>(run_id_.run()) +
204 " after " + boost::lexical_cast<std::string>(fragment_count_) +
" fragments.");
205 pause_requested_.store(
true);
206 generator_ptr_->PauseCmd(timeout, timestamp);
207 logMessage_(
"Completed the Pause transition for run " + boost::lexical_cast<std::string>(run_id_.run()));
213 logMessage_(
"Resuming run " + boost::lexical_cast<std::string>(run_id_.run()));
214 pause_requested_.store(
false);
215 metricMan->do_start();
216 generator_ptr_->ResumeCmd(timeout, timestamp);
217 logMessage_(
"Completed the Resume transition for run " + boost::lexical_cast<std::string>(run_id_.run()));
223 logMessage_(
"Starting Shutdown transition");
224 generator_ptr_->joinThreads();
225 generator_ptr_.reset(
nullptr);
226 metricMan->shutdown();
227 logMessage_(
"Completed Shutdown transition");
233 TLOG(TLVL_DEBUG) <<
"soft_initialize method called with "
234 <<
"ParameterSet = \"" << pset.to_string()
235 <<
"\". Forwarding to initialize.";
236 return initialize(pset, timeout, timestamp);
241 TLOG(TLVL_DEBUG) <<
"reinitialize method called with "
242 <<
"ParameterSet = \"" << pset.to_string()
243 <<
"\". Forwarding to initalize.";
244 return initialize(pset, timeout, timestamp);
249 if (rt_priority_ > 0)
251 #pragma GCC diagnostic push
252 #pragma GCC diagnostic ignored "-Wmissing-field-initializers"
253 sched_param s_param = {};
254 s_param.sched_priority = rt_priority_;
255 if (pthread_setschedparam(pthread_self(), SCHED_RR, &s_param))
256 TLOG(TLVL_WARNING) <<
"setting realtime priority failed";
257 #pragma GCC diagnostic pop
263 if (rt_priority_ > 0)
265 #pragma GCC diagnostic push
266 #pragma GCC diagnostic ignored "-Wmissing-field-initializers"
267 sched_param s_param = {};
268 s_param.sched_priority = rt_priority_;
269 int status = pthread_setschedparam(pthread_self(), SCHED_RR, &s_param);
273 <<
"Failed to set realtime priority to " << rt_priority_
274 <<
", return code = " << status;
276 #pragma GCC diagnostic pop
279 TLOG(TLVL_DEBUG) <<
"Initializing DataSenderManager. my_rank=" << my_rank;
282 TLOG(TLVL_DEBUG) <<
"Waiting for first fragment.";
283 artdaq::MonitoredQuantityStats::TIME_POINT_T startTime;
285 artdaq::FragmentPtrs frags;
286 auto targetFragCount = generator_ptr_->fragmentIDs().size();
292 startTime = artdaq::MonitoredQuantity::getCurrentTime();
294 TLOG(18) <<
"process_fragments getNext start";
295 active = generator_ptr_->getNext(frags);
296 TLOG(18) <<
"process_fragments getNext done (active=" << active <<
")";
303 if (!active && generator_ptr_ && generator_ptr_->exception())
305 parent_application_.in_run_failure();
308 delta_time = artdaq::MonitoredQuantity::getCurrentTime() - startTime;
309 statsHelper_.addSample(INPUT_WAIT_STAT_KEY, delta_time);
311 TLOG(16) <<
"process_fragments INPUT_WAIT=" << delta_time;
313 if (!active) {
break; }
314 statsHelper_.addSample(FRAGMENTS_PER_READ_STAT_KEY, frags.size());
316 for (
auto& fragPtr : frags)
320 TLOG(TLVL_WARNING) <<
"Encountered a bad fragment pointer in fragment " << fragment_count_ <<
". "
321 <<
"This is most likely caused by a problem with the Fragment Generator!";
324 if (fragment_count_ == 0)
326 TLOG(TLVL_DEBUG) <<
"Received first Fragment from Fragment Generator, sequence ID " << fragPtr->sequenceID() <<
", size = " << fragPtr->sizeBytes() <<
" bytes.";
328 artdaq::Fragment::sequence_id_t sequence_id = fragPtr->sequenceID();
329 SetMFIteration(
"Sequence ID " + std::to_string(sequence_id));
330 statsHelper_.addSample(FRAGMENTS_PROCESSED_STAT_KEY, fragPtr->size());
340 if (!skip_seqId_test_ && abs(static_cast<int64_t>(sequence_id) - static_cast<int64_t>(prev_seq_id_)) > 1)
343 <<
"Missing sequence IDs: current sequence ID = "
344 << sequence_id <<
", previous sequence ID = "
345 << prev_seq_id_ <<
".";
347 prev_seq_id_ = sequence_id;
349 startTime = artdaq::MonitoredQuantity::getCurrentTime();
350 TLOG(17) <<
"process_fragments seq=" << sequence_id <<
" sendFragment start";
351 auto res = sender_ptr_->sendFragment(std::move(*fragPtr));
352 if (sender_ptr_->GetSentSequenceIDCount(sequence_id) == targetFragCount)
354 sender_ptr_->RemoveRoutingTableEntry(sequence_id);
358 statsHelper_.addSample(OUTPUT_WAIT_STAT_KEY,
359 artdaq::MonitoredQuantity::getCurrentTime() - startTime);
361 bool readyToReport = statsHelper_.readyToReport();
363 TLOG(TLVL_INFO) << buildStatisticsString_();
366 TLOG(((fragment_count_ == 1) ? TLVL_DEBUG
367 : (((fragment_count_ % 250) == 0 || readyToReport) ? 13 : 14)))
368 << ((fragment_count_ == 1)
369 ?
"Sent first Fragment"
370 :
"Sending fragment " + std::to_string(fragment_count_))
371 <<
" with SeqID " << sequence_id <<
".";
373 if (statsHelper_.statsRollingWindowHasMoved()) { sendMetrics_(); }
377 sender_ptr_.reset(
nullptr);
382 metricMan->do_stop();
384 TLOG(TLVL_DEBUG) <<
"process_fragments loop end";
389 std::string resultString;
392 if (generator_ptr_.get() != 0 && which !=
"core")
394 resultString = generator_ptr_->ReportCmd(which);
395 if (resultString.length() > 0) {
return resultString; }
402 std::string tmpString = app_name +
" run number = ";
403 tmpString.append(boost::lexical_cast<std::string>(run_id_.run()));
405 tmpString.append(
", Sent Fragment count = ");
406 tmpString.append(boost::lexical_cast<std::string>(fragment_count_));
408 if (which !=
"" && which !=
"core")
410 tmpString.append(
". Command=\"" + which +
"\" is not currently supported.");
417 TLOG(TLVL_DEBUG) <<
"metaCommand method called with "
418 <<
"command = \"" << command <<
"\""
419 <<
", arg = \"" << arg <<
"\""
422 if (generator_ptr_)
return generator_ptr_->metaCommand(command, arg);
427 std::string artdaq::BoardReaderCore::buildStatisticsString_()
429 std::ostringstream oss;
430 oss << app_name <<
" statistics:" << std::endl;
432 double fragmentCount = 1.0;
433 artdaq::MonitoredQuantityPtr mqPtr = artdaq::StatisticsCollection::getInstance().getMonitoredQuantity(FRAGMENTS_PROCESSED_STAT_KEY);
434 if (mqPtr.get() != 0)
436 artdaq::MonitoredQuantityStats stats;
437 mqPtr->getStats(stats);
438 oss <<
" Fragment statistics: "
439 << stats.recentSampleCount <<
" fragments received at "
440 << stats.recentSampleRate <<
" fragments/sec, effective data rate = "
441 << (stats.recentValueRate *
sizeof(artdaq::RawDataType) / 1024.0 / 1024.0) <<
" MB/sec, monitor window = "
442 << stats.recentDuration <<
" sec, min::max event size = "
443 << (stats.recentValueMin *
sizeof(artdaq::RawDataType) / 1024.0 / 1024.0)
445 << (stats.recentValueMax *
sizeof(artdaq::RawDataType) / 1024.0 / 1024.0)
446 <<
" MB" << std::endl;
447 fragmentCount = std::max(
double(stats.recentSampleCount), 1.0);
448 oss <<
" Average times per fragment: ";
449 if (stats.recentSampleRate > 0.0)
451 oss <<
" elapsed time = "
452 << (1.0 / stats.recentSampleRate) <<
" sec";
464 mqPtr = artdaq::StatisticsCollection::getInstance().getMonitoredQuantity(INPUT_WAIT_STAT_KEY);
465 if (mqPtr.get() != 0)
467 oss <<
", input wait time = "
468 << (mqPtr->getRecentValueSum() / fragmentCount) <<
" sec";
471 mqPtr = artdaq::StatisticsCollection::getInstance().getMonitoredQuantity(BRSYNC_WAIT_STAT_KEY);
472 if (mqPtr.get() != 0)
474 oss <<
", BRsync wait time = "
475 << (mqPtr->getRecentValueSum() / fragmentCount) <<
" sec";
478 mqPtr = artdaq::StatisticsCollection::getInstance().getMonitoredQuantity(OUTPUT_WAIT_STAT_KEY);
479 if (mqPtr.get() != 0)
481 oss <<
", output wait time = "
482 << (mqPtr->getRecentValueSum() / fragmentCount) <<
" sec";
486 <<
" Fragments per read: ";
487 mqPtr = artdaq::StatisticsCollection::getInstance().getMonitoredQuantity(FRAGMENTS_PER_READ_STAT_KEY);
488 if (mqPtr.get() != 0)
490 artdaq::MonitoredQuantityStats stats;
491 mqPtr->getStats(stats);
493 << stats.recentValueAverage
495 << stats.recentValueMin
497 << stats.recentValueMax;
503 void artdaq::BoardReaderCore::sendMetrics_()
506 double fragmentCount = 1.0;
507 artdaq::MonitoredQuantityPtr mqPtr = artdaq::StatisticsCollection::getInstance().getMonitoredQuantity(FRAGMENTS_PROCESSED_STAT_KEY);
508 if (mqPtr.get() != 0)
510 artdaq::MonitoredQuantityStats stats;
511 mqPtr->getStats(stats);
512 fragmentCount = std::max(
double(stats.recentSampleCount), 1.0);
513 metricMan->sendMetric(
"Fragment Count", static_cast<unsigned long>(stats.fullSampleCount),
"fragments", 1, MetricMode::LastPoint);
514 metricMan->sendMetric(
"Fragment Rate", stats.recentSampleRate,
"fragments/sec", 1, MetricMode::Average);
515 metricMan->sendMetric(
"Average Fragment Size", (stats.recentValueAverage *
sizeof(artdaq::RawDataType)),
"bytes/fragment", 2, MetricMode::Average);
516 metricMan->sendMetric(
"Data Rate", (stats.recentValueRate *
sizeof(artdaq::RawDataType)),
"bytes/sec", 2, MetricMode::Average);
527 mqPtr = artdaq::StatisticsCollection::getInstance().getMonitoredQuantity(INPUT_WAIT_STAT_KEY);
528 if (mqPtr.get() != 0)
530 metricMan->sendMetric(
"Avg Input Wait Time", (mqPtr->getRecentValueSum() / fragmentCount),
"seconds/fragment", 3, MetricMode::Average);
533 mqPtr = artdaq::StatisticsCollection::getInstance().getMonitoredQuantity(BRSYNC_WAIT_STAT_KEY);
534 if (mqPtr.get() != 0)
536 metricMan->sendMetric(
"Avg BoardReader Sync Wait Time", (mqPtr->getRecentValueSum() / fragmentCount),
"seconds/fragment", 3, MetricMode::Average);
539 mqPtr = artdaq::StatisticsCollection::getInstance().getMonitoredQuantity(OUTPUT_WAIT_STAT_KEY);
540 if (mqPtr.get() != 0)
542 metricMan->sendMetric(
"Avg Output Wait Time", (mqPtr->getRecentValueSum() / fragmentCount),
"seconds/fragment", 3, MetricMode::Average);
545 mqPtr = artdaq::StatisticsCollection::getInstance().getMonitoredQuantity(FRAGMENTS_PER_READ_STAT_KEY);
546 if (mqPtr.get() != 0)
548 metricMan->sendMetric(
"Avg Frags Per Read", mqPtr->getRecentValueAverage(),
"fragments/read", 4, MetricMode::Average);
552 void artdaq::BoardReaderCore::logMessage_(std::string
const& text)
556 TLOG(TLVL_INFO) << text;
560 TLOG(TLVL_DEBUG) << text;
void addMonitoredQuantityName(std::string const &statKey)
Add a MonitoredQuantity name to the list.
Commandable is the base class for all artdaq components which implement the artdaq state machine...
Sends Fragment objects using TransferInterface plugins. Uses Routing Tables if confgiured, otherwise will Round-Robin Fragments to the destinations.
bool initialize(fhicl::ParameterSet const &pset, uint64_t timeout, uint64_t timestamp)
Initialize the BoardReaderCore.
static const std::string FRAGMENTS_PROCESSED_STAT_KEY
Key for the Fragments Processed MonitoredQuantity.
bool reinitialize(fhicl::ParameterSet const &pset, uint64_t timeout, uint64_t timestamp)
Reinitialize the BoardReader. No-Op.
static const std::string INPUT_WAIT_STAT_KEY
Key for the Input Wait MonitoredQuantity.
bool stop(uint64_t timeout, uint64_t timestamp)
Stop the BoardReader, and the CommandableFragmentGenerator.
virtual ~BoardReaderCore()
BoardReaderCore Destructor.
static std::string CopyStatusToString(CopyStatus in)
Convert a CopyStatus variable to its string represenatation
BoardReaderCore(Commandable &parent_application)
BoardReaderCore Constructor.
std::unique_ptr< CommandableFragmentGenerator > makeCommandableFragmentGenerator(std::string const &generator_plugin_spec, fhicl::ParameterSet const &ps)
Load a CommandableFragmentGenerator plugin.
static const std::string BRSYNC_WAIT_STAT_KEY
Key for the Sync Wait MonitoredQuantity.
static const std::string FRAGMENTS_PER_READ_STAT_KEY
Key for the Fragments Per Read MonitoredQuantity.
static const std::string OUTPUT_WAIT_STAT_KEY
Key for the Output Wait MonitoredQuantity.
bool soft_initialize(fhicl::ParameterSet const &pset, uint64_t timeout, uint64_t timestamp)
Soft-Initialize the BoardReader. No-Op.
std::string report(std::string const &which) const
Send a report on a given run-time quantity.
void process_fragments()
Main working loop of the BoardReaderCore.
bool shutdown(uint64_t timeout)
Shutdown the BoardReader, and the CommandableFragmentGenerator.
bool start(art::RunID id, uint64_t timeout, uint64_t timestamp)
Start the BoardReader, and the CommandableFragmentGenerator.
bool resume(uint64_t timeout, uint64_t timestamp)
Resume the BoardReader, and the CommandableFragmentGenerator.
bool pause(uint64_t timeout, uint64_t timestamp)
Pause the BoardReader, and the CommandableFragmentGenerator.
bool metaCommand(std::string const &command, std::string const &arg)
Run a user-defined command on the CommandableFragmentGenerator.