2 #include "artdaq/DAQdata/Globals.hh"
3 #define TRACE_NAME (app_name + "_BoardReaderCore").c_str()
5 #include "artdaq-core/Data/Fragment.hh"
6 #include "artdaq-core/Utilities/ExceptionHandler.hh"
7 #include "artdaq/Application/BoardReaderCore.hh"
8 #include "artdaq/Application/TaskType.hh"
9 #include "artdaq/Generators/makeCommandableFragmentGenerator.hh"
14 #include "canvas/Utilities/Exception.h"
15 #include "cetlib_except/exception.h"
28 std::unique_ptr<artdaq::DataSenderManager> artdaq::BoardReaderCore::sender_ptr_ =
nullptr;
31 : parent_application_(parent_application)
33 , generator_ptr_(nullptr)
34 , run_id_(art::RunID::flushRun())
36 , stop_requested_(false)
37 , pause_requested_(false)
39 TLOG(TLVL_DEBUG) <<
"Constructor";
49 TLOG(TLVL_DEBUG) <<
"Destructor";
54 TLOG(TLVL_DEBUG) <<
"initialize method called with "
55 <<
"ParameterSet = \"" << pset.to_string() <<
"\".";
58 fhicl::ParameterSet daq_pset;
61 daq_pset = pset.get<fhicl::ParameterSet>(
"daq");
66 <<
"Unable to find the DAQ parameters in the initialization "
67 <<
"ParameterSet: \"" + pset.to_string() +
"\".";
70 fhicl::ParameterSet fr_pset;
73 fr_pset = daq_pset.get<fhicl::ParameterSet>(
"fragment_receiver");
79 <<
"Unable to find the fragment_receiver parameters in the DAQ "
80 <<
"initialization ParameterSet: \"" + daq_pset.to_string() +
"\".";
85 fhicl::ParameterSet metric_pset;
88 metric_pset = daq_pset.get<fhicl::ParameterSet>(
"metrics");
93 if (metric_pset.is_empty())
95 TLOG(TLVL_INFO) <<
"No metric plugins appear to be defined";
99 metricMan->initialize(metric_pset, app_name);
103 ExceptionHandler(ExceptionHandlerRethrow::no,
104 "Error loading metrics in BoardReaderCore::initialize()");
107 if (daq_pset.has_key(
"rank"))
109 if (my_rank >= 0 && daq_pset.get<
int>(
"rank") != my_rank)
111 TLOG(TLVL_WARNING) <<
"BoardReader rank specified at startup is different than rank specified at configure! Using rank received at configure!";
113 my_rank = daq_pset.get<
int>(
"rank");
117 TLOG(TLVL_ERROR) <<
"BoardReader rank not specified at startup or in configuration! Aborting";
122 std::string frag_gen_name = fr_pset.get<std::string>(
"generator",
"");
123 if (frag_gen_name.length() == 0)
126 <<
"No fragment generator (parameter name = \"generator\") was "
127 <<
"specified in the fragment_receiver ParameterSet. The "
128 <<
"DAQ initialization PSet was \"" << daq_pset.to_string() <<
"\".";
138 std::stringstream exception_string;
139 exception_string <<
"Exception thrown during initialization of fragment generator of type \""
140 << frag_gen_name <<
"\"";
142 ExceptionHandler(ExceptionHandlerRethrow::no, exception_string.str());
144 TLOG(TLVL_DEBUG) <<
"FHiCL parameter set used to initialize the fragment generator which threw an exception: " << fr_pset.to_string();
148 metricMan->setPrefix(generator_ptr_->metricsReportingInstanceName());
150 rt_priority_ = fr_pset.get<
int>(
"rt_priority", 0);
153 statsHelper_.createCollectors(fr_pset, 100, 30.0, 60.0, FRAGMENTS_PROCESSED_STAT_KEY);
156 skip_seqId_test_ = (generator_ptr_->fragmentIDs().size() > 1 || generator_ptr_->request_mode() != RequestMode::Ignored);
158 verbose_ = fr_pset.get<
bool>(
"verbose",
true);
165 TLOG((verbose_ ? TLVL_INFO : TLVL_DEBUG)) <<
"Starting run " <<
id.run();
166 stop_requested_.store(
false);
167 pause_requested_.store(
false);
171 statsHelper_.resetStatistics();
173 metricMan->do_start();
174 generator_ptr_->StartCmd(
id.run(), timeout, timestamp);
177 TLOG((verbose_ ? TLVL_INFO : TLVL_DEBUG)) <<
"Completed the Start transition (Started run) for run " << run_id_.run()
178 <<
", timeout = " << timeout <<
", timestamp = " << timestamp;
184 TLOG((verbose_ ? TLVL_INFO : TLVL_DEBUG)) <<
"Stopping run " << run_id_.run() <<
" after " << fragment_count_ <<
" fragments.";
185 stop_requested_.store(
true);
187 TLOG(TLVL_DEBUG) <<
"Stopping CommandableFragmentGenerator BEGIN";
188 generator_ptr_->StopCmd(timeout, timestamp);
189 TLOG(TLVL_DEBUG) <<
"Stopping CommandableFragmentGenerator END";
191 TLOG(TLVL_DEBUG) <<
"Stopping DataSenderManager";
192 if (sender_ptr_) sender_ptr_->StopSender();
194 TLOG((verbose_ ? TLVL_INFO : TLVL_DEBUG)) <<
"Completed the Stop transition for run " << run_id_.run();
200 TLOG((verbose_ ? TLVL_INFO : TLVL_DEBUG)) <<
"Pausing run " << run_id_.run() <<
" after " << fragment_count_ <<
" fragments.";
201 pause_requested_.store(
true);
202 generator_ptr_->PauseCmd(timeout, timestamp);
203 TLOG((verbose_ ? TLVL_INFO : TLVL_DEBUG)) <<
"Completed the Pause transition for run " << run_id_.run();
209 TLOG((verbose_ ? TLVL_INFO : TLVL_DEBUG)) <<
"Resuming run " << run_id_.run();
210 pause_requested_.store(
false);
211 metricMan->do_start();
212 generator_ptr_->ResumeCmd(timeout, timestamp);
213 TLOG((verbose_ ? TLVL_INFO : TLVL_DEBUG)) <<
"Completed the Resume transition for run " << run_id_.run();
219 TLOG((verbose_ ? TLVL_INFO : TLVL_DEBUG)) <<
"Starting Shutdown transition";
220 generator_ptr_->joinThreads();
221 generator_ptr_.reset(
nullptr);
222 metricMan->shutdown();
223 TLOG((verbose_ ? TLVL_INFO : TLVL_DEBUG)) <<
"Completed Shutdown transition";
229 TLOG(TLVL_DEBUG) <<
"soft_initialize method called with "
230 <<
"ParameterSet = \"" << pset.to_string()
231 <<
"\". Forwarding to initialize.";
232 return initialize(pset, timeout, timestamp);
237 TLOG(TLVL_DEBUG) <<
"reinitialize method called with "
238 <<
"ParameterSet = \"" << pset.to_string()
239 <<
"\". Forwarding to initalize.";
240 return initialize(pset, timeout, timestamp);
245 if (rt_priority_ > 0)
247 #pragma GCC diagnostic push
248 #pragma GCC diagnostic ignored "-Wmissing-field-initializers"
249 sched_param s_param = {};
250 s_param.sched_priority = rt_priority_;
251 if (pthread_setschedparam(pthread_self(), SCHED_RR, &s_param))
252 TLOG(TLVL_WARNING) <<
"setting realtime priority failed";
253 #pragma GCC diagnostic pop
259 if (rt_priority_ > 0)
261 #pragma GCC diagnostic push
262 #pragma GCC diagnostic ignored "-Wmissing-field-initializers"
263 sched_param s_param = {};
264 s_param.sched_priority = rt_priority_;
265 int status = pthread_setschedparam(pthread_self(), SCHED_RR, &s_param);
269 <<
"Failed to set realtime priority to " << rt_priority_
270 <<
", return code = " << status;
272 #pragma GCC diagnostic pop
275 TLOG(TLVL_DEBUG) <<
"Initializing DataSenderManager. my_rank=" << my_rank;
278 TLOG(TLVL_DEBUG) <<
"Waiting for first fragment.";
279 artdaq::MonitoredQuantityStats::TIME_POINT_T startTime;
281 artdaq::FragmentPtrs frags;
282 auto targetFragCount = generator_ptr_->fragmentIDs().size();
288 startTime = artdaq::MonitoredQuantity::getCurrentTime();
290 TLOG(18) <<
"process_fragments getNext start";
291 active = generator_ptr_->getNext(frags);
292 TLOG(18) <<
"process_fragments getNext done (active=" << active <<
")";
299 if (!active && generator_ptr_ && generator_ptr_->exception())
301 parent_application_.in_run_failure();
304 delta_time = artdaq::MonitoredQuantity::getCurrentTime() - startTime;
305 statsHelper_.addSample(INPUT_WAIT_STAT_KEY, delta_time);
307 TLOG(16) <<
"process_fragments INPUT_WAIT=" << delta_time;
309 if (!active) {
break; }
310 statsHelper_.addSample(FRAGMENTS_PER_READ_STAT_KEY, frags.size());
312 for (
auto& fragPtr : frags)
316 TLOG(TLVL_WARNING) <<
"Encountered a bad fragment pointer in fragment " << fragment_count_ <<
". "
317 <<
"This is most likely caused by a problem with the Fragment Generator!";
320 if (fragment_count_ == 0)
322 TLOG(TLVL_DEBUG) <<
"Received first Fragment from Fragment Generator, sequence ID " << fragPtr->sequenceID() <<
", size = " << fragPtr->sizeBytes() <<
" bytes.";
324 artdaq::Fragment::sequence_id_t sequence_id = fragPtr->sequenceID();
325 SetMFIteration(
"Sequence ID " + std::to_string(sequence_id));
326 statsHelper_.addSample(FRAGMENTS_PROCESSED_STAT_KEY, fragPtr->size());
336 if (!skip_seqId_test_ && abs(static_cast<int64_t>(sequence_id) - static_cast<int64_t>(prev_seq_id_)) > 1)
339 <<
"Missing sequence IDs: current sequence ID = "
340 << sequence_id <<
", previous sequence ID = "
341 << prev_seq_id_ <<
".";
343 prev_seq_id_ = sequence_id;
345 startTime = artdaq::MonitoredQuantity::getCurrentTime();
346 TLOG(17) <<
"process_fragments seq=" << sequence_id <<
" sendFragment start";
347 auto res = sender_ptr_->sendFragment(std::move(*fragPtr));
348 if (sender_ptr_->GetSentSequenceIDCount(sequence_id) == targetFragCount)
350 sender_ptr_->RemoveRoutingTableEntry(sequence_id);
354 statsHelper_.addSample(OUTPUT_WAIT_STAT_KEY,
355 artdaq::MonitoredQuantity::getCurrentTime() - startTime);
357 bool readyToReport = statsHelper_.readyToReport();
359 TLOG(TLVL_INFO) << buildStatisticsString_();
362 TLOG(((fragment_count_ == 1) ? TLVL_DEBUG
363 : (((fragment_count_ % 250) == 0 || readyToReport) ? 13 : 14)))
364 << ((fragment_count_ == 1)
365 ?
"Sent first Fragment"
366 :
"Sending fragment " + std::to_string(fragment_count_))
367 <<
" with SeqID " << sequence_id <<
".";
369 if (statsHelper_.statsRollingWindowHasMoved()) { sendMetrics_(); }
373 sender_ptr_.reset(
nullptr);
378 metricMan->do_stop();
380 TLOG(TLVL_DEBUG) <<
"process_fragments loop end";
385 std::string resultString;
388 if (generator_ptr_.get() != 0 && which !=
"core")
390 resultString = generator_ptr_->ReportCmd(which);
391 if (resultString.length() > 0) {
return resultString; }
398 std::string tmpString = app_name +
" run number = ";
399 tmpString.append(boost::lexical_cast<std::string>(run_id_.run()));
401 tmpString.append(
", Sent Fragment count = ");
402 tmpString.append(boost::lexical_cast<std::string>(fragment_count_));
404 if (which !=
"" && which !=
"core")
406 tmpString.append(
". Command=\"" + which +
"\" is not currently supported.");
413 TLOG(TLVL_DEBUG) <<
"metaCommand method called with "
414 <<
"command = \"" << command <<
"\""
415 <<
", arg = \"" << arg <<
"\""
418 if (generator_ptr_)
return generator_ptr_->metaCommand(command, arg);
423 std::string artdaq::BoardReaderCore::buildStatisticsString_()
425 std::ostringstream oss;
426 oss << app_name <<
" statistics:" << std::endl;
428 double fragmentCount = 1.0;
429 artdaq::MonitoredQuantityPtr mqPtr = artdaq::StatisticsCollection::getInstance().getMonitoredQuantity(FRAGMENTS_PROCESSED_STAT_KEY);
430 if (mqPtr.get() != 0)
432 artdaq::MonitoredQuantityStats stats;
433 mqPtr->getStats(stats);
434 oss <<
" Fragment statistics: "
435 << stats.recentSampleCount <<
" fragments received at "
436 << stats.recentSampleRate <<
" fragments/sec, effective data rate = "
437 << (stats.recentValueRate *
sizeof(artdaq::RawDataType) / 1024.0 / 1024.0) <<
" MB/sec, monitor window = "
438 << stats.recentDuration <<
" sec, min::max event size = "
439 << (stats.recentValueMin *
sizeof(artdaq::RawDataType) / 1024.0 / 1024.0)
441 << (stats.recentValueMax *
sizeof(artdaq::RawDataType) / 1024.0 / 1024.0)
442 <<
" MB" << std::endl;
443 fragmentCount = std::max(
double(stats.recentSampleCount), 1.0);
444 oss <<
" Average times per fragment: ";
445 if (stats.recentSampleRate > 0.0)
447 oss <<
" elapsed time = "
448 << (1.0 / stats.recentSampleRate) <<
" sec";
460 mqPtr = artdaq::StatisticsCollection::getInstance().getMonitoredQuantity(INPUT_WAIT_STAT_KEY);
461 if (mqPtr.get() != 0)
463 oss <<
", input wait time = "
464 << (mqPtr->getRecentValueSum() / fragmentCount) <<
" sec";
467 mqPtr = artdaq::StatisticsCollection::getInstance().getMonitoredQuantity(BRSYNC_WAIT_STAT_KEY);
468 if (mqPtr.get() != 0)
470 oss <<
", BRsync wait time = "
471 << (mqPtr->getRecentValueSum() / fragmentCount) <<
" sec";
474 mqPtr = artdaq::StatisticsCollection::getInstance().getMonitoredQuantity(OUTPUT_WAIT_STAT_KEY);
475 if (mqPtr.get() != 0)
477 oss <<
", output wait time = "
478 << (mqPtr->getRecentValueSum() / fragmentCount) <<
" sec";
482 <<
" Fragments per read: ";
483 mqPtr = artdaq::StatisticsCollection::getInstance().getMonitoredQuantity(FRAGMENTS_PER_READ_STAT_KEY);
484 if (mqPtr.get() != 0)
486 artdaq::MonitoredQuantityStats stats;
487 mqPtr->getStats(stats);
489 << stats.recentValueAverage
491 << stats.recentValueMin
493 << stats.recentValueMax;
499 void artdaq::BoardReaderCore::sendMetrics_()
502 double fragmentCount = 1.0;
503 artdaq::MonitoredQuantityPtr mqPtr = artdaq::StatisticsCollection::getInstance().getMonitoredQuantity(FRAGMENTS_PROCESSED_STAT_KEY);
504 if (mqPtr.get() != 0)
506 artdaq::MonitoredQuantityStats stats;
507 mqPtr->getStats(stats);
508 fragmentCount = std::max(
double(stats.recentSampleCount), 1.0);
509 metricMan->sendMetric(
"Fragment Count", static_cast<unsigned long>(stats.fullSampleCount),
"fragments", 1, MetricMode::LastPoint);
510 metricMan->sendMetric(
"Fragment Rate", stats.recentSampleRate,
"fragments/sec", 1, MetricMode::Average);
511 metricMan->sendMetric(
"Average Fragment Size", (stats.recentValueAverage *
sizeof(artdaq::RawDataType)),
"bytes/fragment", 2, MetricMode::Average);
512 metricMan->sendMetric(
"Data Rate", (stats.recentValueRate *
sizeof(artdaq::RawDataType)),
"bytes/sec", 2, MetricMode::Average);
523 mqPtr = artdaq::StatisticsCollection::getInstance().getMonitoredQuantity(INPUT_WAIT_STAT_KEY);
524 if (mqPtr.get() != 0)
526 metricMan->sendMetric(
"Avg Input Wait Time", (mqPtr->getRecentValueSum() / fragmentCount),
"seconds/fragment", 3, MetricMode::Average);
529 mqPtr = artdaq::StatisticsCollection::getInstance().getMonitoredQuantity(BRSYNC_WAIT_STAT_KEY);
530 if (mqPtr.get() != 0)
532 metricMan->sendMetric(
"Avg BoardReader Sync Wait Time", (mqPtr->getRecentValueSum() / fragmentCount),
"seconds/fragment", 3, MetricMode::Average);
535 mqPtr = artdaq::StatisticsCollection::getInstance().getMonitoredQuantity(OUTPUT_WAIT_STAT_KEY);
536 if (mqPtr.get() != 0)
538 metricMan->sendMetric(
"Avg Output Wait Time", (mqPtr->getRecentValueSum() / fragmentCount),
"seconds/fragment", 3, MetricMode::Average);
541 mqPtr = artdaq::StatisticsCollection::getInstance().getMonitoredQuantity(FRAGMENTS_PER_READ_STAT_KEY);
542 if (mqPtr.get() != 0)
544 metricMan->sendMetric(
"Avg Frags Per Read", mqPtr->getRecentValueAverage(),
"fragments/read", 4, MetricMode::Average);
void addMonitoredQuantityName(std::string const &statKey)
Add a MonitoredQuantity name to the list.
Commandable is the base class for all artdaq components which implement the artdaq state machine...
Sends Fragment objects using TransferInterface plugins. Uses Routing Tables if confgiured, otherwise will Round-Robin Fragments to the destinations.
bool initialize(fhicl::ParameterSet const &pset, uint64_t timeout, uint64_t timestamp)
Initialize the BoardReaderCore.
static const std::string FRAGMENTS_PROCESSED_STAT_KEY
Key for the Fragments Processed MonitoredQuantity.
bool reinitialize(fhicl::ParameterSet const &pset, uint64_t timeout, uint64_t timestamp)
Reinitialize the BoardReader. No-Op.
static const std::string INPUT_WAIT_STAT_KEY
Key for the Input Wait MonitoredQuantity.
bool stop(uint64_t timeout, uint64_t timestamp)
Stop the BoardReader, and the CommandableFragmentGenerator.
virtual ~BoardReaderCore()
BoardReaderCore Destructor.
static std::string CopyStatusToString(CopyStatus in)
Convert a CopyStatus variable to its string represenatation
BoardReaderCore(Commandable &parent_application)
BoardReaderCore Constructor.
std::unique_ptr< CommandableFragmentGenerator > makeCommandableFragmentGenerator(std::string const &generator_plugin_spec, fhicl::ParameterSet const &ps)
Load a CommandableFragmentGenerator plugin.
static const std::string BRSYNC_WAIT_STAT_KEY
Key for the Sync Wait MonitoredQuantity.
static const std::string FRAGMENTS_PER_READ_STAT_KEY
Key for the Fragments Per Read MonitoredQuantity.
static const std::string OUTPUT_WAIT_STAT_KEY
Key for the Output Wait MonitoredQuantity.
bool soft_initialize(fhicl::ParameterSet const &pset, uint64_t timeout, uint64_t timestamp)
Soft-Initialize the BoardReader. No-Op.
std::string report(std::string const &which) const
Send a report on a given run-time quantity.
void process_fragments()
Main working loop of the BoardReaderCore.
bool shutdown(uint64_t timeout)
Shutdown the BoardReader, and the CommandableFragmentGenerator.
bool start(art::RunID id, uint64_t timeout, uint64_t timestamp)
Start the BoardReader, and the CommandableFragmentGenerator.
bool resume(uint64_t timeout, uint64_t timestamp)
Resume the BoardReader, and the CommandableFragmentGenerator.
bool pause(uint64_t timeout, uint64_t timestamp)
Pause the BoardReader, and the CommandableFragmentGenerator.
bool metaCommand(std::string const &command, std::string const &arg)
Run a user-defined command on the CommandableFragmentGenerator.