00001 #include "artdaq/Application/TaskType.hh"
00002 #include "artdaq/Application/BoardReaderCore.hh"
00003 #include "artdaq-core/Data/Fragment.hh"
00004 #include "artdaq-core/Utilities/ExceptionHandler.hh"
00005 #include "artdaq/Application/makeCommandableFragmentGenerator.hh"
00006 #include "canvas/Utilities/Exception.h"
00007 #include "cetlib/exception.h"
00008 #include <pthread.h>
00009 #include <sched.h>
00010 #include <algorithm>
00011
00012 #define TRACE_NAME "BoardReaderCore"
00013
00014 const std::string artdaq::BoardReaderCore::
00015 FRAGMENTS_PROCESSED_STAT_KEY("BoardReaderCoreFragmentsProcessed");
00016 const std::string artdaq::BoardReaderCore::
00017 INPUT_WAIT_STAT_KEY("BoardReaderCoreInputWaitTime");
00018 const std::string artdaq::BoardReaderCore::
00019 BRSYNC_WAIT_STAT_KEY("BoardReaderCoreBRSyncWaitTime");
00020 const std::string artdaq::BoardReaderCore::
00021 OUTPUT_WAIT_STAT_KEY("BoardReaderCoreOutputWaitTime");
00022 const std::string artdaq::BoardReaderCore::
00023 FRAGMENTS_PER_READ_STAT_KEY("BoardReaderCoreFragmentsPerRead");
00024
00025 std::unique_ptr<artdaq::DataSenderManager> artdaq::BoardReaderCore::sender_ptr_ = nullptr;
00026
00027 artdaq::BoardReaderCore::BoardReaderCore(Commandable& parent_application,
00028 int rank, std::string name) :
00029 parent_application_(parent_application)
00030
00031 , generator_ptr_(nullptr)
00032 , name_(name)
00033 , stop_requested_(false)
00034 , pause_requested_(false)
00035 {
00036 TLOG_DEBUG(name_) << "Constructor" << TLOG_ENDL;
00037 statsHelper_.addMonitoredQuantityName(FRAGMENTS_PROCESSED_STAT_KEY);
00038 statsHelper_.addMonitoredQuantityName(INPUT_WAIT_STAT_KEY);
00039 statsHelper_.addMonitoredQuantityName(BRSYNC_WAIT_STAT_KEY);
00040 statsHelper_.addMonitoredQuantityName(OUTPUT_WAIT_STAT_KEY);
00041 statsHelper_.addMonitoredQuantityName(FRAGMENTS_PER_READ_STAT_KEY);
00042 metricMan = &metricMan_;
00043 my_rank = rank;
00044 }
00045
00046 artdaq::BoardReaderCore::~BoardReaderCore()
00047 {
00048 TLOG_DEBUG(name_) << "Destructor" << TLOG_ENDL;
00049 }
00050
00051 bool artdaq::BoardReaderCore::initialize(fhicl::ParameterSet const& pset, uint64_t, uint64_t)
00052 {
00053 TLOG_DEBUG(name_) << "initialize method called with "
00054 << "ParameterSet = \"" << pset.to_string()
00055 << "\"." << TLOG_ENDL;
00056
00057
00058 fhicl::ParameterSet daq_pset;
00059 try
00060 {
00061 daq_pset = pset.get<fhicl::ParameterSet>("daq");
00062 }
00063 catch (...)
00064 {
00065 TLOG_ERROR(name_)
00066 << "Unable to find the DAQ parameters in the initialization "
00067 << "ParameterSet: \"" + pset.to_string() + "\"." << TLOG_ENDL;
00068 return false;
00069 }
00070 fhicl::ParameterSet fr_pset;
00071 try
00072 {
00073 fr_pset = daq_pset.get<fhicl::ParameterSet>("fragment_receiver");
00074 data_pset_ = fr_pset;
00075 }
00076 catch (...)
00077 {
00078 TLOG_ERROR(name_)
00079 << "Unable to find the fragment_receiver parameters in the DAQ "
00080 << "initialization ParameterSet: \"" + daq_pset.to_string() + "\"." << TLOG_ENDL;
00081 return false;
00082 }
00083
00084
00085 fhicl::ParameterSet metric_pset;
00086 try
00087 {
00088 metric_pset = daq_pset.get<fhicl::ParameterSet>("metrics");
00089 }
00090 catch (...) {}
00091
00092 if (metric_pset.is_empty())
00093 {
00094 TLOG_INFO(name_) << "No metric plugins appear to be defined" << TLOG_ENDL;
00095 }
00096 try
00097 {
00098 metricMan_.initialize(metric_pset, name_);
00099 }
00100 catch (...)
00101 {
00102 ExceptionHandler(ExceptionHandlerRethrow::no,
00103 "Error loading metrics in BoardReaderCore::initialize()");
00104 }
00105
00106
00107 std::string frag_gen_name = fr_pset.get<std::string>("generator", "");
00108 if (frag_gen_name.length() == 0)
00109 {
00110 TLOG_ERROR(name_)
00111 << "No fragment generator (parameter name = \"generator\") was "
00112 << "specified in the fragment_receiver ParameterSet. The "
00113 << "DAQ initialization PSet was \"" << daq_pset.to_string() << "\"." << TLOG_ENDL;
00114 return false;
00115 }
00116
00117 try
00118 {
00119 generator_ptr_ = artdaq::makeCommandableFragmentGenerator(frag_gen_name, fr_pset);
00120 }
00121 catch (...)
00122 {
00123 std::stringstream exception_string;
00124 exception_string << "Exception thrown during initialization of fragment generator of type \""
00125 << frag_gen_name << "\"";
00126
00127 ExceptionHandler(ExceptionHandlerRethrow::no, exception_string.str());
00128
00129 TLOG_DEBUG(name_) << "FHiCL parameter set used to initialize the fragment generator which threw an exception: " << fr_pset.to_string() << TLOG_ENDL;
00130
00131 return false;
00132 }
00133 metricMan_.setPrefix(generator_ptr_->metricsReportingInstanceName());
00134
00135 rt_priority_ = fr_pset.get<int>("rt_priority", 0);
00136
00137
00138
00139
00140
00141
00142
00143
00144
00145
00146
00147
00148
00149
00150
00151
00152
00153
00154
00155
00156
00157
00158
00159
00160
00161
00162
00163
00164
00165
00166
00167
00168
00169
00170
00171
00172
00173
00174
00175
00176
00177
00178 statsHelper_.createCollectors(fr_pset, 100, 30.0, 60.0, FRAGMENTS_PROCESSED_STAT_KEY);
00179
00180
00181 skip_seqId_test_ = (generator_ptr_->fragmentIDs().size() > 1);
00182
00183 return true;
00184 }
00185
00186 bool artdaq::BoardReaderCore::start(art::RunID id, uint64_t timeout, uint64_t timestamp)
00187 {
00188 stop_requested_.store(false);
00189 pause_requested_.store(false);
00190
00191 fragment_count_ = 0;
00192 prev_seq_id_ = 0;
00193 statsHelper_.resetStatistics();
00194
00195 metricMan_.do_start();
00196 generator_ptr_->StartCmd(id.run(), timeout, timestamp);
00197 run_id_ = id;
00198
00199 TLOG_DEBUG(name_) << "Started run " << run_id_.run() <<
00200 ", timeout = " << timeout << ", timestamp = " << timestamp << TLOG_ENDL;
00201 return true;
00202 }
00203
00204 bool artdaq::BoardReaderCore::stop(uint64_t timeout, uint64_t timestamp)
00205 {
00206 TLOG_DEBUG(name_) << "Stopping run " << run_id_.run()
00207 << " after " << fragment_count_
00208 << " fragments." << TLOG_ENDL;
00209 stop_requested_.store(true);
00210 generator_ptr_->StopCmd(timeout, timestamp);
00211 return true;
00212 }
00213
00214 bool artdaq::BoardReaderCore::pause(uint64_t timeout, uint64_t timestamp)
00215 {
00216 TLOG_DEBUG(name_) << "Pausing run " << run_id_.run()
00217 << " after " << fragment_count_
00218 << " fragments." << TLOG_ENDL;
00219 pause_requested_.store(true);
00220 generator_ptr_->PauseCmd(timeout, timestamp);
00221 return true;
00222 }
00223
00224 bool artdaq::BoardReaderCore::resume(uint64_t timeout, uint64_t timestamp)
00225 {
00226 TLOG_DEBUG(name_) << "Resuming run " << run_id_.run() << TLOG_ENDL;
00227 pause_requested_.store(false);
00228 metricMan_.do_start();
00229 generator_ptr_->ResumeCmd(timeout, timestamp);
00230 return true;
00231 }
00232
00233 bool artdaq::BoardReaderCore::shutdown(uint64_t)
00234 {
00235 generator_ptr_.reset(nullptr);
00236 metricMan_.shutdown();
00237 return true;
00238 }
00239
00240 bool artdaq::BoardReaderCore::soft_initialize(fhicl::ParameterSet const& pset, uint64_t, uint64_t)
00241 {
00242 TLOG_DEBUG(name_) << "soft_initialize method called with "
00243 << "ParameterSet = \"" << pset.to_string()
00244 << "\"." << TLOG_ENDL;
00245 return true;
00246 }
00247
00248 bool artdaq::BoardReaderCore::reinitialize(fhicl::ParameterSet const& pset, uint64_t, uint64_t)
00249 {
00250 TLOG_DEBUG(name_) << "reinitialize method called with "
00251 << "ParameterSet = \"" << pset.to_string()
00252 << "\"." << TLOG_ENDL;
00253 return true;
00254 }
00255
00256 size_t artdaq::BoardReaderCore::process_fragments()
00257 {
00258 if (rt_priority_ > 0)
00259 {
00260 #pragma GCC diagnostic push
00261 #pragma GCC diagnostic ignored "-Wmissing-field-initializers"
00262 sched_param s_param = {};
00263 s_param.sched_priority = rt_priority_;
00264 if (pthread_setschedparam(pthread_self(), SCHED_RR, &s_param))
00265 TLOG_WARNING(name_) << "setting realtime priority failed" << TLOG_ENDL;
00266 #pragma GCC diagnostic pop
00267 }
00268
00269
00270
00271
00272 if (rt_priority_ > 0)
00273 {
00274 #pragma GCC diagnostic push
00275 #pragma GCC diagnostic ignored "-Wmissing-field-initializers"
00276 sched_param s_param = {};
00277 s_param.sched_priority = rt_priority_;
00278 int status = pthread_setschedparam(pthread_self(), SCHED_RR, &s_param);
00279 if (status != 0)
00280 {
00281 TLOG_ERROR(name_)
00282 << "Failed to set realtime priority to " << rt_priority_
00283 << ", return code = " << status << TLOG_ENDL;
00284 }
00285 #pragma GCC diagnostic pop
00286 }
00287
00288 TLOG_DEBUG(name_) << "Initializing DataSenderManager. my_rank=" << my_rank << TLOG_ENDL;
00289 sender_ptr_.reset(new artdaq::DataSenderManager(data_pset_));
00290
00291
00292
00293 TLOG_DEBUG(name_) << "Waiting for first fragment." << TLOG_ENDL;
00294 artdaq::MonitoredQuantityStats::TIME_POINT_T startTime;
00295 double delta_time;
00296 artdaq::FragmentPtrs frags;
00297 bool active = true;
00298
00299
00300 while (active)
00301 {
00302 startTime = artdaq::MonitoredQuantity::getCurrentTime();
00303
00304 TRACE(18, name_ + "::process_fragments getNext start");
00305 active = generator_ptr_->getNext(frags);
00306 TRACE(18, name_ + "::process_fragments getNext done (active=%i)", active);
00307
00308
00309
00310
00311
00312
00313 if (!active && generator_ptr_->exception())
00314 {
00315 parent_application_.in_run_failure();
00316 }
00317
00318 delta_time = artdaq::MonitoredQuantity::getCurrentTime() - startTime;
00319 statsHelper_.addSample(INPUT_WAIT_STAT_KEY, delta_time);
00320
00321 TRACE(16, name_ + "::process_fragments INPUT_WAIT=%f", delta_time);
00322
00323 if (!active) { break; }
00324 statsHelper_.addSample(FRAGMENTS_PER_READ_STAT_KEY, frags.size());
00325
00326 for (auto& fragPtr : frags)
00327 {
00328 if (!fragPtr.get())
00329 {
00330 TLOG_WARNING(name_) << "Encountered a bad fragment pointer in fragment " << fragment_count_ << ". "
00331 << "This is most likely caused by a problem with the Fragment Generator!" << TLOG_ENDL;
00332 continue;
00333 }
00334 artdaq::Fragment::sequence_id_t sequence_id = fragPtr->sequenceID();
00335 statsHelper_.addSample(FRAGMENTS_PROCESSED_STAT_KEY, fragPtr->size());
00336
00337 if ((fragment_count_ % 250) == 0)
00338 {
00339 TLOG_DEBUG(name_)
00340 << "Sending fragment " << fragment_count_
00341 << " with sequence id " << sequence_id << "." << TLOG_ENDL;
00342 }
00343
00344
00345
00346
00347
00348
00349
00350
00351
00352
00353
00354
00355
00356
00357
00358
00359
00360
00361
00362
00363
00364
00365
00366
00367
00368
00369
00370
00371
00372
00373
00374
00375
00376
00377
00378
00379
00380
00381
00382
00383
00384
00385
00386
00387
00388
00389
00390
00391
00392
00393
00394
00395
00396
00397
00398
00399
00400
00401
00402
00403
00404
00405
00406
00407
00408
00409
00410
00411
00412
00413
00414
00415
00416
00417
00418
00419
00420
00421
00422
00423
00424
00425
00426
00427
00428
00429
00430
00431
00432
00433
00434
00435
00436
00437
00438
00439
00440
00441 if (!skip_seqId_test_ && abs(sequence_id - prev_seq_id_) > 1)
00442 {
00443 TLOG_WARNING(name_)
00444 << "Missing sequence IDs: current sequence ID = "
00445 << sequence_id << ", previous sequence ID = "
00446 << prev_seq_id_ << "." << TLOG_ENDL;
00447 }
00448 prev_seq_id_ = sequence_id;
00449
00450 startTime = artdaq::MonitoredQuantity::getCurrentTime();
00451 TRACE(17, name_ + "::process_fragments seq=%lu sendFragment start", sequence_id);
00452 auto res = sender_ptr_->sendFragment(std::move(*fragPtr));
00453 TRACE(17, name_ + "::process_fragments seq=%lu sendFragment done (res=%i)", sequence_id, res);
00454 ++fragment_count_;
00455 statsHelper_.addSample(OUTPUT_WAIT_STAT_KEY,
00456 artdaq::MonitoredQuantity::getCurrentTime() - startTime);
00457
00458 bool readyToReport = statsHelper_.readyToReport(fragment_count_);
00459 if (readyToReport)
00460 {
00461 std::string statString = buildStatisticsString_();
00462 TLOG_DEBUG(name_) << statString << TLOG_ENDL;
00463 }
00464 if (fragment_count_ == 1 || readyToReport)
00465 {
00466 TLOG_DEBUG(name_)
00467 << "Sending fragment " << fragment_count_
00468 << " with sequence id " << sequence_id << "." << TLOG_ENDL;
00469 }
00470 }
00471 if (statsHelper_.statsRollingWindowHasMoved()) { sendMetrics_(); }
00472 frags.clear();
00473 }
00474
00475
00476
00477
00478
00479
00480
00481
00482
00483 metricMan_.do_stop();
00484
00485 sender_ptr_.reset(nullptr);
00486 return fragment_count_;
00487 }
00488
00489 std::string artdaq::BoardReaderCore::report(std::string const& which) const
00490 {
00491 std::string resultString;
00492
00493
00494 if (generator_ptr_.get() != 0)
00495 {
00496 resultString = generator_ptr_->ReportCmd(which);
00497 if (resultString.length() > 0) { return resultString; }
00498 }
00499
00500
00501
00502
00503
00504 std::string tmpString = name_ + " run number = ";
00505 tmpString.append(boost::lexical_cast<std::string>(run_id_.run()));
00506 tmpString.append(". Command=\"" + which + "\" is not currently supported.");
00507 return tmpString;
00508 }
00509
00510 std::string artdaq::BoardReaderCore::buildStatisticsString_()
00511 {
00512 std::ostringstream oss;
00513 oss << name_ << " statistics:" << std::endl;
00514
00515 double fragmentCount = 1.0;
00516 artdaq::MonitoredQuantityPtr mqPtr = artdaq::StatisticsCollection::getInstance().
00517 getMonitoredQuantity(FRAGMENTS_PROCESSED_STAT_KEY);
00518 if (mqPtr.get() != 0)
00519 {
00520 artdaq::MonitoredQuantityStats stats;
00521 mqPtr->getStats(stats);
00522 oss << " Fragment statistics: "
00523 << stats.recentSampleCount << " fragments received at "
00524 << stats.recentSampleRate << " fragments/sec, effective data rate = "
00525 << (stats.recentValueRate * sizeof(artdaq::RawDataType)
00526 / 1024.0 / 1024.0) << " MB/sec, monitor window = "
00527 << stats.recentDuration << " sec, min::max event size = "
00528 << (stats.recentValueMin * sizeof(artdaq::RawDataType)
00529 / 1024.0 / 1024.0)
00530 << "::"
00531 << (stats.recentValueMax * sizeof(artdaq::RawDataType)
00532 / 1024.0 / 1024.0)
00533 << " MB" << std::endl;
00534 fragmentCount = std::max(double(stats.recentSampleCount), 1.0);
00535 oss << " Average times per fragment: ";
00536 if (stats.recentSampleRate > 0.0)
00537 {
00538 oss << " elapsed time = "
00539 << (1.0 / stats.recentSampleRate) << " sec";
00540 }
00541 }
00542
00543
00544
00545
00546
00547
00548
00549
00550
00551 mqPtr = artdaq::StatisticsCollection::getInstance().
00552 getMonitoredQuantity(INPUT_WAIT_STAT_KEY);
00553 if (mqPtr.get() != 0)
00554 {
00555 oss << ", input wait time = "
00556 << (mqPtr->getRecentValueSum() / fragmentCount) << " sec";
00557 }
00558
00559 mqPtr = artdaq::StatisticsCollection::getInstance().
00560 getMonitoredQuantity(BRSYNC_WAIT_STAT_KEY);
00561 if (mqPtr.get() != 0)
00562 {
00563 oss << ", BRsync wait time = "
00564 << (mqPtr->getRecentValueSum() / fragmentCount) << " sec";
00565 }
00566
00567 mqPtr = artdaq::StatisticsCollection::getInstance().
00568 getMonitoredQuantity(OUTPUT_WAIT_STAT_KEY);
00569 if (mqPtr.get() != 0)
00570 {
00571 oss << ", output wait time = "
00572 << (mqPtr->getRecentValueSum() / fragmentCount) << " sec";
00573 }
00574
00575 oss << std::endl << " Fragments per read: ";
00576 mqPtr = artdaq::StatisticsCollection::getInstance().
00577 getMonitoredQuantity(FRAGMENTS_PER_READ_STAT_KEY);
00578 if (mqPtr.get() != 0)
00579 {
00580 artdaq::MonitoredQuantityStats stats;
00581 mqPtr->getStats(stats);
00582 oss << "average = "
00583 << stats.recentValueAverage
00584 << ", min::max = "
00585 << stats.recentValueMin
00586 << "::"
00587 << stats.recentValueMax;
00588 }
00589
00590 return oss.str();
00591 }
00592
00593 void artdaq::BoardReaderCore::sendMetrics_()
00594 {
00595
00596 double fragmentCount = 1.0;
00597 artdaq::MonitoredQuantityPtr mqPtr = artdaq::StatisticsCollection::getInstance().
00598 getMonitoredQuantity(FRAGMENTS_PROCESSED_STAT_KEY);
00599 if (mqPtr.get() != 0)
00600 {
00601 artdaq::MonitoredQuantityStats stats;
00602 mqPtr->getStats(stats);
00603 fragmentCount = std::max(double(stats.recentSampleCount), 1.0);
00604 metricMan_.sendMetric("Fragment Count", static_cast<unsigned long>(stats.fullSampleCount), "fragments", 1, MetricMode::Accumulate);
00605 metricMan_.sendMetric("Fragment Rate", stats.recentSampleRate, "fragments/sec", 1, MetricMode::Average);
00606 metricMan_.sendMetric("Average Fragment Size", (stats.recentValueAverage * sizeof(artdaq::RawDataType)), "bytes/fragment", 2, MetricMode::Average);
00607 metricMan_.sendMetric("Data Rate", (stats.recentValueRate * sizeof(artdaq::RawDataType)), "bytes/sec", 2, MetricMode::Average);
00608 }
00609
00610
00611
00612
00613
00614
00615
00616
00617
00618 mqPtr = artdaq::StatisticsCollection::getInstance().
00619 getMonitoredQuantity(INPUT_WAIT_STAT_KEY);
00620 if (mqPtr.get() != 0)
00621 {
00622 metricMan_.sendMetric("Avg Input Wait Time", (mqPtr->getRecentValueSum() / fragmentCount), "seconds/fragment", 3, MetricMode::Average);
00623 }
00624
00625 mqPtr = artdaq::StatisticsCollection::getInstance().
00626 getMonitoredQuantity(BRSYNC_WAIT_STAT_KEY);
00627 if (mqPtr.get() != 0)
00628 {
00629 metricMan_.sendMetric("Avg BoardReader Sync Wait Time", (mqPtr->getRecentValueSum() / fragmentCount), "seconds/fragment", 3, MetricMode::Average);
00630 }
00631
00632 mqPtr = artdaq::StatisticsCollection::getInstance().
00633 getMonitoredQuantity(OUTPUT_WAIT_STAT_KEY);
00634 if (mqPtr.get() != 0)
00635 {
00636 metricMan_.sendMetric("Avg Output Wait Time", (mqPtr->getRecentValueSum() / fragmentCount), "seconds/fragment", 3, MetricMode::Average);
00637 }
00638
00639 mqPtr = artdaq::StatisticsCollection::getInstance().
00640 getMonitoredQuantity(FRAGMENTS_PER_READ_STAT_KEY);
00641 if (mqPtr.get() != 0)
00642 {
00643 metricMan_.sendMetric("Avg Frags Per Read", mqPtr->getRecentValueAverage(), "fragments/read", 4, MetricMode::Average);
00644 }
00645 }