00001 #include "artdaq/Application/TaskType.hh"
00002 #include "artdaq/Application/BoardReaderCore.hh"
00003 #include "artdaq-core/Data/Fragment.hh"
00004 #include "artdaq-core/Utilities/ExceptionHandler.hh"
00005 #include "artdaq/Application/makeCommandableFragmentGenerator.hh"
00006 #include "canvas/Utilities/Exception.h"
00007 #include "cetlib/exception.h"
00008 #include <pthread.h>
00009 #include <sched.h>
00010 #include <algorithm>
00011
00012 #define TRACE_NAME "BoardReaderCore"
00013
00014 const std::string artdaq::BoardReaderCore::
00015 FRAGMENTS_PROCESSED_STAT_KEY("BoardReaderCoreFragmentsProcessed");
00016 const std::string artdaq::BoardReaderCore::
00017 INPUT_WAIT_STAT_KEY("BoardReaderCoreInputWaitTime");
00018 const std::string artdaq::BoardReaderCore::
00019 BRSYNC_WAIT_STAT_KEY("BoardReaderCoreBRSyncWaitTime");
00020 const std::string artdaq::BoardReaderCore::
00021 OUTPUT_WAIT_STAT_KEY("BoardReaderCoreOutputWaitTime");
00022 const std::string artdaq::BoardReaderCore::
00023 FRAGMENTS_PER_READ_STAT_KEY("BoardReaderCoreFragmentsPerRead");
00024
00025 std::unique_ptr<artdaq::DataSenderManager> artdaq::BoardReaderCore::sender_ptr_ = nullptr;
00026
00027 artdaq::BoardReaderCore::BoardReaderCore(Commandable& parent_application,
00028 int rank, std::string name) :
00029 parent_application_(parent_application)
00030
00031 , generator_ptr_(nullptr)
00032 , name_(name)
00033 , stop_requested_(false)
00034 , pause_requested_(false)
00035 {
00036 TLOG_DEBUG(name_) << "Constructor" << TLOG_ENDL;
00037 statsHelper_.addMonitoredQuantityName(FRAGMENTS_PROCESSED_STAT_KEY);
00038 statsHelper_.addMonitoredQuantityName(INPUT_WAIT_STAT_KEY);
00039 statsHelper_.addMonitoredQuantityName(BRSYNC_WAIT_STAT_KEY);
00040 statsHelper_.addMonitoredQuantityName(OUTPUT_WAIT_STAT_KEY);
00041 statsHelper_.addMonitoredQuantityName(FRAGMENTS_PER_READ_STAT_KEY);
00042 metricMan = &metricMan_;
00043 my_rank = rank;
00044 }
00045
00046 artdaq::BoardReaderCore::~BoardReaderCore()
00047 {
00048 TLOG_DEBUG(name_) << "Destructor" << TLOG_ENDL;
00049 }
00050
00051 bool artdaq::BoardReaderCore::initialize(fhicl::ParameterSet const& pset, uint64_t, uint64_t)
00052 {
00053 TLOG_DEBUG(name_) << "initialize method called with "
00054 << "ParameterSet = \"" << pset.to_string()
00055 << "\"." << TLOG_ENDL;
00056
00057
00058 fhicl::ParameterSet daq_pset;
00059 try
00060 {
00061 daq_pset = pset.get<fhicl::ParameterSet>("daq");
00062 }
00063 catch (...)
00064 {
00065 TLOG_ERROR(name_)
00066 << "Unable to find the DAQ parameters in the initialization "
00067 << "ParameterSet: \"" + pset.to_string() + "\"." << TLOG_ENDL;
00068 return false;
00069 }
00070 fhicl::ParameterSet fr_pset;
00071 try
00072 {
00073 fr_pset = daq_pset.get<fhicl::ParameterSet>("fragment_receiver");
00074 data_pset_ = fr_pset;
00075 }
00076 catch (...)
00077 {
00078 TLOG_ERROR(name_)
00079 << "Unable to find the fragment_receiver parameters in the DAQ "
00080 << "initialization ParameterSet: \"" + daq_pset.to_string() + "\"." << TLOG_ENDL;
00081 return false;
00082 }
00083
00084
00085 fhicl::ParameterSet metric_pset;
00086 try
00087 {
00088 metric_pset = daq_pset.get<fhicl::ParameterSet>("metrics");
00089 }
00090 catch (...) {}
00091
00092 if (metric_pset.is_empty())
00093 {
00094 TLOG_INFO(name_) << "No metric plugins appear to be defined" << TLOG_ENDL;
00095 }
00096 try
00097 {
00098 metricMan_.initialize(metric_pset, name_);
00099 }
00100 catch (...)
00101 {
00102 ExceptionHandler(ExceptionHandlerRethrow::no,
00103 "Error loading metrics in BoardReaderCore::initialize()");
00104 }
00105
00106
00107 std::string frag_gen_name = fr_pset.get<std::string>("generator", "");
00108 if (frag_gen_name.length() == 0)
00109 {
00110 TLOG_ERROR(name_)
00111 << "No fragment generator (parameter name = \"generator\") was "
00112 << "specified in the fragment_receiver ParameterSet. The "
00113 << "DAQ initialization PSet was \"" << daq_pset.to_string() << "\"." << TLOG_ENDL;
00114 return false;
00115 }
00116
00117 try
00118 {
00119 generator_ptr_ = artdaq::makeCommandableFragmentGenerator(frag_gen_name, fr_pset);
00120 }
00121 catch (...)
00122 {
00123 std::stringstream exception_string;
00124 exception_string << "Exception thrown during initialization of fragment generator of type \""
00125 << frag_gen_name << "\"";
00126
00127 ExceptionHandler(ExceptionHandlerRethrow::no, exception_string.str());
00128
00129 TLOG_DEBUG(name_) << "FHiCL parameter set used to initialize the fragment generator which threw an exception: " << fr_pset.to_string() << TLOG_ENDL;
00130
00131 return false;
00132 }
00133 metricMan_.setPrefix(generator_ptr_->metricsReportingInstanceName());
00134
00135 rt_priority_ = fr_pset.get<int>("rt_priority", 0);
00136
00137
00138
00139
00140
00141
00142
00143
00144
00145
00146
00147
00148
00149
00150
00151
00152
00153
00154
00155
00156
00157
00158
00159
00160
00161
00162
00163
00164
00165
00166
00167
00168
00169
00170
00171
00172
00173
00174
00175
00176
00177
00178 statsHelper_.createCollectors(fr_pset, 100, 30.0, 60.0, FRAGMENTS_PROCESSED_STAT_KEY);
00179
00180
00181 skip_seqId_test_ = (generator_ptr_->fragmentIDs().size() > 1);
00182
00183 return true;
00184 }
00185
00186 bool artdaq::BoardReaderCore::start(art::RunID id, uint64_t timeout, uint64_t timestamp)
00187 {
00188 stop_requested_.store(false);
00189 pause_requested_.store(false);
00190
00191 fragment_count_ = 0;
00192 prev_seq_id_ = 0;
00193 statsHelper_.resetStatistics();
00194
00195 metricMan_.do_start();
00196 generator_ptr_->StartCmd(id.run(), timeout, timestamp);
00197 run_id_ = id;
00198
00199 TLOG_DEBUG(name_) << "Started run " << run_id_.run() <<
00200 ", timeout = " << timeout << ", timestamp = " << timestamp << TLOG_ENDL;
00201 return true;
00202 }
00203
00204 bool artdaq::BoardReaderCore::stop(uint64_t timeout, uint64_t timestamp)
00205 {
00206 TLOG_DEBUG(name_) << "Stopping run " << run_id_.run()
00207 << " after " << fragment_count_
00208 << " fragments." << TLOG_ENDL;
00209 stop_requested_.store(true);
00210 generator_ptr_->StopCmd(timeout, timestamp);
00211 return true;
00212 }
00213
00214 bool artdaq::BoardReaderCore::pause(uint64_t timeout, uint64_t timestamp)
00215 {
00216 TLOG_DEBUG(name_) << "Pausing run " << run_id_.run()
00217 << " after " << fragment_count_
00218 << " fragments." << TLOG_ENDL;
00219 pause_requested_.store(true);
00220 generator_ptr_->PauseCmd(timeout, timestamp);
00221 return true;
00222 }
00223
00224 bool artdaq::BoardReaderCore::resume(uint64_t timeout, uint64_t timestamp)
00225 {
00226 TLOG_DEBUG(name_) << "Resuming run " << run_id_.run() << TLOG_ENDL;
00227 pause_requested_.store(false);
00228 metricMan_.do_start();
00229 generator_ptr_->ResumeCmd(timeout, timestamp);
00230 return true;
00231 }
00232
00233 bool artdaq::BoardReaderCore::shutdown(uint64_t)
00234 {
00235 generator_ptr_.reset(nullptr);
00236 metricMan_.shutdown();
00237 return true;
00238 }
00239
00240 bool artdaq::BoardReaderCore::soft_initialize(fhicl::ParameterSet const& pset, uint64_t, uint64_t)
00241 {
00242 TLOG_DEBUG(name_) << "soft_initialize method called with "
00243 << "ParameterSet = \"" << pset.to_string()
00244 << "\"." << TLOG_ENDL;
00245 return true;
00246 }
00247
00248 bool artdaq::BoardReaderCore::reinitialize(fhicl::ParameterSet const& pset, uint64_t, uint64_t)
00249 {
00250 TLOG_DEBUG(name_) << "reinitialize method called with "
00251 << "ParameterSet = \"" << pset.to_string()
00252 << "\"." << TLOG_ENDL;
00253 return true;
00254 }
00255
00256 size_t artdaq::BoardReaderCore::process_fragments()
00257 {
00258 if (rt_priority_ > 0)
00259 {
00260 #pragma GCC diagnostic push
00261 #pragma GCC diagnostic ignored "-Wmissing-field-initializers"
00262 sched_param s_param = {};
00263 s_param.sched_priority = rt_priority_;
00264 if (pthread_setschedparam(pthread_self(), SCHED_RR, &s_param))
00265 TLOG_WARNING(name_) << "setting realtime priority failed" << TLOG_ENDL;
00266 #pragma GCC diagnostic pop
00267 }
00268
00269
00270
00271
00272 if (rt_priority_ > 0)
00273 {
00274 #pragma GCC diagnostic push
00275 #pragma GCC diagnostic ignored "-Wmissing-field-initializers"
00276 sched_param s_param = {};
00277 s_param.sched_priority = rt_priority_;
00278 int status = pthread_setschedparam(pthread_self(), SCHED_RR, &s_param);
00279 if (status != 0)
00280 {
00281 TLOG_ERROR(name_)
00282 << "Failed to set realtime priority to " << rt_priority_
00283 << ", return code = " << status << TLOG_ENDL;
00284 }
00285 #pragma GCC diagnostic pop
00286 }
00287
00288 TLOG_DEBUG(name_) << "Initializing DataSenderManager. my_rank=" << my_rank << TLOG_ENDL;
00289 sender_ptr_.reset(new artdaq::DataSenderManager(data_pset_));
00290
00291
00292
00293 TLOG_DEBUG(name_) << "Waiting for first fragment." << TLOG_ENDL;
00294 artdaq::MonitoredQuantityStats::TIME_POINT_T startTime;
00295 double delta_time;
00296 artdaq::FragmentPtrs frags;
00297 bool active = true;
00298
00299
00300 while (active)
00301 {
00302 startTime = artdaq::MonitoredQuantity::getCurrentTime();
00303
00304 active = generator_ptr_->getNext(frags);
00305
00306
00307
00308
00309
00310
00311 if (!active && generator_ptr_->exception())
00312 {
00313 parent_application_.in_run_failure();
00314 }
00315
00316 delta_time = artdaq::MonitoredQuantity::getCurrentTime() - startTime;
00317 statsHelper_.addSample(INPUT_WAIT_STAT_KEY, delta_time);
00318
00319 TRACE(16, "%s::process_fragments INPUT_WAIT=%f", name_.c_str(), delta_time);
00320
00321 if (!active) { break; }
00322 statsHelper_.addSample(FRAGMENTS_PER_READ_STAT_KEY, frags.size());
00323
00324 for (auto& fragPtr : frags)
00325 {
00326 if (!fragPtr.get())
00327 {
00328 TLOG_WARNING(name_) << "Encountered a bad fragment pointer in fragment " << fragment_count_ << ". "
00329 << "This is most likely caused by a problem with the Fragment Generator!" << TLOG_ENDL;
00330 continue;
00331 }
00332 artdaq::Fragment::sequence_id_t sequence_id = fragPtr->sequenceID();
00333 statsHelper_.addSample(FRAGMENTS_PROCESSED_STAT_KEY, fragPtr->size());
00334
00335 if ((fragment_count_ % 250) == 0)
00336 {
00337 TLOG_DEBUG(name_)
00338 << "Sending fragment " << fragment_count_
00339 << " with sequence id " << sequence_id << "." << TLOG_ENDL;
00340 }
00341
00342
00343
00344
00345
00346
00347
00348
00349
00350
00351
00352
00353
00354
00355
00356
00357
00358
00359
00360
00361
00362
00363
00364
00365
00366
00367
00368
00369
00370
00371
00372
00373
00374
00375
00376
00377
00378
00379
00380
00381
00382
00383
00384
00385
00386
00387
00388
00389
00390
00391
00392
00393
00394
00395
00396
00397
00398
00399
00400
00401
00402
00403
00404
00405
00406
00407
00408
00409
00410
00411
00412
00413
00414
00415
00416
00417
00418
00419
00420
00421
00422
00423
00424
00425
00426
00427
00428
00429
00430
00431
00432
00433
00434
00435
00436
00437
00438
00439 if (!skip_seqId_test_ && abs(sequence_id - prev_seq_id_) > 1)
00440 {
00441 TLOG_WARNING(name_)
00442 << "Missing sequence IDs: current sequence ID = "
00443 << sequence_id << ", previous sequence ID = "
00444 << prev_seq_id_ << "." << TLOG_ENDL;
00445 }
00446 prev_seq_id_ = sequence_id;
00447
00448 startTime = artdaq::MonitoredQuantity::getCurrentTime();
00449 TRACE(17, "%s::process_fragments seq=%lu sendFragment start", name_.c_str(), sequence_id);
00450 auto res = sender_ptr_->sendFragment(std::move(*fragPtr));
00451 TRACE(17, "%s::process_fragments seq=%lu sendFragment done (res=%i)", name_.c_str(), sequence_id,res);
00452 ++fragment_count_;
00453 statsHelper_.addSample(OUTPUT_WAIT_STAT_KEY,
00454 artdaq::MonitoredQuantity::getCurrentTime() - startTime);
00455
00456 bool readyToReport = statsHelper_.readyToReport(fragment_count_);
00457 if (readyToReport)
00458 {
00459 std::string statString = buildStatisticsString_();
00460 TLOG_DEBUG(name_) << statString << TLOG_ENDL;
00461 }
00462 if (fragment_count_ == 1 || readyToReport)
00463 {
00464 TLOG_DEBUG(name_)
00465 << "Sending fragment " << fragment_count_
00466 << " with sequence id " << sequence_id << "." << TLOG_ENDL;
00467 }
00468 }
00469 if (statsHelper_.statsRollingWindowHasMoved()) { sendMetrics_(); }
00470 frags.clear();
00471 }
00472
00473
00474
00475
00476
00477
00478
00479
00480
00481 metricMan_.do_stop();
00482
00483 sender_ptr_.reset(nullptr);
00484 return fragment_count_;
00485 }
00486
00487 std::string artdaq::BoardReaderCore::report(std::string const& which) const
00488 {
00489 std::string resultString;
00490
00491
00492 if (generator_ptr_.get() != 0)
00493 {
00494 resultString = generator_ptr_->ReportCmd(which);
00495 if (resultString.length() > 0) { return resultString; }
00496 }
00497
00498
00499
00500
00501
00502 std::string tmpString = name_ + " run number = ";
00503 tmpString.append(boost::lexical_cast<std::string>(run_id_.run()));
00504 tmpString.append(". Command=\"" + which + "\" is not currently supported.");
00505 return tmpString;
00506 }
00507
00508 std::string artdaq::BoardReaderCore::buildStatisticsString_()
00509 {
00510 std::ostringstream oss;
00511 oss << name_ << " statistics:" << std::endl;
00512
00513 double fragmentCount = 1.0;
00514 artdaq::MonitoredQuantityPtr mqPtr = artdaq::StatisticsCollection::getInstance().
00515 getMonitoredQuantity(FRAGMENTS_PROCESSED_STAT_KEY);
00516 if (mqPtr.get() != 0)
00517 {
00518 artdaq::MonitoredQuantityStats stats;
00519 mqPtr->getStats(stats);
00520 oss << " Fragment statistics: "
00521 << stats.recentSampleCount << " fragments received at "
00522 << stats.recentSampleRate << " fragments/sec, effective data rate = "
00523 << (stats.recentValueRate * sizeof(artdaq::RawDataType)
00524 / 1024.0 / 1024.0) << " MB/sec, monitor window = "
00525 << stats.recentDuration << " sec, min::max event size = "
00526 << (stats.recentValueMin * sizeof(artdaq::RawDataType)
00527 / 1024.0 / 1024.0)
00528 << "::"
00529 << (stats.recentValueMax * sizeof(artdaq::RawDataType)
00530 / 1024.0 / 1024.0)
00531 << " MB" << std::endl;
00532 fragmentCount = std::max(double(stats.recentSampleCount), 1.0);
00533 oss << " Average times per fragment: ";
00534 if (stats.recentSampleRate > 0.0)
00535 {
00536 oss << " elapsed time = "
00537 << (1.0 / stats.recentSampleRate) << " sec";
00538 }
00539 }
00540
00541
00542
00543
00544
00545
00546
00547
00548
00549 mqPtr = artdaq::StatisticsCollection::getInstance().
00550 getMonitoredQuantity(INPUT_WAIT_STAT_KEY);
00551 if (mqPtr.get() != 0)
00552 {
00553 oss << ", input wait time = "
00554 << (mqPtr->getRecentValueSum() / fragmentCount) << " sec";
00555 }
00556
00557 mqPtr = artdaq::StatisticsCollection::getInstance().
00558 getMonitoredQuantity(BRSYNC_WAIT_STAT_KEY);
00559 if (mqPtr.get() != 0)
00560 {
00561 oss << ", BRsync wait time = "
00562 << (mqPtr->getRecentValueSum() / fragmentCount) << " sec";
00563 }
00564
00565 mqPtr = artdaq::StatisticsCollection::getInstance().
00566 getMonitoredQuantity(OUTPUT_WAIT_STAT_KEY);
00567 if (mqPtr.get() != 0)
00568 {
00569 oss << ", output wait time = "
00570 << (mqPtr->getRecentValueSum() / fragmentCount) << " sec";
00571 }
00572
00573 oss << std::endl << " Fragments per read: ";
00574 mqPtr = artdaq::StatisticsCollection::getInstance().
00575 getMonitoredQuantity(FRAGMENTS_PER_READ_STAT_KEY);
00576 if (mqPtr.get() != 0)
00577 {
00578 artdaq::MonitoredQuantityStats stats;
00579 mqPtr->getStats(stats);
00580 oss << "average = "
00581 << stats.recentValueAverage
00582 << ", min::max = "
00583 << stats.recentValueMin
00584 << "::"
00585 << stats.recentValueMax;
00586 }
00587
00588 return oss.str();
00589 }
00590
00591 void artdaq::BoardReaderCore::sendMetrics_()
00592 {
00593
00594 double fragmentCount = 1.0;
00595 artdaq::MonitoredQuantityPtr mqPtr = artdaq::StatisticsCollection::getInstance().
00596 getMonitoredQuantity(FRAGMENTS_PROCESSED_STAT_KEY);
00597 if (mqPtr.get() != 0)
00598 {
00599 artdaq::MonitoredQuantityStats stats;
00600 mqPtr->getStats(stats);
00601 fragmentCount = std::max(double(stats.recentSampleCount), 1.0);
00602 metricMan_.sendMetric("Fragment Count",
00603 static_cast<unsigned long>(stats.fullSampleCount),
00604 "fragments", 1);
00605 metricMan_.sendMetric("Fragment Rate",
00606 stats.recentSampleRate, "fragments/sec", 1);
00607 metricMan_.sendMetric("Average Fragment Size",
00608 (stats.recentValueAverage * sizeof(artdaq::RawDataType)
00609 ), "bytes/fragment", 2);
00610 metricMan_.sendMetric("Data Rate",
00611 (stats.recentValueRate * sizeof(artdaq::RawDataType)
00612 ), "bytes/sec", 2);
00613 }
00614
00615
00616
00617
00618
00619
00620
00621
00622
00623 mqPtr = artdaq::StatisticsCollection::getInstance().
00624 getMonitoredQuantity(INPUT_WAIT_STAT_KEY);
00625 if (mqPtr.get() != 0)
00626 {
00627 metricMan_.sendMetric("Avg Input Wait Time",
00628 (mqPtr->getRecentValueSum() / fragmentCount),
00629 "seconds/fragment", 3, false);
00630 }
00631
00632 mqPtr = artdaq::StatisticsCollection::getInstance().
00633 getMonitoredQuantity(BRSYNC_WAIT_STAT_KEY);
00634 if (mqPtr.get() != 0)
00635 {
00636 metricMan_.sendMetric("Avg BoardReader Sync Wait Time",
00637 (mqPtr->getRecentValueSum() / fragmentCount),
00638 "seconds/fragment", 3, false);
00639 }
00640
00641 mqPtr = artdaq::StatisticsCollection::getInstance().
00642 getMonitoredQuantity(OUTPUT_WAIT_STAT_KEY);
00643 if (mqPtr.get() != 0)
00644 {
00645 metricMan_.sendMetric("Avg Output Wait Time",
00646 (mqPtr->getRecentValueSum() / fragmentCount),
00647 "seconds/fragment", 3, false);
00648 }
00649
00650 mqPtr = artdaq::StatisticsCollection::getInstance().
00651 getMonitoredQuantity(FRAGMENTS_PER_READ_STAT_KEY);
00652 if (mqPtr.get() != 0)
00653 {
00654 metricMan_.sendMetric("Avg Frags Per Read",
00655 mqPtr->getRecentValueAverage(), "fragments/read", 4, false);
00656 }
00657 }