00001 #include "artdaq/Application/CommandableFragmentGenerator.hh"
00002
00003 #include <boost/exception/all.hpp>
00004 #include <boost/throw_exception.hpp>
00005
00006 #include <limits>
00007 #include <iterator>
00008
00009 #include "canvas/Utilities/Exception.h"
00010 #include "cetlib/exception.h"
00011 #include "fhiclcpp/ParameterSet.h"
00012 #include "artdaq-core/Utilities/SimpleLookupPolicy.hh"
00013 #include "artdaq-core/Data/Fragment.hh"
00014 #include "artdaq-core/Data/ContainerFragmentLoader.hh"
00015 #include "artdaq-core/Utilities/ExceptionHandler.hh"
00016
00017 #include <fstream>
00018 #include <iomanip>
00019 #include <iterator>
00020 #include <iostream>
00021 #include <iomanip>
00022 #include <sys/poll.h>
00023 #include "artdaq/DAQdata/TCPConnect.hh"
00024
00025 artdaq::CommandableFragmentGenerator::CommandableFragmentGenerator()
00026 : mutex_()
00027 , request_port_(3001)
00028 , request_addr_("227.128.12.26")
00029 , requests_()
00030 , request_stop_requested_(false)
00031 , end_of_run_timeout_ms_(1000)
00032 , windowOffset_(0)
00033 , windowWidth_(0)
00034 , staleTimeout_(Fragment::InvalidTimestamp)
00035 , maxFragmentCount_(std::numeric_limits<size_t>::max())
00036 , uniqueWindows_(true)
00037 , useDataThread_(false)
00038 , data_thread_running_(false)
00039 , dataBufferDepthFragments_(0)
00040 , dataBufferDepthBytes_(0)
00041 , maxDataBufferDepthFragments_(1000)
00042 , maxDataBufferDepthBytes_(1000)
00043 , useMonitoringThread_(false)
00044 , monitoringInterval_(0)
00045 , lastMonitoringCall_(std::chrono::steady_clock::now())
00046 , isHardwareOK_(true)
00047 , dataBuffer_()
00048 , newDataBuffer_()
00049 , run_number_(-1)
00050 , subrun_number_(-1)
00051 , timeout_(std::numeric_limits<uint64_t>::max())
00052 , timestamp_(std::numeric_limits<uint64_t>::max())
00053 , should_stop_(false)
00054 , exception_(false)
00055 , latest_exception_report_("none")
00056 , ev_counter_(1)
00057 , board_id_(-1)
00058 , instance_name_for_metrics_("FragmentGenerator")
00059 , sleep_on_stop_us_(0) {}
00060
00061 artdaq::CommandableFragmentGenerator::CommandableFragmentGenerator(const fhicl::ParameterSet& ps)
00062 : mutex_()
00063 , request_port_(ps.get<int>("request_port", 3001))
00064 , request_addr_(ps.get<std::string>("request_address", "227.128.12.26"))
00065 , requests_()
00066 , request_stop_requested_(false)
00067 , end_of_run_timeout_ms_(ps.get<size_t>("end_of_run_quiet_timeout_ms", 1000))
00068 , windowOffset_(ps.get<Fragment::timestamp_t>("request_window_offset", 0))
00069 , windowWidth_(ps.get<Fragment::timestamp_t>("request_window_width", 0))
00070 , staleTimeout_(ps.get<Fragment::timestamp_t>("stale_request_timeout", 0xFFFFFFFF))
00071 , uniqueWindows_(ps.get<bool>("request_windows_are_unique", true))
00072 , useDataThread_(ps.get<bool>("separate_data_thread", false))
00073 , data_thread_running_(false)
00074 , dataBufferDepthFragments_(0)
00075 , dataBufferDepthBytes_(0)
00076 , maxDataBufferDepthFragments_(ps.get<int>("data_buffer_depth_fragments", 1000))
00077 , maxDataBufferDepthBytes_(ps.get<size_t>("data_buffer_depth_mb", 1000) * 1024 * 1024)
00078 , useMonitoringThread_(ps.get<bool>("separate_monitoring_thread", false))
00079 , monitoringInterval_(ps.get<int64_t>("hardware_poll_interval_us", 0))
00080 , lastMonitoringCall_(std::chrono::steady_clock::now())
00081 , isHardwareOK_(true)
00082 , dataBuffer_()
00083 , newDataBuffer_()
00084 , run_number_(-1)
00085 , subrun_number_(-1)
00086 , timeout_(std::numeric_limits<uint64_t>::max())
00087 , timestamp_(std::numeric_limits<uint64_t>::max())
00088 , should_stop_(false)
00089 , exception_(false)
00090 , latest_exception_report_("none")
00091 , ev_counter_(1)
00092 , board_id_(-1)
00093 , sleep_on_stop_us_(0)
00094 {
00095 board_id_ = ps.get<int>("board_id");
00096 instance_name_for_metrics_ = "BoardReader." + boost::lexical_cast<std::string>(board_id_);
00097
00098 fragment_ids_ = ps.get<std::vector<artdaq::Fragment::fragment_id_t>>("fragment_ids", std::vector<artdaq::Fragment::fragment_id_t>());
00099
00100 TRACE(24, "artdaq::CommandableFragmentGenerator::CommandableFragmentGenerator(ps)");
00101 int fragment_id = ps.get<int>("fragment_id", -99);
00102
00103 if (fragment_id != -99)
00104 {
00105 if (fragment_ids_.size() != 0)
00106 {
00107 latest_exception_report_ = "Error in CommandableFragmentGenerator: can't both define \"fragment_id\" and \"fragment_ids\" in FHiCL document";
00108 throw cet::exception(latest_exception_report_);
00109 }
00110 else
00111 {
00112 fragment_ids_.emplace_back(fragment_id);
00113 }
00114 }
00115
00116 sleep_on_stop_us_ = ps.get<int>("sleep_on_stop_us", 0);
00117
00118 dataBuffer_.emplace_back(FragmentPtr(new Fragment()));
00119 (*dataBuffer_.begin())->setSystemType(Fragment::EmptyFragmentType);
00120
00121 std::string modeString = ps.get<std::string>("request_mode", "ignored");
00122 if (modeString == "single" || modeString == "Single")
00123 {
00124 TRACE(3, "CommandableFragmentGenerator: RequestMode set to SINGLE");
00125 mode_ = RequestMode::Single;
00126 }
00127 else if (modeString.find("buffer") != std::string::npos || modeString.find("Buffer") != std::string::npos)
00128 {
00129 TRACE(3, "CommandableFragmentGenerator: RequestMode set to BUFFER");
00130 mode_ = RequestMode::Buffer;
00131 }
00132 else if (modeString == "window" || modeString == "Window")
00133 {
00134 TRACE(3, "CommandableFragmentGenerator: RequestMode set to WINDOW");
00135 mode_ = RequestMode::Window;
00136 }
00137 else if (modeString.find("ignore") != std::string::npos || modeString.find("Ignore") != std::string::npos)
00138 {
00139 TRACE(3, "CommandableFragmentGenerator: RequestMode set to IGNORE");
00140 mode_ = RequestMode::Ignored;
00141 }
00142 TLOG_DEBUG("CommandableFragmentGenerator") << "Request mode is " << printMode_() << TLOG_ENDL;
00143
00144 if (mode_ != RequestMode::Ignored)
00145 {
00146 if (!useDataThread_)
00147 {
00148 latest_exception_report_ = "Error in CommandableFragmentGenerator: use_data_thread must be true when request_mode is not \"Ignored\"!";
00149 throw cet::exception(latest_exception_report_);
00150 }
00151 setupRequestListener();
00152 }
00153 }
00154
00155 void artdaq::CommandableFragmentGenerator::setupRequestListener()
00156 {
00157 request_socket_ = socket(AF_INET, SOCK_DGRAM, IPPROTO_UDP);
00158 if (!request_socket_)
00159 {
00160 throw art::Exception(art::errors::Configuration) << "CommandableFragmentGenerator: Error creating socket for receiving data requests!" << std::endl;
00161 exit(1);
00162 }
00163
00164 struct sockaddr_in si_me_request;
00165
00166 int yes = 1;
00167 if (setsockopt(request_socket_, SOL_SOCKET, SO_REUSEADDR, &yes, sizeof(yes)) < 0)
00168 {
00169 throw art::Exception(art::errors::Configuration) <<
00170 "RequestedFragmentGenrator: Unable to enable port reuse on request socket" << std::endl;
00171 exit(1);
00172 }
00173 memset(&si_me_request, 0, sizeof(si_me_request));
00174 si_me_request.sin_family = AF_INET;
00175 si_me_request.sin_port = htons(request_port_);
00176 si_me_request.sin_addr.s_addr = htonl(INADDR_ANY);
00177 if (bind(request_socket_, (struct sockaddr *)&si_me_request, sizeof(si_me_request)) == -1)
00178 {
00179 throw art::Exception(art::errors::Configuration) <<
00180 "CommandableFragmentGenerator: Cannot bind request socket to port " << request_port_ << std::endl;
00181 exit(1);
00182 }
00183
00184 struct ip_mreq mreq;
00185 int sts = ResolveHost(request_addr_.c_str(), mreq.imr_multiaddr);
00186 if (sts == -1)
00187 {
00188 throw art::Exception(art::errors::Configuration) << "Unable to resolve multicast request address" << std::endl;
00189 exit(1);
00190 }
00191 mreq.imr_interface.s_addr = htonl(INADDR_ANY);
00192 if (setsockopt(request_socket_, IPPROTO_IP, IP_ADD_MEMBERSHIP, &mreq, sizeof(mreq)) < 0)
00193 {
00194 throw art::Exception(art::errors::Configuration) <<
00195 "CommandableFragmentGenerator: Unable to join multicast group" << std::endl;
00196 exit(1);
00197 }
00198 }
00199
00200 artdaq::CommandableFragmentGenerator::~CommandableFragmentGenerator()
00201 {
00202 TLOG_DEBUG("CommandableFragmentGenerator") << "Joining dataThread" << TLOG_ENDL;
00203 if (dataThread_.joinable()) dataThread_.join();
00204 TLOG_DEBUG("CommandableFragmentGenerator") << "Joining monitoringThread" << TLOG_ENDL;
00205 if (monitoringThread_.joinable()) monitoringThread_.join();
00206 TLOG_DEBUG("CommandableFragmentGenerator") << "Joining requestThread" << TLOG_ENDL;
00207 if (requestThread_.joinable()) requestThread_.join();
00208 }
00209
00210 bool artdaq::CommandableFragmentGenerator::getNext(FragmentPtrs& output)
00211 {
00212 bool result = true;
00213
00214 if (check_stop()) usleep(sleep_on_stop_us_);
00215 if (exception()) return false;
00216
00217 if (!useMonitoringThread_ && monitoringInterval_ > 0)
00218 {
00219 TRACE(4, "CFG: Collecting Monitoring Data");
00220 auto now = std::chrono::steady_clock::now();
00221 if (std::chrono::duration_cast<std::chrono::microseconds>(now - lastMonitoringCall_).count() >= monitoringInterval_)
00222 {
00223 isHardwareOK_ = checkHWStatus_();
00224 lastMonitoringCall_ = now;
00225 }
00226 }
00227
00228 try
00229 {
00230 std::lock_guard<std::mutex> lk(mutex_);
00231 if (useDataThread_)
00232 {
00233 TRACE(4, "CFG: Calling applyRequests");
00234 result = applyRequests(output);
00235 TRACE(4, "CFG: Done with applyRequests");
00236
00237 if (exception()) {
00238 throw cet::exception("CommandableFragmentGenerator") << "Exception found in BoardReader with board ID " << board_id() << "; BoardReader will now return error status when queried";
00239 }
00240 }
00241 else
00242 {
00243 TRACE(4, "CFG: Calling getNext_ %zu", ev_counter());
00244 try {
00245 result = getNext_(output);
00246 } catch (...) {
00247 throw;
00248 }
00249 TRACE(4, "CFG: Done with getNext_ %zu", ev_counter());
00250 }
00251 }
00252 catch (const cet::exception& e)
00253 {
00254 latest_exception_report_ = "cet::exception caught in getNext(): ";
00255 latest_exception_report_.append(e.what());
00256 TLOG_ERROR("getNext") << "cet::exception caught: " << e << TLOG_ENDL;
00257 set_exception(true);
00258 return false;
00259 }
00260 catch (const boost::exception& e)
00261 {
00262 latest_exception_report_ = "boost::exception caught in getNext(): ";
00263 latest_exception_report_.append(boost::diagnostic_information(e));
00264 TLOG_ERROR("getNext") << "boost::exception caught: " << boost::diagnostic_information(e) << TLOG_ENDL;
00265 set_exception(true);
00266 return false;
00267 }
00268 catch (const std::exception& e)
00269 {
00270 latest_exception_report_ = "std::exception caught in getNext(): ";
00271 latest_exception_report_.append(e.what());
00272 TLOG_ERROR("getNext") << "std::exception caught: " << e.what() << TLOG_ENDL;
00273 set_exception(true);
00274 return false;
00275 }
00276 catch (...)
00277 {
00278 latest_exception_report_ = "Unknown exception caught in getNext().";
00279 TLOG_ERROR("getNext") << "unknown exception caught" << TLOG_ENDL;
00280 set_exception(true);
00281 return false;
00282 }
00283
00284 if (!result)
00285 {
00286 TRACE(4, "CFG:getNext Stopped");
00287 TLOG_DEBUG("getNext") << "stopped " << TLOG_ENDL;
00288 }
00289
00290 return result;
00291 }
00292
00293 bool artdaq::CommandableFragmentGenerator::check_stop()
00294 {
00295 TRACE(4, "CFG::check_stop: should_stop=%i, useDataThread_=%i, requests_.size()=%zu, exception status =%d", should_stop(), useDataThread_, requests_.size(), int(exception()));
00296 if (!should_stop()) return false;
00297 if (!useDataThread_ || mode_ == RequestMode::Ignored) return true;
00298 if (!request_stop_requested_) return false;
00299
00300 auto dur = std::chrono::steady_clock::now() - request_stop_timeout_;
00301 return std::chrono::duration_cast<std::chrono::milliseconds>(dur).count() > static_cast<int>(end_of_run_timeout_ms_);
00302 }
00303
00304 int artdaq::CommandableFragmentGenerator::fragment_id() const
00305 {
00306 if (fragment_ids_.size() != 1)
00307 {
00308 throw cet::exception("Error in CommandableFragmentGenerator: can't call fragment_id() unless member fragment_ids_ vector is length 1");
00309 }
00310 else
00311 {
00312 return fragment_ids_[0];
00313 }
00314 }
00315
00316 size_t artdaq::CommandableFragmentGenerator::ev_counter_inc(size_t step, bool force)
00317 {
00318 if (force || mode_ == RequestMode::Ignored)
00319 {
00320 return ev_counter_.fetch_add(step);
00321 }
00322 return ev_counter_.load();
00323 }
00324
00325 void artdaq::CommandableFragmentGenerator::StartCmd(int run, uint64_t timeout, uint64_t timestamp)
00326 {
00327 if (run < 0) throw cet::exception("CommandableFragmentGenerator") << "negative run number";
00328
00329 timeout_ = timeout;
00330 timestamp_ = timestamp;
00331 ev_counter_.store(1);
00332 should_stop_.store(false);
00333 exception_.store(false);
00334 run_number_ = run;
00335 subrun_number_ = 1;
00336 latest_exception_report_ = "none";
00337 dataBuffer_.clear();
00338 requests_.clear();
00339
00340 start();
00341
00342 std::unique_lock<std::mutex> lk(mutex_);
00343 if (useDataThread_) startDataThread();
00344 if (useMonitoringThread_) startMonitoringThread();
00345 if (mode_ != RequestMode::Ignored) startRequestReceiverThread();
00346 }
00347
00348 void artdaq::CommandableFragmentGenerator::StopCmd(uint64_t timeout, uint64_t timestamp)
00349 {
00350 TLOG_DEBUG("CommandableFragmentGenerator") << "Stop Command received." << TLOG_ENDL;
00351 TRACE(4, "CFG: Stop Command Received");
00352
00353 timeout_ = timeout;
00354 timestamp_ = timestamp;
00355
00356 stopNoMutex();
00357 should_stop_.store(true);
00358 std::unique_lock<std::mutex> lk(mutex_);
00359 stop();
00360 }
00361
00362 void artdaq::CommandableFragmentGenerator::PauseCmd(uint64_t timeout, uint64_t timestamp)
00363 {
00364 timeout_ = timeout;
00365 timestamp_ = timestamp;
00366
00367 pauseNoMutex();
00368 should_stop_.store(true);
00369 std::unique_lock<std::mutex> lk(mutex_);
00370
00371 pause();
00372 }
00373
00374 void artdaq::CommandableFragmentGenerator::ResumeCmd(uint64_t timeout, uint64_t timestamp)
00375 {
00376 timeout_ = timeout;
00377 timestamp_ = timestamp;
00378
00379 subrun_number_ += 1;
00380 should_stop_ = false;
00381
00382 dataBuffer_.clear();
00383 requests_.clear();
00384
00385
00386 resume();
00387
00388 std::unique_lock<std::mutex> lk(mutex_);
00389 if (useDataThread_) startDataThread();
00390 if (useMonitoringThread_) startMonitoringThread();
00391 if (mode_ != RequestMode::Ignored) startRequestReceiverThread();
00392 }
00393
00394 std::string artdaq::CommandableFragmentGenerator::ReportCmd(std::string const& which)
00395 {
00396 std::lock_guard<std::mutex> lk(mutex_);
00397
00398
00399
00400
00401
00402
00403 std::string childReport = reportSpecific(which);
00404 if (childReport.length() > 0) { return childReport; }
00405
00406
00407 if (which == "latest_exception")
00408 {
00409 return latest_exception_report_;
00410 }
00411
00412
00413 childReport = report();
00414 if (childReport.length() > 0) { return childReport; }
00415
00416
00417 std::string tmpString = "The \"" + which + "\" command is not ";
00418 tmpString.append("currently supported by the ");
00419 tmpString.append(metricsReportingInstanceName());
00420 tmpString.append(" fragment generator.");
00421 return tmpString;
00422 }
00423
00424
00425 void artdaq::CommandableFragmentGenerator::pauseNoMutex()
00426 {
00427 #pragma message "Using default implementation of CommandableFragmentGenerator::pauseNoMutex()"
00428 }
00429
00430 void artdaq::CommandableFragmentGenerator::pause()
00431 {
00432 #pragma message "Using default implementation of CommandableFragmentGenerator::pause()"
00433 }
00434
00435 void artdaq::CommandableFragmentGenerator::resume()
00436 {
00437 #pragma message "Using default implementation of CommandableFragmentGenerator::resume()"
00438 }
00439
00440 std::string artdaq::CommandableFragmentGenerator::report()
00441 {
00442 #pragma message "Using default implementation of CommandableFragmentGenerator::report()"
00443 return "";
00444 }
00445
00446 std::string artdaq::CommandableFragmentGenerator::reportSpecific(std::string const&)
00447 {
00448 #pragma message "Using default implementation of CommandableFragmentGenerator::reportSpecific(std::string)"
00449 return "";
00450 }
00451
00452 bool artdaq::CommandableFragmentGenerator::checkHWStatus_()
00453 {
00454 #pragma message "Using default implementation of CommandableFragmentGenerator::checkHWStatus_()"
00455 return true;
00456 }
00457
00458 void artdaq::CommandableFragmentGenerator::startDataThread()
00459 {
00460 if (dataThread_.joinable()) dataThread_.join();
00461 TLOG_INFO("CommandableFragmentGenerator") << "Starting Data Receiver Thread" << TLOG_ENDL;
00462 dataThread_ = std::thread(&CommandableFragmentGenerator::getDataLoop, this);
00463 }
00464
00465 void artdaq::CommandableFragmentGenerator::startMonitoringThread()
00466 {
00467 if (monitoringThread_.joinable()) monitoringThread_.join();
00468 TLOG_INFO("CommandableFragmentGenerator") << "Starting Hardware Monitoring Thread" << TLOG_ENDL;
00469 monitoringThread_ = std::thread(&CommandableFragmentGenerator::getMonitoringDataLoop, this);
00470 }
00471
00472 void artdaq::CommandableFragmentGenerator::startRequestReceiverThread()
00473 {
00474 if (requestThread_.joinable()) requestThread_.join();
00475 TLOG_INFO("CommandableFragmentGenerator") << "Starting Request Reception Thread" << TLOG_ENDL;
00476 requestThread_ = std::thread(&CommandableFragmentGenerator::receiveRequestsLoop, this);
00477 }
00478
00479 std::string artdaq::CommandableFragmentGenerator::printMode_()
00480 {
00481 switch (mode_)
00482 {
00483 case RequestMode::Single:
00484 return "Single";
00485 case RequestMode::Buffer:
00486 return "Buffer";
00487 case RequestMode::Window:
00488 return "Window";
00489 case RequestMode::Ignored:
00490 return "Ignored";
00491 }
00492
00493 return "ERROR";
00494 }
00495
00496 void artdaq::CommandableFragmentGenerator::getDataLoop()
00497 {
00498 data_thread_running_ = true;
00499 while (true)
00500 {
00501 if (!isHardwareOK_)
00502 {
00503 TLOG_DEBUG("CommandableFragmentGenerator") << "getDataLoop: isHardwareOK is " << isHardwareOK_ << ", aborting data thread" << TLOG_ENDL;
00504 data_thread_running_ = false;
00505 return;
00506 }
00507
00508 TRACE(4, "CommandableFragmentGenerator::getDataLoop: calling getNext_");
00509
00510 bool data = false;
00511
00512 try {
00513 data = getNext_(newDataBuffer_);
00514 } catch (...) {
00515 ExceptionHandler(ExceptionHandlerRethrow::no,
00516 "Exception thrown by fragment generator in CommandableFragmentGenerator::getDataLoop; setting exception state to \"true\"");
00517 set_exception(true);
00518
00519 data_thread_running_ = false;
00520 return;
00521 }
00522
00523 auto startwait = std::chrono::steady_clock::now();
00524 auto first = true;
00525 auto lastwaittime = 0;
00526 while (dataBufferIsTooLarge())
00527 {
00528 if (should_stop())
00529 {
00530 TLOG_DEBUG("CommandableFragmentGenerator") << "Run ended while waiting for buffer to shrink!" << TLOG_ENDL;
00531 std::unique_lock<std::mutex> lock(dataBufferMutex_);
00532 getDataBufferStats();
00533 dataCondition_.notify_all();
00534 data_thread_running_ = false;
00535 return;
00536 }
00537 auto waittime = std::chrono::duration_cast<std::chrono::milliseconds>(std::chrono::steady_clock::now() - startwait).count();
00538
00539 if (first || (waittime != lastwaittime && waittime % 1000 == 0))
00540 {
00541 TLOG_WARNING("CommandableFragmentGenerator") << "Bad Omen: Data Buffer has exceeded its size limits. Check the connection between the BoardReader and the EventBuilders! (seq_id=" << ev_counter() << ")" << TLOG_ENDL;
00542 first = false;
00543 }
00544 if (waittime % 5 && waittime != lastwaittime)
00545 {
00546 TRACE(4, "CFG::getDataLoop: Data Retreival paused for %lu ms waiting for data buffer to drain", waittime);
00547 }
00548 lastwaittime = waittime;
00549 usleep(1000);
00550 }
00551
00552 if (data)
00553 {
00554 std::unique_lock<std::mutex> lock(dataBufferMutex_);
00555 switch (mode_)
00556 {
00557 case RequestMode::Single:
00558
00559 while (newDataBuffer_.size() >= fragment_ids_.size())
00560 {
00561 dataBuffer_.clear();
00562 auto it = newDataBuffer_.begin();
00563 std::advance(it, fragment_ids_.size());
00564 dataBuffer_.splice(dataBuffer_.end(), newDataBuffer_, newDataBuffer_.begin(), it);
00565 }
00566 break;
00567 case RequestMode::Buffer:
00568 case RequestMode::Ignored:
00569 case RequestMode::Window:
00570 default:
00571
00572 dataBuffer_.splice(dataBuffer_.end(), newDataBuffer_);
00573 break;
00574 }
00575 getDataBufferStats();
00576 }
00577
00578 {
00579 std::unique_lock<std::mutex> lock(dataBufferMutex_);
00580 if (dataBuffer_.size() > 0)
00581 {
00582 dataCondition_.notify_all();
00583 }
00584 }
00585 if (!data)
00586 {
00587 TLOG_INFO("CommandableFragmentGenerator") << "Data flow has stopped. Ending data collection thread" << TLOG_ENDL;
00588 data_thread_running_ = false;
00589 return;
00590 }
00591 }
00592 }
00593
00594 bool artdaq::CommandableFragmentGenerator::dataBufferIsTooLarge()
00595 {
00596 return (maxDataBufferDepthFragments_ > 0 && dataBufferDepthFragments_ >= maxDataBufferDepthFragments_) || (maxDataBufferDepthBytes_ > 0 && dataBufferDepthBytes_ >= maxDataBufferDepthBytes_);
00597 }
00598
00599 void artdaq::CommandableFragmentGenerator::getDataBufferStats()
00600 {
00602 dataBufferDepthFragments_ = dataBuffer_.size();
00603 size_t acc = 0;
00604 for (auto i = dataBuffer_.begin(); i != dataBuffer_.end(); ++i)
00605 {
00606 acc += (*i)->sizeBytes();
00607 }
00608 dataBufferDepthBytes_ = acc;
00609
00610 if (metricMan)
00611 {
00612 metricMan->sendMetric("Buffer Depth Fragments", dataBufferDepthFragments_.load(), "fragments", 1);
00613 metricMan->sendMetric("Buffer Depth Bytes", dataBufferDepthBytes_.load(), "bytes", 1);
00614 }
00615 TRACE(4, "CFG::getDataBufferStats: frags=%i/%i, sz=%zd/%zd", dataBufferDepthFragments_.load(), maxDataBufferDepthFragments_, dataBufferDepthBytes_.load(), maxDataBufferDepthBytes_);
00616 }
00617
00618 void artdaq::CommandableFragmentGenerator::checkDataBuffer()
00619 {
00620 std::unique_lock<std::mutex> lock(dataBufferMutex_);
00621 dataCondition_.wait_for(lock, std::chrono::milliseconds(10));
00622 if (dataBufferDepthFragments_ > 0)
00623 {
00624 if ((mode_ == RequestMode::Buffer || mode_ == RequestMode::Window))
00625 {
00626
00627 while (dataBufferIsTooLarge())
00628 {
00629 dataBuffer_.erase(dataBuffer_.begin());
00630 getDataBufferStats();
00631 }
00632 if (dataBuffer_.size() > 0)
00633 {
00634 Fragment::timestamp_t last = dataBuffer_.back()->timestamp();
00635 Fragment::timestamp_t min = last > staleTimeout_ ? last - staleTimeout_ : 0;
00636 for (auto it = dataBuffer_.begin(); it != dataBuffer_.end();)
00637 {
00638 if ((*it)->timestamp() < min)
00639 {
00640 it = dataBuffer_.erase(it);
00641 }
00642 else
00643 {
00644 ++it;
00645 }
00646 }
00647 getDataBufferStats();
00648 }
00649 }
00650 else if (mode_ == RequestMode::Single && dataBuffer_.size() > fragment_ids_.size())
00651 {
00652
00653 while (dataBuffer_.size() > fragment_ids_.size())
00654 {
00655 dataBuffer_.erase(dataBuffer_.begin());
00656 }
00657 }
00658 }
00659 }
00660
00661 void artdaq::CommandableFragmentGenerator::getMonitoringDataLoop()
00662 {
00663 while (true)
00664 {
00665 if (should_stop() || monitoringInterval_ <= 0)
00666 {
00667 TLOG_DEBUG("CommandableFragmentGenerator") << "getMonitoringDataLoop: should_stop() is " << std::boolalpha << should_stop()
00668 << " and monitoringInterval is " << monitoringInterval_ << ", returning" << TLOG_ENDL;
00669 return;
00670 }
00671 TRACE(4, "CFG::getMonitoringDataLoop Determining whether to call checkHWStatus_");
00672
00673 auto now = std::chrono::steady_clock::now();
00674 if (std::chrono::duration_cast<std::chrono::microseconds>(now - lastMonitoringCall_).count() >= monitoringInterval_)
00675 {
00676 isHardwareOK_ = checkHWStatus_();
00677 lastMonitoringCall_ = now;
00678 }
00679 usleep(monitoringInterval_ / 10);
00680 }
00681 }
00682
00683 void artdaq::CommandableFragmentGenerator::receiveRequestsLoop()
00684 {
00685 while (true)
00686 {
00687 if (check_stop() || !isHardwareOK_ || exception())
00688 {
00689 TLOG_DEBUG("CommandableFragmentGenerator") << "receiveRequestsLoop: check_stop is " << std::boolalpha << check_stop()
00690 << ", isHardwareOK_ is " << isHardwareOK_ << ", and exception state is " << exception() << ", aborting request reception thread." << TLOG_ENDL;
00691 return;
00692 }
00693
00694
00695 if (mode_ == RequestMode::Ignored) return;
00696 TRACE(4, "CFG::receiveRequestsLoop: Polling Request socket for new requests");
00697
00698 int ms_to_wait = 1000;
00699 struct pollfd ufds[1];
00700 ufds[0].fd = request_socket_;
00701 ufds[0].events = POLLIN | POLLPRI;
00702 int rv = poll(ufds, 1, ms_to_wait);
00703 if (rv > 0)
00704 {
00705 if (ufds[0].revents == POLLIN || ufds[0].revents == POLLPRI)
00706 {
00707 TRACE(4, "CFG: Recieved packet on Request channel");
00708 detail::RequestHeader hdr_buffer;
00709 recv(request_socket_, &hdr_buffer, sizeof(hdr_buffer), 0);
00710 TRACE(4, "CFG: Request header word: 0x%x", (int)hdr_buffer.header);
00711 if (hdr_buffer.isValid())
00712 {
00713 if (hdr_buffer.mode == detail::RequestMessageMode::EndOfRun)
00714 {
00715 TLOG_INFO("CommandableFragmentGenerator") << "Received Request Message with the EndOfRun marker. (Re)Starting 1-second timeout for receiving all outstanding requests..." << TLOG_ENDL;
00716 request_stop_timeout_ = std::chrono::steady_clock::now();
00717 request_stop_requested_ = true;
00718 }
00719 std::vector<detail::RequestPacket> pkt_buffer(hdr_buffer.packet_count);
00720 recv(request_socket_, &pkt_buffer[0], sizeof(detail::RequestPacket) * hdr_buffer.packet_count, 0);
00721 bool anyNew = false;
00722 for (auto& buffer : pkt_buffer)
00723 {
00724 if (!buffer.isValid()) continue;
00725 if (requests_.count(buffer.sequence_id) && requests_[buffer.sequence_id] != buffer.timestamp)
00726 {
00727 TLOG_ERROR("CommandableFragmentGenerator") << "Received conflicting request for SeqID "
00728 << std::to_string(buffer.sequence_id) << "!"
00729 << " Old ts=" << std::to_string(requests_[buffer.sequence_id])
00730 << ", new ts=" << std::to_string(buffer.timestamp) << ". Keeping OLD!" << TLOG_ENDL;
00731 }
00732 else if (!requests_.count(buffer.sequence_id))
00733 {
00734 int delta = buffer.sequence_id - ev_counter();
00735 TRACE(4, "CFG: Recieved request for sequence ID %llu and timestamp %lu (delta: %d)", (unsigned long long)buffer.sequence_id, (unsigned long)buffer.timestamp, delta);
00736 if (delta < 0)
00737 {
00738 TRACE(4, "CFG: Already serviced this request! Ignoring...");
00739 }
00740 else
00741 {
00742 std::unique_lock<std::mutex> tlk(request_mutex_);
00743 requests_[buffer.sequence_id] = buffer.timestamp;
00744 anyNew = true;
00745 }
00746 }
00747 }
00748 if (anyNew)
00749 {
00750 std::unique_lock<std::mutex> lock(request_mutex_);
00751 requestCondition_.notify_all();
00752 }
00753 }
00754 }
00755 }
00756 }
00757 }
00758
00759 bool artdaq::CommandableFragmentGenerator::applyRequests(artdaq::FragmentPtrs& frags)
00760 {
00761 if (check_stop() || exception())
00762 {
00763 return false;
00764 }
00765
00766 if (mode_ == RequestMode::Ignored)
00767 {
00768 while (dataBufferDepthFragments_ <= 0)
00769 {
00770 if (check_stop() || exception()) return false;
00771 std::unique_lock<std::mutex> lock(dataBufferMutex_);
00772 dataCondition_.wait_for(lock, std::chrono::milliseconds(10), [this]() { return dataBufferDepthFragments_ > 0; });
00773 }
00774 }
00775 else
00776 {
00777 if ((check_stop() && requests_.size() <= 0) || exception()) return false;
00778 checkDataBuffer();
00779
00780 while (requests_.size() <= 0)
00781 {
00782 if (check_stop() || exception()) return false;
00783
00784 checkDataBuffer();
00785
00786 std::unique_lock<std::mutex> lock(request_mutex_);
00787 requestCondition_.wait_for(lock, std::chrono::milliseconds(10), [this]() { return requests_.size() > 0; });
00788 }
00789 }
00790
00791 {
00792 std::unique_lock<std::mutex> dlk(dataBufferMutex_);
00793 std::unique_lock<std::mutex> rlk(request_mutex_);
00794
00795
00796 if (mode_ == RequestMode::Ignored)
00797 {
00798
00799 TRACE(4, "CFG: Mode is Ignored; Copying data to output");
00800 std::move(dataBuffer_.begin(), dataBuffer_.end(), std::inserter(frags, frags.end()));
00801 dataBuffer_.clear();
00802 }
00803 else if (mode_ == RequestMode::Single)
00804 {
00805
00806 sendEmptyFragments(frags);
00807
00808 if (dataBuffer_.size() > 0)
00809 {
00810 TRACE(4, "CFG: Mode is Single; Sending copy of last event");
00811 for (auto& fragptr : dataBuffer_)
00812 {
00813
00814 auto frag = fragptr.get();
00815 auto newfrag = std::unique_ptr<artdaq::Fragment>(new Fragment(ev_counter(), frag->fragmentID()));
00816 newfrag->resize(frag->size() - detail::RawFragmentHeader::num_words());
00817 memcpy(newfrag->headerAddress(), frag->headerAddress(), frag->sizeBytes());
00818 newfrag->setTimestamp(requests_[ev_counter()]);
00819 newfrag->setSequenceID(ev_counter());
00820 frags.push_back(std::move(newfrag));
00821 }
00822 }
00823 else
00824 {
00825 sendEmptyFragment(frags, ev_counter(), "No data for");
00826 }
00827 requests_.clear();
00828 ev_counter_inc(1, true);
00829 }
00830 else if (mode_ == RequestMode::Buffer || mode_ == RequestMode::Window)
00831 {
00832 if (mode_ == RequestMode::Buffer)
00833 {
00834
00835 sendEmptyFragments(frags);
00836 }
00837 for (auto req = requests_.begin(); req != requests_.end();)
00838 {
00839 auto ts = req->second;
00840 if (req->first < ev_counter())
00841 {
00842 req = requests_.erase(req);
00843 continue;
00844 }
00845 while (req->first > ev_counter() && request_stop_requested_ && std::chrono::duration_cast<std::chrono::seconds>(std::chrono::steady_clock::now() - request_stop_timeout_).count() > 1)
00846 {
00847 sendEmptyFragment(frags,ev_counter(), "Missing request for");
00848 ev_counter_inc(1, true);
00849 }
00850 if (req->first > ev_counter())
00851 {
00852 ++req;
00853 continue;
00854 }
00855 TRACE(5, "CFG: ApplyRequestS: Checking that data exists for request window %llu (Buffered mode will always succeed)", (unsigned long long)req->first);
00856 Fragment::timestamp_t min = ts > windowOffset_ ? ts - windowOffset_ : 0;
00857 Fragment::timestamp_t max = min + windowWidth_;
00858 TRACE(5, "CFG::ApplyRequests: min is %lu and max is %lu and last point in buffer is %lu (sz=%zu)",
00859 (unsigned long)min, (unsigned long)max, (unsigned long)(dataBuffer_.size() > 0 ? dataBuffer_.back()->timestamp() : 0), dataBuffer_.size());
00860 bool windowClosed = mode_ != RequestMode::Window || (dataBuffer_.size() > 0 && dataBuffer_.back()->timestamp() >= max);
00861 if (windowClosed || !data_thread_running_)
00862 {
00863 TLOG_DEBUG("CommandableFragmentGenerator") << "Creating ContainerFragment for Buffered or Window-requested Fragments" << TLOG_ENDL;
00864 frags.emplace_back(new artdaq::Fragment(ev_counter(), fragment_id()));
00865 frags.back()->setTimestamp(ts);
00866 ContainerFragmentLoader cfl(*frags.back());
00867
00868 if (mode_ == RequestMode::Window && !data_thread_running_ && !windowClosed) cfl.set_missing_data(true);
00869 if (mode_ == RequestMode::Window && dataBuffer_.size() > 0 && dataBuffer_.front()->timestamp() > min)
00870 {
00871 TLOG_DEBUG("CommandableFragmentGenerator") << "Request Window covers data that is either before data collection began or has fallen off the end of the buffer" << TLOG_ENDL;
00872 cfl.set_missing_data(true);
00873 }
00874
00875
00876
00877 for (auto it = dataBuffer_.begin(); it != dataBuffer_.end();)
00878 {
00879 if (mode_ == RequestMode::Window)
00880 {
00881 Fragment::timestamp_t fragT = (*it)->timestamp();
00882 if (fragT < min || fragT > max)
00883 {
00884 ++it;
00885 continue;
00886 }
00887 }
00888
00889 TRACE(5, "CFG::ApplyRequests: Adding Fragment with timestamp %llu to Container", (unsigned long long)(*it)->timestamp());
00890 cfl.addFragment(*it);
00891
00892 if (mode_ == RequestMode::Buffer || (mode_ == RequestMode::Window && uniqueWindows_))
00893 {
00894 it = dataBuffer_.erase(it);
00895 }
00896 else
00897 {
00898 ++it;
00899 }
00900 }
00901 req = requests_.erase(req);
00902 ev_counter_inc(1, true);
00903 }
00904 else
00905 {
00906
00907 break;
00908 }
00909 }
00910 }
00911 getDataBufferStats();
00912 }
00913
00914 if (frags.size() > 0)
00915 TRACE(4, "CFG: Finished Processing Event %lu for fragment_id %i.", ev_counter() + 1, fragment_id());
00916 return true;
00917 }
00918
00919 bool artdaq::CommandableFragmentGenerator::sendEmptyFragment(artdaq::FragmentPtrs& frags, size_t seqId, std::string desc)
00920 {
00921 TLOG_WARNING("CommandableFragmentGenerator") << desc << " request " << seqId << ", sending empty fragment" << TLOG_ENDL;
00922 auto frag = new Fragment();
00923 frag->setSequenceID(seqId);
00924 frag->setSystemType(Fragment::EmptyFragmentType);
00925 frags.emplace_back(FragmentPtr(frag));
00926 return true;
00927 }
00928
00929 void artdaq::CommandableFragmentGenerator::sendEmptyFragments(artdaq::FragmentPtrs& frags)
00930 {
00931 auto sequence_id = Fragment::InvalidSequenceID;
00932 auto timestamp = Fragment::InvalidTimestamp;
00933
00934 for (auto it = requests_.begin(); it != requests_.end();)
00935 {
00936 auto seq = it->first;
00937 auto ts = it->second;
00938
00939
00940 if (++it == requests_.end())
00941 {
00942 sequence_id = seq;
00943 timestamp = ts;
00944 break;
00945 }
00946 if (seq < ev_counter()) continue;
00947
00948
00949 sendEmptyFragment(frags, ev_counter(), "Missed request for");
00950 ev_counter_inc(1, true);
00951 }
00952 requests_.clear();
00953
00954 if (sequence_id < ev_counter()) return;
00955 requests_[sequence_id] = timestamp;
00956 }