$treeview $search $mathjax $extrastylesheet
artdaq
v3_04_01
$projectbrief
|
$projectbrief
|
$searchbox |
00001 #define TRACE_NAME (app_name + "_CommandableFragmentGenerator").c_str() // include these 2 first - 00002 #include "artdaq/DAQdata/Globals.hh" 00003 00004 #include "artdaq/Application/CommandableFragmentGenerator.hh" 00005 00006 #include <boost/exception/all.hpp> 00007 #include <boost/throw_exception.hpp> 00008 00009 #include <limits> 00010 #include <iterator> 00011 00012 #include "canvas/Utilities/Exception.h" 00013 #include "cetlib_except/exception.h" 00014 #include "fhiclcpp/ParameterSet.h" 00015 00016 #include "artdaq-core/Utilities/SimpleLookupPolicy.hh" 00017 #include "artdaq-core/Data/Fragment.hh" 00018 #include "artdaq-core/Data/ContainerFragmentLoader.hh" 00019 #include "artdaq-core/Utilities/ExceptionHandler.hh" 00020 #include "artdaq-core/Utilities/TimeUtils.hh" 00021 00022 #include <fstream> 00023 #include <iomanip> 00024 #include <iterator> 00025 #include <iostream> 00026 #include <iomanip> 00027 #include <algorithm> 00028 #include <sys/poll.h> 00029 #include "artdaq/DAQdata/TCPConnect.hh" 00030 00031 #define TLVL_GETNEXT 10 00032 #define TLVL_GETNEXT_VERBOSE 20 00033 #define TLVL_CHECKSTOP 11 00034 #define TLVL_EVCOUNTERINC 12 00035 #define TLVL_GETDATALOOP 13 00036 #define TLVL_GETDATALOOP_DATABUFFWAIT 21 00037 #define TLVL_GETDATALOOP_VERBOSE 20 00038 #define TLVL_WAITFORBUFFERREADY 15 00039 #define TLVL_GETBUFFERSTATS 16 00040 #define TLVL_CHECKDATABUFFER 17 00041 #define TLVL_GETMONITORINGDATA 18 00042 #define TLVL_APPLYREQUESTS 9 00043 #define TLVL_SENDEMPTYFRAGMENTS 19 00044 #define TLVL_CHECKWINDOWS 14 00045 00046 artdaq::CommandableFragmentGenerator::CommandableFragmentGenerator() 00047 : mutex_() 00048 , requestReceiver_(nullptr) 00049 , windowOffset_(0) 00050 , windowWidth_(0) 00051 , staleTimeout_(Fragment::InvalidTimestamp) 00052 , expectedType_(Fragment::EmptyFragmentType) 00053 , maxFragmentCount_(std::numeric_limits<size_t>::max()) 00054 , uniqueWindows_(true) 00055 , windows_sent_ooo_() 00056 , missing_request_window_timeout_us_(1000000) 00057 , window_close_timeout_us_(2000000) 00058 , useDataThread_(false) 00059 , circularDataBufferMode_(false) 00060 , sleep_on_no_data_us_(0) 00061 , data_thread_running_(false) 00062 , dataBufferDepthFragments_(0) 00063 , dataBufferDepthBytes_(0) 00064 , maxDataBufferDepthFragments_(1000) 00065 , maxDataBufferDepthBytes_(1000) 00066 , useMonitoringThread_(false) 00067 , monitoringInterval_(0) 00068 , lastMonitoringCall_() 00069 , isHardwareOK_(true) 00070 , dataBuffer_() 00071 , newDataBuffer_() 00072 , run_number_(-1) 00073 , subrun_number_(-1) 00074 , timeout_(std::numeric_limits<uint64_t>::max()) 00075 , timestamp_(std::numeric_limits<uint64_t>::max()) 00076 , should_stop_(false) 00077 , exception_(false) 00078 , force_stop_(false) 00079 , latest_exception_report_("none") 00080 , ev_counter_(1) 00081 , board_id_(-1) 00082 , instance_name_for_metrics_("FragmentGenerator") 00083 , sleep_on_stop_us_(0) 00084 {} 00085 00086 artdaq::CommandableFragmentGenerator::CommandableFragmentGenerator(const fhicl::ParameterSet& ps) 00087 : mutex_() 00088 , requestReceiver_(nullptr) 00089 , windowOffset_(ps.get<Fragment::timestamp_t>("request_window_offset", 0)) 00090 , windowWidth_(ps.get<Fragment::timestamp_t>("request_window_width", 0)) 00091 , staleTimeout_(ps.get<Fragment::timestamp_t>("stale_request_timeout", 0xFFFFFFFF)) 00092 , expectedType_(ps.get<Fragment::type_t>("expected_fragment_type", Fragment::type_t(Fragment::EmptyFragmentType))) 00093 , uniqueWindows_(ps.get<bool>("request_windows_are_unique", true)) 00094 , windows_sent_ooo_() 00095 , missing_request_window_timeout_us_(ps.get<size_t>("missing_request_window_timeout_us", 5000000)) 00096 , window_close_timeout_us_(ps.get<size_t>("window_close_timeout_us", 2000000)) 00097 , useDataThread_(ps.get<bool>("separate_data_thread", false)) 00098 , circularDataBufferMode_(ps.get<bool>("circular_buffer_mode", false)) 00099 , sleep_on_no_data_us_(ps.get<size_t>("sleep_on_no_data_us", 0)) 00100 , data_thread_running_(false) 00101 , dataBufferDepthFragments_(0) 00102 , dataBufferDepthBytes_(0) 00103 , maxDataBufferDepthFragments_(ps.get<int>("data_buffer_depth_fragments", 1000)) 00104 , maxDataBufferDepthBytes_(ps.get<size_t>("data_buffer_depth_mb", 1000) * 1024 * 1024) 00105 , useMonitoringThread_(ps.get<bool>("separate_monitoring_thread", false)) 00106 , monitoringInterval_(ps.get<int64_t>("hardware_poll_interval_us", 0)) 00107 , lastMonitoringCall_() 00108 , isHardwareOK_(true) 00109 , dataBuffer_() 00110 , newDataBuffer_() 00111 , run_number_(-1) 00112 , subrun_number_(-1) 00113 , timeout_(std::numeric_limits<uint64_t>::max()) 00114 , timestamp_(std::numeric_limits<uint64_t>::max()) 00115 , should_stop_(false) 00116 , exception_(false) 00117 , force_stop_(false) 00118 , latest_exception_report_("none") 00119 , ev_counter_(1) 00120 , board_id_(-1) 00121 , sleep_on_stop_us_(0) 00122 { 00123 board_id_ = ps.get<int>("board_id"); 00124 instance_name_for_metrics_ = "BoardReader." + boost::lexical_cast<std::string>(board_id_); 00125 00126 fragment_ids_ = ps.get<std::vector<artdaq::Fragment::fragment_id_t>>("fragment_ids", std::vector<artdaq::Fragment::fragment_id_t>()); 00127 00128 TLOG(TLVL_TRACE) << "artdaq::CommandableFragmentGenerator::CommandableFragmentGenerator(ps)"; 00129 int fragment_id = ps.get<int>("fragment_id", -99); 00130 00131 if (fragment_id != -99) 00132 { 00133 if (fragment_ids_.size() != 0) 00134 { 00135 latest_exception_report_ = "Error in CommandableFragmentGenerator: can't both define \"fragment_id\" and \"fragment_ids\" in FHiCL document"; 00136 throw cet::exception(latest_exception_report_); 00137 } 00138 else 00139 { 00140 fragment_ids_.emplace_back(fragment_id); 00141 } 00142 } 00143 00144 sleep_on_stop_us_ = ps.get<int>("sleep_on_stop_us", 0); 00145 00146 std::string modeString = ps.get<std::string>("request_mode", "ignored"); 00147 if (modeString == "single" || modeString == "Single") 00148 { 00149 mode_ = RequestMode::Single; 00150 } 00151 else if (modeString.find("buffer") != std::string::npos || modeString.find("Buffer") != std::string::npos) 00152 { 00153 mode_ = RequestMode::Buffer; 00154 } 00155 else if (modeString == "window" || modeString == "Window") 00156 { 00157 mode_ = RequestMode::Window; 00158 } 00159 else if (modeString.find("ignore") != std::string::npos || modeString.find("Ignore") != std::string::npos) 00160 { 00161 mode_ = RequestMode::Ignored; 00162 } 00163 TLOG(TLVL_DEBUG) << "Request mode is " << printMode_(); 00164 00165 if (mode_ != RequestMode::Ignored) 00166 { 00167 if (!useDataThread_) 00168 { 00169 latest_exception_report_ = "Error in CommandableFragmentGenerator: use_data_thread must be true when request_mode is not \"Ignored\"!"; 00170 throw cet::exception(latest_exception_report_); 00171 } 00172 requestReceiver_.reset(new RequestReceiver(ps)); 00173 } 00174 } 00175 00176 artdaq::CommandableFragmentGenerator::~CommandableFragmentGenerator() 00177 { 00178 joinThreads(); 00179 requestReceiver_.reset(nullptr); 00180 } 00181 00182 void artdaq::CommandableFragmentGenerator::joinThreads() 00183 { 00184 should_stop_ = true; 00185 force_stop_ = true; 00186 TLOG(TLVL_DEBUG) << "Joining dataThread"; 00187 if (dataThread_.joinable()) dataThread_.join(); 00188 TLOG(TLVL_DEBUG) << "Joining monitoringThread"; 00189 if (monitoringThread_.joinable()) monitoringThread_.join(); 00190 TLOG(TLVL_DEBUG) << "joinThreads complete"; 00191 } 00192 00193 bool artdaq::CommandableFragmentGenerator::getNext(FragmentPtrs& output) 00194 { 00195 bool result = true; 00196 00197 if (check_stop()) usleep(sleep_on_stop_us_); 00198 if (exception() || force_stop_) return false; 00199 00200 if (!useMonitoringThread_ && monitoringInterval_ > 0) 00201 { 00202 TLOG(TLVL_GETNEXT) << "getNext: Checking whether to collect Monitoring Data"; 00203 auto now = std::chrono::steady_clock::now(); 00204 00205 if (TimeUtils::GetElapsedTimeMicroseconds(lastMonitoringCall_, now) >= static_cast<size_t>(monitoringInterval_)) 00206 { 00207 TLOG(TLVL_GETNEXT) << "getNext: Collecting Monitoring Data"; 00208 isHardwareOK_ = checkHWStatus_(); 00209 TLOG(TLVL_GETNEXT) << "getNext: isHardwareOK_ is now " << std::boolalpha << isHardwareOK_; 00210 lastMonitoringCall_ = now; 00211 } 00212 } 00213 00214 try 00215 { 00216 std::lock_guard<std::mutex> lk(mutex_); 00217 if (useDataThread_) 00218 { 00219 TLOG(TLVL_TRACE) << "getNext: Calling applyRequests"; 00220 result = applyRequests(output); 00221 TLOG(TLVL_TRACE) << "getNext: Done with applyRequests result=" << std::boolalpha << result; 00222 for (auto dataIter = output.begin(); dataIter != output.end(); ++dataIter) 00223 { 00224 TLOG(20) << "getNext: applyRequests() returned fragment with sequenceID = " << (*dataIter)->sequenceID() 00225 << ", timestamp = " << (*dataIter)->timestamp() << ", and sizeBytes = " << (*dataIter)->sizeBytes(); 00226 } 00227 00228 if (exception()) 00229 { 00230 TLOG(TLVL_ERROR) << "Exception found in BoardReader with board ID " << board_id() << "; BoardReader will now return error status when queried"; 00231 throw cet::exception("CommandableFragmentGenerator") << "Exception found in BoardReader with board ID " << board_id() << "; BoardReader will now return error status when queried"; 00232 } 00233 } 00234 else 00235 { 00236 if (!isHardwareOK_) 00237 { 00238 TLOG(TLVL_ERROR) << "Stopping CFG because the hardware reports bad status!"; 00239 return false; 00240 } 00241 TLOG(TLVL_TRACE) << "getNext: Calling getNext_ w/ ev_counter()=" << ev_counter(); 00242 try 00243 { 00244 result = getNext_(output); 00245 } 00246 catch (...) 00247 { 00248 throw; 00249 } 00250 TLOG(TLVL_TRACE) << "getNext: Done with getNext_ - ev_counter() now " << ev_counter(); 00251 for (auto dataIter = output.begin(); dataIter != output.end(); ++dataIter) 00252 { 00253 TLOG(TLVL_GETNEXT_VERBOSE) << "getNext: getNext_() returned fragment with sequenceID = " << (*dataIter)->sequenceID() 00254 << ", timestamp = " << (*dataIter)->timestamp() << ", and sizeBytes = " << (*dataIter)->sizeBytes(); 00255 } 00256 } 00257 } 00258 catch (const cet::exception& e) 00259 { 00260 latest_exception_report_ = "cet::exception caught in getNext(): "; 00261 latest_exception_report_.append(e.what()); 00262 TLOG(TLVL_ERROR) << "getNext: cet::exception caught: " << e; 00263 set_exception(true); 00264 return false; 00265 } 00266 catch (const boost::exception& e) 00267 { 00268 latest_exception_report_ = "boost::exception caught in getNext(): "; 00269 latest_exception_report_.append(boost::diagnostic_information(e)); 00270 TLOG(TLVL_ERROR) << "getNext: boost::exception caught: " << boost::diagnostic_information(e); 00271 set_exception(true); 00272 return false; 00273 } 00274 catch (const std::exception& e) 00275 { 00276 latest_exception_report_ = "std::exception caught in getNext(): "; 00277 latest_exception_report_.append(e.what()); 00278 TLOG(TLVL_ERROR) << "getNext: std::exception caught: " << e.what(); 00279 set_exception(true); 00280 return false; 00281 } 00282 catch (...) 00283 { 00284 latest_exception_report_ = "Unknown exception caught in getNext()."; 00285 TLOG(TLVL_ERROR) << "getNext: unknown exception caught"; 00286 set_exception(true); 00287 return false; 00288 } 00289 00290 if (!result) 00291 { 00292 TLOG(TLVL_DEBUG) << "getNext: Either getNext_ or applyRequests returned false, stopping"; 00293 } 00294 00295 if (metricMan && !output.empty()) 00296 { 00297 auto timestamp = output.front()->timestamp(); 00298 00299 if (output.size() > 1) 00300 { // Only bother sorting if >1 entry 00301 for (auto& outputfrag : output) 00302 { 00303 if (outputfrag->timestamp() > timestamp) 00304 { 00305 timestamp = outputfrag->timestamp(); 00306 } 00307 } 00308 } 00309 00310 metricMan->sendMetric("Last Timestamp", timestamp, "Ticks", 1, 00311 MetricMode::LastPoint, app_name); 00312 } 00313 00314 return result; 00315 } 00316 00317 bool artdaq::CommandableFragmentGenerator::check_stop() 00318 { 00319 TLOG(TLVL_CHECKSTOP) << "CFG::check_stop: should_stop=" << should_stop() << ", useDataThread_=" << useDataThread_ << ", exception status =" << int(exception()); 00320 00321 if (!should_stop()) return false; 00322 if (!useDataThread_ || mode_ == RequestMode::Ignored) return true; 00323 if (force_stop_) return true; 00324 00325 // check_stop returns true if the CFG should stop. We should wait for the RequestReceiver to stop before stopping. 00326 TLOG(TLVL_DEBUG) << "should_stop is true, force_stop_ is false, requestReceiver_->isRunning() is " << std::boolalpha << requestReceiver_->isRunning(); 00327 return !requestReceiver_->isRunning(); 00328 } 00329 00330 int artdaq::CommandableFragmentGenerator::fragment_id() const 00331 { 00332 if (fragment_ids_.size() != 1) 00333 { 00334 throw cet::exception("Error in CommandableFragmentGenerator: can't call fragment_id() unless member fragment_ids_ vector is length 1"); 00335 } 00336 else 00337 { 00338 return fragment_ids_[0]; 00339 } 00340 } 00341 00342 size_t artdaq::CommandableFragmentGenerator::ev_counter_inc(size_t step, bool force) 00343 { 00344 if (force || mode_ == RequestMode::Ignored) 00345 { 00346 TLOG(TLVL_EVCOUNTERINC) << "ev_counter_inc: Incrementing ev_counter from " << ev_counter() << " by " << step; 00347 return ev_counter_.fetch_add(step); 00348 } 00349 return ev_counter_.load(); 00350 } // returns the prev value 00351 00352 void artdaq::CommandableFragmentGenerator::StartCmd(int run, uint64_t timeout, uint64_t timestamp) 00353 { 00354 TLOG(TLVL_TRACE) << "Start Command received."; 00355 if (run < 0) throw cet::exception("CommandableFragmentGenerator") << "negative run number"; 00356 00357 timeout_ = timeout; 00358 timestamp_ = timestamp; 00359 ev_counter_.store(1); 00360 windows_sent_ooo_.clear(); 00361 { 00362 std::unique_lock<std::mutex> lock(dataBufferMutex_); 00363 dataBufferDepthBytes_ = 0; 00364 dataBufferDepthFragments_ = 0; 00365 dataBuffer_.clear(); 00366 } 00367 should_stop_.store(false); 00368 force_stop_.store(false); 00369 exception_.store(false); 00370 run_number_ = run; 00371 subrun_number_ = 1; 00372 latest_exception_report_ = "none"; 00373 00374 start(); 00375 00376 std::unique_lock<std::mutex> lk(mutex_); 00377 if (useDataThread_) startDataThread(); 00378 if (useMonitoringThread_) startMonitoringThread(); 00379 if (mode_ != RequestMode::Ignored) 00380 { 00381 requestReceiver_->SetRunNumber(static_cast<uint32_t>(run)); 00382 requestReceiver_->startRequestReception(); 00383 } 00384 TLOG(TLVL_TRACE) << "Start Command complete."; 00385 } 00386 00387 void artdaq::CommandableFragmentGenerator::StopCmd(uint64_t timeout, uint64_t timestamp) 00388 { 00389 TLOG(TLVL_TRACE) << "Stop Command received."; 00390 00391 timeout_ = timeout; 00392 timestamp_ = timestamp; 00393 if (requestReceiver_) { 00394 TLOG(TLVL_DEBUG) << "Stopping Request reception BEGIN"; 00395 requestReceiver_->stopRequestReception(); 00396 TLOG(TLVL_DEBUG) << "Stopping Request reception END"; 00397 } 00398 00399 stopNoMutex(); 00400 should_stop_.store(true); 00401 std::unique_lock<std::mutex> lk(mutex_); 00402 stop(); 00403 00404 joinThreads(); 00405 TLOG(TLVL_TRACE) << "Stop Command complete."; 00406 } 00407 00408 void artdaq::CommandableFragmentGenerator::PauseCmd(uint64_t timeout, uint64_t timestamp) 00409 { 00410 TLOG(TLVL_TRACE) << "Pause Command received."; 00411 timeout_ = timeout; 00412 timestamp_ = timestamp; 00413 //if (requestReceiver_->isRunning()) requestReceiver_->stopRequestReceiverThread(); 00414 00415 pauseNoMutex(); 00416 should_stop_.store(true); 00417 std::unique_lock<std::mutex> lk(mutex_); 00418 00419 pause(); 00420 } 00421 00422 void artdaq::CommandableFragmentGenerator::ResumeCmd(uint64_t timeout, uint64_t timestamp) 00423 { 00424 TLOG(TLVL_TRACE) << "Resume Command received."; 00425 timeout_ = timeout; 00426 timestamp_ = timestamp; 00427 00428 subrun_number_ += 1; 00429 should_stop_ = false; 00430 { 00431 std::unique_lock<std::mutex> lk(dataBufferMutex_); 00432 dataBufferDepthBytes_ = 0; 00433 dataBufferDepthFragments_ = 0; 00434 dataBuffer_.clear(); 00435 } 00436 // no lock required: thread not started yet 00437 resume(); 00438 00439 std::unique_lock<std::mutex> lk(mutex_); 00440 //if (useDataThread_) startDataThread(); 00441 //if (useMonitoringThread_) startMonitoringThread(); 00442 //if (mode_ != RequestMode::Ignored && !requestReceiver_->isRunning()) requestReceiver_->startRequestReceiverThread(); 00443 TLOG(TLVL_TRACE) << "Resume Command complete."; 00444 } 00445 00446 std::string artdaq::CommandableFragmentGenerator::ReportCmd(std::string const& which) 00447 { 00448 TLOG(TLVL_TRACE) << "Report Command received."; 00449 std::lock_guard<std::mutex> lk(mutex_); 00450 00451 // 14-May-2015, KAB: please see the comments associated with the report() 00452 // methods in the CommandableFragmentGenerator.hh file for more information 00453 // on the use of those methods in this method. 00454 00455 // check if the child class has something meaningful for this request 00456 std::string childReport = reportSpecific(which); 00457 if (childReport.length() > 0) { return childReport; } 00458 00459 // handle the requests that we can take care of at this level 00460 if (which == "latest_exception") 00461 { 00462 return latest_exception_report_; 00463 } 00464 00465 // check if the child class has provided a catch-all report function 00466 childReport = report(); 00467 if (childReport.length() > 0) { return childReport; } 00468 00469 // if we haven't been able to come up with any report so far, say so 00470 std::string tmpString = "The \"" + which + "\" command is not "; 00471 tmpString.append("currently supported by the "); 00472 tmpString.append(metricsReportingInstanceName()); 00473 tmpString.append(" fragment generator."); 00474 TLOG(TLVL_TRACE) << "Report Command complete."; 00475 return tmpString; 00476 } 00477 00478 // Default implemenetations of state functions 00479 void artdaq::CommandableFragmentGenerator::pauseNoMutex() 00480 { 00481 #pragma message "Using default implementation of CommandableFragmentGenerator::pauseNoMutex()" 00482 } 00483 00484 void artdaq::CommandableFragmentGenerator::pause() 00485 { 00486 #pragma message "Using default implementation of CommandableFragmentGenerator::pause()" 00487 } 00488 00489 void artdaq::CommandableFragmentGenerator::resume() 00490 { 00491 #pragma message "Using default implementation of CommandableFragmentGenerator::resume()" 00492 } 00493 00494 std::string artdaq::CommandableFragmentGenerator::report() 00495 { 00496 #pragma message "Using default implementation of CommandableFragmentGenerator::report()" 00497 return ""; 00498 } 00499 00500 std::string artdaq::CommandableFragmentGenerator::reportSpecific(std::string const&) 00501 { 00502 #pragma message "Using default implementation of CommandableFragmentGenerator::reportSpecific(std::string)" 00503 return ""; 00504 } 00505 00506 bool artdaq::CommandableFragmentGenerator::checkHWStatus_() 00507 { 00508 #pragma message "Using default implementation of CommandableFragmentGenerator::checkHWStatus_()" 00509 return true; 00510 } 00511 00512 bool artdaq::CommandableFragmentGenerator::metaCommand(std::string const&, std::string const&) 00513 { 00514 #pragma message "Using default implementation of CommandableFragmentGenerator::metaCommand(std::string, std::string)" 00515 return true; 00516 } 00517 00518 void artdaq::CommandableFragmentGenerator::startDataThread() 00519 { 00520 if (dataThread_.joinable()) dataThread_.join(); 00521 TLOG(TLVL_INFO) << "Starting Data Receiver Thread"; 00522 try { 00523 dataThread_ = boost::thread(&CommandableFragmentGenerator::getDataLoop, this); 00524 } 00525 catch (const boost::exception& e) 00526 { 00527 TLOG(TLVL_ERROR) << "Caught boost::exception starting Data Receiver thread: " << boost::diagnostic_information(e) << ", errno=" << errno; 00528 std::cerr << "Caught boost::exception starting Data Receiver thread: " << boost::diagnostic_information(e) << ", errno=" << errno << std::endl; 00529 exit(5); 00530 } 00531 } 00532 00533 void artdaq::CommandableFragmentGenerator::startMonitoringThread() 00534 { 00535 if (monitoringThread_.joinable()) monitoringThread_.join(); 00536 TLOG(TLVL_INFO) << "Starting Hardware Monitoring Thread"; 00537 try { 00538 monitoringThread_ = boost::thread(&CommandableFragmentGenerator::getMonitoringDataLoop, this); 00539 } 00540 catch (const boost::exception& e) 00541 { 00542 TLOG(TLVL_ERROR) << "Caught boost::exception starting Hardware Monitoring thread: " << boost::diagnostic_information(e) << ", errno=" << errno; 00543 std::cerr << "Caught boost::exception starting Hardware Monitoring thread: " << boost::diagnostic_information(e) << ", errno=" << errno << std::endl; 00544 exit(5); 00545 } 00546 } 00547 00548 std::string artdaq::CommandableFragmentGenerator::printMode_() 00549 { 00550 switch (mode_) 00551 { 00552 case RequestMode::Single: 00553 return "Single"; 00554 case RequestMode::Buffer: 00555 return "Buffer"; 00556 case RequestMode::Window: 00557 return "Window"; 00558 case RequestMode::Ignored: 00559 return "Ignored"; 00560 } 00561 00562 return "ERROR"; 00563 } 00564 00565 00566 // 00567 // The "useDataThread_" thread 00568 // 00569 void artdaq::CommandableFragmentGenerator::getDataLoop() 00570 { 00571 data_thread_running_ = true; 00572 while (!force_stop_) 00573 { 00574 if (!isHardwareOK_) 00575 { 00576 TLOG(TLVL_DEBUG) << "getDataLoop: isHardwareOK is " << isHardwareOK_ << ", aborting data thread"; 00577 data_thread_running_ = false; 00578 return; 00579 } 00580 00581 TLOG(TLVL_GETDATALOOP) << "getDataLoop: calling getNext_"; 00582 00583 bool data = false; 00584 auto startdata = std::chrono::steady_clock::now(); 00585 00586 try 00587 { 00588 data = getNext_(newDataBuffer_); 00589 } 00590 catch (...) 00591 { 00592 ExceptionHandler(ExceptionHandlerRethrow::no, 00593 "Exception thrown by fragment generator in CommandableFragmentGenerator::getDataLoop; setting exception state to \"true\""); 00594 set_exception(true); 00595 00596 data_thread_running_ = false; 00597 return; 00598 } 00599 00600 size_t newDataBufferDepthBytes = 0; 00601 for (auto dataIter = newDataBuffer_.begin(); dataIter != newDataBuffer_.end(); ++dataIter) 00602 { 00603 TLOG(TLVL_GETDATALOOP_VERBOSE) << "getDataLoop: getNext_() returned fragment with timestamp = " << (*dataIter)->timestamp() << ", and sizeBytes = " << (*dataIter)->sizeBytes(); 00604 newDataBufferDepthBytes += (*dataIter)->sizeBytes(); 00605 } 00606 00607 if (metricMan) 00608 { 00609 metricMan->sendMetric("Avg Data Acquisition Time", TimeUtils::GetElapsedTime(startdata), "s", 3, artdaq::MetricMode::Average); 00610 } 00611 00612 if (newDataBuffer_.size() == 0 && sleep_on_no_data_us_ > 0) 00613 { 00614 usleep(sleep_on_no_data_us_); 00615 } 00616 00617 TLOG(TLVL_GETDATALOOP_DATABUFFWAIT) << "Waiting for data buffer ready"; 00618 if (!waitForDataBufferReady()) return; 00619 TLOG(TLVL_GETDATALOOP_DATABUFFWAIT) << "Done waiting for data buffer ready"; 00620 00621 TLOG(TLVL_GETDATALOOP) << "getDataLoop: processing data"; 00622 if (data && !force_stop_) 00623 { 00624 std::unique_lock<std::mutex> lock(dataBufferMutex_); 00625 switch (mode_) 00626 { 00627 case RequestMode::Single: 00628 // While here, if for some strange reason more than one event's worth of data is returned from getNext_... 00629 while (newDataBuffer_.size() >= fragment_ids_.size()) 00630 { 00631 dataBufferDepthBytes_ = 0; 00632 dataBufferDepthFragments_ = 0; 00633 dataBuffer_.clear(); 00634 auto it = newDataBuffer_.begin(); 00635 std::advance(it, fragment_ids_.size()); 00636 dataBuffer_.splice(dataBuffer_.end(), newDataBuffer_, newDataBuffer_.begin(), it); 00637 00638 for (auto dbit = dataBuffer_.begin(); dbit != dataBuffer_.end(); ++dbit) 00639 { 00640 dataBufferDepthBytes_ += (*dbit)->sizeBytes(); 00641 } 00642 } 00643 break; 00644 case RequestMode::Buffer: 00645 case RequestMode::Ignored: 00646 case RequestMode::Window: 00647 default: 00648 //dataBuffer_.reserve(dataBuffer_.size() + newDataBuffer_.size()); 00649 dataBufferDepthBytes_ += newDataBufferDepthBytes; 00650 dataBuffer_.splice(dataBuffer_.end(), newDataBuffer_); 00651 break; 00652 } 00653 getDataBufferStats(); 00654 } 00655 00656 { 00657 std::unique_lock<std::mutex> lock(dataBufferMutex_); 00658 if (dataBuffer_.size() > 0) 00659 { 00660 dataCondition_.notify_all(); 00661 } 00662 } 00663 if (!data || force_stop_) 00664 { 00665 TLOG(TLVL_INFO) << "Data flow has stopped. Ending data collection thread"; 00666 std::unique_lock<std::mutex> lock(dataBufferMutex_); 00667 data_thread_running_ = false; 00668 if (requestReceiver_) requestReceiver_->ClearRequests(); 00669 newDataBuffer_.clear(); 00670 TLOG(TLVL_INFO) << "getDataLoop: Ending thread"; 00671 return; 00672 } 00673 } 00674 } 00675 00676 bool artdaq::CommandableFragmentGenerator::waitForDataBufferReady() 00677 { 00678 auto startwait = std::chrono::steady_clock::now(); 00679 auto first = true; 00680 auto lastwaittime = 0ULL; 00681 00682 { 00683 std::unique_lock<std::mutex> lock(dataBufferMutex_); 00684 getDataBufferStats(); 00685 } 00686 00687 while (dataBufferIsTooLarge()) 00688 { 00689 if (!circularDataBufferMode_) 00690 { 00691 if (should_stop()) 00692 { 00693 TLOG(TLVL_DEBUG) << "Run ended while waiting for buffer to shrink!"; 00694 std::unique_lock<std::mutex> lock(dataBufferMutex_); 00695 getDataBufferStats(); 00696 dataCondition_.notify_all(); 00697 data_thread_running_ = false; 00698 return false; 00699 } 00700 auto waittime = TimeUtils::GetElapsedTimeMilliseconds(startwait); 00701 00702 if (first || (waittime != lastwaittime && waittime % 1000 == 0)) 00703 { 00704 TLOG(TLVL_WARNING) << "Bad Omen: Data Buffer has exceeded its size limits. " 00705 << "(seq_id=" << ev_counter() 00706 << ", frags=" << dataBufferDepthFragments_ << "/" << maxDataBufferDepthFragments_ 00707 << ", szB=" << dataBufferDepthBytes_ << "/" << maxDataBufferDepthBytes_ << ")"; 00708 TLOG(TLVL_TRACE) << "Bad Omen: Possible causes include requests not getting through or Ignored-mode BR issues"; 00709 first = false; 00710 } 00711 if (waittime % 5 && waittime != lastwaittime) 00712 { 00713 TLOG(TLVL_WAITFORBUFFERREADY) << "getDataLoop: Data Retreival paused for " << waittime << " ms waiting for data buffer to drain"; 00714 } 00715 lastwaittime = waittime; 00716 usleep(1000); 00717 } 00718 else 00719 { 00720 std::unique_lock<std::mutex> lock(dataBufferMutex_); 00721 getDataBufferStats(); // Re-check under lock 00722 if (dataBufferIsTooLarge()) 00723 { 00724 if (dataBuffer_.begin() == dataBuffer_.end()) 00725 { 00726 TLOG(TLVL_WARNING) << "Data buffer is reported as too large, but doesn't contain any Fragments! Possible corrupt memory!"; 00727 continue; 00728 } 00729 if (*dataBuffer_.begin()) 00730 { 00731 TLOG(TLVL_WAITFORBUFFERREADY) << "waitForDataBufferReady: Dropping Fragment with timestamp " << (*dataBuffer_.begin())->timestamp() << " from data buffer (Buffer over-size, circular data buffer mode)"; 00732 } 00733 dataBufferDepthBytes_ -= (*dataBuffer_.begin())->sizeBytes(); 00734 dataBuffer_.erase(dataBuffer_.begin()); 00735 getDataBufferStats(); 00736 } 00737 00738 } 00739 } 00740 return true; 00741 } 00742 00743 bool artdaq::CommandableFragmentGenerator::dataBufferIsTooLarge() 00744 { 00745 return (maxDataBufferDepthFragments_ > 0 && dataBufferDepthFragments_ > maxDataBufferDepthFragments_) || (maxDataBufferDepthBytes_ > 0 && dataBufferDepthBytes_ > maxDataBufferDepthBytes_); 00746 } 00747 00748 void artdaq::CommandableFragmentGenerator::getDataBufferStats() 00749 { 00751 dataBufferDepthFragments_ = dataBuffer_.size(); 00752 00753 if (metricMan) 00754 { 00755 TLOG(TLVL_GETBUFFERSTATS) << "getDataBufferStats: Sending Metrics"; 00756 metricMan->sendMetric("Buffer Depth Fragments", dataBufferDepthFragments_.load(), "fragments", 1, MetricMode::LastPoint); 00757 metricMan->sendMetric("Buffer Depth Bytes", dataBufferDepthBytes_.load(), "bytes", 1, MetricMode::LastPoint); 00758 } 00759 TLOG(TLVL_GETBUFFERSTATS) << "getDataBufferStats: frags=" << dataBufferDepthFragments_.load() << "/" << maxDataBufferDepthFragments_ 00760 << ", sz=" << dataBufferDepthBytes_.load() << "/" << maxDataBufferDepthBytes_; 00761 } 00762 00763 void artdaq::CommandableFragmentGenerator::checkDataBuffer() 00764 { 00765 std::unique_lock<std::mutex> lock(dataBufferMutex_); 00766 dataCondition_.wait_for(lock, std::chrono::milliseconds(10)); 00767 if (dataBufferDepthFragments_ > 0) 00768 { 00769 if ((mode_ == RequestMode::Buffer || mode_ == RequestMode::Window)) 00770 { 00771 // Eliminate extra fragments 00772 getDataBufferStats(); 00773 while (dataBufferIsTooLarge()) 00774 { 00775 TLOG(TLVL_CHECKDATABUFFER) << "checkDataBuffer: Dropping Fragment with timestamp " << (*dataBuffer_.begin())->timestamp() << " from data buffer (Buffer over-size)"; 00776 dataBufferDepthBytes_ -= (*dataBuffer_.begin())->sizeBytes(); 00777 dataBuffer_.erase(dataBuffer_.begin()); 00778 getDataBufferStats(); 00779 } 00780 if (dataBuffer_.size() > 0) 00781 { 00782 TLOG(TLVL_CHECKDATABUFFER) << "Determining if Fragments can be dropped from data buffer"; 00783 Fragment::timestamp_t last = dataBuffer_.back()->timestamp(); 00784 Fragment::timestamp_t min = last > staleTimeout_ ? last - staleTimeout_ : 0; 00785 for (auto it = dataBuffer_.begin(); it != dataBuffer_.end();) 00786 { 00787 if ((*it)->timestamp() < min) 00788 { 00789 TLOG(TLVL_CHECKDATABUFFER) << "checkDataBuffer: Dropping Fragment with timestamp " << (*it)->timestamp() << " from data buffer (timeout=" << staleTimeout_ << ", min=" << min << ")"; 00790 dataBufferDepthBytes_ -= (*it)->sizeBytes(); 00791 it = dataBuffer_.erase(it); 00792 } 00793 else 00794 { 00795 break; 00796 } 00797 } 00798 getDataBufferStats(); 00799 } 00800 } 00801 else if (mode_ == RequestMode::Single && dataBuffer_.size() > fragment_ids_.size()) 00802 { 00803 // Eliminate extra fragments 00804 while (dataBuffer_.size() > fragment_ids_.size()) 00805 { 00806 dataBufferDepthBytes_ -= (*dataBuffer_.begin())->sizeBytes(); 00807 dataBuffer_.erase(dataBuffer_.begin()); 00808 } 00809 } 00810 } 00811 } 00812 00813 void artdaq::CommandableFragmentGenerator::getMonitoringDataLoop() 00814 { 00815 while (!force_stop_) 00816 { 00817 if (should_stop() || monitoringInterval_ <= 0) 00818 { 00819 TLOG(TLVL_DEBUG) << "getMonitoringDataLoop: should_stop() is " << std::boolalpha << should_stop() 00820 << " and monitoringInterval is " << monitoringInterval_ << ", returning"; 00821 return; 00822 } 00823 TLOG(TLVL_GETMONITORINGDATA) << "getMonitoringDataLoop: Determining whether to call checkHWStatus_"; 00824 00825 auto now = std::chrono::steady_clock::now(); 00826 if (TimeUtils::GetElapsedTimeMicroseconds(lastMonitoringCall_, now) >= static_cast<size_t>(monitoringInterval_)) 00827 { 00828 isHardwareOK_ = checkHWStatus_(); 00829 TLOG(TLVL_GETMONITORINGDATA) << "getMonitoringDataLoop: isHardwareOK_ is now " << std::boolalpha << isHardwareOK_; 00830 lastMonitoringCall_ = now; 00831 } 00832 usleep(monitoringInterval_ / 10); 00833 } 00834 } 00835 00836 void artdaq::CommandableFragmentGenerator::applyRequestsIgnoredMode(artdaq::FragmentPtrs& frags) 00837 { 00838 // We just copy everything that's here into the output. 00839 TLOG(TLVL_APPLYREQUESTS) << "Mode is Ignored; Copying data to output"; 00840 std::move(dataBuffer_.begin(), dataBuffer_.end(), std::inserter(frags, frags.end())); 00841 dataBufferDepthBytes_ = 0; 00842 dataBuffer_.clear(); 00843 } 00844 00845 void artdaq::CommandableFragmentGenerator::applyRequestsSingleMode(artdaq::FragmentPtrs& frags) 00846 { 00847 // We only care about the latest request received. Send empties for all others. 00848 auto requests = requestReceiver_->GetRequests(); 00849 while (requests.size() > 1) 00850 { 00851 // std::map is ordered by key => Last sequence ID in the map is the one we care about 00852 requestReceiver_->RemoveRequest(requests.begin()->first); 00853 requests.erase(requests.begin()); 00854 } 00855 sendEmptyFragments(frags, requests); 00856 00857 // If no requests remain after sendEmptyFragments, return 00858 if (requests.size() == 0 || !requests.count(ev_counter())) return; 00859 00860 if (dataBuffer_.size() > 0) 00861 { 00862 TLOG(TLVL_APPLYREQUESTS) << "Mode is Single; Sending copy of last event"; 00863 for (auto& fragptr : dataBuffer_) 00864 { 00865 // Return the latest data point 00866 auto frag = fragptr.get(); 00867 auto newfrag = std::unique_ptr<artdaq::Fragment>(new Fragment(ev_counter(), frag->fragmentID())); 00868 newfrag->resize(frag->size() - detail::RawFragmentHeader::num_words()); 00869 memcpy(newfrag->headerAddress(), frag->headerAddress(), frag->sizeBytes()); 00870 newfrag->setTimestamp(requests[ev_counter()]); 00871 newfrag->setSequenceID(ev_counter()); 00872 frags.push_back(std::move(newfrag)); 00873 } 00874 } 00875 else 00876 { 00877 sendEmptyFragment(frags, ev_counter(), "No data for"); 00878 } 00879 requestReceiver_->RemoveRequest(ev_counter()); 00880 ev_counter_inc(1, true); 00881 } 00882 00883 void artdaq::CommandableFragmentGenerator::applyRequestsBufferMode(artdaq::FragmentPtrs& frags) 00884 { 00885 // We only care about the latest request received. Send empties for all others. 00886 auto requests = requestReceiver_->GetRequests(); 00887 while (requests.size() > 1) 00888 { 00889 // std::map is ordered by key => Last sequence ID in the map is the one we care about 00890 requestReceiver_->RemoveRequest(requests.begin()->first); 00891 requests.erase(requests.begin()); 00892 } 00893 sendEmptyFragments(frags, requests); 00894 00895 // If no requests remain after sendEmptyFragments, return 00896 if (requests.size() == 0 || !requests.count(ev_counter())) return; 00897 00898 TLOG(TLVL_DEBUG) << "Creating ContainerFragment for Buffered Fragments"; 00899 frags.emplace_back(new artdaq::Fragment(ev_counter(), fragment_id())); 00900 frags.back()->setTimestamp(requests[ev_counter()]); 00901 ContainerFragmentLoader cfl(*frags.back()); 00902 cfl.set_missing_data(false); // Buffer mode is never missing data, even if there IS no data. 00903 00904 // Buffer mode TFGs should simply copy out the whole dataBuffer_ into a ContainerFragment 00905 // Window mode TFGs must do a little bit more work to decide which fragments to send for a given request 00906 for (auto it = dataBuffer_.begin(); it != dataBuffer_.end();) 00907 { 00908 TLOG(TLVL_APPLYREQUESTS) << "ApplyRequests: Adding Fragment with timestamp " << (*it)->timestamp() << " to Container with sequence ID " << ev_counter(); 00909 cfl.addFragment(*it); 00910 dataBufferDepthBytes_ -= (*it)->sizeBytes(); 00911 it = dataBuffer_.erase(it); 00912 } 00913 requestReceiver_->RemoveRequest(ev_counter()); 00914 ev_counter_inc(1, true); 00915 } 00916 00917 void artdaq::CommandableFragmentGenerator::applyRequestsWindowMode(artdaq::FragmentPtrs& frags) 00918 { 00919 TLOG(TLVL_APPLYREQUESTS) << "applyRequestsWindowMode BEGIN"; 00920 00921 auto requests = requestReceiver_->GetRequests(); 00922 00923 TLOG(TLVL_APPLYREQUESTS) << "applyRequestsWindowMode: Starting request processing"; 00924 for (auto req = requests.begin(); req != requests.end();) 00925 { 00926 TLOG(TLVL_APPLYREQUESTS) << "applyRequestsWindowMode: processing request with sequence ID " << req->first << ", timestamp " << req->second; 00927 00928 00929 while (req->first < ev_counter() && requests.size() > 0) 00930 { 00931 TLOG(TLVL_APPLYREQUESTS) << "applyRequestsWindowMode: Clearing passed request for sequence ID " << req->first; 00932 requestReceiver_->RemoveRequest(req->first); 00933 req = requests.erase(req); 00934 } 00935 if (requests.size() == 0) break; 00936 00937 auto ts = req->second; 00938 TLOG(TLVL_APPLYREQUESTS) << "applyRequests: Checking that data exists for request window " << req->first; 00939 Fragment::timestamp_t min = ts > windowOffset_ ? ts - windowOffset_ : 0; 00940 Fragment::timestamp_t max = min + windowWidth_; 00941 TLOG(TLVL_APPLYREQUESTS) << "ApplyRequests: min is " << min << ", max is " << max 00942 << " and first/last points in buffer are " << (dataBuffer_.size() > 0 ? dataBuffer_.front()->timestamp() : 0) 00943 << "/" << (dataBuffer_.size() > 0 ? dataBuffer_.back()->timestamp() : 0) 00944 << " (sz=" << dataBuffer_.size() << " [" << dataBufferDepthBytes_.load() 00945 << "/" << maxDataBufferDepthBytes_ << "])"; 00946 bool windowClosed = dataBuffer_.size() > 0 && dataBuffer_.back()->timestamp() >= max; 00947 bool windowTimeout = !windowClosed && TimeUtils::GetElapsedTimeMicroseconds(requestReceiver_->GetRequestTime(req->first)) > window_close_timeout_us_; 00948 if (windowTimeout) 00949 { 00950 TLOG(TLVL_WARNING) << "applyRequests: A timeout occurred waiting for data to close the request window ({" << min << "-" << max 00951 << "}, buffer={" << (dataBuffer_.size() > 0 ? dataBuffer_.front()->timestamp() : 0) << "-" 00952 << (dataBuffer_.size() > 0 ? dataBuffer_.back()->timestamp() : 0) 00953 << "} ). Time waiting: " 00954 << TimeUtils::GetElapsedTimeMicroseconds(requestReceiver_->GetRequestTime(req->first)) << " us " 00955 << "(> " << window_close_timeout_us_ << " us)."; 00956 } 00957 if (windowClosed || !data_thread_running_ || windowTimeout) 00958 { 00959 TLOG(TLVL_DEBUG) << "applyRequests: Creating ContainerFragment for Window-requested Fragments"; 00960 frags.emplace_back(new artdaq::Fragment(req->first, fragment_id())); 00961 frags.back()->setTimestamp(ts); 00962 ContainerFragmentLoader cfl(*frags.back()); 00963 00964 // In the spirit of NOvA's MegaPool: (RS = Request start (min), RE = Request End (max)) 00965 // --- | Buffer Start | --- | Buffer End | --- 00966 //1. RS RE | | | | 00967 //2. RS | | RE | | 00968 //3. RS | | | | RE 00969 //4. | | RS RE | | 00970 //5. | | RS | | RE 00971 //6. | | | | RS RE 00972 // 00973 // If RE (or RS) is after the end of the buffer, we wait for window_close_timeout_us_. If we're here, then that means that windowClosed is false, and the missing_data flag should be set. 00974 // If RS (or RE) is before the start of the buffer, then missing_data should be set to true, as data is assumed to arrive in the buffer in timestamp order 00975 // If the dataBuffer has size 0, then windowClosed will be false 00976 if (!windowClosed || (dataBuffer_.size() > 0 && dataBuffer_.front()->timestamp() > min)) 00977 { 00978 TLOG(TLVL_DEBUG) << "applyRequests: Request window starts before and/or ends after the current data buffer, setting ContainerFragment's missing_data flag!" 00979 << " (requestWindowRange=[" << min << "," << max << "], " 00980 << "buffer={" << (dataBuffer_.size() > 0 ? dataBuffer_.front()->timestamp() : 0) << "-" 00981 << (dataBuffer_.size() > 0 ? dataBuffer_.back()->timestamp() : 0) << "}"; 00982 cfl.set_missing_data(true); 00983 } 00984 00985 // Buffer mode TFGs should simply copy out the whole dataBuffer_ into a ContainerFragment 00986 // Window mode TFGs must do a little bit more work to decide which fragments to send for a given request 00987 int fragCount = 0; 00988 for (auto it = dataBuffer_.begin(); it != dataBuffer_.end();) 00989 { 00990 Fragment::timestamp_t fragT = (*it)->timestamp(); 00991 if (fragT < min || fragT > max || (fragT == max && windowWidth_ > 0)) 00992 { 00993 ++it; 00994 continue; 00995 } 00996 00997 TLOG(TLVL_APPLYREQUESTS) << "applyRequests: Adding (" << (++fragCount) << "th) Fragment with timestamp " 00998 << (*it)->timestamp() << " and sizeBytes " << (*it)->sizeBytes() 00999 << " to Container for sequence ID " << req->first; 01000 cfl.addFragment(*it); 01001 01002 if (uniqueWindows_) 01003 { 01004 dataBufferDepthBytes_ -= (*it)->sizeBytes(); 01005 it = dataBuffer_.erase(it); 01006 } 01007 else 01008 { 01009 ++it; 01010 } 01011 } 01012 requestReceiver_->RemoveRequest(req->first); 01013 checkOutOfOrderWindows(req->first); 01014 requestReceiver_->RemoveRequest(req->first); 01015 req = requests.erase(req); 01016 } 01017 else 01018 { 01019 ++req; 01020 } 01021 } 01022 } 01023 01024 bool artdaq::CommandableFragmentGenerator::applyRequests(artdaq::FragmentPtrs& frags) 01025 { 01026 if (check_stop() || exception()) 01027 { 01028 return false; 01029 } 01030 01031 // Wait for data, if in ignored mode, or a request otherwise 01032 if (mode_ == RequestMode::Ignored) 01033 { 01034 while (dataBufferDepthFragments_ <= 0) 01035 { 01036 if (check_stop() || exception() || !isHardwareOK_) return false; 01037 std::unique_lock<std::mutex> lock(dataBufferMutex_); 01038 dataCondition_.wait_for(lock, std::chrono::milliseconds(10), [this]() { return dataBufferDepthFragments_ > 0; }); 01039 } 01040 } 01041 else 01042 { 01043 if ((check_stop() && requestReceiver_->size() == 0) || exception()) return false; 01044 checkDataBuffer(); 01045 01046 // Wait up to 1000 ms for a request... 01047 auto counter = 0; 01048 01049 while (requestReceiver_->size() == 0 && counter < 100) 01050 { 01051 if (check_stop() || exception()) return false; 01052 01053 checkDataBuffer(); 01054 01055 requestReceiver_->WaitForRequests(10); // milliseconds 01056 counter++; 01057 } 01058 } 01059 01060 { 01061 std::unique_lock<std::mutex> dlk(dataBufferMutex_); 01062 01063 switch (mode_) 01064 { 01065 case RequestMode::Single: 01066 applyRequestsSingleMode(frags); 01067 break; 01068 case RequestMode::Window: 01069 applyRequestsWindowMode(frags); 01070 break; 01071 case RequestMode::Buffer: 01072 applyRequestsBufferMode(frags); 01073 break; 01074 case RequestMode::Ignored: 01075 default: 01076 applyRequestsIgnoredMode(frags); 01077 break; 01078 } 01079 01080 if (!data_thread_running_ || force_stop_) 01081 { 01082 TLOG(TLVL_INFO) << "Data thread has stopped; Clearing data buffer"; 01083 dataBufferDepthBytes_ = 0; 01084 dataBuffer_.clear(); 01085 } 01086 01087 getDataBufferStats(); 01088 } 01089 01090 if (frags.size() > 0) 01091 TLOG(TLVL_APPLYREQUESTS) << "Finished Processing Event " << (*frags.begin())->sequenceID() << " for fragment_id " << fragment_id() << "."; 01092 return true; 01093 } 01094 01095 bool artdaq::CommandableFragmentGenerator::sendEmptyFragment(artdaq::FragmentPtrs& frags, size_t seqId, std::string desc) 01096 { 01097 TLOG(TLVL_WARNING) << desc << " sequence ID " << seqId << ", sending empty fragment"; 01098 for (auto fid : fragment_ids_) 01099 { 01100 auto frag = new Fragment(); 01101 frag->setSequenceID(seqId); 01102 frag->setFragmentID(fid); 01103 frag->setSystemType(Fragment::EmptyFragmentType); 01104 frags.emplace_back(FragmentPtr(frag)); 01105 } 01106 return true; 01107 } 01108 01109 void artdaq::CommandableFragmentGenerator::sendEmptyFragments(artdaq::FragmentPtrs& frags, std::map<Fragment::sequence_id_t, Fragment::timestamp_t>& requests) 01110 { 01111 if (requests.size() > 0) 01112 { 01113 TLOG(TLVL_SENDEMPTYFRAGMENTS) << "Sending Empty Fragments for Sequence IDs from " << ev_counter() << " up to but not including " << requests.begin()->first; 01114 while (requests.begin()->first > ev_counter()) 01115 { 01116 sendEmptyFragment(frags, ev_counter(), "Missed request for"); 01117 ev_counter_inc(1, true); 01118 } 01119 } 01120 } 01121 01122 void artdaq::CommandableFragmentGenerator::checkOutOfOrderWindows(artdaq::Fragment::sequence_id_t seq) 01123 { 01124 windows_sent_ooo_[seq] = std::chrono::steady_clock::now(); 01125 01126 auto it = windows_sent_ooo_.begin(); 01127 while (it != windows_sent_ooo_.end()) 01128 { 01129 if (seq == it->first && it->first == ev_counter()) 01130 { 01131 TLOG(TLVL_CHECKWINDOWS) << "checkOutOfOrderWindows: Sequence ID matches ev_counter, incrementing ev_counter (" << ev_counter() << ")"; 01132 ev_counter_inc(1, true); 01133 it = windows_sent_ooo_.erase(it); 01134 } 01135 else if (it->first <= ev_counter()) 01136 { 01137 TLOG(TLVL_CHECKWINDOWS) << "checkOutOfOrderWindows: Data-taking has caught up to out-of-order window request " << it->first << ", removing from list. ev_counter=" << ev_counter(); 01138 requestReceiver_->RemoveRequest(ev_counter()); 01139 if (it->first == ev_counter()) ev_counter_inc(1, true); 01140 it = windows_sent_ooo_.erase(it); 01141 } 01142 else if (TimeUtils::GetElapsedTimeMicroseconds(it->second) > missing_request_window_timeout_us_) 01143 { 01144 TLOG(TLVL_CHECKWINDOWS) << "checkOutOfOrderWindows: Out-of-order window " << it->first << " has timed out, setting current sequence ID and removing from list"; 01145 while (ev_counter() <= it->first) 01146 { 01147 if (ev_counter() < it->first) TLOG(TLVL_WARNING) << "Missed request for sequence ID " << ev_counter() << "! Will not send any data for this sequence ID!"; 01148 requestReceiver_->RemoveRequest(ev_counter()); 01149 ev_counter_inc(1, true); 01150 } 01151 windows_sent_ooo_.erase(windows_sent_ooo_.begin(), it); 01152 it = windows_sent_ooo_.erase(it); 01153 } 01154 else 01155 { 01156 TLOG(TLVL_CHECKWINDOWS) << "checkOutOfOrderWindows: Out-of-order window " << it->first << " waiting. Current event counter = " << ev_counter(); 01157 ++it; 01158 } 01159 } 01160 } 01161