00001 #define TRACE_NAME "CommandableFragmentGenerator"
00002 #include "tracemf.h"
00003
00004 #include "artdaq/Application/CommandableFragmentGenerator.hh"
00005
00006 #include <boost/exception/all.hpp>
00007 #include <boost/throw_exception.hpp>
00008
00009 #include <limits>
00010 #include <iterator>
00011
00012 #include "canvas/Utilities/Exception.h"
00013 #include "cetlib_except/exception.h"
00014 #include "fhiclcpp/ParameterSet.h"
00015
00016 #include "artdaq-core/Utilities/SimpleLookupPolicy.hh"
00017 #include "artdaq-core/Data/Fragment.hh"
00018 #include "artdaq-core/Data/ContainerFragmentLoader.hh"
00019 #include "artdaq-core/Utilities/ExceptionHandler.hh"
00020 #include "artdaq-core/Utilities/TimeUtils.hh"
00021
00022 #include <fstream>
00023 #include <iomanip>
00024 #include <iterator>
00025 #include <iostream>
00026 #include <iomanip>
00027 #include <algorithm>
00028 #include <sys/poll.h>
00029 #include "artdaq/DAQdata/TCPConnect.hh"
00030
00031 artdaq::CommandableFragmentGenerator::CommandableFragmentGenerator()
00032 : mutex_()
00033 , request_port_(3001)
00034 , request_addr_("227.128.12.26")
00035 , requests_()
00036 , request_stop_requested_(false)
00037 , request_received_(false)
00038 , end_of_run_timeout_ms_(1000)
00039 , windowOffset_(0)
00040 , windowWidth_(0)
00041 , staleTimeout_(Fragment::InvalidTimestamp)
00042 , expectedType_(Fragment::EmptyFragmentType)
00043 , maxFragmentCount_(std::numeric_limits<size_t>::max())
00044 , uniqueWindows_(true)
00045 , missing_request_(true)
00046 , missing_request_time_()
00047 , last_window_send_time_()
00048 , last_window_send_time_set_(false)
00049 , missing_request_window_timeout_us_(1000000)
00050 , window_close_timeout_us_(2000000)
00051 , useDataThread_(false)
00052 , sleep_on_no_data_us_(0)
00053 , data_thread_running_(false)
00054 , dataBufferDepthFragments_(0)
00055 , dataBufferDepthBytes_(0)
00056 , maxDataBufferDepthFragments_(1000)
00057 , maxDataBufferDepthBytes_(1000)
00058 , useMonitoringThread_(false)
00059 , monitoringInterval_(0)
00060 , lastMonitoringCall_()
00061 , isHardwareOK_(true)
00062 , dataBuffer_()
00063 , newDataBuffer_()
00064 , run_number_(-1)
00065 , subrun_number_(-1)
00066 , timeout_(std::numeric_limits<uint64_t>::max())
00067 , timestamp_(std::numeric_limits<uint64_t>::max())
00068 , should_stop_(false)
00069 , exception_(false)
00070 , force_stop_(false)
00071 , latest_exception_report_("none")
00072 , ev_counter_(1)
00073 , board_id_(-1)
00074 , instance_name_for_metrics_("FragmentGenerator")
00075 , sleep_on_stop_us_(0)
00076 {}
00077
00078 artdaq::CommandableFragmentGenerator::CommandableFragmentGenerator(const fhicl::ParameterSet& ps)
00079 : mutex_()
00080 , request_port_(ps.get<int>("request_port", 3001))
00081 , request_addr_(ps.get<std::string>("request_address", "227.128.12.26"))
00082 , requests_()
00083 , request_stop_requested_(false)
00084 , request_received_(false)
00085 , end_of_run_timeout_ms_(ps.get<size_t>("end_of_run_quiet_timeout_ms", 1000))
00086 , windowOffset_(ps.get<Fragment::timestamp_t>("request_window_offset", 0))
00087 , windowWidth_(ps.get<Fragment::timestamp_t>("request_window_width", 0))
00088 , staleTimeout_(ps.get<Fragment::timestamp_t>("stale_request_timeout", 0xFFFFFFFF))
00089 , expectedType_(ps.get<Fragment::type_t>("expected_fragment_type", Fragment::type_t(Fragment::EmptyFragmentType)))
00090 , uniqueWindows_(ps.get<bool>("request_windows_are_unique", true))
00091 , missing_request_(false)
00092 , missing_request_time_(decltype(missing_request_time_)::max())
00093 , last_window_send_time_(decltype(last_window_send_time_)::max())
00094 , last_window_send_time_set_(false)
00095 , missing_request_window_timeout_us_(ps.get<size_t>("missing_request_window_timeout_us", 1000000))
00096 , window_close_timeout_us_(ps.get<size_t>("window_close_timeout_us", 2000000))
00097 , useDataThread_(ps.get<bool>("separate_data_thread", false))
00098 , sleep_on_no_data_us_(ps.get<size_t>("sleep_on_no_data_us", 0))
00099 , data_thread_running_(false)
00100 , dataBufferDepthFragments_(0)
00101 , dataBufferDepthBytes_(0)
00102 , maxDataBufferDepthFragments_(ps.get<int>("data_buffer_depth_fragments", 1000))
00103 , maxDataBufferDepthBytes_(ps.get<size_t>("data_buffer_depth_mb", 1000) * 1024 * 1024)
00104 , useMonitoringThread_(ps.get<bool>("separate_monitoring_thread", false))
00105 , monitoringInterval_(ps.get<int64_t>("hardware_poll_interval_us", 0))
00106 , lastMonitoringCall_()
00107 , isHardwareOK_(true)
00108 , dataBuffer_()
00109 , newDataBuffer_()
00110 , run_number_(-1)
00111 , subrun_number_(-1)
00112 , timeout_(std::numeric_limits<uint64_t>::max())
00113 , timestamp_(std::numeric_limits<uint64_t>::max())
00114 , should_stop_(false)
00115 , exception_(false)
00116 , force_stop_(false)
00117 , latest_exception_report_("none")
00118 , ev_counter_(1)
00119 , board_id_(-1)
00120 , sleep_on_stop_us_(0)
00121 {
00122 board_id_ = ps.get<int>("board_id");
00123 instance_name_for_metrics_ = "BoardReader." + boost::lexical_cast<std::string>(board_id_);
00124
00125 fragment_ids_ = ps.get<std::vector<artdaq::Fragment::fragment_id_t>>("fragment_ids", std::vector<artdaq::Fragment::fragment_id_t>());
00126
00127 TLOG_TRACE("CommandableFragmentGenerator") << "artdaq::CommandableFragmentGenerator::CommandableFragmentGenerator(ps)" << TLOG_ENDL;
00128 int fragment_id = ps.get<int>("fragment_id", -99);
00129
00130 if (fragment_id != -99)
00131 {
00132 if (fragment_ids_.size() != 0)
00133 {
00134 latest_exception_report_ = "Error in CommandableFragmentGenerator: can't both define \"fragment_id\" and \"fragment_ids\" in FHiCL document";
00135 throw cet::exception(latest_exception_report_);
00136 }
00137 else
00138 {
00139 fragment_ids_.emplace_back(fragment_id);
00140 }
00141 }
00142
00143 sleep_on_stop_us_ = ps.get<int>("sleep_on_stop_us", 0);
00144
00145 dataBuffer_.emplace_back(FragmentPtr(new Fragment()));
00146 (*dataBuffer_.begin())->setSystemType(Fragment::EmptyFragmentType);
00147
00148 std::string modeString = ps.get<std::string>("request_mode", "ignored");
00149 if (modeString == "single" || modeString == "Single")
00150 {
00151 mode_ = RequestMode::Single;
00152 }
00153 else if (modeString.find("buffer") != std::string::npos || modeString.find("Buffer") != std::string::npos)
00154 {
00155 mode_ = RequestMode::Buffer;
00156 }
00157 else if (modeString == "window" || modeString == "Window")
00158 {
00159 mode_ = RequestMode::Window;
00160 }
00161 else if (modeString.find("ignore") != std::string::npos || modeString.find("Ignore") != std::string::npos)
00162 {
00163 mode_ = RequestMode::Ignored;
00164 }
00165 TLOG_DEBUG("CommandableFragmentGenerator") << "Request mode is " << printMode_() << TLOG_ENDL;
00166
00167 if (mode_ != RequestMode::Ignored)
00168 {
00169 if (!useDataThread_)
00170 {
00171 latest_exception_report_ = "Error in CommandableFragmentGenerator: use_data_thread must be true when request_mode is not \"Ignored\"!";
00172 throw cet::exception(latest_exception_report_);
00173 }
00174 setupRequestListener();
00175 }
00176 }
00177
00178 void artdaq::CommandableFragmentGenerator::setupRequestListener()
00179 {
00180 request_socket_ = socket(AF_INET, SOCK_DGRAM, IPPROTO_UDP);
00181 if (request_socket_ < 0)
00182 {
00183 throw art::Exception(art::errors::Configuration) << "CommandableFragmentGenerator: Error creating socket for receiving data requests! err=" << strerror(errno) << std::endl;
00184 exit(1);
00185 }
00186
00187 struct sockaddr_in si_me_request;
00188
00189 int yes = 1;
00190 if (setsockopt(request_socket_, SOL_SOCKET, SO_REUSEADDR, &yes, sizeof(yes)) < 0)
00191 {
00192 throw art::Exception(art::errors::Configuration) <<
00193 "RequestedFragmentGenrator: Unable to enable port reuse on request socket, err=" << strerror(errno) << std::endl;
00194 exit(1);
00195 }
00196 memset(&si_me_request, 0, sizeof(si_me_request));
00197 si_me_request.sin_family = AF_INET;
00198 si_me_request.sin_port = htons(request_port_);
00199 si_me_request.sin_addr.s_addr = htonl(INADDR_ANY);
00200 if (bind(request_socket_, (struct sockaddr *)&si_me_request, sizeof(si_me_request)) == -1)
00201 {
00202 throw art::Exception(art::errors::Configuration) <<
00203 "CommandableFragmentGenerator: Cannot bind request socket to port " << request_port_ << ", err=" << strerror(errno) << std::endl;
00204 exit(1);
00205 }
00206
00207 if (request_addr_ != "localhost")
00208 {
00209 struct ip_mreq mreq;
00210 int sts = ResolveHost(request_addr_.c_str(), mreq.imr_multiaddr);
00211 if (sts == -1)
00212 {
00213 throw art::Exception(art::errors::Configuration) << "Unable to resolve multicast request address, err=" << strerror(errno) << std::endl;
00214 exit(1);
00215 }
00216 mreq.imr_interface.s_addr = htonl(INADDR_ANY);
00217 if (setsockopt(request_socket_, IPPROTO_IP, IP_ADD_MEMBERSHIP, &mreq, sizeof(mreq)) < 0)
00218 {
00219 throw art::Exception(art::errors::Configuration) <<
00220 "CommandableFragmentGenerator: Unable to join multicast group, err=" << strerror(errno) << std::endl;
00221 exit(1);
00222 }
00223 }
00224 }
00225
00226 artdaq::CommandableFragmentGenerator::~CommandableFragmentGenerator()
00227 {
00228 joinThreads();
00229 }
00230
00231 void artdaq::CommandableFragmentGenerator::joinThreads()
00232 {
00233 should_stop_ = true;
00234 force_stop_ = true;
00235 TLOG_DEBUG("CommandableFragmentGenerator") << "Joining dataThread" << TLOG_ENDL;
00236 if (dataThread_.joinable()) dataThread_.join();
00237 TLOG_DEBUG("CommandableFragmentGenerator") << "Joining monitoringThread" << TLOG_ENDL;
00238 if (monitoringThread_.joinable()) monitoringThread_.join();
00239 TLOG_DEBUG("CommandableFragmentGenerator") << "Joining requestThread" << TLOG_ENDL;
00240 if (requestThread_.joinable()) requestThread_.join();
00241 if (request_socket_ != -1) close(request_socket_);
00242 }
00243
00244 bool artdaq::CommandableFragmentGenerator::getNext(FragmentPtrs& output)
00245 {
00246 bool result = true;
00247
00248 if (check_stop()) usleep(sleep_on_stop_us_);
00249 if (exception() || force_stop_) return false;
00250
00251 if (!useMonitoringThread_ && monitoringInterval_ > 0)
00252 {
00253 TLOG_ARB(10, "CommandableFragmentGenerator") << "getNext: Checking whether to collect Monitoring Data" << TLOG_ENDL;
00254 auto now = std::chrono::steady_clock::now();
00255
00256 if (TimeUtils::GetElapsedTimeMicroseconds(lastMonitoringCall_, now) >= static_cast<size_t>(monitoringInterval_))
00257 {
00258 TLOG_ARB(10, "CommandableFragmentGenerator") << "getNext: Collecting Monitoring Data" << TLOG_ENDL;
00259 isHardwareOK_ = checkHWStatus_();
00260 TLOG_ARB(10, "CommandableFragmentGenerator") << "getNext: isHardwareOK_ is now " << std::boolalpha << isHardwareOK_ << TLOG_ENDL;
00261 lastMonitoringCall_ = now;
00262 }
00263 }
00264
00265 try
00266 {
00267 std::lock_guard<std::mutex> lk(mutex_);
00268 if (useDataThread_)
00269 {
00270 TLOG_TRACE("CommandableFragmentGenerator") << "getNext: Calling applyRequests" << TLOG_ENDL;
00271 result = applyRequests(output);
00272 TLOG_TRACE("CommandableFragmentGenerator") << "getNext: Done with applyRequests" << TLOG_ENDL;
00273
00274 if (exception())
00275 {
00276 throw cet::exception("CommandableFragmentGenerator") << "Exception found in BoardReader with board ID " << board_id() << "; BoardReader will now return error status when queried";
00277 }
00278 }
00279 else
00280 {
00281 if (!isHardwareOK_)
00282 {
00283 TLOG_ERROR("CommandableFragmentGenerator") << "Stopping CFG because the hardware reports bad status!" << TLOG_ENDL;
00284 return false;
00285 }
00286 TLOG_TRACE("CommandableFragmentGenerator") << "getNext: Calling getNext_ " << std::to_string(ev_counter()) << TLOG_ENDL;
00287 try
00288 {
00289 result = getNext_(output);
00290 }
00291 catch (...)
00292 {
00293 throw;
00294 }
00295 TLOG_TRACE("CommandableFragmentGenerator") << "getNext: Done with getNext_ " << std::to_string(ev_counter()) << TLOG_ENDL;
00296 }
00297 }
00298 catch (const cet::exception& e)
00299 {
00300 latest_exception_report_ = "cet::exception caught in getNext(): ";
00301 latest_exception_report_.append(e.what());
00302 TLOG_ERROR("CommandableFragmentGenerator") << "getNext: cet::exception caught: " << e << TLOG_ENDL;
00303 set_exception(true);
00304 return false;
00305 }
00306 catch (const boost::exception& e)
00307 {
00308 latest_exception_report_ = "boost::exception caught in getNext(): ";
00309 latest_exception_report_.append(boost::diagnostic_information(e));
00310 TLOG_ERROR("CommandableFragmentGenerator") << "getNext: boost::exception caught: " << boost::diagnostic_information(e) << TLOG_ENDL;
00311 set_exception(true);
00312 return false;
00313 }
00314 catch (const std::exception& e)
00315 {
00316 latest_exception_report_ = "std::exception caught in getNext(): ";
00317 latest_exception_report_.append(e.what());
00318 TLOG_ERROR("CommandableFragmentGenerator") << "getNext: std::exception caught: " << e.what() << TLOG_ENDL;
00319 set_exception(true);
00320 return false;
00321 }
00322 catch (...)
00323 {
00324 latest_exception_report_ = "Unknown exception caught in getNext().";
00325 TLOG_ERROR("CommandableFragmentGenerator") << "getNext: unknown exception caught" << TLOG_ENDL;
00326 set_exception(true);
00327 return false;
00328 }
00329
00330 if (!result)
00331 {
00332 TLOG_DEBUG("getNext") << "stopped " << TLOG_ENDL;
00333 }
00334
00335 return result;
00336 }
00337
00338 bool artdaq::CommandableFragmentGenerator::check_stop()
00339 {
00340 TLOG_ARB(14, "CommandableFragmentGeneraotr") << "CFG::check_stop: should_stop=" << should_stop() << ", useDataThread_=" << useDataThread_ << ", requests_.size()=" << std::to_string(requests_.size()) << ", exception status =" << int(exception()) << TLOG_ENDL;
00341
00342 if (!should_stop()) return false;
00343 if (!useDataThread_ || mode_ == RequestMode::Ignored) return true;
00344 if (force_stop_) return true;
00345
00346 if (!request_received_)
00347 {
00348 TLOG_ERROR("CommandableFragmentGenerator") << "Stop request received by request-based CommandableFragmentGenerator, but no requests have been received." << std::endl
00349 << "Check that UDP port " << request_port_ << " is open in the firewall config." << TLOG_ENDL;
00350 return true;
00351 }
00352
00353 if (!request_stop_requested_) return false;
00354
00355 return TimeUtils::GetElapsedTimeMilliseconds(request_stop_timeout_) > end_of_run_timeout_ms_;
00356 }
00357
00358 int artdaq::CommandableFragmentGenerator::fragment_id() const
00359 {
00360 if (fragment_ids_.size() != 1)
00361 {
00362 throw cet::exception("Error in CommandableFragmentGenerator: can't call fragment_id() unless member fragment_ids_ vector is length 1");
00363 }
00364 else
00365 {
00366 return fragment_ids_[0];
00367 }
00368 }
00369
00370 size_t artdaq::CommandableFragmentGenerator::ev_counter_inc(size_t step, bool force)
00371 {
00372 if (force || mode_ == RequestMode::Ignored)
00373 {
00374 return ev_counter_.fetch_add(step);
00375 }
00376 return ev_counter_.load();
00377 }
00378
00379 void artdaq::CommandableFragmentGenerator::StartCmd(int run, uint64_t timeout, uint64_t timestamp)
00380 {
00381 if (run < 0) throw cet::exception("CommandableFragmentGenerator") << "negative run number";
00382
00383 timeout_ = timeout;
00384 timestamp_ = timestamp;
00385 ev_counter_.store(1);
00386 should_stop_.store(false);
00387 exception_.store(false);
00388 run_number_ = run;
00389 subrun_number_ = 1;
00390 latest_exception_report_ = "none";
00391 dataBuffer_.clear();
00392 requests_.clear();
00393 last_window_send_time_set_ = false;
00394
00395 start();
00396
00397 std::unique_lock<std::mutex> lk(mutex_);
00398 if (useDataThread_) startDataThread();
00399 if (useMonitoringThread_) startMonitoringThread();
00400 if (mode_ != RequestMode::Ignored) startRequestReceiverThread();
00401 }
00402
00403 void artdaq::CommandableFragmentGenerator::StopCmd(uint64_t timeout, uint64_t timestamp)
00404 {
00405 TLOG_DEBUG("CommandableFragmentGenerator") << "Stop Command received." << TLOG_ENDL;
00406
00407 timeout_ = timeout;
00408 timestamp_ = timestamp;
00409
00410 stopNoMutex();
00411 should_stop_.store(true);
00412 std::unique_lock<std::mutex> lk(mutex_);
00413 stop();
00414 }
00415
00416 void artdaq::CommandableFragmentGenerator::PauseCmd(uint64_t timeout, uint64_t timestamp)
00417 {
00418 timeout_ = timeout;
00419 timestamp_ = timestamp;
00420
00421 pauseNoMutex();
00422 should_stop_.store(true);
00423 std::unique_lock<std::mutex> lk(mutex_);
00424
00425 pause();
00426 }
00427
00428 void artdaq::CommandableFragmentGenerator::ResumeCmd(uint64_t timeout, uint64_t timestamp)
00429 {
00430 timeout_ = timeout;
00431 timestamp_ = timestamp;
00432
00433 subrun_number_ += 1;
00434 should_stop_ = false;
00435
00436 dataBuffer_.clear();
00437 requests_.clear();
00438
00439
00440 resume();
00441
00442 std::unique_lock<std::mutex> lk(mutex_);
00443 if (useDataThread_) startDataThread();
00444 if (useMonitoringThread_) startMonitoringThread();
00445 if (mode_ != RequestMode::Ignored) startRequestReceiverThread();
00446 }
00447
00448 std::string artdaq::CommandableFragmentGenerator::ReportCmd(std::string const& which)
00449 {
00450 std::lock_guard<std::mutex> lk(mutex_);
00451
00452
00453
00454
00455
00456
00457 std::string childReport = reportSpecific(which);
00458 if (childReport.length() > 0) { return childReport; }
00459
00460
00461 if (which == "latest_exception")
00462 {
00463 return latest_exception_report_;
00464 }
00465
00466
00467 childReport = report();
00468 if (childReport.length() > 0) { return childReport; }
00469
00470
00471 std::string tmpString = "The \"" + which + "\" command is not ";
00472 tmpString.append("currently supported by the ");
00473 tmpString.append(metricsReportingInstanceName());
00474 tmpString.append(" fragment generator.");
00475 return tmpString;
00476 }
00477
00478
00479 void artdaq::CommandableFragmentGenerator::pauseNoMutex()
00480 {
00481 #pragma message "Using default implementation of CommandableFragmentGenerator::pauseNoMutex()"
00482 }
00483
00484 void artdaq::CommandableFragmentGenerator::pause()
00485 {
00486 #pragma message "Using default implementation of CommandableFragmentGenerator::pause()"
00487 }
00488
00489 void artdaq::CommandableFragmentGenerator::resume()
00490 {
00491 #pragma message "Using default implementation of CommandableFragmentGenerator::resume()"
00492 }
00493
00494 std::string artdaq::CommandableFragmentGenerator::report()
00495 {
00496 #pragma message "Using default implementation of CommandableFragmentGenerator::report()"
00497 return "";
00498 }
00499
00500 std::string artdaq::CommandableFragmentGenerator::reportSpecific(std::string const&)
00501 {
00502 #pragma message "Using default implementation of CommandableFragmentGenerator::reportSpecific(std::string)"
00503 return "";
00504 }
00505
00506 bool artdaq::CommandableFragmentGenerator::checkHWStatus_()
00507 {
00508 #pragma message "Using default implementation of CommandableFragmentGenerator::checkHWStatus_()"
00509 return true;
00510 }
00511
00512 void artdaq::CommandableFragmentGenerator::startDataThread()
00513 {
00514 if (dataThread_.joinable()) dataThread_.join();
00515 TLOG_INFO("CommandableFragmentGenerator") << "Starting Data Receiver Thread" << TLOG_ENDL;
00516 dataThread_ = boost::thread(&CommandableFragmentGenerator::getDataLoop, this);
00517 }
00518
00519 void artdaq::CommandableFragmentGenerator::startMonitoringThread()
00520 {
00521 if (monitoringThread_.joinable()) monitoringThread_.join();
00522 TLOG_INFO("CommandableFragmentGenerator") << "Starting Hardware Monitoring Thread" << TLOG_ENDL;
00523 monitoringThread_ = boost::thread(&CommandableFragmentGenerator::getMonitoringDataLoop, this);
00524 }
00525
00526 void artdaq::CommandableFragmentGenerator::startRequestReceiverThread()
00527 {
00528 if (requestThread_.joinable()) requestThread_.join();
00529 TLOG_INFO("CommandableFragmentGenerator") << "Starting Request Reception Thread" << TLOG_ENDL;
00530 requestThread_ = boost::thread(&CommandableFragmentGenerator::receiveRequestsLoop, this);
00531 }
00532
00533 std::string artdaq::CommandableFragmentGenerator::printMode_()
00534 {
00535 switch (mode_)
00536 {
00537 case RequestMode::Single:
00538 return "Single";
00539 case RequestMode::Buffer:
00540 return "Buffer";
00541 case RequestMode::Window:
00542 return "Window";
00543 case RequestMode::Ignored:
00544 return "Ignored";
00545 }
00546
00547 return "ERROR";
00548 }
00549
00550 void artdaq::CommandableFragmentGenerator::getDataLoop()
00551 {
00552 data_thread_running_ = true;
00553 while (!force_stop_)
00554 {
00555 if (!isHardwareOK_)
00556 {
00557 TLOG_DEBUG("CommandableFragmentGenerator") << "getDataLoop: isHardwareOK is " << isHardwareOK_ << ", aborting data thread" << TLOG_ENDL;
00558 data_thread_running_ = false;
00559 return;
00560 }
00561
00562 TLOG_ARB(13, "CommandableFragmentGenerator") << "getDataLoop: calling getNext_" << TLOG_ENDL;
00563
00564 bool data = false;
00565 auto startdata = std::chrono::steady_clock::now();
00566
00567 try
00568 {
00569 data = getNext_(newDataBuffer_);
00570 }
00571 catch (...)
00572 {
00573 ExceptionHandler(ExceptionHandlerRethrow::no,
00574 "Exception thrown by fragment generator in CommandableFragmentGenerator::getDataLoop; setting exception state to \"true\"");
00575 set_exception(true);
00576
00577 data_thread_running_ = false;
00578 return;
00579 }
00580
00581 if (metricMan)
00582 {
00583 metricMan->sendMetric("Avg Data Acquisition Time", TimeUtils::GetElapsedTime(startdata), "s", 3, artdaq::MetricMode::Average);
00584 }
00585
00586 if (newDataBuffer_.size() == 0 && sleep_on_no_data_us_ > 0)
00587 {
00588 usleep(sleep_on_no_data_us_);
00589 }
00590
00591 TLOG_ARB(15, "CommandableFragmentGenerator") << "Waiting for data buffer ready" << TLOG_ENDL;
00592 if (!waitForDataBufferReady()) return;
00593 TLOG_ARB(15, "CommandableFragmentGenerator") << "Done waiting for data buffer ready" << TLOG_ENDL;
00594
00595 TLOG_ARB(13, "CommandableFragmentGenerator") << "getDataLoop: processing data" << TLOG_ENDL;
00596 if (data && !force_stop_)
00597 {
00598 std::unique_lock<std::mutex> lock(dataBufferMutex_);
00599 switch (mode_)
00600 {
00601 case RequestMode::Single:
00602
00603 while (newDataBuffer_.size() >= fragment_ids_.size())
00604 {
00605 dataBuffer_.clear();
00606 auto it = newDataBuffer_.begin();
00607 std::advance(it, fragment_ids_.size());
00608 dataBuffer_.splice(dataBuffer_.end(), newDataBuffer_, newDataBuffer_.begin(), it);
00609 }
00610 break;
00611 case RequestMode::Buffer:
00612 case RequestMode::Ignored:
00613 case RequestMode::Window:
00614 default:
00615
00616 dataBuffer_.splice(dataBuffer_.end(), newDataBuffer_);
00617 break;
00618 }
00619 getDataBufferStats();
00620 }
00621
00622 {
00623 std::unique_lock<std::mutex> lock(dataBufferMutex_);
00624 if (dataBuffer_.size() > 0)
00625 {
00626 dataCondition_.notify_all();
00627 }
00628 }
00629 if (!data || force_stop_)
00630 {
00631 TLOG_INFO("CommandableFragmentGenerator") << "Data flow has stopped. Ending data collection thread" << TLOG_ENDL;
00632 data_thread_running_ = false;
00633 return;
00634 }
00635 }
00636 }
00637
00638 bool artdaq::CommandableFragmentGenerator::waitForDataBufferReady()
00639 {
00640 auto startwait = std::chrono::steady_clock::now();
00641 auto first = true;
00642 auto lastwaittime = 0ULL;
00643 while (dataBufferIsTooLarge())
00644 {
00645 if (should_stop())
00646 {
00647 TLOG_DEBUG("CommandableFragmentGenerator") << "Run ended while waiting for buffer to shrink!" << TLOG_ENDL;
00648 std::unique_lock<std::mutex> lock(dataBufferMutex_);
00649 getDataBufferStats();
00650 dataCondition_.notify_all();
00651 data_thread_running_ = false;
00652 return false;
00653 }
00654 auto waittime = TimeUtils::GetElapsedTimeMilliseconds(startwait);
00655
00656 if (first || (waittime != lastwaittime && waittime % 1000 == 0))
00657 {
00658 TLOG_WARNING("CommandableFragmentGenerator") << "Bad Omen: Data Buffer has exceeded its size limits. Check the connection between the BoardReader and the EventBuilders! (seq_id=" << ev_counter() << ")" << TLOG_ENDL;
00659 first = false;
00660 }
00661 if (waittime % 5 && waittime != lastwaittime)
00662 {
00663 TLOG_ARB(13, "CommandableFragmentGenerator") << "getDataLoop: Data Retreival paused for " << std::to_string(waittime) << " ms waiting for data buffer to drain" << TLOG_ENDL;
00664 }
00665 lastwaittime = waittime;
00666 usleep(1000);
00667 }
00668 return true;
00669 }
00670
00671 bool artdaq::CommandableFragmentGenerator::dataBufferIsTooLarge()
00672 {
00673 return (maxDataBufferDepthFragments_ > 0 && dataBufferDepthFragments_ > maxDataBufferDepthFragments_) || (maxDataBufferDepthBytes_ > 0 && dataBufferDepthBytes_ > maxDataBufferDepthBytes_);
00674 }
00675
00676 void artdaq::CommandableFragmentGenerator::getDataBufferStats()
00677 {
00679 dataBufferDepthFragments_ = dataBuffer_.size();
00680 size_t acc = 0;
00681 TLOG_ARB(15, "CommandableFragmentGenerator") << "getDataBufferStats: Calculating buffer size" << TLOG_ENDL;
00682 for (auto i = dataBuffer_.begin(); i != dataBuffer_.end(); ++i)
00683 {
00684 if (i->get() != nullptr)
00685 {
00686 acc += (*i)->sizeBytes();
00687 }
00688 }
00689 dataBufferDepthBytes_ = acc;
00690
00691 if (metricMan)
00692 {
00693 TLOG_ARB(15, "CommandableFragmentGenerator") << "getDataBufferStats: Sending Metrics" << TLOG_ENDL;
00694 metricMan->sendMetric("Buffer Depth Fragments", dataBufferDepthFragments_.load(), "fragments", 1, MetricMode::LastPoint);
00695 metricMan->sendMetric("Buffer Depth Bytes", dataBufferDepthBytes_.load(), "bytes", 1, MetricMode::LastPoint);
00696 }
00697 TLOG_ARB(15, "CommandableFragmentGenerator") << "getDataBufferStats: frags=" << dataBufferDepthFragments_.load() << "/" << maxDataBufferDepthFragments_
00698 << ", sz=" << std::to_string(dataBufferDepthBytes_.load()) << "/" << std::to_string(maxDataBufferDepthBytes_) << TLOG_ENDL;
00699 }
00700
00701 void artdaq::CommandableFragmentGenerator::checkDataBuffer()
00702 {
00703 std::unique_lock<std::mutex> lock(dataBufferMutex_);
00704 dataCondition_.wait_for(lock, std::chrono::milliseconds(10));
00705 if (dataBufferDepthFragments_ > 0)
00706 {
00707 if ((mode_ == RequestMode::Buffer || mode_ == RequestMode::Window))
00708 {
00709
00710 while (dataBufferIsTooLarge())
00711 {
00712 dataBuffer_.erase(dataBuffer_.begin());
00713 getDataBufferStats();
00714 }
00715 if (dataBuffer_.size() > 0)
00716 {
00717 TLOG_ARB(17, "CommandableFragmentGenerator") << "Determining if Fragments can be dropped from data buffer" << TLOG_ENDL;
00718 Fragment::timestamp_t last = dataBuffer_.back()->timestamp();
00719 Fragment::timestamp_t min = last > staleTimeout_ ? last - staleTimeout_ : 0;
00720 for (auto it = dataBuffer_.begin(); it != dataBuffer_.end();)
00721 {
00722 if ((*it)->timestamp() < min)
00723 {
00724 it = dataBuffer_.erase(it);
00725 }
00726 else
00727 {
00728 ++it;
00729 }
00730 }
00731 getDataBufferStats();
00732 }
00733 }
00734 else if (mode_ == RequestMode::Single && dataBuffer_.size() > fragment_ids_.size())
00735 {
00736
00737 while (dataBuffer_.size() > fragment_ids_.size())
00738 {
00739 dataBuffer_.erase(dataBuffer_.begin());
00740 }
00741 }
00742 }
00743 }
00744
00745 void artdaq::CommandableFragmentGenerator::getMonitoringDataLoop()
00746 {
00747 while (!force_stop_)
00748 {
00749 if (should_stop() || monitoringInterval_ <= 0)
00750 {
00751 TLOG_DEBUG("CommandableFragmentGenerator") << "getMonitoringDataLoop: should_stop() is " << std::boolalpha << should_stop()
00752 << " and monitoringInterval is " << monitoringInterval_ << ", returning" << TLOG_ENDL;
00753 return;
00754 }
00755 TLOG_ARB(12, "CommandableFragmentGenerator") << "getMonitoringDataLoop: Determining whether to call checkHWStatus_" << TLOG_ENDL;
00756
00757 auto now = std::chrono::steady_clock::now();
00758 if (TimeUtils::GetElapsedTimeMicroseconds(lastMonitoringCall_, now) >= static_cast<size_t>(monitoringInterval_))
00759 {
00760 isHardwareOK_ = checkHWStatus_();
00761 TLOG_ARB(12, "CommandableFragmentGenerator") << "getMonitoringDataLoop: isHardwareOK_ is now " << std::boolalpha << isHardwareOK_ << TLOG_ENDL;
00762 lastMonitoringCall_ = now;
00763 }
00764 usleep(monitoringInterval_ / 10);
00765 }
00766 }
00767
00768 void artdaq::CommandableFragmentGenerator::receiveRequestsLoop()
00769 {
00770 while (!force_stop_)
00771 {
00772 if (check_stop() || !isHardwareOK_ || exception())
00773 {
00774 TLOG_DEBUG("CommandableFragmentGenerator") << "receiveRequestsLoop: check_stop is " << std::boolalpha << check_stop()
00775 << ", isHardwareOK_ is " << isHardwareOK_ << ", and exception state is " << exception() << ", aborting request reception thread." << TLOG_ENDL;
00776 return;
00777 }
00778
00779
00780 if (mode_ == RequestMode::Ignored) return;
00781 TLOG_ARB(16, "CommandableFragmentGenerator") << "receiveRequestsLoop: Polling Request socket for new requests" << TLOG_ENDL;
00782
00783 int ms_to_wait = 1000;
00784 struct pollfd ufds[1];
00785 ufds[0].fd = request_socket_;
00786 ufds[0].events = POLLIN | POLLPRI;
00787 int rv = poll(ufds, 1, ms_to_wait);
00788
00789
00790 if (rv <= 0 || (ufds[0].revents != POLLIN && ufds[0].revents != POLLPRI)) continue;
00791
00792 TLOG_ARB(11, "CommandableFragmentGenerator") << "Recieved packet on Request channel" << TLOG_ENDL;
00793 detail::RequestHeader hdr_buffer;
00794 recv(request_socket_, &hdr_buffer, sizeof(hdr_buffer), 0);
00795 TLOG_ARB(11, "CommandableFragmentGenerator") << "Request header word: 0x" << std::hex << hdr_buffer.header << TLOG_ENDL;
00796 if (!hdr_buffer.isValid()) continue;
00797
00798 request_received_ = true;
00799 if (hdr_buffer.mode == detail::RequestMessageMode::EndOfRun)
00800 {
00801 TLOG_INFO("CommandableFragmentGenerator") << "Received Request Message with the EndOfRun marker. (Re)Starting 1-second timeout for receiving all outstanding requests..." << TLOG_ENDL;
00802 request_stop_timeout_ = std::chrono::steady_clock::now();
00803 request_stop_requested_ = true;
00804 }
00805
00806 std::vector<detail::RequestPacket> pkt_buffer(hdr_buffer.packet_count);
00807 recv(request_socket_, &pkt_buffer[0], sizeof(detail::RequestPacket) * hdr_buffer.packet_count, 0);
00808 bool anyNew = false;
00809 for (auto& buffer : pkt_buffer)
00810 {
00811 if (!buffer.isValid()) continue;
00812 if (requests_.count(buffer.sequence_id) && requests_[buffer.sequence_id] != buffer.timestamp)
00813 {
00814 TLOG_ERROR("CommandableFragmentGenerator") << "Received conflicting request for SeqID "
00815 << std::to_string(buffer.sequence_id) << "!"
00816 << " Old ts=" << std::to_string(requests_[buffer.sequence_id])
00817 << ", new ts=" << std::to_string(buffer.timestamp) << ". Keeping OLD!" << TLOG_ENDL;
00818 }
00819 else if (!requests_.count(buffer.sequence_id))
00820 {
00821 int delta = buffer.sequence_id - ev_counter();
00822 TLOG_ARB(11, "CommandableFragmentGenerator") << "Recieved request for sequence ID " << std::to_string(buffer.sequence_id)
00823 << " and timestamp " << std::to_string(buffer.timestamp) << " (delta: " << delta << ")" << TLOG_ENDL;
00824 if (delta < 0)
00825 {
00826 TLOG_ARB(11, "CommandableFragmentGenerator") << "Already serviced this request! Ignoring..." << TLOG_ENDL;
00827 }
00828 else
00829 {
00830 std::unique_lock<std::mutex> tlk(request_mutex_);
00831 requests_[buffer.sequence_id] = buffer.timestamp;
00832 anyNew = true;
00833 }
00834 }
00835 }
00836 if (anyNew)
00837 {
00838 std::unique_lock<std::mutex> lock(request_mutex_);
00839 requestCondition_.notify_all();
00840 }
00841 }
00842 }
00843
00844 void artdaq::CommandableFragmentGenerator::applyRequestsIgnoredMode(artdaq::FragmentPtrs& frags)
00845 {
00846
00847 TLOG_ARB(9, "CommandableFragmentGenerator") << "Mode is Ignored; Copying data to output" << TLOG_ENDL;
00848 std::move(dataBuffer_.begin(), dataBuffer_.end(), std::inserter(frags, frags.end()));
00849 dataBuffer_.clear();
00850 }
00851
00852 void artdaq::CommandableFragmentGenerator::applyRequestsSingleMode(artdaq::FragmentPtrs& frags)
00853 {
00854
00855 sendEmptyFragments(frags);
00856
00857
00858 if (requests_.size() == 0 || !requests_.count(ev_counter())) return;
00859
00860 if (dataBuffer_.size() > 0)
00861 {
00862 TLOG_ARB(9, "CommandableFragmentGenerator") << "Mode is Single; Sending copy of last event" << TLOG_ENDL;
00863 for (auto& fragptr : dataBuffer_)
00864 {
00865
00866 auto frag = fragptr.get();
00867 auto newfrag = std::unique_ptr<artdaq::Fragment>(new Fragment(ev_counter(), frag->fragmentID()));
00868 newfrag->resize(frag->size() - detail::RawFragmentHeader::num_words());
00869 memcpy(newfrag->headerAddress(), frag->headerAddress(), frag->sizeBytes());
00870 newfrag->setTimestamp(requests_[ev_counter()]);
00871 newfrag->setSequenceID(ev_counter());
00872 frags.push_back(std::move(newfrag));
00873 }
00874 }
00875 else
00876 {
00877 sendEmptyFragment(frags, ev_counter(), "No data for");
00878 }
00879 requests_.clear();
00880 ev_counter_inc(1, true);
00881 }
00882
00883 void artdaq::CommandableFragmentGenerator::applyRequestsBufferMode(artdaq::FragmentPtrs& frags)
00884 {
00885 sendEmptyFragments(frags);
00886
00887
00888 if (requests_.size() == 0 || !requests_.count(ev_counter())) return;
00889
00890 TLOG_DEBUG("CommandableFragmentGenerator") << "Creating ContainerFragment for Buffered Fragments" << TLOG_ENDL;
00891 frags.emplace_back(new artdaq::Fragment(ev_counter(), fragment_id()));
00892 frags.back()->setTimestamp(requests_[ev_counter()]);
00893 ContainerFragmentLoader cfl(*frags.back());
00894 cfl.set_missing_data(false);
00895
00896
00897
00898 for (auto it = dataBuffer_.begin(); it != dataBuffer_.end();)
00899 {
00900 TLOG_ARB(9, "CommandableFragmentGenerator") << "ApplyRequests: Adding Fragment with timestamp " << std::to_string((*it)->timestamp()) << " to Container" << TLOG_ENDL;
00901 cfl.addFragment(*it);
00902 it = dataBuffer_.erase(it);
00903 }
00904 requests_.clear();
00905 ev_counter_inc(1, true);
00906 }
00907
00908 void artdaq::CommandableFragmentGenerator::applyRequestsWindowMode(artdaq::FragmentPtrs& frags)
00909 {
00910 if (!last_window_send_time_set_)
00911 {
00912 last_window_send_time_ = std::chrono::steady_clock::now();
00913 last_window_send_time_set_ = true;
00914 }
00915
00916 bool now_have_desired_request = std::any_of(requests_.begin(), requests_.end(),
00917 [this](decltype(requests_)::value_type& request) {
00918 return request.first == ev_counter(); });
00919
00920 if (missing_request_)
00921 {
00922 if (!now_have_desired_request && TimeUtils::GetElapsedTimeMicroseconds(missing_request_time_) > missing_request_window_timeout_us_)
00923 {
00924 TLOG_ERROR("CommandableFragmentGenerator") << "Data-taking has paused for " << TimeUtils::GetElapsedTimeMicroseconds(missing_request_time_) << " us "
00925 << "(> " << std::to_string(missing_request_window_timeout_us_) << " us) while waiting for missing data request messages."
00926 << " Sending Empty Fragments for missing requests!" << TLOG_ENDL;
00927 sendEmptyFragments(frags);
00928
00929 missing_request_ = false;
00930 missing_request_time_ = decltype(missing_request_time_)::max();
00931 }
00932 else if (now_have_desired_request) {
00933 missing_request_ = false;
00934 missing_request_time_ = decltype(missing_request_time_)::max();
00935 }
00936 }
00937
00938 for (auto req = requests_.begin(); req != requests_.end();)
00939 {
00940 auto ts = req->second;
00941 if (req->first < ev_counter())
00942 {
00943 req = requests_.erase(req);
00944 continue;
00945 }
00946 while (req->first > ev_counter() && request_stop_requested_ && TimeUtils::GetElapsedTime(request_stop_timeout_) > 1)
00947 {
00948 sendEmptyFragment(frags, ev_counter(), "Missing request for");
00949 ev_counter_inc(1, true);
00950 }
00951 if (req->first > ev_counter())
00952 {
00953 if (!missing_request_)
00954 {
00955 missing_request_ = true;
00956 missing_request_time_ = std::chrono::steady_clock::now();
00957 }
00958 ++req;
00959 break;
00960 }
00961 TLOG_ARB(9, "CommandableFragmentGenerator") << "ApplyRequests: Checking that data exists for request window " << std::to_string(req->first) << TLOG_ENDL;
00962 Fragment::timestamp_t min = ts > windowOffset_ ? ts - windowOffset_ : 0;
00963 Fragment::timestamp_t max = min + windowWidth_;
00964 TLOG_ARB(9, "CommandableFragmentGenerator") << "ApplyRequests: min is " << std::to_string(min) << ", max is " << std::to_string(max)
00965 << " and last point in buffer is " << std::to_string((dataBuffer_.size() > 0 ? dataBuffer_.back()->timestamp() : 0)) << " (sz=" << std::to_string(dataBuffer_.size()) << ")" << TLOG_ENDL;
00966 bool windowClosed = dataBuffer_.size() > 0 && dataBuffer_.back()->timestamp() >= max;
00967 bool windowTimeout = TimeUtils::GetElapsedTimeMicroseconds(last_window_send_time_) > window_close_timeout_us_;
00968 if (windowTimeout)
00969 {
00970 TLOG_WARNING("CommandableFragmentGenerator") << "A timeout occurred waiting for data to close the request window (max=" << std::to_string(max)
00971 << ", buffer=" << std::to_string((dataBuffer_.size() > 0 ? dataBuffer_.back()->timestamp() : 0))
00972 << " (if no buffer in memory, this is shown as a 0)). Time waiting: "
00973 << TimeUtils::GetElapsedTimeMicroseconds(last_window_send_time_) << " us "
00974 << "(> " << std::to_string(window_close_timeout_us_) << " us)." << TLOG_ENDL;
00975 }
00976 if (windowClosed || !data_thread_running_ || windowTimeout)
00977 {
00978 TLOG_DEBUG("CommandableFragmentGenerator") << "Creating ContainerFragment for Buffered or Window-requested Fragments" << TLOG_ENDL;
00979 frags.emplace_back(new artdaq::Fragment(ev_counter(), fragment_id()));
00980 frags.back()->setTimestamp(ts);
00981 ContainerFragmentLoader cfl(*frags.back());
00982
00983 if (!windowClosed) cfl.set_missing_data(true);
00984 if (dataBuffer_.size() > 0 && dataBuffer_.front()->timestamp() > min)
00985 {
00986 TLOG_DEBUG("CommandableFragmentGenerator") << "Request Window covers data that is either before data collection began or has fallen off the end of the buffer" << TLOG_ENDL;
00987 cfl.set_missing_data(true);
00988 }
00989
00990
00991
00992 for (auto it = dataBuffer_.begin(); it != dataBuffer_.end();)
00993 {
00994 Fragment::timestamp_t fragT = (*it)->timestamp();
00995 if (fragT < min || fragT > max || (fragT == max && windowWidth_ > 0))
00996 {
00997 ++it;
00998 continue;
00999 }
01000
01001 TLOG_ARB(9, "CommandableFragmentGenerator") << "ApplyRequests: Adding Fragment with timestamp " << std::to_string((*it)->timestamp()) << " to Container" << TLOG_ENDL;
01002 cfl.addFragment(*it);
01003
01004 if (uniqueWindows_)
01005 {
01006 it = dataBuffer_.erase(it);
01007 }
01008 else
01009 {
01010 ++it;
01011 }
01012 }
01013 req = requests_.erase(req);
01014 ev_counter_inc(1, true);
01015 last_window_send_time_ = std::chrono::steady_clock::now();
01016 }
01017 else
01018 {
01019
01020 break;
01021 }
01022 }
01023 }
01024
01025 bool artdaq::CommandableFragmentGenerator::applyRequests(artdaq::FragmentPtrs& frags)
01026 {
01027 if (check_stop() || exception())
01028 {
01029 return false;
01030 }
01031
01032
01033 if (mode_ == RequestMode::Ignored)
01034 {
01035 while (dataBufferDepthFragments_ <= 0)
01036 {
01037 if (check_stop() || exception() || !isHardwareOK_) return false;
01038 std::unique_lock<std::mutex> lock(dataBufferMutex_);
01039 dataCondition_.wait_for(lock, std::chrono::milliseconds(10), [this]() { return dataBufferDepthFragments_ > 0; });
01040 }
01041 }
01042 else
01043 {
01044 if ((check_stop() && requests_.size() <= 0) || exception()) return false;
01045 checkDataBuffer();
01046
01047 while (requests_.size() <= 0)
01048 {
01049 if (check_stop() || exception()) return false;
01050
01051 checkDataBuffer();
01052
01053 std::unique_lock<std::mutex> lock(request_mutex_);
01054 requestCondition_.wait_for(lock, std::chrono::milliseconds(10), [this]() { return requests_.size() > 0; });
01055 }
01056 }
01057
01058 {
01059 std::unique_lock<std::mutex> dlk(dataBufferMutex_);
01060 std::unique_lock<std::mutex> rlk(request_mutex_);
01061
01062 switch (mode_)
01063 {
01064 case RequestMode::Single:
01065 applyRequestsSingleMode(frags);
01066 break;
01067 case RequestMode::Window:
01068 applyRequestsWindowMode(frags);
01069 break;
01070 case RequestMode::Buffer:
01071 applyRequestsBufferMode(frags);
01072 break;
01073 case RequestMode::Ignored:
01074 default:
01075 applyRequestsIgnoredMode(frags);
01076 break;
01077 }
01078
01079 getDataBufferStats();
01080 }
01081
01082 if (frags.size() > 0)
01083 TLOG_ARB(9, "CommandableFragmentGenerator") << "Finished Processing Event " << std::to_string(ev_counter() + 1) << " for fragment_id " << fragment_id() << "." << TLOG_ENDL;
01084 return true;
01085 }
01086
01087 bool artdaq::CommandableFragmentGenerator::sendEmptyFragment(artdaq::FragmentPtrs& frags, size_t seqId, std::string desc)
01088 {
01089 TLOG_WARNING("CommandableFragmentGenerator") << desc << " request " << seqId << ", sending empty fragment" << TLOG_ENDL;
01090 for (auto fid : fragment_ids_)
01091 {
01092 auto frag = new Fragment();
01093 frag->setSequenceID(seqId);
01094 frag->setFragmentID(fid);
01095 frag->setSystemType(Fragment::EmptyFragmentType);
01096 frags.emplace_back(FragmentPtr(frag));
01097 }
01098 return true;
01099 }
01100
01101 void artdaq::CommandableFragmentGenerator::sendEmptyFragments(artdaq::FragmentPtrs& frags)
01102 {
01103 auto sequence_id = Fragment::InvalidSequenceID;
01104 auto timestamp = Fragment::InvalidTimestamp;
01105
01106 TLOG_ARB(19, "CommandableFragmentGenerator") << "Sending Empty Fragments" << TLOG_ENDL;
01107 for (auto it = requests_.begin(); it != requests_.end();)
01108 {
01109 auto seq = it->first;
01110 auto ts = it->second;
01111
01112 while (seq > ev_counter())
01113 {
01114
01115 sendEmptyFragment(frags, ev_counter(), "Missed request for");
01116 ev_counter_inc(1, true);
01117 }
01118
01119
01120 if (++it == requests_.end())
01121 {
01122 sequence_id = seq;
01123 timestamp = ts;
01124 break;
01125 }
01126 if (seq < ev_counter()) continue;
01127
01128 }
01129 requests_.clear();
01130
01131 if (sequence_id < ev_counter()) return;
01132 requests_[sequence_id] = timestamp;
01133 }