00001 #define TRACE_NAME (app_name + "_CommandableFragmentGenerator").c_str() // include these 2 first -
00002 #include "artdaq/DAQdata/Globals.hh"
00003
00004 #include "artdaq/Application/CommandableFragmentGenerator.hh"
00005
00006 #include <boost/exception/all.hpp>
00007 #include <boost/throw_exception.hpp>
00008
00009 #include <limits>
00010 #include <iterator>
00011
00012 #include "canvas/Utilities/Exception.h"
00013 #include "cetlib_except/exception.h"
00014 #include "fhiclcpp/ParameterSet.h"
00015
00016 #include "artdaq-core/Utilities/SimpleLookupPolicy.hh"
00017 #include "artdaq-core/Data/Fragment.hh"
00018 #include "artdaq-core/Data/ContainerFragmentLoader.hh"
00019 #include "artdaq-core/Utilities/ExceptionHandler.hh"
00020 #include "artdaq-core/Utilities/TimeUtils.hh"
00021
00022 #include <fstream>
00023 #include <iomanip>
00024 #include <iterator>
00025 #include <iostream>
00026 #include <iomanip>
00027 #include <algorithm>
00028 #include <sys/poll.h>
00029 #include "artdaq/DAQdata/TCPConnect.hh"
00030
00031 #define TLVL_GETNEXT 10
00032 #define TLVL_GETNEXT_VERBOSE 20
00033 #define TLVL_CHECKSTOP 11
00034 #define TLVL_EVCOUNTERINC 12
00035 #define TLVL_GETDATALOOP 13
00036 #define TLVL_GETDATALOOP_DATABUFFWAIT 21
00037 #define TLVL_GETDATALOOP_VERBOSE 20
00038 #define TLVL_WAITFORBUFFERREADY 15
00039 #define TLVL_GETBUFFERSTATS 16
00040 #define TLVL_CHECKDATABUFFER 17
00041 #define TLVL_GETMONITORINGDATA 18
00042 #define TLVL_APPLYREQUESTS 9
00043 #define TLVL_SENDEMPTYFRAGMENTS 19
00044 #define TLVL_CHECKWINDOWS 14
00045
00046 artdaq::CommandableFragmentGenerator::CommandableFragmentGenerator()
00047 : mutex_()
00048 , requestReceiver_(nullptr)
00049 , windowOffset_(0)
00050 , windowWidth_(0)
00051 , staleTimeout_(Fragment::InvalidTimestamp)
00052 , expectedType_(Fragment::EmptyFragmentType)
00053 , maxFragmentCount_(std::numeric_limits<size_t>::max())
00054 , uniqueWindows_(true)
00055 , windows_sent_ooo_()
00056 , missing_request_window_timeout_us_(1000000)
00057 , window_close_timeout_us_(2000000)
00058 , useDataThread_(false)
00059 , circularDataBufferMode_(false)
00060 , sleep_on_no_data_us_(0)
00061 , data_thread_running_(false)
00062 , dataBufferDepthFragments_(0)
00063 , dataBufferDepthBytes_(0)
00064 , maxDataBufferDepthFragments_(1000)
00065 , maxDataBufferDepthBytes_(1000)
00066 , useMonitoringThread_(false)
00067 , monitoringInterval_(0)
00068 , lastMonitoringCall_()
00069 , isHardwareOK_(true)
00070 , dataBuffer_()
00071 , newDataBuffer_()
00072 , run_number_(-1)
00073 , subrun_number_(-1)
00074 , timeout_(std::numeric_limits<uint64_t>::max())
00075 , timestamp_(std::numeric_limits<uint64_t>::max())
00076 , should_stop_(false)
00077 , exception_(false)
00078 , force_stop_(false)
00079 , latest_exception_report_("none")
00080 , ev_counter_(1)
00081 , board_id_(-1)
00082 , instance_name_for_metrics_("FragmentGenerator")
00083 , sleep_on_stop_us_(0)
00084 {}
00085
00086 artdaq::CommandableFragmentGenerator::CommandableFragmentGenerator(const fhicl::ParameterSet& ps)
00087 : mutex_()
00088 , requestReceiver_(nullptr)
00089 , windowOffset_(ps.get<Fragment::timestamp_t>("request_window_offset", 0))
00090 , windowWidth_(ps.get<Fragment::timestamp_t>("request_window_width", 0))
00091 , staleTimeout_(ps.get<Fragment::timestamp_t>("stale_request_timeout", 0xFFFFFFFF))
00092 , expectedType_(ps.get<Fragment::type_t>("expected_fragment_type", Fragment::type_t(Fragment::EmptyFragmentType)))
00093 , uniqueWindows_(ps.get<bool>("request_windows_are_unique", true))
00094 , windows_sent_ooo_()
00095 , missing_request_window_timeout_us_(ps.get<size_t>("missing_request_window_timeout_us", 5000000))
00096 , window_close_timeout_us_(ps.get<size_t>("window_close_timeout_us", 2000000))
00097 , useDataThread_(ps.get<bool>("separate_data_thread", false))
00098 , circularDataBufferMode_(ps.get<bool>("circular_buffer_mode", false))
00099 , sleep_on_no_data_us_(ps.get<size_t>("sleep_on_no_data_us", 0))
00100 , data_thread_running_(false)
00101 , dataBufferDepthFragments_(0)
00102 , dataBufferDepthBytes_(0)
00103 , maxDataBufferDepthFragments_(ps.get<int>("data_buffer_depth_fragments", 1000))
00104 , maxDataBufferDepthBytes_(ps.get<size_t>("data_buffer_depth_mb", 1000) * 1024 * 1024)
00105 , useMonitoringThread_(ps.get<bool>("separate_monitoring_thread", false))
00106 , monitoringInterval_(ps.get<int64_t>("hardware_poll_interval_us", 0))
00107 , lastMonitoringCall_()
00108 , isHardwareOK_(true)
00109 , dataBuffer_()
00110 , newDataBuffer_()
00111 , run_number_(-1)
00112 , subrun_number_(-1)
00113 , timeout_(std::numeric_limits<uint64_t>::max())
00114 , timestamp_(std::numeric_limits<uint64_t>::max())
00115 , should_stop_(false)
00116 , exception_(false)
00117 , force_stop_(false)
00118 , latest_exception_report_("none")
00119 , ev_counter_(1)
00120 , board_id_(-1)
00121 , sleep_on_stop_us_(0)
00122 {
00123 board_id_ = ps.get<int>("board_id");
00124 instance_name_for_metrics_ = "BoardReader." + boost::lexical_cast<std::string>(board_id_);
00125
00126 fragment_ids_ = ps.get<std::vector<artdaq::Fragment::fragment_id_t>>("fragment_ids", std::vector<artdaq::Fragment::fragment_id_t>());
00127
00128 TLOG(TLVL_TRACE) << "artdaq::CommandableFragmentGenerator::CommandableFragmentGenerator(ps)";
00129 int fragment_id = ps.get<int>("fragment_id", -99);
00130
00131 if (fragment_id != -99)
00132 {
00133 if (fragment_ids_.size() != 0)
00134 {
00135 latest_exception_report_ = "Error in CommandableFragmentGenerator: can't both define \"fragment_id\" and \"fragment_ids\" in FHiCL document";
00136 throw cet::exception(latest_exception_report_);
00137 }
00138 else
00139 {
00140 fragment_ids_.emplace_back(fragment_id);
00141 }
00142 }
00143
00144 sleep_on_stop_us_ = ps.get<int>("sleep_on_stop_us", 0);
00145
00146 std::string modeString = ps.get<std::string>("request_mode", "ignored");
00147 if (modeString == "single" || modeString == "Single")
00148 {
00149 mode_ = RequestMode::Single;
00150 }
00151 else if (modeString.find("buffer") != std::string::npos || modeString.find("Buffer") != std::string::npos)
00152 {
00153 mode_ = RequestMode::Buffer;
00154 }
00155 else if (modeString == "window" || modeString == "Window")
00156 {
00157 mode_ = RequestMode::Window;
00158 }
00159 else if (modeString.find("ignore") != std::string::npos || modeString.find("Ignore") != std::string::npos)
00160 {
00161 mode_ = RequestMode::Ignored;
00162 }
00163 TLOG(TLVL_DEBUG) << "Request mode is " << printMode_();
00164
00165 if (mode_ != RequestMode::Ignored)
00166 {
00167 if (!useDataThread_)
00168 {
00169 latest_exception_report_ = "Error in CommandableFragmentGenerator: use_data_thread must be true when request_mode is not \"Ignored\"!";
00170 throw cet::exception(latest_exception_report_);
00171 }
00172 requestReceiver_.reset(new RequestReceiver(ps));
00173 }
00174 }
00175
00176 artdaq::CommandableFragmentGenerator::~CommandableFragmentGenerator()
00177 {
00178 joinThreads();
00179 requestReceiver_.reset(nullptr);
00180 }
00181
00182 void artdaq::CommandableFragmentGenerator::joinThreads()
00183 {
00184 should_stop_ = true;
00185 force_stop_ = true;
00186 TLOG(TLVL_DEBUG) << "Joining dataThread";
00187 if (dataThread_.joinable()) dataThread_.join();
00188 TLOG(TLVL_DEBUG) << "Joining monitoringThread";
00189 if (monitoringThread_.joinable()) monitoringThread_.join();
00190 TLOG(TLVL_DEBUG) << "joinThreads complete";
00191 }
00192
00193 bool artdaq::CommandableFragmentGenerator::getNext(FragmentPtrs& output)
00194 {
00195 bool result = true;
00196
00197 if (check_stop()) usleep(sleep_on_stop_us_);
00198 if (exception() || force_stop_) return false;
00199
00200 if (!useMonitoringThread_ && monitoringInterval_ > 0)
00201 {
00202 TLOG(TLVL_GETNEXT) << "getNext: Checking whether to collect Monitoring Data";
00203 auto now = std::chrono::steady_clock::now();
00204
00205 if (TimeUtils::GetElapsedTimeMicroseconds(lastMonitoringCall_, now) >= static_cast<size_t>(monitoringInterval_))
00206 {
00207 TLOG(TLVL_GETNEXT) << "getNext: Collecting Monitoring Data";
00208 isHardwareOK_ = checkHWStatus_();
00209 TLOG(TLVL_GETNEXT) << "getNext: isHardwareOK_ is now " << std::boolalpha << isHardwareOK_;
00210 lastMonitoringCall_ = now;
00211 }
00212 }
00213
00214 try
00215 {
00216 std::lock_guard<std::mutex> lk(mutex_);
00217 if (useDataThread_)
00218 {
00219 TLOG(TLVL_TRACE) << "getNext: Calling applyRequests";
00220 result = applyRequests(output);
00221 TLOG(TLVL_TRACE) << "getNext: Done with applyRequests result=" << std::boolalpha << result;
00222 for (auto dataIter = output.begin(); dataIter != output.end(); ++dataIter)
00223 {
00224 TLOG(20) << "getNext: applyRequests() returned fragment with sequenceID = " << (*dataIter)->sequenceID()
00225 << ", timestamp = " << (*dataIter)->timestamp() << ", and sizeBytes = " << (*dataIter)->sizeBytes();
00226 }
00227
00228 if (exception())
00229 {
00230 TLOG(TLVL_ERROR) << "Exception found in BoardReader with board ID " << board_id() << "; BoardReader will now return error status when queried";
00231 throw cet::exception("CommandableFragmentGenerator") << "Exception found in BoardReader with board ID " << board_id() << "; BoardReader will now return error status when queried";
00232 }
00233 }
00234 else
00235 {
00236 if (!isHardwareOK_)
00237 {
00238 TLOG(TLVL_ERROR) << "Stopping CFG because the hardware reports bad status!";
00239 return false;
00240 }
00241 TLOG(TLVL_TRACE) << "getNext: Calling getNext_ w/ ev_counter()=" << ev_counter();
00242 try
00243 {
00244 result = getNext_(output);
00245 }
00246 catch (...)
00247 {
00248 throw;
00249 }
00250 TLOG(TLVL_TRACE) << "getNext: Done with getNext_ - ev_counter() now " << ev_counter();
00251 for (auto dataIter = output.begin(); dataIter != output.end(); ++dataIter)
00252 {
00253 TLOG(TLVL_GETNEXT_VERBOSE) << "getNext: getNext_() returned fragment with sequenceID = " << (*dataIter)->sequenceID()
00254 << ", timestamp = " << (*dataIter)->timestamp() << ", and sizeBytes = " << (*dataIter)->sizeBytes();
00255 }
00256 }
00257 }
00258 catch (const cet::exception& e)
00259 {
00260 latest_exception_report_ = "cet::exception caught in getNext(): ";
00261 latest_exception_report_.append(e.what());
00262 TLOG(TLVL_ERROR) << "getNext: cet::exception caught: " << e;
00263 set_exception(true);
00264 return false;
00265 }
00266 catch (const boost::exception& e)
00267 {
00268 latest_exception_report_ = "boost::exception caught in getNext(): ";
00269 latest_exception_report_.append(boost::diagnostic_information(e));
00270 TLOG(TLVL_ERROR) << "getNext: boost::exception caught: " << boost::diagnostic_information(e);
00271 set_exception(true);
00272 return false;
00273 }
00274 catch (const std::exception& e)
00275 {
00276 latest_exception_report_ = "std::exception caught in getNext(): ";
00277 latest_exception_report_.append(e.what());
00278 TLOG(TLVL_ERROR) << "getNext: std::exception caught: " << e.what();
00279 set_exception(true);
00280 return false;
00281 }
00282 catch (...)
00283 {
00284 latest_exception_report_ = "Unknown exception caught in getNext().";
00285 TLOG(TLVL_ERROR) << "getNext: unknown exception caught";
00286 set_exception(true);
00287 return false;
00288 }
00289
00290 if (!result)
00291 {
00292 TLOG(TLVL_DEBUG) << "getNext: Either getNext_ or applyRequests returned false, stopping";
00293 }
00294
00295 if (metricMan && !output.empty())
00296 {
00297 auto timestamp = output.front()->timestamp();
00298
00299 if (output.size() > 1)
00300 {
00301 for (auto& outputfrag : output)
00302 {
00303 if (outputfrag->timestamp() > timestamp)
00304 {
00305 timestamp = outputfrag->timestamp();
00306 }
00307 }
00308 }
00309
00310 metricMan->sendMetric("Last Timestamp", timestamp, "Ticks", 1,
00311 MetricMode::LastPoint, app_name);
00312 }
00313
00314 return result;
00315 }
00316
00317 bool artdaq::CommandableFragmentGenerator::check_stop()
00318 {
00319 TLOG(TLVL_CHECKSTOP) << "CFG::check_stop: should_stop=" << should_stop() << ", useDataThread_=" << useDataThread_ << ", exception status =" << int(exception());
00320
00321 if (!should_stop()) return false;
00322 if (!useDataThread_ || mode_ == RequestMode::Ignored) return true;
00323 if (force_stop_) return true;
00324
00325
00326 TLOG(TLVL_DEBUG) << "should_stop is true, force_stop_ is false, requestReceiver_->isRunning() is " << std::boolalpha << requestReceiver_->isRunning();
00327 return !requestReceiver_->isRunning();
00328 }
00329
00330 int artdaq::CommandableFragmentGenerator::fragment_id() const
00331 {
00332 if (fragment_ids_.size() != 1)
00333 {
00334 throw cet::exception("Error in CommandableFragmentGenerator: can't call fragment_id() unless member fragment_ids_ vector is length 1");
00335 }
00336 else
00337 {
00338 return fragment_ids_[0];
00339 }
00340 }
00341
00342 size_t artdaq::CommandableFragmentGenerator::ev_counter_inc(size_t step, bool force)
00343 {
00344 if (force || mode_ == RequestMode::Ignored)
00345 {
00346 TLOG(TLVL_EVCOUNTERINC) << "ev_counter_inc: Incrementing ev_counter from " << ev_counter() << " by " << step;
00347 return ev_counter_.fetch_add(step);
00348 }
00349 return ev_counter_.load();
00350 }
00351
00352 void artdaq::CommandableFragmentGenerator::StartCmd(int run, uint64_t timeout, uint64_t timestamp)
00353 {
00354 TLOG(TLVL_TRACE) << "Start Command received.";
00355 if (run < 0) throw cet::exception("CommandableFragmentGenerator") << "negative run number";
00356
00357 timeout_ = timeout;
00358 timestamp_ = timestamp;
00359 ev_counter_.store(1);
00360 windows_sent_ooo_.clear();
00361 {
00362 std::unique_lock<std::mutex> lock(dataBufferMutex_);
00363 dataBuffer_.clear();
00364 }
00365 should_stop_.store(false);
00366 force_stop_.store(false);
00367 exception_.store(false);
00368 run_number_ = run;
00369 subrun_number_ = 1;
00370 latest_exception_report_ = "none";
00371
00372 start();
00373
00374 std::unique_lock<std::mutex> lk(mutex_);
00375 if (useDataThread_) startDataThread();
00376 if (useMonitoringThread_) startMonitoringThread();
00377 if (mode_ != RequestMode::Ignored && !requestReceiver_->isRunning()) requestReceiver_->startRequestReceiverThread();
00378 TLOG(TLVL_TRACE) << "Start Command complete.";
00379 }
00380
00381 void artdaq::CommandableFragmentGenerator::StopCmd(uint64_t timeout, uint64_t timestamp)
00382 {
00383 TLOG(TLVL_TRACE) << "Stop Command received.";
00384
00385 timeout_ = timeout;
00386 timestamp_ = timestamp;
00387 if (requestReceiver_ && requestReceiver_->isRunning()) {
00388 TLOG(TLVL_DEBUG) << "Stopping Request receiver thread BEGIN";
00389 requestReceiver_->stopRequestReceiverThread();
00390 TLOG(TLVL_DEBUG) << "Stopping Request receiver thread END";
00391 }
00392
00393 stopNoMutex();
00394 should_stop_.store(true);
00395 std::unique_lock<std::mutex> lk(mutex_);
00396 stop();
00397
00398 joinThreads();
00399 TLOG(TLVL_TRACE) << "Stop Command complete.";
00400 }
00401
00402 void artdaq::CommandableFragmentGenerator::PauseCmd(uint64_t timeout, uint64_t timestamp)
00403 {
00404 TLOG(TLVL_TRACE) << "Pause Command received.";
00405 timeout_ = timeout;
00406 timestamp_ = timestamp;
00407
00408
00409 pauseNoMutex();
00410 should_stop_.store(true);
00411 std::unique_lock<std::mutex> lk(mutex_);
00412
00413 pause();
00414 }
00415
00416 void artdaq::CommandableFragmentGenerator::ResumeCmd(uint64_t timeout, uint64_t timestamp)
00417 {
00418 TLOG(TLVL_TRACE) << "Resume Command received.";
00419 timeout_ = timeout;
00420 timestamp_ = timestamp;
00421
00422 subrun_number_ += 1;
00423 should_stop_ = false;
00424 {
00425 std::unique_lock<std::mutex> lk(dataBufferMutex_);
00426 dataBufferDepthBytes_ = 0;
00427 dataBufferDepthFragments_ = 0;
00428 dataBuffer_.clear();
00429 }
00430
00431 resume();
00432
00433 std::unique_lock<std::mutex> lk(mutex_);
00434
00435
00436
00437 TLOG(TLVL_TRACE) << "Resume Command complete.";
00438 }
00439
00440 std::string artdaq::CommandableFragmentGenerator::ReportCmd(std::string const& which)
00441 {
00442 TLOG(TLVL_TRACE) << "Report Command received.";
00443 std::lock_guard<std::mutex> lk(mutex_);
00444
00445
00446
00447
00448
00449
00450 std::string childReport = reportSpecific(which);
00451 if (childReport.length() > 0) { return childReport; }
00452
00453
00454 if (which == "latest_exception")
00455 {
00456 return latest_exception_report_;
00457 }
00458
00459
00460 childReport = report();
00461 if (childReport.length() > 0) { return childReport; }
00462
00463
00464 std::string tmpString = "The \"" + which + "\" command is not ";
00465 tmpString.append("currently supported by the ");
00466 tmpString.append(metricsReportingInstanceName());
00467 tmpString.append(" fragment generator.");
00468 TLOG(TLVL_TRACE) << "Report Command complete.";
00469 return tmpString;
00470 }
00471
00472
00473 void artdaq::CommandableFragmentGenerator::pauseNoMutex()
00474 {
00475 #pragma message "Using default implementation of CommandableFragmentGenerator::pauseNoMutex()"
00476 }
00477
00478 void artdaq::CommandableFragmentGenerator::pause()
00479 {
00480 #pragma message "Using default implementation of CommandableFragmentGenerator::pause()"
00481 }
00482
00483 void artdaq::CommandableFragmentGenerator::resume()
00484 {
00485 #pragma message "Using default implementation of CommandableFragmentGenerator::resume()"
00486 }
00487
00488 std::string artdaq::CommandableFragmentGenerator::report()
00489 {
00490 #pragma message "Using default implementation of CommandableFragmentGenerator::report()"
00491 return "";
00492 }
00493
00494 std::string artdaq::CommandableFragmentGenerator::reportSpecific(std::string const&)
00495 {
00496 #pragma message "Using default implementation of CommandableFragmentGenerator::reportSpecific(std::string)"
00497 return "";
00498 }
00499
00500 bool artdaq::CommandableFragmentGenerator::checkHWStatus_()
00501 {
00502 #pragma message "Using default implementation of CommandableFragmentGenerator::checkHWStatus_()"
00503 return true;
00504 }
00505
00506 bool artdaq::CommandableFragmentGenerator::metaCommand(std::string const&, std::string const&)
00507 {
00508 #pragma message "Using default implementation of CommandableFragmentGenerator::metaCommand(std::string, std::string)"
00509 return true;
00510 }
00511
00512 void artdaq::CommandableFragmentGenerator::startDataThread()
00513 {
00514 if (dataThread_.joinable()) dataThread_.join();
00515 TLOG(TLVL_INFO) << "Starting Data Receiver Thread";
00516 try {
00517 dataThread_ = boost::thread(&CommandableFragmentGenerator::getDataLoop, this);
00518 }
00519 catch (const boost::exception& e)
00520 {
00521 TLOG(TLVL_ERROR) << "Caught boost::exception starting Data Receiver thread: " << boost::diagnostic_information(e) << ", errno=" << errno;
00522 std::cerr << "Caught boost::exception starting Data Receiver thread: " << boost::diagnostic_information(e) << ", errno=" << errno << std::endl;
00523 exit(5);
00524 }
00525 }
00526
00527 void artdaq::CommandableFragmentGenerator::startMonitoringThread()
00528 {
00529 if (monitoringThread_.joinable()) monitoringThread_.join();
00530 TLOG(TLVL_INFO) << "Starting Hardware Monitoring Thread";
00531 try {
00532 monitoringThread_ = boost::thread(&CommandableFragmentGenerator::getMonitoringDataLoop, this);
00533 }
00534 catch (const boost::exception& e)
00535 {
00536 TLOG(TLVL_ERROR) << "Caught boost::exception starting Hardware Monitoring thread: " << boost::diagnostic_information(e) << ", errno=" << errno;
00537 std::cerr << "Caught boost::exception starting Hardware Monitoring thread: " << boost::diagnostic_information(e) << ", errno=" << errno << std::endl;
00538 exit(5);
00539 }
00540 }
00541
00542 std::string artdaq::CommandableFragmentGenerator::printMode_()
00543 {
00544 switch (mode_)
00545 {
00546 case RequestMode::Single:
00547 return "Single";
00548 case RequestMode::Buffer:
00549 return "Buffer";
00550 case RequestMode::Window:
00551 return "Window";
00552 case RequestMode::Ignored:
00553 return "Ignored";
00554 }
00555
00556 return "ERROR";
00557 }
00558
00559
00560
00561
00562
00563 void artdaq::CommandableFragmentGenerator::getDataLoop()
00564 {
00565 data_thread_running_ = true;
00566 while (!force_stop_)
00567 {
00568 if (!isHardwareOK_)
00569 {
00570 TLOG(TLVL_DEBUG) << "getDataLoop: isHardwareOK is " << isHardwareOK_ << ", aborting data thread";
00571 data_thread_running_ = false;
00572 return;
00573 }
00574
00575 TLOG(TLVL_GETDATALOOP) << "getDataLoop: calling getNext_";
00576
00577 bool data = false;
00578 auto startdata = std::chrono::steady_clock::now();
00579
00580 try
00581 {
00582 data = getNext_(newDataBuffer_);
00583 }
00584 catch (...)
00585 {
00586 ExceptionHandler(ExceptionHandlerRethrow::no,
00587 "Exception thrown by fragment generator in CommandableFragmentGenerator::getDataLoop; setting exception state to \"true\"");
00588 set_exception(true);
00589
00590 data_thread_running_ = false;
00591 return;
00592 }
00593
00594 size_t newDataBufferDepthBytes = 0;
00595 for (auto dataIter = newDataBuffer_.begin(); dataIter != newDataBuffer_.end(); ++dataIter)
00596 {
00597 TLOG(TLVL_GETDATALOOP_VERBOSE) << "getDataLoop: getNext_() returned fragment with timestamp = " << (*dataIter)->timestamp() << ", and sizeBytes = " << (*dataIter)->sizeBytes();
00598 newDataBufferDepthBytes += (*dataIter)->sizeBytes();
00599 }
00600
00601 if (metricMan)
00602 {
00603 metricMan->sendMetric("Avg Data Acquisition Time", TimeUtils::GetElapsedTime(startdata), "s", 3, artdaq::MetricMode::Average);
00604 }
00605
00606 if (newDataBuffer_.size() == 0 && sleep_on_no_data_us_ > 0)
00607 {
00608 usleep(sleep_on_no_data_us_);
00609 }
00610
00611 TLOG(TLVL_GETDATALOOP_DATABUFFWAIT) << "Waiting for data buffer ready";
00612 if (!waitForDataBufferReady()) return;
00613 TLOG(TLVL_GETDATALOOP_DATABUFFWAIT) << "Done waiting for data buffer ready";
00614
00615 TLOG(TLVL_GETDATALOOP) << "getDataLoop: processing data";
00616 if (data && !force_stop_)
00617 {
00618 std::unique_lock<std::mutex> lock(dataBufferMutex_);
00619 switch (mode_)
00620 {
00621 case RequestMode::Single:
00622
00623 while (newDataBuffer_.size() >= fragment_ids_.size())
00624 {
00625 dataBuffer_.clear();
00626 auto it = newDataBuffer_.begin();
00627 std::advance(it, fragment_ids_.size());
00628 dataBuffer_.splice(dataBuffer_.end(), newDataBuffer_, newDataBuffer_.begin(), it);
00629
00630 for (auto dbit = dataBuffer_.begin(); dbit != dataBuffer_.end(); ++dbit)
00631 {
00632 dataBufferDepthBytes_ += (*dbit)->sizeBytes();
00633 }
00634 }
00635 break;
00636 case RequestMode::Buffer:
00637 case RequestMode::Ignored:
00638 case RequestMode::Window:
00639 default:
00640
00641 dataBufferDepthBytes_ += newDataBufferDepthBytes;
00642 dataBuffer_.splice(dataBuffer_.end(), newDataBuffer_);
00643 break;
00644 }
00645 getDataBufferStats();
00646 }
00647
00648 {
00649 std::unique_lock<std::mutex> lock(dataBufferMutex_);
00650 if (dataBuffer_.size() > 0)
00651 {
00652 dataCondition_.notify_all();
00653 }
00654 }
00655 if (!data || force_stop_)
00656 {
00657 TLOG(TLVL_INFO) << "Data flow has stopped. Ending data collection thread";
00658 std::unique_lock<std::mutex> lock(dataBufferMutex_);
00659 data_thread_running_ = false;
00660 if (requestReceiver_) requestReceiver_->ClearRequests();
00661 newDataBuffer_.clear();
00662 TLOG(TLVL_INFO) << "getDataLoop: Ending thread";
00663 return;
00664 }
00665 }
00666 }
00667
00668 bool artdaq::CommandableFragmentGenerator::waitForDataBufferReady()
00669 {
00670 auto startwait = std::chrono::steady_clock::now();
00671 auto first = true;
00672 auto lastwaittime = 0ULL;
00673
00674 {
00675 std::unique_lock<std::mutex> lock(dataBufferMutex_);
00676 getDataBufferStats();
00677 }
00678
00679 while (dataBufferIsTooLarge())
00680 {
00681 if (!circularDataBufferMode_)
00682 {
00683 if (should_stop())
00684 {
00685 TLOG(TLVL_DEBUG) << "Run ended while waiting for buffer to shrink!";
00686 std::unique_lock<std::mutex> lock(dataBufferMutex_);
00687 getDataBufferStats();
00688 dataCondition_.notify_all();
00689 data_thread_running_ = false;
00690 return false;
00691 }
00692 auto waittime = TimeUtils::GetElapsedTimeMilliseconds(startwait);
00693
00694 if (first || (waittime != lastwaittime && waittime % 1000 == 0))
00695 {
00696 TLOG(TLVL_WARNING) << "Bad Omen: Data Buffer has exceeded its size limits. "
00697 << "(seq_id=" << ev_counter()
00698 << ", frags=" << dataBufferDepthFragments_ << "/" << maxDataBufferDepthFragments_
00699 << ", szB=" << dataBufferDepthBytes_ << "/" << maxDataBufferDepthBytes_ << ")";
00700 TLOG(TLVL_TRACE) << "Bad Omen: Possible causes include requests not getting through or Ignored-mode BR issues";
00701 first = false;
00702 }
00703 if (waittime % 5 && waittime != lastwaittime)
00704 {
00705 TLOG(TLVL_WAITFORBUFFERREADY) << "getDataLoop: Data Retreival paused for " << waittime << " ms waiting for data buffer to drain";
00706 }
00707 lastwaittime = waittime;
00708 usleep(1000);
00709 }
00710 else
00711 {
00712 std::unique_lock<std::mutex> lock(dataBufferMutex_);
00713 getDataBufferStats();
00714 if (dataBufferIsTooLarge())
00715 {
00716 if (dataBuffer_.begin() == dataBuffer_.end())
00717 {
00718 TLOG(TLVL_WARNING) << "Data buffer is reported as too large, but doesn't contain any Fragments! Possible corrupt memory!";
00719 continue;
00720 }
00721 if (*dataBuffer_.begin())
00722 {
00723 TLOG(TLVL_WAITFORBUFFERREADY) << "waitForDataBufferReady: Dropping Fragment with timestamp " << (*dataBuffer_.begin())->timestamp() << " from data buffer (Buffer over-size, circular data buffer mode)";
00724 }
00725 dataBufferDepthBytes_ -= (*dataBuffer_.begin())->sizeBytes();
00726 dataBuffer_.erase(dataBuffer_.begin());
00727 getDataBufferStats();
00728 }
00729
00730 }
00731 }
00732 return true;
00733 }
00734
00735 bool artdaq::CommandableFragmentGenerator::dataBufferIsTooLarge()
00736 {
00737 return (maxDataBufferDepthFragments_ > 0 && dataBufferDepthFragments_ > maxDataBufferDepthFragments_) || (maxDataBufferDepthBytes_ > 0 && dataBufferDepthBytes_ > maxDataBufferDepthBytes_);
00738 }
00739
00740 void artdaq::CommandableFragmentGenerator::getDataBufferStats()
00741 {
00743 dataBufferDepthFragments_ = dataBuffer_.size();
00744
00745 if (metricMan)
00746 {
00747 TLOG(TLVL_GETBUFFERSTATS) << "getDataBufferStats: Sending Metrics";
00748 metricMan->sendMetric("Buffer Depth Fragments", dataBufferDepthFragments_.load(), "fragments", 1, MetricMode::LastPoint);
00749 metricMan->sendMetric("Buffer Depth Bytes", dataBufferDepthBytes_.load(), "bytes", 1, MetricMode::LastPoint);
00750 }
00751 TLOG(TLVL_GETBUFFERSTATS) << "getDataBufferStats: frags=" << dataBufferDepthFragments_.load() << "/" << maxDataBufferDepthFragments_
00752 << ", sz=" << dataBufferDepthBytes_.load() << "/" << maxDataBufferDepthBytes_;
00753 }
00754
00755 void artdaq::CommandableFragmentGenerator::checkDataBuffer()
00756 {
00757 std::unique_lock<std::mutex> lock(dataBufferMutex_);
00758 dataCondition_.wait_for(lock, std::chrono::milliseconds(10));
00759 if (dataBufferDepthFragments_ > 0)
00760 {
00761 if ((mode_ == RequestMode::Buffer || mode_ == RequestMode::Window))
00762 {
00763
00764 getDataBufferStats();
00765 while (dataBufferIsTooLarge())
00766 {
00767 TLOG(TLVL_CHECKDATABUFFER) << "checkDataBuffer: Dropping Fragment with timestamp " << (*dataBuffer_.begin())->timestamp() << " from data buffer (Buffer over-size)";
00768 dataBufferDepthBytes_ -= (*dataBuffer_.begin())->sizeBytes();
00769 dataBuffer_.erase(dataBuffer_.begin());
00770 getDataBufferStats();
00771 }
00772 if (dataBuffer_.size() > 0)
00773 {
00774 TLOG(TLVL_CHECKDATABUFFER) << "Determining if Fragments can be dropped from data buffer";
00775 Fragment::timestamp_t last = dataBuffer_.back()->timestamp();
00776 Fragment::timestamp_t min = last > staleTimeout_ ? last - staleTimeout_ : 0;
00777 for (auto it = dataBuffer_.begin(); it != dataBuffer_.end();)
00778 {
00779 if ((*it)->timestamp() < min)
00780 {
00781 TLOG(TLVL_CHECKDATABUFFER) << "checkDataBuffer: Dropping Fragment with timestamp " << (*it)->timestamp() << " from data buffer (timeout=" << staleTimeout_ << ", min=" << min << ")";
00782 dataBufferDepthBytes_ -= (*it)->sizeBytes();
00783 it = dataBuffer_.erase(it);
00784 }
00785 else
00786 {
00787 break;
00788 }
00789 }
00790 getDataBufferStats();
00791 }
00792 }
00793 else if (mode_ == RequestMode::Single && dataBuffer_.size() > fragment_ids_.size())
00794 {
00795
00796 while (dataBuffer_.size() > fragment_ids_.size())
00797 {
00798 dataBufferDepthBytes_ -= (*dataBuffer_.begin())->sizeBytes();
00799 dataBuffer_.erase(dataBuffer_.begin());
00800 }
00801 }
00802 }
00803 }
00804
00805 void artdaq::CommandableFragmentGenerator::getMonitoringDataLoop()
00806 {
00807 while (!force_stop_)
00808 {
00809 if (should_stop() || monitoringInterval_ <= 0)
00810 {
00811 TLOG(TLVL_DEBUG) << "getMonitoringDataLoop: should_stop() is " << std::boolalpha << should_stop()
00812 << " and monitoringInterval is " << monitoringInterval_ << ", returning";
00813 return;
00814 }
00815 TLOG(TLVL_GETMONITORINGDATA) << "getMonitoringDataLoop: Determining whether to call checkHWStatus_";
00816
00817 auto now = std::chrono::steady_clock::now();
00818 if (TimeUtils::GetElapsedTimeMicroseconds(lastMonitoringCall_, now) >= static_cast<size_t>(monitoringInterval_))
00819 {
00820 isHardwareOK_ = checkHWStatus_();
00821 TLOG(TLVL_GETMONITORINGDATA) << "getMonitoringDataLoop: isHardwareOK_ is now " << std::boolalpha << isHardwareOK_;
00822 lastMonitoringCall_ = now;
00823 }
00824 usleep(monitoringInterval_ / 10);
00825 }
00826 }
00827
00828 void artdaq::CommandableFragmentGenerator::applyRequestsIgnoredMode(artdaq::FragmentPtrs& frags)
00829 {
00830
00831 TLOG(TLVL_APPLYREQUESTS) << "Mode is Ignored; Copying data to output";
00832 std::move(dataBuffer_.begin(), dataBuffer_.end(), std::inserter(frags, frags.end()));
00833 dataBufferDepthBytes_ = 0;
00834 dataBuffer_.clear();
00835 }
00836
00837 void artdaq::CommandableFragmentGenerator::applyRequestsSingleMode(artdaq::FragmentPtrs& frags)
00838 {
00839
00840 auto requests = requestReceiver_->GetRequests();
00841 while (requests.size() > 1)
00842 {
00843
00844 requestReceiver_->RemoveRequest(requests.begin()->first);
00845 requests.erase(requests.begin());
00846 }
00847 sendEmptyFragments(frags, requests);
00848
00849
00850 if (requests.size() == 0 || !requests.count(ev_counter())) return;
00851
00852 if (dataBuffer_.size() > 0)
00853 {
00854 TLOG(TLVL_APPLYREQUESTS) << "Mode is Single; Sending copy of last event";
00855 for (auto& fragptr : dataBuffer_)
00856 {
00857
00858 auto frag = fragptr.get();
00859 auto newfrag = std::unique_ptr<artdaq::Fragment>(new Fragment(ev_counter(), frag->fragmentID()));
00860 newfrag->resize(frag->size() - detail::RawFragmentHeader::num_words());
00861 memcpy(newfrag->headerAddress(), frag->headerAddress(), frag->sizeBytes());
00862 newfrag->setTimestamp(requests[ev_counter()]);
00863 newfrag->setSequenceID(ev_counter());
00864 frags.push_back(std::move(newfrag));
00865 }
00866 }
00867 else
00868 {
00869 sendEmptyFragment(frags, ev_counter(), "No data for");
00870 }
00871 requestReceiver_->RemoveRequest(ev_counter());
00872 ev_counter_inc(1, true);
00873 }
00874
00875 void artdaq::CommandableFragmentGenerator::applyRequestsBufferMode(artdaq::FragmentPtrs& frags)
00876 {
00877
00878 auto requests = requestReceiver_->GetRequests();
00879 while (requests.size() > 1)
00880 {
00881
00882 requestReceiver_->RemoveRequest(requests.begin()->first);
00883 requests.erase(requests.begin());
00884 }
00885 sendEmptyFragments(frags, requests);
00886
00887
00888 if (requests.size() == 0 || !requests.count(ev_counter())) return;
00889
00890 TLOG(TLVL_DEBUG) << "Creating ContainerFragment for Buffered Fragments";
00891 frags.emplace_back(new artdaq::Fragment(ev_counter(), fragment_id()));
00892 frags.back()->setTimestamp(requests[ev_counter()]);
00893 ContainerFragmentLoader cfl(*frags.back());
00894 cfl.set_missing_data(false);
00895
00896
00897
00898 for (auto it = dataBuffer_.begin(); it != dataBuffer_.end();)
00899 {
00900 TLOG(TLVL_APPLYREQUESTS) << "ApplyRequests: Adding Fragment with timestamp " << (*it)->timestamp() << " to Container with sequence ID " << ev_counter();
00901 cfl.addFragment(*it);
00902 dataBufferDepthBytes_ -= (*it)->sizeBytes();
00903 it = dataBuffer_.erase(it);
00904 }
00905 requestReceiver_->RemoveRequest(ev_counter());
00906 ev_counter_inc(1, true);
00907 }
00908
00909 void artdaq::CommandableFragmentGenerator::applyRequestsWindowMode(artdaq::FragmentPtrs& frags)
00910 {
00911 TLOG(TLVL_APPLYREQUESTS) << "applyRequestsWindowMode BEGIN";
00912
00913 auto requests = requestReceiver_->GetRequests();
00914
00915 TLOG(TLVL_APPLYREQUESTS) << "applyRequestsWindowMode: Starting request processing";
00916 for (auto req = requests.begin(); req != requests.end();)
00917 {
00918 TLOG(TLVL_APPLYREQUESTS) << "applyRequestsWindowMode: processing request with sequence ID " << req->first << ", timestamp " << req->second;
00919
00920
00921 while (req->first < ev_counter() && requests.size() > 0)
00922 {
00923 TLOG(TLVL_APPLYREQUESTS) << "applyRequestsWindowMode: Clearing passed request for sequence ID " << req->first;
00924 requestReceiver_->RemoveRequest(req->first);
00925 req = requests.erase(req);
00926 }
00927 if (requests.size() == 0) break;
00928
00929 auto ts = req->second;
00930 TLOG(TLVL_APPLYREQUESTS) << "applyRequests: Checking that data exists for request window " << req->first;
00931 Fragment::timestamp_t min = ts > windowOffset_ ? ts - windowOffset_ : 0;
00932 Fragment::timestamp_t max = min + windowWidth_;
00933 TLOG(TLVL_APPLYREQUESTS) << "ApplyRequests: min is " << min << ", max is " << max
00934 << " and first/last points in buffer are " << (dataBuffer_.size() > 0 ? dataBuffer_.front()->timestamp() : 0)
00935 << "/" << (dataBuffer_.size() > 0 ? dataBuffer_.back()->timestamp() : 0)
00936 << " (sz=" << dataBuffer_.size() << " [" << dataBufferDepthBytes_.load()
00937 << "/" << maxDataBufferDepthBytes_ << "])";
00938 bool windowClosed = dataBuffer_.size() > 0 && dataBuffer_.back()->timestamp() >= max;
00939 bool windowTimeout = !windowClosed && TimeUtils::GetElapsedTimeMicroseconds(requestReceiver_->GetRequestTime(req->first)) > window_close_timeout_us_;
00940 if (windowTimeout)
00941 {
00942 TLOG(TLVL_WARNING) << "applyRequests: A timeout occurred waiting for data to close the request window ({" << min << "-" << max
00943 << "}, buffer={" << (dataBuffer_.size() > 0 ? dataBuffer_.front()->timestamp() : 0) << "-"
00944 << (dataBuffer_.size() > 0 ? dataBuffer_.back()->timestamp() : 0)
00945 << "} ). Time waiting: "
00946 << TimeUtils::GetElapsedTimeMicroseconds(requestReceiver_->GetRequestTime(req->first)) << " us "
00947 << "(> " << window_close_timeout_us_ << " us).";
00948 }
00949 if (windowClosed || !data_thread_running_ || windowTimeout)
00950 {
00951 TLOG(TLVL_DEBUG) << "applyRequests: Creating ContainerFragment for Window-requested Fragments";
00952 frags.emplace_back(new artdaq::Fragment(req->first, fragment_id()));
00953 frags.back()->setTimestamp(ts);
00954 ContainerFragmentLoader cfl(*frags.back());
00955
00956
00957
00958
00959
00960
00961
00962
00963
00964
00965
00966
00967
00968 if (!windowClosed || (dataBuffer_.size() > 0 && dataBuffer_.front()->timestamp() > min))
00969 {
00970 TLOG(TLVL_DEBUG) << "applyRequests: Request window starts before and/or ends after the current data buffer, setting ContainerFragment's missing_data flag!"
00971 << " (requestWindowRange=[" << min << "," << max << "], "
00972 << "buffer={" << (dataBuffer_.size() > 0 ? dataBuffer_.front()->timestamp() : 0) << "-"
00973 << (dataBuffer_.size() > 0 ? dataBuffer_.back()->timestamp() : 0) << "}";
00974 cfl.set_missing_data(true);
00975 }
00976
00977
00978
00979 int fragCount = 0;
00980 for (auto it = dataBuffer_.begin(); it != dataBuffer_.end();)
00981 {
00982 Fragment::timestamp_t fragT = (*it)->timestamp();
00983 if (fragT < min || fragT > max || (fragT == max && windowWidth_ > 0))
00984 {
00985 ++it;
00986 continue;
00987 }
00988
00989 TLOG(TLVL_APPLYREQUESTS) << "applyRequests: Adding (" << (++fragCount) << "th) Fragment with timestamp "
00990 << (*it)->timestamp() << " and sizeBytes " << (*it)->sizeBytes()
00991 << " to Container for sequence ID " << req->first;
00992 cfl.addFragment(*it);
00993
00994 if (uniqueWindows_)
00995 {
00996 dataBufferDepthBytes_ -= (*it)->sizeBytes();
00997 it = dataBuffer_.erase(it);
00998 }
00999 else
01000 {
01001 ++it;
01002 }
01003 }
01004 requestReceiver_->RemoveRequest(req->first);
01005 checkOutOfOrderWindows(req->first);
01006 requestReceiver_->RemoveRequest(req->first);
01007 req = requests.erase(req);
01008 }
01009 else
01010 {
01011 ++req;
01012 }
01013 }
01014 }
01015
01016 bool artdaq::CommandableFragmentGenerator::applyRequests(artdaq::FragmentPtrs& frags)
01017 {
01018 if (check_stop() || exception())
01019 {
01020 return false;
01021 }
01022
01023
01024 if (mode_ == RequestMode::Ignored)
01025 {
01026 while (dataBufferDepthFragments_ <= 0)
01027 {
01028 if (check_stop() || exception() || !isHardwareOK_) return false;
01029 std::unique_lock<std::mutex> lock(dataBufferMutex_);
01030 dataCondition_.wait_for(lock, std::chrono::milliseconds(10), [this]() { return dataBufferDepthFragments_ > 0; });
01031 }
01032 }
01033 else
01034 {
01035 if ((check_stop() && requestReceiver_->size() == 0) || exception()) return false;
01036 checkDataBuffer();
01037
01038
01039 auto counter = 0;
01040
01041 while (requestReceiver_->size() == 0 && counter < 100)
01042 {
01043 if (check_stop() || exception()) return false;
01044
01045 checkDataBuffer();
01046
01047 requestReceiver_->WaitForRequests(10);
01048 counter++;
01049 }
01050 }
01051
01052 {
01053 std::unique_lock<std::mutex> dlk(dataBufferMutex_);
01054
01055 switch (mode_)
01056 {
01057 case RequestMode::Single:
01058 applyRequestsSingleMode(frags);
01059 break;
01060 case RequestMode::Window:
01061 applyRequestsWindowMode(frags);
01062 break;
01063 case RequestMode::Buffer:
01064 applyRequestsBufferMode(frags);
01065 break;
01066 case RequestMode::Ignored:
01067 default:
01068 applyRequestsIgnoredMode(frags);
01069 break;
01070 }
01071
01072 if (!data_thread_running_ || force_stop_)
01073 {
01074 TLOG(TLVL_INFO) << "Data thread has stopped; Clearing data buffer";
01075 dataBufferDepthBytes_ = 0;
01076 dataBuffer_.clear();
01077 }
01078
01079 getDataBufferStats();
01080 }
01081
01082 if (frags.size() > 0)
01083 TLOG(TLVL_APPLYREQUESTS) << "Finished Processing Event " << (*frags.begin())->sequenceID() << " for fragment_id " << fragment_id() << ".";
01084 return true;
01085 }
01086
01087 bool artdaq::CommandableFragmentGenerator::sendEmptyFragment(artdaq::FragmentPtrs& frags, size_t seqId, std::string desc)
01088 {
01089 TLOG(TLVL_WARNING) << desc << " sequence ID " << seqId << ", sending empty fragment";
01090 for (auto fid : fragment_ids_)
01091 {
01092 auto frag = new Fragment();
01093 frag->setSequenceID(seqId);
01094 frag->setFragmentID(fid);
01095 frag->setSystemType(Fragment::EmptyFragmentType);
01096 frags.emplace_back(FragmentPtr(frag));
01097 }
01098 return true;
01099 }
01100
01101 void artdaq::CommandableFragmentGenerator::sendEmptyFragments(artdaq::FragmentPtrs& frags, std::map<Fragment::sequence_id_t, Fragment::timestamp_t>& requests)
01102 {
01103 if (requests.size() > 0)
01104 {
01105 TLOG(TLVL_SENDEMPTYFRAGMENTS) << "Sending Empty Fragments for Sequence IDs from " << ev_counter() << " up to but not including " << requests.begin()->first;
01106 while (requests.begin()->first > ev_counter())
01107 {
01108 sendEmptyFragment(frags, ev_counter(), "Missed request for");
01109 ev_counter_inc(1, true);
01110 }
01111 }
01112 }
01113
01114 void artdaq::CommandableFragmentGenerator::checkOutOfOrderWindows(artdaq::Fragment::sequence_id_t seq)
01115 {
01116 windows_sent_ooo_[seq] = std::chrono::steady_clock::now();
01117
01118 auto it = windows_sent_ooo_.begin();
01119 while (it != windows_sent_ooo_.end())
01120 {
01121 if (seq == it->first && it->first == ev_counter())
01122 {
01123 TLOG(TLVL_CHECKWINDOWS) << "checkOutOfOrderWindows: Sequence ID matches ev_counter, incrementing ev_counter (" << ev_counter() << ")";
01124 ev_counter_inc(1, true);
01125 it = windows_sent_ooo_.erase(it);
01126 }
01127 else if (it->first <= ev_counter())
01128 {
01129 TLOG(TLVL_CHECKWINDOWS) << "checkOutOfOrderWindows: Data-taking has caught up to out-of-order window request " << it->first << ", removing from list. ev_counter=" << ev_counter();
01130 requestReceiver_->RemoveRequest(ev_counter());
01131 if (it->first == ev_counter()) ev_counter_inc(1, true);
01132 it = windows_sent_ooo_.erase(it);
01133 }
01134 else if (TimeUtils::GetElapsedTimeMicroseconds(it->second) > missing_request_window_timeout_us_)
01135 {
01136 TLOG(TLVL_CHECKWINDOWS) << "checkOutOfOrderWindows: Out-of-order window " << it->first << " has timed out, setting current sequence ID and removing from list";
01137 while (ev_counter() <= it->first)
01138 {
01139 if (ev_counter() < it->first) TLOG(TLVL_WARNING) << "Missed request for sequence ID " << ev_counter() << "! Will not send any data for this sequence ID!";
01140 requestReceiver_->RemoveRequest(ev_counter());
01141 ev_counter_inc(1, true);
01142 }
01143 windows_sent_ooo_.erase(windows_sent_ooo_.begin(), it);
01144 it = windows_sent_ooo_.erase(it);
01145 }
01146 else
01147 {
01148 TLOG(TLVL_CHECKWINDOWS) << "checkOutOfOrderWindows: Out-of-order window " << it->first << " waiting. Current event counter = " << ev_counter();
01149 ++it;
01150 }
01151 }
01152 }
01153