1 #define TRACE_NAME (app_name + "_CommandableFragmentGenerator").c_str() // include these 2 first -
2 #include "artdaq/DAQdata/Globals.hh"
4 #include "artdaq/Application/CommandableFragmentGenerator.hh"
6 #include <boost/exception/all.hpp>
7 #include <boost/throw_exception.hpp>
12 #include "canvas/Utilities/Exception.h"
13 #include "cetlib_except/exception.h"
14 #include "fhiclcpp/ParameterSet.h"
16 #include "artdaq-core/Utilities/SimpleLookupPolicy.hh"
17 #include "artdaq-core/Data/Fragment.hh"
18 #include "artdaq-core/Data/ContainerFragmentLoader.hh"
19 #include "artdaq-core/Utilities/ExceptionHandler.hh"
20 #include "artdaq-core/Utilities/TimeUtils.hh"
36 , staleTimeout_(Fragment::InvalidTimestamp)
37 , expectedType_(Fragment::EmptyFragmentType)
38 , maxFragmentCount_(std::numeric_limits<size_t>::max())
39 , uniqueWindows_(true)
40 , missing_request_(true)
41 , missing_request_time_()
42 , last_window_send_time_()
43 , last_window_send_time_set_(false)
45 , missing_request_window_timeout_us_(1000000)
46 , window_close_timeout_us_(2000000)
47 , useDataThread_(false)
48 , sleep_on_no_data_us_(0)
49 , data_thread_running_(false)
50 , dataBufferDepthFragments_(0)
51 , dataBufferDepthBytes_(0)
52 , maxDataBufferDepthFragments_(1000)
53 , maxDataBufferDepthBytes_(1000)
54 , useMonitoringThread_(false)
55 , monitoringInterval_(0)
56 , lastMonitoringCall_()
62 , timeout_(std::numeric_limits<uint64_t>::max())
63 , timestamp_(std::numeric_limits<uint64_t>::max())
67 , latest_exception_report_(
"none")
70 , instance_name_for_metrics_(
"FragmentGenerator")
71 , sleep_on_stop_us_(0)
76 , windowOffset_(ps.get<Fragment::timestamp_t>(
"request_window_offset", 0))
77 , windowWidth_(ps.get<Fragment::timestamp_t>(
"request_window_width", 0))
78 , staleTimeout_(ps.get<Fragment::timestamp_t>(
"stale_request_timeout", 0xFFFFFFFF))
79 , expectedType_(ps.get<Fragment::type_t>(
"expected_fragment_type", Fragment::type_t(Fragment::EmptyFragmentType)))
80 , uniqueWindows_(ps.get<bool>(
"request_windows_are_unique", true))
81 , missing_request_(false)
82 , missing_request_time_(decltype(missing_request_time_)::max())
83 , last_window_send_time_(decltype(last_window_send_time_)::max())
84 , last_window_send_time_set_(false)
86 , missing_request_window_timeout_us_(ps.get<size_t>(
"missing_request_window_timeout_us", 1000000))
87 , window_close_timeout_us_(ps.get<size_t>(
"window_close_timeout_us", 2000000))
88 , useDataThread_(ps.get<bool>(
"separate_data_thread", false))
89 , sleep_on_no_data_us_(ps.get<size_t>(
"sleep_on_no_data_us", 0))
90 , data_thread_running_(false)
91 , dataBufferDepthFragments_(0)
92 , dataBufferDepthBytes_(0)
93 , maxDataBufferDepthFragments_(ps.get<int>(
"data_buffer_depth_fragments", 1000))
94 , maxDataBufferDepthBytes_(ps.get<size_t>(
"data_buffer_depth_mb", 1000) * 1024 * 1024)
95 , useMonitoringThread_(ps.get<bool>(
"separate_monitoring_thread", false))
96 , monitoringInterval_(ps.get<int64_t>(
"hardware_poll_interval_us", 0))
97 , lastMonitoringCall_()
103 , timeout_(std::numeric_limits<uint64_t>::max())
104 , timestamp_(std::numeric_limits<uint64_t>::max())
105 , should_stop_(false)
108 , latest_exception_report_(
"none")
111 , sleep_on_stop_us_(0)
113 board_id_ = ps.get<
int>(
"board_id");
114 instance_name_for_metrics_ =
"BoardReader." + boost::lexical_cast<std::string>(board_id_);
116 fragment_ids_ = ps.get<std::vector<artdaq::Fragment::fragment_id_t>>(
"fragment_ids", std::vector<artdaq::Fragment::fragment_id_t>());
118 TLOG(TLVL_TRACE) <<
"artdaq::CommandableFragmentGenerator::CommandableFragmentGenerator(ps)" ;
121 if (fragment_id != -99)
123 if (fragment_ids_.size() != 0)
125 latest_exception_report_ =
"Error in CommandableFragmentGenerator: can't both define \"fragment_id\" and \"fragment_ids\" in FHiCL document";
126 throw cet::exception(latest_exception_report_);
130 fragment_ids_.emplace_back(fragment_id);
134 sleep_on_stop_us_ = ps.get<
int>(
"sleep_on_stop_us", 0);
136 dataBuffer_.emplace_back(FragmentPtr(
new Fragment()));
137 (*dataBuffer_.begin())->setSystemType(Fragment::EmptyFragmentType);
139 std::string modeString = ps.get<std::string>(
"request_mode",
"ignored");
140 if (modeString ==
"single" || modeString ==
"Single")
142 mode_ = RequestMode::Single;
144 else if (modeString.find(
"buffer") != std::string::npos || modeString.find(
"Buffer") != std::string::npos)
146 mode_ = RequestMode::Buffer;
148 else if (modeString ==
"window" || modeString ==
"Window")
150 mode_ = RequestMode::Window;
152 else if (modeString.find(
"ignore") != std::string::npos || modeString.find(
"Ignore") != std::string::npos)
154 mode_ = RequestMode::Ignored;
156 TLOG(TLVL_DEBUG) <<
"Request mode is " <<
printMode_() ;
158 if (mode_ != RequestMode::Ignored)
162 latest_exception_report_ =
"Error in CommandableFragmentGenerator: use_data_thread must be true when request_mode is not \"Ignored\"!";
163 throw cet::exception(latest_exception_report_);
178 TLOG(TLVL_DEBUG) <<
"Joining dataThread" ;
179 if (dataThread_.joinable()) dataThread_.join();
180 TLOG(TLVL_DEBUG) <<
"Joining monitoringThread" ;
181 if (monitoringThread_.joinable()) monitoringThread_.join();
182 requestReceiver_.reset(
nullptr);
189 if (check_stop()) usleep(sleep_on_stop_us_);
190 if (exception() || force_stop_)
return false;
192 if (!useMonitoringThread_ && monitoringInterval_ > 0)
194 TLOG(10) <<
"getNext: Checking whether to collect Monitoring Data" ;
195 auto now = std::chrono::steady_clock::now();
197 if (TimeUtils::GetElapsedTimeMicroseconds(lastMonitoringCall_, now) >= static_cast<size_t>(monitoringInterval_))
199 TLOG(10) <<
"getNext: Collecting Monitoring Data" ;
200 isHardwareOK_ = checkHWStatus_();
201 TLOG(10) <<
"getNext: isHardwareOK_ is now " << std::boolalpha << isHardwareOK_ ;
202 lastMonitoringCall_ = now;
208 std::lock_guard<std::mutex> lk(mutex_);
211 TLOG(TLVL_TRACE) <<
"getNext: Calling applyRequests" ;
212 result = applyRequests(output);
213 TLOG(TLVL_TRACE) <<
"getNext: Done with applyRequests result=" << std::boolalpha << result;
217 TLOG(TLVL_ERROR) <<
"Exception found in BoardReader with board ID " << board_id() <<
"; BoardReader will now return error status when queried";
218 throw cet::exception(
"CommandableFragmentGenerator") <<
"Exception found in BoardReader with board ID " << board_id() <<
"; BoardReader will now return error status when queried";
225 TLOG(TLVL_ERROR) <<
"Stopping CFG because the hardware reports bad status!" ;
228 TLOG(TLVL_TRACE) <<
"getNext: Calling getNext_ " << std::to_string(ev_counter()) ;
231 result = getNext_(output);
237 TLOG(TLVL_TRACE) <<
"getNext: Done with getNext_ " << std::to_string(ev_counter()) ;
240 catch (
const cet::exception& e)
242 latest_exception_report_ =
"cet::exception caught in getNext(): ";
243 latest_exception_report_.append(e.what());
244 TLOG(TLVL_ERROR) <<
"getNext: cet::exception caught: " << e ;
248 catch (
const boost::exception& e)
250 latest_exception_report_ =
"boost::exception caught in getNext(): ";
251 latest_exception_report_.append(boost::diagnostic_information(e));
252 TLOG(TLVL_ERROR) <<
"getNext: boost::exception caught: " << boost::diagnostic_information(e) ;
256 catch (
const std::exception& e)
258 latest_exception_report_ =
"std::exception caught in getNext(): ";
259 latest_exception_report_.append(e.what());
260 TLOG(TLVL_ERROR) <<
"getNext: std::exception caught: " << e.what() ;
266 latest_exception_report_ =
"Unknown exception caught in getNext().";
267 TLOG(TLVL_ERROR) <<
"getNext: unknown exception caught" ;
274 TLOG(TLVL_DEBUG) <<
"stopped " ;
282 TLOG(14) <<
"CFG::check_stop: should_stop=" << should_stop() <<
", useDataThread_=" << useDataThread_ <<
", exception status =" << int(exception()) ;
284 if (!should_stop())
return false;
285 if (!useDataThread_ || mode_ == RequestMode::Ignored)
return true;
286 if (force_stop_)
return true;
289 return !requestReceiver_->isRunning();
294 if (fragment_ids_.size() != 1)
296 throw cet::exception(
"Error in CommandableFragmentGenerator: can't call fragment_id() unless member fragment_ids_ vector is length 1");
300 return fragment_ids_[0];
306 if (force || mode_ == RequestMode::Ignored)
308 return ev_counter_.fetch_add(step);
310 return ev_counter_.load();
315 if (run < 0)
throw cet::exception(
"CommandableFragmentGenerator") <<
"negative run number";
318 timestamp_ = timestamp;
319 ev_counter_.store(1);
320 missing_request_ =
false;
321 should_stop_.store(
false);
322 exception_.store(
false);
325 latest_exception_report_ =
"none";
327 last_window_send_time_set_ =
false;
328 windows_sent_ooo_.clear();
332 std::unique_lock<std::mutex> lk(mutex_);
333 if (useDataThread_) startDataThread();
334 if (useMonitoringThread_) startMonitoringThread();
335 if (mode_ != RequestMode::Ignored && !requestReceiver_->isRunning()) requestReceiver_->startRequestReceiverThread();
340 TLOG(TLVL_DEBUG) <<
"Stop Command received." ;
343 timestamp_ = timestamp;
344 if (requestReceiver_ && requestReceiver_->isRunning()) requestReceiver_->stopRequestReceiverThread();
347 should_stop_.store(
true);
348 std::unique_lock<std::mutex> lk(mutex_);
350 TLOG(TLVL_DEBUG) <<
"Stop command complete.";
356 timestamp_ = timestamp;
357 if (requestReceiver_->isRunning()) requestReceiver_->stopRequestReceiverThread();
360 should_stop_.store(
true);
361 std::unique_lock<std::mutex> lk(mutex_);
369 timestamp_ = timestamp;
372 should_stop_ =
false;
379 std::unique_lock<std::mutex> lk(mutex_);
380 if (useDataThread_) startDataThread();
381 if (useMonitoringThread_) startMonitoringThread();
382 if (mode_ != RequestMode::Ignored && !requestReceiver_->isRunning()) requestReceiver_->startRequestReceiverThread();
387 std::lock_guard<std::mutex> lk(mutex_);
394 std::string childReport = reportSpecific(which);
395 if (childReport.length() > 0) {
return childReport; }
398 if (which ==
"latest_exception")
400 return latest_exception_report_;
404 childReport = report();
405 if (childReport.length() > 0) {
return childReport; }
408 std::string tmpString =
"The \"" + which +
"\" command is not ";
409 tmpString.append(
"currently supported by the ");
410 tmpString.append(metricsReportingInstanceName());
411 tmpString.append(
" fragment generator.");
416 void artdaq::CommandableFragmentGenerator::pauseNoMutex()
418 #pragma message "Using default implementation of CommandableFragmentGenerator::pauseNoMutex()"
421 void artdaq::CommandableFragmentGenerator::pause()
423 #pragma message "Using default implementation of CommandableFragmentGenerator::pause()"
426 void artdaq::CommandableFragmentGenerator::resume()
428 #pragma message "Using default implementation of CommandableFragmentGenerator::resume()"
431 std::string artdaq::CommandableFragmentGenerator::report()
433 #pragma message "Using default implementation of CommandableFragmentGenerator::report()"
437 std::string artdaq::CommandableFragmentGenerator::reportSpecific(std::string
const&)
439 #pragma message "Using default implementation of CommandableFragmentGenerator::reportSpecific(std::string)"
443 bool artdaq::CommandableFragmentGenerator::checkHWStatus_()
445 #pragma message "Using default implementation of CommandableFragmentGenerator::checkHWStatus_()"
451 #pragma message "Using default implementation of CommandableFragmentGenerator::metaCommand(std::string, std::string)"
457 if (dataThread_.joinable()) dataThread_.join();
458 TLOG(TLVL_INFO) <<
"Starting Data Receiver Thread" ;
464 if (monitoringThread_.joinable()) monitoringThread_.join();
465 TLOG(TLVL_INFO) <<
"Starting Hardware Monitoring Thread" ;
473 case RequestMode::Single:
475 case RequestMode::Buffer:
477 case RequestMode::Window:
479 case RequestMode::Ignored:
488 data_thread_running_ =
true;
493 TLOG(TLVL_DEBUG) <<
"getDataLoop: isHardwareOK is " << isHardwareOK_ <<
", aborting data thread" ;
494 data_thread_running_ =
false;
498 TLOG(13) <<
"getDataLoop: calling getNext_" ;
501 auto startdata = std::chrono::steady_clock::now();
505 data = getNext_(newDataBuffer_);
509 ExceptionHandler(ExceptionHandlerRethrow::no,
510 "Exception thrown by fragment generator in CommandableFragmentGenerator::getDataLoop; setting exception state to \"true\"");
513 data_thread_running_ =
false;
519 metricMan->sendMetric(
"Avg Data Acquisition Time", TimeUtils::GetElapsedTime(startdata),
"s", 3, artdaq::MetricMode::Average);
522 if (newDataBuffer_.size() == 0 && sleep_on_no_data_us_ > 0)
524 usleep(sleep_on_no_data_us_);
527 TLOG(15) <<
"Waiting for data buffer ready" ;
528 if (!waitForDataBufferReady())
return;
529 TLOG(15) <<
"Done waiting for data buffer ready" ;
531 TLOG(13) <<
"getDataLoop: processing data" ;
532 if (data && !force_stop_)
534 std::unique_lock<std::mutex> lock(dataBufferMutex_);
537 case RequestMode::Single:
539 while (newDataBuffer_.size() >= fragment_ids_.size())
542 auto it = newDataBuffer_.begin();
543 std::advance(it, fragment_ids_.size());
544 dataBuffer_.splice(dataBuffer_.end(), newDataBuffer_, newDataBuffer_.begin(), it);
547 case RequestMode::Buffer:
548 case RequestMode::Ignored:
549 case RequestMode::Window:
552 dataBuffer_.splice(dataBuffer_.end(), newDataBuffer_);
555 getDataBufferStats();
559 std::unique_lock<std::mutex> lock(dataBufferMutex_);
560 if (dataBuffer_.size() > 0)
562 dataCondition_.notify_all();
565 if (!data || force_stop_)
567 TLOG(TLVL_INFO) <<
"Data flow has stopped. Ending data collection thread" ;
568 data_thread_running_ =
false;
569 if (requestReceiver_) requestReceiver_->ClearRequests();
571 newDataBuffer_.clear();
579 auto startwait = std::chrono::steady_clock::now();
581 auto lastwaittime = 0ULL;
582 while (dataBufferIsTooLarge())
586 TLOG(TLVL_DEBUG) <<
"Run ended while waiting for buffer to shrink!" ;
587 std::unique_lock<std::mutex> lock(dataBufferMutex_);
588 getDataBufferStats();
589 dataCondition_.notify_all();
590 data_thread_running_ =
false;
593 auto waittime = TimeUtils::GetElapsedTimeMilliseconds(startwait);
595 if (first || (waittime != lastwaittime && waittime % 1000 == 0))
597 TLOG(TLVL_WARNING) <<
"Bad Omen: Data Buffer has exceeded its size limits. "
598 <<
"(seq_id=" << ev_counter()
599 <<
", frags=" << dataBufferDepthFragments_ <<
"/" << maxDataBufferDepthFragments_
600 <<
", szB=" << dataBufferDepthBytes_ <<
"/" << maxDataBufferDepthBytes_ <<
")" ;
601 TLOG(TLVL_TRACE) <<
"Bad Omen: Possible causes include requests not getting through or Ignored-mode BR issues" ;
604 if (waittime % 5 && waittime != lastwaittime)
606 TLOG(13) <<
"getDataLoop: Data Retreival paused for " << std::to_string(waittime) <<
" ms waiting for data buffer to drain" ;
608 lastwaittime = waittime;
616 return (maxDataBufferDepthFragments_ > 0 && dataBufferDepthFragments_ > maxDataBufferDepthFragments_) || (maxDataBufferDepthBytes_ > 0 && dataBufferDepthBytes_ > maxDataBufferDepthBytes_);
622 dataBufferDepthFragments_ = dataBuffer_.size();
624 TLOG(15) <<
"getDataBufferStats: Calculating buffer size" ;
625 for (
auto i = dataBuffer_.begin(); i != dataBuffer_.end(); ++i)
627 if (i->get() !=
nullptr)
629 acc += (*i)->sizeBytes();
632 dataBufferDepthBytes_ = acc;
636 TLOG(15) <<
"getDataBufferStats: Sending Metrics" ;
637 metricMan->sendMetric(
"Buffer Depth Fragments", dataBufferDepthFragments_.load(),
"fragments", 1, MetricMode::LastPoint);
638 metricMan->sendMetric(
"Buffer Depth Bytes", dataBufferDepthBytes_.load(),
"bytes", 1, MetricMode::LastPoint);
640 TLOG(15) <<
"getDataBufferStats: frags=" << dataBufferDepthFragments_.load() <<
"/" << maxDataBufferDepthFragments_
641 <<
", sz=" << std::to_string(dataBufferDepthBytes_.load()) <<
"/" << std::to_string(maxDataBufferDepthBytes_) ;
646 std::unique_lock<std::mutex> lock(dataBufferMutex_);
647 dataCondition_.wait_for(lock, std::chrono::milliseconds(10));
648 if (dataBufferDepthFragments_ > 0)
650 if ((mode_ == RequestMode::Buffer || mode_ == RequestMode::Window))
653 while (dataBufferIsTooLarge())
655 dataBuffer_.erase(dataBuffer_.begin());
656 getDataBufferStats();
658 if (dataBuffer_.size() > 0)
660 TLOG(17) <<
"Determining if Fragments can be dropped from data buffer" ;
661 Fragment::timestamp_t last = dataBuffer_.back()->timestamp();
662 Fragment::timestamp_t min = last > staleTimeout_ ? last - staleTimeout_ : 0;
663 for (
auto it = dataBuffer_.begin(); it != dataBuffer_.end();)
665 if ((*it)->timestamp() < min)
667 it = dataBuffer_.erase(it);
674 getDataBufferStats();
677 else if (mode_ == RequestMode::Single && dataBuffer_.size() > fragment_ids_.size())
680 while (dataBuffer_.size() > fragment_ids_.size())
682 dataBuffer_.erase(dataBuffer_.begin());
692 if (should_stop() || monitoringInterval_ <= 0)
694 TLOG(TLVL_DEBUG) <<
"getMonitoringDataLoop: should_stop() is " << std::boolalpha << should_stop()
695 <<
" and monitoringInterval is " << monitoringInterval_ <<
", returning" ;
698 TLOG(12) <<
"getMonitoringDataLoop: Determining whether to call checkHWStatus_" ;
700 auto now = std::chrono::steady_clock::now();
701 if (TimeUtils::GetElapsedTimeMicroseconds(lastMonitoringCall_, now) >= static_cast<size_t>(monitoringInterval_))
703 isHardwareOK_ = checkHWStatus_();
704 TLOG(12) <<
"getMonitoringDataLoop: isHardwareOK_ is now " << std::boolalpha << isHardwareOK_ ;
705 lastMonitoringCall_ = now;
707 usleep(monitoringInterval_ / 10);
714 TLOG(9) <<
"Mode is Ignored; Copying data to output" ;
715 std::move(dataBuffer_.begin(), dataBuffer_.end(), std::inserter(frags, frags.end()));
722 auto requests = requestReceiver_->GetRequests();
723 while (requests.size() > 1) {
725 requestReceiver_->RemoveRequest(requests.begin()->first);
726 requests.erase(requests.begin());
728 sendEmptyFragments(frags, requests);
731 if (requests.size() == 0 || !requests.count(ev_counter()))
return;
733 if (dataBuffer_.size() > 0)
735 TLOG(9) <<
"Mode is Single; Sending copy of last event" ;
736 for (
auto& fragptr : dataBuffer_)
739 auto frag = fragptr.get();
740 auto newfrag = std::unique_ptr<artdaq::Fragment>(
new Fragment(ev_counter(), frag->fragmentID()));
741 newfrag->resize(frag->size() - detail::RawFragmentHeader::num_words());
742 memcpy(newfrag->headerAddress(), frag->headerAddress(), frag->sizeBytes());
743 newfrag->setTimestamp(requests[ev_counter()]);
744 newfrag->setSequenceID(ev_counter());
745 frags.push_back(std::move(newfrag));
750 sendEmptyFragment(frags, ev_counter(),
"No data for");
752 requestReceiver_->RemoveRequest(ev_counter());
753 ev_counter_inc(1,
true);
759 auto requests = requestReceiver_->GetRequests();
760 while (requests.size() > 1) {
762 requestReceiver_->RemoveRequest(requests.begin()->first);
763 requests.erase(requests.begin());
765 sendEmptyFragments(frags, requests);
768 if (requests.size() == 0 || !requests.count(ev_counter()))
return;
770 TLOG(TLVL_DEBUG) <<
"Creating ContainerFragment for Buffered Fragments" ;
771 frags.emplace_back(
new artdaq::Fragment(ev_counter(), fragment_id()));
772 frags.back()->setTimestamp(requests[ev_counter()]);
773 ContainerFragmentLoader cfl(*frags.back());
774 cfl.set_missing_data(
false);
778 for (
auto it = dataBuffer_.begin(); it != dataBuffer_.end();)
780 TLOG(9) <<
"ApplyRequests: Adding Fragment with timestamp " << std::to_string((*it)->timestamp()) <<
" to Container" ;
781 cfl.addFragment(*it);
782 it = dataBuffer_.erase(it);
784 requestReceiver_->RemoveRequest(ev_counter());
785 ev_counter_inc(1,
true);
790 TLOG(10) <<
"applyRequestsWindowMode BEGIN";
791 if (!last_window_send_time_set_)
793 last_window_send_time_ = std::chrono::steady_clock::now();
794 last_window_send_time_set_ =
true;
797 auto requests = requestReceiver_->GetRequests();
798 bool now_have_desired_request = std::any_of(requests.begin(), requests.end(),
799 [
this](decltype(requests)::value_type& request) {
800 return request.first == ev_counter(); });
802 if (missing_request_)
804 if (!now_have_desired_request && TimeUtils::GetElapsedTimeMicroseconds(missing_request_time_) > missing_request_window_timeout_us_)
806 TLOG(TLVL_ERROR) <<
"Data-taking has paused for " << TimeUtils::GetElapsedTimeMicroseconds(missing_request_time_) <<
" us "
807 <<
"(> " << std::to_string(missing_request_window_timeout_us_) <<
" us) while waiting for missing data request messages."
808 <<
" Sending Empty Fragments for missing requests!" ;
809 sendEmptyFragments(frags, requests);
811 missing_request_ =
false;
812 missing_request_time_ = decltype(missing_request_time_)::max();
814 else if (now_have_desired_request) {
815 missing_request_ =
false;
816 missing_request_time_ = decltype(missing_request_time_)::max();
820 TLOG(10) <<
"applyRequestsWindowMode: Starting request processing";
821 for (
auto req = requests.begin(); req != requests.end();)
823 TLOG(10,
"CommandableFragmentGenerator") <<
"applyRequestsWindowMode: processing request with sequence ID " << \
824 req->first <<
", timestamp " << req->second;
827 while (req->first < ev_counter() && requests.size() > 0)
829 TLOG(10) <<
"applyRequestsWindowMode: Clearing passed request for sequence ID " << req->first;
830 requestReceiver_->RemoveRequest(req->first);
831 req = requests.erase(req);
833 if (requests.size() == 0)
break;
834 if (req->first > ev_counter())
836 if (!missing_request_)
838 missing_request_ =
true;
839 missing_request_time_ = std::chrono::steady_clock::now();
842 auto ts = req->second;
843 TLOG(9) <<
"ApplyRequests: Checking that data exists for request window " << std::to_string(req->first) ;
844 Fragment::timestamp_t min = ts > windowOffset_ ? ts - windowOffset_ : 0;
845 Fragment::timestamp_t max = min + windowWidth_;
846 TLOG(9) <<
"ApplyRequests: min is " << std::to_string(min) <<
", max is " << std::to_string(max)
847 <<
" and last point in buffer is " << std::to_string((dataBuffer_.size() > 0 ? dataBuffer_.back()->timestamp() : 0)) <<
" (sz=" << std::to_string(dataBuffer_.size()) <<
")" ;
848 bool windowClosed = dataBuffer_.size() > 0 && dataBuffer_.back()->timestamp() >= max;
849 bool windowTimeout = TimeUtils::GetElapsedTimeMicroseconds(last_window_send_time_) > window_close_timeout_us_;
852 TLOG(TLVL_WARNING) <<
"A timeout occurred waiting for data to close the request window (max=" << std::to_string(max)
853 <<
", buffer=" << std::to_string((dataBuffer_.size() > 0 ? dataBuffer_.back()->timestamp() : 0))
854 <<
" (if no buffer in memory, this is shown as a 0)). Time waiting: "
855 << TimeUtils::GetElapsedTimeMicroseconds(last_window_send_time_) <<
" us "
856 <<
"(> " << std::to_string(window_close_timeout_us_) <<
" us)." ;
858 if (missing_request_) {
859 TLOG(TLVL_ERROR) <<
"A Window timeout has occurred while there are pending requests. Sending empties." ;
860 sendEmptyFragments(frags, requests);
863 if (windowClosed || !data_thread_running_ || windowTimeout)
865 TLOG(TLVL_DEBUG) <<
"Creating ContainerFragment for Buffered or Window-requested Fragments" ;
866 frags.emplace_back(
new artdaq::Fragment(req->first, fragment_id()));
867 frags.back()->setTimestamp(ts);
868 ContainerFragmentLoader cfl(*frags.back());
870 if (!windowClosed) cfl.set_missing_data(
true);
871 if (dataBuffer_.size() > 0 && dataBuffer_.front()->timestamp() > min)
873 TLOG(TLVL_DEBUG) <<
"Request Window covers data that is either before data collection began or has fallen off the end of the buffer" ;
874 cfl.set_missing_data(
true);
879 for (
auto it = dataBuffer_.begin(); it != dataBuffer_.end();)
881 Fragment::timestamp_t fragT = (*it)->timestamp();
882 if (fragT < min || fragT > max || (fragT == max && windowWidth_ > 0))
888 TLOG(9) <<
"ApplyRequests: Adding Fragment with timestamp " << std::to_string((*it)->timestamp()) <<
" to Container" ;
889 cfl.addFragment(*it);
893 it = dataBuffer_.erase(it);
900 if (req->first == ev_counter())
902 ev_counter_inc(1,
true);
903 while (windows_sent_ooo_.count(ev_counter()))
905 TLOG(9) <<
"Data-taking has caught up to out-of-order window request " << ev_counter() <<
", removing from list" ;
906 windows_sent_ooo_.erase(windows_sent_ooo_.begin(), windows_sent_ooo_.find(ev_counter()));
907 ev_counter_inc(1,
true);
912 windows_sent_ooo_.insert(req->first);
914 requestReceiver_->RemoveRequest(req->first);
915 req = requests.erase(req);
916 last_window_send_time_ = std::chrono::steady_clock::now();
927 if (check_stop() || exception())
933 if (mode_ == RequestMode::Ignored)
935 while (dataBufferDepthFragments_ <= 0)
937 if (check_stop() || exception() || !isHardwareOK_)
return false;
938 std::unique_lock<std::mutex> lock(dataBufferMutex_);
939 dataCondition_.wait_for(lock, std::chrono::milliseconds(10), [
this]() {
return dataBufferDepthFragments_ > 0; });
944 if ((check_stop() && requestReceiver_->size() == 0) || exception())
return false;
950 while (requestReceiver_->size() == 0 && counter < 100)
952 if (check_stop() || exception())
return false;
956 requestReceiver_->WaitForRequests(10);
962 std::unique_lock<std::mutex> dlk(dataBufferMutex_);
966 case RequestMode::Single:
967 applyRequestsSingleMode(frags);
969 case RequestMode::Window:
970 applyRequestsWindowMode(frags);
972 case RequestMode::Buffer:
973 applyRequestsBufferMode(frags);
975 case RequestMode::Ignored:
977 applyRequestsIgnoredMode(frags);
981 getDataBufferStats();
984 if (frags.size() > 0)
985 TLOG(9) <<
"Finished Processing Event " << std::to_string(ev_counter() + 1) <<
" for fragment_id " << fragment_id() <<
"." ;
991 TLOG(TLVL_WARNING) << desc <<
" sequence ID " << seqId <<
", sending empty fragment" ;
992 for (
auto fid : fragment_ids_)
994 auto frag =
new Fragment();
995 frag->setSequenceID(seqId);
996 frag->setFragmentID(fid);
997 frag->setSystemType(Fragment::EmptyFragmentType);
998 frags.emplace_back(FragmentPtr(frag));
1005 if (requests.size() == 0 && windows_sent_ooo_.size() == 0)
return;
1007 if (requests.size() > 0) {
1008 TLOG(19) <<
"Sending Empty Fragments for Sequence IDs from " << ev_counter() <<
" up to but not including " << requests.begin()->first ;
1009 while (requests.begin()->first > ev_counter())
1011 sendEmptyFragment(frags, ev_counter(),
"Missed request for");
1012 ev_counter_inc(1,
true);
1015 else if (windows_sent_ooo_.size() > 0)
1017 TLOG(19) <<
"Sending Empty Fragments for Sequence IDs from " << ev_counter() <<
" up to but not including " << *windows_sent_ooo_.begin() ;
1018 while (*windows_sent_ooo_.begin() > ev_counter())
1020 sendEmptyFragment(frags, ev_counter(),
"Missed request for");
1021 ev_counter_inc(1,
true);
1024 while (windows_sent_ooo_.count(ev_counter()))
1026 TLOG(19) <<
"Data-taking has caught up to out-of-order window request " << ev_counter() <<
", removing from list" ;
1027 windows_sent_ooo_.erase(windows_sent_ooo_.begin(), windows_sent_ooo_.find(ev_counter()));
1028 ev_counter_inc(1,
true);
int fragment_id() const
Get the current Fragment ID, if there is only one.
void applyRequestsSingleMode(artdaq::FragmentPtrs &frags)
Create fragments using data buffer for request mode Single. Precondition: dataBufferMutex_ and reques...
virtual ~CommandableFragmentGenerator()
CommandableFragmentGenerator Destructor.
void applyRequestsBufferMode(artdaq::FragmentPtrs &frags)
Create fragments using data buffer for request mode Buffer. Precondition: dataBufferMutex_ and reques...
bool sendEmptyFragment(FragmentPtrs &frags, size_t sequenceId, std::string desc)
Send an EmptyFragmentType Fragment.
void getMonitoringDataLoop()
This function regularly calls checkHWStatus_(), and sets the isHardwareOK flag accordingly.
void startDataThread()
Function that launches the data thread (getDataLoop())
std::string ReportCmd(std::string const &which="")
Get a report about a user-specified run-time quantity.
virtual bool metaCommand(std::string const &command, std::string const &arg)
The meta-command is used for implementing user-specific commands in a CommandableFragmentGenerator.
bool dataBufferIsTooLarge()
Test the configured constraints on the data buffer.
void StopCmd(uint64_t timeout, uint64_t timestamp)
Stop the CommandableFragmentGenerator.
void applyRequestsWindowMode(artdaq::FragmentPtrs &frags)
Create fragments using data buffer for request mode Window. Precondition: dataBufferMutex_ and reques...
void StartCmd(int run, uint64_t timeout, uint64_t timestamp)
Start the CommandableFragmentGenerator.
bool check_stop()
Routine used by applyRequests to make sure that all outstanding requests have been fulfilled before r...
void ResumeCmd(uint64_t timeout, uint64_t timestamp)
Resume the CommandableFragmentGenerator.
CommandableFragmentGenerator()
CommandableFragmentGenerator default constructor.
bool getNext(FragmentPtrs &output) overridefinal
getNext calls either applyRequests or getNext_ to get any data that is ready to be sent to the EventB...
bool waitForDataBufferReady()
Wait for the data buffer to drain (dataBufferIsTooLarge returns false), periodically reporting status...
size_t ev_counter_inc(size_t step=1, bool force=false)
Increment the event counter, if the current RequestMode allows it.
void applyRequestsIgnoredMode(artdaq::FragmentPtrs &frags)
Create fragments using data buffer for request mode Ignored. Precondition: dataBufferMutex_ and reque...
void PauseCmd(uint64_t timeout, uint64_t timestamp)
Pause the CommandableFragmentGenerator.
void getDataLoop()
When separate_data_thread is set to true, this loop repeatedly calls getNext_ and adds returned Fragm...
void sendEmptyFragments(FragmentPtrs &frags, std::map< Fragment::sequence_id_t, Fragment::timestamp_t > &requests)
This function is for Buffered and Single request modes, as they can only respond to one data request ...
void startMonitoringThread()
Function that launches the monitoring thread (getMonitoringDataLoop())
void checkDataBuffer()
Perform data buffer pruning operations. If the RequestMode is Single, removes all but the latest Frag...
std::string printMode_()
Return the string representation of the current RequestMode.
void getDataBufferStats()
Calculate the size of the dataBuffer and report appropriate metrics.
bool applyRequests(FragmentPtrs &output)
See if any requests have been received, and add the corresponding data Fragment objects to the output...
void joinThreads()
Join any data-taking threads. Should be called when destructing CommandableFragmentGenerator.