1 #define TRACE_NAME (app_name + "_CommandableFragmentGenerator").c_str() // include these 2 first -
2 #include "artdaq/DAQdata/Globals.hh"
4 #include "artdaq/Application/CommandableFragmentGenerator.hh"
6 #include <boost/exception/all.hpp>
7 #include <boost/throw_exception.hpp>
12 #include "canvas/Utilities/Exception.h"
13 #include "cetlib_except/exception.h"
14 #include "fhiclcpp/ParameterSet.h"
16 #include "artdaq-core/Utilities/SimpleLookupPolicy.hh"
17 #include "artdaq-core/Data/Fragment.hh"
18 #include "artdaq-core/Data/ContainerFragmentLoader.hh"
19 #include "artdaq-core/Utilities/ExceptionHandler.hh"
20 #include "artdaq-core/Utilities/TimeUtils.hh"
36 , staleTimeout_(Fragment::InvalidTimestamp)
37 , expectedType_(Fragment::EmptyFragmentType)
38 , maxFragmentCount_(std::numeric_limits<size_t>::max())
39 , uniqueWindows_(true)
40 , missing_request_(true)
41 , missing_request_time_()
42 , last_window_send_time_()
43 , last_window_send_time_set_(false)
45 , missing_request_window_timeout_us_(1000000)
46 , window_close_timeout_us_(2000000)
47 , useDataThread_(false)
48 , sleep_on_no_data_us_(0)
49 , data_thread_running_(false)
50 , dataBufferDepthFragments_(0)
51 , dataBufferDepthBytes_(0)
52 , maxDataBufferDepthFragments_(1000)
53 , maxDataBufferDepthBytes_(1000)
54 , useMonitoringThread_(false)
55 , monitoringInterval_(0)
56 , lastMonitoringCall_()
62 , timeout_(std::numeric_limits<uint64_t>::max())
63 , timestamp_(std::numeric_limits<uint64_t>::max())
67 , latest_exception_report_(
"none")
70 , instance_name_for_metrics_(
"FragmentGenerator")
71 , sleep_on_stop_us_(0)
76 , windowOffset_(ps.get<Fragment::timestamp_t>(
"request_window_offset", 0))
77 , windowWidth_(ps.get<Fragment::timestamp_t>(
"request_window_width", 0))
78 , staleTimeout_(ps.get<Fragment::timestamp_t>(
"stale_request_timeout", 0xFFFFFFFF))
79 , expectedType_(ps.get<Fragment::type_t>(
"expected_fragment_type", Fragment::type_t(Fragment::EmptyFragmentType)))
80 , uniqueWindows_(ps.get<bool>(
"request_windows_are_unique", true))
81 , missing_request_(false)
82 , missing_request_time_(decltype(missing_request_time_)::max())
83 , last_window_send_time_(decltype(last_window_send_time_)::max())
84 , last_window_send_time_set_(false)
86 , missing_request_window_timeout_us_(ps.get<size_t>(
"missing_request_window_timeout_us", 1000000))
87 , window_close_timeout_us_(ps.get<size_t>(
"window_close_timeout_us", 2000000))
88 , useDataThread_(ps.get<bool>(
"separate_data_thread", false))
89 , sleep_on_no_data_us_(ps.get<size_t>(
"sleep_on_no_data_us", 0))
90 , data_thread_running_(false)
91 , dataBufferDepthFragments_(0)
92 , dataBufferDepthBytes_(0)
93 , maxDataBufferDepthFragments_(ps.get<int>(
"data_buffer_depth_fragments", 1000))
94 , maxDataBufferDepthBytes_(ps.get<size_t>(
"data_buffer_depth_mb", 1000) * 1024 * 1024)
95 , useMonitoringThread_(ps.get<bool>(
"separate_monitoring_thread", false))
96 , monitoringInterval_(ps.get<int64_t>(
"hardware_poll_interval_us", 0))
97 , lastMonitoringCall_()
103 , timeout_(std::numeric_limits<uint64_t>::max())
104 , timestamp_(std::numeric_limits<uint64_t>::max())
105 , should_stop_(false)
108 , latest_exception_report_(
"none")
111 , sleep_on_stop_us_(0)
113 board_id_ = ps.get<
int>(
"board_id");
114 instance_name_for_metrics_ =
"BoardReader." + boost::lexical_cast<std::string>(board_id_);
116 fragment_ids_ = ps.get<std::vector<artdaq::Fragment::fragment_id_t>>(
"fragment_ids", std::vector<artdaq::Fragment::fragment_id_t>());
118 TLOG(TLVL_TRACE) <<
"artdaq::CommandableFragmentGenerator::CommandableFragmentGenerator(ps)" ;
121 if (fragment_id != -99)
123 if (fragment_ids_.size() != 0)
125 latest_exception_report_ =
"Error in CommandableFragmentGenerator: can't both define \"fragment_id\" and \"fragment_ids\" in FHiCL document";
126 throw cet::exception(latest_exception_report_);
130 fragment_ids_.emplace_back(fragment_id);
134 sleep_on_stop_us_ = ps.get<
int>(
"sleep_on_stop_us", 0);
136 dataBuffer_.emplace_back(FragmentPtr(
new Fragment()));
137 (*dataBuffer_.begin())->setSystemType(Fragment::EmptyFragmentType);
139 std::string modeString = ps.get<std::string>(
"request_mode",
"ignored");
140 if (modeString ==
"single" || modeString ==
"Single")
142 mode_ = RequestMode::Single;
144 else if (modeString.find(
"buffer") != std::string::npos || modeString.find(
"Buffer") != std::string::npos)
146 mode_ = RequestMode::Buffer;
148 else if (modeString ==
"window" || modeString ==
"Window")
150 mode_ = RequestMode::Window;
152 else if (modeString.find(
"ignore") != std::string::npos || modeString.find(
"Ignore") != std::string::npos)
154 mode_ = RequestMode::Ignored;
156 TLOG(TLVL_DEBUG) <<
"Request mode is " <<
printMode_() ;
158 if (mode_ != RequestMode::Ignored)
162 latest_exception_report_ =
"Error in CommandableFragmentGenerator: use_data_thread must be true when request_mode is not \"Ignored\"!";
163 throw cet::exception(latest_exception_report_);
178 TLOG(TLVL_DEBUG) <<
"Joining dataThread" ;
179 if (dataThread_.joinable()) dataThread_.join();
180 TLOG(TLVL_DEBUG) <<
"Joining monitoringThread" ;
181 if (monitoringThread_.joinable()) monitoringThread_.join();
182 requestReceiver_.reset(
nullptr);
189 if (check_stop()) usleep(sleep_on_stop_us_);
190 if (exception() || force_stop_)
return false;
192 if (!useMonitoringThread_ && monitoringInterval_ > 0)
194 TLOG(10) <<
"getNext: Checking whether to collect Monitoring Data" ;
195 auto now = std::chrono::steady_clock::now();
197 if (TimeUtils::GetElapsedTimeMicroseconds(lastMonitoringCall_, now) >= static_cast<size_t>(monitoringInterval_))
199 TLOG(10) <<
"getNext: Collecting Monitoring Data" ;
200 isHardwareOK_ = checkHWStatus_();
201 TLOG(10) <<
"getNext: isHardwareOK_ is now " << std::boolalpha << isHardwareOK_ ;
202 lastMonitoringCall_ = now;
208 std::lock_guard<std::mutex> lk(mutex_);
211 TLOG(TLVL_TRACE) <<
"getNext: Calling applyRequests" ;
212 result = applyRequests(output);
213 TLOG(TLVL_TRACE) <<
"getNext: Done with applyRequests result=" << std::boolalpha << result;
217 TLOG(TLVL_ERROR) <<
"Exception found in BoardReader with board ID " << board_id() <<
"; BoardReader will now return error status when queried";
218 throw cet::exception(
"CommandableFragmentGenerator") <<
"Exception found in BoardReader with board ID " << board_id() <<
"; BoardReader will now return error status when queried";
225 TLOG(TLVL_ERROR) <<
"Stopping CFG because the hardware reports bad status!" ;
228 TLOG(TLVL_TRACE) <<
"getNext: Calling getNext_ " << std::to_string(ev_counter()) ;
231 result = getNext_(output);
237 TLOG(TLVL_TRACE) <<
"getNext: Done with getNext_ " << std::to_string(ev_counter()) ;
238 for (
auto dataIter = output.begin(); dataIter != output.end(); ++dataIter)
240 TLOG(20) <<
"getNext: getNext_() returned fragment with sequenceID = " << (*dataIter)->sequenceID()
241 <<
", timestamp = " << (*dataIter)->timestamp() <<
", and sizeBytes = " << (*dataIter)->sizeBytes();
245 catch (
const cet::exception& e)
247 latest_exception_report_ =
"cet::exception caught in getNext(): ";
248 latest_exception_report_.append(e.what());
249 TLOG(TLVL_ERROR) <<
"getNext: cet::exception caught: " << e ;
253 catch (
const boost::exception& e)
255 latest_exception_report_ =
"boost::exception caught in getNext(): ";
256 latest_exception_report_.append(boost::diagnostic_information(e));
257 TLOG(TLVL_ERROR) <<
"getNext: boost::exception caught: " << boost::diagnostic_information(e) ;
261 catch (
const std::exception& e)
263 latest_exception_report_ =
"std::exception caught in getNext(): ";
264 latest_exception_report_.append(e.what());
265 TLOG(TLVL_ERROR) <<
"getNext: std::exception caught: " << e.what() ;
271 latest_exception_report_ =
"Unknown exception caught in getNext().";
272 TLOG(TLVL_ERROR) <<
"getNext: unknown exception caught" ;
279 TLOG(TLVL_DEBUG) <<
"stopped " ;
282 if (metricMan && !output.empty()) {
284 auto timestamp = output.front()->timestamp();
286 if (output.size() > 1) {
287 for (
auto& outputfrag : output ) {
288 if (outputfrag->timestamp() > timestamp) {
289 timestamp = outputfrag->timestamp();
294 metricMan->sendMetric(
"Last Timestamp", timestamp,
"Ticks", 1,
295 MetricMode::LastPoint, app_name);
303 TLOG(14) <<
"CFG::check_stop: should_stop=" << should_stop() <<
", useDataThread_=" << useDataThread_ <<
", exception status =" << int(exception()) ;
305 if (!should_stop())
return false;
306 if (!useDataThread_ || mode_ == RequestMode::Ignored)
return true;
307 if (force_stop_)
return true;
310 return !requestReceiver_->isRunning();
315 if (fragment_ids_.size() != 1)
317 throw cet::exception(
"Error in CommandableFragmentGenerator: can't call fragment_id() unless member fragment_ids_ vector is length 1");
321 return fragment_ids_[0];
327 if (force || mode_ == RequestMode::Ignored)
329 return ev_counter_.fetch_add(step);
331 return ev_counter_.load();
336 if (run < 0)
throw cet::exception(
"CommandableFragmentGenerator") <<
"negative run number";
339 timestamp_ = timestamp;
340 ev_counter_.store(1);
341 missing_request_ =
false;
342 should_stop_.store(
false);
343 exception_.store(
false);
346 latest_exception_report_ =
"none";
348 last_window_send_time_set_ =
false;
349 windows_sent_ooo_.clear();
353 std::unique_lock<std::mutex> lk(mutex_);
354 if (useDataThread_) startDataThread();
355 if (useMonitoringThread_) startMonitoringThread();
356 if (mode_ != RequestMode::Ignored && !requestReceiver_->isRunning()) requestReceiver_->startRequestReceiverThread();
361 TLOG(TLVL_DEBUG) <<
"Stop Command received." ;
364 timestamp_ = timestamp;
365 if (requestReceiver_ && requestReceiver_->isRunning()) requestReceiver_->stopRequestReceiverThread();
368 should_stop_.store(
true);
369 std::unique_lock<std::mutex> lk(mutex_);
371 TLOG(TLVL_DEBUG) <<
"Stop command complete.";
377 timestamp_ = timestamp;
378 if (requestReceiver_->isRunning()) requestReceiver_->stopRequestReceiverThread();
381 should_stop_.store(
true);
382 std::unique_lock<std::mutex> lk(mutex_);
390 timestamp_ = timestamp;
393 should_stop_ =
false;
400 std::unique_lock<std::mutex> lk(mutex_);
401 if (useDataThread_) startDataThread();
402 if (useMonitoringThread_) startMonitoringThread();
403 if (mode_ != RequestMode::Ignored && !requestReceiver_->isRunning()) requestReceiver_->startRequestReceiverThread();
408 std::lock_guard<std::mutex> lk(mutex_);
415 std::string childReport = reportSpecific(which);
416 if (childReport.length() > 0) {
return childReport; }
419 if (which ==
"latest_exception")
421 return latest_exception_report_;
425 childReport = report();
426 if (childReport.length() > 0) {
return childReport; }
429 std::string tmpString =
"The \"" + which +
"\" command is not ";
430 tmpString.append(
"currently supported by the ");
431 tmpString.append(metricsReportingInstanceName());
432 tmpString.append(
" fragment generator.");
437 void artdaq::CommandableFragmentGenerator::pauseNoMutex()
439 #pragma message "Using default implementation of CommandableFragmentGenerator::pauseNoMutex()"
442 void artdaq::CommandableFragmentGenerator::pause()
444 #pragma message "Using default implementation of CommandableFragmentGenerator::pause()"
447 void artdaq::CommandableFragmentGenerator::resume()
449 #pragma message "Using default implementation of CommandableFragmentGenerator::resume()"
452 std::string artdaq::CommandableFragmentGenerator::report()
454 #pragma message "Using default implementation of CommandableFragmentGenerator::report()"
458 std::string artdaq::CommandableFragmentGenerator::reportSpecific(std::string
const&)
460 #pragma message "Using default implementation of CommandableFragmentGenerator::reportSpecific(std::string)"
464 bool artdaq::CommandableFragmentGenerator::checkHWStatus_()
466 #pragma message "Using default implementation of CommandableFragmentGenerator::checkHWStatus_()"
472 #pragma message "Using default implementation of CommandableFragmentGenerator::metaCommand(std::string, std::string)"
478 if (dataThread_.joinable()) dataThread_.join();
479 TLOG(TLVL_INFO) <<
"Starting Data Receiver Thread" ;
485 if (monitoringThread_.joinable()) monitoringThread_.join();
486 TLOG(TLVL_INFO) <<
"Starting Hardware Monitoring Thread" ;
494 case RequestMode::Single:
496 case RequestMode::Buffer:
498 case RequestMode::Window:
500 case RequestMode::Ignored:
509 data_thread_running_ =
true;
514 TLOG(TLVL_DEBUG) <<
"getDataLoop: isHardwareOK is " << isHardwareOK_ <<
", aborting data thread" ;
515 data_thread_running_ =
false;
519 TLOG(13) <<
"getDataLoop: calling getNext_" ;
522 auto startdata = std::chrono::steady_clock::now();
526 data = getNext_(newDataBuffer_);
530 ExceptionHandler(ExceptionHandlerRethrow::no,
531 "Exception thrown by fragment generator in CommandableFragmentGenerator::getDataLoop; setting exception state to \"true\"");
534 data_thread_running_ =
false;
537 for (
auto dataIter = newDataBuffer_.begin(); dataIter != newDataBuffer_.end(); ++dataIter)
539 TLOG(20) <<
"getDataLoop: getNext_() returned fragment with sequenceID = " << (*dataIter)->sequenceID()
540 <<
", timestamp = " << (*dataIter)->timestamp() <<
", and sizeBytes = " << (*dataIter)->sizeBytes();
545 metricMan->sendMetric(
"Avg Data Acquisition Time", TimeUtils::GetElapsedTime(startdata),
"s", 3, artdaq::MetricMode::Average);
548 if (newDataBuffer_.size() == 0 && sleep_on_no_data_us_ > 0)
550 usleep(sleep_on_no_data_us_);
553 TLOG(15) <<
"Waiting for data buffer ready" ;
554 if (!waitForDataBufferReady())
return;
555 TLOG(15) <<
"Done waiting for data buffer ready" ;
557 TLOG(13) <<
"getDataLoop: processing data" ;
558 if (data && !force_stop_)
560 std::unique_lock<std::mutex> lock(dataBufferMutex_);
563 case RequestMode::Single:
565 while (newDataBuffer_.size() >= fragment_ids_.size())
568 auto it = newDataBuffer_.begin();
569 std::advance(it, fragment_ids_.size());
570 dataBuffer_.splice(dataBuffer_.end(), newDataBuffer_, newDataBuffer_.begin(), it);
573 case RequestMode::Buffer:
574 case RequestMode::Ignored:
575 case RequestMode::Window:
578 dataBuffer_.splice(dataBuffer_.end(), newDataBuffer_);
581 getDataBufferStats();
585 std::unique_lock<std::mutex> lock(dataBufferMutex_);
586 if (dataBuffer_.size() > 0)
588 dataCondition_.notify_all();
591 if (!data || force_stop_)
593 TLOG(TLVL_INFO) <<
"Data flow has stopped. Ending data collection thread" ;
594 data_thread_running_ =
false;
595 if (requestReceiver_) requestReceiver_->ClearRequests();
597 newDataBuffer_.clear();
605 auto startwait = std::chrono::steady_clock::now();
607 auto lastwaittime = 0ULL;
608 while (dataBufferIsTooLarge())
612 TLOG(TLVL_DEBUG) <<
"Run ended while waiting for buffer to shrink!" ;
613 std::unique_lock<std::mutex> lock(dataBufferMutex_);
614 getDataBufferStats();
615 dataCondition_.notify_all();
616 data_thread_running_ =
false;
619 auto waittime = TimeUtils::GetElapsedTimeMilliseconds(startwait);
621 if (first || (waittime != lastwaittime && waittime % 1000 == 0))
623 TLOG(TLVL_WARNING) <<
"Bad Omen: Data Buffer has exceeded its size limits. "
624 <<
"(seq_id=" << ev_counter()
625 <<
", frags=" << dataBufferDepthFragments_ <<
"/" << maxDataBufferDepthFragments_
626 <<
", szB=" << dataBufferDepthBytes_ <<
"/" << maxDataBufferDepthBytes_ <<
")" ;
627 TLOG(TLVL_TRACE) <<
"Bad Omen: Possible causes include requests not getting through or Ignored-mode BR issues" ;
630 if (waittime % 5 && waittime != lastwaittime)
632 TLOG(13) <<
"getDataLoop: Data Retreival paused for " << std::to_string(waittime) <<
" ms waiting for data buffer to drain" ;
634 lastwaittime = waittime;
642 return (maxDataBufferDepthFragments_ > 0 && dataBufferDepthFragments_ > maxDataBufferDepthFragments_) || (maxDataBufferDepthBytes_ > 0 && dataBufferDepthBytes_ > maxDataBufferDepthBytes_);
648 dataBufferDepthFragments_ = dataBuffer_.size();
650 TLOG(15) <<
"getDataBufferStats: Calculating buffer size" ;
651 for (
auto i = dataBuffer_.begin(); i != dataBuffer_.end(); ++i)
653 if (i->get() !=
nullptr)
655 acc += (*i)->sizeBytes();
658 dataBufferDepthBytes_ = acc;
662 TLOG(15) <<
"getDataBufferStats: Sending Metrics" ;
663 metricMan->sendMetric(
"Buffer Depth Fragments", dataBufferDepthFragments_.load(),
"fragments", 1, MetricMode::LastPoint);
664 metricMan->sendMetric(
"Buffer Depth Bytes", dataBufferDepthBytes_.load(),
"bytes", 1, MetricMode::LastPoint);
666 TLOG(15) <<
"getDataBufferStats: frags=" << dataBufferDepthFragments_.load() <<
"/" << maxDataBufferDepthFragments_
667 <<
", sz=" << std::to_string(dataBufferDepthBytes_.load()) <<
"/" << std::to_string(maxDataBufferDepthBytes_) ;
672 std::unique_lock<std::mutex> lock(dataBufferMutex_);
673 dataCondition_.wait_for(lock, std::chrono::milliseconds(10));
674 if (dataBufferDepthFragments_ > 0)
676 if ((mode_ == RequestMode::Buffer || mode_ == RequestMode::Window))
679 while (dataBufferIsTooLarge())
681 dataBuffer_.erase(dataBuffer_.begin());
682 getDataBufferStats();
684 if (dataBuffer_.size() > 0)
686 TLOG(17) <<
"Determining if Fragments can be dropped from data buffer" ;
687 Fragment::timestamp_t last = dataBuffer_.back()->timestamp();
688 Fragment::timestamp_t min = last > staleTimeout_ ? last - staleTimeout_ : 0;
689 for (
auto it = dataBuffer_.begin(); it != dataBuffer_.end();)
691 if ((*it)->timestamp() < min)
693 it = dataBuffer_.erase(it);
700 getDataBufferStats();
703 else if (mode_ == RequestMode::Single && dataBuffer_.size() > fragment_ids_.size())
706 while (dataBuffer_.size() > fragment_ids_.size())
708 dataBuffer_.erase(dataBuffer_.begin());
718 if (should_stop() || monitoringInterval_ <= 0)
720 TLOG(TLVL_DEBUG) <<
"getMonitoringDataLoop: should_stop() is " << std::boolalpha << should_stop()
721 <<
" and monitoringInterval is " << monitoringInterval_ <<
", returning" ;
724 TLOG(12) <<
"getMonitoringDataLoop: Determining whether to call checkHWStatus_" ;
726 auto now = std::chrono::steady_clock::now();
727 if (TimeUtils::GetElapsedTimeMicroseconds(lastMonitoringCall_, now) >= static_cast<size_t>(monitoringInterval_))
729 isHardwareOK_ = checkHWStatus_();
730 TLOG(12) <<
"getMonitoringDataLoop: isHardwareOK_ is now " << std::boolalpha << isHardwareOK_ ;
731 lastMonitoringCall_ = now;
733 usleep(monitoringInterval_ / 10);
740 TLOG(9) <<
"Mode is Ignored; Copying data to output" ;
741 std::move(dataBuffer_.begin(), dataBuffer_.end(), std::inserter(frags, frags.end()));
748 auto requests = requestReceiver_->GetRequests();
749 while (requests.size() > 1) {
751 requestReceiver_->RemoveRequest(requests.begin()->first);
752 requests.erase(requests.begin());
754 sendEmptyFragments(frags, requests);
757 if (requests.size() == 0 || !requests.count(ev_counter()))
return;
759 if (dataBuffer_.size() > 0)
761 TLOG(9) <<
"Mode is Single; Sending copy of last event" ;
762 for (
auto& fragptr : dataBuffer_)
765 auto frag = fragptr.get();
766 auto newfrag = std::unique_ptr<artdaq::Fragment>(
new Fragment(ev_counter(), frag->fragmentID()));
767 newfrag->resize(frag->size() - detail::RawFragmentHeader::num_words());
768 memcpy(newfrag->headerAddress(), frag->headerAddress(), frag->sizeBytes());
769 newfrag->setTimestamp(requests[ev_counter()]);
770 newfrag->setSequenceID(ev_counter());
771 frags.push_back(std::move(newfrag));
776 sendEmptyFragment(frags, ev_counter(),
"No data for");
778 requestReceiver_->RemoveRequest(ev_counter());
779 ev_counter_inc(1,
true);
785 auto requests = requestReceiver_->GetRequests();
786 while (requests.size() > 1) {
788 requestReceiver_->RemoveRequest(requests.begin()->first);
789 requests.erase(requests.begin());
791 sendEmptyFragments(frags, requests);
794 if (requests.size() == 0 || !requests.count(ev_counter()))
return;
796 TLOG(TLVL_DEBUG) <<
"Creating ContainerFragment for Buffered Fragments" ;
797 frags.emplace_back(
new artdaq::Fragment(ev_counter(), fragment_id()));
798 frags.back()->setTimestamp(requests[ev_counter()]);
799 ContainerFragmentLoader cfl(*frags.back());
800 cfl.set_missing_data(
false);
804 for (
auto it = dataBuffer_.begin(); it != dataBuffer_.end();)
806 TLOG(9) <<
"ApplyRequests: Adding Fragment with timestamp " << std::to_string((*it)->timestamp()) <<
" to Container" ;
807 cfl.addFragment(*it);
808 it = dataBuffer_.erase(it);
810 requestReceiver_->RemoveRequest(ev_counter());
811 ev_counter_inc(1,
true);
816 TLOG(10) <<
"applyRequestsWindowMode BEGIN";
817 if (!last_window_send_time_set_)
819 last_window_send_time_ = std::chrono::steady_clock::now();
820 last_window_send_time_set_ =
true;
823 auto requests = requestReceiver_->GetRequests();
824 bool now_have_desired_request = std::any_of(requests.begin(), requests.end(),
825 [
this](decltype(requests)::value_type& request) {
826 return request.first == ev_counter(); });
828 if (missing_request_)
830 if (!now_have_desired_request && TimeUtils::GetElapsedTimeMicroseconds(missing_request_time_) > missing_request_window_timeout_us_)
832 TLOG(TLVL_ERROR) <<
"Data-taking has paused for " << TimeUtils::GetElapsedTimeMicroseconds(missing_request_time_) <<
" us "
833 <<
"(> " << std::to_string(missing_request_window_timeout_us_) <<
" us) while waiting for missing data request messages."
834 <<
" Sending Empty Fragments for missing requests!" ;
835 sendEmptyFragments(frags, requests);
837 missing_request_ =
false;
838 missing_request_time_ = decltype(missing_request_time_)::max();
840 else if (now_have_desired_request) {
841 missing_request_ =
false;
842 missing_request_time_ = decltype(missing_request_time_)::max();
846 TLOG(10) <<
"applyRequestsWindowMode: Starting request processing";
847 for (
auto req = requests.begin(); req != requests.end();)
849 TLOG(10,
"CommandableFragmentGenerator") <<
"applyRequestsWindowMode: processing request with sequence ID " << \
850 req->first <<
", timestamp " << req->second;
853 while (req->first < ev_counter() && requests.size() > 0)
855 TLOG(10) <<
"applyRequestsWindowMode: Clearing passed request for sequence ID " << req->first;
856 requestReceiver_->RemoveRequest(req->first);
857 req = requests.erase(req);
859 if (requests.size() == 0)
break;
860 if (req->first > ev_counter())
862 if (!missing_request_)
864 missing_request_ =
true;
865 missing_request_time_ = std::chrono::steady_clock::now();
868 auto ts = req->second;
869 TLOG(9) <<
"ApplyRequests: Checking that data exists for request window " << std::to_string(req->first) ;
870 Fragment::timestamp_t min = ts > windowOffset_ ? ts - windowOffset_ : 0;
871 Fragment::timestamp_t max = min + windowWidth_;
872 TLOG(9) <<
"ApplyRequests: min is " << std::to_string(min) <<
", max is " << std::to_string(max)
873 <<
" and last point in buffer is " << std::to_string((dataBuffer_.size() > 0 ? dataBuffer_.back()->timestamp() : 0)) <<
" (sz=" << std::to_string(dataBuffer_.size()) <<
")" ;
874 bool windowClosed = dataBuffer_.size() > 0 && dataBuffer_.back()->timestamp() >= max;
875 bool windowTimeout = TimeUtils::GetElapsedTimeMicroseconds(last_window_send_time_) > window_close_timeout_us_;
878 TLOG(TLVL_WARNING) <<
"A timeout occurred waiting for data to close the request window (max=" << std::to_string(max)
879 <<
", buffer=" << std::to_string((dataBuffer_.size() > 0 ? dataBuffer_.back()->timestamp() : 0))
880 <<
" (if no buffer in memory, this is shown as a 0)). Time waiting: "
881 << TimeUtils::GetElapsedTimeMicroseconds(last_window_send_time_) <<
" us "
882 <<
"(> " << std::to_string(window_close_timeout_us_) <<
" us)." ;
884 if (missing_request_) {
885 TLOG(TLVL_ERROR) <<
"A Window timeout has occurred while there are pending requests. Sending empties." ;
886 sendEmptyFragments(frags, requests);
889 if (windowClosed || !data_thread_running_ || windowTimeout)
891 TLOG(TLVL_DEBUG) <<
"Creating ContainerFragment for Buffered or Window-requested Fragments" ;
892 frags.emplace_back(
new artdaq::Fragment(req->first, fragment_id()));
893 frags.back()->setTimestamp(ts);
894 ContainerFragmentLoader cfl(*frags.back());
896 if (!windowClosed) cfl.set_missing_data(
true);
897 if (dataBuffer_.size() > 0 && dataBuffer_.front()->timestamp() > min)
899 TLOG(TLVL_DEBUG) <<
"Request Window covers data that is either before data collection began or has fallen off the end of the buffer" ;
900 cfl.set_missing_data(
true);
905 for (
auto it = dataBuffer_.begin(); it != dataBuffer_.end();)
907 Fragment::timestamp_t fragT = (*it)->timestamp();
908 if (fragT < min || fragT > max || (fragT == max && windowWidth_ > 0))
914 TLOG(9) <<
"ApplyRequests: Adding Fragment with timestamp " << std::to_string((*it)->timestamp()) <<
" to Container" ;
915 cfl.addFragment(*it);
919 it = dataBuffer_.erase(it);
926 if (req->first == ev_counter())
928 ev_counter_inc(1,
true);
929 while (windows_sent_ooo_.count(ev_counter()))
931 TLOG(9) <<
"Data-taking has caught up to out-of-order window request " << ev_counter() <<
", removing from list" ;
932 windows_sent_ooo_.erase(windows_sent_ooo_.begin(), windows_sent_ooo_.find(ev_counter()));
933 ev_counter_inc(1,
true);
938 windows_sent_ooo_.insert(req->first);
940 requestReceiver_->RemoveRequest(req->first);
941 req = requests.erase(req);
942 last_window_send_time_ = std::chrono::steady_clock::now();
953 if (check_stop() || exception())
959 if (mode_ == RequestMode::Ignored)
961 while (dataBufferDepthFragments_ <= 0)
963 if (check_stop() || exception() || !isHardwareOK_)
return false;
964 std::unique_lock<std::mutex> lock(dataBufferMutex_);
965 dataCondition_.wait_for(lock, std::chrono::milliseconds(10), [
this]() {
return dataBufferDepthFragments_ > 0; });
970 if ((check_stop() && requestReceiver_->size() == 0) || exception())
return false;
976 while (requestReceiver_->size() == 0 && counter < 100)
978 if (check_stop() || exception())
return false;
982 requestReceiver_->WaitForRequests(10);
988 std::unique_lock<std::mutex> dlk(dataBufferMutex_);
992 case RequestMode::Single:
993 applyRequestsSingleMode(frags);
995 case RequestMode::Window:
996 applyRequestsWindowMode(frags);
998 case RequestMode::Buffer:
999 applyRequestsBufferMode(frags);
1001 case RequestMode::Ignored:
1003 applyRequestsIgnoredMode(frags);
1007 getDataBufferStats();
1010 if (frags.size() > 0)
1011 TLOG(9) <<
"Finished Processing Event " << std::to_string(ev_counter() + 1) <<
" for fragment_id " << fragment_id() <<
"." ;
1017 TLOG(TLVL_WARNING) << desc <<
" sequence ID " << seqId <<
", sending empty fragment" ;
1018 for (
auto fid : fragment_ids_)
1020 auto frag =
new Fragment();
1021 frag->setSequenceID(seqId);
1022 frag->setFragmentID(fid);
1023 frag->setSystemType(Fragment::EmptyFragmentType);
1024 frags.emplace_back(FragmentPtr(frag));
1031 if (requests.size() == 0 && windows_sent_ooo_.size() == 0)
return;
1033 if (requests.size() > 0) {
1034 TLOG(19) <<
"Sending Empty Fragments for Sequence IDs from " << ev_counter() <<
" up to but not including " << requests.begin()->first ;
1035 while (requests.begin()->first > ev_counter())
1037 sendEmptyFragment(frags, ev_counter(),
"Missed request for");
1038 ev_counter_inc(1,
true);
1041 else if (windows_sent_ooo_.size() > 0)
1043 TLOG(19) <<
"Sending Empty Fragments for Sequence IDs from " << ev_counter() <<
" up to but not including " << *windows_sent_ooo_.begin() ;
1044 while (*windows_sent_ooo_.begin() > ev_counter())
1046 sendEmptyFragment(frags, ev_counter(),
"Missed request for");
1047 ev_counter_inc(1,
true);
1050 while (windows_sent_ooo_.count(ev_counter()))
1052 TLOG(19) <<
"Data-taking has caught up to out-of-order window request " << ev_counter() <<
", removing from list" ;
1053 windows_sent_ooo_.erase(windows_sent_ooo_.begin(), windows_sent_ooo_.find(ev_counter()));
1054 ev_counter_inc(1,
true);
int fragment_id() const
Get the current Fragment ID, if there is only one.
void applyRequestsSingleMode(artdaq::FragmentPtrs &frags)
Create fragments using data buffer for request mode Single. Precondition: dataBufferMutex_ and reques...
virtual ~CommandableFragmentGenerator()
CommandableFragmentGenerator Destructor.
void applyRequestsBufferMode(artdaq::FragmentPtrs &frags)
Create fragments using data buffer for request mode Buffer. Precondition: dataBufferMutex_ and reques...
bool sendEmptyFragment(FragmentPtrs &frags, size_t sequenceId, std::string desc)
Send an EmptyFragmentType Fragment.
void getMonitoringDataLoop()
This function regularly calls checkHWStatus_(), and sets the isHardwareOK flag accordingly.
void startDataThread()
Function that launches the data thread (getDataLoop())
std::string ReportCmd(std::string const &which="")
Get a report about a user-specified run-time quantity.
virtual bool metaCommand(std::string const &command, std::string const &arg)
The meta-command is used for implementing user-specific commands in a CommandableFragmentGenerator.
bool dataBufferIsTooLarge()
Test the configured constraints on the data buffer.
void StopCmd(uint64_t timeout, uint64_t timestamp)
Stop the CommandableFragmentGenerator.
void applyRequestsWindowMode(artdaq::FragmentPtrs &frags)
Create fragments using data buffer for request mode Window. Precondition: dataBufferMutex_ and reques...
void StartCmd(int run, uint64_t timeout, uint64_t timestamp)
Start the CommandableFragmentGenerator.
bool check_stop()
Routine used by applyRequests to make sure that all outstanding requests have been fulfilled before r...
void ResumeCmd(uint64_t timeout, uint64_t timestamp)
Resume the CommandableFragmentGenerator.
CommandableFragmentGenerator()
CommandableFragmentGenerator default constructor.
bool getNext(FragmentPtrs &output) overridefinal
getNext calls either applyRequests or getNext_ to get any data that is ready to be sent to the EventB...
bool waitForDataBufferReady()
Wait for the data buffer to drain (dataBufferIsTooLarge returns false), periodically reporting status...
size_t ev_counter_inc(size_t step=1, bool force=false)
Increment the event counter, if the current RequestMode allows it.
void applyRequestsIgnoredMode(artdaq::FragmentPtrs &frags)
Create fragments using data buffer for request mode Ignored. Precondition: dataBufferMutex_ and reque...
void PauseCmd(uint64_t timeout, uint64_t timestamp)
Pause the CommandableFragmentGenerator.
void getDataLoop()
When separate_data_thread is set to true, this loop repeatedly calls getNext_ and adds returned Fragm...
void sendEmptyFragments(FragmentPtrs &frags, std::map< Fragment::sequence_id_t, Fragment::timestamp_t > &requests)
This function is for Buffered and Single request modes, as they can only respond to one data request ...
void startMonitoringThread()
Function that launches the monitoring thread (getMonitoringDataLoop())
void checkDataBuffer()
Perform data buffer pruning operations. If the RequestMode is Single, removes all but the latest Frag...
std::string printMode_()
Return the string representation of the current RequestMode.
void getDataBufferStats()
Calculate the size of the dataBuffer and report appropriate metrics.
bool applyRequests(FragmentPtrs &output)
See if any requests have been received, and add the corresponding data Fragment objects to the output...
void joinThreads()
Join any data-taking threads. Should be called when destructing CommandableFragmentGenerator.