1 #define TRACE_NAME "CommandableFragmentGenerator"
4 #include "artdaq/Application/CommandableFragmentGenerator.hh"
6 #include <boost/exception/all.hpp>
7 #include <boost/throw_exception.hpp>
12 #include "canvas/Utilities/Exception.h"
13 #include "cetlib_except/exception.h"
14 #include "fhiclcpp/ParameterSet.h"
16 #include "artdaq-core/Utilities/SimpleLookupPolicy.hh"
17 #include "artdaq-core/Data/Fragment.hh"
18 #include "artdaq-core/Data/ContainerFragmentLoader.hh"
19 #include "artdaq-core/Utilities/ExceptionHandler.hh"
20 #include "artdaq-core/Utilities/TimeUtils.hh"
36 , staleTimeout_(Fragment::InvalidTimestamp)
37 , expectedType_(Fragment::EmptyFragmentType)
38 , maxFragmentCount_(std::numeric_limits<size_t>::max())
39 , uniqueWindows_(true)
40 , missing_request_(true)
41 , missing_request_time_()
42 , last_window_send_time_()
43 , last_window_send_time_set_(false)
45 , missing_request_window_timeout_us_(1000000)
46 , window_close_timeout_us_(2000000)
47 , useDataThread_(false)
48 , sleep_on_no_data_us_(0)
49 , data_thread_running_(false)
50 , dataBufferDepthFragments_(0)
51 , dataBufferDepthBytes_(0)
52 , maxDataBufferDepthFragments_(1000)
53 , maxDataBufferDepthBytes_(1000)
54 , useMonitoringThread_(false)
55 , monitoringInterval_(0)
56 , lastMonitoringCall_()
62 , timeout_(std::numeric_limits<uint64_t>::max())
63 , timestamp_(std::numeric_limits<uint64_t>::max())
67 , latest_exception_report_(
"none")
70 , instance_name_for_metrics_(
"FragmentGenerator")
71 , sleep_on_stop_us_(0)
76 , windowOffset_(ps.get<Fragment::timestamp_t>(
"request_window_offset", 0))
77 , windowWidth_(ps.get<Fragment::timestamp_t>(
"request_window_width", 0))
78 , staleTimeout_(ps.get<Fragment::timestamp_t>(
"stale_request_timeout", 0xFFFFFFFF))
79 , expectedType_(ps.get<Fragment::type_t>(
"expected_fragment_type", Fragment::type_t(Fragment::EmptyFragmentType)))
80 , uniqueWindows_(ps.get<bool>(
"request_windows_are_unique", true))
81 , missing_request_(false)
82 , missing_request_time_(decltype(missing_request_time_)::max())
83 , last_window_send_time_(decltype(last_window_send_time_)::max())
84 , last_window_send_time_set_(false)
86 , missing_request_window_timeout_us_(ps.get<size_t>(
"missing_request_window_timeout_us", 1000000))
87 , window_close_timeout_us_(ps.get<size_t>(
"window_close_timeout_us", 2000000))
88 , useDataThread_(ps.get<bool>(
"separate_data_thread", false))
89 , sleep_on_no_data_us_(ps.get<size_t>(
"sleep_on_no_data_us", 0))
90 , data_thread_running_(false)
91 , dataBufferDepthFragments_(0)
92 , dataBufferDepthBytes_(0)
93 , maxDataBufferDepthFragments_(ps.get<int>(
"data_buffer_depth_fragments", 1000))
94 , maxDataBufferDepthBytes_(ps.get<size_t>(
"data_buffer_depth_mb", 1000) * 1024 * 1024)
95 , useMonitoringThread_(ps.get<bool>(
"separate_monitoring_thread", false))
96 , monitoringInterval_(ps.get<int64_t>(
"hardware_poll_interval_us", 0))
97 , lastMonitoringCall_()
103 , timeout_(std::numeric_limits<uint64_t>::max())
104 , timestamp_(std::numeric_limits<uint64_t>::max())
105 , should_stop_(false)
108 , latest_exception_report_(
"none")
111 , sleep_on_stop_us_(0)
113 board_id_ = ps.get<
int>(
"board_id");
114 instance_name_for_metrics_ =
"BoardReader." + boost::lexical_cast<std::string>(board_id_);
116 fragment_ids_ = ps.get<std::vector<artdaq::Fragment::fragment_id_t>>(
"fragment_ids", std::vector<artdaq::Fragment::fragment_id_t>());
118 TLOG_TRACE(
"CommandableFragmentGenerator") <<
"artdaq::CommandableFragmentGenerator::CommandableFragmentGenerator(ps)" << TLOG_ENDL;
121 if (fragment_id != -99)
123 if (fragment_ids_.size() != 0)
125 latest_exception_report_ =
"Error in CommandableFragmentGenerator: can't both define \"fragment_id\" and \"fragment_ids\" in FHiCL document";
126 throw cet::exception(latest_exception_report_);
130 fragment_ids_.emplace_back(fragment_id);
134 sleep_on_stop_us_ = ps.get<
int>(
"sleep_on_stop_us", 0);
136 dataBuffer_.emplace_back(FragmentPtr(
new Fragment()));
137 (*dataBuffer_.begin())->setSystemType(Fragment::EmptyFragmentType);
139 std::string modeString = ps.get<std::string>(
"request_mode",
"ignored");
140 if (modeString ==
"single" || modeString ==
"Single")
142 mode_ = RequestMode::Single;
144 else if (modeString.find(
"buffer") != std::string::npos || modeString.find(
"Buffer") != std::string::npos)
146 mode_ = RequestMode::Buffer;
148 else if (modeString ==
"window" || modeString ==
"Window")
150 mode_ = RequestMode::Window;
152 else if (modeString.find(
"ignore") != std::string::npos || modeString.find(
"Ignore") != std::string::npos)
154 mode_ = RequestMode::Ignored;
156 TLOG_DEBUG(
"CommandableFragmentGenerator") <<
"Request mode is " <<
printMode_() << TLOG_ENDL;
158 if (mode_ != RequestMode::Ignored)
162 latest_exception_report_ =
"Error in CommandableFragmentGenerator: use_data_thread must be true when request_mode is not \"Ignored\"!";
163 throw cet::exception(latest_exception_report_);
178 TLOG_DEBUG(
"CommandableFragmentGenerator") <<
"Joining dataThread" << TLOG_ENDL;
179 if (dataThread_.joinable()) dataThread_.join();
180 TLOG_DEBUG(
"CommandableFragmentGenerator") <<
"Joining monitoringThread" << TLOG_ENDL;
181 if (monitoringThread_.joinable()) monitoringThread_.join();
182 requestReceiver_.reset(
nullptr);
189 if (check_stop()) usleep(sleep_on_stop_us_);
190 if (exception() || force_stop_)
return false;
192 if (!useMonitoringThread_ && monitoringInterval_ > 0)
194 TLOG_ARB(10,
"CommandableFragmentGenerator") <<
"getNext: Checking whether to collect Monitoring Data" << TLOG_ENDL;
195 auto now = std::chrono::steady_clock::now();
197 if (TimeUtils::GetElapsedTimeMicroseconds(lastMonitoringCall_, now) >= static_cast<size_t>(monitoringInterval_))
199 TLOG_ARB(10,
"CommandableFragmentGenerator") <<
"getNext: Collecting Monitoring Data" << TLOG_ENDL;
200 isHardwareOK_ = checkHWStatus_();
201 TLOG_ARB(10,
"CommandableFragmentGenerator") <<
"getNext: isHardwareOK_ is now " << std::boolalpha << isHardwareOK_ << TLOG_ENDL;
202 lastMonitoringCall_ = now;
208 std::lock_guard<std::mutex> lk(mutex_);
211 TLOG_TRACE(
"CommandableFragmentGenerator") <<
"getNext: Calling applyRequests" << TLOG_ENDL;
212 result = applyRequests(output);
213 TLOG_TRACE(
"CommandableFragmentGenerator") <<
"getNext: Done with applyRequests" << TLOG_ENDL;
217 throw cet::exception(
"CommandableFragmentGenerator") <<
"Exception found in BoardReader with board ID " << board_id() <<
"; BoardReader will now return error status when queried";
224 TLOG_ERROR(
"CommandableFragmentGenerator") <<
"Stopping CFG because the hardware reports bad status!" << TLOG_ENDL;
227 TLOG_TRACE(
"CommandableFragmentGenerator") <<
"getNext: Calling getNext_ " << std::to_string(ev_counter()) << TLOG_ENDL;
230 result = getNext_(output);
236 TLOG_TRACE(
"CommandableFragmentGenerator") <<
"getNext: Done with getNext_ " << std::to_string(ev_counter()) << TLOG_ENDL;
239 catch (
const cet::exception& e)
241 latest_exception_report_ =
"cet::exception caught in getNext(): ";
242 latest_exception_report_.append(e.what());
243 TLOG_ERROR(
"CommandableFragmentGenerator") <<
"getNext: cet::exception caught: " << e << TLOG_ENDL;
247 catch (
const boost::exception& e)
249 latest_exception_report_ =
"boost::exception caught in getNext(): ";
250 latest_exception_report_.append(boost::diagnostic_information(e));
251 TLOG_ERROR(
"CommandableFragmentGenerator") <<
"getNext: boost::exception caught: " << boost::diagnostic_information(e) << TLOG_ENDL;
255 catch (
const std::exception& e)
257 latest_exception_report_ =
"std::exception caught in getNext(): ";
258 latest_exception_report_.append(e.what());
259 TLOG_ERROR(
"CommandableFragmentGenerator") <<
"getNext: std::exception caught: " << e.what() << TLOG_ENDL;
265 latest_exception_report_ =
"Unknown exception caught in getNext().";
266 TLOG_ERROR(
"CommandableFragmentGenerator") <<
"getNext: unknown exception caught" << TLOG_ENDL;
273 TLOG_DEBUG(
"getNext") <<
"stopped " << TLOG_ENDL;
281 TLOG_ARB(14,
"CommandableFragmentGeneraotr") <<
"CFG::check_stop: should_stop=" << should_stop() <<
", useDataThread_=" << useDataThread_ <<
", exception status =" << int(exception()) << TLOG_ENDL;
283 if (!should_stop())
return false;
284 if (!useDataThread_ || mode_ == RequestMode::Ignored)
return true;
285 if (force_stop_)
return true;
288 return !requestReceiver_->isRunning();
293 if (fragment_ids_.size() != 1)
295 throw cet::exception(
"Error in CommandableFragmentGenerator: can't call fragment_id() unless member fragment_ids_ vector is length 1");
299 return fragment_ids_[0];
305 if (force || mode_ == RequestMode::Ignored)
307 return ev_counter_.fetch_add(step);
309 return ev_counter_.load();
314 if (run < 0)
throw cet::exception(
"CommandableFragmentGenerator") <<
"negative run number";
317 timestamp_ = timestamp;
318 ev_counter_.store(1);
319 should_stop_.store(
false);
320 exception_.store(
false);
323 latest_exception_report_ =
"none";
325 last_window_send_time_set_ =
false;
329 std::unique_lock<std::mutex> lk(mutex_);
330 if (useDataThread_) startDataThread();
331 if (useMonitoringThread_) startMonitoringThread();
332 if (mode_ != RequestMode::Ignored && !requestReceiver_->isRunning()) requestReceiver_->startRequestReceiverThread();
337 TLOG_DEBUG(
"CommandableFragmentGenerator") <<
"Stop Command received." << TLOG_ENDL;
340 timestamp_ = timestamp;
343 should_stop_.store(
true);
344 std::unique_lock<std::mutex> lk(mutex_);
351 timestamp_ = timestamp;
354 should_stop_.store(
true);
355 std::unique_lock<std::mutex> lk(mutex_);
363 timestamp_ = timestamp;
366 should_stop_ =
false;
373 std::unique_lock<std::mutex> lk(mutex_);
374 if (useDataThread_) startDataThread();
375 if (useMonitoringThread_) startMonitoringThread();
376 if (mode_ != RequestMode::Ignored && !requestReceiver_->isRunning()) requestReceiver_->startRequestReceiverThread();
381 std::lock_guard<std::mutex> lk(mutex_);
388 std::string childReport = reportSpecific(which);
389 if (childReport.length() > 0) {
return childReport; }
392 if (which ==
"latest_exception")
394 return latest_exception_report_;
398 childReport = report();
399 if (childReport.length() > 0) {
return childReport; }
402 std::string tmpString =
"The \"" + which +
"\" command is not ";
403 tmpString.append(
"currently supported by the ");
404 tmpString.append(metricsReportingInstanceName());
405 tmpString.append(
" fragment generator.");
410 void artdaq::CommandableFragmentGenerator::pauseNoMutex()
412 #pragma message "Using default implementation of CommandableFragmentGenerator::pauseNoMutex()"
415 void artdaq::CommandableFragmentGenerator::pause()
417 #pragma message "Using default implementation of CommandableFragmentGenerator::pause()"
420 void artdaq::CommandableFragmentGenerator::resume()
422 #pragma message "Using default implementation of CommandableFragmentGenerator::resume()"
425 std::string artdaq::CommandableFragmentGenerator::report()
427 #pragma message "Using default implementation of CommandableFragmentGenerator::report()"
431 std::string artdaq::CommandableFragmentGenerator::reportSpecific(std::string
const&)
433 #pragma message "Using default implementation of CommandableFragmentGenerator::reportSpecific(std::string)"
437 bool artdaq::CommandableFragmentGenerator::checkHWStatus_()
439 #pragma message "Using default implementation of CommandableFragmentGenerator::checkHWStatus_()"
445 if (dataThread_.joinable()) dataThread_.join();
446 TLOG_INFO(
"CommandableFragmentGenerator") <<
"Starting Data Receiver Thread" << TLOG_ENDL;
452 if (monitoringThread_.joinable()) monitoringThread_.join();
453 TLOG_INFO(
"CommandableFragmentGenerator") <<
"Starting Hardware Monitoring Thread" << TLOG_ENDL;
461 case RequestMode::Single:
463 case RequestMode::Buffer:
465 case RequestMode::Window:
467 case RequestMode::Ignored:
476 data_thread_running_ =
true;
481 TLOG_DEBUG(
"CommandableFragmentGenerator") <<
"getDataLoop: isHardwareOK is " << isHardwareOK_ <<
", aborting data thread" << TLOG_ENDL;
482 data_thread_running_ =
false;
486 TLOG_ARB(13,
"CommandableFragmentGenerator") <<
"getDataLoop: calling getNext_" << TLOG_ENDL;
489 auto startdata = std::chrono::steady_clock::now();
493 data = getNext_(newDataBuffer_);
497 ExceptionHandler(ExceptionHandlerRethrow::no,
498 "Exception thrown by fragment generator in CommandableFragmentGenerator::getDataLoop; setting exception state to \"true\"");
501 data_thread_running_ =
false;
507 metricMan->sendMetric(
"Avg Data Acquisition Time", TimeUtils::GetElapsedTime(startdata),
"s", 3, artdaq::MetricMode::Average);
510 if (newDataBuffer_.size() == 0 && sleep_on_no_data_us_ > 0)
512 usleep(sleep_on_no_data_us_);
515 TLOG_ARB(15,
"CommandableFragmentGenerator") <<
"Waiting for data buffer ready" << TLOG_ENDL;
516 if (!waitForDataBufferReady())
return;
517 TLOG_ARB(15,
"CommandableFragmentGenerator") <<
"Done waiting for data buffer ready" << TLOG_ENDL;
519 TLOG_ARB(13,
"CommandableFragmentGenerator") <<
"getDataLoop: processing data" << TLOG_ENDL;
520 if (data && !force_stop_)
522 std::unique_lock<std::mutex> lock(dataBufferMutex_);
525 case RequestMode::Single:
527 while (newDataBuffer_.size() >= fragment_ids_.size())
530 auto it = newDataBuffer_.begin();
531 std::advance(it, fragment_ids_.size());
532 dataBuffer_.splice(dataBuffer_.end(), newDataBuffer_, newDataBuffer_.begin(), it);
535 case RequestMode::Buffer:
536 case RequestMode::Ignored:
537 case RequestMode::Window:
540 dataBuffer_.splice(dataBuffer_.end(), newDataBuffer_);
543 getDataBufferStats();
547 std::unique_lock<std::mutex> lock(dataBufferMutex_);
548 if (dataBuffer_.size() > 0)
550 dataCondition_.notify_all();
553 if (!data || force_stop_)
555 TLOG_INFO(
"CommandableFragmentGenerator") <<
"Data flow has stopped. Ending data collection thread" << TLOG_ENDL;
556 data_thread_running_ =
false;
564 auto startwait = std::chrono::steady_clock::now();
566 auto lastwaittime = 0ULL;
567 while (dataBufferIsTooLarge())
571 TLOG_DEBUG(
"CommandableFragmentGenerator") <<
"Run ended while waiting for buffer to shrink!" << TLOG_ENDL;
572 std::unique_lock<std::mutex> lock(dataBufferMutex_);
573 getDataBufferStats();
574 dataCondition_.notify_all();
575 data_thread_running_ =
false;
578 auto waittime = TimeUtils::GetElapsedTimeMilliseconds(startwait);
580 if (first || (waittime != lastwaittime && waittime % 1000 == 0))
582 TLOG_WARNING(
"CommandableFragmentGenerator") <<
"Bad Omen: Data Buffer has exceeded its size limits. "
583 <<
"(seq_id=" << ev_counter()
584 <<
", frags=" << dataBufferDepthFragments_ <<
"/" << maxDataBufferDepthFragments_
585 <<
", szB=" << dataBufferDepthBytes_ <<
"/" << maxDataBufferDepthBytes_ <<
")" << TLOG_ENDL;
586 TLOG_TRACE(
"CommandableFragmentGenerator") <<
"Bad Omen: Possible causes include requests not getting through or Ignored-mode BR issues" << TLOG_ENDL;
589 if (waittime % 5 && waittime != lastwaittime)
591 TLOG_ARB(13,
"CommandableFragmentGenerator") <<
"getDataLoop: Data Retreival paused for " << std::to_string(waittime) <<
" ms waiting for data buffer to drain" << TLOG_ENDL;
593 lastwaittime = waittime;
601 return (maxDataBufferDepthFragments_ > 0 && dataBufferDepthFragments_ > maxDataBufferDepthFragments_) || (maxDataBufferDepthBytes_ > 0 && dataBufferDepthBytes_ > maxDataBufferDepthBytes_);
607 dataBufferDepthFragments_ = dataBuffer_.size();
609 TLOG_ARB(15,
"CommandableFragmentGenerator") <<
"getDataBufferStats: Calculating buffer size" << TLOG_ENDL;
610 for (
auto i = dataBuffer_.begin(); i != dataBuffer_.end(); ++i)
612 if (i->get() !=
nullptr)
614 acc += (*i)->sizeBytes();
617 dataBufferDepthBytes_ = acc;
621 TLOG_ARB(15,
"CommandableFragmentGenerator") <<
"getDataBufferStats: Sending Metrics" << TLOG_ENDL;
622 metricMan->sendMetric(
"Buffer Depth Fragments", dataBufferDepthFragments_.load(),
"fragments", 1, MetricMode::LastPoint);
623 metricMan->sendMetric(
"Buffer Depth Bytes", dataBufferDepthBytes_.load(),
"bytes", 1, MetricMode::LastPoint);
625 TLOG_ARB(15,
"CommandableFragmentGenerator") <<
"getDataBufferStats: frags=" << dataBufferDepthFragments_.load() <<
"/" << maxDataBufferDepthFragments_
626 <<
", sz=" << std::to_string(dataBufferDepthBytes_.load()) <<
"/" << std::to_string(maxDataBufferDepthBytes_) << TLOG_ENDL;
631 std::unique_lock<std::mutex> lock(dataBufferMutex_);
632 dataCondition_.wait_for(lock, std::chrono::milliseconds(10));
633 if (dataBufferDepthFragments_ > 0)
635 if ((mode_ == RequestMode::Buffer || mode_ == RequestMode::Window))
638 while (dataBufferIsTooLarge())
640 dataBuffer_.erase(dataBuffer_.begin());
641 getDataBufferStats();
643 if (dataBuffer_.size() > 0)
645 TLOG_ARB(17,
"CommandableFragmentGenerator") <<
"Determining if Fragments can be dropped from data buffer" << TLOG_ENDL;
646 Fragment::timestamp_t last = dataBuffer_.back()->timestamp();
647 Fragment::timestamp_t min = last > staleTimeout_ ? last - staleTimeout_ : 0;
648 for (
auto it = dataBuffer_.begin(); it != dataBuffer_.end();)
650 if ((*it)->timestamp() < min)
652 it = dataBuffer_.erase(it);
659 getDataBufferStats();
662 else if (mode_ == RequestMode::Single && dataBuffer_.size() > fragment_ids_.size())
665 while (dataBuffer_.size() > fragment_ids_.size())
667 dataBuffer_.erase(dataBuffer_.begin());
677 if (should_stop() || monitoringInterval_ <= 0)
679 TLOG_DEBUG(
"CommandableFragmentGenerator") <<
"getMonitoringDataLoop: should_stop() is " << std::boolalpha << should_stop()
680 <<
" and monitoringInterval is " << monitoringInterval_ <<
", returning" << TLOG_ENDL;
683 TLOG_ARB(12,
"CommandableFragmentGenerator") <<
"getMonitoringDataLoop: Determining whether to call checkHWStatus_" << TLOG_ENDL;
685 auto now = std::chrono::steady_clock::now();
686 if (TimeUtils::GetElapsedTimeMicroseconds(lastMonitoringCall_, now) >= static_cast<size_t>(monitoringInterval_))
688 isHardwareOK_ = checkHWStatus_();
689 TLOG_ARB(12,
"CommandableFragmentGenerator") <<
"getMonitoringDataLoop: isHardwareOK_ is now " << std::boolalpha << isHardwareOK_ << TLOG_ENDL;
690 lastMonitoringCall_ = now;
692 usleep(monitoringInterval_ / 10);
699 TLOG_ARB(9,
"CommandableFragmentGenerator") <<
"Mode is Ignored; Copying data to output" << TLOG_ENDL;
700 std::move(dataBuffer_.begin(), dataBuffer_.end(), std::inserter(frags, frags.end()));
707 auto requests = requestReceiver_->GetRequests();
708 while (requests.size() > 1) {
710 requestReceiver_->RemoveRequest(requests.begin()->first);
711 requests.erase(requests.begin());
713 sendEmptyFragments(frags, requests);
716 if (requests.size() == 0 || !requests.count(ev_counter()))
return;
718 if (dataBuffer_.size() > 0)
720 TLOG_ARB(9,
"CommandableFragmentGenerator") <<
"Mode is Single; Sending copy of last event" << TLOG_ENDL;
721 for (
auto& fragptr : dataBuffer_)
724 auto frag = fragptr.get();
725 auto newfrag = std::unique_ptr<artdaq::Fragment>(
new Fragment(ev_counter(), frag->fragmentID()));
726 newfrag->resize(frag->size() - detail::RawFragmentHeader::num_words());
727 memcpy(newfrag->headerAddress(), frag->headerAddress(), frag->sizeBytes());
728 newfrag->setTimestamp(requests[ev_counter()]);
729 newfrag->setSequenceID(ev_counter());
730 frags.push_back(std::move(newfrag));
735 sendEmptyFragment(frags, ev_counter(),
"No data for");
737 requestReceiver_->RemoveRequest(ev_counter());
738 ev_counter_inc(1,
true);
744 auto requests = requestReceiver_->GetRequests();
745 while (requests.size() > 1) {
747 requestReceiver_->RemoveRequest(requests.begin()->first);
748 requests.erase(requests.begin());
750 sendEmptyFragments(frags, requests);
753 if (requests.size() == 0 || !requests.count(ev_counter()))
return;
755 TLOG_DEBUG(
"CommandableFragmentGenerator") <<
"Creating ContainerFragment for Buffered Fragments" << TLOG_ENDL;
756 frags.emplace_back(
new artdaq::Fragment(ev_counter(), fragment_id()));
757 frags.back()->setTimestamp(requests[ev_counter()]);
758 ContainerFragmentLoader cfl(*frags.back());
759 cfl.set_missing_data(
false);
763 for (
auto it = dataBuffer_.begin(); it != dataBuffer_.end();)
765 TLOG_ARB(9,
"CommandableFragmentGenerator") <<
"ApplyRequests: Adding Fragment with timestamp " << std::to_string((*it)->timestamp()) <<
" to Container" << TLOG_ENDL;
766 cfl.addFragment(*it);
767 it = dataBuffer_.erase(it);
769 requestReceiver_->RemoveRequest(ev_counter());
770 ev_counter_inc(1,
true);
775 TLOG(10) <<
"applyRequestsWindowMode BEGIN";
776 if (!last_window_send_time_set_)
778 last_window_send_time_ = std::chrono::steady_clock::now();
779 last_window_send_time_set_ =
true;
782 auto requests = requestReceiver_->GetRequests();
783 bool now_have_desired_request = std::any_of(requests.begin(), requests.end(),
784 [
this](decltype(requests)::value_type& request) {
785 return request.first == ev_counter(); });
787 if (missing_request_)
789 if (!now_have_desired_request && TimeUtils::GetElapsedTimeMicroseconds(missing_request_time_) > missing_request_window_timeout_us_)
791 TLOG_ERROR(
"CommandableFragmentGenerator") <<
"Data-taking has paused for " << TimeUtils::GetElapsedTimeMicroseconds(missing_request_time_) <<
" us "
792 <<
"(> " << std::to_string(missing_request_window_timeout_us_) <<
" us) while waiting for missing data request messages."
793 <<
" Sending Empty Fragments for missing requests!" << TLOG_ENDL;
794 sendEmptyFragments(frags, requests);
796 missing_request_ =
false;
797 missing_request_time_ = decltype(missing_request_time_)::max();
799 else if (now_have_desired_request) {
800 missing_request_ =
false;
801 missing_request_time_ = decltype(missing_request_time_)::max();
805 TLOG(10) <<
"applyRequestsWindowMode: Starting request processing";
806 for (
auto req = requests.begin(); req != requests.end();)
808 while (req->first < ev_counter() && requests.size() > 0)
810 TLOG(10) <<
"applyRequestsWindowMode: Clearing passed request for sequence ID " << req->first;
811 requestReceiver_->RemoveRequest(req->first);
812 req = requests.erase(req);
814 if (requests.size() == 0)
break;
815 if (req->first > ev_counter())
817 if (!missing_request_)
819 missing_request_ =
true;
820 missing_request_time_ = std::chrono::steady_clock::now();
823 auto ts = req->second;
824 TLOG_ARB(9,
"CommandableFragmentGenerator") <<
"ApplyRequests: Checking that data exists for request window " << std::to_string(req->first) << TLOG_ENDL;
825 Fragment::timestamp_t min = ts > windowOffset_ ? ts - windowOffset_ : 0;
826 Fragment::timestamp_t max = min + windowWidth_;
827 TLOG_ARB(9,
"CommandableFragmentGenerator") <<
"ApplyRequests: min is " << std::to_string(min) <<
", max is " << std::to_string(max)
828 <<
" and last point in buffer is " << std::to_string((dataBuffer_.size() > 0 ? dataBuffer_.back()->timestamp() : 0)) <<
" (sz=" << std::to_string(dataBuffer_.size()) <<
")" << TLOG_ENDL;
829 bool windowClosed = dataBuffer_.size() > 0 && dataBuffer_.back()->timestamp() >= max;
830 bool windowTimeout = TimeUtils::GetElapsedTimeMicroseconds(last_window_send_time_) > window_close_timeout_us_;
833 TLOG_WARNING(
"CommandableFragmentGenerator") <<
"A timeout occurred waiting for data to close the request window (max=" << std::to_string(max)
834 <<
", buffer=" << std::to_string((dataBuffer_.size() > 0 ? dataBuffer_.back()->timestamp() : 0))
835 <<
" (if no buffer in memory, this is shown as a 0)). Time waiting: "
836 << TimeUtils::GetElapsedTimeMicroseconds(last_window_send_time_) <<
" us "
837 <<
"(> " << std::to_string(window_close_timeout_us_) <<
" us)." << TLOG_ENDL;
839 if (missing_request_) {
840 TLOG_ERROR(
"CommandableFragmentGenerator") <<
"A Window timeout has occurred while there are pending requests. Sending empties." << TLOG_ENDL;
841 sendEmptyFragments(frags, requests);
844 if (windowClosed || !data_thread_running_ || windowTimeout)
846 TLOG_DEBUG(
"CommandableFragmentGenerator") <<
"Creating ContainerFragment for Buffered or Window-requested Fragments" << TLOG_ENDL;
847 frags.emplace_back(
new artdaq::Fragment(req->first, fragment_id()));
848 frags.back()->setTimestamp(ts);
849 ContainerFragmentLoader cfl(*frags.back());
851 if (!windowClosed) cfl.set_missing_data(
true);
852 if (dataBuffer_.size() > 0 && dataBuffer_.front()->timestamp() > min)
854 TLOG_DEBUG(
"CommandableFragmentGenerator") <<
"Request Window covers data that is either before data collection began or has fallen off the end of the buffer" << TLOG_ENDL;
855 cfl.set_missing_data(
true);
860 for (
auto it = dataBuffer_.begin(); it != dataBuffer_.end();)
862 Fragment::timestamp_t fragT = (*it)->timestamp();
863 if (fragT < min || fragT > max || (fragT == max && windowWidth_ > 0))
869 TLOG_ARB(9,
"CommandableFragmentGenerator") <<
"ApplyRequests: Adding Fragment with timestamp " << std::to_string((*it)->timestamp()) <<
" to Container" << TLOG_ENDL;
870 cfl.addFragment(*it);
874 it = dataBuffer_.erase(it);
881 if (req->first == ev_counter())
883 ev_counter_inc(1,
true);
884 while (windows_sent_ooo_.count(ev_counter()))
886 TLOG_ARB(9,
"CommandableFragmentGenerator") <<
"Data-taking has caught up to out-of-order window request " << ev_counter() <<
", removing from list" << TLOG_ENDL;
887 windows_sent_ooo_.erase(windows_sent_ooo_.begin(), windows_sent_ooo_.find(ev_counter()));
888 ev_counter_inc(1,
true);
893 windows_sent_ooo_.insert(req->first);
895 requestReceiver_->RemoveRequest(req->first);
896 req = requests.erase(req);
897 last_window_send_time_ = std::chrono::steady_clock::now();
908 if (check_stop() || exception())
914 if (mode_ == RequestMode::Ignored)
916 while (dataBufferDepthFragments_ <= 0)
918 if (check_stop() || exception() || !isHardwareOK_)
return false;
919 std::unique_lock<std::mutex> lock(dataBufferMutex_);
920 dataCondition_.wait_for(lock, std::chrono::milliseconds(10), [
this]() {
return dataBufferDepthFragments_ > 0; });
925 if ((check_stop() && requestReceiver_->size() == 0) || exception())
return false;
931 while (requestReceiver_->size() == 0 && counter < 100)
933 if (check_stop() || exception())
return false;
937 requestReceiver_->WaitForRequests(10);
943 std::unique_lock<std::mutex> dlk(dataBufferMutex_);
947 case RequestMode::Single:
948 applyRequestsSingleMode(frags);
950 case RequestMode::Window:
951 applyRequestsWindowMode(frags);
953 case RequestMode::Buffer:
954 applyRequestsBufferMode(frags);
956 case RequestMode::Ignored:
958 applyRequestsIgnoredMode(frags);
962 getDataBufferStats();
965 if (frags.size() > 0)
966 TLOG_ARB(9,
"CommandableFragmentGenerator") <<
"Finished Processing Event " << std::to_string(ev_counter() + 1) <<
" for fragment_id " << fragment_id() <<
"." << TLOG_ENDL;
972 TLOG_WARNING(
"CommandableFragmentGenerator") << desc <<
" sequence ID " << seqId <<
", sending empty fragment" << TLOG_ENDL;
973 for (
auto fid : fragment_ids_)
975 auto frag =
new Fragment();
976 frag->setSequenceID(seqId);
977 frag->setFragmentID(fid);
978 frag->setSystemType(Fragment::EmptyFragmentType);
979 frags.emplace_back(FragmentPtr(frag));
986 if (requests.size() == 0 && windows_sent_ooo_.size() == 0)
return;
988 if (requests.size() > 0) {
989 TLOG_ARB(19,
"CommandableFragmentGenerator") <<
"Sending Empty Fragments for Sequence IDs from " << ev_counter() <<
" up to but not including " << requests.begin()->first << TLOG_ENDL;
990 while (requests.begin()->first > ev_counter())
992 sendEmptyFragment(frags, ev_counter(),
"Missed request for");
993 ev_counter_inc(1,
true);
996 else if (windows_sent_ooo_.size() > 0)
998 TLOG_ARB(19,
"CommandableFragmentGenerator") <<
"Sending Empty Fragments for Sequence IDs from " << ev_counter() <<
" up to but not including " << *windows_sent_ooo_.begin() << TLOG_ENDL;
999 while (*windows_sent_ooo_.begin() > ev_counter())
1001 sendEmptyFragment(frags, ev_counter(),
"Missed request for");
1002 ev_counter_inc(1,
true);
1005 while (windows_sent_ooo_.count(ev_counter()))
1007 TLOG_ARB(19,
"CommandableFragmentGenerator") <<
"Data-taking has caught up to out-of-order window request " << ev_counter() <<
", removing from list" << TLOG_ENDL;
1008 windows_sent_ooo_.erase(windows_sent_ooo_.begin(), windows_sent_ooo_.find(ev_counter()));
1009 ev_counter_inc(1,
true);
int fragment_id() const
Get the current Fragment ID, if there is only one.
void applyRequestsSingleMode(artdaq::FragmentPtrs &frags)
Create fragments using data buffer for request mode Single. Precondition: dataBufferMutex_ and reques...
virtual ~CommandableFragmentGenerator()
CommandableFragmentGenerator Destructor.
void applyRequestsBufferMode(artdaq::FragmentPtrs &frags)
Create fragments using data buffer for request mode Buffer. Precondition: dataBufferMutex_ and reques...
bool sendEmptyFragment(FragmentPtrs &frags, size_t sequenceId, std::string desc)
Send an EmptyFragmentType Fragment.
void getMonitoringDataLoop()
This function regularly calls checkHWStatus_(), and sets the isHardwareOK flag accordingly.
void startDataThread()
Function that launches the data thread (getDataLoop())
std::string ReportCmd(std::string const &which="")
Get a report about a user-specified run-time quantity.
bool dataBufferIsTooLarge()
Test the configured constraints on the data buffer.
void StopCmd(uint64_t timeout, uint64_t timestamp)
Stop the CommandableFragmentGenerator.
void applyRequestsWindowMode(artdaq::FragmentPtrs &frags)
Create fragments using data buffer for request mode Window. Precondition: dataBufferMutex_ and reques...
void StartCmd(int run, uint64_t timeout, uint64_t timestamp)
Start the CommandableFragmentGenerator.
bool check_stop()
Routine used by applyRequests to make sure that all outstanding requests have been fulfilled before r...
void ResumeCmd(uint64_t timeout, uint64_t timestamp)
Resume the CommandableFragmentGenerator.
CommandableFragmentGenerator()
CommandableFragmentGenerator default constructor.
bool getNext(FragmentPtrs &output) overridefinal
getNext calls either applyRequests or getNext_ to get any data that is ready to be sent to the EventB...
bool waitForDataBufferReady()
Wait for the data buffer to drain (dataBufferIsTooLarge returns false), periodically reporting status...
size_t ev_counter_inc(size_t step=1, bool force=false)
Increment the event counter, if the current RequestMode allows it.
void applyRequestsIgnoredMode(artdaq::FragmentPtrs &frags)
Create fragments using data buffer for request mode Ignored. Precondition: dataBufferMutex_ and reque...
void PauseCmd(uint64_t timeout, uint64_t timestamp)
Pause the CommandableFragmentGenerator.
void getDataLoop()
When separate_data_thread is set to true, this loop repeatedly calls getNext_ and adds returned Fragm...
void sendEmptyFragments(FragmentPtrs &frags, std::map< Fragment::sequence_id_t, Fragment::timestamp_t > &requests)
This function is for Buffered and Single request modes, as they can only respond to one data request ...
void startMonitoringThread()
Function that launches the monitoring thread (getMonitoringDataLoop())
void checkDataBuffer()
Perform data buffer pruning operations. If the RequestMode is Single, removes all but the latest Frag...
std::string printMode_()
Return the string representation of the current RequestMode.
void getDataBufferStats()
Calculate the size of the dataBuffer and report appropriate metrics.
bool applyRequests(FragmentPtrs &output)
See if any requests have been received, and add the corresponding data Fragment objects to the output...
void joinThreads()
Join any data-taking threads. Should be called when destructing CommandableFragmentGenerator.