1 #define TRACE_NAME (app_name + "_CommandableFragmentGenerator").c_str() // include these 2 first -
2 #include "artdaq/DAQdata/Globals.hh"
4 #include "artdaq/Application/CommandableFragmentGenerator.hh"
6 #include <boost/exception/all.hpp>
7 #include <boost/throw_exception.hpp>
12 #include "canvas/Utilities/Exception.h"
13 #include "cetlib_except/exception.h"
14 #include "fhiclcpp/ParameterSet.h"
16 #include "artdaq-core/Utilities/SimpleLookupPolicy.hh"
17 #include "artdaq-core/Data/Fragment.hh"
18 #include "artdaq-core/Data/ContainerFragmentLoader.hh"
19 #include "artdaq-core/Utilities/ExceptionHandler.hh"
20 #include "artdaq-core/Utilities/TimeUtils.hh"
31 #define TLVL_GETNEXT 10
32 #define TLVL_GETNEXT_VERBOSE 20
33 #define TLVL_CHECKSTOP 11
34 #define TLVL_EVCOUNTERINC 12
35 #define TLVL_GETDATALOOP 13
36 #define TLVL_GETDATALOOP_DATABUFFWAIT 21
37 #define TLVL_GETDATALOOP_VERBOSE 20
38 #define TLVL_WAITFORBUFFERREADY 15
39 #define TLVL_GETBUFFERSTATS 16
40 #define TLVL_CHECKDATABUFFER 17
41 #define TLVL_GETMONITORINGDATA 18
42 #define TLVL_APPLYREQUESTS 9
43 #define TLVL_SENDEMPTYFRAGMENTS 19
44 #define TLVL_CHECKWINDOWS 14
48 , requestReceiver_(nullptr)
51 , staleTimeout_(Fragment::InvalidTimestamp)
52 , expectedType_(Fragment::EmptyFragmentType)
53 , maxFragmentCount_(std::numeric_limits<size_t>::max())
54 , uniqueWindows_(true)
56 , missing_request_window_timeout_us_(1000000)
57 , window_close_timeout_us_(2000000)
58 , useDataThread_(false)
59 , circularDataBufferMode_(false)
60 , sleep_on_no_data_us_(0)
61 , data_thread_running_(false)
62 , dataBufferDepthFragments_(0)
63 , dataBufferDepthBytes_(0)
64 , maxDataBufferDepthFragments_(1000)
65 , maxDataBufferDepthBytes_(1000)
66 , useMonitoringThread_(false)
67 , monitoringInterval_(0)
68 , lastMonitoringCall_()
74 , timeout_(std::numeric_limits<uint64_t>::max())
75 , timestamp_(std::numeric_limits<uint64_t>::max())
79 , latest_exception_report_(
"none")
82 , instance_name_for_metrics_(
"FragmentGenerator")
83 , sleep_on_stop_us_(0)
88 , requestReceiver_(nullptr)
89 , windowOffset_(ps.get<Fragment::timestamp_t>(
"request_window_offset", 0))
90 , windowWidth_(ps.get<Fragment::timestamp_t>(
"request_window_width", 0))
91 , staleTimeout_(ps.get<Fragment::timestamp_t>(
"stale_request_timeout", 0xFFFFFFFF))
92 , expectedType_(ps.get<Fragment::type_t>(
"expected_fragment_type", Fragment::type_t(Fragment::EmptyFragmentType)))
93 , uniqueWindows_(ps.get<bool>(
"request_windows_are_unique", true))
95 , missing_request_window_timeout_us_(ps.get<size_t>(
"missing_request_window_timeout_us", 5000000))
96 , window_close_timeout_us_(ps.get<size_t>(
"window_close_timeout_us", 2000000))
97 , useDataThread_(ps.get<bool>(
"separate_data_thread", false))
98 , circularDataBufferMode_(ps.get<bool>(
"circular_buffer_mode", false))
99 , sleep_on_no_data_us_(ps.get<size_t>(
"sleep_on_no_data_us", 0))
100 , data_thread_running_(false)
101 , dataBufferDepthFragments_(0)
102 , dataBufferDepthBytes_(0)
103 , maxDataBufferDepthFragments_(ps.get<int>(
"data_buffer_depth_fragments", 1000))
104 , maxDataBufferDepthBytes_(ps.get<size_t>(
"data_buffer_depth_mb", 1000) * 1024 * 1024)
105 , useMonitoringThread_(ps.get<bool>(
"separate_monitoring_thread", false))
106 , monitoringInterval_(ps.get<int64_t>(
"hardware_poll_interval_us", 0))
107 , lastMonitoringCall_()
108 , isHardwareOK_(true)
113 , timeout_(std::numeric_limits<uint64_t>::max())
114 , timestamp_(std::numeric_limits<uint64_t>::max())
115 , should_stop_(false)
118 , latest_exception_report_(
"none")
121 , sleep_on_stop_us_(0)
123 board_id_ = ps.get<
int>(
"board_id");
124 instance_name_for_metrics_ =
"BoardReader." + boost::lexical_cast<std::string>(board_id_);
126 fragment_ids_ = ps.get<std::vector<artdaq::Fragment::fragment_id_t>>(
"fragment_ids", std::vector<artdaq::Fragment::fragment_id_t>());
128 TLOG(TLVL_TRACE) <<
"artdaq::CommandableFragmentGenerator::CommandableFragmentGenerator(ps)";
131 if (fragment_id != -99)
133 if (fragment_ids_.size() != 0)
135 latest_exception_report_ =
"Error in CommandableFragmentGenerator: can't both define \"fragment_id\" and \"fragment_ids\" in FHiCL document";
136 throw cet::exception(latest_exception_report_);
140 fragment_ids_.emplace_back(fragment_id);
144 sleep_on_stop_us_ = ps.get<
int>(
"sleep_on_stop_us", 0);
146 dataBuffer_.emplace_back(FragmentPtr(
new Fragment()));
147 (*dataBuffer_.begin())->setSystemType(Fragment::EmptyFragmentType);
149 std::string modeString = ps.get<std::string>(
"request_mode",
"ignored");
150 if (modeString ==
"single" || modeString ==
"Single")
152 mode_ = RequestMode::Single;
154 else if (modeString.find(
"buffer") != std::string::npos || modeString.find(
"Buffer") != std::string::npos)
156 mode_ = RequestMode::Buffer;
158 else if (modeString ==
"window" || modeString ==
"Window")
160 mode_ = RequestMode::Window;
162 else if (modeString.find(
"ignore") != std::string::npos || modeString.find(
"Ignore") != std::string::npos)
164 mode_ = RequestMode::Ignored;
166 TLOG(TLVL_DEBUG) <<
"Request mode is " <<
printMode_();
168 if (mode_ != RequestMode::Ignored)
172 latest_exception_report_ =
"Error in CommandableFragmentGenerator: use_data_thread must be true when request_mode is not \"Ignored\"!";
173 throw cet::exception(latest_exception_report_);
182 requestReceiver_.reset(
nullptr);
189 TLOG(TLVL_DEBUG) <<
"Joining dataThread";
190 if (dataThread_.joinable()) dataThread_.join();
191 TLOG(TLVL_DEBUG) <<
"Joining monitoringThread";
192 if (monitoringThread_.joinable()) monitoringThread_.join();
193 TLOG(TLVL_DEBUG) <<
"joinThreads complete";
200 if (check_stop()) usleep(sleep_on_stop_us_);
201 if (exception() || force_stop_)
return false;
203 if (!useMonitoringThread_ && monitoringInterval_ > 0)
205 TLOG(TLVL_GETNEXT) <<
"getNext: Checking whether to collect Monitoring Data";
206 auto now = std::chrono::steady_clock::now();
208 if (TimeUtils::GetElapsedTimeMicroseconds(lastMonitoringCall_, now) >= static_cast<size_t>(monitoringInterval_))
210 TLOG(TLVL_GETNEXT) <<
"getNext: Collecting Monitoring Data";
211 isHardwareOK_ = checkHWStatus_();
212 TLOG(TLVL_GETNEXT) <<
"getNext: isHardwareOK_ is now " << std::boolalpha << isHardwareOK_;
213 lastMonitoringCall_ = now;
219 std::lock_guard<std::mutex> lk(mutex_);
222 TLOG(TLVL_TRACE) <<
"getNext: Calling applyRequests";
223 result = applyRequests(output);
224 TLOG(TLVL_TRACE) <<
"getNext: Done with applyRequests result=" << std::boolalpha << result;
225 for (
auto dataIter = output.begin(); dataIter != output.end(); ++dataIter)
227 TLOG(20) <<
"getNext: applyRequests() returned fragment with sequenceID = " << (*dataIter)->sequenceID()
228 <<
", timestamp = " << (*dataIter)->timestamp() <<
", and sizeBytes = " << (*dataIter)->sizeBytes();
233 TLOG(TLVL_ERROR) <<
"Exception found in BoardReader with board ID " << board_id() <<
"; BoardReader will now return error status when queried";
234 throw cet::exception(
"CommandableFragmentGenerator") <<
"Exception found in BoardReader with board ID " << board_id() <<
"; BoardReader will now return error status when queried";
241 TLOG(TLVL_ERROR) <<
"Stopping CFG because the hardware reports bad status!";
244 TLOG(TLVL_TRACE) <<
"getNext: Calling getNext_ w/ ev_counter()=" << ev_counter();
247 result = getNext_(output);
253 TLOG(TLVL_TRACE) <<
"getNext: Done with getNext_ - ev_counter() now " << ev_counter();
254 for (
auto dataIter = output.begin(); dataIter != output.end(); ++dataIter)
256 TLOG(TLVL_GETNEXT_VERBOSE) <<
"getNext: getNext_() returned fragment with sequenceID = " << (*dataIter)->sequenceID()
257 <<
", timestamp = " << (*dataIter)->timestamp() <<
", and sizeBytes = " << (*dataIter)->sizeBytes();
261 catch (
const cet::exception& e)
263 latest_exception_report_ =
"cet::exception caught in getNext(): ";
264 latest_exception_report_.append(e.what());
265 TLOG(TLVL_ERROR) <<
"getNext: cet::exception caught: " << e;
269 catch (
const boost::exception& e)
271 latest_exception_report_ =
"boost::exception caught in getNext(): ";
272 latest_exception_report_.append(boost::diagnostic_information(e));
273 TLOG(TLVL_ERROR) <<
"getNext: boost::exception caught: " << boost::diagnostic_information(e);
277 catch (
const std::exception& e)
279 latest_exception_report_ =
"std::exception caught in getNext(): ";
280 latest_exception_report_.append(e.what());
281 TLOG(TLVL_ERROR) <<
"getNext: std::exception caught: " << e.what();
287 latest_exception_report_ =
"Unknown exception caught in getNext().";
288 TLOG(TLVL_ERROR) <<
"getNext: unknown exception caught";
295 TLOG(TLVL_DEBUG) <<
"getNext: Either getNext_ or applyRequests returned false, stopping";
298 if (metricMan && !output.empty())
300 auto timestamp = output.front()->timestamp();
302 if (output.size() > 1)
304 for (
auto& outputfrag : output)
306 if (outputfrag->timestamp() > timestamp)
308 timestamp = outputfrag->timestamp();
313 metricMan->sendMetric(
"Last Timestamp", timestamp,
"Ticks", 1,
314 MetricMode::LastPoint, app_name);
322 TLOG(TLVL_CHECKSTOP) <<
"CFG::check_stop: should_stop=" << should_stop() <<
", useDataThread_=" << useDataThread_ <<
", exception status =" << int(exception());
324 if (!should_stop())
return false;
325 if (!useDataThread_ || mode_ == RequestMode::Ignored)
return true;
326 if (force_stop_)
return true;
329 TLOG(TLVL_DEBUG) <<
"should_stop is true, force_stop_ is false, requestReceiver_->isRunning() is " << std::boolalpha << requestReceiver_->isRunning();
330 return !requestReceiver_->isRunning();
335 if (fragment_ids_.size() != 1)
337 throw cet::exception(
"Error in CommandableFragmentGenerator: can't call fragment_id() unless member fragment_ids_ vector is length 1");
341 return fragment_ids_[0];
347 if (force || mode_ == RequestMode::Ignored)
349 TLOG(TLVL_EVCOUNTERINC) <<
"ev_counter_inc: Incrementing ev_counter from " << ev_counter() <<
" by " << step;
350 return ev_counter_.fetch_add(step);
352 return ev_counter_.load();
357 TLOG(TLVL_TRACE) <<
"Start Command received.";
358 if (run < 0)
throw cet::exception(
"CommandableFragmentGenerator") <<
"negative run number";
361 timestamp_ = timestamp;
362 ev_counter_.store(1);
363 windows_sent_ooo_.clear();
365 should_stop_.store(
false);
366 force_stop_.store(
false);
367 exception_.store(
false);
370 latest_exception_report_ =
"none";
374 std::unique_lock<std::mutex> lk(mutex_);
375 if (useDataThread_) startDataThread();
376 if (useMonitoringThread_) startMonitoringThread();
377 if (mode_ != RequestMode::Ignored && !requestReceiver_->isRunning()) requestReceiver_->startRequestReceiverThread();
378 TLOG(TLVL_TRACE) <<
"Start Command complete.";
383 TLOG(TLVL_TRACE) <<
"Stop Command received.";
386 timestamp_ = timestamp;
387 if (requestReceiver_ && requestReceiver_->isRunning()) {
388 TLOG(TLVL_DEBUG) <<
"Stopping Request receiver thread BEGIN";
389 requestReceiver_->stopRequestReceiverThread();
390 TLOG(TLVL_DEBUG) <<
"Stopping Request receiver thread END";
394 should_stop_.store(
true);
395 std::unique_lock<std::mutex> lk(mutex_);
399 TLOG(TLVL_TRACE) <<
"Stop Command complete.";
404 TLOG(TLVL_TRACE) <<
"Pause Command received.";
406 timestamp_ = timestamp;
410 should_stop_.store(
true);
411 std::unique_lock<std::mutex> lk(mutex_);
418 TLOG(TLVL_TRACE) <<
"Resume Command received.";
420 timestamp_ = timestamp;
423 should_stop_ =
false;
425 std::unique_lock<std::mutex> lk(dataBufferMutex_);
431 std::unique_lock<std::mutex> lk(mutex_);
435 TLOG(TLVL_TRACE) <<
"Resume Command complete.";
440 TLOG(TLVL_TRACE) <<
"Report Command received.";
441 std::lock_guard<std::mutex> lk(mutex_);
448 std::string childReport = reportSpecific(which);
449 if (childReport.length() > 0) {
return childReport; }
452 if (which ==
"latest_exception")
454 return latest_exception_report_;
458 childReport = report();
459 if (childReport.length() > 0) {
return childReport; }
462 std::string tmpString =
"The \"" + which +
"\" command is not ";
463 tmpString.append(
"currently supported by the ");
464 tmpString.append(metricsReportingInstanceName());
465 tmpString.append(
" fragment generator.");
466 TLOG(TLVL_TRACE) <<
"Report Command complete.";
473 #pragma message "Using default implementation of CommandableFragmentGenerator::pauseNoMutex()"
478 #pragma message "Using default implementation of CommandableFragmentGenerator::pause()"
483 #pragma message "Using default implementation of CommandableFragmentGenerator::resume()"
488 #pragma message "Using default implementation of CommandableFragmentGenerator::report()"
494 #pragma message "Using default implementation of CommandableFragmentGenerator::reportSpecific(std::string)"
500 #pragma message "Using default implementation of CommandableFragmentGenerator::checkHWStatus_()"
506 #pragma message "Using default implementation of CommandableFragmentGenerator::metaCommand(std::string, std::string)"
512 if (dataThread_.joinable()) dataThread_.join();
513 TLOG(TLVL_INFO) <<
"Starting Data Receiver Thread";
517 catch (
const boost::exception& e)
519 TLOG(TLVL_ERROR) <<
"Caught boost::exception starting Data Receiver thread: " << boost::diagnostic_information(e) <<
", errno=" << errno;
520 std::cerr <<
"Caught boost::exception starting Data Receiver thread: " << boost::diagnostic_information(e) <<
", errno=" << errno << std::endl;
527 if (monitoringThread_.joinable()) monitoringThread_.join();
528 TLOG(TLVL_INFO) <<
"Starting Hardware Monitoring Thread";
532 catch (
const boost::exception& e)
534 TLOG(TLVL_ERROR) <<
"Caught boost::exception starting Hardware Monitoring thread: " << boost::diagnostic_information(e) <<
", errno=" << errno;
535 std::cerr <<
"Caught boost::exception starting Hardware Monitoring thread: " << boost::diagnostic_information(e) <<
", errno=" << errno << std::endl;
544 case RequestMode::Single:
546 case RequestMode::Buffer:
548 case RequestMode::Window:
550 case RequestMode::Ignored:
563 data_thread_running_ =
true;
568 TLOG(TLVL_DEBUG) <<
"getDataLoop: isHardwareOK is " << isHardwareOK_ <<
", aborting data thread";
569 data_thread_running_ =
false;
573 TLOG(TLVL_GETDATALOOP) <<
"getDataLoop: calling getNext_";
576 auto startdata = std::chrono::steady_clock::now();
580 data = getNext_(newDataBuffer_);
584 ExceptionHandler(ExceptionHandlerRethrow::no,
585 "Exception thrown by fragment generator in CommandableFragmentGenerator::getDataLoop; setting exception state to \"true\"");
588 data_thread_running_ =
false;
591 for (
auto dataIter = newDataBuffer_.begin(); dataIter != newDataBuffer_.end(); ++dataIter)
593 TLOG(TLVL_GETDATALOOP_VERBOSE) <<
"getDataLoop: getNext_() returned fragment with timestamp = " << (*dataIter)->timestamp() <<
", and sizeBytes = " << (*dataIter)->sizeBytes();
598 metricMan->sendMetric(
"Avg Data Acquisition Time", TimeUtils::GetElapsedTime(startdata),
"s", 3, artdaq::MetricMode::Average);
601 if (newDataBuffer_.size() == 0 && sleep_on_no_data_us_ > 0)
603 usleep(sleep_on_no_data_us_);
606 TLOG(TLVL_GETDATALOOP_DATABUFFWAIT) <<
"Waiting for data buffer ready";
607 if (!waitForDataBufferReady())
return;
608 TLOG(TLVL_GETDATALOOP_DATABUFFWAIT) <<
"Done waiting for data buffer ready";
610 TLOG(TLVL_GETDATALOOP) <<
"getDataLoop: processing data";
611 if (data && !force_stop_)
613 std::unique_lock<std::mutex> lock(dataBufferMutex_);
616 case RequestMode::Single:
618 while (newDataBuffer_.size() >= fragment_ids_.size())
621 auto it = newDataBuffer_.begin();
622 std::advance(it, fragment_ids_.size());
623 dataBuffer_.splice(dataBuffer_.end(), newDataBuffer_, newDataBuffer_.begin(), it);
626 case RequestMode::Buffer:
627 case RequestMode::Ignored:
628 case RequestMode::Window:
631 dataBuffer_.splice(dataBuffer_.end(), newDataBuffer_);
634 getDataBufferStats();
638 std::unique_lock<std::mutex> lock(dataBufferMutex_);
639 if (dataBuffer_.size() > 0)
641 dataCondition_.notify_all();
644 if (!data || force_stop_)
646 TLOG(TLVL_INFO) <<
"Data flow has stopped. Ending data collection thread";
647 std::unique_lock<std::mutex> lock(dataBufferMutex_);
648 data_thread_running_ =
false;
649 if (requestReceiver_) requestReceiver_->ClearRequests();
650 newDataBuffer_.clear();
651 TLOG(TLVL_INFO) <<
"getDataLoop: Ending thread";
659 auto startwait = std::chrono::steady_clock::now();
661 auto lastwaittime = 0ULL;
664 std::unique_lock<std::mutex> lock(dataBufferMutex_);
665 getDataBufferStats();
668 while (dataBufferIsTooLarge())
670 if (!circularDataBufferMode_)
674 TLOG(TLVL_DEBUG) <<
"Run ended while waiting for buffer to shrink!";
675 std::unique_lock<std::mutex> lock(dataBufferMutex_);
676 getDataBufferStats();
677 dataCondition_.notify_all();
678 data_thread_running_ =
false;
681 auto waittime = TimeUtils::GetElapsedTimeMilliseconds(startwait);
683 if (first || (waittime != lastwaittime && waittime % 1000 == 0))
685 TLOG(TLVL_WARNING) <<
"Bad Omen: Data Buffer has exceeded its size limits. "
686 <<
"(seq_id=" << ev_counter()
687 <<
", frags=" << dataBufferDepthFragments_ <<
"/" << maxDataBufferDepthFragments_
688 <<
", szB=" << dataBufferDepthBytes_ <<
"/" << maxDataBufferDepthBytes_ <<
")";
689 TLOG(TLVL_TRACE) <<
"Bad Omen: Possible causes include requests not getting through or Ignored-mode BR issues";
692 if (waittime % 5 && waittime != lastwaittime)
694 TLOG(TLVL_WAITFORBUFFERREADY) <<
"getDataLoop: Data Retreival paused for " << waittime <<
" ms waiting for data buffer to drain";
696 lastwaittime = waittime;
701 std::unique_lock<std::mutex> lock(dataBufferMutex_);
702 getDataBufferStats();
703 if (dataBufferIsTooLarge())
705 if (dataBuffer_.begin() == dataBuffer_.end())
707 TLOG(TLVL_WARNING) <<
"Data buffer is reported as too large, but doesn't contain any Fragments! Possible corrupt memory!";
710 if (*dataBuffer_.begin())
712 TLOG(TLVL_WAITFORBUFFERREADY) <<
"waitForDataBufferReady: Dropping Fragment with timestamp " << (*dataBuffer_.begin())->timestamp() <<
" from data buffer (Buffer over-size, circular data buffer mode)";
714 dataBuffer_.erase(dataBuffer_.begin());
715 getDataBufferStats();
725 return (maxDataBufferDepthFragments_ > 0 && dataBufferDepthFragments_ > maxDataBufferDepthFragments_) || (maxDataBufferDepthBytes_ > 0 && dataBufferDepthBytes_ > maxDataBufferDepthBytes_);
731 dataBufferDepthFragments_ = dataBuffer_.size();
733 TLOG(TLVL_GETBUFFERSTATS) <<
"getDataBufferStats: Calculating buffer size";
734 for (
auto i = dataBuffer_.begin(); i != dataBuffer_.end(); ++i)
736 if (i->get() !=
nullptr)
738 acc += (*i)->sizeBytes();
741 dataBufferDepthBytes_ = acc;
745 TLOG(TLVL_GETBUFFERSTATS) <<
"getDataBufferStats: Sending Metrics";
746 metricMan->sendMetric(
"Buffer Depth Fragments", dataBufferDepthFragments_.load(),
"fragments", 1, MetricMode::LastPoint);
747 metricMan->sendMetric(
"Buffer Depth Bytes", dataBufferDepthBytes_.load(),
"bytes", 1, MetricMode::LastPoint);
749 TLOG(TLVL_GETBUFFERSTATS) <<
"getDataBufferStats: frags=" << dataBufferDepthFragments_.load() <<
"/" << maxDataBufferDepthFragments_
750 <<
", sz=" << dataBufferDepthBytes_.load() <<
"/" << maxDataBufferDepthBytes_;
755 std::unique_lock<std::mutex> lock(dataBufferMutex_);
756 dataCondition_.wait_for(lock, std::chrono::milliseconds(10));
757 if (dataBufferDepthFragments_ > 0)
759 if ((mode_ == RequestMode::Buffer || mode_ == RequestMode::Window))
762 while (dataBufferIsTooLarge())
764 TLOG(TLVL_CHECKDATABUFFER) <<
"checkDataBuffer: Dropping Fragment with timestamp " << (*dataBuffer_.begin())->timestamp() <<
" from data buffer (Buffer over-size)";
765 dataBuffer_.erase(dataBuffer_.begin());
766 getDataBufferStats();
768 if (dataBuffer_.size() > 0)
770 TLOG(TLVL_CHECKDATABUFFER) <<
"Determining if Fragments can be dropped from data buffer";
771 Fragment::timestamp_t last = dataBuffer_.back()->timestamp();
772 Fragment::timestamp_t min = last > staleTimeout_ ? last - staleTimeout_ : 0;
773 for (
auto it = dataBuffer_.begin(); it != dataBuffer_.end();)
775 if ((*it)->timestamp() < min)
777 TLOG(TLVL_CHECKDATABUFFER) <<
"checkDataBuffer: Dropping Fragment with timestamp " << (*it)->timestamp() <<
" from data buffer (timeout=" << staleTimeout_ <<
", min=" << min <<
")";
778 it = dataBuffer_.erase(it);
785 getDataBufferStats();
788 else if (mode_ == RequestMode::Single && dataBuffer_.size() > fragment_ids_.size())
791 while (dataBuffer_.size() > fragment_ids_.size())
793 dataBuffer_.erase(dataBuffer_.begin());
803 if (should_stop() || monitoringInterval_ <= 0)
805 TLOG(TLVL_DEBUG) <<
"getMonitoringDataLoop: should_stop() is " << std::boolalpha << should_stop()
806 <<
" and monitoringInterval is " << monitoringInterval_ <<
", returning";
809 TLOG(TLVL_GETMONITORINGDATA) <<
"getMonitoringDataLoop: Determining whether to call checkHWStatus_";
811 auto now = std::chrono::steady_clock::now();
812 if (TimeUtils::GetElapsedTimeMicroseconds(lastMonitoringCall_, now) >= static_cast<size_t>(monitoringInterval_))
814 isHardwareOK_ = checkHWStatus_();
815 TLOG(TLVL_GETMONITORINGDATA) <<
"getMonitoringDataLoop: isHardwareOK_ is now " << std::boolalpha << isHardwareOK_;
816 lastMonitoringCall_ = now;
818 usleep(monitoringInterval_ / 10);
825 TLOG(TLVL_APPLYREQUESTS) <<
"Mode is Ignored; Copying data to output";
826 std::move(dataBuffer_.begin(), dataBuffer_.end(), std::inserter(frags, frags.end()));
833 auto requests = requestReceiver_->GetRequests();
834 while (requests.size() > 1)
837 requestReceiver_->RemoveRequest(requests.begin()->first);
838 requests.erase(requests.begin());
840 sendEmptyFragments(frags, requests);
843 if (requests.size() == 0 || !requests.count(ev_counter()))
return;
845 if (dataBuffer_.size() > 0)
847 TLOG(TLVL_APPLYREQUESTS) <<
"Mode is Single; Sending copy of last event";
848 for (
auto& fragptr : dataBuffer_)
851 auto frag = fragptr.get();
852 auto newfrag = std::unique_ptr<artdaq::Fragment>(
new Fragment(ev_counter(), frag->fragmentID()));
853 newfrag->resize(frag->size() - detail::RawFragmentHeader::num_words());
854 memcpy(newfrag->headerAddress(), frag->headerAddress(), frag->sizeBytes());
855 newfrag->setTimestamp(requests[ev_counter()]);
856 newfrag->setSequenceID(ev_counter());
857 frags.push_back(std::move(newfrag));
862 sendEmptyFragment(frags, ev_counter(),
"No data for");
864 requestReceiver_->RemoveRequest(ev_counter());
865 ev_counter_inc(1,
true);
871 auto requests = requestReceiver_->GetRequests();
872 while (requests.size() > 1)
875 requestReceiver_->RemoveRequest(requests.begin()->first);
876 requests.erase(requests.begin());
878 sendEmptyFragments(frags, requests);
881 if (requests.size() == 0 || !requests.count(ev_counter()))
return;
883 TLOG(TLVL_DEBUG) <<
"Creating ContainerFragment for Buffered Fragments";
884 frags.emplace_back(
new artdaq::Fragment(ev_counter(), fragment_id()));
885 frags.back()->setTimestamp(requests[ev_counter()]);
886 ContainerFragmentLoader cfl(*frags.back());
887 cfl.set_missing_data(
false);
891 for (
auto it = dataBuffer_.begin(); it != dataBuffer_.end();)
893 TLOG(TLVL_APPLYREQUESTS) <<
"ApplyRequests: Adding Fragment with timestamp " << (*it)->timestamp() <<
" to Container with sequence ID " << ev_counter();
894 cfl.addFragment(*it);
895 it = dataBuffer_.erase(it);
897 requestReceiver_->RemoveRequest(ev_counter());
898 ev_counter_inc(1,
true);
903 TLOG(TLVL_APPLYREQUESTS) <<
"applyRequestsWindowMode BEGIN";
905 auto requests = requestReceiver_->GetRequests();
907 TLOG(TLVL_APPLYREQUESTS) <<
"applyRequestsWindowMode: Starting request processing";
908 for (
auto req = requests.begin(); req != requests.end();)
910 TLOG(TLVL_APPLYREQUESTS) <<
"applyRequestsWindowMode: processing request with sequence ID " << req->first <<
", timestamp " << req->second;
913 while (req->first < ev_counter() && requests.size() > 0)
915 TLOG(TLVL_APPLYREQUESTS) <<
"applyRequestsWindowMode: Clearing passed request for sequence ID " << req->first;
916 requestReceiver_->RemoveRequest(req->first);
917 req = requests.erase(req);
919 if (requests.size() == 0)
break;
921 auto ts = req->second;
922 TLOG(TLVL_APPLYREQUESTS) <<
"applyRequests: Checking that data exists for request window " << req->first;
923 Fragment::timestamp_t min = ts > windowOffset_ ? ts - windowOffset_ : 0;
924 Fragment::timestamp_t max = min + windowWidth_;
925 TLOG(TLVL_APPLYREQUESTS) <<
"ApplyRequests: min is " << min <<
", max is " << max
926 <<
" and last point in buffer is " << (dataBuffer_.size() > 0 ? dataBuffer_.back()->timestamp() : 0) <<
" (sz=" << dataBuffer_.size() <<
")";
927 bool windowClosed = dataBuffer_.size() > 0 && dataBuffer_.back()->timestamp() >= max;
928 bool windowTimeout = !windowClosed && TimeUtils::GetElapsedTimeMicroseconds(requestReceiver_->GetRequestTime(req->first)) > window_close_timeout_us_;
931 TLOG(TLVL_WARNING) <<
"applyRequests: A timeout occurred waiting for data to close the request window ({" << min <<
"-" << max
932 <<
"}, buffer={" << (dataBuffer_.size() > 0 ? dataBuffer_.front()->timestamp() : 0) <<
"-"
933 << (dataBuffer_.size() > 0 ? dataBuffer_.back()->timestamp() : 0)
934 <<
"} ). Time waiting: "
935 << TimeUtils::GetElapsedTimeMicroseconds(requestReceiver_->GetRequestTime(req->first)) <<
" us "
936 <<
"(> " << window_close_timeout_us_ <<
" us).";
938 if (windowClosed || !data_thread_running_ || windowTimeout)
940 TLOG(TLVL_DEBUG) <<
"applyRequests: Creating ContainerFragment for Window-requested Fragments";
941 frags.emplace_back(
new artdaq::Fragment(req->first, fragment_id()));
942 frags.back()->setTimestamp(ts);
943 ContainerFragmentLoader cfl(*frags.back());
957 if (!windowClosed || (dataBuffer_.size() > 0 && dataBuffer_.front()->timestamp() > min))
959 TLOG(TLVL_DEBUG) <<
"applyRequests: Request window starts before and/or ends after the current data buffer, setting ContainerFragment's missing_data flag!"
960 <<
" (requestWindowRange=[" << min <<
"," << max <<
"], "
961 <<
"buffer={" << (dataBuffer_.size() > 0 ? dataBuffer_.front()->timestamp() : 0) <<
"-"
962 << (dataBuffer_.size() > 0 ? dataBuffer_.back()->timestamp() : 0) <<
"}";
963 cfl.set_missing_data(
true);
968 for (
auto it = dataBuffer_.begin(); it != dataBuffer_.end();)
970 Fragment::timestamp_t fragT = (*it)->timestamp();
971 if (fragT < min || fragT > max || (fragT == max && windowWidth_ > 0))
977 TLOG(TLVL_APPLYREQUESTS) <<
"applyRequests: Adding Fragment with timestamp " << (*it)->timestamp() <<
" to Container";
978 cfl.addFragment(*it);
982 it = dataBuffer_.erase(it);
989 requestReceiver_->RemoveRequest(req->first);
990 checkOutOfOrderWindows(req->first);
991 requestReceiver_->RemoveRequest(req->first);
992 req = requests.erase(req);
1003 if (check_stop() || exception())
1009 if (mode_ == RequestMode::Ignored)
1011 while (dataBufferDepthFragments_ <= 0)
1013 if (check_stop() || exception() || !isHardwareOK_)
return false;
1014 std::unique_lock<std::mutex> lock(dataBufferMutex_);
1015 dataCondition_.wait_for(lock, std::chrono::milliseconds(10), [
this]() {
return dataBufferDepthFragments_ > 0; });
1020 if ((check_stop() && requestReceiver_->size() == 0) || exception())
return false;
1026 while (requestReceiver_->size() == 0 && counter < 100)
1028 if (check_stop() || exception())
return false;
1032 requestReceiver_->WaitForRequests(10);
1038 std::unique_lock<std::mutex> dlk(dataBufferMutex_);
1042 case RequestMode::Single:
1043 applyRequestsSingleMode(frags);
1045 case RequestMode::Window:
1046 applyRequestsWindowMode(frags);
1048 case RequestMode::Buffer:
1049 applyRequestsBufferMode(frags);
1051 case RequestMode::Ignored:
1053 applyRequestsIgnoredMode(frags);
1057 if (!data_thread_running_ || force_stop_)
1059 TLOG(TLVL_INFO) <<
"Data thread has stopped; Clearing data buffer";
1060 dataBuffer_.clear();
1063 getDataBufferStats();
1066 if (frags.size() > 0)
1067 TLOG(TLVL_APPLYREQUESTS) <<
"Finished Processing Event " << (*frags.begin())->sequenceID() <<
" for fragment_id " << fragment_id() <<
".";
1073 TLOG(TLVL_WARNING) << desc <<
" sequence ID " << seqId <<
", sending empty fragment";
1074 for (
auto fid : fragment_ids_)
1076 auto frag =
new Fragment();
1077 frag->setSequenceID(seqId);
1078 frag->setFragmentID(fid);
1079 frag->setSystemType(Fragment::EmptyFragmentType);
1080 frags.emplace_back(FragmentPtr(frag));
1087 if (requests.size() > 0)
1089 TLOG(TLVL_SENDEMPTYFRAGMENTS) <<
"Sending Empty Fragments for Sequence IDs from " << ev_counter() <<
" up to but not including " << requests.begin()->first;
1090 while (requests.begin()->first > ev_counter())
1092 sendEmptyFragment(frags, ev_counter(),
"Missed request for");
1093 ev_counter_inc(1,
true);
1100 windows_sent_ooo_[seq] = std::chrono::steady_clock::now();
1102 auto it = windows_sent_ooo_.begin();
1103 while (it != windows_sent_ooo_.end())
1105 if (seq == it->first && it->first == ev_counter())
1107 TLOG(TLVL_CHECKWINDOWS) <<
"checkOutOfOrderWindows: Sequence ID matches ev_counter, incrementing ev_counter (" << ev_counter() <<
")";
1108 ev_counter_inc(1,
true);
1109 it = windows_sent_ooo_.erase(it);
1111 else if (it->first <= ev_counter())
1113 TLOG(TLVL_CHECKWINDOWS) <<
"checkOutOfOrderWindows: Data-taking has caught up to out-of-order window request " << it->first <<
", removing from list. ev_counter=" << ev_counter();
1114 requestReceiver_->RemoveRequest(ev_counter());
1115 if (it->first == ev_counter()) ev_counter_inc(1,
true);
1116 it = windows_sent_ooo_.erase(it);
1118 else if (TimeUtils::GetElapsedTimeMicroseconds(it->second) > missing_request_window_timeout_us_)
1120 TLOG(TLVL_CHECKWINDOWS) <<
"checkOutOfOrderWindows: Out-of-order window " << it->first <<
" has timed out, setting current sequence ID and removing from list";
1121 while (ev_counter() <= it->first)
1123 if (ev_counter() < it->first) TLOG(TLVL_WARNING) <<
"Missed request for sequence ID " << ev_counter() <<
"! Will not send any data for this sequence ID!";
1124 requestReceiver_->RemoveRequest(ev_counter());
1125 ev_counter_inc(1,
true);
1127 windows_sent_ooo_.erase(windows_sent_ooo_.begin(), it);
1128 it = windows_sent_ooo_.erase(it);
1132 TLOG(TLVL_CHECKWINDOWS) <<
"checkOutOfOrderWindows: Out-of-order window " << it->first <<
" waiting. Current event counter = " << ev_counter();
int fragment_id() const
Get the current Fragment ID, if there is only one.
void applyRequestsSingleMode(artdaq::FragmentPtrs &frags)
Create fragments using data buffer for request mode Single. Precondition: dataBufferMutex_ and reques...
virtual bool checkHWStatus_()
Check any relavent hardware status registers. Return false if an error condition exists that should h...
virtual ~CommandableFragmentGenerator()
CommandableFragmentGenerator Destructor.
void applyRequestsBufferMode(artdaq::FragmentPtrs &frags)
Create fragments using data buffer for request mode Buffer. Precondition: dataBufferMutex_ and reques...
bool sendEmptyFragment(FragmentPtrs &frags, size_t sequenceId, std::string desc)
Send an EmptyFragmentType Fragment.
void getMonitoringDataLoop()
This function regularly calls checkHWStatus_(), and sets the isHardwareOK flag accordingly.
void startDataThread()
Function that launches the data thread (getDataLoop())
std::string ReportCmd(std::string const &which="")
Get a report about a user-specified run-time quantity.
virtual bool metaCommand(std::string const &command, std::string const &arg)
The meta-command is used for implementing user-specific commands in a CommandableFragmentGenerator.
bool dataBufferIsTooLarge()
Test the configured constraints on the data buffer.
void StopCmd(uint64_t timeout, uint64_t timestamp)
Stop the CommandableFragmentGenerator.
void applyRequestsWindowMode(artdaq::FragmentPtrs &frags)
Create fragments using data buffer for request mode Window. Precondition: dataBufferMutex_ and reques...
void StartCmd(int run, uint64_t timeout, uint64_t timestamp)
Start the CommandableFragmentGenerator.
virtual void pauseNoMutex()
On call to PauseCmd, pauseNoMutex() is called prior to PauseCmd acquiring the mutex ...
bool check_stop()
Routine used by applyRequests to make sure that all outstanding requests have been fulfilled before r...
void ResumeCmd(uint64_t timeout, uint64_t timestamp)
Resume the CommandableFragmentGenerator.
CommandableFragmentGenerator()
CommandableFragmentGenerator default constructor.
bool getNext(FragmentPtrs &output) overridefinal
getNext calls either applyRequests or getNext_ to get any data that is ready to be sent to the EventB...
bool waitForDataBufferReady()
Wait for the data buffer to drain (dataBufferIsTooLarge returns false), periodically reporting status...
size_t ev_counter_inc(size_t step=1, bool force=false)
Increment the event counter, if the current RequestMode allows it.
void applyRequestsIgnoredMode(artdaq::FragmentPtrs &frags)
Create fragments using data buffer for request mode Ignored. Precondition: dataBufferMutex_ and reque...
void PauseCmd(uint64_t timeout, uint64_t timestamp)
Pause the CommandableFragmentGenerator.
void getDataLoop()
When separate_data_thread is set to true, this loop repeatedly calls getNext_ and adds returned Fragm...
void sendEmptyFragments(FragmentPtrs &frags, std::map< Fragment::sequence_id_t, Fragment::timestamp_t > &requests)
This function is for Buffered and Single request modes, as they can only respond to one data request ...
Receive data requests and make them available to CommandableFragmentGenerator or other interested par...
void startMonitoringThread()
Function that launches the monitoring thread (getMonitoringDataLoop())
virtual void pause()
If a CommandableFragmentGenerator subclass is reading from hardware, the implementation of pause() sh...
virtual void resume()
The subrun number will be incremented before a call to resume.
void checkDataBuffer()
Perform data buffer pruning operations. If the RequestMode is Single, removes all but the latest Frag...
virtual std::string report()
Let's say that the contract with the report() functions is that they return a non-empty string if the...
std::string printMode_()
Return the string representation of the current RequestMode.
void getDataBufferStats()
Calculate the size of the dataBuffer and report appropriate metrics.
void checkOutOfOrderWindows(Fragment::sequence_id_t seq)
Check the windows_sent_ooo_ map for sequence IDs that may be removed.
virtual std::string reportSpecific(std::string const &what)
Report the status of a specific quantity
bool applyRequests(FragmentPtrs &output)
See if any requests have been received, and add the corresponding data Fragment objects to the output...
void joinThreads()
Join any data-taking threads. Should be called when destructing CommandableFragmentGenerator.