1 #define TRACE_NAME (app_name + "_CommandableFragmentGenerator").c_str() // include these 2 first -
2 #include "artdaq/DAQdata/Globals.hh"
4 #include "artdaq/Application/CommandableFragmentGenerator.hh"
6 #include <boost/exception/all.hpp>
7 #include <boost/throw_exception.hpp>
12 #include "canvas/Utilities/Exception.h"
13 #include "cetlib_except/exception.h"
14 #include "fhiclcpp/ParameterSet.h"
16 #include "artdaq-core/Utilities/SimpleLookupPolicy.hh"
17 #include "artdaq-core/Data/Fragment.hh"
18 #include "artdaq-core/Data/ContainerFragmentLoader.hh"
19 #include "artdaq-core/Utilities/ExceptionHandler.hh"
20 #include "artdaq-core/Utilities/TimeUtils.hh"
31 #define TLVL_GETNEXT 10
32 #define TLVL_GETNEXT_VERBOSE 20
33 #define TLVL_CHECKSTOP 11
34 #define TLVL_EVCOUNTERINC 12
35 #define TLVL_GETDATALOOP 13
36 #define TLVL_GETDATALOOP_DATABUFFWAIT 21
37 #define TLVL_GETDATALOOP_VERBOSE 20
38 #define TLVL_WAITFORBUFFERREADY 15
39 #define TLVL_GETBUFFERSTATS 16
40 #define TLVL_CHECKDATABUFFER 17
41 #define TLVL_GETMONITORINGDATA 18
42 #define TLVL_APPLYREQUESTS 9
43 #define TLVL_SENDEMPTYFRAGMENTS 19
44 #define TLVL_CHECKWINDOWS 14
48 , requestReceiver_(nullptr)
51 , staleTimeout_(Fragment::InvalidTimestamp)
52 , expectedType_(Fragment::EmptyFragmentType)
53 , maxFragmentCount_(std::numeric_limits<size_t>::max())
54 , uniqueWindows_(true)
56 , missing_request_window_timeout_us_(1000000)
57 , window_close_timeout_us_(2000000)
58 , useDataThread_(false)
59 , circularDataBufferMode_(false)
60 , sleep_on_no_data_us_(0)
61 , data_thread_running_(false)
62 , dataBufferDepthFragments_(0)
63 , dataBufferDepthBytes_(0)
64 , maxDataBufferDepthFragments_(1000)
65 , maxDataBufferDepthBytes_(1000)
66 , useMonitoringThread_(false)
67 , monitoringInterval_(0)
68 , lastMonitoringCall_()
74 , timeout_(std::numeric_limits<uint64_t>::max())
75 , timestamp_(std::numeric_limits<uint64_t>::max())
79 , latest_exception_report_(
"none")
82 , instance_name_for_metrics_(
"FragmentGenerator")
83 , sleep_on_stop_us_(0)
88 , requestReceiver_(nullptr)
89 , windowOffset_(ps.get<Fragment::timestamp_t>(
"request_window_offset", 0))
90 , windowWidth_(ps.get<Fragment::timestamp_t>(
"request_window_width", 0))
91 , staleTimeout_(ps.get<Fragment::timestamp_t>(
"stale_request_timeout", 0xFFFFFFFF))
92 , expectedType_(ps.get<Fragment::type_t>(
"expected_fragment_type", Fragment::type_t(Fragment::EmptyFragmentType)))
93 , uniqueWindows_(ps.get<bool>(
"request_windows_are_unique", true))
95 , missing_request_window_timeout_us_(ps.get<size_t>(
"missing_request_window_timeout_us", 5000000))
96 , window_close_timeout_us_(ps.get<size_t>(
"window_close_timeout_us", 2000000))
97 , useDataThread_(ps.get<bool>(
"separate_data_thread", false))
98 , circularDataBufferMode_(ps.get<bool>(
"circular_buffer_mode", false))
99 , sleep_on_no_data_us_(ps.get<size_t>(
"sleep_on_no_data_us", 0))
100 , data_thread_running_(false)
101 , dataBufferDepthFragments_(0)
102 , dataBufferDepthBytes_(0)
103 , maxDataBufferDepthFragments_(ps.get<int>(
"data_buffer_depth_fragments", 1000))
104 , maxDataBufferDepthBytes_(ps.get<size_t>(
"data_buffer_depth_mb", 1000) * 1024 * 1024)
105 , useMonitoringThread_(ps.get<bool>(
"separate_monitoring_thread", false))
106 , monitoringInterval_(ps.get<int64_t>(
"hardware_poll_interval_us", 0))
107 , lastMonitoringCall_()
108 , isHardwareOK_(true)
113 , timeout_(std::numeric_limits<uint64_t>::max())
114 , timestamp_(std::numeric_limits<uint64_t>::max())
115 , should_stop_(false)
118 , latest_exception_report_(
"none")
121 , sleep_on_stop_us_(0)
123 board_id_ = ps.get<
int>(
"board_id");
124 instance_name_for_metrics_ =
"BoardReader." + boost::lexical_cast<std::string>(board_id_);
126 fragment_ids_ = ps.get<std::vector<artdaq::Fragment::fragment_id_t>>(
"fragment_ids", std::vector<artdaq::Fragment::fragment_id_t>());
128 TLOG(TLVL_TRACE) <<
"artdaq::CommandableFragmentGenerator::CommandableFragmentGenerator(ps)";
131 if (fragment_id != -99)
133 if (fragment_ids_.size() != 0)
135 latest_exception_report_ =
"Error in CommandableFragmentGenerator: can't both define \"fragment_id\" and \"fragment_ids\" in FHiCL document";
136 throw cet::exception(latest_exception_report_);
140 fragment_ids_.emplace_back(fragment_id);
144 sleep_on_stop_us_ = ps.get<
int>(
"sleep_on_stop_us", 0);
146 std::string modeString = ps.get<std::string>(
"request_mode",
"ignored");
147 if (modeString ==
"single" || modeString ==
"Single")
149 mode_ = RequestMode::Single;
151 else if (modeString.find(
"buffer") != std::string::npos || modeString.find(
"Buffer") != std::string::npos)
153 mode_ = RequestMode::Buffer;
155 else if (modeString ==
"window" || modeString ==
"Window")
157 mode_ = RequestMode::Window;
159 else if (modeString.find(
"ignore") != std::string::npos || modeString.find(
"Ignore") != std::string::npos)
161 mode_ = RequestMode::Ignored;
163 TLOG(TLVL_DEBUG) <<
"Request mode is " <<
printMode_();
165 if (mode_ != RequestMode::Ignored)
169 latest_exception_report_ =
"Error in CommandableFragmentGenerator: use_data_thread must be true when request_mode is not \"Ignored\"!";
170 throw cet::exception(latest_exception_report_);
179 requestReceiver_.reset(
nullptr);
186 TLOG(TLVL_DEBUG) <<
"Joining dataThread";
187 if (dataThread_.joinable()) dataThread_.join();
188 TLOG(TLVL_DEBUG) <<
"Joining monitoringThread";
189 if (monitoringThread_.joinable()) monitoringThread_.join();
190 TLOG(TLVL_DEBUG) <<
"joinThreads complete";
197 if (check_stop()) usleep(sleep_on_stop_us_);
198 if (exception() || force_stop_)
return false;
200 if (!useMonitoringThread_ && monitoringInterval_ > 0)
202 TLOG(TLVL_GETNEXT) <<
"getNext: Checking whether to collect Monitoring Data";
203 auto now = std::chrono::steady_clock::now();
205 if (TimeUtils::GetElapsedTimeMicroseconds(lastMonitoringCall_, now) >= static_cast<size_t>(monitoringInterval_))
207 TLOG(TLVL_GETNEXT) <<
"getNext: Collecting Monitoring Data";
208 isHardwareOK_ = checkHWStatus_();
209 TLOG(TLVL_GETNEXT) <<
"getNext: isHardwareOK_ is now " << std::boolalpha << isHardwareOK_;
210 lastMonitoringCall_ = now;
216 std::lock_guard<std::mutex> lk(mutex_);
219 TLOG(TLVL_TRACE) <<
"getNext: Calling applyRequests";
220 result = applyRequests(output);
221 TLOG(TLVL_TRACE) <<
"getNext: Done with applyRequests result=" << std::boolalpha << result;
222 for (
auto dataIter = output.begin(); dataIter != output.end(); ++dataIter)
224 TLOG(20) <<
"getNext: applyRequests() returned fragment with sequenceID = " << (*dataIter)->sequenceID()
225 <<
", timestamp = " << (*dataIter)->timestamp() <<
", and sizeBytes = " << (*dataIter)->sizeBytes();
230 TLOG(TLVL_ERROR) <<
"Exception found in BoardReader with board ID " << board_id() <<
"; BoardReader will now return error status when queried";
231 throw cet::exception(
"CommandableFragmentGenerator") <<
"Exception found in BoardReader with board ID " << board_id() <<
"; BoardReader will now return error status when queried";
238 TLOG(TLVL_ERROR) <<
"Stopping CFG because the hardware reports bad status!";
241 TLOG(TLVL_TRACE) <<
"getNext: Calling getNext_ w/ ev_counter()=" << ev_counter();
244 result = getNext_(output);
250 TLOG(TLVL_TRACE) <<
"getNext: Done with getNext_ - ev_counter() now " << ev_counter();
251 for (
auto dataIter = output.begin(); dataIter != output.end(); ++dataIter)
253 TLOG(TLVL_GETNEXT_VERBOSE) <<
"getNext: getNext_() returned fragment with sequenceID = " << (*dataIter)->sequenceID()
254 <<
", timestamp = " << (*dataIter)->timestamp() <<
", and sizeBytes = " << (*dataIter)->sizeBytes();
258 catch (
const cet::exception& e)
260 latest_exception_report_ =
"cet::exception caught in getNext(): ";
261 latest_exception_report_.append(e.what());
262 TLOG(TLVL_ERROR) <<
"getNext: cet::exception caught: " << e;
266 catch (
const boost::exception& e)
268 latest_exception_report_ =
"boost::exception caught in getNext(): ";
269 latest_exception_report_.append(boost::diagnostic_information(e));
270 TLOG(TLVL_ERROR) <<
"getNext: boost::exception caught: " << boost::diagnostic_information(e);
274 catch (
const std::exception& e)
276 latest_exception_report_ =
"std::exception caught in getNext(): ";
277 latest_exception_report_.append(e.what());
278 TLOG(TLVL_ERROR) <<
"getNext: std::exception caught: " << e.what();
284 latest_exception_report_ =
"Unknown exception caught in getNext().";
285 TLOG(TLVL_ERROR) <<
"getNext: unknown exception caught";
292 TLOG(TLVL_DEBUG) <<
"getNext: Either getNext_ or applyRequests returned false, stopping";
295 if (metricMan && !output.empty())
297 auto timestamp = output.front()->timestamp();
299 if (output.size() > 1)
301 for (
auto& outputfrag : output)
303 if (outputfrag->timestamp() > timestamp)
305 timestamp = outputfrag->timestamp();
310 metricMan->sendMetric(
"Last Timestamp", timestamp,
"Ticks", 1,
311 MetricMode::LastPoint, app_name);
319 TLOG(TLVL_CHECKSTOP) <<
"CFG::check_stop: should_stop=" << should_stop() <<
", useDataThread_=" << useDataThread_ <<
", exception status =" << int(exception());
321 if (!should_stop())
return false;
322 if (!useDataThread_ || mode_ == RequestMode::Ignored)
return true;
323 if (force_stop_)
return true;
326 TLOG(TLVL_DEBUG) <<
"should_stop is true, force_stop_ is false, requestReceiver_->isRunning() is " << std::boolalpha << requestReceiver_->isRunning();
327 return !requestReceiver_->isRunning();
332 if (fragment_ids_.size() != 1)
334 throw cet::exception(
"Error in CommandableFragmentGenerator: can't call fragment_id() unless member fragment_ids_ vector is length 1");
338 return fragment_ids_[0];
344 if (force || mode_ == RequestMode::Ignored)
346 TLOG(TLVL_EVCOUNTERINC) <<
"ev_counter_inc: Incrementing ev_counter from " << ev_counter() <<
" by " << step;
347 return ev_counter_.fetch_add(step);
349 return ev_counter_.load();
354 TLOG(TLVL_TRACE) <<
"Start Command received.";
355 if (run < 0)
throw cet::exception(
"CommandableFragmentGenerator") <<
"negative run number";
358 timestamp_ = timestamp;
359 ev_counter_.store(1);
360 windows_sent_ooo_.clear();
362 std::unique_lock<std::mutex> lock(dataBufferMutex_);
363 dataBufferDepthBytes_ = 0;
364 dataBufferDepthFragments_ = 0;
367 should_stop_.store(
false);
368 force_stop_.store(
false);
369 exception_.store(
false);
372 latest_exception_report_ =
"none";
376 std::unique_lock<std::mutex> lk(mutex_);
377 if (useDataThread_) startDataThread();
378 if (useMonitoringThread_) startMonitoringThread();
379 if (mode_ != RequestMode::Ignored)
381 requestReceiver_->SetRunNumber(static_cast<uint32_t>(run));
382 requestReceiver_->startRequestReception();
384 TLOG(TLVL_TRACE) <<
"Start Command complete.";
389 TLOG(TLVL_TRACE) <<
"Stop Command received.";
392 timestamp_ = timestamp;
393 if (requestReceiver_) {
394 TLOG(TLVL_DEBUG) <<
"Stopping Request reception BEGIN";
395 requestReceiver_->stopRequestReception();
396 TLOG(TLVL_DEBUG) <<
"Stopping Request reception END";
400 should_stop_.store(
true);
401 std::unique_lock<std::mutex> lk(mutex_);
405 TLOG(TLVL_TRACE) <<
"Stop Command complete.";
410 TLOG(TLVL_TRACE) <<
"Pause Command received.";
412 timestamp_ = timestamp;
416 should_stop_.store(
true);
417 std::unique_lock<std::mutex> lk(mutex_);
424 TLOG(TLVL_TRACE) <<
"Resume Command received.";
426 timestamp_ = timestamp;
429 should_stop_ =
false;
431 std::unique_lock<std::mutex> lk(dataBufferMutex_);
432 dataBufferDepthBytes_ = 0;
433 dataBufferDepthFragments_ = 0;
439 std::unique_lock<std::mutex> lk(mutex_);
443 TLOG(TLVL_TRACE) <<
"Resume Command complete.";
448 TLOG(TLVL_TRACE) <<
"Report Command received.";
449 std::lock_guard<std::mutex> lk(mutex_);
456 std::string childReport = reportSpecific(which);
457 if (childReport.length() > 0) {
return childReport; }
460 if (which ==
"latest_exception")
462 return latest_exception_report_;
466 childReport = report();
467 if (childReport.length() > 0) {
return childReport; }
470 std::string tmpString =
"The \"" + which +
"\" command is not ";
471 tmpString.append(
"currently supported by the ");
472 tmpString.append(metricsReportingInstanceName());
473 tmpString.append(
" fragment generator.");
474 TLOG(TLVL_TRACE) <<
"Report Command complete.";
481 #pragma message "Using default implementation of CommandableFragmentGenerator::pauseNoMutex()"
486 #pragma message "Using default implementation of CommandableFragmentGenerator::pause()"
491 #pragma message "Using default implementation of CommandableFragmentGenerator::resume()"
496 #pragma message "Using default implementation of CommandableFragmentGenerator::report()"
502 #pragma message "Using default implementation of CommandableFragmentGenerator::reportSpecific(std::string)"
508 #pragma message "Using default implementation of CommandableFragmentGenerator::checkHWStatus_()"
514 #pragma message "Using default implementation of CommandableFragmentGenerator::metaCommand(std::string, std::string)"
520 if (dataThread_.joinable()) dataThread_.join();
521 TLOG(TLVL_INFO) <<
"Starting Data Receiver Thread";
525 catch (
const boost::exception& e)
527 TLOG(TLVL_ERROR) <<
"Caught boost::exception starting Data Receiver thread: " << boost::diagnostic_information(e) <<
", errno=" << errno;
528 std::cerr <<
"Caught boost::exception starting Data Receiver thread: " << boost::diagnostic_information(e) <<
", errno=" << errno << std::endl;
535 if (monitoringThread_.joinable()) monitoringThread_.join();
536 TLOG(TLVL_INFO) <<
"Starting Hardware Monitoring Thread";
540 catch (
const boost::exception& e)
542 TLOG(TLVL_ERROR) <<
"Caught boost::exception starting Hardware Monitoring thread: " << boost::diagnostic_information(e) <<
", errno=" << errno;
543 std::cerr <<
"Caught boost::exception starting Hardware Monitoring thread: " << boost::diagnostic_information(e) <<
", errno=" << errno << std::endl;
552 case RequestMode::Single:
554 case RequestMode::Buffer:
556 case RequestMode::Window:
558 case RequestMode::Ignored:
571 data_thread_running_ =
true;
576 TLOG(TLVL_DEBUG) <<
"getDataLoop: isHardwareOK is " << isHardwareOK_ <<
", aborting data thread";
577 data_thread_running_ =
false;
581 TLOG(TLVL_GETDATALOOP) <<
"getDataLoop: calling getNext_";
584 auto startdata = std::chrono::steady_clock::now();
588 data = getNext_(newDataBuffer_);
592 ExceptionHandler(ExceptionHandlerRethrow::no,
593 "Exception thrown by fragment generator in CommandableFragmentGenerator::getDataLoop; setting exception state to \"true\"");
596 data_thread_running_ =
false;
600 size_t newDataBufferDepthBytes = 0;
601 for (
auto dataIter = newDataBuffer_.begin(); dataIter != newDataBuffer_.end(); ++dataIter)
603 TLOG(TLVL_GETDATALOOP_VERBOSE) <<
"getDataLoop: getNext_() returned fragment with timestamp = " << (*dataIter)->timestamp() <<
", and sizeBytes = " << (*dataIter)->sizeBytes();
604 newDataBufferDepthBytes += (*dataIter)->sizeBytes();
609 metricMan->sendMetric(
"Avg Data Acquisition Time", TimeUtils::GetElapsedTime(startdata),
"s", 3, artdaq::MetricMode::Average);
612 if (newDataBuffer_.size() == 0 && sleep_on_no_data_us_ > 0)
614 usleep(sleep_on_no_data_us_);
617 TLOG(TLVL_GETDATALOOP_DATABUFFWAIT) <<
"Waiting for data buffer ready";
618 if (!waitForDataBufferReady())
return;
619 TLOG(TLVL_GETDATALOOP_DATABUFFWAIT) <<
"Done waiting for data buffer ready";
621 TLOG(TLVL_GETDATALOOP) <<
"getDataLoop: processing data";
622 if (data && !force_stop_)
624 std::unique_lock<std::mutex> lock(dataBufferMutex_);
627 case RequestMode::Single:
629 while (newDataBuffer_.size() >= fragment_ids_.size())
631 dataBufferDepthBytes_ = 0;
632 dataBufferDepthFragments_ = 0;
634 auto it = newDataBuffer_.begin();
635 std::advance(it, fragment_ids_.size());
636 dataBuffer_.splice(dataBuffer_.end(), newDataBuffer_, newDataBuffer_.begin(), it);
638 for (
auto dbit = dataBuffer_.begin(); dbit != dataBuffer_.end(); ++dbit)
640 dataBufferDepthBytes_ += (*dbit)->sizeBytes();
644 case RequestMode::Buffer:
645 case RequestMode::Ignored:
646 case RequestMode::Window:
649 dataBufferDepthBytes_ += newDataBufferDepthBytes;
650 dataBuffer_.splice(dataBuffer_.end(), newDataBuffer_);
653 getDataBufferStats();
657 std::unique_lock<std::mutex> lock(dataBufferMutex_);
658 if (dataBuffer_.size() > 0)
660 dataCondition_.notify_all();
663 if (!data || force_stop_)
665 TLOG(TLVL_INFO) <<
"Data flow has stopped. Ending data collection thread";
666 std::unique_lock<std::mutex> lock(dataBufferMutex_);
667 data_thread_running_ =
false;
668 if (requestReceiver_) requestReceiver_->ClearRequests();
669 newDataBuffer_.clear();
670 TLOG(TLVL_INFO) <<
"getDataLoop: Ending thread";
678 auto startwait = std::chrono::steady_clock::now();
680 auto lastwaittime = 0ULL;
683 std::unique_lock<std::mutex> lock(dataBufferMutex_);
684 getDataBufferStats();
687 while (dataBufferIsTooLarge())
689 if (!circularDataBufferMode_)
693 TLOG(TLVL_DEBUG) <<
"Run ended while waiting for buffer to shrink!";
694 std::unique_lock<std::mutex> lock(dataBufferMutex_);
695 getDataBufferStats();
696 dataCondition_.notify_all();
697 data_thread_running_ =
false;
700 auto waittime = TimeUtils::GetElapsedTimeMilliseconds(startwait);
702 if (first || (waittime != lastwaittime && waittime % 1000 == 0))
704 TLOG(TLVL_WARNING) <<
"Bad Omen: Data Buffer has exceeded its size limits. "
705 <<
"(seq_id=" << ev_counter()
706 <<
", frags=" << dataBufferDepthFragments_ <<
"/" << maxDataBufferDepthFragments_
707 <<
", szB=" << dataBufferDepthBytes_ <<
"/" << maxDataBufferDepthBytes_ <<
")";
708 TLOG(TLVL_TRACE) <<
"Bad Omen: Possible causes include requests not getting through or Ignored-mode BR issues";
711 if (waittime % 5 && waittime != lastwaittime)
713 TLOG(TLVL_WAITFORBUFFERREADY) <<
"getDataLoop: Data Retreival paused for " << waittime <<
" ms waiting for data buffer to drain";
715 lastwaittime = waittime;
720 std::unique_lock<std::mutex> lock(dataBufferMutex_);
721 getDataBufferStats();
722 if (dataBufferIsTooLarge())
724 if (dataBuffer_.begin() == dataBuffer_.end())
726 TLOG(TLVL_WARNING) <<
"Data buffer is reported as too large, but doesn't contain any Fragments! Possible corrupt memory!";
729 if (*dataBuffer_.begin())
731 TLOG(TLVL_WAITFORBUFFERREADY) <<
"waitForDataBufferReady: Dropping Fragment with timestamp " << (*dataBuffer_.begin())->timestamp() <<
" from data buffer (Buffer over-size, circular data buffer mode)";
733 dataBufferDepthBytes_ -= (*dataBuffer_.begin())->sizeBytes();
734 dataBuffer_.erase(dataBuffer_.begin());
735 getDataBufferStats();
745 return (maxDataBufferDepthFragments_ > 0 && dataBufferDepthFragments_ > maxDataBufferDepthFragments_) || (maxDataBufferDepthBytes_ > 0 && dataBufferDepthBytes_ > maxDataBufferDepthBytes_);
751 dataBufferDepthFragments_ = dataBuffer_.size();
755 TLOG(TLVL_GETBUFFERSTATS) <<
"getDataBufferStats: Sending Metrics";
756 metricMan->sendMetric(
"Buffer Depth Fragments", dataBufferDepthFragments_.load(),
"fragments", 1, MetricMode::LastPoint);
757 metricMan->sendMetric(
"Buffer Depth Bytes", dataBufferDepthBytes_.load(),
"bytes", 1, MetricMode::LastPoint);
759 TLOG(TLVL_GETBUFFERSTATS) <<
"getDataBufferStats: frags=" << dataBufferDepthFragments_.load() <<
"/" << maxDataBufferDepthFragments_
760 <<
", sz=" << dataBufferDepthBytes_.load() <<
"/" << maxDataBufferDepthBytes_;
765 std::unique_lock<std::mutex> lock(dataBufferMutex_);
766 dataCondition_.wait_for(lock, std::chrono::milliseconds(10));
767 if (dataBufferDepthFragments_ > 0)
769 if ((mode_ == RequestMode::Buffer || mode_ == RequestMode::Window))
772 getDataBufferStats();
773 while (dataBufferIsTooLarge())
775 TLOG(TLVL_CHECKDATABUFFER) <<
"checkDataBuffer: Dropping Fragment with timestamp " << (*dataBuffer_.begin())->timestamp() <<
" from data buffer (Buffer over-size)";
776 dataBufferDepthBytes_ -= (*dataBuffer_.begin())->sizeBytes();
777 dataBuffer_.erase(dataBuffer_.begin());
778 getDataBufferStats();
780 if (dataBuffer_.size() > 0)
782 TLOG(TLVL_CHECKDATABUFFER) <<
"Determining if Fragments can be dropped from data buffer";
783 Fragment::timestamp_t last = dataBuffer_.back()->timestamp();
784 Fragment::timestamp_t min = last > staleTimeout_ ? last - staleTimeout_ : 0;
785 for (
auto it = dataBuffer_.begin(); it != dataBuffer_.end();)
787 if ((*it)->timestamp() < min)
789 TLOG(TLVL_CHECKDATABUFFER) <<
"checkDataBuffer: Dropping Fragment with timestamp " << (*it)->timestamp() <<
" from data buffer (timeout=" << staleTimeout_ <<
", min=" << min <<
")";
790 dataBufferDepthBytes_ -= (*it)->sizeBytes();
791 it = dataBuffer_.erase(it);
798 getDataBufferStats();
801 else if (mode_ == RequestMode::Single && dataBuffer_.size() > fragment_ids_.size())
804 while (dataBuffer_.size() > fragment_ids_.size())
806 dataBufferDepthBytes_ -= (*dataBuffer_.begin())->sizeBytes();
807 dataBuffer_.erase(dataBuffer_.begin());
817 if (should_stop() || monitoringInterval_ <= 0)
819 TLOG(TLVL_DEBUG) <<
"getMonitoringDataLoop: should_stop() is " << std::boolalpha << should_stop()
820 <<
" and monitoringInterval is " << monitoringInterval_ <<
", returning";
823 TLOG(TLVL_GETMONITORINGDATA) <<
"getMonitoringDataLoop: Determining whether to call checkHWStatus_";
825 auto now = std::chrono::steady_clock::now();
826 if (TimeUtils::GetElapsedTimeMicroseconds(lastMonitoringCall_, now) >= static_cast<size_t>(monitoringInterval_))
828 isHardwareOK_ = checkHWStatus_();
829 TLOG(TLVL_GETMONITORINGDATA) <<
"getMonitoringDataLoop: isHardwareOK_ is now " << std::boolalpha << isHardwareOK_;
830 lastMonitoringCall_ = now;
832 usleep(monitoringInterval_ / 10);
839 TLOG(TLVL_APPLYREQUESTS) <<
"Mode is Ignored; Copying data to output";
840 std::move(dataBuffer_.begin(), dataBuffer_.end(), std::inserter(frags, frags.end()));
841 dataBufferDepthBytes_ = 0;
848 auto requests = requestReceiver_->GetRequests();
849 while (requests.size() > 1)
852 requestReceiver_->RemoveRequest(requests.begin()->first);
853 requests.erase(requests.begin());
855 sendEmptyFragments(frags, requests);
858 if (requests.size() == 0 || !requests.count(ev_counter()))
return;
860 if (dataBuffer_.size() > 0)
862 TLOG(TLVL_APPLYREQUESTS) <<
"Mode is Single; Sending copy of last event";
863 for (
auto& fragptr : dataBuffer_)
866 auto frag = fragptr.get();
867 auto newfrag = std::unique_ptr<artdaq::Fragment>(
new Fragment(ev_counter(), frag->fragmentID()));
868 newfrag->resize(frag->size() - detail::RawFragmentHeader::num_words());
869 memcpy(newfrag->headerAddress(), frag->headerAddress(), frag->sizeBytes());
870 newfrag->setTimestamp(requests[ev_counter()]);
871 newfrag->setSequenceID(ev_counter());
872 frags.push_back(std::move(newfrag));
877 sendEmptyFragment(frags, ev_counter(),
"No data for");
879 requestReceiver_->RemoveRequest(ev_counter());
880 ev_counter_inc(1,
true);
886 auto requests = requestReceiver_->GetRequests();
887 while (requests.size() > 1)
890 requestReceiver_->RemoveRequest(requests.begin()->first);
891 requests.erase(requests.begin());
893 sendEmptyFragments(frags, requests);
896 if (requests.size() == 0 || !requests.count(ev_counter()))
return;
898 TLOG(TLVL_DEBUG) <<
"Creating ContainerFragment for Buffered Fragments";
899 frags.emplace_back(
new artdaq::Fragment(ev_counter(), fragment_id()));
900 frags.back()->setTimestamp(requests[ev_counter()]);
901 ContainerFragmentLoader cfl(*frags.back());
902 cfl.set_missing_data(
false);
906 for (
auto it = dataBuffer_.begin(); it != dataBuffer_.end();)
908 TLOG(TLVL_APPLYREQUESTS) <<
"ApplyRequests: Adding Fragment with timestamp " << (*it)->timestamp() <<
" to Container with sequence ID " << ev_counter();
909 cfl.addFragment(*it);
910 dataBufferDepthBytes_ -= (*it)->sizeBytes();
911 it = dataBuffer_.erase(it);
913 requestReceiver_->RemoveRequest(ev_counter());
914 ev_counter_inc(1,
true);
919 TLOG(TLVL_APPLYREQUESTS) <<
"applyRequestsWindowMode BEGIN";
921 auto requests = requestReceiver_->GetRequests();
923 TLOG(TLVL_APPLYREQUESTS) <<
"applyRequestsWindowMode: Starting request processing";
924 for (
auto req = requests.begin(); req != requests.end();)
926 TLOG(TLVL_APPLYREQUESTS) <<
"applyRequestsWindowMode: processing request with sequence ID " << req->first <<
", timestamp " << req->second;
929 while (req->first < ev_counter() && requests.size() > 0)
931 TLOG(TLVL_APPLYREQUESTS) <<
"applyRequestsWindowMode: Clearing passed request for sequence ID " << req->first;
932 requestReceiver_->RemoveRequest(req->first);
933 req = requests.erase(req);
935 if (requests.size() == 0)
break;
937 auto ts = req->second;
938 TLOG(TLVL_APPLYREQUESTS) <<
"applyRequests: Checking that data exists for request window " << req->first;
939 Fragment::timestamp_t min = ts > windowOffset_ ? ts - windowOffset_ : 0;
940 Fragment::timestamp_t max = min + windowWidth_;
941 TLOG(TLVL_APPLYREQUESTS) <<
"ApplyRequests: min is " << min <<
", max is " << max
942 <<
" and first/last points in buffer are " << (dataBuffer_.size() > 0 ? dataBuffer_.front()->timestamp() : 0)
943 <<
"/" << (dataBuffer_.size() > 0 ? dataBuffer_.back()->timestamp() : 0)
944 <<
" (sz=" << dataBuffer_.size() <<
" [" << dataBufferDepthBytes_.load()
945 <<
"/" << maxDataBufferDepthBytes_ <<
"])";
946 bool windowClosed = dataBuffer_.size() > 0 && dataBuffer_.back()->timestamp() >= max;
947 bool windowTimeout = !windowClosed && TimeUtils::GetElapsedTimeMicroseconds(requestReceiver_->GetRequestTime(req->first)) > window_close_timeout_us_;
950 TLOG(TLVL_WARNING) <<
"applyRequests: A timeout occurred waiting for data to close the request window ({" << min <<
"-" << max
951 <<
"}, buffer={" << (dataBuffer_.size() > 0 ? dataBuffer_.front()->timestamp() : 0) <<
"-"
952 << (dataBuffer_.size() > 0 ? dataBuffer_.back()->timestamp() : 0)
953 <<
"} ). Time waiting: "
954 << TimeUtils::GetElapsedTimeMicroseconds(requestReceiver_->GetRequestTime(req->first)) <<
" us "
955 <<
"(> " << window_close_timeout_us_ <<
" us).";
957 if (windowClosed || !data_thread_running_ || windowTimeout)
959 TLOG(TLVL_DEBUG) <<
"applyRequests: Creating ContainerFragment for Window-requested Fragments";
960 frags.emplace_back(
new artdaq::Fragment(req->first, fragment_id()));
961 frags.back()->setTimestamp(ts);
962 ContainerFragmentLoader cfl(*frags.back());
976 if (!windowClosed || (dataBuffer_.size() > 0 && dataBuffer_.front()->timestamp() > min))
978 TLOG(TLVL_DEBUG) <<
"applyRequests: Request window starts before and/or ends after the current data buffer, setting ContainerFragment's missing_data flag!"
979 <<
" (requestWindowRange=[" << min <<
"," << max <<
"], "
980 <<
"buffer={" << (dataBuffer_.size() > 0 ? dataBuffer_.front()->timestamp() : 0) <<
"-"
981 << (dataBuffer_.size() > 0 ? dataBuffer_.back()->timestamp() : 0) <<
"}";
982 cfl.set_missing_data(
true);
988 for (
auto it = dataBuffer_.begin(); it != dataBuffer_.end();)
990 Fragment::timestamp_t fragT = (*it)->timestamp();
991 if (fragT < min || fragT > max || (fragT == max && windowWidth_ > 0))
997 TLOG(TLVL_APPLYREQUESTS) <<
"applyRequests: Adding (" << (++fragCount) <<
"th) Fragment with timestamp "
998 << (*it)->timestamp() <<
" and sizeBytes " << (*it)->sizeBytes()
999 <<
" to Container for sequence ID " << req->first;
1000 cfl.addFragment(*it);
1004 dataBufferDepthBytes_ -= (*it)->sizeBytes();
1005 it = dataBuffer_.erase(it);
1012 requestReceiver_->RemoveRequest(req->first);
1013 checkOutOfOrderWindows(req->first);
1014 requestReceiver_->RemoveRequest(req->first);
1015 req = requests.erase(req);
1026 if (check_stop() || exception())
1032 if (mode_ == RequestMode::Ignored)
1034 while (dataBufferDepthFragments_ <= 0)
1036 if (check_stop() || exception() || !isHardwareOK_)
return false;
1037 std::unique_lock<std::mutex> lock(dataBufferMutex_);
1038 dataCondition_.wait_for(lock, std::chrono::milliseconds(10), [
this]() {
return dataBufferDepthFragments_ > 0; });
1043 if ((check_stop() && requestReceiver_->size() == 0) || exception())
return false;
1049 while (requestReceiver_->size() == 0 && counter < 100)
1051 if (check_stop() || exception())
return false;
1055 requestReceiver_->WaitForRequests(10);
1061 std::unique_lock<std::mutex> dlk(dataBufferMutex_);
1065 case RequestMode::Single:
1066 applyRequestsSingleMode(frags);
1068 case RequestMode::Window:
1069 applyRequestsWindowMode(frags);
1071 case RequestMode::Buffer:
1072 applyRequestsBufferMode(frags);
1074 case RequestMode::Ignored:
1076 applyRequestsIgnoredMode(frags);
1080 if (!data_thread_running_ || force_stop_)
1082 TLOG(TLVL_INFO) <<
"Data thread has stopped; Clearing data buffer";
1083 dataBufferDepthBytes_ = 0;
1084 dataBuffer_.clear();
1087 getDataBufferStats();
1090 if (frags.size() > 0)
1091 TLOG(TLVL_APPLYREQUESTS) <<
"Finished Processing Event " << (*frags.begin())->sequenceID() <<
" for fragment_id " << fragment_id() <<
".";
1097 TLOG(TLVL_WARNING) << desc <<
" sequence ID " << seqId <<
", sending empty fragment";
1098 for (
auto fid : fragment_ids_)
1100 auto frag =
new Fragment();
1101 frag->setSequenceID(seqId);
1102 frag->setFragmentID(fid);
1103 frag->setSystemType(Fragment::EmptyFragmentType);
1104 frags.emplace_back(FragmentPtr(frag));
1111 if (requests.size() > 0)
1113 TLOG(TLVL_SENDEMPTYFRAGMENTS) <<
"Sending Empty Fragments for Sequence IDs from " << ev_counter() <<
" up to but not including " << requests.begin()->first;
1114 while (requests.begin()->first > ev_counter())
1116 sendEmptyFragment(frags, ev_counter(),
"Missed request for");
1117 ev_counter_inc(1,
true);
1124 windows_sent_ooo_[seq] = std::chrono::steady_clock::now();
1126 auto it = windows_sent_ooo_.begin();
1127 while (it != windows_sent_ooo_.end())
1129 if (seq == it->first && it->first == ev_counter())
1131 TLOG(TLVL_CHECKWINDOWS) <<
"checkOutOfOrderWindows: Sequence ID matches ev_counter, incrementing ev_counter (" << ev_counter() <<
")";
1132 ev_counter_inc(1,
true);
1133 it = windows_sent_ooo_.erase(it);
1135 else if (it->first <= ev_counter())
1137 TLOG(TLVL_CHECKWINDOWS) <<
"checkOutOfOrderWindows: Data-taking has caught up to out-of-order window request " << it->first <<
", removing from list. ev_counter=" << ev_counter();
1138 requestReceiver_->RemoveRequest(ev_counter());
1139 if (it->first == ev_counter()) ev_counter_inc(1,
true);
1140 it = windows_sent_ooo_.erase(it);
1142 else if (TimeUtils::GetElapsedTimeMicroseconds(it->second) > missing_request_window_timeout_us_)
1144 TLOG(TLVL_CHECKWINDOWS) <<
"checkOutOfOrderWindows: Out-of-order window " << it->first <<
" has timed out, setting current sequence ID and removing from list";
1145 while (ev_counter() <= it->first)
1147 if (ev_counter() < it->first) TLOG(TLVL_WARNING) <<
"Missed request for sequence ID " << ev_counter() <<
"! Will not send any data for this sequence ID!";
1148 requestReceiver_->RemoveRequest(ev_counter());
1149 ev_counter_inc(1,
true);
1151 windows_sent_ooo_.erase(windows_sent_ooo_.begin(), it);
1152 it = windows_sent_ooo_.erase(it);
1156 TLOG(TLVL_CHECKWINDOWS) <<
"checkOutOfOrderWindows: Out-of-order window " << it->first <<
" waiting. Current event counter = " << ev_counter();
int fragment_id() const
Get the current Fragment ID, if there is only one.
void applyRequestsSingleMode(artdaq::FragmentPtrs &frags)
Create fragments using data buffer for request mode Single. Precondition: dataBufferMutex_ and reques...
virtual bool checkHWStatus_()
Check any relavent hardware status registers. Return false if an error condition exists that should h...
virtual ~CommandableFragmentGenerator()
CommandableFragmentGenerator Destructor.
void applyRequestsBufferMode(artdaq::FragmentPtrs &frags)
Create fragments using data buffer for request mode Buffer. Precondition: dataBufferMutex_ and reques...
bool sendEmptyFragment(FragmentPtrs &frags, size_t sequenceId, std::string desc)
Send an EmptyFragmentType Fragment.
void getMonitoringDataLoop()
This function regularly calls checkHWStatus_(), and sets the isHardwareOK flag accordingly.
void startDataThread()
Function that launches the data thread (getDataLoop())
std::string ReportCmd(std::string const &which="")
Get a report about a user-specified run-time quantity.
virtual bool metaCommand(std::string const &command, std::string const &arg)
The meta-command is used for implementing user-specific commands in a CommandableFragmentGenerator.
bool dataBufferIsTooLarge()
Test the configured constraints on the data buffer.
void StopCmd(uint64_t timeout, uint64_t timestamp)
Stop the CommandableFragmentGenerator.
void applyRequestsWindowMode(artdaq::FragmentPtrs &frags)
Create fragments using data buffer for request mode Window. Precondition: dataBufferMutex_ and reques...
void StartCmd(int run, uint64_t timeout, uint64_t timestamp)
Start the CommandableFragmentGenerator.
virtual void pauseNoMutex()
On call to PauseCmd, pauseNoMutex() is called prior to PauseCmd acquiring the mutex ...
bool check_stop()
Routine used by applyRequests to make sure that all outstanding requests have been fulfilled before r...
void ResumeCmd(uint64_t timeout, uint64_t timestamp)
Resume the CommandableFragmentGenerator.
CommandableFragmentGenerator()
CommandableFragmentGenerator default constructor.
bool getNext(FragmentPtrs &output) overridefinal
getNext calls either applyRequests or getNext_ to get any data that is ready to be sent to the EventB...
bool waitForDataBufferReady()
Wait for the data buffer to drain (dataBufferIsTooLarge returns false), periodically reporting status...
size_t ev_counter_inc(size_t step=1, bool force=false)
Increment the event counter, if the current RequestMode allows it.
void applyRequestsIgnoredMode(artdaq::FragmentPtrs &frags)
Create fragments using data buffer for request mode Ignored. Precondition: dataBufferMutex_ and reque...
void PauseCmd(uint64_t timeout, uint64_t timestamp)
Pause the CommandableFragmentGenerator.
void getDataLoop()
When separate_data_thread is set to true, this loop repeatedly calls getNext_ and adds returned Fragm...
void sendEmptyFragments(FragmentPtrs &frags, std::map< Fragment::sequence_id_t, Fragment::timestamp_t > &requests)
This function is for Buffered and Single request modes, as they can only respond to one data request ...
Receive data requests and make them available to CommandableFragmentGenerator or other interested par...
void startMonitoringThread()
Function that launches the monitoring thread (getMonitoringDataLoop())
virtual void pause()
If a CommandableFragmentGenerator subclass is reading from hardware, the implementation of pause() sh...
virtual void resume()
The subrun number will be incremented before a call to resume.
void checkDataBuffer()
Perform data buffer pruning operations. If the RequestMode is Single, removes all but the latest Frag...
virtual std::string report()
Let's say that the contract with the report() functions is that they return a non-empty string if the...
std::string printMode_()
Return the string representation of the current RequestMode.
void getDataBufferStats()
Calculate the size of the dataBuffer and report appropriate metrics.
void checkOutOfOrderWindows(Fragment::sequence_id_t seq)
Check the windows_sent_ooo_ map for sequence IDs that may be removed.
virtual std::string reportSpecific(std::string const &what)
Report the status of a specific quantity
bool applyRequests(FragmentPtrs &output)
See if any requests have been received, and add the corresponding data Fragment objects to the output...
void joinThreads()
Join any data-taking threads. Should be called when destructing CommandableFragmentGenerator.