1 #define TRACE_NAME (app_name + "_CommandableFragmentGenerator").c_str() // include these 2 first -
2 #include "artdaq/DAQdata/Globals.hh"
4 #include "artdaq/Application/CommandableFragmentGenerator.hh"
6 #include <boost/exception/all.hpp>
7 #include <boost/throw_exception.hpp>
12 #include "canvas/Utilities/Exception.h"
13 #include "cetlib_except/exception.h"
14 #include "fhiclcpp/ParameterSet.h"
16 #include "artdaq-core/Utilities/SimpleLookupPolicy.hh"
17 #include "artdaq-core/Data/Fragment.hh"
18 #include "artdaq-core/Data/ContainerFragmentLoader.hh"
19 #include "artdaq-core/Utilities/ExceptionHandler.hh"
20 #include "artdaq-core/Utilities/TimeUtils.hh"
31 #define TLVL_GETNEXT 10
32 #define TLVL_GETNEXT_VERBOSE 20
33 #define TLVL_CHECKSTOP 11
34 #define TLVL_EVCOUNTERINC 12
35 #define TLVL_GETDATALOOP 13
36 #define TLVL_GETDATALOOP_DATABUFFWAIT 21
37 #define TLVL_GETDATALOOP_VERBOSE 20
38 #define TLVL_WAITFORBUFFERREADY 15
39 #define TLVL_GETBUFFERSTATS 16
40 #define TLVL_CHECKDATABUFFER 17
41 #define TLVL_GETMONITORINGDATA 18
42 #define TLVL_APPLYREQUESTS 9
43 #define TLVL_SENDEMPTYFRAGMENTS 19
44 #define TLVL_CHECKWINDOWS 14
48 , requestReceiver_(nullptr)
51 , staleTimeout_(Fragment::InvalidTimestamp)
52 , expectedType_(Fragment::EmptyFragmentType)
53 , maxFragmentCount_(std::numeric_limits<size_t>::max())
54 , uniqueWindows_(true)
56 , missing_request_window_timeout_us_(1000000)
57 , window_close_timeout_us_(2000000)
58 , useDataThread_(false)
59 , circularDataBufferMode_(false)
60 , sleep_on_no_data_us_(0)
61 , data_thread_running_(false)
62 , dataBufferDepthFragments_(0)
63 , dataBufferDepthBytes_(0)
64 , maxDataBufferDepthFragments_(1000)
65 , maxDataBufferDepthBytes_(1000)
66 , useMonitoringThread_(false)
67 , monitoringInterval_(0)
68 , lastMonitoringCall_()
74 , timeout_(std::numeric_limits<uint64_t>::max())
75 , timestamp_(std::numeric_limits<uint64_t>::max())
79 , latest_exception_report_(
"none")
82 , instance_name_for_metrics_(
"FragmentGenerator")
83 , sleep_on_stop_us_(0)
88 , requestReceiver_(nullptr)
89 , windowOffset_(ps.get<Fragment::timestamp_t>(
"request_window_offset", 0))
90 , windowWidth_(ps.get<Fragment::timestamp_t>(
"request_window_width", 0))
91 , staleTimeout_(ps.get<Fragment::timestamp_t>(
"stale_request_timeout", 0xFFFFFFFF))
92 , expectedType_(ps.get<Fragment::type_t>(
"expected_fragment_type", Fragment::type_t(Fragment::EmptyFragmentType)))
93 , uniqueWindows_(ps.get<bool>(
"request_windows_are_unique", true))
95 , missing_request_window_timeout_us_(ps.get<size_t>(
"missing_request_window_timeout_us", 5000000))
96 , window_close_timeout_us_(ps.get<size_t>(
"window_close_timeout_us", 2000000))
97 , useDataThread_(ps.get<bool>(
"separate_data_thread", false))
98 , circularDataBufferMode_(ps.get<bool>(
"circular_buffer_mode", false))
99 , sleep_on_no_data_us_(ps.get<size_t>(
"sleep_on_no_data_us", 0))
100 , data_thread_running_(false)
101 , dataBufferDepthFragments_(0)
102 , dataBufferDepthBytes_(0)
103 , maxDataBufferDepthFragments_(ps.get<int>(
"data_buffer_depth_fragments", 1000))
104 , maxDataBufferDepthBytes_(ps.get<size_t>(
"data_buffer_depth_mb", 1000) * 1024 * 1024)
105 , useMonitoringThread_(ps.get<bool>(
"separate_monitoring_thread", false))
106 , monitoringInterval_(ps.get<int64_t>(
"hardware_poll_interval_us", 0))
107 , lastMonitoringCall_()
108 , isHardwareOK_(true)
113 , timeout_(std::numeric_limits<uint64_t>::max())
114 , timestamp_(std::numeric_limits<uint64_t>::max())
115 , should_stop_(false)
118 , latest_exception_report_(
"none")
121 , sleep_on_stop_us_(0)
123 board_id_ = ps.get<
int>(
"board_id");
124 instance_name_for_metrics_ =
"BoardReader." + boost::lexical_cast<std::string>(board_id_);
126 fragment_ids_ = ps.get<std::vector<artdaq::Fragment::fragment_id_t>>(
"fragment_ids", std::vector<artdaq::Fragment::fragment_id_t>());
128 TLOG(TLVL_TRACE) <<
"artdaq::CommandableFragmentGenerator::CommandableFragmentGenerator(ps)";
131 if (fragment_id != -99)
133 if (fragment_ids_.size() != 0)
135 latest_exception_report_ =
"Error in CommandableFragmentGenerator: can't both define \"fragment_id\" and \"fragment_ids\" in FHiCL document";
136 throw cet::exception(latest_exception_report_);
140 fragment_ids_.emplace_back(fragment_id);
144 sleep_on_stop_us_ = ps.get<
int>(
"sleep_on_stop_us", 0);
146 std::string modeString = ps.get<std::string>(
"request_mode",
"ignored");
147 if (modeString ==
"single" || modeString ==
"Single")
149 mode_ = RequestMode::Single;
151 else if (modeString.find(
"buffer") != std::string::npos || modeString.find(
"Buffer") != std::string::npos)
153 mode_ = RequestMode::Buffer;
155 else if (modeString ==
"window" || modeString ==
"Window")
157 mode_ = RequestMode::Window;
159 else if (modeString.find(
"ignore") != std::string::npos || modeString.find(
"Ignore") != std::string::npos)
161 mode_ = RequestMode::Ignored;
163 TLOG(TLVL_DEBUG) <<
"Request mode is " <<
printMode_();
165 if (mode_ != RequestMode::Ignored)
169 latest_exception_report_ =
"Error in CommandableFragmentGenerator: use_data_thread must be true when request_mode is not \"Ignored\"!";
170 throw cet::exception(latest_exception_report_);
179 requestReceiver_.reset(
nullptr);
186 TLOG(TLVL_DEBUG) <<
"Joining dataThread";
187 if (dataThread_.joinable()) dataThread_.join();
188 TLOG(TLVL_DEBUG) <<
"Joining monitoringThread";
189 if (monitoringThread_.joinable()) monitoringThread_.join();
190 TLOG(TLVL_DEBUG) <<
"joinThreads complete";
197 if (check_stop()) usleep(sleep_on_stop_us_);
198 if (exception() || force_stop_)
return false;
200 if (!useMonitoringThread_ && monitoringInterval_ > 0)
202 TLOG(TLVL_GETNEXT) <<
"getNext: Checking whether to collect Monitoring Data";
203 auto now = std::chrono::steady_clock::now();
205 if (TimeUtils::GetElapsedTimeMicroseconds(lastMonitoringCall_, now) >= static_cast<size_t>(monitoringInterval_))
207 TLOG(TLVL_GETNEXT) <<
"getNext: Collecting Monitoring Data";
208 isHardwareOK_ = checkHWStatus_();
209 TLOG(TLVL_GETNEXT) <<
"getNext: isHardwareOK_ is now " << std::boolalpha << isHardwareOK_;
210 lastMonitoringCall_ = now;
216 std::lock_guard<std::mutex> lk(mutex_);
219 TLOG(TLVL_TRACE) <<
"getNext: Calling applyRequests";
220 result = applyRequests(output);
221 TLOG(TLVL_TRACE) <<
"getNext: Done with applyRequests result=" << std::boolalpha << result;
222 for (
auto dataIter = output.begin(); dataIter != output.end(); ++dataIter)
224 TLOG(20) <<
"getNext: applyRequests() returned fragment with sequenceID = " << (*dataIter)->sequenceID()
225 <<
", timestamp = " << (*dataIter)->timestamp() <<
", and sizeBytes = " << (*dataIter)->sizeBytes();
230 TLOG(TLVL_ERROR) <<
"Exception found in BoardReader with board ID " << board_id() <<
"; BoardReader will now return error status when queried";
231 throw cet::exception(
"CommandableFragmentGenerator") <<
"Exception found in BoardReader with board ID " << board_id() <<
"; BoardReader will now return error status when queried";
238 TLOG(TLVL_ERROR) <<
"Stopping CFG because the hardware reports bad status!";
241 TLOG(TLVL_TRACE) <<
"getNext: Calling getNext_ w/ ev_counter()=" << ev_counter();
244 result = getNext_(output);
250 TLOG(TLVL_TRACE) <<
"getNext: Done with getNext_ - ev_counter() now " << ev_counter();
251 for (
auto dataIter = output.begin(); dataIter != output.end(); ++dataIter)
253 TLOG(TLVL_GETNEXT_VERBOSE) <<
"getNext: getNext_() returned fragment with sequenceID = " << (*dataIter)->sequenceID()
254 <<
", timestamp = " << (*dataIter)->timestamp() <<
", and sizeBytes = " << (*dataIter)->sizeBytes();
258 catch (
const cet::exception& e)
260 latest_exception_report_ =
"cet::exception caught in getNext(): ";
261 latest_exception_report_.append(e.what());
262 TLOG(TLVL_ERROR) <<
"getNext: cet::exception caught: " << e;
266 catch (
const boost::exception& e)
268 latest_exception_report_ =
"boost::exception caught in getNext(): ";
269 latest_exception_report_.append(boost::diagnostic_information(e));
270 TLOG(TLVL_ERROR) <<
"getNext: boost::exception caught: " << boost::diagnostic_information(e);
274 catch (
const std::exception& e)
276 latest_exception_report_ =
"std::exception caught in getNext(): ";
277 latest_exception_report_.append(e.what());
278 TLOG(TLVL_ERROR) <<
"getNext: std::exception caught: " << e.what();
284 latest_exception_report_ =
"Unknown exception caught in getNext().";
285 TLOG(TLVL_ERROR) <<
"getNext: unknown exception caught";
292 TLOG(TLVL_DEBUG) <<
"getNext: Either getNext_ or applyRequests returned false, stopping";
295 if (metricMan && !output.empty())
297 auto timestamp = output.front()->timestamp();
299 if (output.size() > 1)
301 for (
auto& outputfrag : output)
303 if (outputfrag->timestamp() > timestamp)
305 timestamp = outputfrag->timestamp();
310 metricMan->sendMetric(
"Last Timestamp", timestamp,
"Ticks", 1,
311 MetricMode::LastPoint, app_name);
319 TLOG(TLVL_CHECKSTOP) <<
"CFG::check_stop: should_stop=" << should_stop() <<
", useDataThread_=" << useDataThread_ <<
", exception status =" << int(exception());
321 if (!should_stop())
return false;
322 if (!useDataThread_ || mode_ == RequestMode::Ignored)
return true;
323 if (force_stop_)
return true;
326 TLOG(TLVL_DEBUG) <<
"should_stop is true, force_stop_ is false, requestReceiver_->isRunning() is " << std::boolalpha << requestReceiver_->isRunning();
327 return !requestReceiver_->isRunning();
332 if (fragment_ids_.size() != 1)
334 throw cet::exception(
"Error in CommandableFragmentGenerator: can't call fragment_id() unless member fragment_ids_ vector is length 1");
338 return fragment_ids_[0];
344 if (force || mode_ == RequestMode::Ignored)
346 TLOG(TLVL_EVCOUNTERINC) <<
"ev_counter_inc: Incrementing ev_counter from " << ev_counter() <<
" by " << step;
347 return ev_counter_.fetch_add(step);
349 return ev_counter_.load();
354 TLOG(TLVL_TRACE) <<
"Start Command received.";
355 if (run < 0)
throw cet::exception(
"CommandableFragmentGenerator") <<
"negative run number";
358 timestamp_ = timestamp;
359 ev_counter_.store(1);
360 windows_sent_ooo_.clear();
362 std::unique_lock<std::mutex> lock(dataBufferMutex_);
365 should_stop_.store(
false);
366 force_stop_.store(
false);
367 exception_.store(
false);
370 latest_exception_report_ =
"none";
374 std::unique_lock<std::mutex> lk(mutex_);
375 if (useDataThread_) startDataThread();
376 if (useMonitoringThread_) startMonitoringThread();
377 if (mode_ != RequestMode::Ignored && !requestReceiver_->isRunning()) requestReceiver_->startRequestReceiverThread();
378 TLOG(TLVL_TRACE) <<
"Start Command complete.";
383 TLOG(TLVL_TRACE) <<
"Stop Command received.";
386 timestamp_ = timestamp;
387 if (requestReceiver_ && requestReceiver_->isRunning()) {
388 TLOG(TLVL_DEBUG) <<
"Stopping Request receiver thread BEGIN";
389 requestReceiver_->stopRequestReceiverThread();
390 TLOG(TLVL_DEBUG) <<
"Stopping Request receiver thread END";
394 should_stop_.store(
true);
395 std::unique_lock<std::mutex> lk(mutex_);
399 TLOG(TLVL_TRACE) <<
"Stop Command complete.";
404 TLOG(TLVL_TRACE) <<
"Pause Command received.";
406 timestamp_ = timestamp;
410 should_stop_.store(
true);
411 std::unique_lock<std::mutex> lk(mutex_);
418 TLOG(TLVL_TRACE) <<
"Resume Command received.";
420 timestamp_ = timestamp;
423 should_stop_ =
false;
425 std::unique_lock<std::mutex> lk(dataBufferMutex_);
426 dataBufferDepthBytes_ = 0;
427 dataBufferDepthFragments_ = 0;
433 std::unique_lock<std::mutex> lk(mutex_);
437 TLOG(TLVL_TRACE) <<
"Resume Command complete.";
442 TLOG(TLVL_TRACE) <<
"Report Command received.";
443 std::lock_guard<std::mutex> lk(mutex_);
450 std::string childReport = reportSpecific(which);
451 if (childReport.length() > 0) {
return childReport; }
454 if (which ==
"latest_exception")
456 return latest_exception_report_;
460 childReport = report();
461 if (childReport.length() > 0) {
return childReport; }
464 std::string tmpString =
"The \"" + which +
"\" command is not ";
465 tmpString.append(
"currently supported by the ");
466 tmpString.append(metricsReportingInstanceName());
467 tmpString.append(
" fragment generator.");
468 TLOG(TLVL_TRACE) <<
"Report Command complete.";
475 #pragma message "Using default implementation of CommandableFragmentGenerator::pauseNoMutex()"
480 #pragma message "Using default implementation of CommandableFragmentGenerator::pause()"
485 #pragma message "Using default implementation of CommandableFragmentGenerator::resume()"
490 #pragma message "Using default implementation of CommandableFragmentGenerator::report()"
496 #pragma message "Using default implementation of CommandableFragmentGenerator::reportSpecific(std::string)"
502 #pragma message "Using default implementation of CommandableFragmentGenerator::checkHWStatus_()"
508 #pragma message "Using default implementation of CommandableFragmentGenerator::metaCommand(std::string, std::string)"
514 if (dataThread_.joinable()) dataThread_.join();
515 TLOG(TLVL_INFO) <<
"Starting Data Receiver Thread";
519 catch (
const boost::exception& e)
521 TLOG(TLVL_ERROR) <<
"Caught boost::exception starting Data Receiver thread: " << boost::diagnostic_information(e) <<
", errno=" << errno;
522 std::cerr <<
"Caught boost::exception starting Data Receiver thread: " << boost::diagnostic_information(e) <<
", errno=" << errno << std::endl;
529 if (monitoringThread_.joinable()) monitoringThread_.join();
530 TLOG(TLVL_INFO) <<
"Starting Hardware Monitoring Thread";
534 catch (
const boost::exception& e)
536 TLOG(TLVL_ERROR) <<
"Caught boost::exception starting Hardware Monitoring thread: " << boost::diagnostic_information(e) <<
", errno=" << errno;
537 std::cerr <<
"Caught boost::exception starting Hardware Monitoring thread: " << boost::diagnostic_information(e) <<
", errno=" << errno << std::endl;
546 case RequestMode::Single:
548 case RequestMode::Buffer:
550 case RequestMode::Window:
552 case RequestMode::Ignored:
565 data_thread_running_ =
true;
570 TLOG(TLVL_DEBUG) <<
"getDataLoop: isHardwareOK is " << isHardwareOK_ <<
", aborting data thread";
571 data_thread_running_ =
false;
575 TLOG(TLVL_GETDATALOOP) <<
"getDataLoop: calling getNext_";
578 auto startdata = std::chrono::steady_clock::now();
582 data = getNext_(newDataBuffer_);
586 ExceptionHandler(ExceptionHandlerRethrow::no,
587 "Exception thrown by fragment generator in CommandableFragmentGenerator::getDataLoop; setting exception state to \"true\"");
590 data_thread_running_ =
false;
594 size_t newDataBufferDepthBytes = 0;
595 for (
auto dataIter = newDataBuffer_.begin(); dataIter != newDataBuffer_.end(); ++dataIter)
597 TLOG(TLVL_GETDATALOOP_VERBOSE) <<
"getDataLoop: getNext_() returned fragment with timestamp = " << (*dataIter)->timestamp() <<
", and sizeBytes = " << (*dataIter)->sizeBytes();
598 newDataBufferDepthBytes += (*dataIter)->sizeBytes();
603 metricMan->sendMetric(
"Avg Data Acquisition Time", TimeUtils::GetElapsedTime(startdata),
"s", 3, artdaq::MetricMode::Average);
606 if (newDataBuffer_.size() == 0 && sleep_on_no_data_us_ > 0)
608 usleep(sleep_on_no_data_us_);
611 TLOG(TLVL_GETDATALOOP_DATABUFFWAIT) <<
"Waiting for data buffer ready";
612 if (!waitForDataBufferReady())
return;
613 TLOG(TLVL_GETDATALOOP_DATABUFFWAIT) <<
"Done waiting for data buffer ready";
615 TLOG(TLVL_GETDATALOOP) <<
"getDataLoop: processing data";
616 if (data && !force_stop_)
618 std::unique_lock<std::mutex> lock(dataBufferMutex_);
621 case RequestMode::Single:
623 while (newDataBuffer_.size() >= fragment_ids_.size())
626 auto it = newDataBuffer_.begin();
627 std::advance(it, fragment_ids_.size());
628 dataBuffer_.splice(dataBuffer_.end(), newDataBuffer_, newDataBuffer_.begin(), it);
630 for (
auto dbit = dataBuffer_.begin(); dbit != dataBuffer_.end(); ++dbit)
632 dataBufferDepthBytes_ += (*dbit)->sizeBytes();
636 case RequestMode::Buffer:
637 case RequestMode::Ignored:
638 case RequestMode::Window:
641 dataBufferDepthBytes_ += newDataBufferDepthBytes;
642 dataBuffer_.splice(dataBuffer_.end(), newDataBuffer_);
645 getDataBufferStats();
649 std::unique_lock<std::mutex> lock(dataBufferMutex_);
650 if (dataBuffer_.size() > 0)
652 dataCondition_.notify_all();
655 if (!data || force_stop_)
657 TLOG(TLVL_INFO) <<
"Data flow has stopped. Ending data collection thread";
658 std::unique_lock<std::mutex> lock(dataBufferMutex_);
659 data_thread_running_ =
false;
660 if (requestReceiver_) requestReceiver_->ClearRequests();
661 newDataBuffer_.clear();
662 TLOG(TLVL_INFO) <<
"getDataLoop: Ending thread";
670 auto startwait = std::chrono::steady_clock::now();
672 auto lastwaittime = 0ULL;
675 std::unique_lock<std::mutex> lock(dataBufferMutex_);
676 getDataBufferStats();
679 while (dataBufferIsTooLarge())
681 if (!circularDataBufferMode_)
685 TLOG(TLVL_DEBUG) <<
"Run ended while waiting for buffer to shrink!";
686 std::unique_lock<std::mutex> lock(dataBufferMutex_);
687 getDataBufferStats();
688 dataCondition_.notify_all();
689 data_thread_running_ =
false;
692 auto waittime = TimeUtils::GetElapsedTimeMilliseconds(startwait);
694 if (first || (waittime != lastwaittime && waittime % 1000 == 0))
696 TLOG(TLVL_WARNING) <<
"Bad Omen: Data Buffer has exceeded its size limits. "
697 <<
"(seq_id=" << ev_counter()
698 <<
", frags=" << dataBufferDepthFragments_ <<
"/" << maxDataBufferDepthFragments_
699 <<
", szB=" << dataBufferDepthBytes_ <<
"/" << maxDataBufferDepthBytes_ <<
")";
700 TLOG(TLVL_TRACE) <<
"Bad Omen: Possible causes include requests not getting through or Ignored-mode BR issues";
703 if (waittime % 5 && waittime != lastwaittime)
705 TLOG(TLVL_WAITFORBUFFERREADY) <<
"getDataLoop: Data Retreival paused for " << waittime <<
" ms waiting for data buffer to drain";
707 lastwaittime = waittime;
712 std::unique_lock<std::mutex> lock(dataBufferMutex_);
713 getDataBufferStats();
714 if (dataBufferIsTooLarge())
716 if (dataBuffer_.begin() == dataBuffer_.end())
718 TLOG(TLVL_WARNING) <<
"Data buffer is reported as too large, but doesn't contain any Fragments! Possible corrupt memory!";
721 if (*dataBuffer_.begin())
723 TLOG(TLVL_WAITFORBUFFERREADY) <<
"waitForDataBufferReady: Dropping Fragment with timestamp " << (*dataBuffer_.begin())->timestamp() <<
" from data buffer (Buffer over-size, circular data buffer mode)";
725 dataBufferDepthBytes_ -= (*dataBuffer_.begin())->sizeBytes();
726 dataBuffer_.erase(dataBuffer_.begin());
727 getDataBufferStats();
737 return (maxDataBufferDepthFragments_ > 0 && dataBufferDepthFragments_ > maxDataBufferDepthFragments_) || (maxDataBufferDepthBytes_ > 0 && dataBufferDepthBytes_ > maxDataBufferDepthBytes_);
743 dataBufferDepthFragments_ = dataBuffer_.size();
747 TLOG(TLVL_GETBUFFERSTATS) <<
"getDataBufferStats: Sending Metrics";
748 metricMan->sendMetric(
"Buffer Depth Fragments", dataBufferDepthFragments_.load(),
"fragments", 1, MetricMode::LastPoint);
749 metricMan->sendMetric(
"Buffer Depth Bytes", dataBufferDepthBytes_.load(),
"bytes", 1, MetricMode::LastPoint);
751 TLOG(TLVL_GETBUFFERSTATS) <<
"getDataBufferStats: frags=" << dataBufferDepthFragments_.load() <<
"/" << maxDataBufferDepthFragments_
752 <<
", sz=" << dataBufferDepthBytes_.load() <<
"/" << maxDataBufferDepthBytes_;
757 std::unique_lock<std::mutex> lock(dataBufferMutex_);
758 dataCondition_.wait_for(lock, std::chrono::milliseconds(10));
759 if (dataBufferDepthFragments_ > 0)
761 if ((mode_ == RequestMode::Buffer || mode_ == RequestMode::Window))
764 getDataBufferStats();
765 while (dataBufferIsTooLarge())
767 TLOG(TLVL_CHECKDATABUFFER) <<
"checkDataBuffer: Dropping Fragment with timestamp " << (*dataBuffer_.begin())->timestamp() <<
" from data buffer (Buffer over-size)";
768 dataBufferDepthBytes_ -= (*dataBuffer_.begin())->sizeBytes();
769 dataBuffer_.erase(dataBuffer_.begin());
770 getDataBufferStats();
772 if (dataBuffer_.size() > 0)
774 TLOG(TLVL_CHECKDATABUFFER) <<
"Determining if Fragments can be dropped from data buffer";
775 Fragment::timestamp_t last = dataBuffer_.back()->timestamp();
776 Fragment::timestamp_t min = last > staleTimeout_ ? last - staleTimeout_ : 0;
777 for (
auto it = dataBuffer_.begin(); it != dataBuffer_.end();)
779 if ((*it)->timestamp() < min)
781 TLOG(TLVL_CHECKDATABUFFER) <<
"checkDataBuffer: Dropping Fragment with timestamp " << (*it)->timestamp() <<
" from data buffer (timeout=" << staleTimeout_ <<
", min=" << min <<
")";
782 dataBufferDepthBytes_ -= (*it)->sizeBytes();
783 it = dataBuffer_.erase(it);
790 getDataBufferStats();
793 else if (mode_ == RequestMode::Single && dataBuffer_.size() > fragment_ids_.size())
796 while (dataBuffer_.size() > fragment_ids_.size())
798 dataBufferDepthBytes_ -= (*dataBuffer_.begin())->sizeBytes();
799 dataBuffer_.erase(dataBuffer_.begin());
809 if (should_stop() || monitoringInterval_ <= 0)
811 TLOG(TLVL_DEBUG) <<
"getMonitoringDataLoop: should_stop() is " << std::boolalpha << should_stop()
812 <<
" and monitoringInterval is " << monitoringInterval_ <<
", returning";
815 TLOG(TLVL_GETMONITORINGDATA) <<
"getMonitoringDataLoop: Determining whether to call checkHWStatus_";
817 auto now = std::chrono::steady_clock::now();
818 if (TimeUtils::GetElapsedTimeMicroseconds(lastMonitoringCall_, now) >= static_cast<size_t>(monitoringInterval_))
820 isHardwareOK_ = checkHWStatus_();
821 TLOG(TLVL_GETMONITORINGDATA) <<
"getMonitoringDataLoop: isHardwareOK_ is now " << std::boolalpha << isHardwareOK_;
822 lastMonitoringCall_ = now;
824 usleep(monitoringInterval_ / 10);
831 TLOG(TLVL_APPLYREQUESTS) <<
"Mode is Ignored; Copying data to output";
832 std::move(dataBuffer_.begin(), dataBuffer_.end(), std::inserter(frags, frags.end()));
833 dataBufferDepthBytes_ = 0;
840 auto requests = requestReceiver_->GetRequests();
841 while (requests.size() > 1)
844 requestReceiver_->RemoveRequest(requests.begin()->first);
845 requests.erase(requests.begin());
847 sendEmptyFragments(frags, requests);
850 if (requests.size() == 0 || !requests.count(ev_counter()))
return;
852 if (dataBuffer_.size() > 0)
854 TLOG(TLVL_APPLYREQUESTS) <<
"Mode is Single; Sending copy of last event";
855 for (
auto& fragptr : dataBuffer_)
858 auto frag = fragptr.get();
859 auto newfrag = std::unique_ptr<artdaq::Fragment>(
new Fragment(ev_counter(), frag->fragmentID()));
860 newfrag->resize(frag->size() - detail::RawFragmentHeader::num_words());
861 memcpy(newfrag->headerAddress(), frag->headerAddress(), frag->sizeBytes());
862 newfrag->setTimestamp(requests[ev_counter()]);
863 newfrag->setSequenceID(ev_counter());
864 frags.push_back(std::move(newfrag));
869 sendEmptyFragment(frags, ev_counter(),
"No data for");
871 requestReceiver_->RemoveRequest(ev_counter());
872 ev_counter_inc(1,
true);
878 auto requests = requestReceiver_->GetRequests();
879 while (requests.size() > 1)
882 requestReceiver_->RemoveRequest(requests.begin()->first);
883 requests.erase(requests.begin());
885 sendEmptyFragments(frags, requests);
888 if (requests.size() == 0 || !requests.count(ev_counter()))
return;
890 TLOG(TLVL_DEBUG) <<
"Creating ContainerFragment for Buffered Fragments";
891 frags.emplace_back(
new artdaq::Fragment(ev_counter(), fragment_id()));
892 frags.back()->setTimestamp(requests[ev_counter()]);
893 ContainerFragmentLoader cfl(*frags.back());
894 cfl.set_missing_data(
false);
898 for (
auto it = dataBuffer_.begin(); it != dataBuffer_.end();)
900 TLOG(TLVL_APPLYREQUESTS) <<
"ApplyRequests: Adding Fragment with timestamp " << (*it)->timestamp() <<
" to Container with sequence ID " << ev_counter();
901 cfl.addFragment(*it);
902 dataBufferDepthBytes_ -= (*it)->sizeBytes();
903 it = dataBuffer_.erase(it);
905 requestReceiver_->RemoveRequest(ev_counter());
906 ev_counter_inc(1,
true);
911 TLOG(TLVL_APPLYREQUESTS) <<
"applyRequestsWindowMode BEGIN";
913 auto requests = requestReceiver_->GetRequests();
915 TLOG(TLVL_APPLYREQUESTS) <<
"applyRequestsWindowMode: Starting request processing";
916 for (
auto req = requests.begin(); req != requests.end();)
918 TLOG(TLVL_APPLYREQUESTS) <<
"applyRequestsWindowMode: processing request with sequence ID " << req->first <<
", timestamp " << req->second;
921 while (req->first < ev_counter() && requests.size() > 0)
923 TLOG(TLVL_APPLYREQUESTS) <<
"applyRequestsWindowMode: Clearing passed request for sequence ID " << req->first;
924 requestReceiver_->RemoveRequest(req->first);
925 req = requests.erase(req);
927 if (requests.size() == 0)
break;
929 auto ts = req->second;
930 TLOG(TLVL_APPLYREQUESTS) <<
"applyRequests: Checking that data exists for request window " << req->first;
931 Fragment::timestamp_t min = ts > windowOffset_ ? ts - windowOffset_ : 0;
932 Fragment::timestamp_t max = min + windowWidth_;
933 TLOG(TLVL_APPLYREQUESTS) <<
"ApplyRequests: min is " << min <<
", max is " << max
934 <<
" and first/last points in buffer are " << (dataBuffer_.size() > 0 ? dataBuffer_.front()->timestamp() : 0)
935 <<
"/" << (dataBuffer_.size() > 0 ? dataBuffer_.back()->timestamp() : 0)
936 <<
" (sz=" << dataBuffer_.size() <<
" [" << dataBufferDepthBytes_.load()
937 <<
"/" << maxDataBufferDepthBytes_ <<
"])";
938 bool windowClosed = dataBuffer_.size() > 0 && dataBuffer_.back()->timestamp() >= max;
939 bool windowTimeout = !windowClosed && TimeUtils::GetElapsedTimeMicroseconds(requestReceiver_->GetRequestTime(req->first)) > window_close_timeout_us_;
942 TLOG(TLVL_WARNING) <<
"applyRequests: A timeout occurred waiting for data to close the request window ({" << min <<
"-" << max
943 <<
"}, buffer={" << (dataBuffer_.size() > 0 ? dataBuffer_.front()->timestamp() : 0) <<
"-"
944 << (dataBuffer_.size() > 0 ? dataBuffer_.back()->timestamp() : 0)
945 <<
"} ). Time waiting: "
946 << TimeUtils::GetElapsedTimeMicroseconds(requestReceiver_->GetRequestTime(req->first)) <<
" us "
947 <<
"(> " << window_close_timeout_us_ <<
" us).";
949 if (windowClosed || !data_thread_running_ || windowTimeout)
951 TLOG(TLVL_DEBUG) <<
"applyRequests: Creating ContainerFragment for Window-requested Fragments";
952 frags.emplace_back(
new artdaq::Fragment(req->first, fragment_id()));
953 frags.back()->setTimestamp(ts);
954 ContainerFragmentLoader cfl(*frags.back());
968 if (!windowClosed || (dataBuffer_.size() > 0 && dataBuffer_.front()->timestamp() > min))
970 TLOG(TLVL_DEBUG) <<
"applyRequests: Request window starts before and/or ends after the current data buffer, setting ContainerFragment's missing_data flag!"
971 <<
" (requestWindowRange=[" << min <<
"," << max <<
"], "
972 <<
"buffer={" << (dataBuffer_.size() > 0 ? dataBuffer_.front()->timestamp() : 0) <<
"-"
973 << (dataBuffer_.size() > 0 ? dataBuffer_.back()->timestamp() : 0) <<
"}";
974 cfl.set_missing_data(
true);
980 for (
auto it = dataBuffer_.begin(); it != dataBuffer_.end();)
982 Fragment::timestamp_t fragT = (*it)->timestamp();
983 if (fragT < min || fragT > max || (fragT == max && windowWidth_ > 0))
989 TLOG(TLVL_APPLYREQUESTS) <<
"applyRequests: Adding (" << (++fragCount) <<
"th) Fragment with timestamp "
990 << (*it)->timestamp() <<
" and sizeBytes " << (*it)->sizeBytes()
991 <<
" to Container for sequence ID " << req->first;
992 cfl.addFragment(*it);
996 dataBufferDepthBytes_ -= (*it)->sizeBytes();
997 it = dataBuffer_.erase(it);
1004 requestReceiver_->RemoveRequest(req->first);
1005 checkOutOfOrderWindows(req->first);
1006 requestReceiver_->RemoveRequest(req->first);
1007 req = requests.erase(req);
1018 if (check_stop() || exception())
1024 if (mode_ == RequestMode::Ignored)
1026 while (dataBufferDepthFragments_ <= 0)
1028 if (check_stop() || exception() || !isHardwareOK_)
return false;
1029 std::unique_lock<std::mutex> lock(dataBufferMutex_);
1030 dataCondition_.wait_for(lock, std::chrono::milliseconds(10), [
this]() {
return dataBufferDepthFragments_ > 0; });
1035 if ((check_stop() && requestReceiver_->size() == 0) || exception())
return false;
1041 while (requestReceiver_->size() == 0 && counter < 100)
1043 if (check_stop() || exception())
return false;
1047 requestReceiver_->WaitForRequests(10);
1053 std::unique_lock<std::mutex> dlk(dataBufferMutex_);
1057 case RequestMode::Single:
1058 applyRequestsSingleMode(frags);
1060 case RequestMode::Window:
1061 applyRequestsWindowMode(frags);
1063 case RequestMode::Buffer:
1064 applyRequestsBufferMode(frags);
1066 case RequestMode::Ignored:
1068 applyRequestsIgnoredMode(frags);
1072 if (!data_thread_running_ || force_stop_)
1074 TLOG(TLVL_INFO) <<
"Data thread has stopped; Clearing data buffer";
1075 dataBufferDepthBytes_ = 0;
1076 dataBuffer_.clear();
1079 getDataBufferStats();
1082 if (frags.size() > 0)
1083 TLOG(TLVL_APPLYREQUESTS) <<
"Finished Processing Event " << (*frags.begin())->sequenceID() <<
" for fragment_id " << fragment_id() <<
".";
1089 TLOG(TLVL_WARNING) << desc <<
" sequence ID " << seqId <<
", sending empty fragment";
1090 for (
auto fid : fragment_ids_)
1092 auto frag =
new Fragment();
1093 frag->setSequenceID(seqId);
1094 frag->setFragmentID(fid);
1095 frag->setSystemType(Fragment::EmptyFragmentType);
1096 frags.emplace_back(FragmentPtr(frag));
1103 if (requests.size() > 0)
1105 TLOG(TLVL_SENDEMPTYFRAGMENTS) <<
"Sending Empty Fragments for Sequence IDs from " << ev_counter() <<
" up to but not including " << requests.begin()->first;
1106 while (requests.begin()->first > ev_counter())
1108 sendEmptyFragment(frags, ev_counter(),
"Missed request for");
1109 ev_counter_inc(1,
true);
1116 windows_sent_ooo_[seq] = std::chrono::steady_clock::now();
1118 auto it = windows_sent_ooo_.begin();
1119 while (it != windows_sent_ooo_.end())
1121 if (seq == it->first && it->first == ev_counter())
1123 TLOG(TLVL_CHECKWINDOWS) <<
"checkOutOfOrderWindows: Sequence ID matches ev_counter, incrementing ev_counter (" << ev_counter() <<
")";
1124 ev_counter_inc(1,
true);
1125 it = windows_sent_ooo_.erase(it);
1127 else if (it->first <= ev_counter())
1129 TLOG(TLVL_CHECKWINDOWS) <<
"checkOutOfOrderWindows: Data-taking has caught up to out-of-order window request " << it->first <<
", removing from list. ev_counter=" << ev_counter();
1130 requestReceiver_->RemoveRequest(ev_counter());
1131 if (it->first == ev_counter()) ev_counter_inc(1,
true);
1132 it = windows_sent_ooo_.erase(it);
1134 else if (TimeUtils::GetElapsedTimeMicroseconds(it->second) > missing_request_window_timeout_us_)
1136 TLOG(TLVL_CHECKWINDOWS) <<
"checkOutOfOrderWindows: Out-of-order window " << it->first <<
" has timed out, setting current sequence ID and removing from list";
1137 while (ev_counter() <= it->first)
1139 if (ev_counter() < it->first) TLOG(TLVL_WARNING) <<
"Missed request for sequence ID " << ev_counter() <<
"! Will not send any data for this sequence ID!";
1140 requestReceiver_->RemoveRequest(ev_counter());
1141 ev_counter_inc(1,
true);
1143 windows_sent_ooo_.erase(windows_sent_ooo_.begin(), it);
1144 it = windows_sent_ooo_.erase(it);
1148 TLOG(TLVL_CHECKWINDOWS) <<
"checkOutOfOrderWindows: Out-of-order window " << it->first <<
" waiting. Current event counter = " << ev_counter();
int fragment_id() const
Get the current Fragment ID, if there is only one.
void applyRequestsSingleMode(artdaq::FragmentPtrs &frags)
Create fragments using data buffer for request mode Single. Precondition: dataBufferMutex_ and reques...
virtual bool checkHWStatus_()
Check any relavent hardware status registers. Return false if an error condition exists that should h...
virtual ~CommandableFragmentGenerator()
CommandableFragmentGenerator Destructor.
void applyRequestsBufferMode(artdaq::FragmentPtrs &frags)
Create fragments using data buffer for request mode Buffer. Precondition: dataBufferMutex_ and reques...
bool sendEmptyFragment(FragmentPtrs &frags, size_t sequenceId, std::string desc)
Send an EmptyFragmentType Fragment.
void getMonitoringDataLoop()
This function regularly calls checkHWStatus_(), and sets the isHardwareOK flag accordingly.
void startDataThread()
Function that launches the data thread (getDataLoop())
std::string ReportCmd(std::string const &which="")
Get a report about a user-specified run-time quantity.
virtual bool metaCommand(std::string const &command, std::string const &arg)
The meta-command is used for implementing user-specific commands in a CommandableFragmentGenerator.
bool dataBufferIsTooLarge()
Test the configured constraints on the data buffer.
void StopCmd(uint64_t timeout, uint64_t timestamp)
Stop the CommandableFragmentGenerator.
void applyRequestsWindowMode(artdaq::FragmentPtrs &frags)
Create fragments using data buffer for request mode Window. Precondition: dataBufferMutex_ and reques...
void StartCmd(int run, uint64_t timeout, uint64_t timestamp)
Start the CommandableFragmentGenerator.
virtual void pauseNoMutex()
On call to PauseCmd, pauseNoMutex() is called prior to PauseCmd acquiring the mutex ...
bool check_stop()
Routine used by applyRequests to make sure that all outstanding requests have been fulfilled before r...
void ResumeCmd(uint64_t timeout, uint64_t timestamp)
Resume the CommandableFragmentGenerator.
CommandableFragmentGenerator()
CommandableFragmentGenerator default constructor.
bool getNext(FragmentPtrs &output) overridefinal
getNext calls either applyRequests or getNext_ to get any data that is ready to be sent to the EventB...
bool waitForDataBufferReady()
Wait for the data buffer to drain (dataBufferIsTooLarge returns false), periodically reporting status...
size_t ev_counter_inc(size_t step=1, bool force=false)
Increment the event counter, if the current RequestMode allows it.
void applyRequestsIgnoredMode(artdaq::FragmentPtrs &frags)
Create fragments using data buffer for request mode Ignored. Precondition: dataBufferMutex_ and reque...
void PauseCmd(uint64_t timeout, uint64_t timestamp)
Pause the CommandableFragmentGenerator.
void getDataLoop()
When separate_data_thread is set to true, this loop repeatedly calls getNext_ and adds returned Fragm...
void sendEmptyFragments(FragmentPtrs &frags, std::map< Fragment::sequence_id_t, Fragment::timestamp_t > &requests)
This function is for Buffered and Single request modes, as they can only respond to one data request ...
Receive data requests and make them available to CommandableFragmentGenerator or other interested par...
void startMonitoringThread()
Function that launches the monitoring thread (getMonitoringDataLoop())
virtual void pause()
If a CommandableFragmentGenerator subclass is reading from hardware, the implementation of pause() sh...
virtual void resume()
The subrun number will be incremented before a call to resume.
void checkDataBuffer()
Perform data buffer pruning operations. If the RequestMode is Single, removes all but the latest Frag...
virtual std::string report()
Let's say that the contract with the report() functions is that they return a non-empty string if the...
std::string printMode_()
Return the string representation of the current RequestMode.
void getDataBufferStats()
Calculate the size of the dataBuffer and report appropriate metrics.
void checkOutOfOrderWindows(Fragment::sequence_id_t seq)
Check the windows_sent_ooo_ map for sequence IDs that may be removed.
virtual std::string reportSpecific(std::string const &what)
Report the status of a specific quantity
bool applyRequests(FragmentPtrs &output)
See if any requests have been received, and add the corresponding data Fragment objects to the output...
void joinThreads()
Join any data-taking threads. Should be called when destructing CommandableFragmentGenerator.