1 #define TRACE_NAME (app_name + "_CommandableFragmentGenerator").c_str() // include these 2 first -
2 #include "artdaq/DAQdata/Globals.hh"
4 #include "artdaq/Application/CommandableFragmentGenerator.hh"
6 #include <boost/exception/all.hpp>
7 #include <boost/throw_exception.hpp>
12 #include "canvas/Utilities/Exception.h"
13 #include "cetlib_except/exception.h"
14 #include "fhiclcpp/ParameterSet.h"
16 #include "artdaq-core/Utilities/SimpleLookupPolicy.hh"
17 #include "artdaq-core/Data/Fragment.hh"
18 #include "artdaq-core/Data/ContainerFragmentLoader.hh"
19 #include "artdaq-core/Utilities/ExceptionHandler.hh"
20 #include "artdaq-core/Utilities/TimeUtils.hh"
31 #define TLVL_GETNEXT 10
32 #define TLVL_GETNEXT_VERBOSE 20
33 #define TLVL_CHECKSTOP 11
34 #define TLVL_EVCOUNTERINC 12
35 #define TLVL_GETDATALOOP 13
36 #define TLVL_GETDATALOOP_DATABUFFWAIT 21
37 #define TLVL_GETDATALOOP_VERBOSE 20
38 #define TLVL_WAITFORBUFFERREADY 15
39 #define TLVL_GETBUFFERSTATS 16
40 #define TLVL_CHECKDATABUFFER 17
41 #define TLVL_GETMONITORINGDATA 18
42 #define TLVL_APPLYREQUESTS 9
43 #define TLVL_SENDEMPTYFRAGMENTS 19
44 #define TLVL_CHECKWINDOWS 14
48 , requestReceiver_(nullptr)
51 , staleTimeout_(Fragment::InvalidTimestamp)
52 , expectedType_(Fragment::EmptyFragmentType)
53 , maxFragmentCount_(std::numeric_limits<size_t>::max())
54 , uniqueWindows_(true)
56 , missing_request_window_timeout_us_(1000000)
57 , window_close_timeout_us_(2000000)
58 , useDataThread_(false)
59 , sleep_on_no_data_us_(0)
60 , data_thread_running_(false)
61 , dataBufferDepthFragments_(0)
62 , dataBufferDepthBytes_(0)
63 , maxDataBufferDepthFragments_(1000)
64 , maxDataBufferDepthBytes_(1000)
65 , useMonitoringThread_(false)
66 , monitoringInterval_(0)
67 , lastMonitoringCall_()
73 , timeout_(std::numeric_limits<uint64_t>::max())
74 , timestamp_(std::numeric_limits<uint64_t>::max())
78 , latest_exception_report_(
"none")
81 , instance_name_for_metrics_(
"FragmentGenerator")
82 , sleep_on_stop_us_(0)
87 , requestReceiver_(nullptr)
88 , windowOffset_(ps.get<Fragment::timestamp_t>(
"request_window_offset", 0))
89 , windowWidth_(ps.get<Fragment::timestamp_t>(
"request_window_width", 0))
90 , staleTimeout_(ps.get<Fragment::timestamp_t>(
"stale_request_timeout", 0xFFFFFFFF))
91 , expectedType_(ps.get<Fragment::type_t>(
"expected_fragment_type", Fragment::type_t(Fragment::EmptyFragmentType)))
92 , uniqueWindows_(ps.get<bool>(
"request_windows_are_unique", true))
94 , missing_request_window_timeout_us_(ps.get<size_t>(
"missing_request_window_timeout_us", 5000000))
95 , window_close_timeout_us_(ps.get<size_t>(
"window_close_timeout_us", 2000000))
96 , useDataThread_(ps.get<bool>(
"separate_data_thread", false))
97 , sleep_on_no_data_us_(ps.get<size_t>(
"sleep_on_no_data_us", 0))
98 , data_thread_running_(false)
99 , dataBufferDepthFragments_(0)
100 , dataBufferDepthBytes_(0)
101 , maxDataBufferDepthFragments_(ps.get<int>(
"data_buffer_depth_fragments", 1000))
102 , maxDataBufferDepthBytes_(ps.get<size_t>(
"data_buffer_depth_mb", 1000) * 1024 * 1024)
103 , useMonitoringThread_(ps.get<bool>(
"separate_monitoring_thread", false))
104 , monitoringInterval_(ps.get<int64_t>(
"hardware_poll_interval_us", 0))
105 , lastMonitoringCall_()
106 , isHardwareOK_(true)
111 , timeout_(std::numeric_limits<uint64_t>::max())
112 , timestamp_(std::numeric_limits<uint64_t>::max())
113 , should_stop_(false)
116 , latest_exception_report_(
"none")
119 , sleep_on_stop_us_(0)
121 board_id_ = ps.get<
int>(
"board_id");
122 instance_name_for_metrics_ =
"BoardReader." + boost::lexical_cast<std::string>(board_id_);
124 fragment_ids_ = ps.get<std::vector<artdaq::Fragment::fragment_id_t>>(
"fragment_ids", std::vector<artdaq::Fragment::fragment_id_t>());
126 TLOG(TLVL_TRACE) <<
"artdaq::CommandableFragmentGenerator::CommandableFragmentGenerator(ps)";
129 if (fragment_id != -99)
131 if (fragment_ids_.size() != 0)
133 latest_exception_report_ =
"Error in CommandableFragmentGenerator: can't both define \"fragment_id\" and \"fragment_ids\" in FHiCL document";
134 throw cet::exception(latest_exception_report_);
138 fragment_ids_.emplace_back(fragment_id);
142 sleep_on_stop_us_ = ps.get<
int>(
"sleep_on_stop_us", 0);
144 dataBuffer_.emplace_back(FragmentPtr(
new Fragment()));
145 (*dataBuffer_.begin())->setSystemType(Fragment::EmptyFragmentType);
147 std::string modeString = ps.get<std::string>(
"request_mode",
"ignored");
148 if (modeString ==
"single" || modeString ==
"Single")
150 mode_ = RequestMode::Single;
152 else if (modeString.find(
"buffer") != std::string::npos || modeString.find(
"Buffer") != std::string::npos)
154 mode_ = RequestMode::Buffer;
156 else if (modeString ==
"window" || modeString ==
"Window")
158 mode_ = RequestMode::Window;
160 else if (modeString.find(
"ignore") != std::string::npos || modeString.find(
"Ignore") != std::string::npos)
162 mode_ = RequestMode::Ignored;
164 TLOG(TLVL_DEBUG) <<
"Request mode is " <<
printMode_();
166 if (mode_ != RequestMode::Ignored)
170 latest_exception_report_ =
"Error in CommandableFragmentGenerator: use_data_thread must be true when request_mode is not \"Ignored\"!";
171 throw cet::exception(latest_exception_report_);
186 TLOG(TLVL_DEBUG) <<
"Joining dataThread";
187 if (dataThread_.joinable()) dataThread_.join();
188 TLOG(TLVL_DEBUG) <<
"Joining monitoringThread";
189 if (monitoringThread_.joinable()) monitoringThread_.join();
190 requestReceiver_.reset(
nullptr);
197 if (check_stop()) usleep(sleep_on_stop_us_);
198 if (exception() || force_stop_)
return false;
200 if (!useMonitoringThread_ && monitoringInterval_ > 0)
202 TLOG(TLVL_GETNEXT) <<
"getNext: Checking whether to collect Monitoring Data";
203 auto now = std::chrono::steady_clock::now();
205 if (TimeUtils::GetElapsedTimeMicroseconds(lastMonitoringCall_, now) >= static_cast<size_t>(monitoringInterval_))
207 TLOG(TLVL_GETNEXT) <<
"getNext: Collecting Monitoring Data";
208 isHardwareOK_ = checkHWStatus_();
209 TLOG(TLVL_GETNEXT) <<
"getNext: isHardwareOK_ is now " << std::boolalpha << isHardwareOK_;
210 lastMonitoringCall_ = now;
216 std::lock_guard<std::mutex> lk(mutex_);
219 TLOG(TLVL_TRACE) <<
"getNext: Calling applyRequests";
220 result = applyRequests(output);
221 TLOG(TLVL_TRACE) <<
"getNext: Done with applyRequests result=" << std::boolalpha << result;
225 TLOG(TLVL_ERROR) <<
"Exception found in BoardReader with board ID " << board_id() <<
"; BoardReader will now return error status when queried";
226 throw cet::exception(
"CommandableFragmentGenerator") <<
"Exception found in BoardReader with board ID " << board_id() <<
"; BoardReader will now return error status when queried";
233 TLOG(TLVL_ERROR) <<
"Stopping CFG because the hardware reports bad status!";
236 TLOG(TLVL_TRACE) <<
"getNext: Calling getNext_ " << ev_counter();
239 result = getNext_(output);
245 TLOG(TLVL_TRACE) <<
"getNext: Done with getNext_ " << ev_counter();
246 for (
auto dataIter = output.begin(); dataIter != output.end(); ++dataIter)
248 TLOG(TLVL_GETNEXT_VERBOSE) <<
"getNext: getNext_() returned fragment with sequenceID = " << (*dataIter)->sequenceID()
249 <<
", timestamp = " << (*dataIter)->timestamp() <<
", and sizeBytes = " << (*dataIter)->sizeBytes();
253 catch (
const cet::exception& e)
255 latest_exception_report_ =
"cet::exception caught in getNext(): ";
256 latest_exception_report_.append(e.what());
257 TLOG(TLVL_ERROR) <<
"getNext: cet::exception caught: " << e;
261 catch (
const boost::exception& e)
263 latest_exception_report_ =
"boost::exception caught in getNext(): ";
264 latest_exception_report_.append(boost::diagnostic_information(e));
265 TLOG(TLVL_ERROR) <<
"getNext: boost::exception caught: " << boost::diagnostic_information(e);
269 catch (
const std::exception& e)
271 latest_exception_report_ =
"std::exception caught in getNext(): ";
272 latest_exception_report_.append(e.what());
273 TLOG(TLVL_ERROR) <<
"getNext: std::exception caught: " << e.what();
279 latest_exception_report_ =
"Unknown exception caught in getNext().";
280 TLOG(TLVL_ERROR) <<
"getNext: unknown exception caught";
287 TLOG(TLVL_DEBUG) <<
"stopped ";
290 if (metricMan && !output.empty())
292 auto timestamp = output.front()->timestamp();
294 if (output.size() > 1)
296 for (
auto& outputfrag : output)
298 if (outputfrag->timestamp() > timestamp)
300 timestamp = outputfrag->timestamp();
305 metricMan->sendMetric(
"Last Timestamp", timestamp,
"Ticks", 1,
306 MetricMode::LastPoint, app_name);
314 TLOG(TLVL_CHECKSTOP) <<
"CFG::check_stop: should_stop=" << should_stop() <<
", useDataThread_=" << useDataThread_ <<
", exception status =" << int(exception());
316 if (!should_stop())
return false;
317 if (!useDataThread_ || mode_ == RequestMode::Ignored)
return true;
318 if (force_stop_)
return true;
321 return !requestReceiver_->isRunning();
326 if (fragment_ids_.size() != 1)
328 throw cet::exception(
"Error in CommandableFragmentGenerator: can't call fragment_id() unless member fragment_ids_ vector is length 1");
332 return fragment_ids_[0];
338 if (force || mode_ == RequestMode::Ignored)
340 TLOG(TLVL_EVCOUNTERINC) <<
"ev_counter_inc: Incrementing ev_counter from " << ev_counter() <<
" by " << step;
341 return ev_counter_.fetch_add(step);
343 return ev_counter_.load();
348 TLOG(TLVL_TRACE) <<
"Start Command received.";
349 if (run < 0)
throw cet::exception(
"CommandableFragmentGenerator") <<
"negative run number";
352 timestamp_ = timestamp;
353 ev_counter_.store(1);
354 should_stop_.store(
false);
355 exception_.store(
false);
358 latest_exception_report_ =
"none";
360 windows_sent_ooo_.clear();
364 std::unique_lock<std::mutex> lk(mutex_);
365 if (useDataThread_) startDataThread();
366 if (useMonitoringThread_) startMonitoringThread();
367 if (mode_ != RequestMode::Ignored && !requestReceiver_->isRunning()) requestReceiver_->startRequestReceiverThread();
368 TLOG(TLVL_TRACE) <<
"Start Command complete.";
373 TLOG(TLVL_TRACE) <<
"Stop Command received.";
376 timestamp_ = timestamp;
377 if (requestReceiver_ && requestReceiver_->isRunning()) requestReceiver_->stopRequestReceiverThread();
380 should_stop_.store(
true);
381 std::unique_lock<std::mutex> lk(mutex_);
383 TLOG(TLVL_TRACE) <<
"Stop command complete.";
388 TLOG(TLVL_TRACE) <<
"Pause Command received.";
390 timestamp_ = timestamp;
391 if (requestReceiver_->isRunning()) requestReceiver_->stopRequestReceiverThread();
394 should_stop_.store(
true);
395 std::unique_lock<std::mutex> lk(mutex_);
398 TLOG(TLVL_TRACE) <<
"Pause Command complete.";
403 TLOG(TLVL_TRACE) <<
"Resume Command received.";
405 timestamp_ = timestamp;
408 should_stop_ =
false;
415 std::unique_lock<std::mutex> lk(mutex_);
416 if (useDataThread_) startDataThread();
417 if (useMonitoringThread_) startMonitoringThread();
418 if (mode_ != RequestMode::Ignored && !requestReceiver_->isRunning()) requestReceiver_->startRequestReceiverThread();
419 TLOG(TLVL_TRACE) <<
"Resume Command complete.";
424 TLOG(TLVL_TRACE) <<
"Report Command received.";
425 std::lock_guard<std::mutex> lk(mutex_);
432 std::string childReport = reportSpecific(which);
433 if (childReport.length() > 0) {
return childReport; }
436 if (which ==
"latest_exception")
438 return latest_exception_report_;
442 childReport = report();
443 if (childReport.length() > 0) {
return childReport; }
446 std::string tmpString =
"The \"" + which +
"\" command is not ";
447 tmpString.append(
"currently supported by the ");
448 tmpString.append(metricsReportingInstanceName());
449 tmpString.append(
" fragment generator.");
450 TLOG(TLVL_TRACE) <<
"Report Command complete.";
457 #pragma message "Using default implementation of CommandableFragmentGenerator::pauseNoMutex()"
462 #pragma message "Using default implementation of CommandableFragmentGenerator::pause()"
467 #pragma message "Using default implementation of CommandableFragmentGenerator::resume()"
472 #pragma message "Using default implementation of CommandableFragmentGenerator::report()"
478 #pragma message "Using default implementation of CommandableFragmentGenerator::reportSpecific(std::string)"
484 #pragma message "Using default implementation of CommandableFragmentGenerator::checkHWStatus_()"
490 #pragma message "Using default implementation of CommandableFragmentGenerator::metaCommand(std::string, std::string)"
496 if (dataThread_.joinable()) dataThread_.join();
497 TLOG(TLVL_INFO) <<
"Starting Data Receiver Thread";
503 if (monitoringThread_.joinable()) monitoringThread_.join();
504 TLOG(TLVL_INFO) <<
"Starting Hardware Monitoring Thread";
512 case RequestMode::Single:
514 case RequestMode::Buffer:
516 case RequestMode::Window:
518 case RequestMode::Ignored:
527 data_thread_running_ =
true;
532 TLOG(TLVL_DEBUG) <<
"getDataLoop: isHardwareOK is " << isHardwareOK_ <<
", aborting data thread";
533 data_thread_running_ =
false;
537 TLOG(TLVL_GETDATALOOP) <<
"getDataLoop: calling getNext_";
540 auto startdata = std::chrono::steady_clock::now();
544 data = getNext_(newDataBuffer_);
548 ExceptionHandler(ExceptionHandlerRethrow::no,
549 "Exception thrown by fragment generator in CommandableFragmentGenerator::getDataLoop; setting exception state to \"true\"");
552 data_thread_running_ =
false;
555 for (
auto dataIter = newDataBuffer_.begin(); dataIter != newDataBuffer_.end(); ++dataIter)
557 TLOG(TLVL_GETDATALOOP_VERBOSE) <<
"getDataLoop: getNext_() returned fragment with sequenceID = " << (*dataIter)->sequenceID()
558 <<
", timestamp = " << (*dataIter)->timestamp() <<
", and sizeBytes = " << (*dataIter)->sizeBytes();
563 metricMan->sendMetric(
"Avg Data Acquisition Time", TimeUtils::GetElapsedTime(startdata),
"s", 3, artdaq::MetricMode::Average);
566 if (newDataBuffer_.size() == 0 && sleep_on_no_data_us_ > 0)
568 usleep(sleep_on_no_data_us_);
571 TLOG(TLVL_GETDATALOOP_DATABUFFWAIT) <<
"Waiting for data buffer ready";
572 if (!waitForDataBufferReady())
return;
573 TLOG(TLVL_GETDATALOOP_DATABUFFWAIT) <<
"Done waiting for data buffer ready";
575 TLOG(TLVL_GETDATALOOP) <<
"getDataLoop: processing data";
576 if (data && !force_stop_)
578 std::unique_lock<std::mutex> lock(dataBufferMutex_);
581 case RequestMode::Single:
583 while (newDataBuffer_.size() >= fragment_ids_.size())
586 auto it = newDataBuffer_.begin();
587 std::advance(it, fragment_ids_.size());
588 dataBuffer_.splice(dataBuffer_.end(), newDataBuffer_, newDataBuffer_.begin(), it);
591 case RequestMode::Buffer:
592 case RequestMode::Ignored:
593 case RequestMode::Window:
596 dataBuffer_.splice(dataBuffer_.end(), newDataBuffer_);
599 getDataBufferStats();
603 std::unique_lock<std::mutex> lock(dataBufferMutex_);
604 if (dataBuffer_.size() > 0)
606 dataCondition_.notify_all();
609 if (!data || force_stop_)
611 TLOG(TLVL_INFO) <<
"Data flow has stopped. Ending data collection thread";
612 data_thread_running_ =
false;
613 if (requestReceiver_) requestReceiver_->ClearRequests();
615 newDataBuffer_.clear();
623 auto startwait = std::chrono::steady_clock::now();
625 auto lastwaittime = 0ULL;
626 while (dataBufferIsTooLarge())
630 TLOG(TLVL_DEBUG) <<
"Run ended while waiting for buffer to shrink!";
631 std::unique_lock<std::mutex> lock(dataBufferMutex_);
632 getDataBufferStats();
633 dataCondition_.notify_all();
634 data_thread_running_ =
false;
637 auto waittime = TimeUtils::GetElapsedTimeMilliseconds(startwait);
639 if (first || (waittime != lastwaittime && waittime % 1000 == 0))
641 TLOG(TLVL_WARNING) <<
"Bad Omen: Data Buffer has exceeded its size limits. "
642 <<
"(seq_id=" << ev_counter()
643 <<
", frags=" << dataBufferDepthFragments_ <<
"/" << maxDataBufferDepthFragments_
644 <<
", szB=" << dataBufferDepthBytes_ <<
"/" << maxDataBufferDepthBytes_ <<
")";
645 TLOG(TLVL_TRACE) <<
"Bad Omen: Possible causes include requests not getting through or Ignored-mode BR issues";
648 if (waittime % 5 && waittime != lastwaittime)
650 TLOG(TLVL_WAITFORBUFFERREADY) <<
"getDataLoop: Data Retreival paused for " << waittime <<
" ms waiting for data buffer to drain";
652 lastwaittime = waittime;
660 return (maxDataBufferDepthFragments_ > 0 && dataBufferDepthFragments_ > maxDataBufferDepthFragments_) || (maxDataBufferDepthBytes_ > 0 && dataBufferDepthBytes_ > maxDataBufferDepthBytes_);
666 dataBufferDepthFragments_ = dataBuffer_.size();
668 TLOG(TLVL_GETBUFFERSTATS) <<
"getDataBufferStats: Calculating buffer size";
669 for (
auto i = dataBuffer_.begin(); i != dataBuffer_.end(); ++i)
671 if (i->get() !=
nullptr)
673 acc += (*i)->sizeBytes();
676 dataBufferDepthBytes_ = acc;
680 TLOG(TLVL_GETBUFFERSTATS) <<
"getDataBufferStats: Sending Metrics";
681 metricMan->sendMetric(
"Buffer Depth Fragments", dataBufferDepthFragments_.load(),
"fragments", 1, MetricMode::LastPoint);
682 metricMan->sendMetric(
"Buffer Depth Bytes", dataBufferDepthBytes_.load(),
"bytes", 1, MetricMode::LastPoint);
684 TLOG(TLVL_GETBUFFERSTATS) <<
"getDataBufferStats: frags=" << dataBufferDepthFragments_.load() <<
"/" << maxDataBufferDepthFragments_
685 <<
", sz=" << dataBufferDepthBytes_.load() <<
"/" << maxDataBufferDepthBytes_;
690 std::unique_lock<std::mutex> lock(dataBufferMutex_);
691 dataCondition_.wait_for(lock, std::chrono::milliseconds(10));
692 if (dataBufferDepthFragments_ > 0)
694 if ((mode_ == RequestMode::Buffer || mode_ == RequestMode::Window))
697 while (dataBufferIsTooLarge())
699 dataBuffer_.erase(dataBuffer_.begin());
700 getDataBufferStats();
702 if (dataBuffer_.size() > 0)
704 TLOG(TLVL_CHECKDATABUFFER) <<
"Determining if Fragments can be dropped from data buffer";
705 Fragment::timestamp_t last = dataBuffer_.back()->timestamp();
706 Fragment::timestamp_t min = last > staleTimeout_ ? last - staleTimeout_ : 0;
707 for (
auto it = dataBuffer_.begin(); it != dataBuffer_.end();)
709 if ((*it)->timestamp() < min)
711 it = dataBuffer_.erase(it);
718 getDataBufferStats();
721 else if (mode_ == RequestMode::Single && dataBuffer_.size() > fragment_ids_.size())
724 while (dataBuffer_.size() > fragment_ids_.size())
726 dataBuffer_.erase(dataBuffer_.begin());
736 if (should_stop() || monitoringInterval_ <= 0)
738 TLOG(TLVL_DEBUG) <<
"getMonitoringDataLoop: should_stop() is " << std::boolalpha << should_stop()
739 <<
" and monitoringInterval is " << monitoringInterval_ <<
", returning";
742 TLOG(TLVL_GETMONITORINGDATA) <<
"getMonitoringDataLoop: Determining whether to call checkHWStatus_";
744 auto now = std::chrono::steady_clock::now();
745 if (TimeUtils::GetElapsedTimeMicroseconds(lastMonitoringCall_, now) >= static_cast<size_t>(monitoringInterval_))
747 isHardwareOK_ = checkHWStatus_();
748 TLOG(TLVL_GETMONITORINGDATA) <<
"getMonitoringDataLoop: isHardwareOK_ is now " << std::boolalpha << isHardwareOK_;
749 lastMonitoringCall_ = now;
751 usleep(monitoringInterval_ / 10);
758 TLOG(TLVL_APPLYREQUESTS) <<
"Mode is Ignored; Copying data to output";
759 std::move(dataBuffer_.begin(), dataBuffer_.end(), std::inserter(frags, frags.end()));
766 auto requests = requestReceiver_->GetRequests();
767 while (requests.size() > 1)
770 requestReceiver_->RemoveRequest(requests.begin()->first);
771 requests.erase(requests.begin());
773 sendEmptyFragments(frags, requests);
776 if (requests.size() == 0 || !requests.count(ev_counter()))
return;
778 if (dataBuffer_.size() > 0)
780 TLOG(TLVL_APPLYREQUESTS) <<
"Mode is Single; Sending copy of last event";
781 for (
auto& fragptr : dataBuffer_)
784 auto frag = fragptr.get();
785 auto newfrag = std::unique_ptr<artdaq::Fragment>(
new Fragment(ev_counter(), frag->fragmentID()));
786 newfrag->resize(frag->size() - detail::RawFragmentHeader::num_words());
787 memcpy(newfrag->headerAddress(), frag->headerAddress(), frag->sizeBytes());
788 newfrag->setTimestamp(requests[ev_counter()]);
789 newfrag->setSequenceID(ev_counter());
790 frags.push_back(std::move(newfrag));
795 sendEmptyFragment(frags, ev_counter(),
"No data for");
797 requestReceiver_->RemoveRequest(ev_counter());
798 ev_counter_inc(1,
true);
804 auto requests = requestReceiver_->GetRequests();
805 while (requests.size() > 1)
808 requestReceiver_->RemoveRequest(requests.begin()->first);
809 requests.erase(requests.begin());
811 sendEmptyFragments(frags, requests);
814 if (requests.size() == 0 || !requests.count(ev_counter()))
return;
816 TLOG(TLVL_DEBUG) <<
"Creating ContainerFragment for Buffered Fragments";
817 frags.emplace_back(
new artdaq::Fragment(ev_counter(), fragment_id()));
818 frags.back()->setTimestamp(requests[ev_counter()]);
819 ContainerFragmentLoader cfl(*frags.back());
820 cfl.set_missing_data(
false);
824 for (
auto it = dataBuffer_.begin(); it != dataBuffer_.end();)
826 TLOG(TLVL_APPLYREQUESTS) <<
"ApplyRequests: Adding Fragment with timestamp " << (*it)->timestamp() <<
" to Container";
827 cfl.addFragment(*it);
828 it = dataBuffer_.erase(it);
830 requestReceiver_->RemoveRequest(ev_counter());
831 ev_counter_inc(1,
true);
836 TLOG(TLVL_APPLYREQUESTS) <<
"applyRequestsWindowMode BEGIN";
838 auto requests = requestReceiver_->GetRequests();
840 TLOG(TLVL_APPLYREQUESTS) <<
"applyRequestsWindowMode: Starting request processing";
841 for (
auto req = requests.begin(); req != requests.end();)
843 TLOG(TLVL_APPLYREQUESTS) <<
"applyRequestsWindowMode: processing request with sequence ID " << req->first <<
", timestamp " << req->second;
846 while (req->first < ev_counter() && requests.size() > 0)
848 TLOG(TLVL_APPLYREQUESTS) <<
"applyRequestsWindowMode: Clearing passed request for sequence ID " << req->first;
849 requestReceiver_->RemoveRequest(req->first);
850 req = requests.erase(req);
852 if (requests.size() == 0)
break;
854 auto ts = req->second;
855 TLOG(TLVL_APPLYREQUESTS) <<
"ApplyRequests: Checking that data exists for request window " << req->first;
856 Fragment::timestamp_t min = ts > windowOffset_ ? ts - windowOffset_ : 0;
857 Fragment::timestamp_t max = min + windowWidth_;
858 TLOG(TLVL_APPLYREQUESTS) <<
"ApplyRequests: min is " << min <<
", max is " << max
859 <<
" and last point in buffer is " << (dataBuffer_.size() > 0 ? dataBuffer_.back()->timestamp() : 0) <<
" (sz=" << dataBuffer_.size() <<
")";
860 bool windowClosed = dataBuffer_.size() > 0 && dataBuffer_.back()->timestamp() >= max;
861 bool windowTimeout = !windowClosed && TimeUtils::GetElapsedTimeMicroseconds(requestReceiver_->GetRequestTime(req->first)) > window_close_timeout_us_;
864 TLOG(TLVL_WARNING) <<
"A timeout occurred waiting for data to close the request window ({" << min <<
"-" << max
865 <<
"}, buffer={" << (dataBuffer_.size() > 0 ? dataBuffer_.front()->timestamp() : 0) <<
"-"
866 << (dataBuffer_.size() > 0 ? dataBuffer_.back()->timestamp() : 0)
867 <<
"} ). Time waiting: "
868 << TimeUtils::GetElapsedTimeMicroseconds(requestReceiver_->GetRequestTime(req->first)) <<
" us "
869 <<
"(> " << window_close_timeout_us_ <<
" us).";
871 if (windowClosed || !data_thread_running_ || windowTimeout)
873 TLOG(TLVL_DEBUG) <<
"Creating ContainerFragment for Buffered or Window-requested Fragments";
874 frags.emplace_back(
new artdaq::Fragment(req->first, fragment_id()));
875 frags.back()->setTimestamp(ts);
876 ContainerFragmentLoader cfl(*frags.back());
878 if (!windowClosed) cfl.set_missing_data(
true);
879 if (dataBuffer_.size() > 0 && dataBuffer_.front()->timestamp() > min)
881 TLOG(TLVL_DEBUG) <<
"Request Window covers data that is either before data collection began or has fallen off the end of the buffer";
882 cfl.set_missing_data(
true);
887 for (
auto it = dataBuffer_.begin(); it != dataBuffer_.end();)
889 Fragment::timestamp_t fragT = (*it)->timestamp();
890 if (fragT < min || fragT > max || (fragT == max && windowWidth_ > 0))
896 TLOG(TLVL_APPLYREQUESTS) <<
"ApplyRequests: Adding Fragment with timestamp " << (*it)->timestamp() <<
" to Container";
897 cfl.addFragment(*it);
901 it = dataBuffer_.erase(it);
908 requestReceiver_->RemoveRequest(req->first);
909 checkOutOfOrderWindows(req->first);
910 requestReceiver_->RemoveRequest(req->first);
911 req = requests.erase(req);
922 if (check_stop() || exception())
928 if (mode_ == RequestMode::Ignored)
930 while (dataBufferDepthFragments_ <= 0)
932 if (check_stop() || exception() || !isHardwareOK_)
return false;
933 std::unique_lock<std::mutex> lock(dataBufferMutex_);
934 dataCondition_.wait_for(lock, std::chrono::milliseconds(10), [
this]() {
return dataBufferDepthFragments_ > 0; });
939 if ((check_stop() && requestReceiver_->size() == 0) || exception())
return false;
945 while (requestReceiver_->size() == 0 && counter < 100)
947 if (check_stop() || exception())
return false;
951 requestReceiver_->WaitForRequests(10);
957 std::unique_lock<std::mutex> dlk(dataBufferMutex_);
961 case RequestMode::Single:
962 applyRequestsSingleMode(frags);
964 case RequestMode::Window:
965 applyRequestsWindowMode(frags);
967 case RequestMode::Buffer:
968 applyRequestsBufferMode(frags);
970 case RequestMode::Ignored:
972 applyRequestsIgnoredMode(frags);
976 getDataBufferStats();
979 if (frags.size() > 0)
980 TLOG(TLVL_APPLYREQUESTS) <<
"Finished Processing Event " << (*frags.begin())->sequenceID() <<
" for fragment_id " << fragment_id() <<
".";
986 TLOG(TLVL_WARNING) << desc <<
" sequence ID " << seqId <<
", sending empty fragment";
987 for (
auto fid : fragment_ids_)
989 auto frag =
new Fragment();
990 frag->setSequenceID(seqId);
991 frag->setFragmentID(fid);
992 frag->setSystemType(Fragment::EmptyFragmentType);
993 frags.emplace_back(FragmentPtr(frag));
1000 if (requests.size() > 0)
1002 TLOG(TLVL_SENDEMPTYFRAGMENTS) <<
"Sending Empty Fragments for Sequence IDs from " << ev_counter() <<
" up to but not including " << requests.begin()->first;
1003 while (requests.begin()->first > ev_counter())
1005 sendEmptyFragment(frags, ev_counter(),
"Missed request for");
1006 ev_counter_inc(1,
true);
1013 windows_sent_ooo_[seq] = std::chrono::steady_clock::now();
1015 auto it = windows_sent_ooo_.begin();
1016 while (it != windows_sent_ooo_.end())
1018 if (seq == it->first && it->first == ev_counter())
1020 TLOG(TLVL_CHECKWINDOWS) <<
"checkOutOfOrderWindows: Sequence ID matches ev_counter, incrementing ev_counter (" << ev_counter() <<
")";
1021 ev_counter_inc(1,
true);
1022 it = windows_sent_ooo_.erase(it);
1024 else if (it->first <= ev_counter())
1026 TLOG(TLVL_CHECKWINDOWS) <<
"checkOutOfOrderWindows: Data-taking has caught up to out-of-order window request " << it->first <<
", removing from list. ev_counter=" << ev_counter();
1027 requestReceiver_->RemoveRequest(ev_counter());
1028 if (it->first == ev_counter()) ev_counter_inc(1,
true);
1029 it = windows_sent_ooo_.erase(it);
1031 else if (TimeUtils::GetElapsedTimeMicroseconds(it->second) > missing_request_window_timeout_us_)
1033 TLOG(TLVL_CHECKWINDOWS) <<
"checkOutOfOrderWindows: Out-of-order window " << it->first <<
" has timed out, setting current sequence ID and removing from list";
1034 while (ev_counter() <= it->first)
1036 if (ev_counter() < it->first) TLOG(TLVL_WARNING) <<
"Missed request for sequence ID " << ev_counter() <<
"! Will not send any data for this sequence ID!";
1037 requestReceiver_->RemoveRequest(ev_counter());
1038 ev_counter_inc(1,
true);
1040 windows_sent_ooo_.erase(windows_sent_ooo_.begin(), it);
1041 it = windows_sent_ooo_.erase(it);
int fragment_id() const
Get the current Fragment ID, if there is only one.
void applyRequestsSingleMode(artdaq::FragmentPtrs &frags)
Create fragments using data buffer for request mode Single. Precondition: dataBufferMutex_ and reques...
virtual bool checkHWStatus_()
Check any relavent hardware status registers. Return false if an error condition exists that should h...
virtual ~CommandableFragmentGenerator()
CommandableFragmentGenerator Destructor.
void applyRequestsBufferMode(artdaq::FragmentPtrs &frags)
Create fragments using data buffer for request mode Buffer. Precondition: dataBufferMutex_ and reques...
bool sendEmptyFragment(FragmentPtrs &frags, size_t sequenceId, std::string desc)
Send an EmptyFragmentType Fragment.
void getMonitoringDataLoop()
This function regularly calls checkHWStatus_(), and sets the isHardwareOK flag accordingly.
void startDataThread()
Function that launches the data thread (getDataLoop())
std::string ReportCmd(std::string const &which="")
Get a report about a user-specified run-time quantity.
virtual bool metaCommand(std::string const &command, std::string const &arg)
The meta-command is used for implementing user-specific commands in a CommandableFragmentGenerator.
bool dataBufferIsTooLarge()
Test the configured constraints on the data buffer.
void StopCmd(uint64_t timeout, uint64_t timestamp)
Stop the CommandableFragmentGenerator.
void applyRequestsWindowMode(artdaq::FragmentPtrs &frags)
Create fragments using data buffer for request mode Window. Precondition: dataBufferMutex_ and reques...
void StartCmd(int run, uint64_t timeout, uint64_t timestamp)
Start the CommandableFragmentGenerator.
virtual void pauseNoMutex()
On call to PauseCmd, pauseNoMutex() is called prior to PauseCmd acquiring the mutex ...
bool check_stop()
Routine used by applyRequests to make sure that all outstanding requests have been fulfilled before r...
void ResumeCmd(uint64_t timeout, uint64_t timestamp)
Resume the CommandableFragmentGenerator.
CommandableFragmentGenerator()
CommandableFragmentGenerator default constructor.
bool getNext(FragmentPtrs &output) overridefinal
getNext calls either applyRequests or getNext_ to get any data that is ready to be sent to the EventB...
bool waitForDataBufferReady()
Wait for the data buffer to drain (dataBufferIsTooLarge returns false), periodically reporting status...
size_t ev_counter_inc(size_t step=1, bool force=false)
Increment the event counter, if the current RequestMode allows it.
void applyRequestsIgnoredMode(artdaq::FragmentPtrs &frags)
Create fragments using data buffer for request mode Ignored. Precondition: dataBufferMutex_ and reque...
void PauseCmd(uint64_t timeout, uint64_t timestamp)
Pause the CommandableFragmentGenerator.
void getDataLoop()
When separate_data_thread is set to true, this loop repeatedly calls getNext_ and adds returned Fragm...
void sendEmptyFragments(FragmentPtrs &frags, std::map< Fragment::sequence_id_t, Fragment::timestamp_t > &requests)
This function is for Buffered and Single request modes, as they can only respond to one data request ...
Receive data requests and make them available to CommandableFragmentGenerator or other interested par...
void startMonitoringThread()
Function that launches the monitoring thread (getMonitoringDataLoop())
virtual void pause()
If a CommandableFragmentGenerator subclass is reading from hardware, the implementation of pause() sh...
virtual void resume()
The subrun number will be incremented before a call to resume.
void checkDataBuffer()
Perform data buffer pruning operations. If the RequestMode is Single, removes all but the latest Frag...
virtual std::string report()
Let's say that the contract with the report() functions is that they return a non-empty string if the...
std::string printMode_()
Return the string representation of the current RequestMode.
void getDataBufferStats()
Calculate the size of the dataBuffer and report appropriate metrics.
void checkOutOfOrderWindows(Fragment::sequence_id_t seq)
Check the windows_sent_ooo_ map for sequence IDs that may be removed.
virtual std::string reportSpecific(std::string const &what)
Report the status of a specific quantity
bool applyRequests(FragmentPtrs &output)
See if any requests have been received, and add the corresponding data Fragment objects to the output...
void joinThreads()
Join any data-taking threads. Should be called when destructing CommandableFragmentGenerator.