1 #include "artdaq/DAQdata/Globals.hh"
2 #define TRACE_NAME (app_name + "_CommandableFragmentGenerator").c_str() // include these 2 first -
4 #include "artdaq/Generators/CommandableFragmentGenerator.hh"
6 #include <boost/exception/all.hpp>
7 #include <boost/throw_exception.hpp>
12 #include "canvas/Utilities/Exception.h"
13 #include "cetlib_except/exception.h"
14 #include "fhiclcpp/ParameterSet.h"
16 #include "artdaq-core/Utilities/SimpleLookupPolicy.hh"
17 #include "artdaq-core/Data/Fragment.hh"
18 #include "artdaq-core/Data/ContainerFragmentLoader.hh"
19 #include "artdaq-core/Utilities/ExceptionHandler.hh"
20 #include "artdaq-core/Utilities/TimeUtils.hh"
31 #define TLVL_GETNEXT 10
32 #define TLVL_GETNEXT_VERBOSE 20
33 #define TLVL_CHECKSTOP 11
34 #define TLVL_EVCOUNTERINC 12
35 #define TLVL_GETDATALOOP 13
36 #define TLVL_GETDATALOOP_DATABUFFWAIT 21
37 #define TLVL_GETDATALOOP_VERBOSE 20
38 #define TLVL_WAITFORBUFFERREADY 15
39 #define TLVL_GETBUFFERSTATS 16
40 #define TLVL_CHECKDATABUFFER 17
41 #define TLVL_GETMONITORINGDATA 18
42 #define TLVL_APPLYREQUESTS 9
43 #define TLVL_SENDEMPTYFRAGMENTS 19
44 #define TLVL_CHECKWINDOWS 14
48 , requestReceiver_(nullptr)
49 , windowOffset_(ps.get<Fragment::timestamp_t>(
"request_window_offset", 0))
50 , windowWidth_(ps.get<Fragment::timestamp_t>(
"request_window_width", 0))
51 , staleTimeout_(ps.get<Fragment::timestamp_t>(
"stale_request_timeout", 0xFFFFFFFF))
52 , expectedType_(ps.get<Fragment::type_t>(
"expected_fragment_type", Fragment::type_t(Fragment::EmptyFragmentType)))
53 , uniqueWindows_(ps.get<bool>(
"request_windows_are_unique", true))
54 , missing_request_window_timeout_us_(ps.get<size_t>(
"missing_request_window_timeout_us", 5000000))
55 , window_close_timeout_us_(ps.get<size_t>(
"window_close_timeout_us", 2000000))
56 , useDataThread_(ps.get<bool>(
"separate_data_thread", false))
57 , circularDataBufferMode_(ps.get<bool>(
"circular_buffer_mode", false))
58 , sleep_on_no_data_us_(ps.get<size_t>(
"sleep_on_no_data_us", 0))
59 , data_thread_running_(false)
60 , maxDataBufferDepthFragments_(ps.get<int>(
"data_buffer_depth_fragments", 1000))
61 , maxDataBufferDepthBytes_(ps.get<size_t>(
"data_buffer_depth_mb", 1000) * 1024 * 1024)
62 , useMonitoringThread_(ps.get<bool>(
"separate_monitoring_thread", false))
63 , monitoringInterval_(ps.get<int64_t>(
"hardware_poll_interval_us", 0))
64 , lastMonitoringCall_()
68 , timeout_(std::numeric_limits<uint64_t>::max())
69 , timestamp_(std::numeric_limits<uint64_t>::max())
73 , latest_exception_report_(
"none")
76 , sleep_on_stop_us_(0)
78 board_id_ = ps.get<
int>(
"board_id");
79 instance_name_for_metrics_ =
"BoardReader." + boost::lexical_cast<std::string>(board_id_);
81 auto fragment_ids = ps.get<std::vector<artdaq::Fragment::fragment_id_t>>(
"fragment_ids", std::vector<artdaq::Fragment::fragment_id_t>());
83 TLOG(TLVL_TRACE) <<
"artdaq::CommandableFragmentGenerator::CommandableFragmentGenerator(ps)";
84 int fragment_id = ps.get<
int>(
"fragment_id", -99);
86 if (fragment_id != -99)
88 if (fragment_ids.size() != 0)
90 latest_exception_report_ =
"Error in CommandableFragmentGenerator: can't both define \"fragment_id\" and \"fragment_ids\" in FHiCL document";
91 throw cet::exception(latest_exception_report_);
95 fragment_ids.emplace_back(fragment_id);
99 for (
auto&
id : fragment_ids)
101 dataBuffers_[id].DataBufferDepthBytes = 0;
102 dataBuffers_[id].DataBufferDepthFragments = 0;
103 dataBuffers_[id].HighestRequestSeen = 0;
104 dataBuffers_[id].DataBuffer.emplace_back(FragmentPtr(
new Fragment()));
105 (*dataBuffers_[id].DataBuffer.begin())->setSystemType(Fragment::EmptyFragmentType);
108 sleep_on_stop_us_ = ps.get<
int>(
"sleep_on_stop_us", 0);
110 std::string modeString = ps.get<std::string>(
"request_mode",
"ignored");
111 if (modeString ==
"single" || modeString ==
"Single")
113 mode_ = RequestMode::Single;
115 else if (modeString.find(
"buffer") != std::string::npos || modeString.find(
"Buffer") != std::string::npos)
117 mode_ = RequestMode::Buffer;
119 else if (modeString ==
"window" || modeString ==
"Window")
121 mode_ = RequestMode::Window;
123 else if (modeString.find(
"ignore") != std::string::npos || modeString.find(
"Ignore") != std::string::npos)
125 mode_ = RequestMode::Ignored;
127 TLOG(TLVL_DEBUG) <<
"Request mode is " <<
printMode_();
129 if (mode_ != RequestMode::Ignored)
133 latest_exception_report_ =
"Error in CommandableFragmentGenerator: use_data_thread must be true when request_mode is not \"Ignored\"!";
134 throw cet::exception(latest_exception_report_);
143 requestReceiver_.reset(
nullptr);
150 TLOG(TLVL_DEBUG) <<
"Joining dataThread";
151 if (dataThread_.joinable()) dataThread_.join();
152 TLOG(TLVL_DEBUG) <<
"Joining monitoringThread";
153 if (monitoringThread_.joinable()) monitoringThread_.join();
154 TLOG(TLVL_DEBUG) <<
"joinThreads complete";
161 if (check_stop()) usleep(sleep_on_stop_us_);
162 if (exception() || force_stop_)
return false;
164 if (!useMonitoringThread_ && monitoringInterval_ > 0)
166 TLOG(TLVL_GETNEXT) <<
"getNext: Checking whether to collect Monitoring Data";
167 auto now = std::chrono::steady_clock::now();
169 if (TimeUtils::GetElapsedTimeMicroseconds(lastMonitoringCall_, now) >= static_cast<size_t>(monitoringInterval_))
171 TLOG(TLVL_GETNEXT) <<
"getNext: Collecting Monitoring Data";
172 isHardwareOK_ = checkHWStatus_();
173 TLOG(TLVL_GETNEXT) <<
"getNext: isHardwareOK_ is now " << std::boolalpha << isHardwareOK_;
174 lastMonitoringCall_ = now;
180 std::lock_guard<std::mutex> lk(mutex_);
183 TLOG(TLVL_TRACE) <<
"getNext: Calling applyRequests";
184 result = applyRequests(output);
185 TLOG(TLVL_TRACE) <<
"getNext: Done with applyRequests result=" << std::boolalpha << result;
186 for (
auto dataIter = output.begin(); dataIter != output.end(); ++dataIter)
188 TLOG(20) <<
"getNext: applyRequests() returned fragment with sequenceID = " << (*dataIter)->sequenceID()
189 <<
", type = " << (*dataIter)->typeString() <<
", id = " << std::to_string((*dataIter)->fragmentID())
190 <<
", timestamp = " << (*dataIter)->timestamp() <<
", and sizeBytes = " << (*dataIter)->sizeBytes();
195 TLOG(TLVL_ERROR) <<
"Exception found in BoardReader with board ID " << board_id() <<
"; BoardReader will now return error status when queried";
196 throw cet::exception(
"CommandableFragmentGenerator") <<
"Exception found in BoardReader with board ID " << board_id() <<
"; BoardReader will now return error status when queried";
203 TLOG(TLVL_ERROR) <<
"Stopping CFG because the hardware reports bad status!";
206 TLOG(TLVL_TRACE) <<
"getNext: Calling getNext_ w/ ev_counter()=" << ev_counter();
209 result = getNext_(output);
215 TLOG(TLVL_TRACE) <<
"getNext: Done with getNext_ - ev_counter() now " << ev_counter();
216 for (
auto dataIter = output.begin(); dataIter != output.end(); ++dataIter)
218 TLOG(TLVL_GETNEXT_VERBOSE) <<
"getNext: getNext_() returned fragment with sequenceID = " << (*dataIter)->sequenceID()
219 <<
", type = " << (*dataIter)->typeString() <<
", id = " << std::to_string((*dataIter)->fragmentID())
220 <<
", timestamp = " << (*dataIter)->timestamp() <<
", and sizeBytes = " << (*dataIter)->sizeBytes();
224 catch (
const cet::exception& e)
226 latest_exception_report_ =
"cet::exception caught in getNext(): ";
227 latest_exception_report_.append(e.what());
228 TLOG(TLVL_ERROR) <<
"getNext: cet::exception caught: " << e;
232 catch (
const boost::exception& e)
234 latest_exception_report_ =
"boost::exception caught in getNext(): ";
235 latest_exception_report_.append(boost::diagnostic_information(e));
236 TLOG(TLVL_ERROR) <<
"getNext: boost::exception caught: " << boost::diagnostic_information(e);
240 catch (
const std::exception& e)
242 latest_exception_report_ =
"std::exception caught in getNext(): ";
243 latest_exception_report_.append(e.what());
244 TLOG(TLVL_ERROR) <<
"getNext: std::exception caught: " << e.what();
250 latest_exception_report_ =
"Unknown exception caught in getNext().";
251 TLOG(TLVL_ERROR) <<
"getNext: unknown exception caught";
258 TLOG(TLVL_DEBUG) <<
"getNext: Either getNext_ or applyRequests returned false, stopping";
261 if (metricMan && !output.empty())
263 auto timestamp = output.front()->timestamp();
265 if (output.size() > 1)
267 for (
auto& outputfrag : output)
269 if (outputfrag->timestamp() > timestamp)
271 timestamp = outputfrag->timestamp();
276 metricMan->sendMetric(
"Last Timestamp", timestamp,
"Ticks", 1,
277 MetricMode::LastPoint, app_name);
285 TLOG(TLVL_CHECKSTOP) <<
"CFG::check_stop: should_stop=" << should_stop() <<
", useDataThread_=" << useDataThread_ <<
", exception status =" << int(exception());
287 if (!should_stop())
return false;
288 if (!useDataThread_ || mode_ == RequestMode::Ignored)
return true;
289 if (force_stop_)
return true;
292 TLOG(TLVL_DEBUG) <<
"should_stop is true, force_stop_ is false, requestReceiver_->isRunning() is " << std::boolalpha << requestReceiver_->isRunning();
293 return !requestReceiver_->isRunning();
298 if (force || mode_ == RequestMode::Ignored)
300 TLOG(TLVL_EVCOUNTERINC) <<
"ev_counter_inc: Incrementing ev_counter from " << ev_counter() <<
" by " << step;
301 return ev_counter_.fetch_add(step);
303 return ev_counter_.load();
308 TLOG(TLVL_TRACE) <<
"Start Command received.";
309 if (run < 0)
throw cet::exception(
"CommandableFragmentGenerator") <<
"negative run number";
312 timestamp_ = timestamp;
313 ev_counter_.store(1);
315 std::unique_lock<std::mutex> lk(dataBuffersMutex_);
316 for (
auto&
id : dataBuffers_)
318 id.second.DataBufferDepthBytes = 0;
319 id.second.DataBufferDepthFragments = 0;
320 id.second.HighestRequestSeen = 0;
321 id.second.DataBuffer.clear();
322 id.second.WindowsSent.clear();
326 should_stop_.store(
false);
327 force_stop_.store(
false);
328 exception_.store(
false);
331 latest_exception_report_ =
"none";
335 std::unique_lock<std::mutex> lk(mutex_);
336 if (useDataThread_) startDataThread();
337 if (useMonitoringThread_) startMonitoringThread();
338 if (mode_ != RequestMode::Ignored)
340 requestReceiver_->SetRunNumber(static_cast<uint32_t>(run));
341 requestReceiver_->startRequestReception();
343 TLOG(TLVL_TRACE) <<
"Start Command complete.";
348 TLOG(TLVL_TRACE) <<
"Stop Command received.";
351 timestamp_ = timestamp;
352 if (requestReceiver_) {
353 TLOG(TLVL_DEBUG) <<
"Stopping Request reception BEGIN";
354 requestReceiver_->stopRequestReception();
355 TLOG(TLVL_DEBUG) <<
"Stopping Request reception END";
359 should_stop_.store(
true);
360 std::unique_lock<std::mutex> lk(mutex_);
364 TLOG(TLVL_TRACE) <<
"Stop Command complete.";
369 TLOG(TLVL_TRACE) <<
"Pause Command received.";
371 timestamp_ = timestamp;
375 should_stop_.store(
true);
376 std::unique_lock<std::mutex> lk(mutex_);
383 TLOG(TLVL_TRACE) <<
"Resume Command received.";
385 timestamp_ = timestamp;
388 should_stop_ =
false;
391 std::unique_lock<std::mutex> lk(dataBuffersMutex_);
392 for (
auto&
id : dataBuffers_)
394 id.second.DataBufferDepthBytes = 0;
395 id.second.DataBufferDepthFragments = 0;
396 id.second.DataBuffer.clear();
402 std::unique_lock<std::mutex> lk(mutex_);
406 TLOG(TLVL_TRACE) <<
"Resume Command complete.";
411 TLOG(TLVL_TRACE) <<
"Report Command received.";
412 std::lock_guard<std::mutex> lk(mutex_);
419 std::string childReport = reportSpecific(which);
420 if (childReport.length() > 0) {
return childReport; }
423 if (which ==
"latest_exception")
425 return latest_exception_report_;
429 childReport = report();
430 if (childReport.length() > 0) {
return childReport; }
433 std::string tmpString =
"The \"" + which +
"\" command is not ";
434 tmpString.append(
"currently supported by the ");
435 tmpString.append(metricsReportingInstanceName());
436 tmpString.append(
" fragment generator.");
437 TLOG(TLVL_TRACE) <<
"Report Command complete.";
444 #pragma message "Using default implementation of CommandableFragmentGenerator::pauseNoMutex()"
449 #pragma message "Using default implementation of CommandableFragmentGenerator::pause()"
454 #pragma message "Using default implementation of CommandableFragmentGenerator::resume()"
459 #pragma message "Using default implementation of CommandableFragmentGenerator::report()"
465 #pragma message "Using default implementation of CommandableFragmentGenerator::reportSpecific(std::string)"
471 #pragma message "Using default implementation of CommandableFragmentGenerator::checkHWStatus_()"
477 #pragma message "Using default implementation of CommandableFragmentGenerator::metaCommand(std::string, std::string)"
483 if (dataThread_.joinable()) dataThread_.join();
484 TLOG(TLVL_INFO) <<
"Starting Data Receiver Thread";
488 catch (
const boost::exception& e)
490 TLOG(TLVL_ERROR) <<
"Caught boost::exception starting Data Receiver thread: " << boost::diagnostic_information(e) <<
", errno=" << errno;
491 std::cerr <<
"Caught boost::exception starting Data Receiver thread: " << boost::diagnostic_information(e) <<
", errno=" << errno << std::endl;
498 if (monitoringThread_.joinable()) monitoringThread_.join();
499 TLOG(TLVL_INFO) <<
"Starting Hardware Monitoring Thread";
503 catch (
const boost::exception& e)
505 TLOG(TLVL_ERROR) <<
"Caught boost::exception starting Hardware Monitoring thread: " << boost::diagnostic_information(e) <<
", errno=" << errno;
506 std::cerr <<
"Caught boost::exception starting Hardware Monitoring thread: " << boost::diagnostic_information(e) <<
", errno=" << errno << std::endl;
515 case RequestMode::Single:
517 case RequestMode::Buffer:
519 case RequestMode::Window:
521 case RequestMode::Ignored:
534 data_thread_running_ =
true;
539 TLOG(TLVL_DEBUG) <<
"getDataLoop: isHardwareOK is " << isHardwareOK_ <<
", aborting data thread";
540 data_thread_running_ =
false;
544 TLOG(TLVL_GETDATALOOP) <<
"getDataLoop: calling getNext_";
547 auto startdata = std::chrono::steady_clock::now();
548 FragmentPtrs newData;
552 data = getNext_(newData);
556 ExceptionHandler(ExceptionHandlerRethrow::no,
557 "Exception thrown by fragment generator in CommandableFragmentGenerator::getDataLoop; setting exception state to \"true\"");
560 data_thread_running_ =
false;
566 metricMan->sendMetric(
"Avg Data Acquisition Time", TimeUtils::GetElapsedTime(startdata),
"s", 3, artdaq::MetricMode::Average);
569 if (newData.size() == 0 && sleep_on_no_data_us_ > 0)
571 usleep(sleep_on_no_data_us_);
574 auto dataIter = newData.begin();
575 while (dataIter != newData.end())
577 TLOG(TLVL_GETDATALOOP_VERBOSE) <<
"getDataLoop: getNext_() returned fragment with timestamp = " << (*dataIter)->timestamp() <<
", and sizeBytes = " << (*dataIter)->sizeBytes();
579 auto frag_id = (*dataIter)->fragmentID();
580 if (!dataBuffers_.count(frag_id)) {
581 throw cet::exception(
"DataBufferError") <<
"Error in CommandableFragmentGenerator: Recevied Fragment with fragment_id " << frag_id <<
", but this ID was not declared in fragment_ids!";
584 TLOG(TLVL_GETDATALOOP_DATABUFFWAIT) <<
"Waiting for data buffer ready";
585 if (!waitForDataBufferReady(frag_id))
return;
586 TLOG(TLVL_GETDATALOOP_DATABUFFWAIT) <<
"Done waiting for data buffer ready";
588 TLOG(TLVL_GETDATALOOP) <<
"getDataLoop: processing data";
589 if (data && !force_stop_)
591 std::unique_lock<std::mutex> lock(dataBuffersMutex_);
594 case RequestMode::Single:
595 dataBuffers_[frag_id].DataBuffer.clear();
596 dataBuffers_[frag_id].DataBufferDepthBytes = (*dataIter)->sizeBytes();
597 dataBuffers_[frag_id].DataBuffer.emplace_back(std::move(*dataIter));
598 dataIter = newData.erase(dataIter);
600 case RequestMode::Buffer:
601 case RequestMode::Ignored:
602 case RequestMode::Window:
605 dataBuffers_[frag_id].DataBufferDepthBytes += (*dataIter)->sizeBytes();
606 dataBuffers_[frag_id].DataBuffer.emplace_back(std::move(*dataIter));
607 dataIter = newData.erase(dataIter);
610 getDataBufferStats(frag_id);
618 std::unique_lock<std::mutex> lock(dataBuffersMutex_);
619 for (
auto&
id : dataBuffers_) {
620 if (
id.second.DataBuffer.size() > 0)
622 dataCondition_.notify_all();
627 if (!data || force_stop_)
629 TLOG(TLVL_INFO) <<
"Data flow has stopped. Ending data collection thread";
630 std::unique_lock<std::mutex> lock(dataBuffersMutex_);
631 data_thread_running_ =
false;
632 if (requestReceiver_) requestReceiver_->ClearRequests();
634 TLOG(TLVL_INFO) <<
"getDataLoop: Ending thread";
642 if (!dataBuffers_.count(
id)) {
643 throw cet::exception(
"DataBufferError") <<
"Error in CommandableFragmentGenerator: Cannot wait for data buffer for ID " <<
id <<
" because it does not exist!";
645 auto startwait = std::chrono::steady_clock::now();
647 auto lastwaittime = 0ULL;
650 std::unique_lock<std::mutex> lock(dataBuffersMutex_);
651 getDataBufferStats(
id);
654 while (dataBufferIsTooLarge(
id))
656 if (!circularDataBufferMode_)
660 TLOG(TLVL_DEBUG) <<
"Run ended while waiting for buffer to shrink!";
661 std::unique_lock<std::mutex> lock(dataBuffersMutex_);
662 getDataBufferStats(
id);
663 dataCondition_.notify_all();
664 data_thread_running_ =
false;
667 auto waittime = TimeUtils::GetElapsedTimeMilliseconds(startwait);
669 if (first || (waittime != lastwaittime && waittime % 1000 == 0))
671 TLOG(TLVL_WARNING) <<
"Bad Omen: Data Buffer has exceeded its size limits. "
672 <<
"(seq_id=" << ev_counter() <<
", frag_id=" <<
id
673 <<
", frags=" << dataBuffers_[id].DataBufferDepthFragments <<
"/" << maxDataBufferDepthFragments_
674 <<
", szB=" << dataBuffers_[id].DataBufferDepthBytes <<
"/" << maxDataBufferDepthBytes_ <<
")";
675 TLOG(TLVL_TRACE) <<
"Bad Omen: Possible causes include requests not getting through or Ignored-mode BR issues";
678 if (waittime % 5 && waittime != lastwaittime)
680 TLOG(TLVL_WAITFORBUFFERREADY) <<
"getDataLoop: Data Retreival paused for " << waittime <<
" ms waiting for data buffer to drain";
682 lastwaittime = waittime;
687 std::unique_lock<std::mutex> lock(dataBuffersMutex_);
688 getDataBufferStats(
id);
689 if (dataBufferIsTooLarge(
id))
691 auto begin = dataBuffers_[id].DataBuffer.begin();
692 if (begin == dataBuffers_[
id].DataBuffer.end())
694 TLOG(TLVL_WARNING) <<
"Data buffer is reported as too large, but doesn't contain any Fragments! Possible corrupt memory!";
699 TLOG(TLVL_WAITFORBUFFERREADY) <<
"waitForDataBufferReady: Dropping Fragment with timestamp " << (*begin)->timestamp() <<
" from data buffer (Buffer over-size, circular data buffer mode)";
701 dataBuffers_[id].DataBufferDepthBytes -= (*begin)->sizeBytes();
702 dataBuffers_[id].DataBuffer.erase(begin);
703 getDataBufferStats(
id);
713 if (!dataBuffers_.count(
id)) {
714 throw cet::exception(
"DataBufferError") <<
"Error in CommandableFragmentGenerator: Cannot check size of data buffer for ID " <<
id <<
" because it does not exist!";
716 return (maxDataBufferDepthFragments_ > 0 && dataBuffers_[
id].DataBufferDepthFragments > maxDataBufferDepthFragments_) || (maxDataBufferDepthBytes_ > 0 && dataBuffers_[id].DataBufferDepthBytes > maxDataBufferDepthBytes_);
721 if (!dataBuffers_.count(
id)) {
722 throw cet::exception(
"DataBufferError") <<
"Error in CommandableFragmentGenerator: Cannot get stats of data buffer for ID " <<
id <<
" because it does not exist!";
725 dataBuffers_[id].DataBufferDepthFragments = dataBuffers_[id].DataBuffer.size();
729 TLOG(TLVL_GETBUFFERSTATS) <<
"getDataBufferStats: Sending Metrics";
730 metricMan->sendMetric(
"Buffer Depth Fragments", dataBuffers_[
id].DataBufferDepthFragments.load(),
"fragments", 1, MetricMode::LastPoint);
731 metricMan->sendMetric(
"Buffer Depth Bytes", dataBuffers_[
id].DataBufferDepthBytes.load(),
"bytes", 1, MetricMode::LastPoint);
733 TLOG(TLVL_GETBUFFERSTATS) <<
"getDataBufferStats: frags=" << dataBuffers_[id].DataBufferDepthFragments.load() <<
"/" << maxDataBufferDepthFragments_
734 <<
", sz=" << dataBuffers_[id].DataBufferDepthBytes.load() <<
"/" << maxDataBufferDepthBytes_;
739 if (!dataBuffers_.count(
id)) {
740 throw cet::exception(
"DataBufferError") <<
"Error in CommandableFragmentGenerator: Cannot check data buffer for ID " <<
id <<
" because it does not exist!";
743 if (dataBuffers_[
id].DataBufferDepthFragments > 0 && mode_ != RequestMode::Single && mode_ != RequestMode::Ignored)
746 getDataBufferStats(
id);
747 while (dataBufferIsTooLarge(
id))
749 auto begin = dataBuffers_[id].DataBuffer.begin();
750 TLOG(TLVL_CHECKDATABUFFER) <<
"checkDataBuffer: Dropping Fragment with timestamp " << (*begin)->timestamp() <<
" from data buffer (Buffer over-size)";
751 dataBuffers_[id].DataBufferDepthBytes -= (*begin)->sizeBytes();
752 dataBuffers_[id].DataBuffer.erase(begin);
753 getDataBufferStats(
id);
755 if (dataBuffers_[
id].DataBuffer.size() > 0)
757 TLOG(TLVL_CHECKDATABUFFER) <<
"Determining if Fragments can be dropped from data buffer";
758 Fragment::timestamp_t last = dataBuffers_[id].DataBuffer.back()->timestamp();
759 Fragment::timestamp_t min = last > staleTimeout_ ? last - staleTimeout_ : 0;
760 for (
auto it = dataBuffers_[
id].DataBuffer.begin(); it != dataBuffers_[id].DataBuffer.end();)
762 if ((*it)->timestamp() < min)
764 TLOG(TLVL_CHECKDATABUFFER) <<
"checkDataBuffer: Dropping Fragment with timestamp " << (*it)->timestamp() <<
" from data buffer (timeout=" << staleTimeout_ <<
", min=" << min <<
")";
765 dataBuffers_[id].DataBufferDepthBytes -= (*it)->sizeBytes();
766 it = dataBuffers_[id].DataBuffer.erase(it);
773 getDataBufferStats(
id);
782 if (should_stop() || monitoringInterval_ <= 0)
784 TLOG(TLVL_DEBUG) <<
"getMonitoringDataLoop: should_stop() is " << std::boolalpha << should_stop()
785 <<
" and monitoringInterval is " << monitoringInterval_ <<
", returning";
788 TLOG(TLVL_GETMONITORINGDATA) <<
"getMonitoringDataLoop: Determining whether to call checkHWStatus_";
790 auto now = std::chrono::steady_clock::now();
791 if (TimeUtils::GetElapsedTimeMicroseconds(lastMonitoringCall_, now) >= static_cast<size_t>(monitoringInterval_))
793 isHardwareOK_ = checkHWStatus_();
794 TLOG(TLVL_GETMONITORINGDATA) <<
"getMonitoringDataLoop: isHardwareOK_ is now " << std::boolalpha << isHardwareOK_;
795 lastMonitoringCall_ = now;
797 usleep(monitoringInterval_ / 10);
805 TLOG(TLVL_APPLYREQUESTS) <<
"Mode is Ignored; Copying data to output";
806 for (
auto&
id : dataBuffers_) {
807 std::move(
id.second.DataBuffer.begin(),
id.second.DataBuffer.end(), std::inserter(frags, frags.end()));
808 id.second.DataBufferDepthBytes = 0;
809 id.second.DataBufferDepthFragments = 0;
810 id.second.DataBuffer.clear();
817 auto requests = requestReceiver_->GetRequests();
818 while (requests.size() > 1)
821 requestReceiver_->RemoveRequest(requests.begin()->first);
822 requests.erase(requests.begin());
824 sendEmptyFragments(frags, requests);
827 if (requests.size() == 0 || !requests.count(ev_counter()))
return;
829 for (
auto&
id : dataBuffers_) {
830 if (
id.second.DataBuffer.size() > 0)
832 assert(
id.second.DataBuffer.size() == 1);
833 TLOG(TLVL_APPLYREQUESTS) <<
"Mode is Single; Sending copy of last event";
834 for (
auto& fragptr :
id.second.DataBuffer)
837 auto frag = fragptr.get();
838 auto newfrag = std::unique_ptr<artdaq::Fragment>(
new Fragment(ev_counter(), frag->fragmentID()));
839 newfrag->resize(frag->size() - detail::RawFragmentHeader::num_words());
840 memcpy(newfrag->headerAddress(), frag->headerAddress(), frag->sizeBytes());
841 newfrag->setTimestamp(requests[ev_counter()]);
842 newfrag->setSequenceID(ev_counter());
843 frags.push_back(std::move(newfrag));
848 sendEmptyFragment(frags, ev_counter(),
id.first,
"No data for");
851 requestReceiver_->RemoveRequest(ev_counter());
852 ev_counter_inc(1,
true);
858 auto requests = requestReceiver_->GetRequests();
859 while (requests.size() > 1)
862 requestReceiver_->RemoveRequest(requests.begin()->first);
863 requests.erase(requests.begin());
865 sendEmptyFragments(frags, requests);
868 if (requests.size() == 0 || !requests.count(ev_counter()))
return;
870 for (
auto&
id : dataBuffers_) {
872 TLOG(TLVL_DEBUG) <<
"Creating ContainerFragment for Buffered Fragments";
873 frags.emplace_back(
new artdaq::Fragment(ev_counter(),
id.first));
874 frags.back()->setTimestamp(requests[ev_counter()]);
875 ContainerFragmentLoader cfl(*frags.back());
876 cfl.set_missing_data(
false);
880 for (
auto it =
id.second.DataBuffer.begin(); it !=
id.second.DataBuffer.end();)
882 TLOG(TLVL_APPLYREQUESTS) <<
"ApplyRequests: Adding Fragment with timestamp " << (*it)->timestamp() <<
" to Container with sequence ID " << ev_counter();
883 cfl.addFragment(*it);
884 id.second.DataBufferDepthBytes -= (*it)->sizeBytes();
885 it =
id.second.DataBuffer.erase(it);
888 requestReceiver_->RemoveRequest(ev_counter());
889 ev_counter_inc(1,
true);
893 void artdaq::CommandableFragmentGenerator::applyRequestsWindowMode_CheckAndFillDataBuffer(artdaq::FragmentPtrs& frags, artdaq::Fragment::fragment_id_t
id, artdaq::Fragment::sequence_id_t seq, artdaq::Fragment::timestamp_t ts)
895 TLOG(TLVL_APPLYREQUESTS) <<
"applyRequestsWindowMode_CheckAndFillDataBuffer: Checking that data exists for request window " << seq;
896 Fragment::timestamp_t min = ts > windowOffset_ ? ts - windowOffset_ : 0;
897 Fragment::timestamp_t max = min + windowWidth_;
898 TLOG(TLVL_APPLYREQUESTS) <<
"ApplyRequestsWindowsMode_CheckAndFillDataBuffer: min is " << min <<
", max is " << max
899 <<
" and first/last points in buffer are " << (dataBuffers_[id].DataBuffer.size() > 0 ? dataBuffers_[id].DataBuffer.front()->timestamp() : 0)
900 <<
"/" << (dataBuffers_[
id].DataBuffer.size() > 0 ? dataBuffers_[id].DataBuffer.back()->timestamp() : 0)
901 <<
" (sz=" << dataBuffers_[
id].DataBuffer.size() <<
" [" << dataBuffers_[id].DataBufferDepthBytes.load()
902 <<
"/" << maxDataBufferDepthBytes_ <<
"])";
903 bool windowClosed = dataBuffers_[id].DataBuffer.size() > 0 && dataBuffers_[id].DataBuffer.back()->timestamp() >= max;
904 bool windowTimeout = !windowClosed && TimeUtils::GetElapsedTimeMicroseconds(requestReceiver_->GetRequestTime(seq)) > window_close_timeout_us_;
907 TLOG(TLVL_WARNING) <<
"applyRequestsWindowMode_CheckAndFillDataBuffer: A timeout occurred waiting for data to close the request window ({" << min <<
"-" << max
908 <<
"}, buffer={" << (dataBuffers_[id].DataBuffer.size() > 0 ? dataBuffers_[id].DataBuffer.front()->timestamp() : 0) <<
"-"
909 << (dataBuffers_[
id].DataBuffer.size() > 0 ? dataBuffers_[id].DataBuffer.back()->timestamp() : 0)
910 <<
"} ). Time waiting: "
911 << TimeUtils::GetElapsedTimeMicroseconds(requestReceiver_->GetRequestTime(seq)) <<
" us "
912 <<
"(> " << window_close_timeout_us_ <<
" us).";
914 if (windowClosed || !data_thread_running_ || windowTimeout)
916 TLOG(TLVL_DEBUG) <<
"applyRequestsWindowMode_CheckAndFillDataBuffer: Creating ContainerFragment for Window-requested Fragments";
917 frags.emplace_back(
new artdaq::Fragment(seq,
id));
918 frags.back()->setTimestamp(ts);
919 ContainerFragmentLoader cfl(*frags.back());
933 if (!windowClosed || (dataBuffers_[
id].DataBuffer.size() > 0 && dataBuffers_[id].DataBuffer.front()->timestamp() > min))
935 TLOG(TLVL_DEBUG) <<
"applyRequestsWindowMode_CheckAndFillDataBuffer: Request window starts before and/or ends after the current data buffer, setting ContainerFragment's missing_data flag!"
936 <<
" (requestWindowRange=[" << min <<
"," << max <<
"], "
937 <<
"buffer={" << (dataBuffers_[id].DataBuffer.size() > 0 ? dataBuffers_[id].DataBuffer.front()->timestamp() : 0) <<
"-"
938 << (dataBuffers_[
id].DataBuffer.size() > 0 ? dataBuffers_[id].DataBuffer.back()->timestamp() : 0) <<
"}";
939 cfl.set_missing_data(
true);
943 for (
auto it = dataBuffers_[
id].DataBuffer.begin(); it != dataBuffers_[id].DataBuffer.end();)
945 Fragment::timestamp_t fragT = (*it)->timestamp();
946 if (fragT < min || fragT > max || (fragT == max && windowWidth_ > 0))
952 TLOG(TLVL_APPLYREQUESTS) <<
"applyRequestsWindowMode_CheckAndFillDataBuffer: Adding Fragment with timestamp " << (*it)->timestamp() <<
" to Container";
953 cfl.addFragment(*it);
957 dataBuffers_[id].DataBufferDepthBytes -= (*it)->sizeBytes();
958 it = dataBuffers_[id].DataBuffer.erase(it);
966 dataBuffers_[id].WindowsSent[seq] = std::chrono::steady_clock::now();
967 if (seq > dataBuffers_[
id].HighestRequestSeen) dataBuffers_[id].HighestRequestSeen = seq;
973 TLOG(TLVL_APPLYREQUESTS) <<
"applyRequestsWindowMode BEGIN";
975 auto requests = requestReceiver_->GetRequests();
977 TLOG(TLVL_APPLYREQUESTS) <<
"applyRequestsWindowMode: Starting request processing";
978 for (
auto req = requests.begin(); req != requests.end();)
980 TLOG(TLVL_APPLYREQUESTS) <<
"applyRequestsWindowMode: processing request with sequence ID " << req->first <<
", timestamp " << req->second;
983 while (req->first < ev_counter() && requests.size() > 0)
985 TLOG(TLVL_APPLYREQUESTS) <<
"applyRequestsWindowMode: Clearing passed request for sequence ID " << req->first;
986 requestReceiver_->RemoveRequest(req->first);
987 req = requests.erase(req);
989 if (requests.size() == 0)
break;
991 for (
auto&
id : dataBuffers_)
993 if (!
id.second.WindowsSent.count(req->first)) {
994 applyRequestsWindowMode_CheckAndFillDataBuffer(frags,
id.first, req->first, req->second);
997 checkSentWindows(req->first);
1002 for (
auto&
id : dataBuffers_) {
1003 std::set<artdaq::Fragment::sequence_id_t> seqs;
1004 for (
auto& seq :
id.second.WindowsSent) {
1005 seqs.insert(seq.first);
1007 for (
auto& seq : seqs)
1009 checkSentWindows(seq);
1016 if (check_stop() || exception())
1022 if (mode_ == RequestMode::Ignored)
1024 while (dataBufferFragmentCount_() == 0)
1026 if (check_stop() || exception() || !isHardwareOK_)
return false;
1027 std::unique_lock<std::mutex> lock(dataBuffersMutex_);
1028 dataCondition_.wait_for(lock, std::chrono::milliseconds(10), [
this]() {
return dataBufferFragmentCount_() > 0; });
1033 if ((check_stop() && requestReceiver_->size() == 0) || exception())
return false;
1035 std::unique_lock<std::mutex> lock(dataBuffersMutex_);
1036 dataCondition_.wait_for(lock, std::chrono::milliseconds(10));
1043 while (requestReceiver_->size() == 0 && counter < 100)
1045 if (check_stop() || exception())
return false;
1049 requestReceiver_->WaitForRequests(10);
1055 std::unique_lock<std::mutex> dlk(dataBuffersMutex_);
1059 case RequestMode::Single:
1060 applyRequestsSingleMode(frags);
1062 case RequestMode::Window:
1063 applyRequestsWindowMode(frags);
1065 case RequestMode::Buffer:
1066 applyRequestsBufferMode(frags);
1068 case RequestMode::Ignored:
1070 applyRequestsIgnoredMode(frags);
1074 if (!data_thread_running_ || force_stop_)
1076 TLOG(TLVL_INFO) <<
"Data thread has stopped; Clearing data buffers";
1077 for (
auto&
id : dataBuffers_) {
1078 id.second.DataBufferDepthBytes = 0;
1079 id.second.DataBufferDepthFragments = 0;
1080 id.second.DataBuffer.clear();
1084 getDataBuffersStats();
1087 if (frags.size() > 0)
1088 TLOG(TLVL_APPLYREQUESTS) <<
"Finished Processing requests, returning " << frags.size() <<
" fragments, current ev_counter is " << ev_counter();
1094 TLOG(TLVL_WARNING) << desc <<
" sequence ID " << seqId <<
", sending empty fragment";
1095 auto frag =
new Fragment();
1096 frag->setSequenceID(seqId);
1097 frag->setFragmentID(fragmentId);
1098 frag->setSystemType(Fragment::EmptyFragmentType);
1099 frags.emplace_back(FragmentPtr(frag));
1105 if (requests.size() > 0)
1107 TLOG(TLVL_SENDEMPTYFRAGMENTS) <<
"Sending Empty Fragments for Sequence IDs from " << ev_counter() <<
" up to but not including " << requests.begin()->first;
1108 while (requests.begin()->first > ev_counter())
1110 for (
auto& fid : dataBuffers_)
1112 sendEmptyFragment(frags, ev_counter(), fid.first,
"Missed request for");
1114 ev_counter_inc(1,
true);
1121 bool seqComplete =
true;
1122 bool seqTimeout =
false;
1123 for (
auto&
id : dataBuffers_) {
1124 if (!
id.second.WindowsSent.count(seq) ||
id.second.HighestRequestSeen < seq) {
1125 seqComplete =
false;
1127 if (
id.second.WindowsSent.count(seq) && TimeUtils::GetElapsedTimeMicroseconds(
id.second.WindowsSent[seq]) > missing_request_window_timeout_us_) {
1132 TLOG(TLVL_CHECKWINDOWS) <<
"checkSentWindows: Request " << seq <<
" is complete, removing from requestReceiver.";
1133 requestReceiver_->RemoveRequest(seq);
1135 if (ev_counter() == seq) {
1136 TLOG(TLVL_CHECKWINDOWS) <<
"checkSentWindows: Sequence ID matches ev_counter, incrementing ev_counter (" << ev_counter() <<
")";
1138 for (
auto&
id : dataBuffers_) {
1139 id.second.WindowsSent.erase(seq);
1142 ev_counter_inc(1,
true);
1146 TLOG(TLVL_CHECKWINDOWS) <<
"checkSentWindows: Sent Window history indicates that requests between " << ev_counter() <<
" and " << seq <<
" have timed out.";
1147 while (ev_counter() <= seq)
1149 if (ev_counter() < seq) TLOG(TLVL_WARNING) <<
"Missed request for sequence ID " << ev_counter() <<
"! Will not send any data for this sequence ID!";
1150 requestReceiver_->RemoveRequest(ev_counter());
1152 for (
auto&
id : dataBuffers_) {
1153 id.second.WindowsSent.erase(ev_counter());
1156 ev_counter_inc(1,
true);
CommandableFragmentGenerator(const fhicl::ParameterSet &ps)
CommandableFragmentGenerator Constructor.
void applyRequestsSingleMode(artdaq::FragmentPtrs &frags)
Create fragments using data buffer for request mode Single. Precondition: dataBufferMutex_ and reques...
virtual bool checkHWStatus_()
Check any relavent hardware status registers. Return false if an error condition exists that should h...
virtual ~CommandableFragmentGenerator()
CommandableFragmentGenerator Destructor.
void applyRequestsBufferMode(artdaq::FragmentPtrs &frags)
Create fragments using data buffer for request mode Buffer. Precondition: dataBufferMutex_ and reques...
void getDataBufferStats(Fragment::fragment_id_t id)
Calculate the size of the dataBuffer and report appropriate metrics.
void getMonitoringDataLoop()
This function regularly calls checkHWStatus_(), and sets the isHardwareOK flag accordingly.
void startDataThread()
Function that launches the data thread (getDataLoop())
std::string ReportCmd(std::string const &which="")
Get a report about a user-specified run-time quantity.
virtual bool metaCommand(std::string const &command, std::string const &arg)
The meta-command is used for implementing user-specific commands in a CommandableFragmentGenerator.
void StopCmd(uint64_t timeout, uint64_t timestamp)
Stop the CommandableFragmentGenerator.
void checkSentWindows(Fragment::sequence_id_t seq)
Check the windows_sent_ooo_ map for sequence IDs that may be removed.
bool sendEmptyFragment(FragmentPtrs &frags, size_t sequenceId, Fragment::fragment_id_t fragmentId, std::string desc)
Send an EmptyFragmentType Fragment.
void applyRequestsWindowMode(artdaq::FragmentPtrs &frags)
Create fragments using data buffer for request mode Window. Precondition: dataBufferMutex_ and reques...
void StartCmd(int run, uint64_t timeout, uint64_t timestamp)
Start the CommandableFragmentGenerator.
virtual void pauseNoMutex()
On call to PauseCmd, pauseNoMutex() is called prior to PauseCmd acquiring the mutex ...
bool check_stop()
Routine used by applyRequests to make sure that all outstanding requests have been fulfilled before r...
void ResumeCmd(uint64_t timeout, uint64_t timestamp)
Resume the CommandableFragmentGenerator.
bool getNext(FragmentPtrs &output) overridefinal
getNext calls either applyRequests or getNext_ to get any data that is ready to be sent to the EventB...
size_t ev_counter_inc(size_t step=1, bool force=false)
Increment the event counter, if the current RequestMode allows it.
bool applyRequests(FragmentPtrs &frags)
See if any requests have been received, and add the corresponding data Fragment objects to the output...
void applyRequestsIgnoredMode(artdaq::FragmentPtrs &frags)
Create fragments using data buffer for request mode Ignored. Precondition: dataBufferMutex_ and reque...
void PauseCmd(uint64_t timeout, uint64_t timestamp)
Pause the CommandableFragmentGenerator.
void getDataLoop()
When separate_data_thread is set to true, this loop repeatedly calls getNext_ and adds returned Fragm...
void sendEmptyFragments(FragmentPtrs &frags, std::map< Fragment::sequence_id_t, Fragment::timestamp_t > &requests)
This function is for Buffered and Single request modes, as they can only respond to one data request ...
Receive data requests and make them available to CommandableFragmentGenerator or other interested par...
void startMonitoringThread()
Function that launches the monitoring thread (getMonitoringDataLoop())
virtual void pause()
If a CommandableFragmentGenerator subclass is reading from hardware, the implementation of pause() sh...
virtual void resume()
The subrun number will be incremented before a call to resume.
bool waitForDataBufferReady(Fragment::fragment_id_t id)
Wait for the data buffer to drain (dataBufferIsTooLarge returns false), periodically reporting status...
virtual std::string report()
Let's say that the contract with the report() functions is that they return a non-empty string if the...
std::string printMode_()
Return the string representation of the current RequestMode.
void checkDataBuffer(Fragment::fragment_id_t id)
Perform data buffer pruning operations. If the RequestMode is Single, removes all but the latest Frag...
virtual std::string reportSpecific(std::string const &what)
Report the status of a specific quantity
bool dataBufferIsTooLarge(Fragment::fragment_id_t id)
Test the configured constraints on the data buffer.
void joinThreads()
Join any data-taking threads. Should be called when destructing CommandableFragmentGenerator.