1 #include "artdaq/DAQdata/Globals.hh"
2 #define TRACE_NAME (app_name + "_CommandableFragmentGenerator").c_str() // include these 2 first -
4 #include "artdaq/Generators/CommandableFragmentGenerator.hh"
6 #include <boost/exception/all.hpp>
7 #include <boost/throw_exception.hpp>
12 #include "canvas/Utilities/Exception.h"
13 #include "cetlib_except/exception.h"
14 #include "fhiclcpp/ParameterSet.h"
16 #include "artdaq-core/Data/ContainerFragmentLoader.hh"
17 #include "artdaq-core/Data/Fragment.hh"
18 #include "artdaq-core/Utilities/ExceptionHandler.hh"
19 #include "artdaq-core/Utilities/SimpleLookupPolicy.hh"
20 #include "artdaq-core/Utilities/TimeUtils.hh"
30 #define TLVL_GETNEXT 10
31 #define TLVL_GETNEXT_VERBOSE 20
32 #define TLVL_CHECKSTOP 11
33 #define TLVL_EVCOUNTERINC 12
34 #define TLVL_GETDATALOOP 13
35 #define TLVL_GETDATALOOP_DATABUFFWAIT 21
36 #define TLVL_GETDATALOOP_VERBOSE 20
37 #define TLVL_WAITFORBUFFERREADY 15
38 #define TLVL_GETBUFFERSTATS 16
39 #define TLVL_CHECKDATABUFFER 17
40 #define TLVL_GETMONITORINGDATA 18
41 #define TLVL_APPLYREQUESTS 9
42 #define TLVL_SENDEMPTYFRAGMENTS 19
43 #define TLVL_CHECKWINDOWS 14
47 , requestReceiver_(nullptr)
48 , bufferModeKeepLatest_(ps.get<bool>(
"buffer_mode_keep_latest", false))
49 , windowOffset_(ps.get<Fragment::timestamp_t>(
"request_window_offset", 0))
50 , windowWidth_(ps.get<Fragment::timestamp_t>(
"request_window_width", 0))
51 , staleTimeout_(ps.get<Fragment::timestamp_t>(
"stale_request_timeout", 0xFFFFFFFF))
52 , expectedType_(ps.get<Fragment::type_t>(
"expected_fragment_type", Fragment::type_t(Fragment::EmptyFragmentType)))
53 , uniqueWindows_(ps.get<bool>(
"request_windows_are_unique", true))
54 , missing_request_window_timeout_us_(ps.get<size_t>(
"missing_request_window_timeout_us", 5000000))
55 , window_close_timeout_us_(ps.get<size_t>(
"window_close_timeout_us", 2000000))
56 , useDataThread_(ps.get<bool>(
"separate_data_thread", false))
57 , circularDataBufferMode_(ps.get<bool>(
"circular_buffer_mode", false))
58 , sleep_on_no_data_us_(ps.get<size_t>(
"sleep_on_no_data_us", 0))
59 , data_thread_running_(false)
60 , maxDataBufferDepthFragments_(ps.get<int>(
"data_buffer_depth_fragments", 1000))
61 , maxDataBufferDepthBytes_(ps.get<size_t>(
"data_buffer_depth_mb", 1000) * 1024 * 1024)
62 , useMonitoringThread_(ps.get<bool>(
"separate_monitoring_thread", false))
63 , monitoringInterval_(ps.get<int64_t>(
"hardware_poll_interval_us", 0))
64 , lastMonitoringCall_()
68 , timeout_(std::numeric_limits<uint64_t>::max())
69 , timestamp_(std::numeric_limits<uint64_t>::max())
73 , latest_exception_report_(
"none")
76 , sleep_on_stop_us_(0)
78 board_id_ = ps.get<
int>(
"board_id");
79 instance_name_for_metrics_ =
"BoardReader." + boost::lexical_cast<std::string>(board_id_);
81 auto fragment_ids = ps.get<std::vector<artdaq::Fragment::fragment_id_t>>(
"fragment_ids", std::vector<artdaq::Fragment::fragment_id_t>());
83 TLOG(TLVL_TRACE) <<
"artdaq::CommandableFragmentGenerator::CommandableFragmentGenerator(ps)";
86 if (fragment_id != -99)
88 if (fragment_ids.size() != 0)
90 latest_exception_report_ =
"Error in CommandableFragmentGenerator: can't both define \"fragment_id\" and \"fragment_ids\" in FHiCL document";
91 throw cet::exception(latest_exception_report_);
95 fragment_ids.emplace_back(fragment_id);
99 for (
auto&
id : fragment_ids)
101 dataBuffers_[id].DataBufferDepthBytes = 0;
102 dataBuffers_[id].DataBufferDepthFragments = 0;
103 dataBuffers_[id].HighestRequestSeen = 0;
104 dataBuffers_[id].BufferFragmentKept =
false;
105 dataBuffers_[id].DataBuffer.emplace_back(FragmentPtr(
new Fragment()));
106 (*dataBuffers_[id].DataBuffer.begin())->setSystemType(Fragment::EmptyFragmentType);
109 sleep_on_stop_us_ = ps.get<
int>(
"sleep_on_stop_us", 0);
111 std::string modeString = ps.get<std::string>(
"request_mode",
"ignored");
112 if (modeString ==
"single" || modeString ==
"Single")
114 mode_ = RequestMode::Single;
116 else if (modeString.find(
"buffer") != std::string::npos || modeString.find(
"Buffer") != std::string::npos)
118 mode_ = RequestMode::Buffer;
120 else if (modeString ==
"window" || modeString ==
"Window")
122 mode_ = RequestMode::Window;
124 else if (modeString.find(
"ignore") != std::string::npos || modeString.find(
"Ignore") != std::string::npos)
126 mode_ = RequestMode::Ignored;
128 else if (modeString.find(
"sequence") != std::string::npos || modeString.find(
"Sequence") != std::string::npos)
130 mode_ = RequestMode::SequenceID;
132 TLOG(TLVL_DEBUG) <<
"Request mode is " <<
printMode_();
134 if (mode_ != RequestMode::Ignored)
138 latest_exception_report_ =
"Error in CommandableFragmentGenerator: use_data_thread must be true when request_mode is not \"Ignored\"!";
139 throw cet::exception(latest_exception_report_);
148 requestReceiver_.reset(
nullptr);
155 TLOG(TLVL_DEBUG) <<
"Joining dataThread";
156 if (dataThread_.joinable()) dataThread_.join();
157 TLOG(TLVL_DEBUG) <<
"Joining monitoringThread";
158 if (monitoringThread_.joinable()) monitoringThread_.join();
159 TLOG(TLVL_DEBUG) <<
"joinThreads complete";
166 if (check_stop()) usleep(sleep_on_stop_us_);
167 if (exception() || force_stop_)
return false;
169 if (!useMonitoringThread_ && monitoringInterval_ > 0)
171 TLOG(TLVL_GETNEXT) <<
"getNext: Checking whether to collect Monitoring Data";
172 auto now = std::chrono::steady_clock::now();
174 if (TimeUtils::GetElapsedTimeMicroseconds(lastMonitoringCall_, now) >= static_cast<size_t>(monitoringInterval_))
176 TLOG(TLVL_GETNEXT) <<
"getNext: Collecting Monitoring Data";
177 isHardwareOK_ = checkHWStatus_();
178 TLOG(TLVL_GETNEXT) <<
"getNext: isHardwareOK_ is now " << std::boolalpha << isHardwareOK_;
179 lastMonitoringCall_ = now;
185 std::lock_guard<std::mutex> lk(mutex_);
188 TLOG(TLVL_TRACE) <<
"getNext: Calling applyRequests";
189 result = applyRequests(output);
190 TLOG(TLVL_TRACE) <<
"getNext: Done with applyRequests result=" << std::boolalpha << result;
191 for (
auto dataIter = output.begin(); dataIter != output.end(); ++dataIter)
193 TLOG(20) <<
"getNext: applyRequests() returned fragment with sequenceID = " << (*dataIter)->sequenceID()
194 <<
", type = " << (*dataIter)->typeString() <<
", id = " << std::to_string((*dataIter)->fragmentID())
195 <<
", timestamp = " << (*dataIter)->timestamp() <<
", and sizeBytes = " << (*dataIter)->sizeBytes();
200 TLOG(TLVL_ERROR) <<
"Exception found in BoardReader with board ID " << board_id() <<
"; BoardReader will now return error status when queried";
201 throw cet::exception(
"CommandableFragmentGenerator") <<
"Exception found in BoardReader with board ID " << board_id() <<
"; BoardReader will now return error status when queried";
208 TLOG(TLVL_ERROR) <<
"Stopping CFG because the hardware reports bad status!";
211 TLOG(TLVL_TRACE) <<
"getNext: Calling getNext_ w/ ev_counter()=" << ev_counter();
214 result = getNext_(output);
220 TLOG(TLVL_TRACE) <<
"getNext: Done with getNext_ - ev_counter() now " << ev_counter();
221 for (
auto dataIter = output.begin(); dataIter != output.end(); ++dataIter)
223 TLOG(TLVL_GETNEXT_VERBOSE) <<
"getNext: getNext_() returned fragment with sequenceID = " << (*dataIter)->sequenceID()
224 <<
", type = " << (*dataIter)->typeString() <<
", id = " << std::to_string((*dataIter)->fragmentID())
225 <<
", timestamp = " << (*dataIter)->timestamp() <<
", and sizeBytes = " << (*dataIter)->sizeBytes();
229 catch (
const cet::exception& e)
231 latest_exception_report_ =
"cet::exception caught in getNext(): ";
232 latest_exception_report_.append(e.what());
233 TLOG(TLVL_ERROR) <<
"getNext: cet::exception caught: " << e;
237 catch (
const boost::exception& e)
239 latest_exception_report_ =
"boost::exception caught in getNext(): ";
240 latest_exception_report_.append(boost::diagnostic_information(e));
241 TLOG(TLVL_ERROR) <<
"getNext: boost::exception caught: " << boost::diagnostic_information(e);
245 catch (
const std::exception& e)
247 latest_exception_report_ =
"std::exception caught in getNext(): ";
248 latest_exception_report_.append(e.what());
249 TLOG(TLVL_ERROR) <<
"getNext: std::exception caught: " << e.what();
255 latest_exception_report_ =
"Unknown exception caught in getNext().";
256 TLOG(TLVL_ERROR) <<
"getNext: unknown exception caught";
263 TLOG(TLVL_DEBUG) <<
"getNext: Either getNext_ or applyRequests returned false, stopping";
266 if (metricMan && !output.empty())
268 auto timestamp = output.front()->timestamp();
270 if (output.size() > 1)
272 for (
auto& outputfrag : output)
274 if (outputfrag->timestamp() > timestamp)
276 timestamp = outputfrag->timestamp();
281 metricMan->sendMetric(
"Last Timestamp", timestamp,
"Ticks", 1, MetricMode::LastPoint);
289 TLOG(TLVL_CHECKSTOP) <<
"CFG::check_stop: should_stop=" << should_stop() <<
", useDataThread_=" << useDataThread_ <<
", exception status =" << int(exception());
291 if (!should_stop())
return false;
292 if (!useDataThread_ || mode_ == RequestMode::Ignored)
return true;
293 if (force_stop_)
return true;
296 TLOG(TLVL_DEBUG) <<
"should_stop is true, force_stop_ is false, requestReceiver_->isRunning() is " << std::boolalpha << requestReceiver_->isRunning();
297 return !requestReceiver_->isRunning();
302 if (force || mode_ == RequestMode::Ignored || mode_ == RequestMode::SequenceID)
304 TLOG(TLVL_EVCOUNTERINC) <<
"ev_counter_inc: Incrementing ev_counter from " << ev_counter() <<
" by " << step;
305 return ev_counter_.fetch_add(step);
307 return ev_counter_.load();
312 TLOG(TLVL_TRACE) <<
"Start Command received.";
313 if (run < 0)
throw cet::exception(
"CommandableFragmentGenerator") <<
"negative run number";
316 timestamp_ = timestamp;
317 ev_counter_.store(1);
319 std::unique_lock<std::mutex> lk(dataBuffersMutex_);
320 for (
auto&
id : dataBuffers_)
322 id.second.DataBufferDepthBytes = 0;
323 id.second.DataBufferDepthFragments = 0;
324 id.second.HighestRequestSeen = 0;
325 id.second.BufferFragmentKept =
false;
326 id.second.DataBuffer.clear();
327 id.second.WindowsSent.clear();
331 should_stop_.store(
false);
332 force_stop_.store(
false);
333 exception_.store(
false);
336 latest_exception_report_ =
"none";
340 std::unique_lock<std::mutex> lk(mutex_);
341 if (useDataThread_) startDataThread();
342 if (useMonitoringThread_) startMonitoringThread();
343 if (mode_ != RequestMode::Ignored)
345 requestReceiver_->SetRunNumber(static_cast<uint32_t>(run));
346 requestReceiver_->startRequestReception();
348 TLOG(TLVL_TRACE) <<
"Start Command complete.";
353 TLOG(TLVL_TRACE) <<
"Stop Command received.";
356 timestamp_ = timestamp;
357 if (requestReceiver_)
359 TLOG(TLVL_DEBUG) <<
"Stopping Request reception BEGIN";
360 requestReceiver_->stopRequestReception();
361 TLOG(TLVL_DEBUG) <<
"Stopping Request reception END";
365 should_stop_.store(
true);
366 std::unique_lock<std::mutex> lk(mutex_);
370 TLOG(TLVL_TRACE) <<
"Stop Command complete.";
375 TLOG(TLVL_TRACE) <<
"Pause Command received.";
377 timestamp_ = timestamp;
381 should_stop_.store(
true);
382 std::unique_lock<std::mutex> lk(mutex_);
389 TLOG(TLVL_TRACE) <<
"Resume Command received.";
391 timestamp_ = timestamp;
394 should_stop_ =
false;
397 std::unique_lock<std::mutex> lk(dataBuffersMutex_);
398 for (
auto&
id : dataBuffers_)
400 id.second.DataBufferDepthBytes = 0;
401 id.second.DataBufferDepthFragments = 0;
402 id.second.BufferFragmentKept =
false;
403 id.second.DataBuffer.clear();
409 std::unique_lock<std::mutex> lk(mutex_);
413 TLOG(TLVL_TRACE) <<
"Resume Command complete.";
418 TLOG(TLVL_TRACE) <<
"Report Command received.";
419 std::lock_guard<std::mutex> lk(mutex_);
426 std::string childReport = reportSpecific(which);
427 if (childReport.length() > 0) {
return childReport; }
430 if (which ==
"latest_exception")
432 return latest_exception_report_;
436 childReport = report();
437 if (childReport.length() > 0) {
return childReport; }
447 TLOG(TLVL_TRACE) <<
"Report Command complete.";
454 #pragma message "Using default implementation of CommandableFragmentGenerator::pauseNoMutex()"
459 #pragma message "Using default implementation of CommandableFragmentGenerator::pause()"
463 #pragma message "Using default implementation of CommandableFragmentGenerator::resume()"
468 #pragma message "Using default implementation of CommandableFragmentGenerator::report()"
474 #pragma message "Using default implementation of CommandableFragmentGenerator::reportSpecific(std::string)"
480 #pragma message "Using default implementation of CommandableFragmentGenerator::checkHWStatus_()"
486 #pragma message "Using default implementation of CommandableFragmentGenerator::metaCommand(std::string, std::string)"
492 if (dataThread_.joinable()) dataThread_.join();
493 TLOG(TLVL_INFO) <<
"Starting Data Receiver Thread";
498 catch (
const boost::exception& e)
500 TLOG(TLVL_ERROR) <<
"Caught boost::exception starting Data Receiver thread: " << boost::diagnostic_information(e) <<
", errno=" << errno;
501 std::cerr <<
"Caught boost::exception starting Data Receiver thread: " << boost::diagnostic_information(e) <<
", errno=" << errno << std::endl;
508 if (monitoringThread_.joinable()) monitoringThread_.join();
509 TLOG(TLVL_INFO) <<
"Starting Hardware Monitoring Thread";
514 catch (
const boost::exception& e)
516 TLOG(TLVL_ERROR) <<
"Caught boost::exception starting Hardware Monitoring thread: " << boost::diagnostic_information(e) <<
", errno=" << errno;
517 std::cerr <<
"Caught boost::exception starting Hardware Monitoring thread: " << boost::diagnostic_information(e) <<
", errno=" << errno << std::endl;
526 case RequestMode::Single:
528 case RequestMode::Buffer:
530 case RequestMode::Window:
532 case RequestMode::Ignored:
534 case RequestMode::SequenceID:
546 data_thread_running_ =
true;
551 TLOG(TLVL_DEBUG) <<
"getDataLoop: isHardwareOK is " << isHardwareOK_ <<
", aborting data thread";
552 data_thread_running_ =
false;
556 TLOG(TLVL_GETDATALOOP) <<
"getDataLoop: calling getNext_";
559 auto startdata = std::chrono::steady_clock::now();
560 FragmentPtrs newData;
564 data = getNext_(newData);
568 ExceptionHandler(ExceptionHandlerRethrow::no,
569 "Exception thrown by fragment generator in CommandableFragmentGenerator::getDataLoop; setting exception state to \"true\"");
572 data_thread_running_ =
false;
578 metricMan->sendMetric(
"Avg Data Acquisition Time", TimeUtils::GetElapsedTime(startdata),
"s", 3, artdaq::MetricMode::Average);
581 if (newData.size() == 0 && sleep_on_no_data_us_ > 0)
583 usleep(sleep_on_no_data_us_);
586 auto dataIter = newData.begin();
587 while (dataIter != newData.end())
589 TLOG(TLVL_GETDATALOOP_VERBOSE) <<
"getDataLoop: getNext_() returned fragment with timestamp = " << (*dataIter)->timestamp() <<
", and sizeBytes = " << (*dataIter)->sizeBytes();
591 auto frag_id = (*dataIter)->fragmentID();
592 if (!dataBuffers_.count(frag_id))
594 throw cet::exception(
"DataBufferError") <<
"Error in CommandableFragmentGenerator: Recevied Fragment with fragment_id " << frag_id <<
", but this ID was not declared in fragment_ids!";
597 TLOG(TLVL_GETDATALOOP_DATABUFFWAIT) <<
"Waiting for data buffer ready";
598 if (!waitForDataBufferReady(frag_id))
return;
599 TLOG(TLVL_GETDATALOOP_DATABUFFWAIT) <<
"Done waiting for data buffer ready";
601 TLOG(TLVL_GETDATALOOP) <<
"getDataLoop: processing data";
602 if (data && !force_stop_)
604 std::unique_lock<std::mutex> lock(dataBuffersMutex_);
607 case RequestMode::Single:
608 dataBuffers_[frag_id].DataBuffer.clear();
609 dataBuffers_[frag_id].DataBufferDepthBytes = (*dataIter)->sizeBytes();
610 dataBuffers_[frag_id].DataBuffer.emplace_back(std::move(*dataIter));
611 dataIter = newData.erase(dataIter);
613 case RequestMode::Buffer:
614 case RequestMode::Ignored:
615 case RequestMode::Window:
616 case RequestMode::SequenceID:
619 dataBuffers_[frag_id].DataBufferDepthBytes += (*dataIter)->sizeBytes();
620 dataBuffers_[frag_id].DataBuffer.emplace_back(std::move(*dataIter));
621 dataIter = newData.erase(dataIter);
624 getDataBufferStats(frag_id);
633 std::unique_lock<std::mutex> lock(dataBuffersMutex_);
634 for (
auto&
id : dataBuffers_)
636 if (
id.second.DataBuffer.size() > 0)
638 dataCondition_.notify_all();
643 if (!data || force_stop_)
645 TLOG(TLVL_INFO) <<
"Data flow has stopped. Ending data collection thread";
646 std::unique_lock<std::mutex> lock(dataBuffersMutex_);
647 data_thread_running_ =
false;
648 if (requestReceiver_) requestReceiver_->ClearRequests();
650 TLOG(TLVL_INFO) <<
"getDataLoop: Ending thread";
658 if (!dataBuffers_.count(
id))
660 throw cet::exception(
"DataBufferError") <<
"Error in CommandableFragmentGenerator: Cannot wait for data buffer for ID " <<
id <<
" because it does not exist!";
662 auto startwait = std::chrono::steady_clock::now();
664 auto lastwaittime = 0ULL;
667 std::unique_lock<std::mutex> lock(dataBuffersMutex_);
668 getDataBufferStats(
id);
671 while (dataBufferIsTooLarge(
id))
673 if (!circularDataBufferMode_)
677 TLOG(TLVL_DEBUG) <<
"Run ended while waiting for buffer to shrink!";
678 std::unique_lock<std::mutex> lock(dataBuffersMutex_);
679 getDataBufferStats(
id);
680 dataCondition_.notify_all();
681 data_thread_running_ =
false;
684 auto waittime = TimeUtils::GetElapsedTimeMilliseconds(startwait);
686 if (first || (waittime != lastwaittime && waittime % 1000 == 0))
688 TLOG(TLVL_WARNING) <<
"Bad Omen: Data Buffer has exceeded its size limits. "
689 <<
"(seq_id=" << ev_counter() <<
", frag_id=" <<
id
690 <<
", frags=" << dataBuffers_[id].DataBufferDepthFragments <<
"/" << maxDataBufferDepthFragments_
691 <<
", szB=" << dataBuffers_[id].DataBufferDepthBytes <<
"/" << maxDataBufferDepthBytes_ <<
")";
692 TLOG(TLVL_TRACE) <<
"Bad Omen: Possible causes include requests not getting through or Ignored-mode BR issues";
695 if (waittime % 5 && waittime != lastwaittime)
697 TLOG(TLVL_WAITFORBUFFERREADY) <<
"getDataLoop: Data Retreival paused for " << waittime <<
" ms waiting for data buffer to drain";
699 lastwaittime = waittime;
704 std::unique_lock<std::mutex> lock(dataBuffersMutex_);
705 getDataBufferStats(
id);
706 if (dataBufferIsTooLarge(
id))
708 auto begin = dataBuffers_[id].DataBuffer.begin();
709 if (begin == dataBuffers_[
id].DataBuffer.end())
711 TLOG(TLVL_WARNING) <<
"Data buffer is reported as too large, but doesn't contain any Fragments! Possible corrupt memory!";
716 TLOG(TLVL_WAITFORBUFFERREADY) <<
"waitForDataBufferReady: Dropping Fragment with timestamp " << (*begin)->timestamp() <<
" from data buffer (Buffer over-size, circular data buffer mode)";
718 dataBuffers_[id].DataBufferDepthBytes -= (*begin)->sizeBytes();
719 dataBuffers_[id].DataBuffer.erase(begin);
720 dataBuffers_[id].BufferFragmentKept =
false;
721 getDataBufferStats(
id);
730 if (!dataBuffers_.count(
id))
732 throw cet::exception(
"DataBufferError") <<
"Error in CommandableFragmentGenerator: Cannot check size of data buffer for ID " <<
id <<
" because it does not exist!";
734 return (maxDataBufferDepthFragments_ > 0 && dataBuffers_[
id].DataBufferDepthFragments > maxDataBufferDepthFragments_) || (maxDataBufferDepthBytes_ > 0 && dataBuffers_[id].DataBufferDepthBytes > maxDataBufferDepthBytes_);
739 if (!dataBuffers_.count(
id))
741 throw cet::exception(
"DataBufferError") <<
"Error in CommandableFragmentGenerator: Cannot get stats of data buffer for ID " <<
id <<
" because it does not exist!";
744 dataBuffers_[id].DataBufferDepthFragments = dataBuffers_[id].DataBuffer.size();
748 TLOG(TLVL_GETBUFFERSTATS) <<
"getDataBufferStats: Sending Metrics";
749 metricMan->sendMetric(
"Buffer Depth Fragments", dataBuffers_[
id].DataBufferDepthFragments.load(),
"fragments", 1, MetricMode::LastPoint);
750 metricMan->sendMetric(
"Buffer Depth Bytes", dataBuffers_[
id].DataBufferDepthBytes.load(),
"bytes", 1, MetricMode::LastPoint);
752 auto bufferDepthFragmentsPercent = dataBuffers_[id].DataBufferDepthFragments.load() * 100 /
static_cast<double>(maxDataBufferDepthFragments_);
753 auto bufferDepthBytesPercent = dataBuffers_[id].DataBufferDepthBytes.load() * 100 /
static_cast<double>(maxDataBufferDepthBytes_);
754 metricMan->sendMetric(
"Fragment Generator Buffer Full %Fragments", bufferDepthFragmentsPercent,
"%", 3, MetricMode::LastPoint);
755 metricMan->sendMetric(
"Fragment Generator Buffer Full %Bytes", bufferDepthBytesPercent,
"%", 3, MetricMode::LastPoint);
756 metricMan->sendMetric(
"Fragment Generator Buffer Full %", bufferDepthFragmentsPercent > bufferDepthBytesPercent ? bufferDepthFragmentsPercent : bufferDepthBytesPercent,
"%", 1, MetricMode::LastPoint);
758 TLOG(TLVL_GETBUFFERSTATS) <<
"getDataBufferStats: frags=" << dataBuffers_[id].DataBufferDepthFragments.load() <<
"/" << maxDataBufferDepthFragments_
759 <<
", sz=" << dataBuffers_[id].DataBufferDepthBytes.load() <<
"/" << maxDataBufferDepthBytes_;
764 if (!dataBuffers_.count(
id))
766 throw cet::exception(
"DataBufferError") <<
"Error in CommandableFragmentGenerator: Cannot check data buffer for ID " <<
id <<
" because it does not exist!";
769 if (dataBuffers_[
id].DataBufferDepthFragments > 0 && mode_ != RequestMode::Single && mode_ != RequestMode::Ignored)
772 getDataBufferStats(
id);
773 while (dataBufferIsTooLarge(
id))
775 auto begin = dataBuffers_[id].DataBuffer.begin();
776 TLOG(TLVL_CHECKDATABUFFER) <<
"checkDataBuffer: Dropping Fragment with timestamp " << (*begin)->timestamp() <<
" from data buffer (Buffer over-size)";
777 dataBuffers_[id].DataBufferDepthBytes -= (*begin)->sizeBytes();
778 dataBuffers_[id].DataBuffer.erase(begin);
779 dataBuffers_[id].BufferFragmentKept =
false;
780 getDataBufferStats(
id);
782 if (dataBuffers_[
id].DataBuffer.size() > 0)
784 TLOG(TLVL_CHECKDATABUFFER) <<
"Determining if Fragments can be dropped from data buffer";
785 Fragment::timestamp_t last = dataBuffers_[id].DataBuffer.back()->timestamp();
786 Fragment::timestamp_t min = last > staleTimeout_ ? last - staleTimeout_ : 0;
787 for (
auto it = dataBuffers_[
id].DataBuffer.begin(); it != dataBuffers_[id].DataBuffer.end();)
789 if ((*it)->timestamp() < min)
791 TLOG(TLVL_CHECKDATABUFFER) <<
"checkDataBuffer: Dropping Fragment with timestamp " << (*it)->timestamp() <<
" from data buffer (timeout=" << staleTimeout_ <<
", min=" << min <<
")";
792 dataBuffers_[id].DataBufferDepthBytes -= (*it)->sizeBytes();
793 dataBuffers_[id].BufferFragmentKept =
false;
794 it = dataBuffers_[id].DataBuffer.erase(it);
801 getDataBufferStats(
id);
810 if (should_stop() || monitoringInterval_ <= 0)
812 TLOG(TLVL_DEBUG) <<
"getMonitoringDataLoop: should_stop() is " << std::boolalpha << should_stop()
813 <<
" and monitoringInterval is " << monitoringInterval_ <<
", returning";
816 TLOG(TLVL_GETMONITORINGDATA) <<
"getMonitoringDataLoop: Determining whether to call checkHWStatus_";
818 auto now = std::chrono::steady_clock::now();
819 if (TimeUtils::GetElapsedTimeMicroseconds(lastMonitoringCall_, now) >= static_cast<size_t>(monitoringInterval_))
821 isHardwareOK_ = checkHWStatus_();
822 TLOG(TLVL_GETMONITORINGDATA) <<
"getMonitoringDataLoop: isHardwareOK_ is now " << std::boolalpha << isHardwareOK_;
823 lastMonitoringCall_ = now;
825 usleep(monitoringInterval_ / 10);
833 TLOG(TLVL_APPLYREQUESTS) <<
"Mode is Ignored; Copying data to output";
834 for (
auto&
id : dataBuffers_)
836 std::move(
id.second.DataBuffer.begin(),
id.second.DataBuffer.end(), std::inserter(frags, frags.end()));
837 id.second.DataBufferDepthBytes = 0;
838 id.second.DataBufferDepthFragments = 0;
839 id.second.BufferFragmentKept =
false;
840 id.second.DataBuffer.clear();
847 auto requests = requestReceiver_->GetRequests();
848 while (requests.size() > 1)
851 requestReceiver_->RemoveRequest(requests.begin()->first);
852 requests.erase(requests.begin());
854 sendEmptyFragments(frags, requests);
857 if (requests.size() == 0 || !requests.count(ev_counter()))
return;
859 for (
auto&
id : dataBuffers_)
861 if (
id.second.DataBuffer.size() > 0)
863 assert(
id.second.DataBuffer.size() == 1);
864 TLOG(TLVL_APPLYREQUESTS) <<
"Mode is Single; Sending copy of last event";
865 for (
auto& fragptr :
id.second.DataBuffer)
868 auto frag = fragptr.get();
869 auto newfrag = std::unique_ptr<artdaq::Fragment>(
new Fragment(ev_counter(), frag->fragmentID()));
870 newfrag->resize(frag->size() - detail::RawFragmentHeader::num_words());
871 memcpy(newfrag->headerAddress(), frag->headerAddress(), frag->sizeBytes());
872 newfrag->setTimestamp(requests[ev_counter()]);
873 newfrag->setSequenceID(ev_counter());
874 frags.push_back(std::move(newfrag));
879 sendEmptyFragment(frags, ev_counter(),
id.first,
"No data for");
882 requestReceiver_->RemoveRequest(ev_counter());
883 ev_counter_inc(1,
true);
889 auto requests = requestReceiver_->GetRequests();
890 while (requests.size() > 1)
893 requestReceiver_->RemoveRequest(requests.begin()->first);
894 requests.erase(requests.begin());
896 sendEmptyFragments(frags, requests);
899 if (requests.size() == 0 || !requests.count(ev_counter()))
return;
901 for (
auto&
id : dataBuffers_)
903 TLOG(TLVL_DEBUG) <<
"Creating ContainerFragment for Buffered Fragments";
904 frags.emplace_back(
new artdaq::Fragment(ev_counter(),
id.first));
905 frags.back()->setTimestamp(requests[ev_counter()]);
906 ContainerFragmentLoader cfl(*frags.back());
907 cfl.set_missing_data(
false);
910 auto it =
id.second.DataBuffer.begin();
911 if (
id.second.BufferFragmentKept &&
id.second.DataBuffer.size() > 1)
913 id.second.DataBufferDepthBytes -= (*it)->sizeBytes();
914 it =
id.second.DataBuffer.erase(it);
918 while (it !=
id.second.DataBuffer.end())
920 TLOG(TLVL_APPLYREQUESTS) <<
"ApplyRequests: Adding Fragment with timestamp " << (*it)->timestamp() <<
" to Container with sequence ID " << ev_counter();
921 cfl.addFragment(*it);
922 if (bufferModeKeepLatest_ &&
id.second.DataBuffer.size() == 1)
924 id.second.BufferFragmentKept =
true;
927 id.second.DataBufferDepthBytes -= (*it)->sizeBytes();
928 it =
id.second.DataBuffer.erase(it);
931 requestReceiver_->RemoveRequest(ev_counter());
932 ev_counter_inc(1,
true);
937 TLOG(TLVL_APPLYREQUESTS) <<
"applyRequestsWindowMode_CheckAndFillDataBuffer: Checking that data exists for request window " << seq;
938 Fragment::timestamp_t min = ts > windowOffset_ ? ts - windowOffset_ : 0;
939 Fragment::timestamp_t max = min + windowWidth_;
940 TLOG(TLVL_APPLYREQUESTS) <<
"ApplyRequestsWindowsMode_CheckAndFillDataBuffer: min is " << min <<
", max is " << max
941 <<
" and first/last points in buffer are " << (dataBuffers_[id].DataBuffer.size() > 0 ? dataBuffers_[id].DataBuffer.front()->timestamp() : 0)
942 <<
"/" << (dataBuffers_[
id].DataBuffer.size() > 0 ? dataBuffers_[id].DataBuffer.back()->timestamp() : 0)
943 <<
" (sz=" << dataBuffers_[
id].DataBuffer.size() <<
" [" << dataBuffers_[id].DataBufferDepthBytes.load()
944 <<
"/" << maxDataBufferDepthBytes_ <<
"])";
945 bool windowClosed = dataBuffers_[id].DataBuffer.size() > 0 && dataBuffers_[id].DataBuffer.back()->timestamp() >= max;
946 bool windowTimeout = !windowClosed && TimeUtils::GetElapsedTimeMicroseconds(requestReceiver_->GetRequestTime(seq)) > window_close_timeout_us_;
949 TLOG(TLVL_WARNING) <<
"applyRequestsWindowMode_CheckAndFillDataBuffer: A timeout occurred waiting for data to close the request window ({" << min <<
"-" << max
950 <<
"}, buffer={" << (dataBuffers_[id].DataBuffer.size() > 0 ? dataBuffers_[id].DataBuffer.front()->timestamp() : 0) <<
"-"
951 << (dataBuffers_[
id].DataBuffer.size() > 0 ? dataBuffers_[id].DataBuffer.back()->timestamp() : 0)
952 <<
"} ). Time waiting: "
953 << TimeUtils::GetElapsedTimeMicroseconds(requestReceiver_->GetRequestTime(seq)) <<
" us "
954 <<
"(> " << window_close_timeout_us_ <<
" us).";
956 if (windowClosed || !data_thread_running_ || windowTimeout)
958 TLOG(TLVL_DEBUG) <<
"applyRequestsWindowMode_CheckAndFillDataBuffer: Creating ContainerFragment for Window-requested Fragments";
959 frags.emplace_back(
new artdaq::Fragment(seq,
id));
960 frags.back()->setTimestamp(ts);
961 ContainerFragmentLoader cfl(*frags.back());
975 if (!windowClosed || (dataBuffers_[
id].DataBuffer.size() > 0 && dataBuffers_[id].DataBuffer.front()->timestamp() > min))
977 TLOG(TLVL_DEBUG) <<
"applyRequestsWindowMode_CheckAndFillDataBuffer: Request window starts before and/or ends after the current data buffer, setting ContainerFragment's missing_data flag!"
978 <<
" (requestWindowRange=[" << min <<
"," << max <<
"], "
979 <<
"buffer={" << (dataBuffers_[id].DataBuffer.size() > 0 ? dataBuffers_[id].DataBuffer.front()->timestamp() : 0) <<
"-"
980 << (dataBuffers_[
id].DataBuffer.size() > 0 ? dataBuffers_[id].DataBuffer.back()->timestamp() : 0) <<
"}";
981 cfl.set_missing_data(
true);
985 for (
auto it = dataBuffers_[
id].DataBuffer.begin(); it != dataBuffers_[id].DataBuffer.end();)
987 Fragment::timestamp_t fragT = (*it)->timestamp();
988 if (fragT < min || fragT > max || (fragT == max && windowWidth_ > 0))
994 TLOG(TLVL_APPLYREQUESTS) <<
"applyRequestsWindowMode_CheckAndFillDataBuffer: Adding Fragment with timestamp " << (*it)->timestamp() <<
" to Container";
995 cfl.addFragment(*it);
999 dataBuffers_[id].DataBufferDepthBytes -= (*it)->sizeBytes();
1000 it = dataBuffers_[id].DataBuffer.erase(it);
1008 dataBuffers_[id].WindowsSent[seq] = std::chrono::steady_clock::now();
1009 if (seq > dataBuffers_[
id].HighestRequestSeen) dataBuffers_[id].HighestRequestSeen = seq;
1015 TLOG(TLVL_APPLYREQUESTS) <<
"applyRequestsWindowMode BEGIN";
1017 auto requests = requestReceiver_->GetRequests();
1019 TLOG(TLVL_APPLYREQUESTS) <<
"applyRequestsWindowMode: Starting request processing";
1020 for (
auto req = requests.begin(); req != requests.end();)
1022 TLOG(TLVL_APPLYREQUESTS) <<
"applyRequestsWindowMode: processing request with sequence ID " << req->first <<
", timestamp " << req->second;
1024 while (req->first < ev_counter() && requests.size() > 0)
1026 TLOG(TLVL_APPLYREQUESTS) <<
"applyRequestsWindowMode: Clearing passed request for sequence ID " << req->first;
1027 requestReceiver_->RemoveRequest(req->first);
1028 req = requests.erase(req);
1030 if (requests.size() == 0)
break;
1032 for (
auto&
id : dataBuffers_)
1034 if (!
id.second.WindowsSent.count(req->first))
1036 applyRequestsWindowMode_CheckAndFillDataBuffer(frags,
id.first, req->first, req->second);
1039 checkSentWindows(req->first);
1044 for (
auto&
id : dataBuffers_)
1046 std::set<artdaq::Fragment::sequence_id_t> seqs;
1047 for (
auto& seq :
id.second.WindowsSent)
1049 seqs.insert(seq.first);
1051 for (
auto& seq : seqs)
1053 checkSentWindows(seq);
1060 TLOG(TLVL_APPLYREQUESTS) <<
"applyRequestsSequenceIDMode BEGIN";
1062 auto requests = requestReceiver_->GetRequests();
1064 TLOG(TLVL_APPLYREQUESTS) <<
"applyRequestsSequenceIDMode: Starting request processing";
1065 for (
auto req = requests.begin(); req != requests.end();)
1067 TLOG(TLVL_APPLYREQUESTS) <<
"applyRequestsSequenceIDMode: Checking that data exists for request SequenceID " << req->first;
1069 for (
auto&
id : dataBuffers_)
1071 if (!
id.second.WindowsSent.count(req->first))
1073 TLOG(29) <<
"Searching id " <<
id.first <<
" for Fragments with Sequence ID " << req->first;
1074 for (
auto it =
id.second.DataBuffer.begin(); it !=
id.second.DataBuffer.end();)
1076 auto seq = (*it)->sequenceID();
1077 TLOG(29) <<
"applyRequestsSequenceIDMode: Fragment SeqID " << seq <<
", request ID " << req->first;
1078 if (seq == req->first)
1080 TLOG(29) <<
"applyRequestsSequenceIDMode: Adding Fragment to output";
1081 id.second.WindowsSent[req->first] = std::chrono::steady_clock::now();
1082 id.second.DataBufferDepthBytes -= (*it)->sizeBytes();
1083 frags.push_back(std::move(*it));
1084 it =
id.second.DataBuffer.erase(it);
1092 if (req->first >
id.second.HighestRequestSeen)
id.second.HighestRequestSeen = req->first;
1094 checkSentWindows(req->first);
1099 for (
auto&
id : dataBuffers_)
1101 std::set<artdaq::Fragment::sequence_id_t> seqs;
1102 for (
auto& seq :
id.second.WindowsSent)
1104 seqs.insert(seq.first);
1106 for (
auto& seq : seqs)
1108 checkSentWindows(seq);
1115 if (check_stop() || exception())
1121 if (mode_ == RequestMode::Ignored)
1123 while (dataBufferFragmentCount_() == 0)
1125 if (check_stop() || exception() || !isHardwareOK_)
return false;
1126 std::unique_lock<std::mutex> lock(dataBuffersMutex_);
1127 dataCondition_.wait_for(lock, std::chrono::milliseconds(10), [
this]() {
return dataBufferFragmentCount_() > 0; });
1132 if ((check_stop() && requestReceiver_->size() == 0) || exception())
return false;
1134 std::unique_lock<std::mutex> lock(dataBuffersMutex_);
1135 dataCondition_.wait_for(lock, std::chrono::milliseconds(10));
1142 while (requestReceiver_->size() == 0 && counter < 100)
1144 if (check_stop() || exception())
return false;
1148 requestReceiver_->WaitForRequests(10);
1154 std::unique_lock<std::mutex> dlk(dataBuffersMutex_);
1158 case RequestMode::Single:
1159 applyRequestsSingleMode(frags);
1161 case RequestMode::Window:
1162 applyRequestsWindowMode(frags);
1164 case RequestMode::Buffer:
1165 applyRequestsBufferMode(frags);
1167 case RequestMode::SequenceID:
1168 applyRequestsSequenceIDMode(frags);
1170 case RequestMode::Ignored:
1172 applyRequestsIgnoredMode(frags);
1176 if (!data_thread_running_ || force_stop_)
1178 TLOG(TLVL_INFO) <<
"Data thread has stopped; Clearing data buffers";
1179 for (
auto&
id : dataBuffers_)
1181 id.second.DataBufferDepthBytes = 0;
1182 id.second.DataBufferDepthFragments = 0;
1183 id.second.BufferFragmentKept =
false;
1184 id.second.DataBuffer.clear();
1188 getDataBuffersStats();
1191 if (frags.size() > 0)
1192 TLOG(TLVL_APPLYREQUESTS) <<
"Finished Processing requests, returning " << frags.size() <<
" fragments, current ev_counter is " << ev_counter();
1198 TLOG(TLVL_WARNING) << desc <<
" sequence ID " << seqId <<
", sending empty fragment";
1199 auto frag =
new Fragment();
1200 frag->setSequenceID(seqId);
1201 frag->setFragmentID(fragmentId);
1202 frag->setSystemType(Fragment::EmptyFragmentType);
1203 frags.emplace_back(FragmentPtr(frag));
1209 if (requests.size() > 0)
1211 TLOG(TLVL_SENDEMPTYFRAGMENTS) <<
"Sending Empty Fragments for Sequence IDs from " << ev_counter() <<
" up to but not including " << requests.begin()->first;
1212 while (requests.begin()->first > ev_counter())
1214 for (
auto& fid : dataBuffers_)
1216 sendEmptyFragment(frags, ev_counter(), fid.first,
"Missed request for");
1218 ev_counter_inc(1,
true);
1225 TLOG(TLVL_CHECKWINDOWS) <<
"checkSentWindows: Checking if request " << seq <<
" can be removed from request list";
1226 bool seqComplete =
true;
1227 bool seqTimeout =
false;
1228 for (
auto&
id : dataBuffers_)
1230 if (!
id.second.WindowsSent.count(seq) ||
id.second.HighestRequestSeen < seq)
1232 seqComplete =
false;
1234 if (
id.second.WindowsSent.count(seq) && TimeUtils::GetElapsedTimeMicroseconds(
id.second.WindowsSent[seq]) > missing_request_window_timeout_us_)
1241 TLOG(TLVL_CHECKWINDOWS) <<
"checkSentWindows: Request " << seq <<
" is complete, removing from requestReceiver.";
1242 requestReceiver_->RemoveRequest(seq);
1244 if (ev_counter() == seq)
1246 TLOG(TLVL_CHECKWINDOWS) <<
"checkSentWindows: Sequence ID matches ev_counter, incrementing ev_counter (" << ev_counter() <<
")";
1248 for (
auto&
id : dataBuffers_)
1250 id.second.WindowsSent.erase(seq);
1253 ev_counter_inc(1,
true);
1258 TLOG(TLVL_CHECKWINDOWS) <<
"checkSentWindows: Sent Window history indicates that requests between " << ev_counter() <<
" and " << seq <<
" have timed out.";
1259 while (ev_counter() <= seq)
1261 if (ev_counter() < seq) TLOG(TLVL_WARNING) <<
"Missed request for sequence ID " << ev_counter() <<
"! Will not send any data for this sequence ID!";
1262 requestReceiver_->RemoveRequest(ev_counter());
1264 for (
auto&
id : dataBuffers_)
1266 id.second.WindowsSent.erase(ev_counter());
1269 ev_counter_inc(1,
true);
CommandableFragmentGenerator(const fhicl::ParameterSet &ps)
CommandableFragmentGenerator Constructor.
void applyRequestsSingleMode(artdaq::FragmentPtrs &frags)
Create fragments using data buffer for request mode Single. Precondition: dataBufferMutex_ and reques...
virtual bool checkHWStatus_()
Check any relavent hardware status registers. Return false if an error condition exists that should h...
virtual ~CommandableFragmentGenerator()
CommandableFragmentGenerator Destructor.
void applyRequestsBufferMode(artdaq::FragmentPtrs &frags)
Create fragments using data buffer for request mode Buffer. Precondition: dataBufferMutex_ and reques...
void getDataBufferStats(Fragment::fragment_id_t id)
Calculate the size of the dataBuffer and report appropriate metrics.
void getMonitoringDataLoop()
This function regularly calls checkHWStatus_(), and sets the isHardwareOK flag accordingly.
void startDataThread()
Function that launches the data thread (getDataLoop())
artdaq::Fragment::fragment_id_t fragment_id() const
Get the Fragment ID of this Fragment generator.
std::string ReportCmd(std::string const &which="")
Get a report about a user-specified run-time quantity.
virtual bool metaCommand(std::string const &command, std::string const &arg)
The meta-command is used for implementing user-specific commands in a CommandableFragmentGenerator.
void StopCmd(uint64_t timeout, uint64_t timestamp)
Stop the CommandableFragmentGenerator.
void applyRequestsWindowMode_CheckAndFillDataBuffer(artdaq::FragmentPtrs &frags, artdaq::Fragment::fragment_id_t id, artdaq::Fragment::sequence_id_t seq, artdaq::Fragment::timestamp_t ts)
void checkSentWindows(Fragment::sequence_id_t seq)
Check the windows_sent_ooo_ map for sequence IDs that may be removed.
bool sendEmptyFragment(FragmentPtrs &frags, size_t sequenceId, Fragment::fragment_id_t fragmentId, std::string desc)
Send an EmptyFragmentType Fragment.
void applyRequestsWindowMode(artdaq::FragmentPtrs &frags)
Create fragments using data buffer for request mode Window. Precondition: dataBufferMutex_ and reques...
void StartCmd(int run, uint64_t timeout, uint64_t timestamp)
Start the CommandableFragmentGenerator.
virtual void pauseNoMutex()
On call to PauseCmd, pauseNoMutex() is called prior to PauseCmd acquiring the mutex ...
bool check_stop()
Routine used by applyRequests to make sure that all outstanding requests have been fulfilled before r...
void ResumeCmd(uint64_t timeout, uint64_t timestamp)
Resume the CommandableFragmentGenerator.
bool getNext(FragmentPtrs &output) overridefinal
getNext calls either applyRequests or getNext_ to get any data that is ready to be sent to the EventB...
size_t ev_counter_inc(size_t step=1, bool force=false)
Increment the event counter, if the current RequestMode allows it.
bool applyRequests(FragmentPtrs &frags)
See if any requests have been received, and add the corresponding data Fragment objects to the output...
void applyRequestsIgnoredMode(artdaq::FragmentPtrs &frags)
Create fragments using data buffer for request mode Ignored. Precondition: dataBufferMutex_ and reque...
void PauseCmd(uint64_t timeout, uint64_t timestamp)
Pause the CommandableFragmentGenerator.
void getDataLoop()
When separate_data_thread is set to true, this loop repeatedly calls getNext_ and adds returned Fragm...
void sendEmptyFragments(FragmentPtrs &frags, std::map< Fragment::sequence_id_t, Fragment::timestamp_t > &requests)
This function is for Buffered and Single request modes, as they can only respond to one data request ...
Receive data requests and make them available to CommandableFragmentGenerator or other interested par...
void startMonitoringThread()
Function that launches the monitoring thread (getMonitoringDataLoop())
virtual void pause()
If a CommandableFragmentGenerator subclass is reading from hardware, the implementation of pause() sh...
virtual void resume()
The subrun number will be incremented before a call to resume.
void applyRequestsSequenceIDMode(artdaq::FragmentPtrs &frags)
Create fragments using data buffer for request mode SequenceID. Precondition: dataBufferMutex_ and re...
bool waitForDataBufferReady(Fragment::fragment_id_t id)
Wait for the data buffer to drain (dataBufferIsTooLarge returns false), periodically reporting status...
virtual std::string report()
Let's say that the contract with the report() functions is that they return a non-empty string if the...
std::string printMode_()
Return the string representation of the current RequestMode.
void checkDataBuffer(Fragment::fragment_id_t id)
Perform data buffer pruning operations for the given buffer. If the RequestMode is Single...
virtual std::string reportSpecific(std::string const &what)
Report the status of a specific quantity
bool dataBufferIsTooLarge(Fragment::fragment_id_t id)
Test the configured constraints on the data buffer.
void joinThreads()
Join any data-taking threads. Should be called when destructing CommandableFragmentGenerator.