1 #include "artdaq/DAQdata/Globals.hh"
2 #define TRACE_NAME (app_name + "_CommandableFragmentGenerator").c_str() // include these 2 first -
4 #include "artdaq/Generators/CommandableFragmentGenerator.hh"
6 #include <boost/exception/all.hpp>
7 #include <boost/throw_exception.hpp>
12 #include "canvas/Utilities/Exception.h"
13 #include "cetlib_except/exception.h"
14 #include "fhiclcpp/ParameterSet.h"
16 #include "artdaq-core/Data/ContainerFragmentLoader.hh"
17 #include "artdaq-core/Data/Fragment.hh"
18 #include "artdaq-core/Utilities/ExceptionHandler.hh"
19 #include "artdaq-core/Utilities/SimpleLookupPolicy.hh"
20 #include "artdaq-core/Utilities/TimeUtils.hh"
30 #define TLVL_GETNEXT 10
31 #define TLVL_GETNEXT_VERBOSE 20
32 #define TLVL_CHECKSTOP 11
33 #define TLVL_EVCOUNTERINC 12
34 #define TLVL_GETDATALOOP 13
35 #define TLVL_GETDATALOOP_DATABUFFWAIT 21
36 #define TLVL_GETDATALOOP_VERBOSE 20
37 #define TLVL_WAITFORBUFFERREADY 15
38 #define TLVL_GETBUFFERSTATS 16
39 #define TLVL_CHECKDATABUFFER 17
40 #define TLVL_GETMONITORINGDATA 18
41 #define TLVL_APPLYREQUESTS 9
42 #define TLVL_SENDEMPTYFRAGMENTS 19
43 #define TLVL_CHECKWINDOWS 14
47 , requestReceiver_(nullptr)
48 , bufferModeKeepLatest_(ps.get<bool>(
"buffer_mode_keep_latest", false))
49 , windowOffset_(ps.get<Fragment::timestamp_t>(
"request_window_offset", 0))
50 , windowWidth_(ps.get<Fragment::timestamp_t>(
"request_window_width", 0))
51 , staleTimeout_(ps.get<Fragment::timestamp_t>(
"stale_request_timeout", 0xFFFFFFFF))
52 , expectedType_(ps.get<Fragment::type_t>(
"expected_fragment_type", Fragment::type_t(Fragment::EmptyFragmentType)))
53 , uniqueWindows_(ps.get<bool>(
"request_windows_are_unique", true))
54 , missing_request_window_timeout_us_(ps.get<size_t>(
"missing_request_window_timeout_us", 5000000))
55 , window_close_timeout_us_(ps.get<size_t>(
"window_close_timeout_us", 2000000))
56 , useDataThread_(ps.get<bool>(
"separate_data_thread", false))
57 , circularDataBufferMode_(ps.get<bool>(
"circular_buffer_mode", false))
58 , sleep_on_no_data_us_(ps.get<size_t>(
"sleep_on_no_data_us", 0))
59 , data_thread_running_(false)
60 , maxDataBufferDepthFragments_(ps.get<int>(
"data_buffer_depth_fragments", 1000))
61 , maxDataBufferDepthBytes_(ps.get<size_t>(
"data_buffer_depth_mb", 1000) * 1024 * 1024)
62 , useMonitoringThread_(ps.get<bool>(
"separate_monitoring_thread", false))
63 , monitoringInterval_(ps.get<int64_t>(
"hardware_poll_interval_us", 0))
64 , lastMonitoringCall_()
68 , timeout_(std::numeric_limits<uint64_t>::max())
69 , timestamp_(std::numeric_limits<uint64_t>::max())
73 , latest_exception_report_(
"none")
76 , sleep_on_stop_us_(0)
78 board_id_ = ps.get<
int>(
"board_id");
79 instance_name_for_metrics_ =
"BoardReader." + boost::lexical_cast<std::string>(board_id_);
81 auto fragment_ids = ps.get<std::vector<artdaq::Fragment::fragment_id_t>>(
"fragment_ids", std::vector<artdaq::Fragment::fragment_id_t>());
83 TLOG(TLVL_TRACE) <<
"artdaq::CommandableFragmentGenerator::CommandableFragmentGenerator(ps)";
86 if (fragment_id != -99)
88 if (fragment_ids.size() != 0)
90 latest_exception_report_ =
"Error in CommandableFragmentGenerator: can't both define \"fragment_id\" and \"fragment_ids\" in FHiCL document";
91 throw cet::exception(latest_exception_report_);
95 fragment_ids.emplace_back(fragment_id);
99 for (
auto&
id : fragment_ids)
101 dataBuffers_[id].DataBufferDepthBytes = 0;
102 dataBuffers_[id].DataBufferDepthFragments = 0;
103 dataBuffers_[id].HighestRequestSeen = 0;
104 dataBuffers_[id].BufferFragmentKept =
false;
105 dataBuffers_[id].DataBuffer.emplace_back(FragmentPtr(
new Fragment()));
106 (*dataBuffers_[id].DataBuffer.begin())->setSystemType(Fragment::EmptyFragmentType);
109 sleep_on_stop_us_ = ps.get<
int>(
"sleep_on_stop_us", 0);
111 std::string modeString = ps.get<std::string>(
"request_mode",
"ignored");
112 if (modeString ==
"single" || modeString ==
"Single")
114 mode_ = RequestMode::Single;
116 else if (modeString.find(
"buffer") != std::string::npos || modeString.find(
"Buffer") != std::string::npos)
118 mode_ = RequestMode::Buffer;
120 else if (modeString ==
"window" || modeString ==
"Window")
122 mode_ = RequestMode::Window;
124 else if (modeString.find(
"ignore") != std::string::npos || modeString.find(
"Ignore") != std::string::npos)
126 mode_ = RequestMode::Ignored;
128 TLOG(TLVL_DEBUG) <<
"Request mode is " <<
printMode_();
130 if (mode_ != RequestMode::Ignored)
134 latest_exception_report_ =
"Error in CommandableFragmentGenerator: use_data_thread must be true when request_mode is not \"Ignored\"!";
135 throw cet::exception(latest_exception_report_);
144 requestReceiver_.reset(
nullptr);
151 TLOG(TLVL_DEBUG) <<
"Joining dataThread";
152 if (dataThread_.joinable()) dataThread_.join();
153 TLOG(TLVL_DEBUG) <<
"Joining monitoringThread";
154 if (monitoringThread_.joinable()) monitoringThread_.join();
155 TLOG(TLVL_DEBUG) <<
"joinThreads complete";
162 if (check_stop()) usleep(sleep_on_stop_us_);
163 if (exception() || force_stop_)
return false;
165 if (!useMonitoringThread_ && monitoringInterval_ > 0)
167 TLOG(TLVL_GETNEXT) <<
"getNext: Checking whether to collect Monitoring Data";
168 auto now = std::chrono::steady_clock::now();
170 if (TimeUtils::GetElapsedTimeMicroseconds(lastMonitoringCall_, now) >= static_cast<size_t>(monitoringInterval_))
172 TLOG(TLVL_GETNEXT) <<
"getNext: Collecting Monitoring Data";
173 isHardwareOK_ = checkHWStatus_();
174 TLOG(TLVL_GETNEXT) <<
"getNext: isHardwareOK_ is now " << std::boolalpha << isHardwareOK_;
175 lastMonitoringCall_ = now;
181 std::lock_guard<std::mutex> lk(mutex_);
184 TLOG(TLVL_TRACE) <<
"getNext: Calling applyRequests";
185 result = applyRequests(output);
186 TLOG(TLVL_TRACE) <<
"getNext: Done with applyRequests result=" << std::boolalpha << result;
187 for (
auto dataIter = output.begin(); dataIter != output.end(); ++dataIter)
189 TLOG(20) <<
"getNext: applyRequests() returned fragment with sequenceID = " << (*dataIter)->sequenceID()
190 <<
", type = " << (*dataIter)->typeString() <<
", id = " << std::to_string((*dataIter)->fragmentID())
191 <<
", timestamp = " << (*dataIter)->timestamp() <<
", and sizeBytes = " << (*dataIter)->sizeBytes();
196 TLOG(TLVL_ERROR) <<
"Exception found in BoardReader with board ID " << board_id() <<
"; BoardReader will now return error status when queried";
197 throw cet::exception(
"CommandableFragmentGenerator") <<
"Exception found in BoardReader with board ID " << board_id() <<
"; BoardReader will now return error status when queried";
204 TLOG(TLVL_ERROR) <<
"Stopping CFG because the hardware reports bad status!";
207 TLOG(TLVL_TRACE) <<
"getNext: Calling getNext_ w/ ev_counter()=" << ev_counter();
210 result = getNext_(output);
216 TLOG(TLVL_TRACE) <<
"getNext: Done with getNext_ - ev_counter() now " << ev_counter();
217 for (
auto dataIter = output.begin(); dataIter != output.end(); ++dataIter)
219 TLOG(TLVL_GETNEXT_VERBOSE) <<
"getNext: getNext_() returned fragment with sequenceID = " << (*dataIter)->sequenceID()
220 <<
", type = " << (*dataIter)->typeString() <<
", id = " << std::to_string((*dataIter)->fragmentID())
221 <<
", timestamp = " << (*dataIter)->timestamp() <<
", and sizeBytes = " << (*dataIter)->sizeBytes();
225 catch (
const cet::exception& e)
227 latest_exception_report_ =
"cet::exception caught in getNext(): ";
228 latest_exception_report_.append(e.what());
229 TLOG(TLVL_ERROR) <<
"getNext: cet::exception caught: " << e;
233 catch (
const boost::exception& e)
235 latest_exception_report_ =
"boost::exception caught in getNext(): ";
236 latest_exception_report_.append(boost::diagnostic_information(e));
237 TLOG(TLVL_ERROR) <<
"getNext: boost::exception caught: " << boost::diagnostic_information(e);
241 catch (
const std::exception& e)
243 latest_exception_report_ =
"std::exception caught in getNext(): ";
244 latest_exception_report_.append(e.what());
245 TLOG(TLVL_ERROR) <<
"getNext: std::exception caught: " << e.what();
251 latest_exception_report_ =
"Unknown exception caught in getNext().";
252 TLOG(TLVL_ERROR) <<
"getNext: unknown exception caught";
259 TLOG(TLVL_DEBUG) <<
"getNext: Either getNext_ or applyRequests returned false, stopping";
262 if (metricMan && !output.empty())
264 auto timestamp = output.front()->timestamp();
266 if (output.size() > 1)
268 for (
auto& outputfrag : output)
270 if (outputfrag->timestamp() > timestamp)
272 timestamp = outputfrag->timestamp();
277 metricMan->sendMetric(
"Last Timestamp", timestamp,
"Ticks", 1, MetricMode::LastPoint);
285 TLOG(TLVL_CHECKSTOP) <<
"CFG::check_stop: should_stop=" << should_stop() <<
", useDataThread_=" << useDataThread_ <<
", exception status =" << int(exception());
287 if (!should_stop())
return false;
288 if (!useDataThread_ || mode_ == RequestMode::Ignored)
return true;
289 if (force_stop_)
return true;
292 TLOG(TLVL_DEBUG) <<
"should_stop is true, force_stop_ is false, requestReceiver_->isRunning() is " << std::boolalpha << requestReceiver_->isRunning();
293 return !requestReceiver_->isRunning();
298 if (force || mode_ == RequestMode::Ignored)
300 TLOG(TLVL_EVCOUNTERINC) <<
"ev_counter_inc: Incrementing ev_counter from " << ev_counter() <<
" by " << step;
301 return ev_counter_.fetch_add(step);
303 return ev_counter_.load();
308 TLOG(TLVL_TRACE) <<
"Start Command received.";
309 if (run < 0)
throw cet::exception(
"CommandableFragmentGenerator") <<
"negative run number";
312 timestamp_ = timestamp;
313 ev_counter_.store(1);
315 std::unique_lock<std::mutex> lk(dataBuffersMutex_);
316 for (
auto&
id : dataBuffers_)
318 id.second.DataBufferDepthBytes = 0;
319 id.second.DataBufferDepthFragments = 0;
320 id.second.HighestRequestSeen = 0;
321 id.second.BufferFragmentKept =
false;
322 id.second.DataBuffer.clear();
323 id.second.WindowsSent.clear();
327 should_stop_.store(
false);
328 force_stop_.store(
false);
329 exception_.store(
false);
332 latest_exception_report_ =
"none";
336 std::unique_lock<std::mutex> lk(mutex_);
337 if (useDataThread_) startDataThread();
338 if (useMonitoringThread_) startMonitoringThread();
339 if (mode_ != RequestMode::Ignored)
341 requestReceiver_->SetRunNumber(static_cast<uint32_t>(run));
342 requestReceiver_->startRequestReception();
344 TLOG(TLVL_TRACE) <<
"Start Command complete.";
349 TLOG(TLVL_TRACE) <<
"Stop Command received.";
352 timestamp_ = timestamp;
353 if (requestReceiver_)
355 TLOG(TLVL_DEBUG) <<
"Stopping Request reception BEGIN";
356 requestReceiver_->stopRequestReception();
357 TLOG(TLVL_DEBUG) <<
"Stopping Request reception END";
361 should_stop_.store(
true);
362 std::unique_lock<std::mutex> lk(mutex_);
366 TLOG(TLVL_TRACE) <<
"Stop Command complete.";
371 TLOG(TLVL_TRACE) <<
"Pause Command received.";
373 timestamp_ = timestamp;
377 should_stop_.store(
true);
378 std::unique_lock<std::mutex> lk(mutex_);
385 TLOG(TLVL_TRACE) <<
"Resume Command received.";
387 timestamp_ = timestamp;
390 should_stop_ =
false;
393 std::unique_lock<std::mutex> lk(dataBuffersMutex_);
394 for (
auto&
id : dataBuffers_)
396 id.second.DataBufferDepthBytes = 0;
397 id.second.DataBufferDepthFragments = 0;
398 id.second.BufferFragmentKept =
false;
399 id.second.DataBuffer.clear();
405 std::unique_lock<std::mutex> lk(mutex_);
409 TLOG(TLVL_TRACE) <<
"Resume Command complete.";
414 TLOG(TLVL_TRACE) <<
"Report Command received.";
415 std::lock_guard<std::mutex> lk(mutex_);
422 std::string childReport = reportSpecific(which);
423 if (childReport.length() > 0) {
return childReport; }
426 if (which ==
"latest_exception")
428 return latest_exception_report_;
432 childReport = report();
433 if (childReport.length() > 0) {
return childReport; }
443 TLOG(TLVL_TRACE) <<
"Report Command complete.";
450 #pragma message "Using default implementation of CommandableFragmentGenerator::pauseNoMutex()"
455 #pragma message "Using default implementation of CommandableFragmentGenerator::pause()"
459 #pragma message "Using default implementation of CommandableFragmentGenerator::resume()"
464 #pragma message "Using default implementation of CommandableFragmentGenerator::report()"
470 #pragma message "Using default implementation of CommandableFragmentGenerator::reportSpecific(std::string)"
476 #pragma message "Using default implementation of CommandableFragmentGenerator::checkHWStatus_()"
482 #pragma message "Using default implementation of CommandableFragmentGenerator::metaCommand(std::string, std::string)"
488 if (dataThread_.joinable()) dataThread_.join();
489 TLOG(TLVL_INFO) <<
"Starting Data Receiver Thread";
494 catch (
const boost::exception& e)
496 TLOG(TLVL_ERROR) <<
"Caught boost::exception starting Data Receiver thread: " << boost::diagnostic_information(e) <<
", errno=" << errno;
497 std::cerr <<
"Caught boost::exception starting Data Receiver thread: " << boost::diagnostic_information(e) <<
", errno=" << errno << std::endl;
504 if (monitoringThread_.joinable()) monitoringThread_.join();
505 TLOG(TLVL_INFO) <<
"Starting Hardware Monitoring Thread";
510 catch (
const boost::exception& e)
512 TLOG(TLVL_ERROR) <<
"Caught boost::exception starting Hardware Monitoring thread: " << boost::diagnostic_information(e) <<
", errno=" << errno;
513 std::cerr <<
"Caught boost::exception starting Hardware Monitoring thread: " << boost::diagnostic_information(e) <<
", errno=" << errno << std::endl;
522 case RequestMode::Single:
524 case RequestMode::Buffer:
526 case RequestMode::Window:
528 case RequestMode::Ignored:
540 data_thread_running_ =
true;
545 TLOG(TLVL_DEBUG) <<
"getDataLoop: isHardwareOK is " << isHardwareOK_ <<
", aborting data thread";
546 data_thread_running_ =
false;
550 TLOG(TLVL_GETDATALOOP) <<
"getDataLoop: calling getNext_";
553 auto startdata = std::chrono::steady_clock::now();
554 FragmentPtrs newData;
558 data = getNext_(newData);
562 ExceptionHandler(ExceptionHandlerRethrow::no,
563 "Exception thrown by fragment generator in CommandableFragmentGenerator::getDataLoop; setting exception state to \"true\"");
566 data_thread_running_ =
false;
572 metricMan->sendMetric(
"Avg Data Acquisition Time", TimeUtils::GetElapsedTime(startdata),
"s", 3, artdaq::MetricMode::Average);
575 if (newData.size() == 0 && sleep_on_no_data_us_ > 0)
577 usleep(sleep_on_no_data_us_);
580 auto dataIter = newData.begin();
581 while (dataIter != newData.end())
583 TLOG(TLVL_GETDATALOOP_VERBOSE) <<
"getDataLoop: getNext_() returned fragment with timestamp = " << (*dataIter)->timestamp() <<
", and sizeBytes = " << (*dataIter)->sizeBytes();
585 auto frag_id = (*dataIter)->fragmentID();
586 if (!dataBuffers_.count(frag_id))
588 throw cet::exception(
"DataBufferError") <<
"Error in CommandableFragmentGenerator: Recevied Fragment with fragment_id " << frag_id <<
", but this ID was not declared in fragment_ids!";
591 TLOG(TLVL_GETDATALOOP_DATABUFFWAIT) <<
"Waiting for data buffer ready";
592 if (!waitForDataBufferReady(frag_id))
return;
593 TLOG(TLVL_GETDATALOOP_DATABUFFWAIT) <<
"Done waiting for data buffer ready";
595 TLOG(TLVL_GETDATALOOP) <<
"getDataLoop: processing data";
596 if (data && !force_stop_)
598 std::unique_lock<std::mutex> lock(dataBuffersMutex_);
601 case RequestMode::Single:
602 dataBuffers_[frag_id].DataBuffer.clear();
603 dataBuffers_[frag_id].DataBufferDepthBytes = (*dataIter)->sizeBytes();
604 dataBuffers_[frag_id].DataBuffer.emplace_back(std::move(*dataIter));
605 dataIter = newData.erase(dataIter);
607 case RequestMode::Buffer:
608 case RequestMode::Ignored:
609 case RequestMode::Window:
612 dataBuffers_[frag_id].DataBufferDepthBytes += (*dataIter)->sizeBytes();
613 dataBuffers_[frag_id].DataBuffer.emplace_back(std::move(*dataIter));
614 dataIter = newData.erase(dataIter);
617 getDataBufferStats(frag_id);
626 std::unique_lock<std::mutex> lock(dataBuffersMutex_);
627 for (
auto&
id : dataBuffers_)
629 if (
id.second.DataBuffer.size() > 0)
631 dataCondition_.notify_all();
636 if (!data || force_stop_)
638 TLOG(TLVL_INFO) <<
"Data flow has stopped. Ending data collection thread";
639 std::unique_lock<std::mutex> lock(dataBuffersMutex_);
640 data_thread_running_ =
false;
641 if (requestReceiver_) requestReceiver_->ClearRequests();
643 TLOG(TLVL_INFO) <<
"getDataLoop: Ending thread";
651 if (!dataBuffers_.count(
id))
653 throw cet::exception(
"DataBufferError") <<
"Error in CommandableFragmentGenerator: Cannot wait for data buffer for ID " <<
id <<
" because it does not exist!";
655 auto startwait = std::chrono::steady_clock::now();
657 auto lastwaittime = 0ULL;
660 std::unique_lock<std::mutex> lock(dataBuffersMutex_);
661 getDataBufferStats(
id);
664 while (dataBufferIsTooLarge(
id))
666 if (!circularDataBufferMode_)
670 TLOG(TLVL_DEBUG) <<
"Run ended while waiting for buffer to shrink!";
671 std::unique_lock<std::mutex> lock(dataBuffersMutex_);
672 getDataBufferStats(
id);
673 dataCondition_.notify_all();
674 data_thread_running_ =
false;
677 auto waittime = TimeUtils::GetElapsedTimeMilliseconds(startwait);
679 if (first || (waittime != lastwaittime && waittime % 1000 == 0))
681 TLOG(TLVL_WARNING) <<
"Bad Omen: Data Buffer has exceeded its size limits. "
682 <<
"(seq_id=" << ev_counter() <<
", frag_id=" <<
id
683 <<
", frags=" << dataBuffers_[id].DataBufferDepthFragments <<
"/" << maxDataBufferDepthFragments_
684 <<
", szB=" << dataBuffers_[id].DataBufferDepthBytes <<
"/" << maxDataBufferDepthBytes_ <<
")";
685 TLOG(TLVL_TRACE) <<
"Bad Omen: Possible causes include requests not getting through or Ignored-mode BR issues";
688 if (waittime % 5 && waittime != lastwaittime)
690 TLOG(TLVL_WAITFORBUFFERREADY) <<
"getDataLoop: Data Retreival paused for " << waittime <<
" ms waiting for data buffer to drain";
692 lastwaittime = waittime;
697 std::unique_lock<std::mutex> lock(dataBuffersMutex_);
698 getDataBufferStats(
id);
699 if (dataBufferIsTooLarge(
id))
701 auto begin = dataBuffers_[id].DataBuffer.begin();
702 if (begin == dataBuffers_[
id].DataBuffer.end())
704 TLOG(TLVL_WARNING) <<
"Data buffer is reported as too large, but doesn't contain any Fragments! Possible corrupt memory!";
709 TLOG(TLVL_WAITFORBUFFERREADY) <<
"waitForDataBufferReady: Dropping Fragment with timestamp " << (*begin)->timestamp() <<
" from data buffer (Buffer over-size, circular data buffer mode)";
711 dataBuffers_[id].DataBufferDepthBytes -= (*begin)->sizeBytes();
712 dataBuffers_[id].DataBuffer.erase(begin);
713 dataBuffers_[id].BufferFragmentKept =
false;
714 getDataBufferStats(
id);
723 if (!dataBuffers_.count(
id))
725 throw cet::exception(
"DataBufferError") <<
"Error in CommandableFragmentGenerator: Cannot check size of data buffer for ID " <<
id <<
" because it does not exist!";
727 return (maxDataBufferDepthFragments_ > 0 && dataBuffers_[
id].DataBufferDepthFragments > maxDataBufferDepthFragments_) || (maxDataBufferDepthBytes_ > 0 && dataBuffers_[id].DataBufferDepthBytes > maxDataBufferDepthBytes_);
732 if (!dataBuffers_.count(
id))
734 throw cet::exception(
"DataBufferError") <<
"Error in CommandableFragmentGenerator: Cannot get stats of data buffer for ID " <<
id <<
" because it does not exist!";
737 dataBuffers_[id].DataBufferDepthFragments = dataBuffers_[id].DataBuffer.size();
741 TLOG(TLVL_GETBUFFERSTATS) <<
"getDataBufferStats: Sending Metrics";
742 metricMan->sendMetric(
"Buffer Depth Fragments", dataBuffers_[
id].DataBufferDepthFragments.load(),
"fragments", 1, MetricMode::LastPoint);
743 metricMan->sendMetric(
"Buffer Depth Bytes", dataBuffers_[
id].DataBufferDepthBytes.load(),
"bytes", 1, MetricMode::LastPoint);
745 auto bufferDepthFragmentsPercent = dataBuffers_[id].DataBufferDepthFragments.load() * 100 /
static_cast<double>(maxDataBufferDepthFragments_);
746 auto bufferDepthBytesPercent = dataBuffers_[id].DataBufferDepthBytes.load() * 100 /
static_cast<double>(maxDataBufferDepthBytes_);
747 metricMan->sendMetric(
"Fragment Generator Buffer Full %Fragments", bufferDepthFragmentsPercent,
"%", 3, MetricMode::LastPoint);
748 metricMan->sendMetric(
"Fragment Generator Buffer Full %Bytes", bufferDepthBytesPercent,
"%", 3, MetricMode::LastPoint);
749 metricMan->sendMetric(
"Fragment Generator Buffer Full %", bufferDepthFragmentsPercent > bufferDepthBytesPercent ? bufferDepthFragmentsPercent : bufferDepthBytesPercent,
"%", 1, MetricMode::LastPoint);
751 TLOG(TLVL_GETBUFFERSTATS) <<
"getDataBufferStats: frags=" << dataBuffers_[id].DataBufferDepthFragments.load() <<
"/" << maxDataBufferDepthFragments_
752 <<
", sz=" << dataBuffers_[id].DataBufferDepthBytes.load() <<
"/" << maxDataBufferDepthBytes_;
757 if (!dataBuffers_.count(
id))
759 throw cet::exception(
"DataBufferError") <<
"Error in CommandableFragmentGenerator: Cannot check data buffer for ID " <<
id <<
" because it does not exist!";
762 if (dataBuffers_[
id].DataBufferDepthFragments > 0 && mode_ != RequestMode::Single && mode_ != RequestMode::Ignored)
765 getDataBufferStats(
id);
766 while (dataBufferIsTooLarge(
id))
768 auto begin = dataBuffers_[id].DataBuffer.begin();
769 TLOG(TLVL_CHECKDATABUFFER) <<
"checkDataBuffer: Dropping Fragment with timestamp " << (*begin)->timestamp() <<
" from data buffer (Buffer over-size)";
770 dataBuffers_[id].DataBufferDepthBytes -= (*begin)->sizeBytes();
771 dataBuffers_[id].DataBuffer.erase(begin);
772 dataBuffers_[id].BufferFragmentKept =
false;
773 getDataBufferStats(
id);
775 if (dataBuffers_[
id].DataBuffer.size() > 0)
777 TLOG(TLVL_CHECKDATABUFFER) <<
"Determining if Fragments can be dropped from data buffer";
778 Fragment::timestamp_t last = dataBuffers_[id].DataBuffer.back()->timestamp();
779 Fragment::timestamp_t min = last > staleTimeout_ ? last - staleTimeout_ : 0;
780 for (
auto it = dataBuffers_[
id].DataBuffer.begin(); it != dataBuffers_[id].DataBuffer.end();)
782 if ((*it)->timestamp() < min)
784 TLOG(TLVL_CHECKDATABUFFER) <<
"checkDataBuffer: Dropping Fragment with timestamp " << (*it)->timestamp() <<
" from data buffer (timeout=" << staleTimeout_ <<
", min=" << min <<
")";
785 dataBuffers_[id].DataBufferDepthBytes -= (*it)->sizeBytes();
786 dataBuffers_[id].BufferFragmentKept =
false;
787 it = dataBuffers_[id].DataBuffer.erase(it);
794 getDataBufferStats(
id);
803 if (should_stop() || monitoringInterval_ <= 0)
805 TLOG(TLVL_DEBUG) <<
"getMonitoringDataLoop: should_stop() is " << std::boolalpha << should_stop()
806 <<
" and monitoringInterval is " << monitoringInterval_ <<
", returning";
809 TLOG(TLVL_GETMONITORINGDATA) <<
"getMonitoringDataLoop: Determining whether to call checkHWStatus_";
811 auto now = std::chrono::steady_clock::now();
812 if (TimeUtils::GetElapsedTimeMicroseconds(lastMonitoringCall_, now) >= static_cast<size_t>(monitoringInterval_))
814 isHardwareOK_ = checkHWStatus_();
815 TLOG(TLVL_GETMONITORINGDATA) <<
"getMonitoringDataLoop: isHardwareOK_ is now " << std::boolalpha << isHardwareOK_;
816 lastMonitoringCall_ = now;
818 usleep(monitoringInterval_ / 10);
826 TLOG(TLVL_APPLYREQUESTS) <<
"Mode is Ignored; Copying data to output";
827 for (
auto&
id : dataBuffers_)
829 std::move(
id.second.DataBuffer.begin(),
id.second.DataBuffer.end(), std::inserter(frags, frags.end()));
830 id.second.DataBufferDepthBytes = 0;
831 id.second.DataBufferDepthFragments = 0;
832 id.second.BufferFragmentKept =
false;
833 id.second.DataBuffer.clear();
840 auto requests = requestReceiver_->GetRequests();
841 while (requests.size() > 1)
844 requestReceiver_->RemoveRequest(requests.begin()->first);
845 requests.erase(requests.begin());
847 sendEmptyFragments(frags, requests);
850 if (requests.size() == 0 || !requests.count(ev_counter()))
return;
852 for (
auto&
id : dataBuffers_)
854 if (
id.second.DataBuffer.size() > 0)
856 assert(
id.second.DataBuffer.size() == 1);
857 TLOG(TLVL_APPLYREQUESTS) <<
"Mode is Single; Sending copy of last event";
858 for (
auto& fragptr :
id.second.DataBuffer)
861 auto frag = fragptr.get();
862 auto newfrag = std::unique_ptr<artdaq::Fragment>(
new Fragment(ev_counter(), frag->fragmentID()));
863 newfrag->resize(frag->size() - detail::RawFragmentHeader::num_words());
864 memcpy(newfrag->headerAddress(), frag->headerAddress(), frag->sizeBytes());
865 newfrag->setTimestamp(requests[ev_counter()]);
866 newfrag->setSequenceID(ev_counter());
867 frags.push_back(std::move(newfrag));
872 sendEmptyFragment(frags, ev_counter(),
id.first,
"No data for");
875 requestReceiver_->RemoveRequest(ev_counter());
876 ev_counter_inc(1,
true);
882 auto requests = requestReceiver_->GetRequests();
883 while (requests.size() > 1)
886 requestReceiver_->RemoveRequest(requests.begin()->first);
887 requests.erase(requests.begin());
889 sendEmptyFragments(frags, requests);
892 if (requests.size() == 0 || !requests.count(ev_counter()))
return;
894 for (
auto&
id : dataBuffers_)
896 TLOG(TLVL_DEBUG) <<
"Creating ContainerFragment for Buffered Fragments";
897 frags.emplace_back(
new artdaq::Fragment(ev_counter(),
id.first));
898 frags.back()->setTimestamp(requests[ev_counter()]);
899 ContainerFragmentLoader cfl(*frags.back());
900 cfl.set_missing_data(
false);
903 auto it =
id.second.DataBuffer.begin();
904 if (
id.second.BufferFragmentKept &&
id.second.DataBuffer.size() > 1)
906 id.second.DataBufferDepthBytes -= (*it)->sizeBytes();
907 it =
id.second.DataBuffer.erase(it);
911 while (it !=
id.second.DataBuffer.end())
913 TLOG(TLVL_APPLYREQUESTS) <<
"ApplyRequests: Adding Fragment with timestamp " << (*it)->timestamp() <<
" to Container with sequence ID " << ev_counter();
914 cfl.addFragment(*it);
915 if (bufferModeKeepLatest_ &&
id.second.DataBuffer.size() == 1)
917 id.second.BufferFragmentKept =
true;
920 id.second.DataBufferDepthBytes -= (*it)->sizeBytes();
921 it =
id.second.DataBuffer.erase(it);
924 requestReceiver_->RemoveRequest(ev_counter());
925 ev_counter_inc(1,
true);
930 TLOG(TLVL_APPLYREQUESTS) <<
"applyRequestsWindowMode_CheckAndFillDataBuffer: Checking that data exists for request window " << seq;
931 Fragment::timestamp_t min = ts > windowOffset_ ? ts - windowOffset_ : 0;
932 Fragment::timestamp_t max = min + windowWidth_;
933 TLOG(TLVL_APPLYREQUESTS) <<
"ApplyRequestsWindowsMode_CheckAndFillDataBuffer: min is " << min <<
", max is " << max
934 <<
" and first/last points in buffer are " << (dataBuffers_[id].DataBuffer.size() > 0 ? dataBuffers_[id].DataBuffer.front()->timestamp() : 0)
935 <<
"/" << (dataBuffers_[
id].DataBuffer.size() > 0 ? dataBuffers_[id].DataBuffer.back()->timestamp() : 0)
936 <<
" (sz=" << dataBuffers_[
id].DataBuffer.size() <<
" [" << dataBuffers_[id].DataBufferDepthBytes.load()
937 <<
"/" << maxDataBufferDepthBytes_ <<
"])";
938 bool windowClosed = dataBuffers_[id].DataBuffer.size() > 0 && dataBuffers_[id].DataBuffer.back()->timestamp() >= max;
939 bool windowTimeout = !windowClosed && TimeUtils::GetElapsedTimeMicroseconds(requestReceiver_->GetRequestTime(seq)) > window_close_timeout_us_;
942 TLOG(TLVL_WARNING) <<
"applyRequestsWindowMode_CheckAndFillDataBuffer: A timeout occurred waiting for data to close the request window ({" << min <<
"-" << max
943 <<
"}, buffer={" << (dataBuffers_[id].DataBuffer.size() > 0 ? dataBuffers_[id].DataBuffer.front()->timestamp() : 0) <<
"-"
944 << (dataBuffers_[
id].DataBuffer.size() > 0 ? dataBuffers_[id].DataBuffer.back()->timestamp() : 0)
945 <<
"} ). Time waiting: "
946 << TimeUtils::GetElapsedTimeMicroseconds(requestReceiver_->GetRequestTime(seq)) <<
" us "
947 <<
"(> " << window_close_timeout_us_ <<
" us).";
949 if (windowClosed || !data_thread_running_ || windowTimeout)
951 TLOG(TLVL_DEBUG) <<
"applyRequestsWindowMode_CheckAndFillDataBuffer: Creating ContainerFragment for Window-requested Fragments";
952 frags.emplace_back(
new artdaq::Fragment(seq,
id));
953 frags.back()->setTimestamp(ts);
954 ContainerFragmentLoader cfl(*frags.back());
968 if (!windowClosed || (dataBuffers_[
id].DataBuffer.size() > 0 && dataBuffers_[id].DataBuffer.front()->timestamp() > min))
970 TLOG(TLVL_DEBUG) <<
"applyRequestsWindowMode_CheckAndFillDataBuffer: Request window starts before and/or ends after the current data buffer, setting ContainerFragment's missing_data flag!"
971 <<
" (requestWindowRange=[" << min <<
"," << max <<
"], "
972 <<
"buffer={" << (dataBuffers_[id].DataBuffer.size() > 0 ? dataBuffers_[id].DataBuffer.front()->timestamp() : 0) <<
"-"
973 << (dataBuffers_[
id].DataBuffer.size() > 0 ? dataBuffers_[id].DataBuffer.back()->timestamp() : 0) <<
"}";
974 cfl.set_missing_data(
true);
978 for (
auto it = dataBuffers_[
id].DataBuffer.begin(); it != dataBuffers_[id].DataBuffer.end();)
980 Fragment::timestamp_t fragT = (*it)->timestamp();
981 if (fragT < min || fragT > max || (fragT == max && windowWidth_ > 0))
987 TLOG(TLVL_APPLYREQUESTS) <<
"applyRequestsWindowMode_CheckAndFillDataBuffer: Adding Fragment with timestamp " << (*it)->timestamp() <<
" to Container";
988 cfl.addFragment(*it);
992 dataBuffers_[id].DataBufferDepthBytes -= (*it)->sizeBytes();
993 it = dataBuffers_[id].DataBuffer.erase(it);
1001 dataBuffers_[id].WindowsSent[seq] = std::chrono::steady_clock::now();
1002 if (seq > dataBuffers_[
id].HighestRequestSeen) dataBuffers_[id].HighestRequestSeen = seq;
1008 TLOG(TLVL_APPLYREQUESTS) <<
"applyRequestsWindowMode BEGIN";
1010 auto requests = requestReceiver_->GetRequests();
1012 TLOG(TLVL_APPLYREQUESTS) <<
"applyRequestsWindowMode: Starting request processing";
1013 for (
auto req = requests.begin(); req != requests.end();)
1015 TLOG(TLVL_APPLYREQUESTS) <<
"applyRequestsWindowMode: processing request with sequence ID " << req->first <<
", timestamp " << req->second;
1017 while (req->first < ev_counter() && requests.size() > 0)
1019 TLOG(TLVL_APPLYREQUESTS) <<
"applyRequestsWindowMode: Clearing passed request for sequence ID " << req->first;
1020 requestReceiver_->RemoveRequest(req->first);
1021 req = requests.erase(req);
1023 if (requests.size() == 0)
break;
1025 for (
auto&
id : dataBuffers_)
1027 if (!
id.second.WindowsSent.count(req->first))
1029 applyRequestsWindowMode_CheckAndFillDataBuffer(frags,
id.first, req->first, req->second);
1032 checkSentWindows(req->first);
1037 for (
auto&
id : dataBuffers_)
1039 std::set<artdaq::Fragment::sequence_id_t> seqs;
1040 for (
auto& seq :
id.second.WindowsSent)
1042 seqs.insert(seq.first);
1044 for (
auto& seq : seqs)
1046 checkSentWindows(seq);
1053 if (check_stop() || exception())
1059 if (mode_ == RequestMode::Ignored)
1061 while (dataBufferFragmentCount_() == 0)
1063 if (check_stop() || exception() || !isHardwareOK_)
return false;
1064 std::unique_lock<std::mutex> lock(dataBuffersMutex_);
1065 dataCondition_.wait_for(lock, std::chrono::milliseconds(10), [
this]() {
return dataBufferFragmentCount_() > 0; });
1070 if ((check_stop() && requestReceiver_->size() == 0) || exception())
return false;
1072 std::unique_lock<std::mutex> lock(dataBuffersMutex_);
1073 dataCondition_.wait_for(lock, std::chrono::milliseconds(10));
1080 while (requestReceiver_->size() == 0 && counter < 100)
1082 if (check_stop() || exception())
return false;
1086 requestReceiver_->WaitForRequests(10);
1092 std::unique_lock<std::mutex> dlk(dataBuffersMutex_);
1096 case RequestMode::Single:
1097 applyRequestsSingleMode(frags);
1099 case RequestMode::Window:
1100 applyRequestsWindowMode(frags);
1102 case RequestMode::Buffer:
1103 applyRequestsBufferMode(frags);
1105 case RequestMode::Ignored:
1107 applyRequestsIgnoredMode(frags);
1111 if (!data_thread_running_ || force_stop_)
1113 TLOG(TLVL_INFO) <<
"Data thread has stopped; Clearing data buffers";
1114 for (
auto&
id : dataBuffers_)
1116 id.second.DataBufferDepthBytes = 0;
1117 id.second.DataBufferDepthFragments = 0;
1118 id.second.BufferFragmentKept =
false;
1119 id.second.DataBuffer.clear();
1123 getDataBuffersStats();
1126 if (frags.size() > 0)
1127 TLOG(TLVL_APPLYREQUESTS) <<
"Finished Processing requests, returning " << frags.size() <<
" fragments, current ev_counter is " << ev_counter();
1133 TLOG(TLVL_WARNING) << desc <<
" sequence ID " << seqId <<
", sending empty fragment";
1134 auto frag =
new Fragment();
1135 frag->setSequenceID(seqId);
1136 frag->setFragmentID(fragmentId);
1137 frag->setSystemType(Fragment::EmptyFragmentType);
1138 frags.emplace_back(FragmentPtr(frag));
1144 if (requests.size() > 0)
1146 TLOG(TLVL_SENDEMPTYFRAGMENTS) <<
"Sending Empty Fragments for Sequence IDs from " << ev_counter() <<
" up to but not including " << requests.begin()->first;
1147 while (requests.begin()->first > ev_counter())
1149 for (
auto& fid : dataBuffers_)
1151 sendEmptyFragment(frags, ev_counter(), fid.first,
"Missed request for");
1153 ev_counter_inc(1,
true);
1160 bool seqComplete =
true;
1161 bool seqTimeout =
false;
1162 for (
auto&
id : dataBuffers_)
1164 if (!
id.second.WindowsSent.count(seq) ||
id.second.HighestRequestSeen < seq)
1166 seqComplete =
false;
1168 if (
id.second.WindowsSent.count(seq) && TimeUtils::GetElapsedTimeMicroseconds(
id.second.WindowsSent[seq]) > missing_request_window_timeout_us_)
1175 TLOG(TLVL_CHECKWINDOWS) <<
"checkSentWindows: Request " << seq <<
" is complete, removing from requestReceiver.";
1176 requestReceiver_->RemoveRequest(seq);
1178 if (ev_counter() == seq)
1180 TLOG(TLVL_CHECKWINDOWS) <<
"checkSentWindows: Sequence ID matches ev_counter, incrementing ev_counter (" << ev_counter() <<
")";
1182 for (
auto&
id : dataBuffers_)
1184 id.second.WindowsSent.erase(seq);
1187 ev_counter_inc(1,
true);
1192 TLOG(TLVL_CHECKWINDOWS) <<
"checkSentWindows: Sent Window history indicates that requests between " << ev_counter() <<
" and " << seq <<
" have timed out.";
1193 while (ev_counter() <= seq)
1195 if (ev_counter() < seq) TLOG(TLVL_WARNING) <<
"Missed request for sequence ID " << ev_counter() <<
"! Will not send any data for this sequence ID!";
1196 requestReceiver_->RemoveRequest(ev_counter());
1198 for (
auto&
id : dataBuffers_)
1200 id.second.WindowsSent.erase(ev_counter());
1203 ev_counter_inc(1,
true);
CommandableFragmentGenerator(const fhicl::ParameterSet &ps)
CommandableFragmentGenerator Constructor.
void applyRequestsSingleMode(artdaq::FragmentPtrs &frags)
Create fragments using data buffer for request mode Single. Precondition: dataBufferMutex_ and reques...
virtual bool checkHWStatus_()
Check any relavent hardware status registers. Return false if an error condition exists that should h...
virtual ~CommandableFragmentGenerator()
CommandableFragmentGenerator Destructor.
void applyRequestsBufferMode(artdaq::FragmentPtrs &frags)
Create fragments using data buffer for request mode Buffer. Precondition: dataBufferMutex_ and reques...
void getDataBufferStats(Fragment::fragment_id_t id)
Calculate the size of the dataBuffer and report appropriate metrics.
void getMonitoringDataLoop()
This function regularly calls checkHWStatus_(), and sets the isHardwareOK flag accordingly.
void startDataThread()
Function that launches the data thread (getDataLoop())
artdaq::Fragment::fragment_id_t fragment_id() const
Get the Fragment ID of this Fragment generator.
std::string ReportCmd(std::string const &which="")
Get a report about a user-specified run-time quantity.
virtual bool metaCommand(std::string const &command, std::string const &arg)
The meta-command is used for implementing user-specific commands in a CommandableFragmentGenerator.
void StopCmd(uint64_t timeout, uint64_t timestamp)
Stop the CommandableFragmentGenerator.
void applyRequestsWindowMode_CheckAndFillDataBuffer(artdaq::FragmentPtrs &frags, artdaq::Fragment::fragment_id_t id, artdaq::Fragment::sequence_id_t seq, artdaq::Fragment::timestamp_t ts)
Copy data from the relevant data buffer that matches the given timestamp.
void checkSentWindows(Fragment::sequence_id_t seq)
Check the windows_sent_ooo_ map for sequence IDs that may be removed.
bool sendEmptyFragment(FragmentPtrs &frags, size_t sequenceId, Fragment::fragment_id_t fragmentId, std::string desc)
Send an EmptyFragmentType Fragment.
void applyRequestsWindowMode(artdaq::FragmentPtrs &frags)
Create fragments using data buffer for request mode Window. Precondition: dataBufferMutex_ and reques...
void StartCmd(int run, uint64_t timeout, uint64_t timestamp)
Start the CommandableFragmentGenerator.
virtual void pauseNoMutex()
On call to PauseCmd, pauseNoMutex() is called prior to PauseCmd acquiring the mutex ...
bool check_stop()
Routine used by applyRequests to make sure that all outstanding requests have been fulfilled before r...
void ResumeCmd(uint64_t timeout, uint64_t timestamp)
Resume the CommandableFragmentGenerator.
bool getNext(FragmentPtrs &output) overridefinal
getNext calls either applyRequests or getNext_ to get any data that is ready to be sent to the EventB...
size_t ev_counter_inc(size_t step=1, bool force=false)
Increment the event counter, if the current RequestMode allows it.
bool applyRequests(FragmentPtrs &frags)
See if any requests have been received, and add the corresponding data Fragment objects to the output...
void applyRequestsIgnoredMode(artdaq::FragmentPtrs &frags)
Create fragments using data buffer for request mode Ignored. Precondition: dataBufferMutex_ and reque...
void PauseCmd(uint64_t timeout, uint64_t timestamp)
Pause the CommandableFragmentGenerator.
void getDataLoop()
When separate_data_thread is set to true, this loop repeatedly calls getNext_ and adds returned Fragm...
void sendEmptyFragments(FragmentPtrs &frags, std::map< Fragment::sequence_id_t, Fragment::timestamp_t > &requests)
This function is for Buffered and Single request modes, as they can only respond to one data request ...
Receive data requests and make them available to CommandableFragmentGenerator or other interested par...
void startMonitoringThread()
Function that launches the monitoring thread (getMonitoringDataLoop())
virtual void pause()
If a CommandableFragmentGenerator subclass is reading from hardware, the implementation of pause() sh...
virtual void resume()
The subrun number will be incremented before a call to resume.
bool waitForDataBufferReady(Fragment::fragment_id_t id)
Wait for the data buffer to drain (dataBufferIsTooLarge returns false), periodically reporting status...
virtual std::string report()
Let's say that the contract with the report() functions is that they return a non-empty string if the...
std::string printMode_()
Return the string representation of the current RequestMode.
void checkDataBuffer(Fragment::fragment_id_t id)
Perform data buffer pruning operations for the given buffer. If the RequestMode is Single...
virtual std::string reportSpecific(std::string const &what)
Report the status of a specific quantity
bool dataBufferIsTooLarge(Fragment::fragment_id_t id)
Test the configured constraints on the data buffer.
void joinThreads()
Join any data-taking threads. Should be called when destructing CommandableFragmentGenerator.