1 #include "artdaq/DAQdata/Globals.hh"
2 #define TRACE_NAME (app_name + "_CommandableFragmentGenerator").c_str() // include these 2 first -
4 #include "artdaq/Generators/CommandableFragmentGenerator.hh"
6 #include <boost/exception/all.hpp>
7 #include <boost/throw_exception.hpp>
12 #include "canvas/Utilities/Exception.h"
13 #include "cetlib_except/exception.h"
14 #include "fhiclcpp/ParameterSet.h"
16 #include "artdaq-core/Data/ContainerFragmentLoader.hh"
17 #include "artdaq-core/Data/Fragment.hh"
18 #include "artdaq-core/Utilities/ExceptionHandler.hh"
19 #include "artdaq-core/Utilities/SimpleLookupPolicy.hh"
20 #include "artdaq-core/Utilities/TimeUtils.hh"
30 #define TLVL_GETNEXT 10
31 #define TLVL_GETNEXT_VERBOSE 20
32 #define TLVL_CHECKSTOP 11
33 #define TLVL_EVCOUNTERINC 12
34 #define TLVL_GETDATALOOP 13
35 #define TLVL_GETDATALOOP_DATABUFFWAIT 21
36 #define TLVL_GETDATALOOP_VERBOSE 20
37 #define TLVL_WAITFORBUFFERREADY 15
38 #define TLVL_GETBUFFERSTATS 16
39 #define TLVL_CHECKDATABUFFER 17
40 #define TLVL_GETMONITORINGDATA 18
41 #define TLVL_APPLYREQUESTS 9
42 #define TLVL_SENDEMPTYFRAGMENTS 19
43 #define TLVL_CHECKWINDOWS 14
47 , requestReceiver_(nullptr)
48 , windowOffset_(ps.get<Fragment::timestamp_t>(
"request_window_offset", 0))
49 , windowWidth_(ps.get<Fragment::timestamp_t>(
"request_window_width", 0))
50 , staleTimeout_(ps.get<Fragment::timestamp_t>(
"stale_request_timeout", 0xFFFFFFFF))
51 , expectedType_(ps.get<Fragment::type_t>(
"expected_fragment_type", Fragment::type_t(Fragment::EmptyFragmentType)))
52 , uniqueWindows_(ps.get<bool>(
"request_windows_are_unique", true))
53 , missing_request_window_timeout_us_(ps.get<size_t>(
"missing_request_window_timeout_us", 5000000))
54 , window_close_timeout_us_(ps.get<size_t>(
"window_close_timeout_us", 2000000))
55 , useDataThread_(ps.get<bool>(
"separate_data_thread", false))
56 , circularDataBufferMode_(ps.get<bool>(
"circular_buffer_mode", false))
57 , sleep_on_no_data_us_(ps.get<size_t>(
"sleep_on_no_data_us", 0))
58 , data_thread_running_(false)
59 , maxDataBufferDepthFragments_(ps.get<int>(
"data_buffer_depth_fragments", 1000))
60 , maxDataBufferDepthBytes_(ps.get<size_t>(
"data_buffer_depth_mb", 1000) * 1024 * 1024)
61 , useMonitoringThread_(ps.get<bool>(
"separate_monitoring_thread", false))
62 , monitoringInterval_(ps.get<int64_t>(
"hardware_poll_interval_us", 0))
63 , lastMonitoringCall_()
67 , timeout_(std::numeric_limits<uint64_t>::max())
68 , timestamp_(std::numeric_limits<uint64_t>::max())
72 , latest_exception_report_(
"none")
75 , sleep_on_stop_us_(0)
77 board_id_ = ps.get<
int>(
"board_id");
78 instance_name_for_metrics_ =
"BoardReader." + boost::lexical_cast<std::string>(board_id_);
80 auto fragment_ids = ps.get<std::vector<artdaq::Fragment::fragment_id_t>>(
"fragment_ids", std::vector<artdaq::Fragment::fragment_id_t>());
82 TLOG(TLVL_TRACE) <<
"artdaq::CommandableFragmentGenerator::CommandableFragmentGenerator(ps)";
85 if (fragment_id != -99)
87 if (fragment_ids.size() != 0)
89 latest_exception_report_ =
"Error in CommandableFragmentGenerator: can't both define \"fragment_id\" and \"fragment_ids\" in FHiCL document";
90 throw cet::exception(latest_exception_report_);
94 fragment_ids.emplace_back(fragment_id);
98 for (
auto&
id : fragment_ids)
100 dataBuffers_[id].DataBufferDepthBytes = 0;
101 dataBuffers_[id].DataBufferDepthFragments = 0;
102 dataBuffers_[id].HighestRequestSeen = 0;
103 dataBuffers_[id].DataBuffer.emplace_back(FragmentPtr(
new Fragment()));
104 (*dataBuffers_[id].DataBuffer.begin())->setSystemType(Fragment::EmptyFragmentType);
107 sleep_on_stop_us_ = ps.get<
int>(
"sleep_on_stop_us", 0);
109 std::string modeString = ps.get<std::string>(
"request_mode",
"ignored");
110 if (modeString ==
"single" || modeString ==
"Single")
112 mode_ = RequestMode::Single;
114 else if (modeString.find(
"buffer") != std::string::npos || modeString.find(
"Buffer") != std::string::npos)
116 mode_ = RequestMode::Buffer;
118 else if (modeString ==
"window" || modeString ==
"Window")
120 mode_ = RequestMode::Window;
122 else if (modeString.find(
"ignore") != std::string::npos || modeString.find(
"Ignore") != std::string::npos)
124 mode_ = RequestMode::Ignored;
126 TLOG(TLVL_DEBUG) <<
"Request mode is " <<
printMode_();
128 if (mode_ != RequestMode::Ignored)
132 latest_exception_report_ =
"Error in CommandableFragmentGenerator: use_data_thread must be true when request_mode is not \"Ignored\"!";
133 throw cet::exception(latest_exception_report_);
142 requestReceiver_.reset(
nullptr);
149 TLOG(TLVL_DEBUG) <<
"Joining dataThread";
150 if (dataThread_.joinable()) dataThread_.join();
151 TLOG(TLVL_DEBUG) <<
"Joining monitoringThread";
152 if (monitoringThread_.joinable()) monitoringThread_.join();
153 TLOG(TLVL_DEBUG) <<
"joinThreads complete";
160 if (check_stop()) usleep(sleep_on_stop_us_);
161 if (exception() || force_stop_)
return false;
163 if (!useMonitoringThread_ && monitoringInterval_ > 0)
165 TLOG(TLVL_GETNEXT) <<
"getNext: Checking whether to collect Monitoring Data";
166 auto now = std::chrono::steady_clock::now();
168 if (TimeUtils::GetElapsedTimeMicroseconds(lastMonitoringCall_, now) >= static_cast<size_t>(monitoringInterval_))
170 TLOG(TLVL_GETNEXT) <<
"getNext: Collecting Monitoring Data";
171 isHardwareOK_ = checkHWStatus_();
172 TLOG(TLVL_GETNEXT) <<
"getNext: isHardwareOK_ is now " << std::boolalpha << isHardwareOK_;
173 lastMonitoringCall_ = now;
179 std::lock_guard<std::mutex> lk(mutex_);
182 TLOG(TLVL_TRACE) <<
"getNext: Calling applyRequests";
183 result = applyRequests(output);
184 TLOG(TLVL_TRACE) <<
"getNext: Done with applyRequests result=" << std::boolalpha << result;
185 for (
auto dataIter = output.begin(); dataIter != output.end(); ++dataIter)
187 TLOG(20) <<
"getNext: applyRequests() returned fragment with sequenceID = " << (*dataIter)->sequenceID()
188 <<
", type = " << (*dataIter)->typeString() <<
", id = " << std::to_string((*dataIter)->fragmentID())
189 <<
", timestamp = " << (*dataIter)->timestamp() <<
", and sizeBytes = " << (*dataIter)->sizeBytes();
194 TLOG(TLVL_ERROR) <<
"Exception found in BoardReader with board ID " << board_id() <<
"; BoardReader will now return error status when queried";
195 throw cet::exception(
"CommandableFragmentGenerator") <<
"Exception found in BoardReader with board ID " << board_id() <<
"; BoardReader will now return error status when queried";
202 TLOG(TLVL_ERROR) <<
"Stopping CFG because the hardware reports bad status!";
205 TLOG(TLVL_TRACE) <<
"getNext: Calling getNext_ w/ ev_counter()=" << ev_counter();
208 result = getNext_(output);
214 TLOG(TLVL_TRACE) <<
"getNext: Done with getNext_ - ev_counter() now " << ev_counter();
215 for (
auto dataIter = output.begin(); dataIter != output.end(); ++dataIter)
217 TLOG(TLVL_GETNEXT_VERBOSE) <<
"getNext: getNext_() returned fragment with sequenceID = " << (*dataIter)->sequenceID()
218 <<
", type = " << (*dataIter)->typeString() <<
", id = " << std::to_string((*dataIter)->fragmentID())
219 <<
", timestamp = " << (*dataIter)->timestamp() <<
", and sizeBytes = " << (*dataIter)->sizeBytes();
223 catch (
const cet::exception& e)
225 latest_exception_report_ =
"cet::exception caught in getNext(): ";
226 latest_exception_report_.append(e.what());
227 TLOG(TLVL_ERROR) <<
"getNext: cet::exception caught: " << e;
231 catch (
const boost::exception& e)
233 latest_exception_report_ =
"boost::exception caught in getNext(): ";
234 latest_exception_report_.append(boost::diagnostic_information(e));
235 TLOG(TLVL_ERROR) <<
"getNext: boost::exception caught: " << boost::diagnostic_information(e);
239 catch (
const std::exception& e)
241 latest_exception_report_ =
"std::exception caught in getNext(): ";
242 latest_exception_report_.append(e.what());
243 TLOG(TLVL_ERROR) <<
"getNext: std::exception caught: " << e.what();
249 latest_exception_report_ =
"Unknown exception caught in getNext().";
250 TLOG(TLVL_ERROR) <<
"getNext: unknown exception caught";
257 TLOG(TLVL_DEBUG) <<
"getNext: Either getNext_ or applyRequests returned false, stopping";
260 if (metricMan && !output.empty())
262 auto timestamp = output.front()->timestamp();
264 if (output.size() > 1)
266 for (
auto& outputfrag : output)
268 if (outputfrag->timestamp() > timestamp)
270 timestamp = outputfrag->timestamp();
275 metricMan->sendMetric(
"Last Timestamp", timestamp,
"Ticks", 1,
276 MetricMode::LastPoint, app_name);
284 TLOG(TLVL_CHECKSTOP) <<
"CFG::check_stop: should_stop=" << should_stop() <<
", useDataThread_=" << useDataThread_ <<
", exception status =" << int(exception());
286 if (!should_stop())
return false;
287 if (!useDataThread_ || mode_ == RequestMode::Ignored)
return true;
288 if (force_stop_)
return true;
291 TLOG(TLVL_DEBUG) <<
"should_stop is true, force_stop_ is false, requestReceiver_->isRunning() is " << std::boolalpha << requestReceiver_->isRunning();
292 return !requestReceiver_->isRunning();
297 if (force || mode_ == RequestMode::Ignored)
299 TLOG(TLVL_EVCOUNTERINC) <<
"ev_counter_inc: Incrementing ev_counter from " << ev_counter() <<
" by " << step;
300 return ev_counter_.fetch_add(step);
302 return ev_counter_.load();
307 TLOG(TLVL_TRACE) <<
"Start Command received.";
308 if (run < 0)
throw cet::exception(
"CommandableFragmentGenerator") <<
"negative run number";
311 timestamp_ = timestamp;
312 ev_counter_.store(1);
314 std::unique_lock<std::mutex> lk(dataBuffersMutex_);
315 for (
auto&
id : dataBuffers_)
317 id.second.DataBufferDepthBytes = 0;
318 id.second.DataBufferDepthFragments = 0;
319 id.second.HighestRequestSeen = 0;
320 id.second.DataBuffer.clear();
321 id.second.WindowsSent.clear();
325 should_stop_.store(
false);
326 force_stop_.store(
false);
327 exception_.store(
false);
330 latest_exception_report_ =
"none";
334 std::unique_lock<std::mutex> lk(mutex_);
335 if (useDataThread_) startDataThread();
336 if (useMonitoringThread_) startMonitoringThread();
337 if (mode_ != RequestMode::Ignored)
339 requestReceiver_->SetRunNumber(static_cast<uint32_t>(run));
340 requestReceiver_->startRequestReception();
342 TLOG(TLVL_TRACE) <<
"Start Command complete.";
347 TLOG(TLVL_TRACE) <<
"Stop Command received.";
350 timestamp_ = timestamp;
351 if (requestReceiver_)
353 TLOG(TLVL_DEBUG) <<
"Stopping Request reception BEGIN";
354 requestReceiver_->stopRequestReception();
355 TLOG(TLVL_DEBUG) <<
"Stopping Request reception END";
359 should_stop_.store(
true);
360 std::unique_lock<std::mutex> lk(mutex_);
364 TLOG(TLVL_TRACE) <<
"Stop Command complete.";
369 TLOG(TLVL_TRACE) <<
"Pause Command received.";
371 timestamp_ = timestamp;
375 should_stop_.store(
true);
376 std::unique_lock<std::mutex> lk(mutex_);
383 TLOG(TLVL_TRACE) <<
"Resume Command received.";
385 timestamp_ = timestamp;
388 should_stop_ =
false;
391 std::unique_lock<std::mutex> lk(dataBuffersMutex_);
392 for (
auto&
id : dataBuffers_)
394 id.second.DataBufferDepthBytes = 0;
395 id.second.DataBufferDepthFragments = 0;
396 id.second.DataBuffer.clear();
402 std::unique_lock<std::mutex> lk(mutex_);
406 TLOG(TLVL_TRACE) <<
"Resume Command complete.";
411 TLOG(TLVL_TRACE) <<
"Report Command received.";
412 std::lock_guard<std::mutex> lk(mutex_);
419 std::string childReport = reportSpecific(which);
420 if (childReport.length() > 0) {
return childReport; }
423 if (which ==
"latest_exception")
425 return latest_exception_report_;
429 childReport = report();
430 if (childReport.length() > 0) {
return childReport; }
440 TLOG(TLVL_TRACE) <<
"Report Command complete.";
447 #pragma message "Using default implementation of CommandableFragmentGenerator::pauseNoMutex()"
452 #pragma message "Using default implementation of CommandableFragmentGenerator::pause()"
456 #pragma message "Using default implementation of CommandableFragmentGenerator::resume()"
461 #pragma message "Using default implementation of CommandableFragmentGenerator::report()"
467 #pragma message "Using default implementation of CommandableFragmentGenerator::reportSpecific(std::string)"
473 #pragma message "Using default implementation of CommandableFragmentGenerator::checkHWStatus_()"
479 #pragma message "Using default implementation of CommandableFragmentGenerator::metaCommand(std::string, std::string)"
485 if (dataThread_.joinable()) dataThread_.join();
486 TLOG(TLVL_INFO) <<
"Starting Data Receiver Thread";
491 catch (
const boost::exception& e)
493 TLOG(TLVL_ERROR) <<
"Caught boost::exception starting Data Receiver thread: " << boost::diagnostic_information(e) <<
", errno=" << errno;
494 std::cerr <<
"Caught boost::exception starting Data Receiver thread: " << boost::diagnostic_information(e) <<
", errno=" << errno << std::endl;
501 if (monitoringThread_.joinable()) monitoringThread_.join();
502 TLOG(TLVL_INFO) <<
"Starting Hardware Monitoring Thread";
507 catch (
const boost::exception& e)
509 TLOG(TLVL_ERROR) <<
"Caught boost::exception starting Hardware Monitoring thread: " << boost::diagnostic_information(e) <<
", errno=" << errno;
510 std::cerr <<
"Caught boost::exception starting Hardware Monitoring thread: " << boost::diagnostic_information(e) <<
", errno=" << errno << std::endl;
519 case RequestMode::Single:
521 case RequestMode::Buffer:
523 case RequestMode::Window:
525 case RequestMode::Ignored:
537 data_thread_running_ =
true;
542 TLOG(TLVL_DEBUG) <<
"getDataLoop: isHardwareOK is " << isHardwareOK_ <<
", aborting data thread";
543 data_thread_running_ =
false;
547 TLOG(TLVL_GETDATALOOP) <<
"getDataLoop: calling getNext_";
550 auto startdata = std::chrono::steady_clock::now();
551 FragmentPtrs newData;
555 data = getNext_(newData);
559 ExceptionHandler(ExceptionHandlerRethrow::no,
560 "Exception thrown by fragment generator in CommandableFragmentGenerator::getDataLoop; setting exception state to \"true\"");
563 data_thread_running_ =
false;
569 metricMan->sendMetric(
"Avg Data Acquisition Time", TimeUtils::GetElapsedTime(startdata),
"s", 3, artdaq::MetricMode::Average);
572 if (newData.size() == 0 && sleep_on_no_data_us_ > 0)
574 usleep(sleep_on_no_data_us_);
577 auto dataIter = newData.begin();
578 while (dataIter != newData.end())
580 TLOG(TLVL_GETDATALOOP_VERBOSE) <<
"getDataLoop: getNext_() returned fragment with timestamp = " << (*dataIter)->timestamp() <<
", and sizeBytes = " << (*dataIter)->sizeBytes();
582 auto frag_id = (*dataIter)->fragmentID();
583 if (!dataBuffers_.count(frag_id))
585 throw cet::exception(
"DataBufferError") <<
"Error in CommandableFragmentGenerator: Recevied Fragment with fragment_id " << frag_id <<
", but this ID was not declared in fragment_ids!";
588 TLOG(TLVL_GETDATALOOP_DATABUFFWAIT) <<
"Waiting for data buffer ready";
589 if (!waitForDataBufferReady(frag_id))
return;
590 TLOG(TLVL_GETDATALOOP_DATABUFFWAIT) <<
"Done waiting for data buffer ready";
592 TLOG(TLVL_GETDATALOOP) <<
"getDataLoop: processing data";
593 if (data && !force_stop_)
595 std::unique_lock<std::mutex> lock(dataBuffersMutex_);
598 case RequestMode::Single:
599 dataBuffers_[frag_id].DataBuffer.clear();
600 dataBuffers_[frag_id].DataBufferDepthBytes = (*dataIter)->sizeBytes();
601 dataBuffers_[frag_id].DataBuffer.emplace_back(std::move(*dataIter));
602 dataIter = newData.erase(dataIter);
604 case RequestMode::Buffer:
605 case RequestMode::Ignored:
606 case RequestMode::Window:
609 dataBuffers_[frag_id].DataBufferDepthBytes += (*dataIter)->sizeBytes();
610 dataBuffers_[frag_id].DataBuffer.emplace_back(std::move(*dataIter));
611 dataIter = newData.erase(dataIter);
614 getDataBufferStats(frag_id);
623 std::unique_lock<std::mutex> lock(dataBuffersMutex_);
624 for (
auto&
id : dataBuffers_)
626 if (
id.second.DataBuffer.size() > 0)
628 dataCondition_.notify_all();
633 if (!data || force_stop_)
635 TLOG(TLVL_INFO) <<
"Data flow has stopped. Ending data collection thread";
636 std::unique_lock<std::mutex> lock(dataBuffersMutex_);
637 data_thread_running_ =
false;
638 if (requestReceiver_) requestReceiver_->ClearRequests();
640 TLOG(TLVL_INFO) <<
"getDataLoop: Ending thread";
648 if (!dataBuffers_.count(
id))
650 throw cet::exception(
"DataBufferError") <<
"Error in CommandableFragmentGenerator: Cannot wait for data buffer for ID " <<
id <<
" because it does not exist!";
652 auto startwait = std::chrono::steady_clock::now();
654 auto lastwaittime = 0ULL;
657 std::unique_lock<std::mutex> lock(dataBuffersMutex_);
658 getDataBufferStats(
id);
661 while (dataBufferIsTooLarge(
id))
663 if (!circularDataBufferMode_)
667 TLOG(TLVL_DEBUG) <<
"Run ended while waiting for buffer to shrink!";
668 std::unique_lock<std::mutex> lock(dataBuffersMutex_);
669 getDataBufferStats(
id);
670 dataCondition_.notify_all();
671 data_thread_running_ =
false;
674 auto waittime = TimeUtils::GetElapsedTimeMilliseconds(startwait);
676 if (first || (waittime != lastwaittime && waittime % 1000 == 0))
678 TLOG(TLVL_WARNING) <<
"Bad Omen: Data Buffer has exceeded its size limits. "
679 <<
"(seq_id=" << ev_counter() <<
", frag_id=" <<
id
680 <<
", frags=" << dataBuffers_[id].DataBufferDepthFragments <<
"/" << maxDataBufferDepthFragments_
681 <<
", szB=" << dataBuffers_[id].DataBufferDepthBytes <<
"/" << maxDataBufferDepthBytes_ <<
")";
682 TLOG(TLVL_TRACE) <<
"Bad Omen: Possible causes include requests not getting through or Ignored-mode BR issues";
685 if (waittime % 5 && waittime != lastwaittime)
687 TLOG(TLVL_WAITFORBUFFERREADY) <<
"getDataLoop: Data Retreival paused for " << waittime <<
" ms waiting for data buffer to drain";
689 lastwaittime = waittime;
694 std::unique_lock<std::mutex> lock(dataBuffersMutex_);
695 getDataBufferStats(
id);
696 if (dataBufferIsTooLarge(
id))
698 auto begin = dataBuffers_[id].DataBuffer.begin();
699 if (begin == dataBuffers_[
id].DataBuffer.end())
701 TLOG(TLVL_WARNING) <<
"Data buffer is reported as too large, but doesn't contain any Fragments! Possible corrupt memory!";
706 TLOG(TLVL_WAITFORBUFFERREADY) <<
"waitForDataBufferReady: Dropping Fragment with timestamp " << (*begin)->timestamp() <<
" from data buffer (Buffer over-size, circular data buffer mode)";
708 dataBuffers_[id].DataBufferDepthBytes -= (*begin)->sizeBytes();
709 dataBuffers_[id].DataBuffer.erase(begin);
710 getDataBufferStats(
id);
719 if (!dataBuffers_.count(
id))
721 throw cet::exception(
"DataBufferError") <<
"Error in CommandableFragmentGenerator: Cannot check size of data buffer for ID " <<
id <<
" because it does not exist!";
723 return (maxDataBufferDepthFragments_ > 0 && dataBuffers_[
id].DataBufferDepthFragments > maxDataBufferDepthFragments_) || (maxDataBufferDepthBytes_ > 0 && dataBuffers_[id].DataBufferDepthBytes > maxDataBufferDepthBytes_);
728 if (!dataBuffers_.count(
id))
730 throw cet::exception(
"DataBufferError") <<
"Error in CommandableFragmentGenerator: Cannot get stats of data buffer for ID " <<
id <<
" because it does not exist!";
733 dataBuffers_[id].DataBufferDepthFragments = dataBuffers_[id].DataBuffer.size();
737 TLOG(TLVL_GETBUFFERSTATS) <<
"getDataBufferStats: Sending Metrics";
738 metricMan->sendMetric(
"Buffer Depth Fragments", dataBuffers_[
id].DataBufferDepthFragments.load(),
"fragments", 1, MetricMode::LastPoint);
739 metricMan->sendMetric(
"Buffer Depth Bytes", dataBuffers_[
id].DataBufferDepthBytes.load(),
"bytes", 1, MetricMode::LastPoint);
741 TLOG(TLVL_GETBUFFERSTATS) <<
"getDataBufferStats: frags=" << dataBuffers_[id].DataBufferDepthFragments.load() <<
"/" << maxDataBufferDepthFragments_
742 <<
", sz=" << dataBuffers_[id].DataBufferDepthBytes.load() <<
"/" << maxDataBufferDepthBytes_;
747 if (!dataBuffers_.count(
id))
749 throw cet::exception(
"DataBufferError") <<
"Error in CommandableFragmentGenerator: Cannot check data buffer for ID " <<
id <<
" because it does not exist!";
752 if (dataBuffers_[
id].DataBufferDepthFragments > 0 && mode_ != RequestMode::Single && mode_ != RequestMode::Ignored)
755 getDataBufferStats(
id);
756 while (dataBufferIsTooLarge(
id))
758 auto begin = dataBuffers_[id].DataBuffer.begin();
759 TLOG(TLVL_CHECKDATABUFFER) <<
"checkDataBuffer: Dropping Fragment with timestamp " << (*begin)->timestamp() <<
" from data buffer (Buffer over-size)";
760 dataBuffers_[id].DataBufferDepthBytes -= (*begin)->sizeBytes();
761 dataBuffers_[id].DataBuffer.erase(begin);
762 getDataBufferStats(
id);
764 if (dataBuffers_[
id].DataBuffer.size() > 0)
766 TLOG(TLVL_CHECKDATABUFFER) <<
"Determining if Fragments can be dropped from data buffer";
767 Fragment::timestamp_t last = dataBuffers_[id].DataBuffer.back()->timestamp();
768 Fragment::timestamp_t min = last > staleTimeout_ ? last - staleTimeout_ : 0;
769 for (
auto it = dataBuffers_[
id].DataBuffer.begin(); it != dataBuffers_[id].DataBuffer.end();)
771 if ((*it)->timestamp() < min)
773 TLOG(TLVL_CHECKDATABUFFER) <<
"checkDataBuffer: Dropping Fragment with timestamp " << (*it)->timestamp() <<
" from data buffer (timeout=" << staleTimeout_ <<
", min=" << min <<
")";
774 dataBuffers_[id].DataBufferDepthBytes -= (*it)->sizeBytes();
775 it = dataBuffers_[id].DataBuffer.erase(it);
782 getDataBufferStats(
id);
791 if (should_stop() || monitoringInterval_ <= 0)
793 TLOG(TLVL_DEBUG) <<
"getMonitoringDataLoop: should_stop() is " << std::boolalpha << should_stop()
794 <<
" and monitoringInterval is " << monitoringInterval_ <<
", returning";
797 TLOG(TLVL_GETMONITORINGDATA) <<
"getMonitoringDataLoop: Determining whether to call checkHWStatus_";
799 auto now = std::chrono::steady_clock::now();
800 if (TimeUtils::GetElapsedTimeMicroseconds(lastMonitoringCall_, now) >= static_cast<size_t>(monitoringInterval_))
802 isHardwareOK_ = checkHWStatus_();
803 TLOG(TLVL_GETMONITORINGDATA) <<
"getMonitoringDataLoop: isHardwareOK_ is now " << std::boolalpha << isHardwareOK_;
804 lastMonitoringCall_ = now;
806 usleep(monitoringInterval_ / 10);
814 TLOG(TLVL_APPLYREQUESTS) <<
"Mode is Ignored; Copying data to output";
815 for (
auto&
id : dataBuffers_)
817 std::move(
id.second.DataBuffer.begin(),
id.second.DataBuffer.end(), std::inserter(frags, frags.end()));
818 id.second.DataBufferDepthBytes = 0;
819 id.second.DataBufferDepthFragments = 0;
820 id.second.DataBuffer.clear();
827 auto requests = requestReceiver_->GetRequests();
828 while (requests.size() > 1)
831 requestReceiver_->RemoveRequest(requests.begin()->first);
832 requests.erase(requests.begin());
834 sendEmptyFragments(frags, requests);
837 if (requests.size() == 0 || !requests.count(ev_counter()))
return;
839 for (
auto&
id : dataBuffers_)
841 if (
id.second.DataBuffer.size() > 0)
843 assert(
id.second.DataBuffer.size() == 1);
844 TLOG(TLVL_APPLYREQUESTS) <<
"Mode is Single; Sending copy of last event";
845 for (
auto& fragptr :
id.second.DataBuffer)
848 auto frag = fragptr.get();
849 auto newfrag = std::unique_ptr<artdaq::Fragment>(
new Fragment(ev_counter(), frag->fragmentID()));
850 newfrag->resize(frag->size() - detail::RawFragmentHeader::num_words());
851 memcpy(newfrag->headerAddress(), frag->headerAddress(), frag->sizeBytes());
852 newfrag->setTimestamp(requests[ev_counter()]);
853 newfrag->setSequenceID(ev_counter());
854 frags.push_back(std::move(newfrag));
859 sendEmptyFragment(frags, ev_counter(),
id.first,
"No data for");
862 requestReceiver_->RemoveRequest(ev_counter());
863 ev_counter_inc(1,
true);
869 auto requests = requestReceiver_->GetRequests();
870 while (requests.size() > 1)
873 requestReceiver_->RemoveRequest(requests.begin()->first);
874 requests.erase(requests.begin());
876 sendEmptyFragments(frags, requests);
879 if (requests.size() == 0 || !requests.count(ev_counter()))
return;
881 for (
auto&
id : dataBuffers_)
883 TLOG(TLVL_DEBUG) <<
"Creating ContainerFragment for Buffered Fragments";
884 frags.emplace_back(
new artdaq::Fragment(ev_counter(),
id.first));
885 frags.back()->setTimestamp(requests[ev_counter()]);
886 ContainerFragmentLoader cfl(*frags.back());
887 cfl.set_missing_data(
false);
891 for (
auto it =
id.second.DataBuffer.begin(); it !=
id.second.DataBuffer.end();)
893 TLOG(TLVL_APPLYREQUESTS) <<
"ApplyRequests: Adding Fragment with timestamp " << (*it)->timestamp() <<
" to Container with sequence ID " << ev_counter();
894 cfl.addFragment(*it);
895 id.second.DataBufferDepthBytes -= (*it)->sizeBytes();
896 it =
id.second.DataBuffer.erase(it);
899 requestReceiver_->RemoveRequest(ev_counter());
900 ev_counter_inc(1,
true);
905 TLOG(TLVL_APPLYREQUESTS) <<
"applyRequestsWindowMode_CheckAndFillDataBuffer: Checking that data exists for request window " << seq;
906 Fragment::timestamp_t min = ts > windowOffset_ ? ts - windowOffset_ : 0;
907 Fragment::timestamp_t max = min + windowWidth_;
908 TLOG(TLVL_APPLYREQUESTS) <<
"ApplyRequestsWindowsMode_CheckAndFillDataBuffer: min is " << min <<
", max is " << max
909 <<
" and first/last points in buffer are " << (dataBuffers_[id].DataBuffer.size() > 0 ? dataBuffers_[id].DataBuffer.front()->timestamp() : 0)
910 <<
"/" << (dataBuffers_[
id].DataBuffer.size() > 0 ? dataBuffers_[id].DataBuffer.back()->timestamp() : 0)
911 <<
" (sz=" << dataBuffers_[
id].DataBuffer.size() <<
" [" << dataBuffers_[id].DataBufferDepthBytes.load()
912 <<
"/" << maxDataBufferDepthBytes_ <<
"])";
913 bool windowClosed = dataBuffers_[id].DataBuffer.size() > 0 && dataBuffers_[id].DataBuffer.back()->timestamp() >= max;
914 bool windowTimeout = !windowClosed && TimeUtils::GetElapsedTimeMicroseconds(requestReceiver_->GetRequestTime(seq)) > window_close_timeout_us_;
917 TLOG(TLVL_WARNING) <<
"applyRequestsWindowMode_CheckAndFillDataBuffer: A timeout occurred waiting for data to close the request window ({" << min <<
"-" << max
918 <<
"}, buffer={" << (dataBuffers_[id].DataBuffer.size() > 0 ? dataBuffers_[id].DataBuffer.front()->timestamp() : 0) <<
"-"
919 << (dataBuffers_[
id].DataBuffer.size() > 0 ? dataBuffers_[id].DataBuffer.back()->timestamp() : 0)
920 <<
"} ). Time waiting: "
921 << TimeUtils::GetElapsedTimeMicroseconds(requestReceiver_->GetRequestTime(seq)) <<
" us "
922 <<
"(> " << window_close_timeout_us_ <<
" us).";
924 if (windowClosed || !data_thread_running_ || windowTimeout)
926 TLOG(TLVL_DEBUG) <<
"applyRequestsWindowMode_CheckAndFillDataBuffer: Creating ContainerFragment for Window-requested Fragments";
927 frags.emplace_back(
new artdaq::Fragment(seq,
id));
928 frags.back()->setTimestamp(ts);
929 ContainerFragmentLoader cfl(*frags.back());
943 if (!windowClosed || (dataBuffers_[
id].DataBuffer.size() > 0 && dataBuffers_[id].DataBuffer.front()->timestamp() > min))
945 TLOG(TLVL_DEBUG) <<
"applyRequestsWindowMode_CheckAndFillDataBuffer: Request window starts before and/or ends after the current data buffer, setting ContainerFragment's missing_data flag!"
946 <<
" (requestWindowRange=[" << min <<
"," << max <<
"], "
947 <<
"buffer={" << (dataBuffers_[id].DataBuffer.size() > 0 ? dataBuffers_[id].DataBuffer.front()->timestamp() : 0) <<
"-"
948 << (dataBuffers_[
id].DataBuffer.size() > 0 ? dataBuffers_[id].DataBuffer.back()->timestamp() : 0) <<
"}";
949 cfl.set_missing_data(
true);
953 for (
auto it = dataBuffers_[
id].DataBuffer.begin(); it != dataBuffers_[id].DataBuffer.end();)
955 Fragment::timestamp_t fragT = (*it)->timestamp();
956 if (fragT < min || fragT > max || (fragT == max && windowWidth_ > 0))
962 TLOG(TLVL_APPLYREQUESTS) <<
"applyRequestsWindowMode_CheckAndFillDataBuffer: Adding Fragment with timestamp " << (*it)->timestamp() <<
" to Container";
963 cfl.addFragment(*it);
967 dataBuffers_[id].DataBufferDepthBytes -= (*it)->sizeBytes();
968 it = dataBuffers_[id].DataBuffer.erase(it);
976 dataBuffers_[id].WindowsSent[seq] = std::chrono::steady_clock::now();
977 if (seq > dataBuffers_[
id].HighestRequestSeen) dataBuffers_[id].HighestRequestSeen = seq;
983 TLOG(TLVL_APPLYREQUESTS) <<
"applyRequestsWindowMode BEGIN";
985 auto requests = requestReceiver_->GetRequests();
987 TLOG(TLVL_APPLYREQUESTS) <<
"applyRequestsWindowMode: Starting request processing";
988 for (
auto req = requests.begin(); req != requests.end();)
990 TLOG(TLVL_APPLYREQUESTS) <<
"applyRequestsWindowMode: processing request with sequence ID " << req->first <<
", timestamp " << req->second;
992 while (req->first < ev_counter() && requests.size() > 0)
994 TLOG(TLVL_APPLYREQUESTS) <<
"applyRequestsWindowMode: Clearing passed request for sequence ID " << req->first;
995 requestReceiver_->RemoveRequest(req->first);
996 req = requests.erase(req);
998 if (requests.size() == 0)
break;
1000 for (
auto&
id : dataBuffers_)
1002 if (!
id.second.WindowsSent.count(req->first))
1004 applyRequestsWindowMode_CheckAndFillDataBuffer(frags,
id.first, req->first, req->second);
1007 checkSentWindows(req->first);
1012 for (
auto&
id : dataBuffers_)
1014 std::set<artdaq::Fragment::sequence_id_t> seqs;
1015 for (
auto& seq :
id.second.WindowsSent)
1017 seqs.insert(seq.first);
1019 for (
auto& seq : seqs)
1021 checkSentWindows(seq);
1028 if (check_stop() || exception())
1034 if (mode_ == RequestMode::Ignored)
1036 while (dataBufferFragmentCount_() == 0)
1038 if (check_stop() || exception() || !isHardwareOK_)
return false;
1039 std::unique_lock<std::mutex> lock(dataBuffersMutex_);
1040 dataCondition_.wait_for(lock, std::chrono::milliseconds(10), [
this]() {
return dataBufferFragmentCount_() > 0; });
1045 if ((check_stop() && requestReceiver_->size() == 0) || exception())
return false;
1047 std::unique_lock<std::mutex> lock(dataBuffersMutex_);
1048 dataCondition_.wait_for(lock, std::chrono::milliseconds(10));
1055 while (requestReceiver_->size() == 0 && counter < 100)
1057 if (check_stop() || exception())
return false;
1061 requestReceiver_->WaitForRequests(10);
1067 std::unique_lock<std::mutex> dlk(dataBuffersMutex_);
1071 case RequestMode::Single:
1072 applyRequestsSingleMode(frags);
1074 case RequestMode::Window:
1075 applyRequestsWindowMode(frags);
1077 case RequestMode::Buffer:
1078 applyRequestsBufferMode(frags);
1080 case RequestMode::Ignored:
1082 applyRequestsIgnoredMode(frags);
1086 if (!data_thread_running_ || force_stop_)
1088 TLOG(TLVL_INFO) <<
"Data thread has stopped; Clearing data buffers";
1089 for (
auto&
id : dataBuffers_)
1091 id.second.DataBufferDepthBytes = 0;
1092 id.second.DataBufferDepthFragments = 0;
1093 id.second.DataBuffer.clear();
1097 getDataBuffersStats();
1100 if (frags.size() > 0)
1101 TLOG(TLVL_APPLYREQUESTS) <<
"Finished Processing requests, returning " << frags.size() <<
" fragments, current ev_counter is " << ev_counter();
1107 TLOG(TLVL_WARNING) << desc <<
" sequence ID " << seqId <<
", sending empty fragment";
1108 auto frag =
new Fragment();
1109 frag->setSequenceID(seqId);
1110 frag->setFragmentID(fragmentId);
1111 frag->setSystemType(Fragment::EmptyFragmentType);
1112 frags.emplace_back(FragmentPtr(frag));
1118 if (requests.size() > 0)
1120 TLOG(TLVL_SENDEMPTYFRAGMENTS) <<
"Sending Empty Fragments for Sequence IDs from " << ev_counter() <<
" up to but not including " << requests.begin()->first;
1121 while (requests.begin()->first > ev_counter())
1123 for (
auto& fid : dataBuffers_)
1125 sendEmptyFragment(frags, ev_counter(), fid.first,
"Missed request for");
1127 ev_counter_inc(1,
true);
1134 bool seqComplete =
true;
1135 bool seqTimeout =
false;
1136 for (
auto&
id : dataBuffers_)
1138 if (!
id.second.WindowsSent.count(seq) ||
id.second.HighestRequestSeen < seq)
1140 seqComplete =
false;
1142 if (
id.second.WindowsSent.count(seq) && TimeUtils::GetElapsedTimeMicroseconds(
id.second.WindowsSent[seq]) > missing_request_window_timeout_us_)
1149 TLOG(TLVL_CHECKWINDOWS) <<
"checkSentWindows: Request " << seq <<
" is complete, removing from requestReceiver.";
1150 requestReceiver_->RemoveRequest(seq);
1152 if (ev_counter() == seq)
1154 TLOG(TLVL_CHECKWINDOWS) <<
"checkSentWindows: Sequence ID matches ev_counter, incrementing ev_counter (" << ev_counter() <<
")";
1156 for (
auto&
id : dataBuffers_)
1158 id.second.WindowsSent.erase(seq);
1161 ev_counter_inc(1,
true);
1166 TLOG(TLVL_CHECKWINDOWS) <<
"checkSentWindows: Sent Window history indicates that requests between " << ev_counter() <<
" and " << seq <<
" have timed out.";
1167 while (ev_counter() <= seq)
1169 if (ev_counter() < seq) TLOG(TLVL_WARNING) <<
"Missed request for sequence ID " << ev_counter() <<
"! Will not send any data for this sequence ID!";
1170 requestReceiver_->RemoveRequest(ev_counter());
1172 for (
auto&
id : dataBuffers_)
1174 id.second.WindowsSent.erase(ev_counter());
1177 ev_counter_inc(1,
true);
CommandableFragmentGenerator(const fhicl::ParameterSet &ps)
CommandableFragmentGenerator Constructor.
void applyRequestsSingleMode(artdaq::FragmentPtrs &frags)
Create fragments using data buffer for request mode Single. Precondition: dataBufferMutex_ and reques...
virtual bool checkHWStatus_()
Check any relavent hardware status registers. Return false if an error condition exists that should h...
virtual ~CommandableFragmentGenerator()
CommandableFragmentGenerator Destructor.
void applyRequestsBufferMode(artdaq::FragmentPtrs &frags)
Create fragments using data buffer for request mode Buffer. Precondition: dataBufferMutex_ and reques...
void getDataBufferStats(Fragment::fragment_id_t id)
Calculate the size of the dataBuffer and report appropriate metrics.
void getMonitoringDataLoop()
This function regularly calls checkHWStatus_(), and sets the isHardwareOK flag accordingly.
void startDataThread()
Function that launches the data thread (getDataLoop())
artdaq::Fragment::fragment_id_t fragment_id() const
Get the Fragment ID of this Fragment generator.
std::string ReportCmd(std::string const &which="")
Get a report about a user-specified run-time quantity.
virtual bool metaCommand(std::string const &command, std::string const &arg)
The meta-command is used for implementing user-specific commands in a CommandableFragmentGenerator.
void StopCmd(uint64_t timeout, uint64_t timestamp)
Stop the CommandableFragmentGenerator.
void applyRequestsWindowMode_CheckAndFillDataBuffer(artdaq::FragmentPtrs &frags, artdaq::Fragment::fragment_id_t id, artdaq::Fragment::sequence_id_t seq, artdaq::Fragment::timestamp_t ts)
Copy data from the relevant data buffer that matches the given timestamp.
void checkSentWindows(Fragment::sequence_id_t seq)
Check the windows_sent_ooo_ map for sequence IDs that may be removed.
bool sendEmptyFragment(FragmentPtrs &frags, size_t sequenceId, Fragment::fragment_id_t fragmentId, std::string desc)
Send an EmptyFragmentType Fragment.
void applyRequestsWindowMode(artdaq::FragmentPtrs &frags)
Create fragments using data buffer for request mode Window. Precondition: dataBufferMutex_ and reques...
void StartCmd(int run, uint64_t timeout, uint64_t timestamp)
Start the CommandableFragmentGenerator.
virtual void pauseNoMutex()
On call to PauseCmd, pauseNoMutex() is called prior to PauseCmd acquiring the mutex ...
bool check_stop()
Routine used by applyRequests to make sure that all outstanding requests have been fulfilled before r...
void ResumeCmd(uint64_t timeout, uint64_t timestamp)
Resume the CommandableFragmentGenerator.
bool getNext(FragmentPtrs &output) overridefinal
getNext calls either applyRequests or getNext_ to get any data that is ready to be sent to the EventB...
size_t ev_counter_inc(size_t step=1, bool force=false)
Increment the event counter, if the current RequestMode allows it.
bool applyRequests(FragmentPtrs &frags)
See if any requests have been received, and add the corresponding data Fragment objects to the output...
void applyRequestsIgnoredMode(artdaq::FragmentPtrs &frags)
Create fragments using data buffer for request mode Ignored. Precondition: dataBufferMutex_ and reque...
void PauseCmd(uint64_t timeout, uint64_t timestamp)
Pause the CommandableFragmentGenerator.
void getDataLoop()
When separate_data_thread is set to true, this loop repeatedly calls getNext_ and adds returned Fragm...
void sendEmptyFragments(FragmentPtrs &frags, std::map< Fragment::sequence_id_t, Fragment::timestamp_t > &requests)
This function is for Buffered and Single request modes, as they can only respond to one data request ...
Receive data requests and make them available to CommandableFragmentGenerator or other interested par...
void startMonitoringThread()
Function that launches the monitoring thread (getMonitoringDataLoop())
virtual void pause()
If a CommandableFragmentGenerator subclass is reading from hardware, the implementation of pause() sh...
virtual void resume()
The subrun number will be incremented before a call to resume.
bool waitForDataBufferReady(Fragment::fragment_id_t id)
Wait for the data buffer to drain (dataBufferIsTooLarge returns false), periodically reporting status...
virtual std::string report()
Let's say that the contract with the report() functions is that they return a non-empty string if the...
std::string printMode_()
Return the string representation of the current RequestMode.
void checkDataBuffer(Fragment::fragment_id_t id)
Perform data buffer pruning operations for the given buffer. If the RequestMode is Single...
virtual std::string reportSpecific(std::string const &what)
Report the status of a specific quantity
bool dataBufferIsTooLarge(Fragment::fragment_id_t id)
Test the configured constraints on the data buffer.
void joinThreads()
Join any data-taking threads. Should be called when destructing CommandableFragmentGenerator.