1 #define TRACE_NAME "CommandableFragmentGenerator"
4 #include "artdaq/Application/CommandableFragmentGenerator.hh"
6 #include <boost/exception/all.hpp>
7 #include <boost/throw_exception.hpp>
12 #include "canvas/Utilities/Exception.h"
13 #include "cetlib_except/exception.h"
14 #include "fhiclcpp/ParameterSet.h"
16 #include "artdaq-core/Utilities/SimpleLookupPolicy.hh"
17 #include "artdaq-core/Data/Fragment.hh"
18 #include "artdaq-core/Data/ContainerFragmentLoader.hh"
19 #include "artdaq-core/Utilities/ExceptionHandler.hh"
20 #include "artdaq-core/Utilities/TimeUtils.hh"
34 , request_addr_(
"227.128.12.26")
36 , request_stop_requested_(false)
37 , request_received_(false)
38 , end_of_run_timeout_ms_(1000)
41 , staleTimeout_(Fragment::InvalidTimestamp)
42 , expectedType_(Fragment::EmptyFragmentType)
43 , maxFragmentCount_(std::numeric_limits<size_t>::max())
44 , uniqueWindows_(true)
45 , missing_request_(true)
46 , missing_request_time_()
47 , last_window_send_time_()
48 , last_window_send_time_set_(false)
49 , missing_request_window_timeout_us_(1000000)
50 , window_close_timeout_us_(2000000)
51 , useDataThread_(false)
52 , sleep_on_no_data_us_(0)
53 , data_thread_running_(false)
54 , dataBufferDepthFragments_(0)
55 , dataBufferDepthBytes_(0)
56 , maxDataBufferDepthFragments_(1000)
57 , maxDataBufferDepthBytes_(1000)
58 , useMonitoringThread_(false)
59 , monitoringInterval_(0)
60 , lastMonitoringCall_()
66 , timeout_(std::numeric_limits<uint64_t>::max())
67 , timestamp_(std::numeric_limits<uint64_t>::max())
71 , latest_exception_report_(
"none")
74 , instance_name_for_metrics_(
"FragmentGenerator")
75 , sleep_on_stop_us_(0)
80 , request_port_(ps.get<int>(
"request_port", 3001))
81 , request_addr_(ps.get<std::string>(
"request_address",
"227.128.12.26"))
83 , request_stop_requested_(false)
84 , request_received_(false)
85 , end_of_run_timeout_ms_(ps.get<size_t>(
"end_of_run_quiet_timeout_ms", 1000))
86 , windowOffset_(ps.get<Fragment::timestamp_t>(
"request_window_offset", 0))
87 , windowWidth_(ps.get<Fragment::timestamp_t>(
"request_window_width", 0))
88 , staleTimeout_(ps.get<Fragment::timestamp_t>(
"stale_request_timeout", 0xFFFFFFFF))
89 , expectedType_(ps.get<Fragment::type_t>(
"expected_fragment_type", Fragment::type_t(Fragment::EmptyFragmentType)))
90 , uniqueWindows_(ps.get<bool>(
"request_windows_are_unique", true))
91 , missing_request_(false)
92 , missing_request_time_(decltype(missing_request_time_)::max())
93 , last_window_send_time_(decltype(last_window_send_time_)::max())
94 , last_window_send_time_set_(false)
95 , missing_request_window_timeout_us_(ps.get<size_t>(
"missing_request_window_timeout_us", 1000000))
96 , window_close_timeout_us_(ps.get<size_t>(
"window_close_timeout_us", 2000000))
97 , useDataThread_(ps.get<bool>(
"separate_data_thread", false))
98 , sleep_on_no_data_us_(ps.get<size_t>(
"sleep_on_no_data_us", 0))
99 , data_thread_running_(false)
100 , dataBufferDepthFragments_(0)
101 , dataBufferDepthBytes_(0)
102 , maxDataBufferDepthFragments_(ps.get<int>(
"data_buffer_depth_fragments", 1000))
103 , maxDataBufferDepthBytes_(ps.get<size_t>(
"data_buffer_depth_mb", 1000) * 1024 * 1024)
104 , useMonitoringThread_(ps.get<bool>(
"separate_monitoring_thread", false))
105 , monitoringInterval_(ps.get<int64_t>(
"hardware_poll_interval_us", 0))
106 , lastMonitoringCall_()
107 , isHardwareOK_(true)
112 , timeout_(std::numeric_limits<uint64_t>::max())
113 , timestamp_(std::numeric_limits<uint64_t>::max())
114 , should_stop_(false)
117 , latest_exception_report_(
"none")
120 , sleep_on_stop_us_(0)
122 board_id_ = ps.get<
int>(
"board_id");
123 instance_name_for_metrics_ =
"BoardReader." + boost::lexical_cast<std::string>(board_id_);
125 fragment_ids_ = ps.get<std::vector<artdaq::Fragment::fragment_id_t>>(
"fragment_ids", std::vector<artdaq::Fragment::fragment_id_t>());
127 TLOG_TRACE(
"CommandableFragmentGenerator") <<
"artdaq::CommandableFragmentGenerator::CommandableFragmentGenerator(ps)" << TLOG_ENDL;
130 if (fragment_id != -99)
132 if (fragment_ids_.size() != 0)
134 latest_exception_report_ =
"Error in CommandableFragmentGenerator: can't both define \"fragment_id\" and \"fragment_ids\" in FHiCL document";
135 throw cet::exception(latest_exception_report_);
139 fragment_ids_.emplace_back(fragment_id);
143 sleep_on_stop_us_ = ps.get<
int>(
"sleep_on_stop_us", 0);
145 dataBuffer_.emplace_back(FragmentPtr(
new Fragment()));
146 (*dataBuffer_.begin())->setSystemType(Fragment::EmptyFragmentType);
148 std::string modeString = ps.get<std::string>(
"request_mode",
"ignored");
149 if (modeString ==
"single" || modeString ==
"Single")
151 mode_ = RequestMode::Single;
153 else if (modeString.find(
"buffer") != std::string::npos || modeString.find(
"Buffer") != std::string::npos)
155 mode_ = RequestMode::Buffer;
157 else if (modeString ==
"window" || modeString ==
"Window")
159 mode_ = RequestMode::Window;
161 else if (modeString.find(
"ignore") != std::string::npos || modeString.find(
"Ignore") != std::string::npos)
163 mode_ = RequestMode::Ignored;
165 TLOG_DEBUG(
"CommandableFragmentGenerator") <<
"Request mode is " <<
printMode_() << TLOG_ENDL;
167 if (mode_ != RequestMode::Ignored)
171 latest_exception_report_ =
"Error in CommandableFragmentGenerator: use_data_thread must be true when request_mode is not \"Ignored\"!";
172 throw cet::exception(latest_exception_report_);
180 request_socket_ = socket(AF_INET, SOCK_DGRAM, IPPROTO_UDP);
181 if (request_socket_ < 0)
183 throw art::Exception(art::errors::Configuration) <<
"CommandableFragmentGenerator: Error creating socket for receiving data requests! err=" << strerror(errno) << std::endl;
187 struct sockaddr_in si_me_request;
190 if (setsockopt(request_socket_, SOL_SOCKET, SO_REUSEADDR, &yes,
sizeof(yes)) < 0)
192 throw art::Exception(art::errors::Configuration) <<
193 "RequestedFragmentGenrator: Unable to enable port reuse on request socket, err=" << strerror(errno) << std::endl;
196 memset(&si_me_request, 0,
sizeof(si_me_request));
197 si_me_request.sin_family = AF_INET;
198 si_me_request.sin_port = htons(request_port_);
199 si_me_request.sin_addr.s_addr = htonl(INADDR_ANY);
200 if (bind(request_socket_, (
struct sockaddr *)&si_me_request,
sizeof(si_me_request)) == -1)
202 throw art::Exception(art::errors::Configuration) <<
203 "CommandableFragmentGenerator: Cannot bind request socket to port " << request_port_ <<
", err=" << strerror(errno) << std::endl;
207 if (request_addr_ !=
"localhost")
210 int sts =
ResolveHost(request_addr_.c_str(), mreq.imr_multiaddr);
213 throw art::Exception(art::errors::Configuration) <<
"Unable to resolve multicast request address, err=" << strerror(errno) << std::endl;
216 mreq.imr_interface.s_addr = htonl(INADDR_ANY);
217 if (setsockopt(request_socket_, IPPROTO_IP, IP_ADD_MEMBERSHIP, &mreq,
sizeof(mreq)) < 0)
219 throw art::Exception(art::errors::Configuration) <<
220 "CommandableFragmentGenerator: Unable to join multicast group, err=" << strerror(errno) << std::endl;
235 TLOG_DEBUG(
"CommandableFragmentGenerator") <<
"Joining dataThread" << TLOG_ENDL;
236 if (dataThread_.joinable()) dataThread_.join();
237 TLOG_DEBUG(
"CommandableFragmentGenerator") <<
"Joining monitoringThread" << TLOG_ENDL;
238 if (monitoringThread_.joinable()) monitoringThread_.join();
239 TLOG_DEBUG(
"CommandableFragmentGenerator") <<
"Joining requestThread" << TLOG_ENDL;
240 if (requestThread_.joinable()) requestThread_.join();
241 if (request_socket_ != -1) close(request_socket_);
248 if (check_stop()) usleep(sleep_on_stop_us_);
249 if (exception() || force_stop_)
return false;
251 if (!useMonitoringThread_ && monitoringInterval_ > 0)
253 TLOG_ARB(10,
"CommandableFragmentGenerator") <<
"getNext: Checking whether to collect Monitoring Data" << TLOG_ENDL;
254 auto now = std::chrono::steady_clock::now();
256 if (TimeUtils::GetElapsedTimeMicroseconds(lastMonitoringCall_, now) >= static_cast<size_t>(monitoringInterval_))
258 TLOG_ARB(10,
"CommandableFragmentGenerator") <<
"getNext: Collecting Monitoring Data" << TLOG_ENDL;
259 isHardwareOK_ = checkHWStatus_();
260 TLOG_ARB(10,
"CommandableFragmentGenerator") <<
"getNext: isHardwareOK_ is now " << std::boolalpha << isHardwareOK_ << TLOG_ENDL;
261 lastMonitoringCall_ = now;
267 std::lock_guard<std::mutex> lk(mutex_);
270 TLOG_TRACE(
"CommandableFragmentGenerator") <<
"getNext: Calling applyRequests" << TLOG_ENDL;
271 result = applyRequests(output);
272 TLOG_TRACE(
"CommandableFragmentGenerator") <<
"getNext: Done with applyRequests" << TLOG_ENDL;
276 throw cet::exception(
"CommandableFragmentGenerator") <<
"Exception found in BoardReader with board ID " << board_id() <<
"; BoardReader will now return error status when queried";
283 TLOG_ERROR(
"CommandableFragmentGenerator") <<
"Stopping CFG because the hardware reports bad status!" << TLOG_ENDL;
286 TLOG_TRACE(
"CommandableFragmentGenerator") <<
"getNext: Calling getNext_ " << std::to_string(ev_counter()) << TLOG_ENDL;
289 result = getNext_(output);
295 TLOG_TRACE(
"CommandableFragmentGenerator") <<
"getNext: Done with getNext_ " << std::to_string(ev_counter()) << TLOG_ENDL;
298 catch (
const cet::exception& e)
300 latest_exception_report_ =
"cet::exception caught in getNext(): ";
301 latest_exception_report_.append(e.what());
302 TLOG_ERROR(
"CommandableFragmentGenerator") <<
"getNext: cet::exception caught: " << e << TLOG_ENDL;
306 catch (
const boost::exception& e)
308 latest_exception_report_ =
"boost::exception caught in getNext(): ";
309 latest_exception_report_.append(boost::diagnostic_information(e));
310 TLOG_ERROR(
"CommandableFragmentGenerator") <<
"getNext: boost::exception caught: " << boost::diagnostic_information(e) << TLOG_ENDL;
314 catch (
const std::exception& e)
316 latest_exception_report_ =
"std::exception caught in getNext(): ";
317 latest_exception_report_.append(e.what());
318 TLOG_ERROR(
"CommandableFragmentGenerator") <<
"getNext: std::exception caught: " << e.what() << TLOG_ENDL;
324 latest_exception_report_ =
"Unknown exception caught in getNext().";
325 TLOG_ERROR(
"CommandableFragmentGenerator") <<
"getNext: unknown exception caught" << TLOG_ENDL;
332 TLOG_DEBUG(
"getNext") <<
"stopped " << TLOG_ENDL;
340 TLOG_ARB(14,
"CommandableFragmentGeneraotr") <<
"CFG::check_stop: should_stop=" << should_stop() <<
", useDataThread_=" << useDataThread_ <<
", requests_.size()=" << std::to_string(requests_.size()) <<
", exception status =" <<
int(exception()) << TLOG_ENDL;
342 if (!should_stop())
return false;
343 if (!useDataThread_ || mode_ == RequestMode::Ignored)
return true;
344 if (force_stop_)
return true;
346 if (!request_received_)
348 TLOG_ERROR(
"CommandableFragmentGenerator") <<
"Stop request received by request-based CommandableFragmentGenerator, but no requests have been received." << std::endl
349 <<
"Check that UDP port " << request_port_ <<
" is open in the firewall config." << TLOG_ENDL;
353 if (!request_stop_requested_)
return false;
355 return TimeUtils::GetElapsedTimeMilliseconds(request_stop_timeout_) > end_of_run_timeout_ms_;
360 if (fragment_ids_.size() != 1)
362 throw cet::exception(
"Error in CommandableFragmentGenerator: can't call fragment_id() unless member fragment_ids_ vector is length 1");
366 return fragment_ids_[0];
372 if (force || mode_ == RequestMode::Ignored)
374 return ev_counter_.fetch_add(step);
376 return ev_counter_.load();
381 if (run < 0)
throw cet::exception(
"CommandableFragmentGenerator") <<
"negative run number";
384 timestamp_ = timestamp;
385 ev_counter_.store(1);
386 should_stop_.store(
false);
387 exception_.store(
false);
390 latest_exception_report_ =
"none";
393 last_window_send_time_set_ =
false;
397 std::unique_lock<std::mutex> lk(mutex_);
398 if (useDataThread_) startDataThread();
399 if (useMonitoringThread_) startMonitoringThread();
400 if (mode_ != RequestMode::Ignored) startRequestReceiverThread();
405 TLOG_DEBUG(
"CommandableFragmentGenerator") <<
"Stop Command received." << TLOG_ENDL;
408 timestamp_ = timestamp;
411 should_stop_.store(
true);
412 std::unique_lock<std::mutex> lk(mutex_);
419 timestamp_ = timestamp;
422 should_stop_.store(
true);
423 std::unique_lock<std::mutex> lk(mutex_);
431 timestamp_ = timestamp;
434 should_stop_ =
false;
442 std::unique_lock<std::mutex> lk(mutex_);
443 if (useDataThread_) startDataThread();
444 if (useMonitoringThread_) startMonitoringThread();
445 if (mode_ != RequestMode::Ignored) startRequestReceiverThread();
450 std::lock_guard<std::mutex> lk(mutex_);
457 std::string childReport = reportSpecific(which);
458 if (childReport.length() > 0) {
return childReport; }
461 if (which ==
"latest_exception")
463 return latest_exception_report_;
467 childReport = report();
468 if (childReport.length() > 0) {
return childReport; }
471 std::string tmpString =
"The \"" + which +
"\" command is not ";
472 tmpString.append(
"currently supported by the ");
473 tmpString.append(metricsReportingInstanceName());
474 tmpString.append(
" fragment generator.");
479 void artdaq::CommandableFragmentGenerator::pauseNoMutex()
481 #pragma message "Using default implementation of CommandableFragmentGenerator::pauseNoMutex()"
484 void artdaq::CommandableFragmentGenerator::pause()
486 #pragma message "Using default implementation of CommandableFragmentGenerator::pause()"
489 void artdaq::CommandableFragmentGenerator::resume()
491 #pragma message "Using default implementation of CommandableFragmentGenerator::resume()"
494 std::string artdaq::CommandableFragmentGenerator::report()
496 #pragma message "Using default implementation of CommandableFragmentGenerator::report()"
500 std::string artdaq::CommandableFragmentGenerator::reportSpecific(std::string
const&)
502 #pragma message "Using default implementation of CommandableFragmentGenerator::reportSpecific(std::string)"
506 bool artdaq::CommandableFragmentGenerator::checkHWStatus_()
508 #pragma message "Using default implementation of CommandableFragmentGenerator::checkHWStatus_()"
514 if (dataThread_.joinable()) dataThread_.join();
515 TLOG_INFO(
"CommandableFragmentGenerator") <<
"Starting Data Receiver Thread" << TLOG_ENDL;
521 if (monitoringThread_.joinable()) monitoringThread_.join();
522 TLOG_INFO(
"CommandableFragmentGenerator") <<
"Starting Hardware Monitoring Thread" << TLOG_ENDL;
528 if (requestThread_.joinable()) requestThread_.join();
529 TLOG_INFO(
"CommandableFragmentGenerator") <<
"Starting Request Reception Thread" << TLOG_ENDL;
537 case RequestMode::Single:
539 case RequestMode::Buffer:
541 case RequestMode::Window:
543 case RequestMode::Ignored:
552 data_thread_running_ =
true;
557 TLOG_DEBUG(
"CommandableFragmentGenerator") <<
"getDataLoop: isHardwareOK is " << isHardwareOK_ <<
", aborting data thread" << TLOG_ENDL;
558 data_thread_running_ =
false;
562 TLOG_ARB(13,
"CommandableFragmentGenerator") <<
"getDataLoop: calling getNext_" << TLOG_ENDL;
565 auto startdata = std::chrono::steady_clock::now();
569 data = getNext_(newDataBuffer_);
573 ExceptionHandler(ExceptionHandlerRethrow::no,
574 "Exception thrown by fragment generator in CommandableFragmentGenerator::getDataLoop; setting exception state to \"true\"");
577 data_thread_running_ =
false;
583 metricMan->sendMetric(
"Avg Data Acquisition Time", TimeUtils::GetElapsedTime(startdata),
"s", 3, artdaq::MetricMode::Average);
586 if (newDataBuffer_.size() == 0 && sleep_on_no_data_us_ > 0)
588 usleep(sleep_on_no_data_us_);
591 TLOG_ARB(15,
"CommandableFragmentGenerator") <<
"Waiting for data buffer ready" << TLOG_ENDL;
592 if (!waitForDataBufferReady())
return;
593 TLOG_ARB(15,
"CommandableFragmentGenerator") <<
"Done waiting for data buffer ready" << TLOG_ENDL;
595 TLOG_ARB(13,
"CommandableFragmentGenerator") <<
"getDataLoop: processing data" << TLOG_ENDL;
596 if (data && !force_stop_)
598 std::unique_lock<std::mutex> lock(dataBufferMutex_);
601 case RequestMode::Single:
603 while (newDataBuffer_.size() >= fragment_ids_.size())
606 auto it = newDataBuffer_.begin();
607 std::advance(it, fragment_ids_.size());
608 dataBuffer_.splice(dataBuffer_.end(), newDataBuffer_, newDataBuffer_.begin(), it);
611 case RequestMode::Buffer:
612 case RequestMode::Ignored:
613 case RequestMode::Window:
616 dataBuffer_.splice(dataBuffer_.end(), newDataBuffer_);
619 getDataBufferStats();
623 std::unique_lock<std::mutex> lock(dataBufferMutex_);
624 if (dataBuffer_.size() > 0)
626 dataCondition_.notify_all();
629 if (!data || force_stop_)
631 TLOG_INFO(
"CommandableFragmentGenerator") <<
"Data flow has stopped. Ending data collection thread" << TLOG_ENDL;
632 data_thread_running_ =
false;
640 auto startwait = std::chrono::steady_clock::now();
642 auto lastwaittime = 0ULL;
643 while (dataBufferIsTooLarge())
647 TLOG_DEBUG(
"CommandableFragmentGenerator") <<
"Run ended while waiting for buffer to shrink!" << TLOG_ENDL;
648 std::unique_lock<std::mutex> lock(dataBufferMutex_);
649 getDataBufferStats();
650 dataCondition_.notify_all();
651 data_thread_running_ =
false;
654 auto waittime = TimeUtils::GetElapsedTimeMilliseconds(startwait);
656 if (first || (waittime != lastwaittime && waittime % 1000 == 0))
658 TLOG_WARNING(
"CommandableFragmentGenerator") <<
"Bad Omen: Data Buffer has exceeded its size limits. Check the connection between the BoardReader and the EventBuilders! (seq_id=" << ev_counter() <<
")" << TLOG_ENDL;
661 if (waittime % 5 && waittime != lastwaittime)
663 TLOG_ARB(13,
"CommandableFragmentGenerator") <<
"getDataLoop: Data Retreival paused for " << std::to_string(waittime) <<
" ms waiting for data buffer to drain" << TLOG_ENDL;
665 lastwaittime = waittime;
673 return (maxDataBufferDepthFragments_ > 0 && dataBufferDepthFragments_ > maxDataBufferDepthFragments_) || (maxDataBufferDepthBytes_ > 0 && dataBufferDepthBytes_ > maxDataBufferDepthBytes_);
679 dataBufferDepthFragments_ = dataBuffer_.size();
681 TLOG_ARB(15,
"CommandableFragmentGenerator") <<
"getDataBufferStats: Calculating buffer size" << TLOG_ENDL;
682 for (
auto i = dataBuffer_.begin(); i != dataBuffer_.end(); ++i)
684 if (i->get() !=
nullptr)
686 acc += (*i)->sizeBytes();
689 dataBufferDepthBytes_ = acc;
693 TLOG_ARB(15,
"CommandableFragmentGenerator") <<
"getDataBufferStats: Sending Metrics" << TLOG_ENDL;
694 metricMan->sendMetric(
"Buffer Depth Fragments", dataBufferDepthFragments_.load(),
"fragments", 1, MetricMode::LastPoint);
695 metricMan->sendMetric(
"Buffer Depth Bytes", dataBufferDepthBytes_.load(),
"bytes", 1, MetricMode::LastPoint);
697 TLOG_ARB(15,
"CommandableFragmentGenerator") <<
"getDataBufferStats: frags=" << dataBufferDepthFragments_.load() <<
"/" << maxDataBufferDepthFragments_
698 <<
", sz=" << std::to_string(dataBufferDepthBytes_.load()) <<
"/" << std::to_string(maxDataBufferDepthBytes_) << TLOG_ENDL;
703 std::unique_lock<std::mutex> lock(dataBufferMutex_);
704 dataCondition_.wait_for(lock, std::chrono::milliseconds(10));
705 if (dataBufferDepthFragments_ > 0)
707 if ((mode_ == RequestMode::Buffer || mode_ == RequestMode::Window))
710 while (dataBufferIsTooLarge())
712 dataBuffer_.erase(dataBuffer_.begin());
713 getDataBufferStats();
715 if (dataBuffer_.size() > 0)
717 TLOG_ARB(17,
"CommandableFragmentGenerator") <<
"Determining if Fragments can be dropped from data buffer" << TLOG_ENDL;
718 Fragment::timestamp_t last = dataBuffer_.back()->timestamp();
719 Fragment::timestamp_t min = last > staleTimeout_ ? last - staleTimeout_ : 0;
720 for (
auto it = dataBuffer_.begin(); it != dataBuffer_.end();)
722 if ((*it)->timestamp() < min)
724 it = dataBuffer_.erase(it);
731 getDataBufferStats();
734 else if (mode_ == RequestMode::Single && dataBuffer_.size() > fragment_ids_.size())
737 while (dataBuffer_.size() > fragment_ids_.size())
739 dataBuffer_.erase(dataBuffer_.begin());
749 if (should_stop() || monitoringInterval_ <= 0)
751 TLOG_DEBUG(
"CommandableFragmentGenerator") <<
"getMonitoringDataLoop: should_stop() is " << std::boolalpha << should_stop()
752 <<
" and monitoringInterval is " << monitoringInterval_ <<
", returning" << TLOG_ENDL;
755 TLOG_ARB(12,
"CommandableFragmentGenerator") <<
"getMonitoringDataLoop: Determining whether to call checkHWStatus_" << TLOG_ENDL;
757 auto now = std::chrono::steady_clock::now();
758 if (TimeUtils::GetElapsedTimeMicroseconds(lastMonitoringCall_, now) >= static_cast<size_t>(monitoringInterval_))
760 isHardwareOK_ = checkHWStatus_();
761 TLOG_ARB(12,
"CommandableFragmentGenerator") <<
"getMonitoringDataLoop: isHardwareOK_ is now " << std::boolalpha << isHardwareOK_ << TLOG_ENDL;
762 lastMonitoringCall_ = now;
764 usleep(monitoringInterval_ / 10);
772 if (check_stop() || !isHardwareOK_ || exception())
774 TLOG_DEBUG(
"CommandableFragmentGenerator") <<
"receiveRequestsLoop: check_stop is " << std::boolalpha << check_stop()
775 <<
", isHardwareOK_ is " << isHardwareOK_ <<
", and exception state is " << exception() <<
", aborting request reception thread." << TLOG_ENDL;
780 if (mode_ == RequestMode::Ignored)
return;
781 TLOG_ARB(16,
"CommandableFragmentGenerator") <<
"receiveRequestsLoop: Polling Request socket for new requests" << TLOG_ENDL;
783 int ms_to_wait = 1000;
784 struct pollfd ufds[1];
785 ufds[0].fd = request_socket_;
786 ufds[0].events = POLLIN | POLLPRI;
787 int rv = poll(ufds, 1, ms_to_wait);
790 if (rv <= 0 || (ufds[0].revents != POLLIN && ufds[0].revents != POLLPRI))
continue;
792 TLOG_ARB(11,
"CommandableFragmentGenerator") <<
"Recieved packet on Request channel" << TLOG_ENDL;
794 recv(request_socket_, &hdr_buffer,
sizeof(hdr_buffer), 0);
795 TLOG_ARB(11,
"CommandableFragmentGenerator") <<
"Request header word: 0x" << std::hex << hdr_buffer.
header << TLOG_ENDL;
796 if (!hdr_buffer.
isValid())
continue;
798 request_received_ =
true;
801 TLOG_INFO(
"CommandableFragmentGenerator") <<
"Received Request Message with the EndOfRun marker. (Re)Starting 1-second timeout for receiving all outstanding requests..." << TLOG_ENDL;
802 request_stop_timeout_ = std::chrono::steady_clock::now();
803 request_stop_requested_ =
true;
806 std::vector<detail::RequestPacket> pkt_buffer(hdr_buffer.
packet_count);
809 for (
auto& buffer : pkt_buffer)
811 if (!buffer.isValid())
continue;
812 if (requests_.count(buffer.sequence_id) && requests_[buffer.sequence_id] != buffer.timestamp)
814 TLOG_ERROR(
"CommandableFragmentGenerator") <<
"Received conflicting request for SeqID "
815 << std::to_string(buffer.sequence_id) <<
"!"
816 <<
" Old ts=" << std::to_string(requests_[buffer.sequence_id])
817 <<
", new ts=" << std::to_string(buffer.timestamp) <<
". Keeping OLD!" << TLOG_ENDL;
819 else if (!requests_.count(buffer.sequence_id))
821 int delta = buffer.sequence_id - ev_counter();
822 TLOG_ARB(11,
"CommandableFragmentGenerator") <<
"Recieved request for sequence ID " << std::to_string(buffer.sequence_id)
823 <<
" and timestamp " << std::to_string(buffer.timestamp) <<
" (delta: " << delta <<
")" << TLOG_ENDL;
826 TLOG_ARB(11,
"CommandableFragmentGenerator") <<
"Already serviced this request! Ignoring..." << TLOG_ENDL;
830 std::unique_lock<std::mutex> tlk(request_mutex_);
831 requests_[buffer.sequence_id] = buffer.timestamp;
838 std::unique_lock<std::mutex> lock(request_mutex_);
839 requestCondition_.notify_all();
847 TLOG_ARB(9,
"CommandableFragmentGenerator") <<
"Mode is Ignored; Copying data to output" << TLOG_ENDL;
848 std::move(dataBuffer_.begin(), dataBuffer_.end(), std::inserter(frags, frags.end()));
855 sendEmptyFragments(frags);
858 if (requests_.size() == 0 || !requests_.count(ev_counter()))
return;
860 if (dataBuffer_.size() > 0)
862 TLOG_ARB(9,
"CommandableFragmentGenerator") <<
"Mode is Single; Sending copy of last event" << TLOG_ENDL;
863 for (
auto& fragptr : dataBuffer_)
866 auto frag = fragptr.get();
867 auto newfrag = std::unique_ptr<artdaq::Fragment>(
new Fragment(ev_counter(), frag->fragmentID()));
868 newfrag->resize(frag->size() - detail::RawFragmentHeader::num_words());
869 memcpy(newfrag->headerAddress(), frag->headerAddress(), frag->sizeBytes());
870 newfrag->setTimestamp(requests_[ev_counter()]);
871 newfrag->setSequenceID(ev_counter());
872 frags.push_back(std::move(newfrag));
877 sendEmptyFragment(frags, ev_counter(),
"No data for");
880 ev_counter_inc(1,
true);
885 sendEmptyFragments(frags);
888 if (requests_.size() == 0 || !requests_.count(ev_counter()))
return;
890 TLOG_DEBUG(
"CommandableFragmentGenerator") <<
"Creating ContainerFragment for Buffered Fragments" << TLOG_ENDL;
891 frags.emplace_back(
new artdaq::Fragment(ev_counter(), fragment_id()));
892 frags.back()->setTimestamp(requests_[ev_counter()]);
893 ContainerFragmentLoader cfl(*frags.back());
894 cfl.set_missing_data(
false);
898 for (
auto it = dataBuffer_.begin(); it != dataBuffer_.end();)
900 TLOG_ARB(9,
"CommandableFragmentGenerator") <<
"ApplyRequests: Adding Fragment with timestamp " << std::to_string((*it)->timestamp()) <<
" to Container" << TLOG_ENDL;
901 cfl.addFragment(*it);
902 it = dataBuffer_.erase(it);
905 ev_counter_inc(1,
true);
910 if (!last_window_send_time_set_)
912 last_window_send_time_ = std::chrono::steady_clock::now();
913 last_window_send_time_set_ =
true;
916 bool now_have_desired_request = std::any_of(requests_.begin(), requests_.end(),
917 [
this](decltype(requests_)::value_type& request) {
918 return request.first == ev_counter(); });
920 if (missing_request_)
922 if (!now_have_desired_request && TimeUtils::GetElapsedTimeMicroseconds(missing_request_time_) > missing_request_window_timeout_us_)
924 TLOG_ERROR(
"CommandableFragmentGenerator") <<
"Data-taking has paused for " << TimeUtils::GetElapsedTimeMicroseconds(missing_request_time_) <<
" us "
925 <<
"(> " << std::to_string(missing_request_window_timeout_us_) <<
" us) while waiting for missing data request messages."
926 <<
" Sending Empty Fragments for missing requests!" << TLOG_ENDL;
927 sendEmptyFragments(frags);
929 missing_request_ =
false;
930 missing_request_time_ = decltype(missing_request_time_)::max();
932 else if (now_have_desired_request) {
933 missing_request_ =
false;
934 missing_request_time_ = decltype(missing_request_time_)::max();
938 for (
auto req = requests_.begin(); req != requests_.end();)
940 auto ts = req->second;
941 if (req->first < ev_counter())
943 req = requests_.erase(req);
946 while (req->first > ev_counter() && request_stop_requested_ && TimeUtils::GetElapsedTime(request_stop_timeout_) > 1)
948 sendEmptyFragment(frags, ev_counter(),
"Missing request for");
949 ev_counter_inc(1,
true);
951 if (req->first > ev_counter())
953 if (!missing_request_)
955 missing_request_ =
true;
956 missing_request_time_ = std::chrono::steady_clock::now();
961 TLOG_ARB(9,
"CommandableFragmentGenerator") <<
"ApplyRequests: Checking that data exists for request window " << std::to_string(req->first) << TLOG_ENDL;
962 Fragment::timestamp_t min = ts > windowOffset_ ? ts - windowOffset_ : 0;
963 Fragment::timestamp_t max = min + windowWidth_;
964 TLOG_ARB(9,
"CommandableFragmentGenerator") <<
"ApplyRequests: min is " << std::to_string(min) <<
", max is " << std::to_string(max)
965 <<
" and last point in buffer is " << std::to_string((dataBuffer_.size() > 0 ? dataBuffer_.back()->timestamp() : 0)) <<
" (sz=" << std::to_string(dataBuffer_.size()) <<
")" << TLOG_ENDL;
966 bool windowClosed = dataBuffer_.size() > 0 && dataBuffer_.back()->timestamp() >= max;
967 bool windowTimeout = TimeUtils::GetElapsedTimeMicroseconds(last_window_send_time_) > window_close_timeout_us_;
970 TLOG_WARNING(
"CommandableFragmentGenerator") <<
"A timeout occurred waiting for data to close the request window (max=" << std::to_string(max)
971 <<
", buffer=" << std::to_string((dataBuffer_.size() > 0 ? dataBuffer_.back()->timestamp() : 0))
972 <<
" (if no buffer in memory, this is shown as a 0)). Time waiting: "
973 << TimeUtils::GetElapsedTimeMicroseconds(last_window_send_time_) <<
" us "
974 <<
"(> " << std::to_string(window_close_timeout_us_) <<
" us)." << TLOG_ENDL;
976 if (windowClosed || !data_thread_running_ || windowTimeout)
978 TLOG_DEBUG(
"CommandableFragmentGenerator") <<
"Creating ContainerFragment for Buffered or Window-requested Fragments" << TLOG_ENDL;
979 frags.emplace_back(
new artdaq::Fragment(ev_counter(), fragment_id()));
980 frags.back()->setTimestamp(ts);
981 ContainerFragmentLoader cfl(*frags.back());
983 if (!windowClosed) cfl.set_missing_data(
true);
984 if (dataBuffer_.size() > 0 && dataBuffer_.front()->timestamp() > min)
986 TLOG_DEBUG(
"CommandableFragmentGenerator") <<
"Request Window covers data that is either before data collection began or has fallen off the end of the buffer" << TLOG_ENDL;
987 cfl.set_missing_data(
true);
992 for (
auto it = dataBuffer_.begin(); it != dataBuffer_.end();)
994 Fragment::timestamp_t fragT = (*it)->timestamp();
995 if (fragT < min || fragT > max || (fragT == max && windowWidth_ > 0))
1001 TLOG_ARB(9,
"CommandableFragmentGenerator") <<
"ApplyRequests: Adding Fragment with timestamp " << std::to_string((*it)->timestamp()) <<
" to Container" << TLOG_ENDL;
1002 cfl.addFragment(*it);
1006 it = dataBuffer_.erase(it);
1013 req = requests_.erase(req);
1014 ev_counter_inc(1,
true);
1015 last_window_send_time_ = std::chrono::steady_clock::now();
1027 if (check_stop() || exception())
1033 if (mode_ == RequestMode::Ignored)
1035 while (dataBufferDepthFragments_ <= 0)
1037 if (check_stop() || exception() || !isHardwareOK_)
return false;
1038 std::unique_lock<std::mutex> lock(dataBufferMutex_);
1039 dataCondition_.wait_for(lock, std::chrono::milliseconds(10), [
this]() {
return dataBufferDepthFragments_ > 0; });
1044 if ((check_stop() && requests_.size() <= 0) || exception())
return false;
1047 while (requests_.size() <= 0)
1049 if (check_stop() || exception())
return false;
1053 std::unique_lock<std::mutex> lock(request_mutex_);
1054 requestCondition_.wait_for(lock, std::chrono::milliseconds(10), [
this]() {
return requests_.size() > 0; });
1059 std::unique_lock<std::mutex> dlk(dataBufferMutex_);
1060 std::unique_lock<std::mutex> rlk(request_mutex_);
1064 case RequestMode::Single:
1065 applyRequestsSingleMode(frags);
1067 case RequestMode::Window:
1068 applyRequestsWindowMode(frags);
1070 case RequestMode::Buffer:
1071 applyRequestsBufferMode(frags);
1073 case RequestMode::Ignored:
1075 applyRequestsIgnoredMode(frags);
1079 getDataBufferStats();
1082 if (frags.size() > 0)
1083 TLOG_ARB(9,
"CommandableFragmentGenerator") <<
"Finished Processing Event " << std::to_string(ev_counter() + 1) <<
" for fragment_id " << fragment_id() <<
"." << TLOG_ENDL;
1089 TLOG_WARNING(
"CommandableFragmentGenerator") << desc <<
" request " << seqId <<
", sending empty fragment" << TLOG_ENDL;
1090 for (
auto fid : fragment_ids_)
1092 auto frag =
new Fragment();
1093 frag->setSequenceID(seqId);
1094 frag->setFragmentID(fid);
1095 frag->setSystemType(Fragment::EmptyFragmentType);
1096 frags.emplace_back(FragmentPtr(frag));
1103 auto sequence_id = Fragment::InvalidSequenceID;
1104 auto timestamp = Fragment::InvalidTimestamp;
1106 TLOG_ARB(19,
"CommandableFragmentGenerator") <<
"Sending Empty Fragments" << TLOG_ENDL;
1107 for (
auto it = requests_.begin(); it != requests_.end();)
1109 auto seq = it->first;
1110 auto ts = it->second;
1112 while (seq > ev_counter())
1115 sendEmptyFragment(frags, ev_counter(),
"Missed request for");
1116 ev_counter_inc(1,
true);
1120 if (++it == requests_.end())
1126 if (seq < ev_counter())
continue;
1131 if (sequence_id < ev_counter())
return;
1132 requests_[sequence_id] = timestamp;
int fragment_id() const
Get the current Fragment ID, if there is only one.
void applyRequestsSingleMode(artdaq::FragmentPtrs &frags)
Create fragments using data buffer for request mode Single. Precondition: dataBufferMutex_ and reques...
int ResolveHost(char const *host_in, in_addr &addr)
Convert a string hostname to a in_addr suitable for socket communication.
virtual ~CommandableFragmentGenerator()
CommandableFragmentGenerator Destructor.
void applyRequestsBufferMode(artdaq::FragmentPtrs &frags)
Create fragments using data buffer for request mode Buffer. Precondition: dataBufferMutex_ and reques...
bool sendEmptyFragment(FragmentPtrs &frags, size_t sequenceId, std::string desc)
Send an EmptyFragmentType Fragment.
void getMonitoringDataLoop()
This function regularly calls checkHWStatus_(), and sets the isHardwareOK flag accordingly.
End of Run mode (Used to end request processing on receiver)
void startDataThread()
Function that launches the data thread (getDataLoop())
std::string ReportCmd(std::string const &which="")
Get a report about a user-specified run-time quantity.
bool dataBufferIsTooLarge()
Test the configured constraints on the data buffer.
void StopCmd(uint64_t timeout, uint64_t timestamp)
Stop the CommandableFragmentGenerator.
void applyRequestsWindowMode(artdaq::FragmentPtrs &frags)
Create fragments using data buffer for request mode Window. Precondition: dataBufferMutex_ and reques...
void StartCmd(int run, uint64_t timeout, uint64_t timestamp)
Start the CommandableFragmentGenerator.
bool check_stop()
Routine used by applyRequests to make sure that all outstanding requests have been fulfilled before r...
void ResumeCmd(uint64_t timeout, uint64_t timestamp)
Resume the CommandableFragmentGenerator.
CommandableFragmentGenerator()
CommandableFragmentGenerator default constructor.
bool getNext(FragmentPtrs &output) overridefinal
getNext calls either applyRequests or getNext_ to get any data that is ready to be sent to the EventB...
bool waitForDataBufferReady()
Wait for the data buffer to drain (dataBufferIsTooLarge returns false), periodically reporting status...
size_t ev_counter_inc(size_t step=1, bool force=false)
Increment the event counter, if the current RequestMode allows it.
void applyRequestsIgnoredMode(artdaq::FragmentPtrs &frags)
Create fragments using data buffer for request mode Ignored. Precondition: dataBufferMutex_ and reque...
void PauseCmd(uint64_t timeout, uint64_t timestamp)
Pause the CommandableFragmentGenerator.
void getDataLoop()
When separate_data_thread is set to true, this loop repeatedly calls getNext_ and adds returned Fragm...
The RequestPacket contains information about a single data request.
void startRequestReceiverThread()
Function that launches the data request receiver thread (receiveRequestsLoop())
void startMonitoringThread()
Function that launches the monitoring thread (getMonitoringDataLoop())
void checkDataBuffer()
Perform data buffer pruning operations. If the RequestMode is Single, removes all but the latest Frag...
std::string printMode_()
Return the string representation of the current RequestMode.
void sendEmptyFragments(FragmentPtrs &frags)
This function is for Buffered and Single request modes, as they can only respond to one data request ...
void getDataBufferStats()
Calculate the size of the dataBuffer and report appropriate metrics.
bool applyRequests(FragmentPtrs &output)
See if any requests have been received, and add the corresponding data Fragment objects to the output...
void receiveRequestsLoop()
This function receives data request packets, adding new requests to the request list.
void setupRequestListener()
Opens the socket used to listen for data requests.
void joinThreads()
Join any data-taking threads. Should be called when destructing CommandableFragmentGenerator.