1 #include "artdaq/Application/CommandableFragmentGenerator.hh"
3 #include <boost/exception/all.hpp>
4 #include <boost/throw_exception.hpp>
9 #include "canvas/Utilities/Exception.h"
10 #include "cetlib/exception.h"
11 #include "fhiclcpp/ParameterSet.h"
12 #include "artdaq-core/Utilities/SimpleLookupPolicy.hh"
13 #include "artdaq-core/Data/Fragment.hh"
14 #include "artdaq-core/Data/ContainerFragmentLoader.hh"
15 #include "artdaq-core/Utilities/ExceptionHandler.hh"
28 , request_addr_(
"227.128.12.26")
30 , request_stop_requested_(false)
31 , end_of_run_timeout_ms_(1000)
34 , staleTimeout_(Fragment::InvalidTimestamp)
35 , maxFragmentCount_(std::numeric_limits<size_t>::max())
36 , uniqueWindows_(true)
37 , useDataThread_(false)
38 , data_thread_running_(false)
39 , dataBufferDepthFragments_(0)
40 , dataBufferDepthBytes_(0)
41 , maxDataBufferDepthFragments_(1000)
42 , maxDataBufferDepthBytes_(1000)
43 , useMonitoringThread_(false)
44 , monitoringInterval_(0)
45 , lastMonitoringCall_(std::chrono::steady_clock::now())
51 , timeout_(std::numeric_limits<uint64_t>::max())
52 , timestamp_(std::numeric_limits<uint64_t>::max())
55 , latest_exception_report_(
"none")
58 , instance_name_for_metrics_(
"FragmentGenerator")
59 , sleep_on_stop_us_(0) {}
63 , request_port_(ps.get<int>(
"request_port", 3001))
64 , request_addr_(ps.get<std::string>(
"request_address",
"227.128.12.26"))
66 , request_stop_requested_(false)
67 , end_of_run_timeout_ms_(ps.get<size_t>(
"end_of_run_quiet_timeout_ms", 1000))
68 , windowOffset_(ps.get<Fragment::timestamp_t>(
"request_window_offset", 0))
69 , windowWidth_(ps.get<Fragment::timestamp_t>(
"request_window_width", 0))
70 , staleTimeout_(ps.get<Fragment::timestamp_t>(
"stale_request_timeout", 0xFFFFFFFF))
71 , uniqueWindows_(ps.get<bool>(
"request_windows_are_unique", true))
72 , useDataThread_(ps.get<bool>(
"separate_data_thread", false))
73 , data_thread_running_(false)
74 , dataBufferDepthFragments_(0)
75 , dataBufferDepthBytes_(0)
76 , maxDataBufferDepthFragments_(ps.get<int>(
"data_buffer_depth_fragments", 1000))
77 , maxDataBufferDepthBytes_(ps.get<size_t>(
"data_buffer_depth_mb", 1000) * 1024 * 1024)
78 , useMonitoringThread_(ps.get<bool>(
"separate_monitoring_thread", false))
79 , monitoringInterval_(ps.get<int64_t>(
"hardware_poll_interval_us", 0))
80 , lastMonitoringCall_(std::chrono::steady_clock::now())
86 , timeout_(std::numeric_limits<uint64_t>::max())
87 , timestamp_(std::numeric_limits<uint64_t>::max())
90 , latest_exception_report_(
"none")
93 , sleep_on_stop_us_(0)
95 board_id_ = ps.get<
int>(
"board_id");
96 instance_name_for_metrics_ =
"BoardReader." + boost::lexical_cast<std::string>(board_id_);
98 fragment_ids_ = ps.get<std::vector<artdaq::Fragment::fragment_id_t>>(
"fragment_ids", std::vector<artdaq::Fragment::fragment_id_t>());
100 TRACE(24,
"artdaq::CommandableFragmentGenerator::CommandableFragmentGenerator(ps)");
103 if (fragment_id != -99)
105 if (fragment_ids_.size() != 0)
107 latest_exception_report_ =
"Error in CommandableFragmentGenerator: can't both define \"fragment_id\" and \"fragment_ids\" in FHiCL document";
108 throw cet::exception(latest_exception_report_);
112 fragment_ids_.emplace_back(fragment_id);
116 sleep_on_stop_us_ = ps.get<
int>(
"sleep_on_stop_us", 0);
118 dataBuffer_.emplace_back(FragmentPtr(
new Fragment()));
119 (*dataBuffer_.begin())->setSystemType(Fragment::EmptyFragmentType);
121 std::string modeString = ps.get<std::string>(
"request_mode",
"ignored");
122 if (modeString ==
"single" || modeString ==
"Single")
124 TRACE(3,
"CommandableFragmentGenerator: RequestMode set to SINGLE");
125 mode_ = RequestMode::Single;
127 else if (modeString.find(
"buffer") != std::string::npos || modeString.find(
"Buffer") != std::string::npos)
129 TRACE(3,
"CommandableFragmentGenerator: RequestMode set to BUFFER");
130 mode_ = RequestMode::Buffer;
132 else if (modeString ==
"window" || modeString ==
"Window")
134 TRACE(3,
"CommandableFragmentGenerator: RequestMode set to WINDOW");
135 mode_ = RequestMode::Window;
137 else if (modeString.find(
"ignore") != std::string::npos || modeString.find(
"Ignore") != std::string::npos)
139 TRACE(3,
"CommandableFragmentGenerator: RequestMode set to IGNORE");
140 mode_ = RequestMode::Ignored;
142 TLOG_DEBUG(
"CommandableFragmentGenerator") <<
"Request mode is " <<
printMode_() << TLOG_ENDL;
144 if (mode_ != RequestMode::Ignored)
148 latest_exception_report_ =
"Error in CommandableFragmentGenerator: use_data_thread must be true when request_mode is not \"Ignored\"!";
149 throw cet::exception(latest_exception_report_);
157 request_socket_ = socket(AF_INET, SOCK_DGRAM, IPPROTO_UDP);
158 if (!request_socket_)
160 throw art::Exception(art::errors::Configuration) <<
"CommandableFragmentGenerator: Error creating socket for receiving data requests!" << std::endl;
164 struct sockaddr_in si_me_request;
167 if (setsockopt(request_socket_, SOL_SOCKET, SO_REUSEADDR, &yes,
sizeof(yes)) < 0)
169 throw art::Exception(art::errors::Configuration) <<
170 "RequestedFragmentGenrator: Unable to enable port reuse on request socket" << std::endl;
173 memset(&si_me_request, 0,
sizeof(si_me_request));
174 si_me_request.sin_family = AF_INET;
175 si_me_request.sin_port = htons(request_port_);
176 si_me_request.sin_addr.s_addr = htonl(INADDR_ANY);
177 if (bind(request_socket_, (
struct sockaddr *)&si_me_request,
sizeof(si_me_request)) == -1)
179 throw art::Exception(art::errors::Configuration) <<
180 "CommandableFragmentGenerator: Cannot bind request socket to port " << request_port_ << std::endl;
185 int sts =
ResolveHost(request_addr_.c_str(), mreq.imr_multiaddr);
188 throw art::Exception(art::errors::Configuration) <<
"Unable to resolve multicast request address" << std::endl;
191 mreq.imr_interface.s_addr = htonl(INADDR_ANY);
192 if (setsockopt(request_socket_, IPPROTO_IP, IP_ADD_MEMBERSHIP, &mreq,
sizeof(mreq)) < 0)
194 throw art::Exception(art::errors::Configuration) <<
195 "CommandableFragmentGenerator: Unable to join multicast group" << std::endl;
202 TLOG_DEBUG(
"CommandableFragmentGenerator") <<
"Joining dataThread" << TLOG_ENDL;
203 if (dataThread_.joinable()) dataThread_.join();
204 TLOG_DEBUG(
"CommandableFragmentGenerator") <<
"Joining monitoringThread" << TLOG_ENDL;
205 if (monitoringThread_.joinable()) monitoringThread_.join();
206 TLOG_DEBUG(
"CommandableFragmentGenerator") <<
"Joining requestThread" << TLOG_ENDL;
207 if (requestThread_.joinable()) requestThread_.join();
214 if (check_stop()) usleep(sleep_on_stop_us_);
215 if (exception())
return false;
217 if (!useMonitoringThread_ && monitoringInterval_ > 0)
219 TRACE(4,
"CFG: Collecting Monitoring Data");
220 auto now = std::chrono::steady_clock::now();
221 if (std::chrono::duration_cast<std::chrono::microseconds>(now - lastMonitoringCall_).count() >= monitoringInterval_)
223 isHardwareOK_ = checkHWStatus_();
224 lastMonitoringCall_ = now;
230 std::lock_guard<std::mutex> lk(mutex_);
233 TRACE(4,
"CFG: Calling applyRequests");
234 result = applyRequests(output);
235 TRACE(4,
"CFG: Done with applyRequests");
238 throw cet::exception(
"CommandableFragmentGenerator") <<
"Exception found in BoardReader with board ID " << board_id() <<
"; BoardReader will now return error status when queried";
243 TRACE(4,
"CFG: Calling getNext_ %zu", ev_counter());
245 result = getNext_(output);
249 TRACE(4,
"CFG: Done with getNext_ %zu", ev_counter());
252 catch (
const cet::exception& e)
254 latest_exception_report_ =
"cet::exception caught in getNext(): ";
255 latest_exception_report_.append(e.what());
256 TLOG_ERROR(
"getNext") <<
"cet::exception caught: " << e << TLOG_ENDL;
260 catch (
const boost::exception& e)
262 latest_exception_report_ =
"boost::exception caught in getNext(): ";
263 latest_exception_report_.append(boost::diagnostic_information(e));
264 TLOG_ERROR(
"getNext") <<
"boost::exception caught: " << boost::diagnostic_information(e) << TLOG_ENDL;
268 catch (
const std::exception& e)
270 latest_exception_report_ =
"std::exception caught in getNext(): ";
271 latest_exception_report_.append(e.what());
272 TLOG_ERROR(
"getNext") <<
"std::exception caught: " << e.what() << TLOG_ENDL;
278 latest_exception_report_ =
"Unknown exception caught in getNext().";
279 TLOG_ERROR(
"getNext") <<
"unknown exception caught" << TLOG_ENDL;
286 TRACE(4,
"CFG:getNext Stopped");
287 TLOG_DEBUG(
"getNext") <<
"stopped " << TLOG_ENDL;
295 TRACE(4,
"CFG::check_stop: should_stop=%i, useDataThread_=%i, requests_.size()=%zu, exception status =%d", should_stop(), useDataThread_, requests_.size(), int(exception()));
296 if (!should_stop())
return false;
297 if (!useDataThread_ || mode_ == RequestMode::Ignored)
return true;
298 if (!request_stop_requested_)
return false;
300 auto dur = std::chrono::steady_clock::now() - request_stop_timeout_;
301 return std::chrono::duration_cast<std::chrono::milliseconds>(dur).count() >
static_cast<int>(end_of_run_timeout_ms_);
306 if (fragment_ids_.size() != 1)
308 throw cet::exception(
"Error in CommandableFragmentGenerator: can't call fragment_id() unless member fragment_ids_ vector is length 1");
312 return fragment_ids_[0];
318 if (force || mode_ == RequestMode::Ignored)
320 return ev_counter_.fetch_add(step);
322 return ev_counter_.load();
327 if (run < 0)
throw cet::exception(
"CommandableFragmentGenerator") <<
"negative run number";
330 timestamp_ = timestamp;
331 ev_counter_.store(1);
332 should_stop_.store(
false);
333 exception_.store(
false);
336 latest_exception_report_ =
"none";
342 std::unique_lock<std::mutex> lk(mutex_);
343 if (useDataThread_) startDataThread();
344 if (useMonitoringThread_) startMonitoringThread();
345 if (mode_ != RequestMode::Ignored) startRequestReceiverThread();
350 TLOG_DEBUG(
"CommandableFragmentGenerator") <<
"Stop Command received." << TLOG_ENDL;
351 TRACE(4,
"CFG: Stop Command Received");
354 timestamp_ = timestamp;
357 should_stop_.store(
true);
358 std::unique_lock<std::mutex> lk(mutex_);
365 timestamp_ = timestamp;
368 should_stop_.store(
true);
369 std::unique_lock<std::mutex> lk(mutex_);
377 timestamp_ = timestamp;
380 should_stop_ =
false;
388 std::unique_lock<std::mutex> lk(mutex_);
389 if (useDataThread_) startDataThread();
390 if (useMonitoringThread_) startMonitoringThread();
391 if (mode_ != RequestMode::Ignored) startRequestReceiverThread();
396 std::lock_guard<std::mutex> lk(mutex_);
403 std::string childReport = reportSpecific(which);
404 if (childReport.length() > 0) {
return childReport; }
407 if (which ==
"latest_exception")
409 return latest_exception_report_;
413 childReport = report();
414 if (childReport.length() > 0) {
return childReport; }
417 std::string tmpString =
"The \"" + which +
"\" command is not ";
418 tmpString.append(
"currently supported by the ");
419 tmpString.append(metricsReportingInstanceName());
420 tmpString.append(
" fragment generator.");
425 void artdaq::CommandableFragmentGenerator::pauseNoMutex()
427 #pragma message "Using default implementation of CommandableFragmentGenerator::pauseNoMutex()"
430 void artdaq::CommandableFragmentGenerator::pause()
432 #pragma message "Using default implementation of CommandableFragmentGenerator::pause()"
435 void artdaq::CommandableFragmentGenerator::resume()
437 #pragma message "Using default implementation of CommandableFragmentGenerator::resume()"
440 std::string artdaq::CommandableFragmentGenerator::report()
442 #pragma message "Using default implementation of CommandableFragmentGenerator::report()"
446 std::string artdaq::CommandableFragmentGenerator::reportSpecific(std::string
const&)
448 #pragma message "Using default implementation of CommandableFragmentGenerator::reportSpecific(std::string)"
452 bool artdaq::CommandableFragmentGenerator::checkHWStatus_()
454 #pragma message "Using default implementation of CommandableFragmentGenerator::checkHWStatus_()"
460 if (dataThread_.joinable()) dataThread_.join();
461 TLOG_INFO(
"CommandableFragmentGenerator") <<
"Starting Data Receiver Thread" << TLOG_ENDL;
467 if (monitoringThread_.joinable()) monitoringThread_.join();
468 TLOG_INFO(
"CommandableFragmentGenerator") <<
"Starting Hardware Monitoring Thread" << TLOG_ENDL;
474 if (requestThread_.joinable()) requestThread_.join();
475 TLOG_INFO(
"CommandableFragmentGenerator") <<
"Starting Request Reception Thread" << TLOG_ENDL;
483 case RequestMode::Single:
485 case RequestMode::Buffer:
487 case RequestMode::Window:
489 case RequestMode::Ignored:
498 data_thread_running_ =
true;
503 TLOG_DEBUG(
"CommandableFragmentGenerator") <<
"getDataLoop: isHardwareOK is " << isHardwareOK_ <<
", aborting data thread" << TLOG_ENDL;
504 data_thread_running_ =
false;
508 TRACE(4,
"CommandableFragmentGenerator::getDataLoop: calling getNext_");
513 data = getNext_(newDataBuffer_);
515 ExceptionHandler(ExceptionHandlerRethrow::no,
516 "Exception thrown by fragment generator in CommandableFragmentGenerator::getDataLoop; setting exception state to \"true\"");
519 data_thread_running_ =
false;
523 auto startwait = std::chrono::steady_clock::now();
525 auto lastwaittime = 0;
526 while (dataBufferIsTooLarge())
530 TLOG_DEBUG(
"CommandableFragmentGenerator") <<
"Run ended while waiting for buffer to shrink!" << TLOG_ENDL;
531 std::unique_lock<std::mutex> lock(dataBufferMutex_);
532 getDataBufferStats();
533 dataCondition_.notify_all();
534 data_thread_running_ =
false;
537 auto waittime = std::chrono::duration_cast<std::chrono::milliseconds>(std::chrono::steady_clock::now() - startwait).count();
539 if (first || (waittime != lastwaittime && waittime % 1000 == 0))
541 TLOG_WARNING(
"CommandableFragmentGenerator") <<
"Bad Omen: Data Buffer has exceeded its size limits. Check the connection between the BoardReader and the EventBuilders! (seq_id=" << ev_counter() <<
")" << TLOG_ENDL;
544 if (waittime % 5 && waittime != lastwaittime)
546 TRACE(4,
"CFG::getDataLoop: Data Retreival paused for %lu ms waiting for data buffer to drain", waittime);
548 lastwaittime = waittime;
554 std::unique_lock<std::mutex> lock(dataBufferMutex_);
557 case RequestMode::Single:
559 while (newDataBuffer_.size() >= fragment_ids_.size())
562 auto it = newDataBuffer_.begin();
563 std::advance(it, fragment_ids_.size());
564 dataBuffer_.splice(dataBuffer_.end(), newDataBuffer_, newDataBuffer_.begin(), it);
567 case RequestMode::Buffer:
568 case RequestMode::Ignored:
569 case RequestMode::Window:
572 dataBuffer_.splice(dataBuffer_.end(), newDataBuffer_);
575 getDataBufferStats();
579 std::unique_lock<std::mutex> lock(dataBufferMutex_);
580 if (dataBuffer_.size() > 0)
582 dataCondition_.notify_all();
587 TLOG_INFO(
"CommandableFragmentGenerator") <<
"Data flow has stopped. Ending data collection thread" << TLOG_ENDL;
588 data_thread_running_ =
false;
596 return (maxDataBufferDepthFragments_ > 0 && dataBufferDepthFragments_ >= maxDataBufferDepthFragments_) || (maxDataBufferDepthBytes_ > 0 && dataBufferDepthBytes_ >= maxDataBufferDepthBytes_);
602 dataBufferDepthFragments_ = dataBuffer_.size();
604 for (
auto i = dataBuffer_.begin(); i != dataBuffer_.end(); ++i)
606 acc += (*i)->sizeBytes();
608 dataBufferDepthBytes_ = acc;
612 metricMan->sendMetric(
"Buffer Depth Fragments", dataBufferDepthFragments_.load(),
"fragments", 1);
613 metricMan->sendMetric(
"Buffer Depth Bytes", dataBufferDepthBytes_.load(),
"bytes", 1);
615 TRACE(4,
"CFG::getDataBufferStats: frags=%i/%i, sz=%zd/%zd", dataBufferDepthFragments_.load(), maxDataBufferDepthFragments_, dataBufferDepthBytes_.load(), maxDataBufferDepthBytes_);
620 std::unique_lock<std::mutex> lock(dataBufferMutex_);
621 dataCondition_.wait_for(lock, std::chrono::milliseconds(10));
622 if (dataBufferDepthFragments_ > 0)
624 if ((mode_ == RequestMode::Buffer || mode_ == RequestMode::Window))
627 while (dataBufferIsTooLarge())
629 dataBuffer_.erase(dataBuffer_.begin());
630 getDataBufferStats();
632 if (dataBuffer_.size() > 0)
634 Fragment::timestamp_t last = dataBuffer_.back()->timestamp();
635 Fragment::timestamp_t min = last > staleTimeout_ ? last - staleTimeout_ : 0;
636 for (
auto it = dataBuffer_.begin(); it != dataBuffer_.end();)
638 if ((*it)->timestamp() < min)
640 it = dataBuffer_.erase(it);
647 getDataBufferStats();
650 else if (mode_ == RequestMode::Single && dataBuffer_.size() > fragment_ids_.size())
653 while (dataBuffer_.size() > fragment_ids_.size())
655 dataBuffer_.erase(dataBuffer_.begin());
665 if (should_stop() || monitoringInterval_ <= 0)
667 TLOG_DEBUG(
"CommandableFragmentGenerator") <<
"getMonitoringDataLoop: should_stop() is " << std::boolalpha << should_stop()
668 <<
" and monitoringInterval is " << monitoringInterval_ <<
", returning" << TLOG_ENDL;
671 TRACE(4,
"CFG::getMonitoringDataLoop Determining whether to call checkHWStatus_");
673 auto now = std::chrono::steady_clock::now();
674 if (std::chrono::duration_cast<std::chrono::microseconds>(now - lastMonitoringCall_).count() >= monitoringInterval_)
676 isHardwareOK_ = checkHWStatus_();
677 lastMonitoringCall_ = now;
679 usleep(monitoringInterval_ / 10);
687 if (check_stop() || !isHardwareOK_ || exception())
689 TLOG_DEBUG(
"CommandableFragmentGenerator") <<
"receiveRequestsLoop: check_stop is " << std::boolalpha << check_stop()
690 <<
", isHardwareOK_ is " << isHardwareOK_ <<
", and exception state is " << exception() <<
", aborting request reception thread." << TLOG_ENDL;
695 if (mode_ == RequestMode::Ignored)
return;
696 TRACE(4,
"CFG::receiveRequestsLoop: Polling Request socket for new requests");
698 int ms_to_wait = 1000;
699 struct pollfd ufds[1];
700 ufds[0].fd = request_socket_;
701 ufds[0].events = POLLIN | POLLPRI;
702 int rv = poll(ufds, 1, ms_to_wait);
705 if (ufds[0].revents == POLLIN || ufds[0].revents == POLLPRI)
707 TRACE(4,
"CFG: Recieved packet on Request channel");
709 recv(request_socket_, &hdr_buffer,
sizeof(hdr_buffer), 0);
710 TRACE(4,
"CFG: Request header word: 0x%x", (
int)hdr_buffer.
header);
713 if (hdr_buffer.
mode == detail::RequestMessageMode::EndOfRun)
715 TLOG_INFO(
"CommandableFragmentGenerator") <<
"Received Request Message with the EndOfRun marker. (Re)Starting 1-second timeout for receiving all outstanding requests..." << TLOG_ENDL;
716 request_stop_timeout_ = std::chrono::steady_clock::now();
717 request_stop_requested_ =
true;
719 std::vector<detail::RequestPacket> pkt_buffer(hdr_buffer.
packet_count);
722 for (
auto& buffer : pkt_buffer)
724 if (!buffer.isValid())
continue;
725 if (requests_.count(buffer.sequence_id) && requests_[buffer.sequence_id] != buffer.timestamp)
727 TLOG_ERROR(
"CommandableFragmentGenerator") <<
"Received conflicting request for SeqID "
728 << std::to_string(buffer.sequence_id) <<
"!"
729 <<
" Old ts=" << std::to_string(requests_[buffer.sequence_id])
730 <<
", new ts=" << std::to_string(buffer.timestamp) <<
". Keeping OLD!" << TLOG_ENDL;
732 else if (!requests_.count(buffer.sequence_id))
734 int delta = buffer.sequence_id - ev_counter();
735 TRACE(4,
"CFG: Recieved request for sequence ID %llu and timestamp %lu (delta: %d)", (
unsigned long long)buffer.sequence_id, (
unsigned long)buffer.timestamp, delta);
738 TRACE(4,
"CFG: Already serviced this request! Ignoring...");
742 std::unique_lock<std::mutex> tlk(request_mutex_);
743 requests_[buffer.sequence_id] = buffer.timestamp;
750 std::unique_lock<std::mutex> lock(request_mutex_);
751 requestCondition_.notify_all();
761 if (check_stop() || exception())
766 if (mode_ == RequestMode::Ignored)
768 while (dataBufferDepthFragments_ <= 0)
770 if (check_stop() || exception())
return false;
771 std::unique_lock<std::mutex> lock(dataBufferMutex_);
772 dataCondition_.wait_for(lock, std::chrono::milliseconds(10), [
this]() {
return dataBufferDepthFragments_ > 0; });
777 if ((check_stop() && requests_.size() <= 0) || exception())
return false;
780 while (requests_.size() <= 0)
782 if (check_stop() || exception())
return false;
786 std::unique_lock<std::mutex> lock(request_mutex_);
787 requestCondition_.wait_for(lock, std::chrono::milliseconds(10), [
this]() {
return requests_.size() > 0; });
792 std::unique_lock<std::mutex> dlk(dataBufferMutex_);
793 std::unique_lock<std::mutex> rlk(request_mutex_);
796 if (mode_ == RequestMode::Ignored)
799 TRACE(4,
"CFG: Mode is Ignored; Copying data to output");
800 std::move(dataBuffer_.begin(), dataBuffer_.end(), std::inserter(frags, frags.end()));
803 else if (mode_ == RequestMode::Single)
806 sendEmptyFragments(frags);
808 if (dataBuffer_.size() > 0)
810 TRACE(4,
"CFG: Mode is Single; Sending copy of last event");
811 for (
auto& fragptr : dataBuffer_)
814 auto frag = fragptr.get();
815 auto newfrag = std::unique_ptr<artdaq::Fragment>(
new Fragment(ev_counter(), frag->fragmentID()));
816 newfrag->resize(frag->size() - detail::RawFragmentHeader::num_words());
817 memcpy(newfrag->headerAddress(), frag->headerAddress(), frag->sizeBytes());
818 newfrag->setTimestamp(requests_[ev_counter()]);
819 newfrag->setSequenceID(ev_counter());
820 frags.push_back(std::move(newfrag));
825 sendEmptyFragment(frags, ev_counter(),
"No data for");
828 ev_counter_inc(1,
true);
830 else if (mode_ == RequestMode::Buffer || mode_ == RequestMode::Window)
832 if (mode_ == RequestMode::Buffer)
835 sendEmptyFragments(frags);
837 for (
auto req = requests_.begin(); req != requests_.end();)
839 auto ts = req->second;
840 if (req->first < ev_counter())
842 req = requests_.erase(req);
845 while (req->first > ev_counter() && request_stop_requested_ && std::chrono::duration_cast<std::chrono::seconds>(std::chrono::steady_clock::now() - request_stop_timeout_).count() > 1)
847 sendEmptyFragment(frags,ev_counter(),
"Missing request for");
848 ev_counter_inc(1,
true);
850 if (req->first > ev_counter())
855 TRACE(5,
"CFG: ApplyRequestS: Checking that data exists for request window %llu (Buffered mode will always succeed)", (
unsigned long long)req->first);
856 Fragment::timestamp_t min = ts > windowOffset_ ? ts - windowOffset_ : 0;
857 Fragment::timestamp_t max = min + windowWidth_;
858 TRACE(5,
"CFG::ApplyRequests: min is %lu and max is %lu and last point in buffer is %lu (sz=%zu)",
859 (
unsigned long)min, (
unsigned long)max, (
unsigned long)(dataBuffer_.size() > 0 ? dataBuffer_.back()->timestamp() : 0), dataBuffer_.size());
860 bool windowClosed = mode_ != RequestMode::Window || (dataBuffer_.size() > 0 && dataBuffer_.back()->timestamp() >= max);
861 if (windowClosed || !data_thread_running_)
863 TLOG_DEBUG(
"CommandableFragmentGenerator") <<
"Creating ContainerFragment for Buffered or Window-requested Fragments" << TLOG_ENDL;
864 frags.emplace_back(
new artdaq::Fragment(ev_counter(), fragment_id()));
865 frags.back()->setTimestamp(ts);
866 ContainerFragmentLoader cfl(*frags.back());
868 if (mode_ == RequestMode::Window && !data_thread_running_ && !windowClosed) cfl.set_missing_data(
true);
869 if (mode_ == RequestMode::Window && dataBuffer_.size() > 0 && dataBuffer_.front()->timestamp() > min)
871 TLOG_DEBUG(
"CommandableFragmentGenerator") <<
"Request Window covers data that is either before data collection began or has fallen off the end of the buffer" << TLOG_ENDL;
872 cfl.set_missing_data(
true);
877 for (
auto it = dataBuffer_.begin(); it != dataBuffer_.end();)
879 if (mode_ == RequestMode::Window)
881 Fragment::timestamp_t fragT = (*it)->timestamp();
882 if (fragT < min || fragT > max)
889 TRACE(5,
"CFG::ApplyRequests: Adding Fragment with timestamp %llu to Container", (
unsigned long long)(*it)->timestamp());
890 cfl.addFragment(*it);
892 if (mode_ == RequestMode::Buffer || (mode_ == RequestMode::Window && uniqueWindows_))
894 it = dataBuffer_.erase(it);
901 req = requests_.erase(req);
902 ev_counter_inc(1,
true);
911 getDataBufferStats();
914 if (frags.size() > 0)
915 TRACE(4,
"CFG: Finished Processing Event %lu for fragment_id %i.", ev_counter() + 1, fragment_id());
921 TLOG_WARNING(
"CommandableFragmentGenerator") << desc <<
" request " << seqId <<
", sending empty fragment" << TLOG_ENDL;
922 auto frag =
new Fragment();
923 frag->setSequenceID(seqId);
924 frag->setSystemType(Fragment::EmptyFragmentType);
925 frags.emplace_back(FragmentPtr(frag));
931 auto sequence_id = Fragment::InvalidSequenceID;
932 auto timestamp = Fragment::InvalidTimestamp;
934 for (
auto it = requests_.begin(); it != requests_.end();)
936 auto seq = it->first;
937 auto ts = it->second;
940 if (++it == requests_.end())
946 if (seq < ev_counter())
continue;
949 sendEmptyFragment(frags, ev_counter(),
"Missed request for");
950 ev_counter_inc(1,
true);
954 if (sequence_id < ev_counter())
return;
955 requests_[sequence_id] = timestamp;
int fragment_id() const
Get the current Fragment ID, if there is only one.
int ResolveHost(char const *host_in, in_addr &addr)
Convert a string hostname to a in_addr suitable for socket communication.
virtual ~CommandableFragmentGenerator()
CommandableFragmentGenerator Destructor.
bool sendEmptyFragment(FragmentPtrs &frags, size_t sequenceId, std::string desc)
Send an EmptyFragmentType Fragment.
void getMonitoringDataLoop()
This function regularly calls checkHWStatus_(), and sets the isHardwareOK flag accordingly.
void startDataThread()
Function that launches the data thread (getDataLoop())
std::string ReportCmd(std::string const &which="")
Get a report about a user-specified run-time quantity.
bool dataBufferIsTooLarge()
Test the configured constraints on the data buffer.
void StopCmd(uint64_t timeout, uint64_t timestamp)
Stop the CommandableFragmentGenerator.
void StartCmd(int run, uint64_t timeout, uint64_t timestamp)
Start the CommandableFragmentGenerator.
bool check_stop()
Routine used by applyRequests to make sure that all outstanding requests have been fulfilled before r...
void ResumeCmd(uint64_t timeout, uint64_t timestamp)
Resume the CommandableFragmentGenerator.
CommandableFragmentGenerator()
CommandableFragmentGenerator default constructor.
bool getNext(FragmentPtrs &output) overridefinal
getNext calls either applyRequests or getNext_ to get any data that is ready to be sent to the EventB...
size_t ev_counter_inc(size_t step=1, bool force=false)
Increment the event counter, if the current RequestMode allows it.
void PauseCmd(uint64_t timeout, uint64_t timestamp)
Pause the CommandableFragmentGenerator.
void getDataLoop()
When separate_data_thread is set to true, this loop repeatedly calls getNext_ and adds returned Fragm...
The RequestPacket contains information about a single data request.
void startRequestReceiverThread()
Function that launches the data request receiver thread (receiveRequestsLoop())
void startMonitoringThread()
Function that launches the monitoring thread (getMonitoringDataLoop())
void checkDataBuffer()
Perform data buffer pruning operations. If the RequestMode is Single, removes all but the latest Frag...
std::string printMode_()
Return the string representation of the current RequestMode.
void sendEmptyFragments(FragmentPtrs &frags)
This function is for Buffered and Single request modes, as they can only respond to one data request ...
void getDataBufferStats()
Calculate the size of the dataBuffer and report appropriate metrics.
bool applyRequests(FragmentPtrs &output)
See if any requests have been received, and add the corresponding data Fragment objects to the output...
void receiveRequestsLoop()
This function receives data request packets, adding new requests to the request list.
void setupRequestListener()
Opens the socket used to listen for data requests.