1 #define TRACE_NAME "RequestReceiver"
3 #include "artdaq/DAQrate/RequestReceiver.hh"
4 #include "artdaq/DAQdata/Globals.hh"
5 #include "artdaq/DAQrate/detail/RequestMessage.hh"
7 #include <boost/exception/all.hpp>
8 #include <boost/throw_exception.hpp>
13 #include "canvas/Utilities/Exception.h"
14 #include "cetlib_except/exception.h"
15 #include "fhiclcpp/ParameterSet.h"
17 #include "artdaq-core/Utilities/SimpleLookupPolicy.hh"
18 #include "artdaq-core/Data/Fragment.hh"
19 #include "artdaq-core/Data/ContainerFragmentLoader.hh"
20 #include "artdaq-core/Utilities/ExceptionHandler.hh"
21 #include "artdaq-core/Utilities/TimeUtils.hh"
32 artdaq::RequestReceiver::RequestReceiver()
34 , request_addr_(
"227.128.12.26")
38 , request_stop_requested_(false)
39 , request_received_(false)
40 , end_of_run_timeout_ms_(1000)
42 , highest_seen_request_(0)
45 artdaq::RequestReceiver::RequestReceiver(
const fhicl::ParameterSet& ps)
46 : request_port_(ps.get<int>(
"request_port", 3001))
47 , request_addr_(ps.get<std::string>(
"request_address",
"227.128.12.26"))
51 , request_stop_requested_(false)
52 , request_received_(false)
53 , end_of_run_timeout_ms_(ps.get<size_t>(
"end_of_run_quiet_timeout_ms", 1000))
55 , highest_seen_request_(0)
57 setupRequestListener();
62 TLOG(TLVL_INFO) <<
"Setting up request listen socket, rank=" << my_rank <<
", address=" << request_addr_ <<
":" << request_port_ ;
63 request_socket_ = socket(AF_INET, SOCK_DGRAM, IPPROTO_UDP);
64 if (request_socket_ < 0)
66 TLOG(TLVL_ERROR) <<
"Error creating socket for receiving data requests! err=" << strerror(errno) ;
70 struct sockaddr_in si_me_request;
73 if (setsockopt(request_socket_, SOL_SOCKET, SO_REUSEADDR, &yes,
sizeof(yes)) < 0)
75 TLOG(TLVL_ERROR) <<
"Unable to enable port reuse on request socket, err=" << strerror(errno) ;
78 memset(&si_me_request, 0,
sizeof(si_me_request));
79 si_me_request.sin_family = AF_INET;
80 si_me_request.sin_port = htons(request_port_);
81 si_me_request.sin_addr.s_addr = htonl(INADDR_ANY);
82 if (bind(request_socket_, (
struct sockaddr *)&si_me_request,
sizeof(si_me_request)) == -1)
84 TLOG(TLVL_ERROR) <<
"Cannot bind request socket to port " << request_port_ <<
", err=" << strerror(errno) ;
88 if (request_addr_ !=
"localhost")
91 int sts =
ResolveHost(request_addr_.c_str(), mreq.imr_multiaddr);
94 TLOG(TLVL_ERROR) <<
"Unable to resolve multicast request address, err=" << strerror(errno) ;
97 mreq.imr_interface.s_addr = htonl(INADDR_ANY);
98 if (setsockopt(request_socket_, IPPROTO_IP, IP_ADD_MEMBERSHIP, &mreq,
sizeof(mreq)) < 0)
100 TLOG(TLVL_ERROR) <<
"Unable to join multicast group, err=" << strerror(errno) ;
104 TLOG(TLVL_INFO) <<
"Done setting up request socket, rank=" << my_rank ;
107 artdaq::RequestReceiver::~RequestReceiver()
109 stopRequestReceiverThread();
114 std::unique_lock<std::mutex> lk(state_mutex_);
115 if (!request_received_)
117 TLOG(TLVL_ERROR) <<
"Stop request received by RequestReceiver, but no requests have ever been received." << std::endl
118 <<
"Check that UDP port " << request_port_ <<
" is open in the firewall config.";
121 TLOG(TLVL_DEBUG) <<
"Joining requestThread";
122 if (requestThread_.joinable()) requestThread_.join();
123 while (running_) usleep(10000);
125 if (request_socket_ != -1) {
126 close(request_socket_);
127 request_socket_ = -1;
129 request_received_ =
false;
130 highest_seen_request_ = 0;
135 std::unique_lock<std::mutex> lk(state_mutex_);
136 if (requestThread_.joinable()) requestThread_.join();
137 should_stop_ =
false;
138 request_stop_requested_ =
false;
140 if (request_socket_ == -1) {
141 TLOG(TLVL_INFO) <<
"Connecting Request Reception socket";
142 setupRequestListener();
145 TLOG(TLVL_INFO) <<
"Starting Request Reception Thread" ;
152 while (!should_stop_)
154 TLOG(16) <<
"receiveRequestsLoop: Polling Request socket for new requests" ;
157 struct pollfd ufds[1];
158 ufds[0].fd = request_socket_;
159 ufds[0].events = POLLIN | POLLPRI;
160 int rv = poll(ufds, 1, ms_to_wait);
163 if (rv <= 0 || (ufds[0].revents != POLLIN && ufds[0].revents != POLLPRI))
165 if (request_stop_requested_ && TimeUtils::GetElapsedTimeMilliseconds(request_stop_timeout_) > end_of_run_timeout_ms_)
172 TLOG(11) <<
"Recieved packet on Request channel" ;
174 recv(request_socket_, &hdr_buffer,
sizeof(hdr_buffer), 0);
175 TLOG(11) <<
"Request header word: 0x" << std::hex << hdr_buffer.
header ;
176 if (!hdr_buffer.
isValid())
continue;
178 request_received_ =
true;
181 TLOG(TLVL_INFO) <<
"Received Request Message with the EndOfRun marker. (Re)Starting 1-second timeout for receiving all outstanding requests..." ;
182 request_stop_timeout_ = std::chrono::steady_clock::now();
183 request_stop_requested_ =
true;
186 std::vector<artdaq::detail::RequestPacket> pkt_buffer(hdr_buffer.
packet_count);
190 if (should_stop_)
break;
192 for (
auto& buffer : pkt_buffer)
194 if (!buffer.isValid())
continue;
195 if (requests_.count(buffer.sequence_id) && requests_[buffer.sequence_id] != buffer.timestamp)
197 TLOG(TLVL_ERROR) <<
"Received conflicting request for SeqID "
198 << std::to_string(buffer.sequence_id) <<
"!"
199 <<
" Old ts=" << std::to_string(requests_[buffer.sequence_id])
200 <<
", new ts=" << std::to_string(buffer.timestamp) <<
". Keeping OLD!" ;
202 else if (!requests_.count(buffer.sequence_id))
204 int delta = buffer.sequence_id - highest_seen_request_;
205 TLOG(11) <<
"Recieved request for sequence ID " << std::to_string(buffer.sequence_id)
206 <<
" and timestamp " << std::to_string(buffer.timestamp) <<
" (delta: " << delta <<
")" ;
209 TLOG(11) <<
"Already serviced this request! Ignoring..." ;
213 std::unique_lock<std::mutex> tlk(request_mutex_);
214 requests_[buffer.sequence_id] = buffer.timestamp;
215 request_timing_[buffer.sequence_id] = std::chrono::steady_clock::now();
221 request_cv_.notify_all();
224 TLOG(TLVL_DEBUG) <<
"Ending Request Thread";
228 void artdaq::RequestReceiver::RemoveRequest(artdaq::Fragment::sequence_id_t reqID)
230 std::unique_lock<std::mutex> lk(request_mutex_);
231 requests_.erase(reqID);
232 if (reqID > highest_seen_request_ && !should_stop_)
234 TLOG(18) <<
"Setting highest_seen_request_ to " << reqID;
235 highest_seen_request_ = reqID;
240 metricMan->sendMetric(
"Request Response Time", TimeUtils::GetElapsedTime(request_timing_[reqID]),
"seconds", 2, MetricMode::Average);
242 request_timing_.erase(reqID);
int ResolveHost(char const *host_in, in_addr &addr)
Convert a string hostname to a in_addr suitable for socket communication.
End of Run mode (Used to end request processing on receiver)
void startRequestReceiverThread()
Function that launches the data request receiver thread (receiveRequestsLoop())
void setupRequestListener()
Opens the socket used to listen for data requests.
void receiveRequestsLoop()
This function receives data request packets, adding new requests to the request list.
The RequestPacket contains information about a single data request.
void stopRequestReceiverThread()
Stop the data request receiver thread (receiveRequestsLoop)