1 #include "artdaq/DAQdata/Globals.hh"
2 #define TRACE_NAME (app_name + "_RequestReceiver").c_str()
4 #include "artdaq/DAQdata/Globals.hh"
5 #include "artdaq/DAQrate/RequestReceiver.hh"
6 #include "artdaq/DAQrate/detail/RequestMessage.hh"
8 #include <boost/exception/all.hpp>
9 #include <boost/throw_exception.hpp>
14 #include "canvas/Utilities/Exception.h"
15 #include "cetlib_except/exception.h"
16 #include "fhiclcpp/ParameterSet.h"
18 #include "artdaq-core/Data/ContainerFragmentLoader.hh"
19 #include "artdaq-core/Data/Fragment.hh"
20 #include "artdaq-core/Utilities/ExceptionHandler.hh"
21 #include "artdaq-core/Utilities/SimpleLookupPolicy.hh"
22 #include "artdaq-core/Utilities/TimeUtils.hh"
24 #include <arpa/inet.h>
25 #include <netinet/in.h>
36 , request_addr_(
"227.128.12.26")
42 , request_stop_requested_(false)
43 , request_received_(false)
44 , end_of_run_timeout_ms_(1000)
46 , highest_seen_request_(0)
47 , out_of_order_requests_()
48 , request_increment_(1)
52 : request_port_(ps.get<int>(
"request_port", 3001))
53 , request_addr_(ps.get<std::string>(
"request_address",
"227.128.12.26"))
54 , multicast_in_addr_(ps.get<std::string>(
"multicast_interface_ip",
"0.0.0.0"))
60 , request_stop_requested_(false)
61 , request_received_(false)
62 , end_of_run_timeout_ms_(ps.get<size_t>(
"end_of_run_quiet_timeout_ms", 1000))
64 , highest_seen_request_(0)
65 , out_of_order_requests_()
66 , request_increment_(ps.get<artdaq::Fragment::sequence_id_t>(
"request_increment", 1))
73 TLOG(TLVL_INFO) <<
"Setting up request listen socket, rank=" << my_rank <<
", address=" << request_addr_ <<
":" << request_port_
74 <<
", multicast interface=" << multicast_in_addr_;
75 request_socket_ = socket(AF_INET, SOCK_DGRAM, IPPROTO_UDP);
76 if (request_socket_ < 0)
78 TLOG(TLVL_ERROR) <<
"Error creating socket for receiving data requests! err=" << strerror(errno);
82 struct sockaddr_in si_me_request;
85 if (setsockopt(request_socket_, SOL_SOCKET, SO_REUSEADDR, &yes,
sizeof(yes)) < 0)
87 TLOG(TLVL_ERROR) <<
"Unable to enable port reuse on request socket, err=" << strerror(errno);
90 memset(&si_me_request, 0,
sizeof(si_me_request));
91 si_me_request.sin_family = AF_INET;
92 si_me_request.sin_port = htons(request_port_);
93 si_me_request.sin_addr.s_addr = htonl(INADDR_ANY);
94 if (bind(request_socket_, (
struct sockaddr*)&si_me_request,
sizeof(si_me_request)) == -1)
96 TLOG(TLVL_ERROR) <<
"Cannot bind request socket to port " << request_port_ <<
", err=" << strerror(errno);
100 if (request_addr_ !=
"localhost")
103 int sts =
ResolveHost(request_addr_.c_str(), mreq.imr_multiaddr);
106 TLOG(TLVL_ERROR) <<
"Unable to resolve multicast request address, err=" << strerror(errno);
112 TLOG(TLVL_ERROR) <<
"Unable to resolve hostname for " << multicast_in_addr_;
115 if (setsockopt(request_socket_, IPPROTO_IP, IP_ADD_MEMBERSHIP, &mreq,
sizeof(mreq)) < 0)
117 TLOG(TLVL_ERROR) <<
"Unable to join multicast group, err=" << strerror(errno);
121 TLOG(TLVL_INFO) <<
"Done setting up request socket, rank=" << my_rank;
124 artdaq::RequestReceiver::~RequestReceiver()
126 stopRequestReception(
true);
131 std::unique_lock<std::mutex> lk(state_mutex_);
132 if (!request_received_ && !force)
134 TLOG(TLVL_ERROR) <<
"Stop request received by RequestReceiver, but no requests have ever been received." << std::endl
135 <<
"Check that UDP port " << request_port_ <<
" is open in the firewall config.";
140 TLOG(TLVL_DEBUG) <<
"Joining requestThread";
141 if (requestThread_.joinable()) requestThread_.join();
145 if (once) TLOG(TLVL_ERROR) <<
"running_ is true after thread join! Should NOT happen";
151 if (request_socket_ != -1)
153 close(request_socket_);
154 request_socket_ = -1;
156 request_received_ =
false;
157 highest_seen_request_ = 0;
162 std::unique_lock<std::mutex> lk(state_mutex_);
163 if (requestThread_.joinable()) requestThread_.join();
164 should_stop_ =
false;
165 request_stop_requested_ =
false;
167 if (request_socket_ == -1)
169 TLOG(TLVL_INFO) <<
"Connecting Request Reception socket";
170 setupRequestListener();
173 TLOG(TLVL_INFO) <<
"Starting Request Reception Thread";
178 catch (
const boost::exception& e)
180 TLOG(TLVL_ERROR) <<
"Caught boost::exception starting Request Receiver thread: " << boost::diagnostic_information(e) <<
", errno=" << errno;
181 std::cerr <<
"Caught boost::exception starting Request Receiver thread: " << boost::diagnostic_information(e) <<
", errno=" << errno << std::endl;
189 while (!should_stop_)
191 TLOG(16) <<
"receiveRequestsLoop: Polling Request socket for new requests";
193 if (request_socket_ == -1)
195 setupRequestListener();
199 struct pollfd ufds[1];
200 ufds[0].fd = request_socket_;
201 ufds[0].events = POLLIN | POLLPRI | POLLERR;
202 int rv = poll(ufds, 1, ms_to_wait);
205 if (rv <= 0 || (ufds[0].revents != POLLIN && ufds[0].revents != POLLPRI))
207 if (rv == 1 && (ufds[0].revents & (POLLNVAL | POLLERR | POLLHUP)))
209 close(request_socket_);
210 request_socket_ = -1;
212 if (request_stop_requested_ && TimeUtils::GetElapsedTimeMilliseconds(request_stop_timeout_) > end_of_run_timeout_ms_)
219 TLOG(11) <<
"Received packet on Request channel";
220 std::vector<uint8_t> buffer(MAX_REQUEST_MESSAGE_SIZE);
221 struct sockaddr_in from;
222 socklen_t len =
sizeof(from);
223 auto sts = recvfrom(request_socket_, &buffer[0], MAX_REQUEST_MESSAGE_SIZE, 0, (
struct sockaddr*)&from, &len);
226 TLOG(TLVL_ERROR) <<
"Error receiving request message header err=" << strerror(errno);
227 close(request_socket_);
228 request_socket_ = -1;
233 TLOG(11) <<
"Request header word: 0x" << std::hex << hdr_buffer->
header << std::dec <<
", packet_count: " << hdr_buffer->packet_count <<
" from rank " << hdr_buffer->rank <<
", " << inet_ntoa(from.sin_addr) <<
":" << from.sin_port <<
", run number: " << hdr_buffer->run_number;
234 if (!hdr_buffer->isValid())
continue;
236 request_received_ =
true;
239 if (run_number_ != 0 && hdr_buffer->run_number != run_number_)
241 TLOG(TLVL_WARNING) <<
"Received a Request Message with the wrong run number ("
242 << hdr_buffer->run_number <<
"), expected " << run_number_
243 <<
", ignoring this request.";
249 TLOG(TLVL_INFO) <<
"Received Request Message with the EndOfRun marker. (Re)Starting 1-second timeout for receiving all outstanding requests...";
250 request_stop_timeout_ = std::chrono::steady_clock::now();
251 request_stop_requested_ =
true;
254 std::vector<artdaq::detail::RequestPacket> pkt_buffer(hdr_buffer->packet_count);
258 if (should_stop_)
break;
260 for (
auto& buffer : pkt_buffer)
262 TLOG(20) <<
"Request Packet: hdr=" << buffer.header <<
", seq=" << buffer.sequence_id <<
", ts=" << buffer.timestamp;
263 if (!buffer.isValid())
continue;
264 std::unique_lock<std::mutex> tlk(request_mutex_);
265 if (requests_.count(buffer.sequence_id) && requests_[buffer.sequence_id] != buffer.timestamp)
267 TLOG(TLVL_ERROR) <<
"Received conflicting request for SeqID "
268 << buffer.sequence_id <<
"!"
269 <<
" Old ts=" << requests_[buffer.sequence_id]
270 <<
", new ts=" << buffer.timestamp <<
". Keeping OLD!";
272 else if (!requests_.count(buffer.sequence_id))
274 int delta = buffer.sequence_id - highest_seen_request_;
275 TLOG(11) <<
"Received request for sequence ID " << buffer.sequence_id
276 <<
" and timestamp " << buffer.timestamp <<
" (delta: " << delta <<
")";
277 if (delta <= 0 || out_of_order_requests_.count(buffer.sequence_id))
279 TLOG(11) <<
"Already serviced this request ( sequence ID " << buffer.sequence_id <<
")! Ignoring...";
283 requests_[buffer.sequence_id] = buffer.timestamp;
284 request_timing_[buffer.sequence_id] = std::chrono::steady_clock::now();
291 request_cv_.notify_all();
294 TLOG(TLVL_DEBUG) <<
"Ending Request Thread";
300 TLOG(10) <<
"RemoveRequest: Removing request for id " << reqID;
301 std::unique_lock<std::mutex> lk(request_mutex_);
302 requests_.erase(reqID);
304 if (reqID > highest_seen_request_)
306 TLOG(10) <<
"RemoveRequest: out_of_order_requests_.size() == " << out_of_order_requests_.size() <<
", reqID=" << reqID <<
", expected=" << highest_seen_request_ + request_increment_;
307 if (out_of_order_requests_.size() || reqID != highest_seen_request_ + request_increment_)
309 out_of_order_requests_.insert(reqID);
311 auto it = out_of_order_requests_.begin();
312 while (it != out_of_order_requests_.end() && !should_stop_)
314 if (*it == highest_seen_request_ + request_increment_)
316 highest_seen_request_ = *it;
317 it = out_of_order_requests_.erase(it);
327 highest_seen_request_ = reqID;
329 TLOG(10) <<
"RemoveRequest: reqID=" << reqID <<
" Setting highest_seen_request_ to " << highest_seen_request_;
331 if (metricMan && request_timing_.count(reqID))
333 metricMan->sendMetric(
"Request Response Time", TimeUtils::GetElapsedTime(request_timing_[reqID]),
"seconds", 2, MetricMode::Average);
335 request_timing_.erase(reqID);
int ResolveHost(char const *host_in, in_addr &addr)
Convert a string hostname to a in_addr suitable for socket communication.
void startRequestReception()
Enables (starts) the reception of data requests.
End of Run mode (Used to end request processing on receiver)
RequestReceiver()
RequestReceiver Default Constructor.
int GetInterfaceForNetwork(char const *host_in, in_addr &addr)
Convert an IP address to the network address of the interface sharing the subnet mask.
void setupRequestListener()
Opens the socket used to listen for data requests.
void receiveRequestsLoop()
This function receives data request packets, adding new requests to the request list.
The RequestPacket contains information about a single data request.
void stopRequestReception(bool force=false)
Disables (stops) the reception of data requests.
void RemoveRequest(artdaq::Fragment::sequence_id_t reqID)
Remove the request with the given sequence ID from the request map