1 #define TRACE_NAME "RequestReceiver"
3 #include "artdaq/DAQrate/RequestReceiver.hh"
4 #include "artdaq/DAQdata/Globals.hh"
5 #include "artdaq/DAQrate/detail/RequestMessage.hh"
7 #include <boost/exception/all.hpp>
8 #include <boost/throw_exception.hpp>
13 #include "canvas/Utilities/Exception.h"
14 #include "cetlib_except/exception.h"
15 #include "fhiclcpp/ParameterSet.h"
17 #include "artdaq-core/Utilities/SimpleLookupPolicy.hh"
18 #include "artdaq-core/Data/Fragment.hh"
19 #include "artdaq-core/Data/ContainerFragmentLoader.hh"
20 #include "artdaq-core/Utilities/ExceptionHandler.hh"
21 #include "artdaq-core/Utilities/TimeUtils.hh"
32 artdaq::RequestReceiver::RequestReceiver()
34 , request_addr_(
"227.128.12.26")
38 , request_stop_requested_(false)
39 , request_received_(false)
40 , end_of_run_timeout_ms_(1000)
42 , highest_seen_request_(0)
45 artdaq::RequestReceiver::RequestReceiver(
const fhicl::ParameterSet& ps)
46 : request_port_(ps.get<int>(
"request_port", 3001))
47 , request_addr_(ps.get<std::string>(
"request_address",
"227.128.12.26"))
48 , multicast_out_addr_(ps.get<std::string>(
"multicast_interface_ip",
"0.0.0.0"))
52 , request_stop_requested_(false)
53 , request_received_(false)
54 , end_of_run_timeout_ms_(ps.get<size_t>(
"end_of_run_quiet_timeout_ms", 1000))
56 , highest_seen_request_(0)
58 setupRequestListener();
63 TLOG(TLVL_INFO) <<
"Setting up request listen socket, rank=" << my_rank <<
", address=" << request_addr_ <<
":" << request_port_ ;
64 request_socket_ = socket(AF_INET, SOCK_DGRAM, IPPROTO_UDP);
65 if (request_socket_ < 0)
67 TLOG(TLVL_ERROR) <<
"Error creating socket for receiving data requests! err=" << strerror(errno) ;
71 struct sockaddr_in si_me_request;
74 if (setsockopt(request_socket_, SOL_SOCKET, SO_REUSEADDR, &yes,
sizeof(yes)) < 0)
76 TLOG(TLVL_ERROR) <<
"Unable to enable port reuse on request socket, err=" << strerror(errno) ;
79 memset(&si_me_request, 0,
sizeof(si_me_request));
80 si_me_request.sin_family = AF_INET;
81 si_me_request.sin_port = htons(request_port_);
82 si_me_request.sin_addr.s_addr = htonl(INADDR_ANY);
83 if (bind(request_socket_, (
struct sockaddr *)&si_me_request,
sizeof(si_me_request)) == -1)
85 TLOG(TLVL_ERROR) <<
"Cannot bind request socket to port " << request_port_ <<
", err=" << strerror(errno) ;
89 if (request_addr_ !=
"localhost")
92 int sts =
ResolveHost(request_addr_.c_str(), mreq.imr_multiaddr);
95 TLOG(TLVL_ERROR) <<
"Unable to resolve multicast request address, err=" << strerror(errno) ;
101 TLOG(TLVL_ERROR) <<
"Unable to resolve hostname for " << multicast_out_addr_;
104 if (setsockopt(request_socket_, IPPROTO_IP, IP_ADD_MEMBERSHIP, &mreq,
sizeof(mreq)) < 0)
106 TLOG(TLVL_ERROR) <<
"Unable to join multicast group, err=" << strerror(errno) ;
110 TLOG(TLVL_INFO) <<
"Done setting up request socket, rank=" << my_rank ;
113 artdaq::RequestReceiver::~RequestReceiver()
115 stopRequestReceiverThread();
120 std::unique_lock<std::mutex> lk(state_mutex_);
121 if (!request_received_)
123 TLOG(TLVL_ERROR) <<
"Stop request received by RequestReceiver, but no requests have ever been received." << std::endl
124 <<
"Check that UDP port " << request_port_ <<
" is open in the firewall config.";
127 TLOG(TLVL_DEBUG) <<
"Joining requestThread";
128 if (requestThread_.joinable()) requestThread_.join();
129 while (running_) usleep(10000);
131 if (request_socket_ != -1) {
132 close(request_socket_);
133 request_socket_ = -1;
135 request_received_ =
false;
136 highest_seen_request_ = 0;
141 std::unique_lock<std::mutex> lk(state_mutex_);
142 if (requestThread_.joinable()) requestThread_.join();
143 should_stop_ =
false;
144 request_stop_requested_ =
false;
146 if (request_socket_ == -1) {
147 TLOG(TLVL_INFO) <<
"Connecting Request Reception socket";
148 setupRequestListener();
151 TLOG(TLVL_INFO) <<
"Starting Request Reception Thread" ;
158 while (!should_stop_)
160 TLOG(16) <<
"receiveRequestsLoop: Polling Request socket for new requests" ;
162 if (request_socket_ == -1)
164 setupRequestListener();
168 struct pollfd ufds[1];
169 ufds[0].fd = request_socket_;
170 ufds[0].events = POLLIN | POLLPRI | POLLERR;
171 int rv = poll(ufds, 1, ms_to_wait);
174 if (rv <= 0 || (ufds[0].revents != POLLIN && ufds[0].revents != POLLPRI))
176 if (rv == 1 && (ufds[0].revents == POLLNVAL || ufds[0].revents == POLLERR))
178 close(request_socket_);
179 request_socket_ = -1;
181 if (request_stop_requested_ && TimeUtils::GetElapsedTimeMilliseconds(request_stop_timeout_) > end_of_run_timeout_ms_)
188 TLOG(11) <<
"Recieved packet on Request channel" ;
190 auto sts = recv(request_socket_, &hdr_buffer,
sizeof(hdr_buffer), 0);
193 TLOG(TLVL_ERROR) <<
"Error receiving request message header err=" << strerror(errno);
194 close(request_socket_);
195 request_socket_ = -1;
198 TLOG(11) <<
"Request header word: 0x" << std::hex << hdr_buffer.
header ;
199 if (!hdr_buffer.
isValid())
continue;
201 request_received_ =
true;
204 TLOG(TLVL_INFO) <<
"Received Request Message with the EndOfRun marker. (Re)Starting 1-second timeout for receiving all outstanding requests..." ;
205 request_stop_timeout_ = std::chrono::steady_clock::now();
206 request_stop_requested_ =
true;
209 std::vector<artdaq::detail::RequestPacket> pkt_buffer(hdr_buffer.
packet_count);
216 TLOG(TLVL_ERROR) <<
"Error receiving request message data err=" << strerror(errno);
217 close(request_socket_);
218 request_socket_ = -1;
226 if (should_stop_)
break;
228 for (
auto& buffer : pkt_buffer)
230 if (!buffer.isValid())
continue;
231 if (requests_.count(buffer.sequence_id) && requests_[buffer.sequence_id] != buffer.timestamp)
233 TLOG(TLVL_ERROR) <<
"Received conflicting request for SeqID "
234 << std::to_string(buffer.sequence_id) <<
"!"
235 <<
" Old ts=" << std::to_string(requests_[buffer.sequence_id])
236 <<
", new ts=" << std::to_string(buffer.timestamp) <<
". Keeping OLD!" ;
238 else if (!requests_.count(buffer.sequence_id))
240 int delta = buffer.sequence_id - highest_seen_request_;
241 TLOG(11) <<
"Recieved request for sequence ID " << std::to_string(buffer.sequence_id)
242 <<
" and timestamp " << std::to_string(buffer.timestamp) <<
" (delta: " << delta <<
")" ;
245 TLOG(11) <<
"Already serviced this request! Ignoring..." ;
249 std::unique_lock<std::mutex> tlk(request_mutex_);
250 requests_[buffer.sequence_id] = buffer.timestamp;
251 request_timing_[buffer.sequence_id] = std::chrono::steady_clock::now();
257 request_cv_.notify_all();
260 TLOG(TLVL_DEBUG) <<
"Ending Request Thread";
264 void artdaq::RequestReceiver::RemoveRequest(artdaq::Fragment::sequence_id_t reqID)
266 std::unique_lock<std::mutex> lk(request_mutex_);
267 requests_.erase(reqID);
268 if (reqID > highest_seen_request_ && !should_stop_)
270 TLOG(18) <<
"Setting highest_seen_request_ to " << reqID;
271 highest_seen_request_ = reqID;
276 metricMan->sendMetric(
"Request Response Time", TimeUtils::GetElapsedTime(request_timing_[reqID]),
"seconds", 2, MetricMode::Average);
278 request_timing_.erase(reqID);
int ResolveHost(char const *host_in, in_addr &addr)
Convert a string hostname to a in_addr suitable for socket communication.
End of Run mode (Used to end request processing on receiver)
void startRequestReceiverThread()
Function that launches the data request receiver thread (receiveRequestsLoop())
int GetInterfaceForNetwork(char const *host_in, in_addr &addr)
Convert an IP address to the network address of the interface sharing the subnet mask.
void setupRequestListener()
Opens the socket used to listen for data requests.
void receiveRequestsLoop()
This function receives data request packets, adding new requests to the request list.
The RequestPacket contains information about a single data request.
void stopRequestReceiverThread()
Stop the data request receiver thread (receiveRequestsLoop)