1 #include "artdaq/DAQdata/Globals.hh"
2 #define TRACE_NAME (app_name + "_RequestReceiver").c_str()
4 #include "artdaq/DAQdata/Globals.hh"
5 #include "artdaq/DAQrate/RequestReceiver.hh"
6 #include "artdaq/DAQrate/detail/RequestMessage.hh"
8 #include <boost/exception/all.hpp>
9 #include <boost/throw_exception.hpp>
14 #include "canvas/Utilities/Exception.h"
15 #include "cetlib_except/exception.h"
16 #include "fhiclcpp/ParameterSet.h"
18 #include "artdaq-core/Data/ContainerFragmentLoader.hh"
19 #include "artdaq-core/Data/Fragment.hh"
20 #include "artdaq-core/Utilities/ExceptionHandler.hh"
21 #include "artdaq-core/Utilities/SimpleLookupPolicy.hh"
22 #include "artdaq-core/Utilities/TimeUtils.hh"
24 #include <arpa/inet.h>
25 #include <netinet/in.h>
35 : request_stop_requested_(false)
36 , request_received_(false)
38 , request_addr_(
"227.128.12.26")
39 , receive_requests_(false)
43 : request_stop_requested_(false)
44 , request_received_(false)
46 , request_port_(ps.get<int>(
"request_port", 3001))
47 , request_addr_(ps.get<std::string>(
"request_address",
"227.128.12.26"))
48 , multicast_in_addr_(ps.get<std::string>(
"multicast_interface_ip",
"0.0.0.0"))
49 , receive_requests_(ps.get<bool>(
"receive_requests", false))
50 , end_of_run_timeout_ms_(ps.get<size_t>(
"end_of_run_quiet_timeout_ms", 1000))
51 , requests_(output_buffer)
53 if (receive_requests_)
61 TLOG(TLVL_INFO) <<
"Setting up request listen socket, rank=" << my_rank <<
", address=" << request_addr_ <<
":" << request_port_
62 <<
", multicast interface=" << multicast_in_addr_;
63 request_socket_ = socket(AF_INET, SOCK_DGRAM, IPPROTO_UDP);
64 if (request_socket_ < 0)
66 TLOG(TLVL_ERROR) <<
"Error creating socket for receiving data requests! err=" << strerror(errno);
70 struct sockaddr_in si_me_request;
73 if (setsockopt(request_socket_, SOL_SOCKET, SO_REUSEADDR, &yes,
sizeof(yes)) < 0)
75 TLOG(TLVL_ERROR) <<
"Unable to enable port reuse on request socket, err=" << strerror(errno);
78 memset(&si_me_request, 0,
sizeof(si_me_request));
79 si_me_request.sin_family = AF_INET;
80 si_me_request.sin_port = htons(request_port_);
81 si_me_request.sin_addr.s_addr = htonl(INADDR_ANY);
82 if (bind(request_socket_, reinterpret_cast<struct sockaddr*>(&si_me_request),
sizeof(si_me_request)) == -1)
84 TLOG(TLVL_ERROR) <<
"Cannot bind request socket to port " << request_port_ <<
", err=" << strerror(errno);
88 if (request_addr_ !=
"localhost")
91 int sts =
ResolveHost(request_addr_.c_str(), mreq.imr_multiaddr);
94 TLOG(TLVL_ERROR) <<
"Unable to resolve multicast request address, err=" << strerror(errno);
100 TLOG(TLVL_ERROR) <<
"Unable to determine the multicast network interface for " << multicast_in_addr_;
103 char addr_str[INET_ADDRSTRLEN];
104 inet_ntop(AF_INET, &(mreq.imr_interface), addr_str, INET_ADDRSTRLEN);
105 TLOG(TLVL_INFO) <<
"Successfully determined the multicast network interface for " << multicast_in_addr_ <<
": " << addr_str <<
" (RequestReceiver)";
106 if (setsockopt(request_socket_, IPPROTO_IP, IP_ADD_MEMBERSHIP, &mreq,
sizeof(mreq)) < 0)
108 TLOG(TLVL_ERROR) <<
"Unable to join multicast group, err=" << strerror(errno);
112 TLOG(TLVL_INFO) <<
"Done setting up request socket, rank=" << my_rank;
117 stopRequestReception(
true);
122 std::unique_lock<std::mutex> lk(state_mutex_);
123 if (!receive_requests_)
return;
124 if (!request_received_ && !force)
126 TLOG(TLVL_ERROR) <<
"Stop request received by RequestReceiver, but no requests have ever been received." << std::endl
127 <<
"Check that UDP port " << request_port_ <<
" is open in the firewall config.";
132 TLOG(TLVL_DEBUG) <<
"Joining requestThread";
135 if (requestThread_.joinable())
137 requestThread_.join();
149 TLOG(TLVL_ERROR) <<
"running_ is true after thread join! Should NOT happen";
156 if (request_socket_ != -1)
158 close(request_socket_);
159 request_socket_ = -1;
161 request_received_ =
false;
166 if (!receive_requests_)
return;
167 std::unique_lock<std::mutex> lk(state_mutex_);
168 if (requestThread_.joinable())
170 requestThread_.join();
172 should_stop_ =
false;
173 request_stop_requested_ =
false;
175 if (request_socket_ == -1)
177 TLOG(TLVL_INFO) <<
"Connecting Request Reception socket";
178 setupRequestListener();
181 TLOG(TLVL_INFO) <<
"Starting Request Reception Thread";
186 snprintf(tname,
sizeof(tname) - 1,
"%d-ReqRecv", my_rank);
187 tname[
sizeof(tname) - 1] =
'\0';
188 auto handle = requestThread_.native_handle();
189 pthread_setname_np(handle, tname);
191 catch (
const boost::exception& e)
193 TLOG(TLVL_ERROR) <<
"Caught boost::exception starting Request Receiver thread: " << boost::diagnostic_information(e) <<
", errno=" << errno;
194 std::cerr <<
"Caught boost::exception starting Request Receiver thread: " << boost::diagnostic_information(e) <<
", errno=" << errno << std::endl;
203 requests_->setRunning(
true);
204 while (!should_stop_)
206 TLOG(16) <<
"receiveRequestsLoop: Polling Request socket for new requests";
208 if (request_socket_ == -1)
210 setupRequestListener();
214 struct pollfd ufds[1];
215 ufds[0].fd = request_socket_;
216 ufds[0].events = POLLIN | POLLPRI | POLLERR;
217 int rv = poll(ufds, 1, ms_to_wait);
220 if (rv <= 0 || (ufds[0].revents != POLLIN && ufds[0].revents != POLLPRI))
222 if (rv == 1 && ((ufds[0].revents & (POLLNVAL | POLLERR | POLLHUP)) != 0))
224 close(request_socket_);
225 request_socket_ = -1;
227 if (request_stop_requested_ && TimeUtils::GetElapsedTimeMilliseconds(request_stop_timeout_) > end_of_run_timeout_ms_)
234 TLOG(11) <<
"Received packet on Request channel";
235 std::vector<uint8_t> buffer(MAX_REQUEST_MESSAGE_SIZE);
236 struct sockaddr_in from;
237 socklen_t len =
sizeof(from);
238 auto sts = recvfrom(request_socket_, &buffer[0], MAX_REQUEST_MESSAGE_SIZE, 0, reinterpret_cast<struct sockaddr*>(&from), &len);
241 TLOG(TLVL_ERROR) <<
"Error receiving request message header err=" << strerror(errno);
242 close(request_socket_);
243 request_socket_ = -1;
248 TLOG(11) <<
"Request header word: 0x" << std::hex << hdr_buffer->
header << std::dec <<
", packet_count: " << hdr_buffer->packet_count <<
" from rank " << hdr_buffer->rank <<
", " << inet_ntoa(from.sin_addr) <<
":" << from.sin_port <<
", run number: " << hdr_buffer->run_number;
249 if (!hdr_buffer->isValid())
254 request_received_ =
true;
257 if (run_number_ != 0 && hdr_buffer->run_number != run_number_)
259 TLOG(TLVL_WARNING) <<
"Received a Request Message with the wrong run number ("
260 << hdr_buffer->run_number <<
"), expected " << run_number_
261 <<
", ignoring this request.";
267 TLOG(TLVL_INFO) <<
"Received Request Message with the EndOfRun marker. (Re)Starting 1-second timeout for receiving all outstanding requests...";
268 request_stop_timeout_ = std::chrono::steady_clock::now();
269 request_stop_requested_ =
true;
272 std::vector<artdaq::detail::RequestPacket> pkt_buffer(hdr_buffer->packet_count);
280 for (
auto& buffer : pkt_buffer)
282 TLOG(20) <<
"Request Packet: hdr=" << buffer.header <<
", seq=" << buffer.sequence_id <<
", ts=" << buffer.timestamp;
283 if (!buffer.isValid())
continue;
284 requests_->push(buffer.sequence_id, buffer.timestamp);
287 TLOG(TLVL_DEBUG) <<
"Ending Request Thread";
289 requests_->setRunning(
false);
int ResolveHost(char const *host_in, in_addr &addr)
Convert a string hostname to a in_addr suitable for socket communication.
void startRequestReception()
Enables (starts) the reception of data requests.
End of Run mode (Used to end request processing on receiver)
RequestReceiver()
RequestReceiver Default Constructor.
int GetInterfaceForNetwork(char const *host_in, in_addr &addr)
Convert an IP address to the network address of the interface sharing the subnet mask.
void setupRequestListener()
Opens the socket used to listen for data requests.
void receiveRequestsLoop()
This function receives data request packets, adding new requests to the request list.
The RequestPacket contains information about a single data request.
virtual ~RequestReceiver()
RequestReceiver Destructor.
void stopRequestReception(bool force=false)
Disables (stops) the reception of data requests.