1 #include "artdaq/DAQdata/Globals.hh"
2 #define TRACE_NAME (app_name + "_RequestReceiver").c_str()
4 #include "artdaq/DAQdata/Globals.hh"
5 #include "artdaq/DAQrate/detail/RequestMessage.hh"
6 #include "artdaq/DAQrate/detail/RequestReceiver.hh"
8 #include <boost/exception/all.hpp>
9 #include <boost/throw_exception.hpp>
14 #include "canvas/Utilities/Exception.h"
15 #include "cetlib_except/exception.h"
16 #include "fhiclcpp/ParameterSet.h"
18 #include "artdaq-core/Data/ContainerFragmentLoader.hh"
19 #include "artdaq-core/Data/Fragment.hh"
20 #include "artdaq-core/Utilities/ExceptionHandler.hh"
21 #include "artdaq-core/Utilities/SimpleLookupPolicy.hh"
22 #include "artdaq-core/Utilities/TimeUtils.hh"
24 #include <arpa/inet.h>
25 #include <netinet/in.h>
35 : request_stop_requested_(false)
36 , requests_received_(0)
38 , request_addr_(
"227.128.12.26")
39 , receive_requests_(false)
43 : request_stop_requested_(false)
44 , requests_received_(0)
46 , request_port_(ps.get<int>(
"request_port", 3001))
47 , request_addr_(ps.get<std::string>(
"request_address",
"227.128.12.26"))
48 , multicast_in_addr_(ps.get<std::string>(
"multicast_interface_ip",
"0.0.0.0"))
49 , receive_requests_(ps.get<bool>(
"receive_requests", false))
50 , end_of_run_timeout_ms_(ps.get<size_t>(
"end_of_run_quiet_timeout_ms", 1000))
51 , requests_(output_buffer)
53 TLOG(TLVL_DEBUG + 32) <<
"RequestReceiver CONSTRUCTOR ps: " << ps.to_string();
54 if (receive_requests_)
62 TLOG(TLVL_INFO) <<
"Setting up request listen socket, rank=" << my_rank <<
", address=" << request_addr_ <<
":" << request_port_
63 <<
", multicast interface=" << multicast_in_addr_;
64 request_socket_ = socket(AF_INET, SOCK_DGRAM, IPPROTO_UDP);
65 if (request_socket_ < 0)
67 TLOG(TLVL_ERROR) <<
"Error creating socket for receiving data requests! err=" << strerror(errno);
71 struct sockaddr_in si_me_request;
74 if (setsockopt(request_socket_, SOL_SOCKET, SO_REUSEADDR, &yes,
sizeof(yes)) < 0)
76 TLOG(TLVL_ERROR) <<
"Unable to enable port reuse on request socket, err=" << strerror(errno);
79 memset(&si_me_request, 0,
sizeof(si_me_request));
80 si_me_request.sin_family = AF_INET;
81 si_me_request.sin_port = htons(request_port_);
82 si_me_request.sin_addr.s_addr = htonl(INADDR_ANY);
83 if (bind(request_socket_, reinterpret_cast<struct sockaddr*>(&si_me_request),
sizeof(si_me_request)) == -1)
85 TLOG(TLVL_ERROR) <<
"Cannot bind request socket to port " << request_port_ <<
", err=" << strerror(errno);
89 if (request_addr_ !=
"localhost")
92 int sts =
ResolveHost(request_addr_.c_str(), mreq.imr_multiaddr);
95 TLOG(TLVL_ERROR) <<
"Unable to resolve multicast request address, err=" << strerror(errno);
101 TLOG(TLVL_ERROR) <<
"Unable to determine the multicast network interface for " << multicast_in_addr_;
104 char addr_str[INET_ADDRSTRLEN];
105 inet_ntop(AF_INET, &(mreq.imr_interface), addr_str, INET_ADDRSTRLEN);
106 TLOG(TLVL_INFO) <<
"Successfully determined the multicast network interface for " << multicast_in_addr_ <<
": " << addr_str <<
" (RequestReceiver)";
107 if (setsockopt(request_socket_, IPPROTO_IP, IP_ADD_MEMBERSHIP, &mreq,
sizeof(mreq)) < 0)
109 TLOG(TLVL_ERROR) <<
"Unable to join multicast group, err=" << strerror(errno);
113 TLOG(TLVL_INFO) <<
"Done setting up request socket, rank=" << my_rank;
118 stopRequestReception(
true);
123 std::unique_lock<std::mutex> lk(state_mutex_);
124 if (!receive_requests_)
return;
125 if (requests_received_ == 0 && !force)
127 TLOG(TLVL_ERROR) <<
"Stop request received by RequestReceiver, but no requests have ever been received." << std::endl
128 <<
"Check that UDP port " << request_port_ <<
" is open in the firewall config.";
133 TLOG(TLVL_DEBUG + 32) <<
"Joining requestThread";
136 if (requestThread_.joinable())
138 requestThread_.join();
150 TLOG(TLVL_ERROR) <<
"running_ is true after thread join! Should NOT happen";
157 if (request_socket_ != -1)
159 close(request_socket_);
160 request_socket_ = -1;
162 TLOG(TLVL_INFO) <<
"RequestReceiver stopped, received " << requests_received_ <<
" request messages";
163 requests_received_ = 0;
168 if (!receive_requests_)
return;
169 std::unique_lock<std::mutex> lk(state_mutex_);
170 if (requestThread_.joinable())
172 requestThread_.join();
174 should_stop_ =
false;
175 request_stop_requested_ =
false;
177 if (request_socket_ == -1)
179 TLOG(TLVL_INFO) <<
"Connecting Request Reception socket";
180 setupRequestListener();
183 TLOG(TLVL_INFO) <<
"Starting Request Reception Thread";
188 snprintf(tname,
sizeof(tname) - 1,
"%d-ReqRecv", my_rank);
189 tname[
sizeof(tname) - 1] =
'\0';
190 auto handle = requestThread_.native_handle();
191 pthread_setname_np(handle, tname);
193 catch (
const boost::exception& e)
195 TLOG(TLVL_ERROR) <<
"Caught boost::exception starting Request Receiver thread: " << boost::diagnostic_information(e) <<
", errno=" << errno;
196 std::cerr <<
"Caught boost::exception starting Request Receiver thread: " << boost::diagnostic_information(e) <<
", errno=" << errno << std::endl;
205 requests_->setRunning(
true);
206 while (!should_stop_)
208 TLOG(TLVL_DEBUG + 35) <<
"receiveRequestsLoop: Polling Request socket for new requests";
210 if (request_socket_ == -1)
212 setupRequestListener();
216 struct pollfd ufds[1];
217 ufds[0].fd = request_socket_;
218 ufds[0].events = POLLIN | POLLPRI | POLLERR;
219 int rv = poll(ufds, 1, ms_to_wait);
222 if (rv <= 0 || (ufds[0].revents != POLLIN && ufds[0].revents != POLLPRI))
224 if (rv == 1 && ((ufds[0].revents & (POLLNVAL | POLLERR | POLLHUP)) != 0))
226 close(request_socket_);
227 request_socket_ = -1;
229 if (request_stop_requested_ && TimeUtils::GetElapsedTimeMilliseconds(request_stop_timeout_) > end_of_run_timeout_ms_)
236 TLOG(TLVL_DEBUG + 34) <<
"Received packet on Request channel";
237 std::vector<uint8_t> buffer(MAX_REQUEST_MESSAGE_SIZE);
238 struct sockaddr_in from;
239 socklen_t len =
sizeof(from);
240 auto sts = recvfrom(request_socket_, &buffer[0], MAX_REQUEST_MESSAGE_SIZE, 0, reinterpret_cast<struct sockaddr*>(&from), &len);
243 TLOG(TLVL_ERROR) <<
"Error receiving request message header err=" << strerror(errno);
244 close(request_socket_);
245 request_socket_ = -1;
250 TLOG(TLVL_DEBUG + 34) <<
"Request header word: 0x" << std::hex << hdr_buffer->
header << std::dec <<
", packet_count: " << hdr_buffer->packet_count <<
" from rank " << hdr_buffer->rank <<
", " << inet_ntoa(from.sin_addr) <<
":" << from.sin_port <<
", run number: " << hdr_buffer->run_number;
251 if (!hdr_buffer->isValid())
257 if (run_number_ != 0 && hdr_buffer->run_number != run_number_)
259 TLOG(TLVL_WARNING) <<
"Received a Request Message with the wrong run number ("
260 << hdr_buffer->run_number <<
"), expected " << run_number_
261 <<
", ignoring this request.";
265 requests_received_++;
269 TLOG(TLVL_INFO) <<
"Received Request Message with the EndOfRun marker. (Re)Starting 1-second timeout for receiving all outstanding requests...";
270 request_stop_timeout_ = std::chrono::steady_clock::now();
271 request_stop_requested_ =
true;
274 std::vector<artdaq::detail::RequestPacket> pkt_buffer(hdr_buffer->packet_count);
282 for (
auto& buffer : pkt_buffer)
284 TLOG(TLVL_DEBUG + 36) <<
"Request Packet: hdr=" << buffer.header <<
", seq=" << buffer.sequence_id <<
", ts=" << buffer.timestamp;
285 if (!buffer.isValid())
continue;
286 requests_->push(buffer.sequence_id, buffer.timestamp);
289 TLOG(TLVL_DEBUG + 32) <<
"Ending Request Thread";
291 requests_->setRunning(
false);
int ResolveHost(char const *host_in, in_addr &addr)
Convert a string hostname to a in_addr suitable for socket communication.
void startRequestReception()
Enables (starts) the reception of data requests.
End of Run mode (Used to end request processing on receiver)
RequestReceiver()
RequestReceiver Default Constructor.
int GetInterfaceForNetwork(char const *host_in, in_addr &addr)
Convert an IP address to the network address of the interface sharing the subnet mask.
void setupRequestListener()
Opens the socket used to listen for data requests.
void receiveRequestsLoop()
This function receives data request packets, adding new requests to the request list.
The RequestPacket contains information about a single data request.
virtual ~RequestReceiver()
RequestReceiver Destructor.
void stopRequestReception(bool force=false)
Disables (stops) the reception of data requests.