artdaq  v3_12_02
RequestReceiver.cc
1 #include "artdaq/DAQdata/Globals.hh"
2 #define TRACE_NAME (app_name + "_RequestReceiver").c_str()
3 
4 #include "artdaq/DAQdata/Globals.hh"
5 #include "artdaq/DAQrate/detail/RequestMessage.hh"
6 #include "artdaq/DAQrate/detail/RequestReceiver.hh"
7 
8 #include <boost/exception/all.hpp>
9 #include <boost/throw_exception.hpp>
10 
11 #include <iterator>
12 #include <limits>
13 
14 #include "canvas/Utilities/Exception.h"
15 #include "cetlib_except/exception.h"
16 #include "fhiclcpp/ParameterSet.h"
17 
18 #include "artdaq-core/Data/ContainerFragmentLoader.hh"
19 #include "artdaq-core/Data/Fragment.hh"
20 #include "artdaq-core/Utilities/ExceptionHandler.hh"
21 #include "artdaq-core/Utilities/SimpleLookupPolicy.hh"
22 #include "artdaq-core/Utilities/TimeUtils.hh"
23 
24 #include <arpa/inet.h>
25 #include <netinet/in.h>
26 #include <sys/poll.h>
27 #include <algorithm>
28 #include <fstream>
29 #include <iomanip>
30 #include <iostream>
31 #include <iterator>
33 
35  : request_stop_requested_(false)
36  , request_received_(false)
37  , should_stop_(false)
38  , request_addr_("227.128.12.26")
39  , receive_requests_(false)
40 {}
41 
42 artdaq::RequestReceiver::RequestReceiver(const fhicl::ParameterSet& ps, std::shared_ptr<RequestBuffer> output_buffer)
43  : request_stop_requested_(false)
44  , request_received_(false)
45  , should_stop_(false)
46  , request_port_(ps.get<int>("request_port", 3001))
47  , request_addr_(ps.get<std::string>("request_address", "227.128.12.26"))
48  , multicast_in_addr_(ps.get<std::string>("multicast_interface_ip", "0.0.0.0"))
49  , receive_requests_(ps.get<bool>("receive_requests", false))
50  , end_of_run_timeout_ms_(ps.get<size_t>("end_of_run_quiet_timeout_ms", 1000))
51  , requests_(output_buffer)
52 {
53  TLOG(TLVL_DEBUG + 32) << "RequestReceiver CONSTRUCTOR ps: " << ps.to_string();
54  if (receive_requests_)
55  {
57  }
58 }
59 
61 {
62  TLOG(TLVL_INFO) << "Setting up request listen socket, rank=" << my_rank << ", address=" << request_addr_ << ":" << request_port_
63  << ", multicast interface=" << multicast_in_addr_;
64  request_socket_ = socket(AF_INET, SOCK_DGRAM, IPPROTO_UDP);
65  if (request_socket_ < 0)
66  {
67  TLOG(TLVL_ERROR) << "Error creating socket for receiving data requests! err=" << strerror(errno);
68  exit(1);
69  }
70 
71  struct sockaddr_in si_me_request;
72 
73  int yes = 1;
74  if (setsockopt(request_socket_, SOL_SOCKET, SO_REUSEADDR, &yes, sizeof(yes)) < 0)
75  {
76  TLOG(TLVL_ERROR) << "Unable to enable port reuse on request socket, err=" << strerror(errno);
77  exit(1);
78  }
79  memset(&si_me_request, 0, sizeof(si_me_request));
80  si_me_request.sin_family = AF_INET;
81  si_me_request.sin_port = htons(request_port_);
82  si_me_request.sin_addr.s_addr = htonl(INADDR_ANY);
83  if (bind(request_socket_, reinterpret_cast<struct sockaddr*>(&si_me_request), sizeof(si_me_request)) == -1) // NOLINT(cppcoreguidelines-pro-type-reinterpret-cast)
84  {
85  TLOG(TLVL_ERROR) << "Cannot bind request socket to port " << request_port_ << ", err=" << strerror(errno);
86  exit(1);
87  }
88 
89  if (request_addr_ != "localhost")
90  {
91  struct ip_mreq mreq;
92  int sts = ResolveHost(request_addr_.c_str(), mreq.imr_multiaddr);
93  if (sts == -1)
94  {
95  TLOG(TLVL_ERROR) << "Unable to resolve multicast request address, err=" << strerror(errno);
96  exit(1);
97  }
98  sts = GetInterfaceForNetwork(multicast_in_addr_.c_str(), mreq.imr_interface);
99  if (sts == -1)
100  {
101  TLOG(TLVL_ERROR) << "Unable to determine the multicast network interface for " << multicast_in_addr_;
102  exit(1);
103  }
104  char addr_str[INET_ADDRSTRLEN];
105  inet_ntop(AF_INET, &(mreq.imr_interface), addr_str, INET_ADDRSTRLEN);
106  TLOG(TLVL_INFO) << "Successfully determined the multicast network interface for " << multicast_in_addr_ << ": " << addr_str << " (RequestReceiver)";
107  if (setsockopt(request_socket_, IPPROTO_IP, IP_ADD_MEMBERSHIP, &mreq, sizeof(mreq)) < 0)
108  {
109  TLOG(TLVL_ERROR) << "Unable to join multicast group, err=" << strerror(errno);
110  exit(1);
111  }
112  }
113  TLOG(TLVL_INFO) << "Done setting up request socket, rank=" << my_rank;
114 }
115 
117 {
118  stopRequestReception(true);
119 }
120 
122 {
123  std::unique_lock<std::mutex> lk(state_mutex_);
124  if (!receive_requests_) return;
125  if (!request_received_ && !force)
126  {
127  TLOG(TLVL_ERROR) << "Stop request received by RequestReceiver, but no requests have ever been received." << std::endl
128  << "Check that UDP port " << request_port_ << " is open in the firewall config.";
129  }
130  should_stop_ = true;
131  if (running_)
132  {
133  TLOG(TLVL_DEBUG + 32) << "Joining requestThread";
134  try
135  {
136  if (requestThread_.joinable())
137  {
138  requestThread_.join();
139  }
140  }
141  catch (...)
142  {
143  // IGNORED
144  }
145  bool once = true;
146  while (running_)
147  {
148  if (once)
149  {
150  TLOG(TLVL_ERROR) << "running_ is true after thread join! Should NOT happen";
151  }
152  once = false;
153  usleep(10000);
154  }
155  }
156 
157  if (request_socket_ != -1)
158  {
159  close(request_socket_);
160  request_socket_ = -1;
161  }
162  request_received_ = false;
163 }
164 
166 {
167  if (!receive_requests_) return;
168  std::unique_lock<std::mutex> lk(state_mutex_);
169  if (requestThread_.joinable())
170  {
171  requestThread_.join();
172  }
173  should_stop_ = false;
174  request_stop_requested_ = false;
175 
176  if (request_socket_ == -1)
177  {
178  TLOG(TLVL_INFO) << "Connecting Request Reception socket";
179  setupRequestListener();
180  }
181 
182  TLOG(TLVL_INFO) << "Starting Request Reception Thread";
183  try
184  {
185  requestThread_ = boost::thread(&RequestReceiver::receiveRequestsLoop, this);
186  char tname[16]; // Size 16 - see man page pthread_setname_np(3) and/or prctl(2)
187  snprintf(tname, sizeof(tname) - 1, "%d-ReqRecv", my_rank); // NOLINT
188  tname[sizeof(tname) - 1] = '\0'; // assure term. snprintf is not too evil :)
189  auto handle = requestThread_.native_handle();
190  pthread_setname_np(handle, tname);
191  }
192  catch (const boost::exception& e)
193  {
194  TLOG(TLVL_ERROR) << "Caught boost::exception starting Request Receiver thread: " << boost::diagnostic_information(e) << ", errno=" << errno;
195  std::cerr << "Caught boost::exception starting Request Receiver thread: " << boost::diagnostic_information(e) << ", errno=" << errno << std::endl;
196  exit(5);
197  }
198 }
199 
201 {
202  running_ = true;
203  requests_->reset();
204  requests_->setRunning(true);
205  while (!should_stop_)
206  {
207  TLOG(TLVL_DEBUG + 35) << "receiveRequestsLoop: Polling Request socket for new requests";
208 
209  if (request_socket_ == -1)
210  {
211  setupRequestListener();
212  }
213 
214  int ms_to_wait = 10;
215  struct pollfd ufds[1];
216  ufds[0].fd = request_socket_;
217  ufds[0].events = POLLIN | POLLPRI | POLLERR;
218  int rv = poll(ufds, 1, ms_to_wait);
219 
220  // Continue loop if no message received or message does not have correct event ID
221  if (rv <= 0 || (ufds[0].revents != POLLIN && ufds[0].revents != POLLPRI))
222  {
223  if (rv == 1 && ((ufds[0].revents & (POLLNVAL | POLLERR | POLLHUP)) != 0))
224  {
225  close(request_socket_);
226  request_socket_ = -1;
227  }
228  if (request_stop_requested_ && TimeUtils::GetElapsedTimeMilliseconds(request_stop_timeout_) > end_of_run_timeout_ms_)
229  {
230  break;
231  }
232  continue;
233  }
234 
235  TLOG(TLVL_DEBUG + 34) << "Received packet on Request channel";
236  std::vector<uint8_t> buffer(MAX_REQUEST_MESSAGE_SIZE);
237  struct sockaddr_in from;
238  socklen_t len = sizeof(from);
239  auto sts = recvfrom(request_socket_, &buffer[0], MAX_REQUEST_MESSAGE_SIZE, 0, reinterpret_cast<struct sockaddr*>(&from), &len); // NOLINT(cppcoreguidelines-pro-type-reinterpret-cast)
240  if (sts < 0)
241  {
242  TLOG(TLVL_ERROR) << "Error receiving request message header err=" << strerror(errno);
243  close(request_socket_);
244  request_socket_ = -1;
245  continue;
246  }
247 
248  auto hdr_buffer = reinterpret_cast<artdaq::detail::RequestHeader*>(&buffer[0]); // NOLINT(cppcoreguidelines-pro-type-reinterpret-cast)
249  TLOG(TLVL_DEBUG + 34) << "Request header word: 0x" << std::hex << hdr_buffer->header << std::dec << ", packet_count: " << hdr_buffer->packet_count << " from rank " << hdr_buffer->rank << ", " << inet_ntoa(from.sin_addr) << ":" << from.sin_port << ", run number: " << hdr_buffer->run_number;
250  if (!hdr_buffer->isValid())
251  {
252  continue;
253  }
254 
255  request_received_ = true;
256 
257  // 19-Dec-2018, KAB: added check on current run number
258  if (run_number_ != 0 && hdr_buffer->run_number != run_number_)
259  {
260  TLOG(TLVL_WARNING) << "Received a Request Message with the wrong run number ("
261  << hdr_buffer->run_number << "), expected " << run_number_
262  << ", ignoring this request.";
263  continue;
264  }
265 
266  if (hdr_buffer->mode == artdaq::detail::RequestMessageMode::EndOfRun)
267  {
268  TLOG(TLVL_INFO) << "Received Request Message with the EndOfRun marker. (Re)Starting 1-second timeout for receiving all outstanding requests...";
269  request_stop_timeout_ = std::chrono::steady_clock::now();
270  request_stop_requested_ = true;
271  }
272 
273  std::vector<artdaq::detail::RequestPacket> pkt_buffer(hdr_buffer->packet_count);
274  memcpy(&pkt_buffer[0], &buffer[sizeof(artdaq::detail::RequestHeader)], sizeof(artdaq::detail::RequestPacket) * hdr_buffer->packet_count);
275 
276  if (should_stop_)
277  {
278  break;
279  }
280 
281  for (auto& buffer : pkt_buffer)
282  {
283  TLOG(TLVL_DEBUG + 36) << "Request Packet: hdr=" << /*std::dec <<*/ buffer.header << ", seq=" << buffer.sequence_id << ", ts=" << buffer.timestamp;
284  if (!buffer.isValid()) continue;
285  requests_->push(buffer.sequence_id, buffer.timestamp);
286  }
287  }
288  TLOG(TLVL_DEBUG + 32) << "Ending Request Thread";
289  running_ = false;
290  requests_->setRunning(false);
291 }
int ResolveHost(char const *host_in, in_addr &addr)
Convert a string hostname to a in_addr suitable for socket communication.
Definition: TCPConnect.cc:34
void startRequestReception()
Enables (starts) the reception of data requests.
End of Run mode (Used to end request processing on receiver)
RequestReceiver()
RequestReceiver Default Constructor.
int GetInterfaceForNetwork(char const *host_in, in_addr &addr)
Convert an IP address to the network address of the interface sharing the subnet mask.
Definition: TCPConnect.cc:224
Header of a RequestMessage. Contains magic bytes for validation and a count of expected RequestPacket...
void setupRequestListener()
Opens the socket used to listen for data requests.
void receiveRequestsLoop()
This function receives data request packets, adding new requests to the request list.
The RequestPacket contains information about a single data request.
virtual ~RequestReceiver()
RequestReceiver Destructor.
void stopRequestReception(bool force=false)
Disables (stops) the reception of data requests.