artdaq  v3_01_00
RequestReceiver.cc
1 #define TRACE_NAME "RequestReceiver"
2 
3 #include "artdaq/DAQrate/RequestReceiver.hh"
4 #include "artdaq/DAQdata/Globals.hh"
5 #include "artdaq/DAQrate/detail/RequestMessage.hh"
6 
7 #include <boost/exception/all.hpp>
8 #include <boost/throw_exception.hpp>
9 
10 #include <limits>
11 #include <iterator>
12 
13 #include "canvas/Utilities/Exception.h"
14 #include "cetlib_except/exception.h"
15 #include "fhiclcpp/ParameterSet.h"
16 
17 #include "artdaq-core/Utilities/SimpleLookupPolicy.hh"
18 #include "artdaq-core/Data/Fragment.hh"
19 #include "artdaq-core/Data/ContainerFragmentLoader.hh"
20 #include "artdaq-core/Utilities/ExceptionHandler.hh"
21 #include "artdaq-core/Utilities/TimeUtils.hh"
22 
23 #include <fstream>
24 #include <iomanip>
25 #include <iterator>
26 #include <iostream>
27 #include <iomanip>
28 #include <algorithm>
29 #include <sys/poll.h>
31 
32 artdaq::RequestReceiver::RequestReceiver()
33  : request_port_(3001)
34  , request_addr_("227.128.12.26")
35  , running_(false)
36  , requests_()
37  , request_timing_()
38  , request_stop_requested_(false)
39  , request_received_(false)
40  , end_of_run_timeout_ms_(1000)
41  , should_stop_(false)
42  , highest_seen_request_(0)
43 {}
44 
45 artdaq::RequestReceiver::RequestReceiver(const fhicl::ParameterSet& ps)
46  : request_port_(ps.get<int>("request_port", 3001))
47  , request_addr_(ps.get<std::string>("request_address", "227.128.12.26"))
48  , running_(false)
49  , requests_()
50  , request_timing_()
51  , request_stop_requested_(false)
52  , request_received_(false)
53  , end_of_run_timeout_ms_(ps.get<size_t>("end_of_run_quiet_timeout_ms", 1000))
54  , should_stop_(false)
55  , highest_seen_request_(0)
56 {
57  setupRequestListener();
58 }
59 
61 {
62  TLOG(TLVL_INFO) << "Setting up request listen socket, rank=" << my_rank << ", address=" << request_addr_ << ":" << request_port_ ;
63  request_socket_ = socket(AF_INET, SOCK_DGRAM, IPPROTO_UDP);
64  if (request_socket_ < 0)
65  {
66  TLOG(TLVL_ERROR) << "Error creating socket for receiving data requests! err=" << strerror(errno) ;
67  exit(1);
68  }
69 
70  struct sockaddr_in si_me_request;
71 
72  int yes = 1;
73  if (setsockopt(request_socket_, SOL_SOCKET, SO_REUSEADDR, &yes, sizeof(yes)) < 0)
74  {
75  TLOG(TLVL_ERROR) << "Unable to enable port reuse on request socket, err=" << strerror(errno) ;
76  exit(1);
77  }
78  memset(&si_me_request, 0, sizeof(si_me_request));
79  si_me_request.sin_family = AF_INET;
80  si_me_request.sin_port = htons(request_port_);
81  si_me_request.sin_addr.s_addr = htonl(INADDR_ANY);
82  if (bind(request_socket_, (struct sockaddr *)&si_me_request, sizeof(si_me_request)) == -1)
83  {
84  TLOG(TLVL_ERROR) << "Cannot bind request socket to port " << request_port_ << ", err=" << strerror(errno) ;
85  exit(1);
86  }
87 
88  if (request_addr_ != "localhost")
89  {
90  struct ip_mreq mreq;
91  int sts = ResolveHost(request_addr_.c_str(), mreq.imr_multiaddr);
92  if (sts == -1)
93  {
94  TLOG(TLVL_ERROR) << "Unable to resolve multicast request address, err=" << strerror(errno) ;
95  exit(1);
96  }
97  mreq.imr_interface.s_addr = htonl(INADDR_ANY);
98  if (setsockopt(request_socket_, IPPROTO_IP, IP_ADD_MEMBERSHIP, &mreq, sizeof(mreq)) < 0)
99  {
100  TLOG(TLVL_ERROR) << "Unable to join multicast group, err=" << strerror(errno) ;
101  exit(1);
102  }
103  }
104  TLOG(TLVL_INFO) << "Done setting up request socket, rank=" << my_rank ;
105 }
106 
107 artdaq::RequestReceiver::~RequestReceiver()
108 {
109  stopRequestReceiverThread();
110 }
111 
113 {
114  std::unique_lock<std::mutex> lk(state_mutex_);
115  if (!request_received_)
116  {
117  TLOG(TLVL_ERROR) << "Stop request received by RequestReceiver, but no requests have ever been received." << std::endl
118  << "Check that UDP port " << request_port_ << " is open in the firewall config.";
119  }
120  should_stop_ = true;
121  TLOG(TLVL_DEBUG) << "Joining requestThread";
122  if (requestThread_.joinable()) requestThread_.join();
123  while (running_) usleep(10000);
124 
125  if (request_socket_ != -1) {
126  close(request_socket_);
127  request_socket_ = -1;
128  }
129  request_received_ = false;
130  highest_seen_request_ = 0;
131 }
132 
134 {
135  std::unique_lock<std::mutex> lk(state_mutex_);
136  if (requestThread_.joinable()) requestThread_.join();
137  should_stop_ = false;
138  request_stop_requested_ = false;
139 
140  if (request_socket_ == -1) {
141  TLOG(TLVL_INFO) << "Connecting Request Reception socket";
142  setupRequestListener();
143  }
144 
145  TLOG(TLVL_INFO) << "Starting Request Reception Thread" ;
146  requestThread_ = boost::thread(&RequestReceiver::receiveRequestsLoop, this);
147  running_ = true;
148 }
149 
151 {
152  while (!should_stop_)
153  {
154  TLOG(16) << "receiveRequestsLoop: Polling Request socket for new requests" ;
155 
156  int ms_to_wait = 10;
157  struct pollfd ufds[1];
158  ufds[0].fd = request_socket_;
159  ufds[0].events = POLLIN | POLLPRI;
160  int rv = poll(ufds, 1, ms_to_wait);
161 
162  // Continue loop if no message received or message does not have correct event ID
163  if (rv <= 0 || (ufds[0].revents != POLLIN && ufds[0].revents != POLLPRI))
164  {
165  if (request_stop_requested_ && TimeUtils::GetElapsedTimeMilliseconds(request_stop_timeout_) > end_of_run_timeout_ms_)
166  {
167  break;
168  }
169  continue;
170  }
171 
172  TLOG(11) << "Recieved packet on Request channel" ;
174  recv(request_socket_, &hdr_buffer, sizeof(hdr_buffer), 0);
175  TLOG(11) << "Request header word: 0x" << std::hex << hdr_buffer.header ;
176  if (!hdr_buffer.isValid()) continue;
177 
178  request_received_ = true;
180  {
181  TLOG(TLVL_INFO) << "Received Request Message with the EndOfRun marker. (Re)Starting 1-second timeout for receiving all outstanding requests..." ;
182  request_stop_timeout_ = std::chrono::steady_clock::now();
183  request_stop_requested_ = true;
184  }
185 
186  std::vector<artdaq::detail::RequestPacket> pkt_buffer(hdr_buffer.packet_count);
187  recv(request_socket_, &pkt_buffer[0], sizeof(artdaq::detail::RequestPacket) * hdr_buffer.packet_count, 0);
188  bool anyNew = false;
189 
190  if (should_stop_) break;
191 
192  for (auto& buffer : pkt_buffer)
193  {
194  if (!buffer.isValid()) continue;
195  if (requests_.count(buffer.sequence_id) && requests_[buffer.sequence_id] != buffer.timestamp)
196  {
197  TLOG(TLVL_ERROR) << "Received conflicting request for SeqID "
198  << std::to_string(buffer.sequence_id) << "!"
199  << " Old ts=" << std::to_string(requests_[buffer.sequence_id])
200  << ", new ts=" << std::to_string(buffer.timestamp) << ". Keeping OLD!" ;
201  }
202  else if (!requests_.count(buffer.sequence_id))
203  {
204  int delta = buffer.sequence_id - highest_seen_request_;
205  TLOG(11) << "Recieved request for sequence ID " << std::to_string(buffer.sequence_id)
206  << " and timestamp " << std::to_string(buffer.timestamp) << " (delta: " << delta << ")" ;
207  if (delta < 0)
208  {
209  TLOG(11) << "Already serviced this request! Ignoring..." ;
210  }
211  else
212  {
213  std::unique_lock<std::mutex> tlk(request_mutex_);
214  requests_[buffer.sequence_id] = buffer.timestamp;
215  request_timing_[buffer.sequence_id] = std::chrono::steady_clock::now();
216  anyNew = true;
217  }
218  }
219  }
220  if (anyNew) {
221  request_cv_.notify_all();
222  }
223  }
224  TLOG(TLVL_DEBUG) << "Ending Request Thread";
225  running_ = false;
226 }
227 
228 void artdaq::RequestReceiver::RemoveRequest(artdaq::Fragment::sequence_id_t reqID)
229 {
230  std::unique_lock<std::mutex> lk(request_mutex_);
231  requests_.erase(reqID);
232  if (reqID > highest_seen_request_ && !should_stop_) // Stop accounting for requests after stop
233  {
234  TLOG(18) << "Setting highest_seen_request_ to " << reqID;
235  highest_seen_request_ = reqID;
236  }
237 
238  if (metricMan)
239  {
240  metricMan->sendMetric("Request Response Time", TimeUtils::GetElapsedTime(request_timing_[reqID]), "seconds", 2, MetricMode::Average);
241  }
242  request_timing_.erase(reqID);
243 }
int ResolveHost(char const *host_in, in_addr &addr)
Convert a string hostname to a in_addr suitable for socket communication.
Definition: TCPConnect.cc:29
End of Run mode (Used to end request processing on receiver)
RequestMessageMode mode
Communicates additional information to the Request receiver.
void startRequestReceiverThread()
Function that launches the data request receiver thread (receiveRequestsLoop())
bool isValid() const
Check the magic bytes of the packet.
Header of a RequestMessage. Contains magic bytes for validation and a count of expected RequestPacket...
uint32_t packet_count
The number of RequestPackets in this Request message.
void setupRequestListener()
Opens the socket used to listen for data requests.
void receiveRequestsLoop()
This function receives data request packets, adding new requests to the request list.
The RequestPacket contains information about a single data request.
void stopRequestReceiverThread()
Stop the data request receiver thread (receiveRequestsLoop)