artdaq  v3_09_06a
RequestSender.cc
1 #include "artdaq/DAQdata/Globals.hh" // Before trace.h gets included in ConcurrentQueue (from GlobalQueue)
2 #define TRACE_NAME (app_name + "_RequestSender").c_str()
3 #include <dlfcn.h>
4 #include <chrono>
5 #include <cstring>
6 #include <fstream>
7 #include <iomanip>
8 #include <sstream>
9 #include <utility>
10 #include "artdaq/DAQrate/RequestSender.hh"
11 
12 #include "artdaq-core/Core/StatisticsCollection.hh"
14 #include "artdaq/DAQrate/detail/RoutingPacket.hh"
15 #include "cetlib_except/exception.h"
16 
17 namespace artdaq {
18 RequestSender::RequestSender(const fhicl::ParameterSet& pset)
19  : send_requests_(pset.get<bool>("send_requests", false))
20  , initialized_(false)
21  , request_address_(pset.get<std::string>("request_address", "227.128.12.26"))
22  , request_port_(pset.get<int>("request_port", 3001))
23  , request_delay_(pset.get<size_t>("request_delay_ms", 0) * 1000)
24  , request_shutdown_timeout_us_(pset.get<size_t>("request_shutdown_timeout_us", 100000))
25  , request_socket_(-1)
26  , multicast_out_addr_(pset.get<std::string>("multicast_interface_ip", pset.get<std::string>("output_address", "0.0.0.0")))
27  , request_mode_(detail::RequestMessageMode::Normal)
28  , token_socket_(-1)
29  , request_sending_(0)
30  , tokens_sent_(0)
31  , run_number_(0)
32 {
33  TLOG(TLVL_DEBUG) << "RequestSender CONSTRUCTOR";
34  setup_requests_();
35 
36  auto rmConfig = pset.get<fhicl::ParameterSet>("routing_token_config", fhicl::ParameterSet());
37  send_routing_tokens_ = rmConfig.get<bool>("use_routing_manager", false);
38  token_port_ = rmConfig.get<int>("routing_token_port", 35555);
39  token_address_ = rmConfig.get<std::string>("routing_manager_hostname", "localhost");
40  setup_tokens_();
41  TLOG(12) << "artdaq::RequestSender::RequestSender ctor - reader_thread_ initialized";
42  initialized_ = true;
43 }
44 
46 {
47  TLOG(TLVL_INFO) << "Shutting down RequestSender: Waiting for " << request_sending_.load() << " requests to be sent";
48 
49  auto start_time = std::chrono::steady_clock::now();
50 
51  while (request_sending_.load() > 0 && request_shutdown_timeout_us_ + request_delay_ > TimeUtils::GetElapsedTimeMicroseconds(start_time))
52  {
53  usleep(1000);
54  }
55  {
56  std::lock_guard<std::mutex> lk(request_mutex_);
57  std::lock_guard<std::mutex> lk2(request_send_mutex_);
58  }
59  TLOG(TLVL_INFO) << "Shutting down RequestSender: request_socket_: " << request_socket_ << ", token_socket_: " << token_socket_;
60  if (request_socket_ != -1)
61  {
62  if (shutdown(request_socket_, 2) != 0 && errno == ENOTSOCK)
63  {
64  TLOG(TLVL_ERROR) << "Shutdown of request_socket_ resulted in ENOTSOCK. NOT Closing file descriptor!";
65  }
66  else
67  {
68  close(request_socket_);
69  }
70  request_socket_ = -1;
71  }
72  if (token_socket_ != -1)
73  {
74  if (shutdown(token_socket_, 2) != 0 && errno == ENOTSOCK)
75  {
76  TLOG(TLVL_ERROR) << "Shutdown of token_socket_ resulted in ENOTSOCK. NOT Closing file descriptor!";
77  }
78  else
79  {
80  close(token_socket_);
81  }
82  token_socket_ = -1;
83  }
84 }
85 
87 {
88  {
89  std::lock_guard<std::mutex> lk(request_mutex_);
90  request_mode_ = mode;
91  }
92  SendRequest(true);
93 }
94 
95 void RequestSender::setup_requests_()
96 {
97  if (send_requests_)
98  {
99  request_socket_ = socket(AF_INET, SOCK_DGRAM, IPPROTO_UDP);
100  if (request_socket_ < 0)
101  {
102  TLOG(TLVL_ERROR) << "I failed to create the socket for sending Data Requests! err=" << strerror(errno);
103  exit(1);
104  }
105  int sts = ResolveHost(request_address_.c_str(), request_port_, request_addr_);
106  if (sts == -1)
107  {
108  TLOG(TLVL_ERROR) << "Unable to resolve Data Request address, err=" << strerror(errno);
109  exit(1);
110  }
111 
112  /* if (multicast_out_addr_ == "0.0.0.0")
113  {
114  char hostname[HOST_NAME_MAX];
115  sts = gethostname(hostname, HOST_NAME_MAX);
116  multicast_out_addr_ = std::string(hostname);
117  if (sts < 0)
118  {
119  TLOG(TLVL_ERROR) << "Could not get current hostname, err=" << strerror(errno);
120  exit(1);
121  }
122  }*/
123 
124  // For 0.0.0.0, use system-specified IP_MULTICAST_IF
125  if (multicast_out_addr_ != "localhost" && multicast_out_addr_ != "0.0.0.0")
126  {
127  struct in_addr addr;
128  sts = GetInterfaceForNetwork(multicast_out_addr_.c_str(), addr);
129  //sts = ResolveHost(multicast_out_addr_.c_str(), addr);
130  if (sts == -1)
131  {
132  TLOG(TLVL_ERROR) << "Unable to determine the multicast interface address for " << multicast_out_addr_ << ", err=" << strerror(errno);
133  exit(1);
134  }
135  char addr_str[INET_ADDRSTRLEN];
136  inet_ntop(AF_INET, &(addr), addr_str, INET_ADDRSTRLEN);
137  TLOG(TLVL_INFO) << "Successfully determined the multicast network interface for " << multicast_out_addr_ << ": " << addr_str;
138 
139  if (setsockopt(request_socket_, IPPROTO_IP, IP_MULTICAST_IF, &addr, sizeof(addr)) == -1)
140  {
141  TLOG(TLVL_ERROR) << "Cannot set outgoing interface, err=" << strerror(errno);
142  exit(1);
143  }
144  }
145  int yes = 1;
146  if (setsockopt(request_socket_, SOL_SOCKET, SO_REUSEADDR, &yes, sizeof(yes)) < 0)
147  {
148  TLOG(TLVL_ERROR) << "Unable to enable port reuse on request socket, err=" << strerror(errno);
149  exit(1);
150  }
151  if (setsockopt(request_socket_, IPPROTO_IP, IP_MULTICAST_LOOP, &yes, sizeof(yes)) < 0)
152  {
153  TLOG(TLVL_ERROR) << "Unable to enable multicast loopback on request socket, err=" << strerror(errno);
154  exit(1);
155  }
156  if (setsockopt(request_socket_, SOL_SOCKET, SO_BROADCAST, &yes, sizeof(yes)) == -1)
157  {
158  TLOG(TLVL_ERROR) << "Cannot set request socket to broadcast, err=" << strerror(errno);
159  exit(1);
160  }
161  }
162 }
163 
164 void RequestSender::setup_tokens_()
165 {
166  if (send_routing_tokens_)
167  {
168  TLOG(TLVL_DEBUG) << "Creating Routing Token sending socket";
169  auto start_time = std::chrono::steady_clock::now();
170  while (token_socket_ < 0 && TimeUtils::GetElapsedTime(start_time) < 30)
171  {
172  token_socket_ = TCPConnect(token_address_.c_str(), token_port_, 0, sizeof(detail::RoutingToken));
173  if (token_socket_ < 0)
174  {
175  TLOG(TLVL_TRACE) << "Waited " << TimeUtils::GetElapsedTime(start_time) << " s for Routing Manager to open token socket";
176  usleep(100000);
177  }
178  }
179  if (token_socket_ < 0)
180  {
181  TLOG(TLVL_ERROR) << "I failed to create the socket for sending Routing Tokens! err=" << strerror(errno);
182  exit(1);
183  }
184  TLOG(TLVL_INFO) << "Routing Token sending socket created successfully for address " << token_address_;
185  }
186 }
187 
188 void RequestSender::do_send_request_()
189 {
190  if (!send_requests_)
191  {
192  request_sending_--;
193  return;
194  }
195  if (request_socket_ == -1)
196  {
197  setup_requests_();
198  }
199 
200  TLOG(TLVL_TRACE) << "Waiting for " << request_delay_ << " microseconds.";
201  std::this_thread::sleep_for(std::chrono::microseconds(request_delay_));
202 
203  TLOG(TLVL_TRACE) << "Creating RequestMessage";
204  detail::RequestMessage message;
205  message.setRank(my_rank);
206  message.setRunNumber(run_number_);
207  {
208  std::lock_guard<std::mutex> lk(request_mutex_);
209  for (auto& req : active_requests_)
210  {
211  TLOG(12) << "Adding a request with sequence ID " << req.first << ", timestamp " << req.second << " to request message";
212  message.addRequest(req.first, req.second);
213  }
214  TLOG(TLVL_TRACE) << "Setting mode flag in Message Header to " << request_mode_;
215  message.setMode(request_mode_);
216  }
217  char str[INET_ADDRSTRLEN];
218  inet_ntop(AF_INET, &(request_addr_.sin_addr), str, INET_ADDRSTRLEN);
219  std::lock_guard<std::mutex> lk2(request_send_mutex_);
220  TLOG(TLVL_TRACE) << "Sending request for " << message.size() << " events to multicast group " << str
221  << ", port " << request_port_ << ", interface " << multicast_out_addr_;
222  auto buf = message.GetMessage();
223  auto sts = sendto(request_socket_, &buf[0], buf.size(), 0, reinterpret_cast<struct sockaddr*>(&request_addr_), sizeof(request_addr_)); // NOLINT(cppcoreguidelines-pro-type-reinterpret-cast)
224  if (sts < 0 || static_cast<size_t>(sts) != buf.size())
225  {
226  TLOG(TLVL_ERROR) << "Error sending request message err=" << strerror(errno) << "sts=" << sts;
227  request_socket_ = -1;
228  request_sending_--;
229  return;
230  }
231  TLOG(TLVL_TRACE) << "Done sending request sts=" << sts;
232  request_sending_--;
233 }
234 
235 void RequestSender::send_routing_token_(int nSlots, int run_number)
236 {
237  TLOG(TLVL_TRACE) << "send_routing_token_ called, send_routing_tokens_=" << std::boolalpha << send_routing_tokens_;
238  if (!send_routing_tokens_)
239  {
240  return;
241  }
242  if (token_socket_ == -1)
243  {
244  setup_tokens_();
245  }
246  detail::RoutingToken token;
247  token.header = TOKEN_MAGIC;
248  token.rank = my_rank;
249  token.new_slots_free = nSlots;
250  token.run_number = run_number;
251 
252  TLOG(TLVL_TRACE) << "Sending RoutingToken to " << token_address_ << ":" << token_port_;
253  size_t sts = 0;
254  while (sts < sizeof(detail::RoutingToken))
255  {
256  auto res = send(token_socket_, reinterpret_cast<uint8_t*>(&token) + sts, sizeof(detail::RoutingToken) - sts, 0); // NOLINT(cppcoreguidelines-pro-type-reinterpret-cast,cppcoreguidelines-pro-bounds-pointer-arithmetic)
257  if (res < 0)
258  {
259  TLOG(TLVL_WARNING) << "Error on token_socket, reconnecting";
260  close(token_socket_);
261  token_socket_ = -1;
262  sts = 0;
263  setup_tokens_();
264  continue;
265  }
266  sts += res;
267  }
268  tokens_sent_ += nSlots;
269  TLOG(TLVL_TRACE) << "Done sending RoutingToken to " << token_address_ << ":" << token_port_;
270 }
271 
272 void RequestSender::SendRoutingToken(int nSlots, int run_number)
273 {
274  while (!initialized_)
275  {
276  usleep(1000);
277  }
278  if (!send_routing_tokens_)
279  {
280  return;
281  }
282  boost::thread token([=] { send_routing_token_(nSlots, run_number); });
283  token.detach();
284  usleep(0); // Give up time slice
285 }
286 
287 void RequestSender::SendRequest(bool endOfRunOnly)
288 {
289  while (!initialized_)
290  {
291  usleep(1000);
292  }
293 
294  if (!send_requests_)
295  {
296  return;
297  }
298  {
299  std::lock_guard<std::mutex> lk(request_mutex_);
300  if (endOfRunOnly && request_mode_ != detail::RequestMessageMode::EndOfRun)
301  {
302  return;
303  }
304  }
305  request_sending_++;
306  boost::thread request([=] { do_send_request_(); });
307  request.detach();
308 }
309 
310 void RequestSender::AddRequest(Fragment::sequence_id_t seqID, Fragment::timestamp_t timestamp)
311 {
312  while (!initialized_)
313  {
314  usleep(1000);
315  }
316 
317  {
318  std::lock_guard<std::mutex> lk(request_mutex_);
319  if (active_requests_.count(seqID) == 0u)
320  {
321  TLOG(12) << "Adding request for sequence ID " << seqID << " and timestamp " << timestamp << " to request list.";
322  active_requests_[seqID] = timestamp;
323  }
324  }
325  SendRequest();
326 }
327 
328 void RequestSender::RemoveRequest(Fragment::sequence_id_t seqID)
329 {
330  while (!initialized_)
331  {
332  usleep(1000);
333  }
334  std::lock_guard<std::mutex> lk(request_mutex_);
335  TLOG(12) << "Removing request for sequence ID " << seqID << " from request list.";
336  active_requests_.erase(seqID);
337 }
338 } // namespace artdaq
void RemoveRequest(Fragment::sequence_id_t seqID)
Remove a request from the request list.
int ResolveHost(char const *host_in, in_addr &addr)
Convert a string hostname to a in_addr suitable for socket communication.
Definition: TCPConnect.cc:33
End of Run mode (Used to end request processing on receiver)
int TCPConnect(char const *host_in, int dflt_port, int64_t flags=0, int sndbufsiz=0)
Connect to a host on a given port.
Definition: TCPConnect.cc:376
void SendRoutingToken(int nSlots, int run_number)
Send a RoutingToken message indicating that slots are available.
virtual ~RequestSender()
RequestSender Destructor.
void SendRequest(bool endOfRunOnly=false)
Send a request message containing all current requests.
int GetInterfaceForNetwork(char const *host_in, in_addr &addr)
Convert an IP address to the network address of the interface sharing the subnet mask.
Definition: TCPConnect.cc:223
RequestSender()=delete
Default Constructor is deleted.
void SetRequestMode(detail::RequestMessageMode mode)
Set the mode for RequestMessages. Used to indicate when RequestSender should enter &quot;EndOfRun&quot; mode...
RequestMessageMode
Mode used to indicate current run conditions to the request receiver.
void AddRequest(Fragment::sequence_id_t seqID, Fragment::timestamp_t timestamp)
Add a request to the request list.