artdaq  v3_05_00
RequestSender.cc
1 #define TRACE_NAME (app_name + "_RequestSender").c_str()
2 #include "artdaq/DAQdata/Globals.hh" // Before trace.h gets included in ConcurrentQueue (from GlobalQueue)
3 #include "artdaq/DAQrate/RequestSender.hh"
4 #include <utility>
5 #include <cstring>
6 #include <dlfcn.h>
7 #include <iomanip>
8 #include <fstream>
9 #include <sstream>
10 #include <chrono>
11 
12 #include "cetlib_except/exception.h"
13 #include "artdaq-core/Core/StatisticsCollection.hh"
14 #include "artdaq-core/Core/SimpleMemoryReader.hh"
15 #include "artdaq/DAQrate/detail/RoutingPacket.hh"
17 
18 namespace artdaq
19 {
20  RequestSender::RequestSender(const fhicl::ParameterSet& pset)
21  : send_requests_(pset.get<bool>("send_requests", false))
22  , initialized_(false)
23  , active_requests_()
24  , request_address_(pset.get<std::string>("request_address", "227.128.12.26"))
25  , request_port_(pset.get<int>("request_port", 3001))
26  , request_delay_(pset.get<size_t>("request_delay_ms", 0) * 1000)
27  , request_shutdown_timeout_us_(pset.get<size_t>("request_shutdown_timeout_us", 100000))
28  , request_socket_(-1)
29  , multicast_out_addr_(pset.get<std::string>("multicast_interface_ip", pset.get<std::string>("output_address", "0.0.0.0")))
30  , request_mode_(detail::RequestMessageMode::Normal)
31  , token_socket_(-1)
32  , request_sending_(0)
33  , tokens_sent_(0)
34  , run_number_(0)
35  {
36  TLOG(TLVL_DEBUG) << "RequestSender CONSTRUCTOR";
37  setup_requests_();
38 
39  auto rmConfig = pset.get<fhicl::ParameterSet>("routing_token_config", fhicl::ParameterSet());
40  send_routing_tokens_ = rmConfig.get<bool>("use_routing_master", false);
41  token_port_ = rmConfig.get<int>("routing_token_port", 35555);
42  token_address_ = rmConfig.get<std::string>("routing_master_hostname", "localhost");
43  setup_tokens_();
44  TLOG(12) << "artdaq::RequestSender::RequestSender ctor - reader_thread_ initialized";
45  initialized_ = true;
46  }
47 
48 
50  {
51  TLOG(TLVL_INFO) << "Shutting down RequestSender: Waiting for " << request_sending_.load() << " requests to be sent";
52 
53  auto start_time = std::chrono::steady_clock::now();
54 
55  while (request_sending_.load() > 0 && request_shutdown_timeout_us_ + request_delay_ > TimeUtils::GetElapsedTimeMicroseconds(start_time))
56  {
57  usleep(1000);
58  }
59  {
60  std::unique_lock<std::mutex> lk(request_mutex_);
61  std::unique_lock<std::mutex> lk2(request_send_mutex_);
62  }
63  TLOG(TLVL_INFO) << "Shutting down RequestSender: request_socket_: " << request_socket_ << ", token_socket_: " << token_socket_;
64  if (request_socket_ != -1)
65  {
66  if (shutdown(request_socket_, 2) != 0 && errno == ENOTSOCK)
67  {
68  TLOG(TLVL_ERROR) << "Shutdown of request_socket_ resulted in ENOTSOCK. NOT Closing file descriptor!";
69  }
70  else
71  {
72  close(request_socket_);
73  }
74  request_socket_ = -1;
75  }
76  if (token_socket_ != -1)
77  {
78  if(shutdown(token_socket_, 2) != 0 && errno == ENOTSOCK)
79  {
80  TLOG(TLVL_ERROR) << "Shutdown of token_socket_ resulted in ENOTSOCK. NOT Closing file descriptor!";
81  }
82  else
83  {
84  close(token_socket_);
85  }
86  token_socket_ = -1;
87  }
88  }
89 
90 
92  {
93  request_mode_ = mode;
94  SendRequest(true);
95  }
96 
97  void RequestSender::setup_requests_()
98  {
99  if (send_requests_)
100  {
101  request_socket_ = socket(AF_INET, SOCK_DGRAM, IPPROTO_UDP);
102  if (request_socket_ < 0)
103  {
104  TLOG(TLVL_ERROR) << "I failed to create the socket for sending Data Requests! err=" << strerror(errno);
105  exit(1);
106  }
107  int sts = ResolveHost(request_address_.c_str(), request_port_, request_addr_);
108  if (sts == -1)
109  {
110  TLOG(TLVL_ERROR) << "Unable to resolve Data Request address, err=" << strerror(errno);
111  exit(1);
112  }
113 
114  if (multicast_out_addr_ == "0.0.0.0")
115  {
116  multicast_out_addr_.reserve(HOST_NAME_MAX);
117  sts = gethostname(&multicast_out_addr_[0], HOST_NAME_MAX);
118  if (sts < 0)
119  {
120  TLOG(TLVL_ERROR) << "Could not get current hostname, err=" << strerror(errno);
121  exit(1);
122  }
123  }
124 
125  if (multicast_out_addr_ != "localhost")
126  {
127  struct in_addr addr;
128  sts = GetInterfaceForNetwork(multicast_out_addr_.c_str(), addr);
129  //sts = ResolveHost(multicast_out_addr_.c_str(), addr);
130  if (sts == -1)
131  {
132  TLOG(TLVL_ERROR) << "Unable to resolve multicast interface address, err=" << strerror(errno);
133  exit(1);
134  }
135 
136  if (setsockopt(request_socket_, IPPROTO_IP, IP_MULTICAST_IF, &addr, sizeof(addr)) == -1)
137  {
138  TLOG(TLVL_ERROR) << "Cannot set outgoing interface, err=" << strerror(errno);
139  exit(1);
140  }
141  }
142  int yes = 1;
143  if (setsockopt(request_socket_, SOL_SOCKET, SO_REUSEADDR, &yes, sizeof(yes)) < 0)
144  {
145  TLOG(TLVL_ERROR) << "Unable to enable port reuse on request socket, err=" << strerror(errno);
146  exit(1);
147  }
148  if (setsockopt(request_socket_, IPPROTO_IP, IP_MULTICAST_LOOP, &yes, sizeof(yes)) < 0)
149  {
150  TLOG(TLVL_ERROR) << "Unable to enable multicast loopback on request socket, err=" << strerror(errno);
151  exit(1);
152  }
153  if (setsockopt(request_socket_, SOL_SOCKET, SO_BROADCAST, (void*)&yes, sizeof(int)) == -1)
154  {
155  TLOG(TLVL_ERROR) << "Cannot set request socket to broadcast, err=" << strerror(errno);
156  exit(1);
157  }
158  }
159  }
160 
161  void
162  RequestSender::setup_tokens_()
163  {
164  if (send_routing_tokens_)
165  {
166  TLOG(TLVL_DEBUG) << "Creating Routing Token sending socket";
167  auto start_time = std::chrono::steady_clock::now();
168  while (token_socket_ < 0 && TimeUtils::GetElapsedTime(start_time) < 30)
169  {
170  token_socket_ = TCPConnect(token_address_.c_str(), token_port_, 0, sizeof(detail::RoutingToken));
171  if (token_socket_ < 0)
172  {
173  TLOG(TLVL_TRACE) << "Waited " << TimeUtils::GetElapsedTime(start_time) << " s for Routing Master to open token socket";
174  usleep(100000);
175  }
176  }
177  if (token_socket_ < 0)
178  {
179  TLOG(TLVL_ERROR) << "I failed to create the socket for sending Routing Tokens! err=" << strerror(errno);
180  exit(1);
181  }
182  TLOG(TLVL_DEBUG) << "Routing Token sending socket created successfully";
183  }
184  }
185 
186  void RequestSender::do_send_request_()
187  {
188  if (!send_requests_) {
189  request_sending_--;
190  return;
191  }
192  if (request_socket_ == -1) setup_requests_();
193 
194  TLOG(TLVL_TRACE) << "Waiting for " << request_delay_ << " microseconds.";
195  std::this_thread::sleep_for(std::chrono::microseconds(request_delay_));
196 
197  TLOG(TLVL_TRACE) << "Creating RequestMessage";
198  detail::RequestMessage message;
199  message.setRank(my_rank);
200  message.setRunNumber(run_number_);
201  {
202  std::unique_lock<std::mutex> lk(request_mutex_);
203  for (auto& req : active_requests_)
204  {
205  TLOG(12) << "Adding a request with sequence ID " << req.first << ", timestamp " << req.second << " to request message";
206  message.addRequest(req.first, req.second);
207  }
208  }
209  TLOG(TLVL_TRACE) << "Setting mode flag in Message Header to " << request_mode_;
210  message.setMode(request_mode_);
211  char str[INET_ADDRSTRLEN];
212  inet_ntop(AF_INET, &(request_addr_.sin_addr), str, INET_ADDRSTRLEN);
213  std::unique_lock<std::mutex> lk2(request_send_mutex_);
214  TLOG(TLVL_TRACE) << "Sending request for " << message.size() << " events to multicast group " << str;
215  auto buf = message.GetMessage();
216  auto sts=sendto(request_socket_, &buf[0], buf.size(), 0, (struct sockaddr *)&request_addr_, sizeof(request_addr_));
217  if (sts < 0 || static_cast<size_t>(sts) != buf.size())
218  {
219  TLOG(TLVL_ERROR) << "Error sending request message err=" << strerror(errno)<< "sts="<<sts;
220  request_socket_ = -1;
221  request_sending_--;
222  return;
223  }
224  TLOG(TLVL_TRACE) << "Done sending request sts="<<sts;
225  request_sending_--;
226  }
227 
228  void RequestSender::send_routing_token_(int nSlots, int run_number)
229  {
230  TLOG(TLVL_TRACE) << "send_routing_token_ called, send_routing_tokens_=" << std::boolalpha << send_routing_tokens_;
231  if (!send_routing_tokens_) return;
232  if (token_socket_ == -1) setup_tokens_();
233  detail::RoutingToken token;
234  token.header = TOKEN_MAGIC;
235  token.rank = my_rank;
236  token.new_slots_free = nSlots;
237  token.run_number = run_number;
238 
239  TLOG(TLVL_TRACE) << "Sending RoutingToken to " << token_address_ << ":" << token_port_;
240  size_t sts = 0;
241  while (sts < sizeof(detail::RoutingToken))
242  {
243  auto res = send(token_socket_, reinterpret_cast<uint8_t*>(&token) + sts, sizeof(detail::RoutingToken) - sts, 0);
244  if (res < 0)
245  {
246  close(token_socket_);
247  token_socket_ = -1;
248  sts = 0;
249  setup_tokens_();
250  continue;
251  }
252  sts += res;
253  }
254  tokens_sent_ += nSlots;
255  TLOG(TLVL_TRACE) << "Done sending RoutingToken to " << token_address_ << ":" << token_port_;
256  }
257 
258  void RequestSender::SendRoutingToken(int nSlots, int run_number)
259  {
260  while (!initialized_) usleep(1000);
261  if (!send_routing_tokens_) return;
262  boost::thread token([=] { send_routing_token_(nSlots, run_number); });
263  token.detach();
264  usleep(0); // Give up time slice
265  }
266 
267  void RequestSender::SendRequest(bool endOfRunOnly)
268  {
269  while (!initialized_) usleep(1000);
270 
271  if (!send_requests_) return;
272  if (endOfRunOnly && request_mode_ != detail::RequestMessageMode::EndOfRun) return;
273  request_sending_++;
274  boost::thread request([=] { do_send_request_(); });
275  request.detach();
276  }
277 
278  void RequestSender::AddRequest(Fragment::sequence_id_t seqID, Fragment::timestamp_t timestamp)
279  {
280  while (!initialized_) usleep(1000);
281 
282  {
283  std::lock_guard<std::mutex> lk(request_mutex_);
284  if (!active_requests_.count(seqID)) {
285  TLOG(12) << "Adding request for sequence ID " << seqID << " and timestamp " << timestamp << " to request list.";
286  active_requests_[seqID] = timestamp;
287  }
288  }
289  SendRequest();
290  }
291 
292  void RequestSender::RemoveRequest(Fragment::sequence_id_t seqID)
293  {
294  while (!initialized_) usleep(1000);
295  std::lock_guard<std::mutex> lk(request_mutex_);
296  TLOG(12) << "Removing request for sequence ID " << seqID << " from request list.";
297  active_requests_.erase(seqID);
298  }
299 }
void RemoveRequest(Fragment::sequence_id_t seqID)
Remove a request from the request list.
int ResolveHost(char const *host_in, in_addr &addr)
Convert a string hostname to a in_addr suitable for socket communication.
Definition: TCPConnect.cc:33
int TCPConnect(char const *host_in, int dflt_port, long flags=0, int sndbufsiz=0)
Connect to a host on a given port.
Definition: TCPConnect.cc:358
End of Run mode (Used to end request processing on receiver)
void SendRoutingToken(int nSlots, int run_number)
Send a RoutingToken message indicating that slots are available.
virtual ~RequestSender()
RequestSender Destructor.
void SendRequest(bool endOfRunOnly=false)
Send a request message containing all current requests.
int GetInterfaceForNetwork(char const *host_in, in_addr &addr)
Convert an IP address to the network address of the interface sharing the subnet mask.
Definition: TCPConnect.cc:217
RequestSender()=delete
Default Constructor is deleted.
void SetRequestMode(detail::RequestMessageMode mode)
Set the mode for RequestMessages. Used to indicate when RequestSender should enter &quot;EndOfRun&quot; mode...
RequestMessageMode
Mode used to indicate current run conditions to the request receiver.
void AddRequest(Fragment::sequence_id_t seqID, Fragment::timestamp_t timestamp)
Add a request to the request list.