artdaq  v3_08_00
RequestSender.cc
1 #include "artdaq/DAQdata/Globals.hh" // Before trace.h gets included in ConcurrentQueue (from GlobalQueue)
2 #define TRACE_NAME (app_name + "_RequestSender").c_str()
3 #include <dlfcn.h>
4 #include <chrono>
5 #include <cstring>
6 #include <fstream>
7 #include <iomanip>
8 #include <sstream>
9 #include <utility>
10 #include "artdaq/DAQrate/RequestSender.hh"
11 
12 #include "artdaq-core/Core/SimpleMemoryReader.hh"
13 #include "artdaq-core/Core/StatisticsCollection.hh"
15 #include "artdaq/DAQrate/detail/RoutingPacket.hh"
16 #include "cetlib_except/exception.h"
17 
18 namespace artdaq {
19 RequestSender::RequestSender(const fhicl::ParameterSet& pset)
20  : send_requests_(pset.get<bool>("send_requests", false))
21  , initialized_(false)
22  , active_requests_()
23  , request_address_(pset.get<std::string>("request_address", "227.128.12.26"))
24  , request_port_(pset.get<int>("request_port", 3001))
25  , request_delay_(pset.get<size_t>("request_delay_ms", 0) * 1000)
26  , request_shutdown_timeout_us_(pset.get<size_t>("request_shutdown_timeout_us", 100000))
27  , request_socket_(-1)
28  , multicast_out_addr_(pset.get<std::string>("multicast_interface_ip", pset.get<std::string>("output_address", "0.0.0.0")))
29  , request_mode_(detail::RequestMessageMode::Normal)
30  , token_socket_(-1)
31  , request_sending_(0)
32  , tokens_sent_(0)
33  , run_number_(0)
34 {
35  TLOG(TLVL_DEBUG) << "RequestSender CONSTRUCTOR";
36  setup_requests_();
37 
38  auto rmConfig = pset.get<fhicl::ParameterSet>("routing_token_config", fhicl::ParameterSet());
39  send_routing_tokens_ = rmConfig.get<bool>("use_routing_master", false);
40  token_port_ = rmConfig.get<int>("routing_token_port", 35555);
41  token_address_ = rmConfig.get<std::string>("routing_master_hostname", "localhost");
42  setup_tokens_();
43  TLOG(12) << "artdaq::RequestSender::RequestSender ctor - reader_thread_ initialized";
44  initialized_ = true;
45 }
46 
48 {
49  TLOG(TLVL_INFO) << "Shutting down RequestSender: Waiting for " << request_sending_.load() << " requests to be sent";
50 
51  auto start_time = std::chrono::steady_clock::now();
52 
53  while (request_sending_.load() > 0 && request_shutdown_timeout_us_ + request_delay_ > TimeUtils::GetElapsedTimeMicroseconds(start_time))
54  {
55  usleep(1000);
56  }
57  {
58  std::unique_lock<std::mutex> lk(request_mutex_);
59  std::unique_lock<std::mutex> lk2(request_send_mutex_);
60  }
61  TLOG(TLVL_INFO) << "Shutting down RequestSender: request_socket_: " << request_socket_ << ", token_socket_: " << token_socket_;
62  if (request_socket_ != -1)
63  {
64  if (shutdown(request_socket_, 2) != 0 && errno == ENOTSOCK)
65  {
66  TLOG(TLVL_ERROR) << "Shutdown of request_socket_ resulted in ENOTSOCK. NOT Closing file descriptor!";
67  }
68  else
69  {
70  close(request_socket_);
71  }
72  request_socket_ = -1;
73  }
74  if (token_socket_ != -1)
75  {
76  if (shutdown(token_socket_, 2) != 0 && errno == ENOTSOCK)
77  {
78  TLOG(TLVL_ERROR) << "Shutdown of token_socket_ resulted in ENOTSOCK. NOT Closing file descriptor!";
79  }
80  else
81  {
82  close(token_socket_);
83  }
84  token_socket_ = -1;
85  }
86 }
87 
89 {
90  request_mode_ = mode;
91  SendRequest(true);
92 }
93 
94 void RequestSender::setup_requests_()
95 {
96  if (send_requests_)
97  {
98  request_socket_ = socket(AF_INET, SOCK_DGRAM, IPPROTO_UDP);
99  if (request_socket_ < 0)
100  {
101  TLOG(TLVL_ERROR) << "I failed to create the socket for sending Data Requests! err=" << strerror(errno);
102  exit(1);
103  }
104  int sts = ResolveHost(request_address_.c_str(), request_port_, request_addr_);
105  if (sts == -1)
106  {
107  TLOG(TLVL_ERROR) << "Unable to resolve Data Request address, err=" << strerror(errno);
108  exit(1);
109  }
110 
111 /* if (multicast_out_addr_ == "0.0.0.0")
112  {
113  char hostname[HOST_NAME_MAX];
114  sts = gethostname(hostname, HOST_NAME_MAX);
115  multicast_out_addr_ = std::string(hostname);
116  if (sts < 0)
117  {
118  TLOG(TLVL_ERROR) << "Could not get current hostname, err=" << strerror(errno);
119  exit(1);
120  }
121  }*/
122 
123  // For 0.0.0.0, use system-specified IP_MULTICAST_IF
124  if (multicast_out_addr_ != "localhost" && multicast_out_addr_ != "0.0.0.0")
125  {
126  struct in_addr addr;
127  sts = GetInterfaceForNetwork(multicast_out_addr_.c_str(), addr);
128  //sts = ResolveHost(multicast_out_addr_.c_str(), addr);
129  if (sts == -1)
130  {
131  TLOG(TLVL_ERROR) << "Unable to determine the multicast interface address for " << multicast_out_addr_ << ", err=" << strerror(errno);
132  exit(1);
133  }
134  char addr_str[INET_ADDRSTRLEN];
135  inet_ntop(AF_INET, &(addr), addr_str, INET_ADDRSTRLEN);
136  TLOG(TLVL_INFO) << "Successfully determined the multicast network interface for " << multicast_out_addr_ << ": " << addr_str;
137 
138  if (setsockopt(request_socket_, IPPROTO_IP, IP_MULTICAST_IF, &addr, sizeof(addr)) == -1)
139  {
140  TLOG(TLVL_ERROR) << "Cannot set outgoing interface, err=" << strerror(errno);
141  exit(1);
142  }
143  }
144  int yes = 1;
145  if (setsockopt(request_socket_, SOL_SOCKET, SO_REUSEADDR, &yes, sizeof(yes)) < 0)
146  {
147  TLOG(TLVL_ERROR) << "Unable to enable port reuse on request socket, err=" << strerror(errno);
148  exit(1);
149  }
150  if (setsockopt(request_socket_, IPPROTO_IP, IP_MULTICAST_LOOP, &yes, sizeof(yes)) < 0)
151  {
152  TLOG(TLVL_ERROR) << "Unable to enable multicast loopback on request socket, err=" << strerror(errno);
153  exit(1);
154  }
155  if (setsockopt(request_socket_, SOL_SOCKET, SO_BROADCAST, (void*)&yes, sizeof(int)) == -1)
156  {
157  TLOG(TLVL_ERROR) << "Cannot set request socket to broadcast, err=" << strerror(errno);
158  exit(1);
159  }
160  }
161 }
162 
163 void RequestSender::setup_tokens_()
164 {
165  if (send_routing_tokens_)
166  {
167  TLOG(TLVL_DEBUG) << "Creating Routing Token sending socket";
168  auto start_time = std::chrono::steady_clock::now();
169  while (token_socket_ < 0 && TimeUtils::GetElapsedTime(start_time) < 30)
170  {
171  token_socket_ = TCPConnect(token_address_.c_str(), token_port_, 0, sizeof(detail::RoutingToken));
172  if (token_socket_ < 0)
173  {
174  TLOG(TLVL_TRACE) << "Waited " << TimeUtils::GetElapsedTime(start_time) << " s for Routing Master to open token socket";
175  usleep(100000);
176  }
177  }
178  if (token_socket_ < 0)
179  {
180  TLOG(TLVL_ERROR) << "I failed to create the socket for sending Routing Tokens! err=" << strerror(errno);
181  exit(1);
182  }
183  TLOG(TLVL_INFO) << "Routing Token sending socket created successfully for address " << token_address_;
184  }
185 }
186 
187 void RequestSender::do_send_request_()
188 {
189  if (!send_requests_)
190  {
191  request_sending_--;
192  return;
193  }
194  if (request_socket_ == -1) setup_requests_();
195 
196  TLOG(TLVL_TRACE) << "Waiting for " << request_delay_ << " microseconds.";
197  std::this_thread::sleep_for(std::chrono::microseconds(request_delay_));
198 
199  TLOG(TLVL_TRACE) << "Creating RequestMessage";
200  detail::RequestMessage message;
201  message.setRank(my_rank);
202  message.setRunNumber(run_number_);
203  {
204  std::unique_lock<std::mutex> lk(request_mutex_);
205  for (auto& req : active_requests_)
206  {
207  TLOG(12) << "Adding a request with sequence ID " << req.first << ", timestamp " << req.second << " to request message";
208  message.addRequest(req.first, req.second);
209  }
210  }
211  TLOG(TLVL_TRACE) << "Setting mode flag in Message Header to " << request_mode_;
212  message.setMode(request_mode_);
213  char str[INET_ADDRSTRLEN];
214  inet_ntop(AF_INET, &(request_addr_.sin_addr), str, INET_ADDRSTRLEN);
215  std::unique_lock<std::mutex> lk2(request_send_mutex_);
216  TLOG(TLVL_TRACE) << "Sending request for " << message.size() << " events to multicast group " << str
217  << ", port " << request_port_ << ", interface " << multicast_out_addr_;
218  auto buf = message.GetMessage();
219  auto sts = sendto(request_socket_, &buf[0], buf.size(), 0, (struct sockaddr*)&request_addr_, sizeof(request_addr_));
220  if (sts < 0 || static_cast<size_t>(sts) != buf.size())
221  {
222  TLOG(TLVL_ERROR) << "Error sending request message err=" << strerror(errno) << "sts=" << sts;
223  request_socket_ = -1;
224  request_sending_--;
225  return;
226  }
227  TLOG(TLVL_TRACE) << "Done sending request sts=" << sts;
228  request_sending_--;
229 }
230 
231 void RequestSender::send_routing_token_(int nSlots, int run_number)
232 {
233  TLOG(TLVL_TRACE) << "send_routing_token_ called, send_routing_tokens_=" << std::boolalpha << send_routing_tokens_;
234  if (!send_routing_tokens_) return;
235  if (token_socket_ == -1) setup_tokens_();
236  detail::RoutingToken token;
237  token.header = TOKEN_MAGIC;
238  token.rank = my_rank;
239  token.new_slots_free = nSlots;
240  token.run_number = run_number;
241 
242  TLOG(TLVL_TRACE) << "Sending RoutingToken to " << token_address_ << ":" << token_port_;
243  size_t sts = 0;
244  while (sts < sizeof(detail::RoutingToken))
245  {
246  auto res = send(token_socket_, reinterpret_cast<uint8_t*>(&token) + sts, sizeof(detail::RoutingToken) - sts, 0);
247  if (res < 0)
248  {
249  TLOG(TLVL_WARNING) << "Error on token_socket, reconnecting";
250  close(token_socket_);
251  token_socket_ = -1;
252  sts = 0;
253  setup_tokens_();
254  continue;
255  }
256  sts += res;
257  }
258  tokens_sent_ += nSlots;
259  TLOG(TLVL_TRACE) << "Done sending RoutingToken to " << token_address_ << ":" << token_port_;
260 }
261 
262 void RequestSender::SendRoutingToken(int nSlots, int run_number)
263 {
264  while (!initialized_) usleep(1000);
265  if (!send_routing_tokens_) return;
266  boost::thread token([=] { send_routing_token_(nSlots, run_number); });
267  token.detach();
268  usleep(0); // Give up time slice
269 }
270 
271 void RequestSender::SendRequest(bool endOfRunOnly)
272 {
273  while (!initialized_) usleep(1000);
274 
275  if (!send_requests_) return;
276  if (endOfRunOnly && request_mode_ != detail::RequestMessageMode::EndOfRun) return;
277  request_sending_++;
278  boost::thread request([=] { do_send_request_(); });
279  request.detach();
280 }
281 
282 void RequestSender::AddRequest(Fragment::sequence_id_t seqID, Fragment::timestamp_t timestamp)
283 {
284  while (!initialized_) usleep(1000);
285 
286  {
287  std::lock_guard<std::mutex> lk(request_mutex_);
288  if (!active_requests_.count(seqID))
289  {
290  TLOG(12) << "Adding request for sequence ID " << seqID << " and timestamp " << timestamp << " to request list.";
291  active_requests_[seqID] = timestamp;
292  }
293  }
294  SendRequest();
295 }
296 
297 void RequestSender::RemoveRequest(Fragment::sequence_id_t seqID)
298 {
299  while (!initialized_) usleep(1000);
300  std::lock_guard<std::mutex> lk(request_mutex_);
301  TLOG(12) << "Removing request for sequence ID " << seqID << " from request list.";
302  active_requests_.erase(seqID);
303 }
304 } // namespace artdaq
void RemoveRequest(Fragment::sequence_id_t seqID)
Remove a request from the request list.
int ResolveHost(char const *host_in, in_addr &addr)
Convert a string hostname to a in_addr suitable for socket communication.
Definition: TCPConnect.cc:33
int TCPConnect(char const *host_in, int dflt_port, long flags=0, int sndbufsiz=0)
Connect to a host on a given port.
Definition: TCPConnect.cc:361
End of Run mode (Used to end request processing on receiver)
void SendRoutingToken(int nSlots, int run_number)
Send a RoutingToken message indicating that slots are available.
virtual ~RequestSender()
RequestSender Destructor.
void SendRequest(bool endOfRunOnly=false)
Send a request message containing all current requests.
int GetInterfaceForNetwork(char const *host_in, in_addr &addr)
Convert an IP address to the network address of the interface sharing the subnet mask.
Definition: TCPConnect.cc:217
RequestSender()=delete
Default Constructor is deleted.
void SetRequestMode(detail::RequestMessageMode mode)
Set the mode for RequestMessages. Used to indicate when RequestSender should enter &quot;EndOfRun&quot; mode...
RequestMessageMode
Mode used to indicate current run conditions to the request receiver.
void AddRequest(Fragment::sequence_id_t seqID, Fragment::timestamp_t timestamp)
Add a request to the request list.