artdaq  v3_02_01
RequestSender.cc
1 #define TRACE_NAME "RequestSender"
2 #include "artdaq/DAQdata/Globals.hh" // Before trace.h gets included in ConcurrentQueue (from GlobalQueue)
3 #include "artdaq/DAQrate/RequestSender.hh"
4 #include <utility>
5 #include <cstring>
6 #include <dlfcn.h>
7 #include <iomanip>
8 #include <fstream>
9 #include <sstream>
10 #include <chrono>
11 
12 #include "cetlib_except/exception.h"
13 #include "artdaq-core/Core/StatisticsCollection.hh"
14 #include "artdaq-core/Core/SimpleMemoryReader.hh"
15 #include "artdaq/DAQrate/detail/RoutingPacket.hh"
17 
18 namespace artdaq
19 {
20  RequestSender::RequestSender(const fhicl::ParameterSet& pset)
21  : send_requests_(pset.get<bool>("send_requests", false))
22  , active_requests_()
23  , request_address_(pset.get<std::string>("request_address", "227.128.12.26"))
24  , request_port_(pset.get<int>("request_port", 3001))
25  , request_delay_(pset.get<size_t>("request_delay_ms", 10) * 1000)
26  , request_shutdown_timeout_us_(pset.get<size_t>("request_shutdown_timeout_us", 100000))
27  , multicast_out_addr_(pset.get<std::string>("multicast_interface_ip", pset.get<std::string>("output_address", "0.0.0.0")))
28  , request_mode_(detail::RequestMessageMode::Normal)
29  , token_socket_(-1)
30  , request_sending_(0)
31  {
32  TLOG(TLVL_DEBUG) << "RequestSender CONSTRUCTOR";
33  setup_requests_();
34 
35  auto rmConfig = pset.get<fhicl::ParameterSet>("routing_token_config", fhicl::ParameterSet());
36  send_routing_tokens_ = rmConfig.get<bool>("use_routing_master", false);
37  token_port_ = rmConfig.get<int>("routing_token_port", 35555);
38  token_address_ = rmConfig.get<std::string>("routing_master_hostname", "localhost");
39  setup_tokens_();
40  TLOG(12) << "artdaq::RequestSender::RequestSender ctor - reader_thread_ initialized";
41  }
42 
43 
45  {
46  TLOG(TLVL_INFO) << "Shutting down RequestSender: Waiting for requests to be sent";
47 
48  auto start_time = std::chrono::steady_clock::now();
49 
50  while (request_sending_ > 0 && request_shutdown_timeout_us_ > TimeUtils::GetElapsedTimeMicroseconds(start_time))
51  {
52  usleep(1000);
53  }
54  TLOG(TLVL_INFO) << "Shutting down RequestSender";
55  if (request_socket_ > 0)
56  {
57  shutdown(request_socket_, 2);
58  close(request_socket_);
59  }
60  if (token_socket_ > 0)
61  {
62  shutdown(token_socket_, 2);
63  close(token_socket_);
64  }
65  }
66 
67 
69  {
70  request_mode_ = mode;
71  SendRequest(true);
72  }
73 
74  void
75  RequestSender::setup_requests_()
76  {
77  if (send_requests_)
78  {
79  request_socket_ = socket(AF_INET, SOCK_DGRAM, IPPROTO_UDP);
80  if (request_socket_ < 0)
81  {
82  TLOG(TLVL_ERROR) << "I failed to create the socket for sending Data Requests! err=" << strerror(errno);
83  exit(1);
84  }
85  int sts = ResolveHost(request_address_.c_str(), request_port_, request_addr_);
86  if (sts == -1)
87  {
88  TLOG(TLVL_ERROR) << "Unable to resolve Data Request address, err=" << strerror(errno);
89  exit(1);
90  }
91 
92  if (multicast_out_addr_ == "0.0.0.0")
93  {
94  multicast_out_addr_.reserve(HOST_NAME_MAX);
95  sts = gethostname(&multicast_out_addr_[0], HOST_NAME_MAX);
96  if (sts < 0)
97  {
98  TLOG(TLVL_ERROR) << "Could not get current hostname, err=" << strerror(errno);
99  exit(1);
100  }
101  }
102 
103  if (multicast_out_addr_ != "localhost")
104  {
105  struct in_addr addr;
106  sts = GetInterfaceForNetwork(multicast_out_addr_.c_str(), addr);
107  //sts = ResolveHost(multicast_out_addr_.c_str(), addr);
108  if (sts == -1)
109  {
110  TLOG(TLVL_ERROR) << "Unable to resolve multicast interface address, err=" << strerror(errno);
111  exit(1);
112  }
113 
114  if (setsockopt(request_socket_, IPPROTO_IP, IP_MULTICAST_IF, &addr, sizeof(addr)) == -1)
115  {
116  TLOG(TLVL_ERROR) << "Cannot set outgoing interface, err=" << strerror(errno);
117  exit(1);
118  }
119  }
120  int yes = 1;
121  if (setsockopt(request_socket_, SOL_SOCKET, SO_REUSEADDR, &yes, sizeof(yes)) < 0)
122  {
123  TLOG(TLVL_ERROR) << "Unable to enable port reuse on request socket, err=" << strerror(errno);
124  exit(1);
125  }
126  if (setsockopt(request_socket_, IPPROTO_IP, IP_MULTICAST_LOOP, &yes, sizeof(yes)) < 0)
127  {
128  TLOG(TLVL_ERROR) << "Unable to enable multicast loopback on request socket, err=" << strerror(errno);
129  exit(1);
130  }
131  if (setsockopt(request_socket_, SOL_SOCKET, SO_BROADCAST, (void*)&yes, sizeof(int)) == -1)
132  {
133  TLOG(TLVL_ERROR) << "Cannot set request socket to broadcast, err=" << strerror(errno);
134  exit(1);
135  }
136  }
137  }
138 
139  void
140  RequestSender::setup_tokens_()
141  {
142  if (send_routing_tokens_)
143  {
144  TLOG(TLVL_DEBUG) << "Creating Routing Token sending socket";
145  int retry = 5;
146  while (retry > 0 && token_socket_ < 0)
147  {
148  token_socket_ = TCPConnect(token_address_.c_str(), token_port_);
149  if (token_socket_ < 0) usleep(100000);
150  retry--;
151  }
152  if (token_socket_ < 0)
153  {
154  TLOG(TLVL_ERROR) << "I failed to create the socket for sending Routing Tokens! err=" << strerror(errno);
155  exit(1);
156  }
157  TLOG(TLVL_DEBUG) << "Routing Token sending socket created successfully";
158  }
159  }
160 
161  void RequestSender::do_send_request_()
162  {
163  if (!send_requests_) return;
164  if (request_socket_ == -1) setup_requests_();
165 
166  TLOG(TLVL_TRACE) << "Waiting for " << request_delay_ << " microseconds.";
167  std::this_thread::sleep_for(std::chrono::microseconds(request_delay_));
168 
169  TLOG(TLVL_TRACE) << "Creating RequestMessage";
170  detail::RequestMessage message;
171  {
172  std::lock_guard<std::mutex> lk(request_mutex_);
173  for (auto& req : active_requests_)
174  {
175  TLOG(12, "RequestSender") << "Adding a request with sequence ID " << req.first << ", timestamp " << req.second;
176  message.addRequest(req.first, req.second);
177  }
178  }
179  TLOG(TLVL_TRACE) << "Setting mode flag in Message Header";
180  message.header()->mode = request_mode_;
181  char str[INET_ADDRSTRLEN];
182  inet_ntop(AF_INET, &(request_addr_.sin_addr), str, INET_ADDRSTRLEN);
183  std::lock_guard<std::mutex> lk2(request_send_mutex_);
184  TLOG(TLVL_TRACE) << "Sending request for " << message.size() << " events to multicast group " << str;
185  if (sendto(request_socket_, message.header(), sizeof(detail::RequestHeader), 0, (struct sockaddr *)&request_addr_, sizeof(request_addr_)) < 0)
186  {
187  TLOG(TLVL_ERROR) << "Error sending request message header err=" << strerror(errno);
188  request_socket_ = 1;
189  request_sending_--;
190  return;
191  }
192  size_t sent = 0;
193  while (sent < sizeof(detail::RequestPacket) * message.size())
194  {
195  ssize_t thisSent = sendto(request_socket_, reinterpret_cast<uint8_t*>(message.buffer()) + sent, sizeof(detail::RequestPacket) * message.size() - sent, 0, (struct sockaddr *)&request_addr_, sizeof(request_addr_));
196  if (thisSent < 0)
197  {
198  TLOG(TLVL_ERROR) << "Error sending request message data err=" << strerror(errno);
199  request_socket_ = -1;
200  request_sending_--;
201  return;
202  }
203  sent += thisSent;
204  }
205  TLOG(TLVL_TRACE) << "Done sending request";
206  request_sending_--;
207  }
208 
209  void RequestSender::send_routing_token_(int nSlots)
210  {
211  TLOG(TLVL_TRACE) << "send_routing_token_ called, send_routing_tokens_=" << std::boolalpha << send_routing_tokens_;
212  if (!send_routing_tokens_) return;
213  if (token_socket_ == -1) setup_tokens_();
214  detail::RoutingToken token;
215  token.header = TOKEN_MAGIC;
216  token.rank = my_rank;
217  token.new_slots_free = nSlots;
218 
219  TLOG(TLVL_TRACE) << "Sending RoutingToken to " << token_address_ << ":" << token_port_;
220  size_t sts = 0;
221  while (sts < sizeof(detail::RoutingToken))
222  {
223  auto res = send(token_socket_, reinterpret_cast<uint8_t*>(&token) + sts, sizeof(detail::RoutingToken) - sts, 0);
224  if (res == -1)
225  {
226  usleep(1000);
227  continue;
228  }
229  sts += res;
230  }
231  TLOG(TLVL_TRACE) << "Done sending RoutingToken to " << token_address_ << ":" << token_port_;
232  }
233 
235  {
236  if (!send_routing_tokens_) return;
237  boost::thread token([=] { send_routing_token_(nSlots); });
238  token.detach();
239  usleep(0); // Give up time slice
240  }
241 
242  void RequestSender::SendRequest(bool endOfRunOnly)
243  {
244  if (!send_requests_) return;
245  if (endOfRunOnly && request_mode_ != detail::RequestMessageMode::EndOfRun) return;
246  request_sending_++;
247  boost::thread request([=] { do_send_request_(); });
248  request.detach();
249  }
250 
251  void RequestSender::AddRequest(Fragment::sequence_id_t seqID, Fragment::timestamp_t timestamp)
252  {
253  {
254  std::lock_guard<std::mutex> lk(request_mutex_);
255  if (!active_requests_.count(seqID)) active_requests_[seqID] = timestamp;
256  }
257  SendRequest();
258  }
259 
260  void RequestSender::RemoveRequest(Fragment::sequence_id_t seqID)
261  {
262  std::lock_guard<std::mutex> lk(request_mutex_);
263  active_requests_.erase(seqID);
264  }
265 }
void RemoveRequest(Fragment::sequence_id_t seqID)
Remove a request from the request list.
int ResolveHost(char const *host_in, in_addr &addr)
Convert a string hostname to a in_addr suitable for socket communication.
Definition: TCPConnect.cc:33
int TCPConnect(char const *host_in, int dflt_port, long flags=0, int sndbufsiz=0)
Connect to a host on a given port.
Definition: TCPConnect.cc:356
End of Run mode (Used to end request processing on receiver)
virtual ~RequestSender()
RequestSender Destructor.
void SendRequest(bool endOfRunOnly=false)
Send a request message containing all current requests.
int GetInterfaceForNetwork(char const *host_in, in_addr &addr)
Convert an IP address to the network address of the interface sharing the subnet mask.
Definition: TCPConnect.cc:216
void SendRoutingToken(int nSlots)
Send a RoutingToken message indicating that slots are available.
RequestSender()=delete
Default Constructor is deleted.
void SetRequestMode(detail::RequestMessageMode mode)
Set the mode for RequestMessages. Used to indicate when RequestSender should enter &quot;EndOfRun&quot; mode...
RequestMessageMode
Mode used to indicate current run conditions to the request receiver.
void AddRequest(Fragment::sequence_id_t seqID, Fragment::timestamp_t timestamp)
Add a request to the request list.