artdaq  v3_00_03
RequestSender.cc
1 #define TRACE_NAME "RequestSender"
2 #include "artdaq/DAQdata/Globals.hh" // Before trace.h gets included in ConcurrentQueue (from GlobalQueue)
3 #include "artdaq/DAQrate/RequestSender.hh"
4 #include <utility>
5 #include <cstring>
6 #include <dlfcn.h>
7 #include <iomanip>
8 #include <fstream>
9 #include <sstream>
10 #include <chrono>
11 
12 #include "cetlib_except/exception.h"
13 #include "artdaq-core/Core/StatisticsCollection.hh"
14 #include "artdaq-core/Core/SimpleMemoryReader.hh"
15 #include "artdaq/DAQrate/detail/RoutingPacket.hh"
17 
18 namespace artdaq
19 {
20 
21  RequestSender::RequestSender(const fhicl::ParameterSet& pset)
22  : send_requests_(pset.get<bool>("send_requests", false))
23  , active_requests_()
24  , request_port_(pset.get<int>("request_port", 3001))
25  , request_delay_(pset.get<size_t>("request_delay_ms", 10) * 1000)
26  , request_shutdown_timeout_us_(pset.get<size_t>("request_shutdown_timeout_us", 100000))
27  , multicast_out_addr_(pset.get<std::string>("output_address", "0.0.0.0"))
28  , request_mode_(detail::RequestMessageMode::Normal)
29  , token_socket_(-1)
30  {
31  TLOG_DEBUG("RequestSender") << "RequestSender CONSTRUCTOR" << TLOG_ENDL;
32  setup_requests_(pset.get<std::string>("request_address", "227.128.12.26"));
33 
34  auto rmConfig = pset.get<fhicl::ParameterSet>("routing_token_config", fhicl::ParameterSet());
35  send_routing_tokens_ = rmConfig.get<bool>("use_routing_master", false);
36  token_port_ = rmConfig.get<int>("routing_token_port", 35555);
37  token_address_ = rmConfig.get<std::string>("routing_master_hostname", "localhost");
38  setup_tokens_();
39  TRACE(12, "artdaq::RequestSender::RequestSender ctor - reader_thread_ initialized");
40  }
41 
42 
44  {
45  TLOG_TRACE("RequestSender") << "Shutting down RequestSender: Waiting for requests to be sent" << TLOG_ENDL;
46 
47  auto start_time = std::chrono::steady_clock::now();
48 
49  while (request_sending_ && request_shutdown_timeout_us_ > TimeUtils::GetElapsedTimeMicroseconds(start_time))
50  {
51  usleep(1000);
52  }
53  TLOG_TRACE("RequestSender") << "Shutting down RequestSender" << TLOG_ENDL;
54  if (request_socket_ > 0)
55  {
56  shutdown(request_socket_, 2);
57  close(request_socket_);
58  }
59  if (token_socket_ > 0)
60  {
61  shutdown(token_socket_, 2);
62  close(token_socket_);
63  }
64  }
65 
66 
68  {
69  request_mode_ = mode;
70  SendRequest(true);
71  }
72 
73  void
74  RequestSender::setup_requests_(std::string request_address)
75  {
76  if (send_requests_)
77  {
78  request_socket_ = socket(AF_INET, SOCK_DGRAM, IPPROTO_UDP);
79  if (request_socket_ < 0)
80  {
81  TLOG_ERROR("RequestSender") << "I failed to create the socket for sending Data Requests! err=" << strerror(errno) << TLOG_ENDL;
82  exit(1);
83  }
84  int sts = ResolveHost(request_address.c_str(), request_port_, request_addr_);
85  if (sts == -1)
86  {
87  TLOG_ERROR("RequestSender") << "Unable to resolve Data Request address, err=" << strerror(errno) << TLOG_ENDL;
88  exit(1);
89  }
90 
91  if (multicast_out_addr_ == "0.0.0.0")
92  {
93  multicast_out_addr_.reserve(HOST_NAME_MAX);
94  sts = gethostname(&multicast_out_addr_[0], HOST_NAME_MAX);
95  if (sts < 0)
96  {
97  TLOG_ERROR("RequestSender") << "Could not get current hostname, err=" << strerror(errno) << TLOG_ENDL;
98  exit(1);
99  }
100  }
101 
102  if (multicast_out_addr_ != "localhost")
103  {
104  struct in_addr addr;
105  sts = ResolveHost(multicast_out_addr_.c_str(), addr);
106  if (sts == -1)
107  {
108  TLOG_ERROR("RequestSender") << "Unable to resolve multicast interface address, err=" << strerror(errno) << TLOG_ENDL;
109  exit(1);
110  }
111 
112  int yes = 1;
113  if (setsockopt(request_socket_, SOL_SOCKET, SO_REUSEADDR, &yes, sizeof(yes)) < 0)
114  {
115  TLOG_ERROR("RequestSender") << "Unable to enable port reuse on request socket, err=" << strerror(errno) << TLOG_ENDL;
116  exit(1);
117  }
118  if (setsockopt(request_socket_, IPPROTO_IP, IP_MULTICAST_LOOP, &yes, sizeof(yes)) < 0)
119  {
120  TLOG_ERROR("RequestSender") << "Unable to enable multicast loopback on request socket, err=" << strerror(errno) << TLOG_ENDL;
121  exit(1);
122  }
123  if (setsockopt(request_socket_, IPPROTO_IP, IP_MULTICAST_IF, &addr, sizeof(addr)) == -1)
124  {
125  TLOG_ERROR("RequestSender") << "Cannot set outgoing interface, err=" << strerror(errno) << TLOG_ENDL;
126  exit(1);
127  }
128  }
129  int yes = 1;
130  if (setsockopt(request_socket_, SOL_SOCKET, SO_BROADCAST, (void*)&yes, sizeof(int)) == -1)
131  {
132  TLOG_ERROR("RequestSender") << "Cannot set request socket to broadcast, err=" << strerror(errno) << TLOG_ENDL;
133  exit(1);
134  }
135  }
136  }
137 
138  void
139  RequestSender::setup_tokens_()
140  {
141  if (send_routing_tokens_)
142  {
143  TLOG_DEBUG("RequestSender") << "Creating Routing Token sending socket" << TLOG_ENDL;
144  int retry = 5;
145  while (retry > 0 && token_socket_ < 0) {
146  token_socket_ = TCPConnect(token_address_.c_str(), token_port_);
147  if (token_socket_ < 0) usleep(100000);
148  retry--;
149  }
150  if (token_socket_ < 0)
151  {
152  TLOG_ERROR("RequestSender") << "I failed to create the socket for sending Routing Tokens! err=" << strerror(errno) << TLOG_ENDL;
153  exit(1);
154  }
155  TLOG_DEBUG("RequestSender") << "Routing Token sending socket created successfully" << TLOG_ENDL;
156  }
157  }
158 
159  void RequestSender::do_send_request_()
160  {
161  if (!send_requests_) return;
162  TLOG_TRACE("RequestSender") << "Waiting for " << request_delay_ << " microseconds." << TLOG_ENDL;
163  std::this_thread::sleep_for(std::chrono::microseconds(request_delay_));
164 
165  TLOG_TRACE("RequestSender") << "Creating RequestMessage" << TLOG_ENDL;
166  detail::RequestMessage message;
167  {
168  std::lock_guard<std::mutex> lk(request_mutex_);
169  for (auto& req : active_requests_)
170  {
171  message.addRequest(req.first, req.second);
172  }
173  }
174  TLOG_TRACE("RequestSender") << "Setting mode flag in Message Header" << TLOG_ENDL;
175  message.header()->mode = request_mode_;
176  char str[INET_ADDRSTRLEN];
177  inet_ntop(AF_INET, &(request_addr_.sin_addr), str, INET_ADDRSTRLEN);
178  std::lock_guard<std::mutex> lk2(request_send_mutex_);
179  TLOG_TRACE("RequestSender") << "Sending request for " << std::to_string(message.size()) << " events to multicast group " << str << TLOG_ENDL;
180  if (sendto(request_socket_, message.header(), sizeof(detail::RequestHeader), 0, (struct sockaddr *)&request_addr_, sizeof(request_addr_)) < 0)
181  {
182  TLOG_ERROR("RequestSender") << "Error sending request message header err=" << strerror(errno) << TLOG_ENDL;
183  }
184  if (sendto(request_socket_, message.buffer(), sizeof(detail::RequestPacket) * message.size(), 0, (struct sockaddr *)&request_addr_, sizeof(request_addr_)) < 0)
185  {
186  TLOG_ERROR("RequestSender") << "Error sending request message data err=" << strerror(errno) << TLOG_ENDL;
187  }
188  request_sending_ = false;
189  }
190 
191  void RequestSender::send_routing_token_(int nSlots)
192  {
193  TLOG_TRACE("RequestSender") << "send_routing_token_ called, send_routing_tokens_=" << std::boolalpha << send_routing_tokens_ << TLOG_ENDL;
194  if (!send_routing_tokens_) return;
195  if (token_socket_ == -1) setup_tokens_();
196  detail::RoutingToken token;
197  token.header = TOKEN_MAGIC;
198  token.rank = my_rank;
199  token.new_slots_free = nSlots;
200 
201  TLOG_TRACE("RequestSender") << "Sending RoutingToken to " << token_address_ << ":" << token_port_ << TLOG_ENDL;
202  size_t sts = 0;
203  while (sts < sizeof(detail::RoutingToken))
204  {
205  auto res = send(token_socket_, reinterpret_cast<uint8_t*>(&token) + sts, sizeof(detail::RoutingToken) - sts, 0);
206  if (res == -1)
207  {
208  usleep(1000);
209  continue;
210  }
211  sts += res;
212  }
213  TLOG_TRACE("RequestSender") << "Done sending RoutingToken to " << token_address_ << ":" << token_port_ << TLOG_ENDL;
214  }
215 
217  {
218  if (!send_routing_tokens_) return;
219  boost::thread token([=] {send_routing_token_(nSlots); });
220  token.detach();
221  usleep(0); // Give up time slice
222  }
223 
224  void RequestSender::SendRequest(bool endOfRunOnly)
225  {
226  if (!send_requests_) return;
227  if (endOfRunOnly && request_mode_ != detail::RequestMessageMode::EndOfRun) return;
228  request_sending_ = true;
229  boost::thread request([=] { do_send_request_(); });
230  request.detach();
231  }
232 
233  void RequestSender::AddRequest(Fragment::sequence_id_t seqID, Fragment::timestamp_t timestamp)
234  {
235  {
236  std::lock_guard<std::mutex> lk(request_mutex_);
237  if (!active_requests_.count(seqID)) active_requests_[seqID] = timestamp;
238  }
239  SendRequest();
240  }
241 
242  void RequestSender::RemoveRequest(Fragment::sequence_id_t seqID)
243  {
244  std::lock_guard<std::mutex> lk(request_mutex_);
245  active_requests_.erase(seqID);
246  }
247 }
void RemoveRequest(Fragment::sequence_id_t seqID)
Remove a request from the request list.
int ResolveHost(char const *host_in, in_addr &addr)
Convert a string hostname to a in_addr suitable for socket communication.
Definition: TCPConnect.cc:27
int TCPConnect(char const *host_in, int dflt_port, long flags=0, int sndbufsiz=0)
Connect to a host on a given port.
Definition: TCPConnect.cc:122
End of Run mode (Used to end request processing on receiver)
virtual ~RequestSender()
RequestSender Destructor.
void SendRequest(bool endOfRunOnly=false)
Send a request message containing all current requests.
void SendRoutingToken(int nSlots)
Send a RoutingToken message indicating that slots are available.
RequestSender()=delete
Default Constructor is deleted.
void SetRequestMode(detail::RequestMessageMode mode)
Set the mode for RequestMessages. Used to indicate when RequestSender should enter &quot;EndOfRun&quot; mode...
RequestMessageMode
Mode used to indicate current run conditions to the request receiver.
void AddRequest(Fragment::sequence_id_t seqID, Fragment::timestamp_t timestamp)
Add a request to the request list.