artdaq  v3_12_02
RequestSender.cc
1 #include "TRACE/tracemf.h"
2 #include "artdaq/DAQdata/Globals.hh" // Before trace.h gets included in ConcurrentQueue (from GlobalQueue)
3 #define TRACE_NAME (app_name + "_RequestSender").c_str()
4 #include "artdaq/DAQrate/detail/RequestSender.hh"
5 
7 
8 #include "fhiclcpp/ParameterSet.h"
9 
10 #include <boost/thread.hpp>
11 
12 #include <dlfcn.h>
13 #include <chrono>
14 #include <cstring>
15 #include <fstream>
16 #include <iomanip>
17 #include <mutex>
18 #include <sstream>
19 #include <thread>
20 #include <utility>
21 
22 namespace artdaq {
23 RequestSender::RequestSender(const fhicl::ParameterSet& pset)
24  : send_requests_(pset.get<bool>("send_requests", false))
25  , initialized_(false)
26  , request_address_(pset.get<std::string>("request_address", "227.128.12.26"))
27  , request_port_(pset.get<int>("request_port", 3001))
28  , request_delay_(pset.get<size_t>("request_delay_ms", 0) * 1000)
29  , request_shutdown_timeout_us_(pset.get<size_t>("request_shutdown_timeout_us", 100000))
30  , request_socket_(-1)
31  , multicast_out_addr_(pset.get<std::string>("multicast_interface_ip", pset.get<std::string>("output_address", "0.0.0.0")))
32  , request_mode_(detail::RequestMessageMode::Normal)
33  , min_request_interval_ms_(pset.get<size_t>("min_request_interval_ms", 100))
34  , request_sending_(0)
35  , run_number_(0)
36 {
37  TLOG(TLVL_DEBUG) << "RequestSender CONSTRUCTOR pset=" << pset.to_string();
38  setup_requests_();
39 
40  TLOG(TLVL_DEBUG + 35) << "artdaq::RequestSender::RequestSender ctor - reader_thread_ initialized";
41  initialized_ = true;
42 }
43 
45 {
46  TLOG(TLVL_INFO) << "Shutting down RequestSender: Waiting for " << request_sending_.load() << " requests to be sent";
47 
48  auto start_time = std::chrono::steady_clock::now();
49 
50  while (request_sending_.load() > 0 && request_shutdown_timeout_us_ + request_delay_ > TimeUtils::GetElapsedTimeMicroseconds(start_time))
51  {
52  usleep(1000);
53  }
54  {
55  std::lock_guard<std::mutex> lk(request_mutex_);
56  std::lock_guard<std::mutex> lk2(request_send_mutex_);
57  }
58  TLOG(TLVL_INFO) << "Shutting down RequestSender: request_socket_: " << request_socket_;
59  if (request_socket_ != -1)
60  {
61  if (shutdown(request_socket_, 2) != 0 && errno == ENOTSOCK)
62  {
63  TLOG(TLVL_ERROR) << "Shutdown of request_socket_ resulted in ENOTSOCK. NOT Closing file descriptor!";
64  }
65  else
66  {
67  close(request_socket_);
68  }
69  request_socket_ = -1;
70  }
71 }
72 
74 {
75  {
76  std::lock_guard<std::mutex> lk(request_mutex_);
77  request_mode_ = mode;
78  }
79  SendRequest(true);
80 }
81 
82 void RequestSender::setup_requests_()
83 {
84  if (send_requests_)
85  {
86  request_socket_ = socket(AF_INET, SOCK_DGRAM, IPPROTO_UDP);
87  if (request_socket_ < 0)
88  {
89  TLOG(TLVL_ERROR) << "I failed to create the socket for sending Data Requests! err=" << strerror(errno);
90  exit(1);
91  }
92  int sts = ResolveHost(request_address_.c_str(), request_port_, request_addr_);
93  if (sts == -1)
94  {
95  TLOG(TLVL_ERROR) << "Unable to resolve Data Request address, err=" << strerror(errno);
96  exit(1);
97  }
98 
99  /* if (multicast_out_addr_ == "0.0.0.0")
100  {
101  char hostname[HOST_NAME_MAX];
102  sts = gethostname(hostname, HOST_NAME_MAX);
103  multicast_out_addr_ = std::string(hostname);
104  if (sts < 0)
105  {
106  TLOG(TLVL_ERROR) << "Could not get current hostname, err=" << strerror(errno);
107  exit(1);
108  }
109  }*/
110 
111  // For 0.0.0.0, use system-specified IP_MULTICAST_IF
112  if (multicast_out_addr_ != "localhost" && multicast_out_addr_ != "0.0.0.0")
113  {
114  struct in_addr addr;
115  sts = GetInterfaceForNetwork(multicast_out_addr_.c_str(), addr);
116  // sts = ResolveHost(multicast_out_addr_.c_str(), addr);
117  if (sts == -1)
118  {
119  TLOG(TLVL_ERROR) << "Unable to determine the multicast interface address for " << multicast_out_addr_ << ", err=" << strerror(errno);
120  exit(1);
121  }
122  char addr_str[INET_ADDRSTRLEN];
123  inet_ntop(AF_INET, &(addr), addr_str, INET_ADDRSTRLEN);
124  TLOG(TLVL_INFO) << "Successfully determined the multicast network interface for " << multicast_out_addr_ << ": " << addr_str;
125 
126  if (setsockopt(request_socket_, IPPROTO_IP, IP_MULTICAST_IF, &addr, sizeof(addr)) == -1)
127  {
128  TLOG(TLVL_ERROR) << "Cannot set outgoing interface, err=" << strerror(errno);
129  exit(1);
130  }
131  }
132  int yes = 1;
133  if (setsockopt(request_socket_, SOL_SOCKET, SO_REUSEADDR, &yes, sizeof(yes)) < 0)
134  {
135  TLOG(TLVL_ERROR) << "Unable to enable port reuse on request socket, err=" << strerror(errno);
136  exit(1);
137  }
138  if (setsockopt(request_socket_, IPPROTO_IP, IP_MULTICAST_LOOP, &yes, sizeof(yes)) < 0)
139  {
140  TLOG(TLVL_ERROR) << "Unable to enable multicast loopback on request socket, err=" << strerror(errno);
141  exit(1);
142  }
143  if (setsockopt(request_socket_, SOL_SOCKET, SO_BROADCAST, &yes, sizeof(yes)) == -1)
144  {
145  TLOG(TLVL_ERROR) << "Cannot set request socket to broadcast, err=" << strerror(errno);
146  exit(1);
147  }
148  }
149 }
150 
151 void RequestSender::do_send_request_()
152 {
153  if (!send_requests_)
154  {
155  request_sending_--;
156  return;
157  }
158  if (request_socket_ == -1)
159  {
160  setup_requests_();
161  }
162 
163  TLOG(TLVL_DEBUG + 33) << "Waiting for " << request_delay_ << " microseconds.";
164  std::this_thread::sleep_for(std::chrono::microseconds(request_delay_));
165 
166  TLOG(TLVL_DEBUG + 33) << "Creating RequestMessage";
167  detail::RequestMessage message;
168  message.setRank(my_rank);
169  message.setRunNumber(run_number_);
170  {
171  std::lock_guard<std::mutex> lk(request_mutex_);
172  for (auto& req : active_requests_)
173  {
174  TLOG(TLVL_DEBUG + 36) << "Adding a request with sequence ID " << req.first << ", timestamp " << req.second << " to request message";
175  message.addRequest(req.first, req.second);
176  }
177  TLOG(TLVL_DEBUG + 33) << "Setting mode flag in Message Header to " << static_cast<int>(request_mode_);
178  message.setMode(request_mode_);
179  }
180  char str[INET_ADDRSTRLEN];
181  inet_ntop(AF_INET, &(request_addr_.sin_addr), str, INET_ADDRSTRLEN);
182  std::lock_guard<std::mutex> lk2(request_send_mutex_);
183  TLOG(TLVL_DEBUG + 33) << "Sending request for " << message.size() << " events to multicast group " << str
184  << ", port " << request_port_ << ", interface " << multicast_out_addr_;
185  auto buf = message.GetMessage();
186  auto sts = sendto(request_socket_, &buf[0], buf.size(), 0, reinterpret_cast<struct sockaddr*>(&request_addr_), sizeof(request_addr_)); // NOLINT(cppcoreguidelines-pro-type-reinterpret-cast)
187  if (sts < 0 || static_cast<size_t>(sts) != buf.size())
188  {
189  TLOG(TLVL_ERROR) << "Error sending request message err=" << strerror(errno) << "sts=" << sts;
190  request_socket_ = -1;
191  request_sending_--;
192  return;
193  }
194  TLOG(TLVL_DEBUG + 33) << "Done sending request sts=" << sts;
195  request_sending_--;
196 }
197 
198 void RequestSender::SendRequest(bool endOfRunOnly)
199 {
200  while (!initialized_)
201  {
202  usleep(1000);
203  }
204 
205  if (!send_requests_)
206  {
207  return;
208  }
209  {
210  std::lock_guard<std::mutex> lk(request_mutex_);
211  if (endOfRunOnly && request_mode_ != detail::RequestMessageMode::EndOfRun)
212  {
213  return;
214  }
215  }
216  last_request_send_time_ = std::chrono::steady_clock::now();
217  request_sending_++;
218  boost::thread request([=] { do_send_request_(); });
219  request.detach();
220 }
221 
222 void RequestSender::AddRequest(Fragment::sequence_id_t seqID, Fragment::timestamp_t timestamp)
223 {
224  while (!initialized_)
225  {
226  usleep(1000);
227  }
228 
229  {
230  std::lock_guard<std::mutex> lk(request_mutex_);
231  if (active_requests_.count(seqID) == 0u)
232  {
233  TLOG(TLVL_DEBUG + 37) << "Adding request for sequence ID " << seqID << " and timestamp " << timestamp << " to request list.";
234  active_requests_[seqID] = timestamp;
235  }
236 
237  while (active_requests_.size() > detail::RequestMessage::max_request_count())
238  {
239  TLOG(TLVL_WARNING) << "Erasing request with seqID " << active_requests_.begin()->first << " due to over-large request list size! (" << active_requests_.size() << " / " << detail::RequestMessage::max_request_count() << ")";
240  active_requests_.erase(active_requests_.begin());
241  }
242  }
243  SendRequest(TimeUtils::GetElapsedTimeMilliseconds(last_request_send_time_) < min_request_interval_ms_);
244 }
245 
246 void RequestSender::RemoveRequest(Fragment::sequence_id_t seqID)
247 {
248  while (!initialized_)
249  {
250  usleep(1000);
251  }
252  std::lock_guard<std::mutex> lk(request_mutex_);
253  TLOG(TLVL_DEBUG + 38) << "Removing request for sequence ID " << seqID << " from request list.";
254  active_requests_.erase(seqID);
255 }
256 } // namespace artdaq
void RemoveRequest(Fragment::sequence_id_t seqID)
Remove a request from the request list.
int ResolveHost(char const *host_in, in_addr &addr)
Convert a string hostname to a in_addr suitable for socket communication.
Definition: TCPConnect.cc:34
End of Run mode (Used to end request processing on receiver)
virtual ~RequestSender()
RequestSender Destructor.
void SendRequest(bool endOfRunOnly=false)
Send a request message containing all current requests.
int GetInterfaceForNetwork(char const *host_in, in_addr &addr)
Convert an IP address to the network address of the interface sharing the subnet mask.
Definition: TCPConnect.cc:224
static size_t max_request_count()
Get the maximum number of requests that can be sent in a single RequestMessage.
RequestSender()=delete
Default Constructor is deleted.
void SetRequestMode(detail::RequestMessageMode mode)
Set the mode for RequestMessages. Used to indicate when RequestSender should enter &quot;EndOfRun&quot; mode...
RequestMessageMode
Mode used to indicate current run conditions to the request receiver.
void AddRequest(Fragment::sequence_id_t seqID, Fragment::timestamp_t timestamp)
Add a request to the request list.