1 #include "artdaq/DAQdata/Globals.hh"
2 #define TRACE_NAME (app_name + "_RequestSender").c_str()
10 #include "artdaq/DAQrate/RequestSender.hh"
12 #include "artdaq-core/Core/SimpleMemoryReader.hh"
13 #include "artdaq-core/Core/StatisticsCollection.hh"
15 #include "artdaq/DAQrate/detail/RoutingPacket.hh"
16 #include "cetlib_except/exception.h"
20 : send_requests_(pset.get<bool>(
"send_requests", false))
23 , request_address_(pset.get<std::string>(
"request_address",
"227.128.12.26"))
24 , request_port_(pset.get<int>(
"request_port", 3001))
25 , request_delay_(pset.get<size_t>(
"request_delay_ms", 0) * 1000)
26 , request_shutdown_timeout_us_(pset.get<size_t>(
"request_shutdown_timeout_us", 100000))
28 , multicast_out_addr_(pset.get<std::string>(
"multicast_interface_ip", pset.get<std::string>(
"output_address",
"0.0.0.0")))
29 , request_mode_(detail::RequestMessageMode::Normal)
35 TLOG(TLVL_DEBUG) <<
"RequestSender CONSTRUCTOR";
38 auto rmConfig = pset.get<fhicl::ParameterSet>(
"routing_token_config", fhicl::ParameterSet());
39 send_routing_tokens_ = rmConfig.get<
bool>(
"use_routing_master",
false);
40 token_port_ = rmConfig.get<
int>(
"routing_token_port", 35555);
41 token_address_ = rmConfig.get<std::string>(
"routing_master_hostname",
"localhost");
43 TLOG(12) <<
"artdaq::RequestSender::RequestSender ctor - reader_thread_ initialized";
49 TLOG(TLVL_INFO) <<
"Shutting down RequestSender: Waiting for " << request_sending_.load() <<
" requests to be sent";
51 auto start_time = std::chrono::steady_clock::now();
53 while (request_sending_.load() > 0 && request_shutdown_timeout_us_ + request_delay_ > TimeUtils::GetElapsedTimeMicroseconds(start_time))
58 std::unique_lock<std::mutex> lk(request_mutex_);
59 std::unique_lock<std::mutex> lk2(request_send_mutex_);
61 TLOG(TLVL_INFO) <<
"Shutting down RequestSender: request_socket_: " << request_socket_ <<
", token_socket_: " << token_socket_;
62 if (request_socket_ != -1)
64 if (shutdown(request_socket_, 2) != 0 && errno == ENOTSOCK)
66 TLOG(TLVL_ERROR) <<
"Shutdown of request_socket_ resulted in ENOTSOCK. NOT Closing file descriptor!";
70 close(request_socket_);
74 if (token_socket_ != -1)
76 if (shutdown(token_socket_, 2) != 0 && errno == ENOTSOCK)
78 TLOG(TLVL_ERROR) <<
"Shutdown of token_socket_ resulted in ENOTSOCK. NOT Closing file descriptor!";
94 void RequestSender::setup_requests_()
98 request_socket_ = socket(AF_INET, SOCK_DGRAM, IPPROTO_UDP);
99 if (request_socket_ < 0)
101 TLOG(TLVL_ERROR) <<
"I failed to create the socket for sending Data Requests! err=" << strerror(errno);
104 int sts =
ResolveHost(request_address_.c_str(), request_port_, request_addr_);
107 TLOG(TLVL_ERROR) <<
"Unable to resolve Data Request address, err=" << strerror(errno);
111 if (multicast_out_addr_ ==
"0.0.0.0")
113 char hostname[HOST_NAME_MAX];
114 sts = gethostname(hostname, HOST_NAME_MAX);
115 multicast_out_addr_ = std::string(hostname);
118 TLOG(TLVL_ERROR) <<
"Could not get current hostname, err=" << strerror(errno);
123 if (multicast_out_addr_ !=
"localhost")
130 TLOG(TLVL_ERROR) <<
"Unable to determine the multicast interface address for " << multicast_out_addr_ <<
", err=" << strerror(errno);
133 char addr_str[INET_ADDRSTRLEN];
134 inet_ntop(AF_INET, &(addr), addr_str, INET_ADDRSTRLEN);
135 TLOG(TLVL_INFO) <<
"Successfully determined the multicast network interface for " << multicast_out_addr_ <<
": " << addr_str;
137 if (setsockopt(request_socket_, IPPROTO_IP, IP_MULTICAST_IF, &addr,
sizeof(addr)) == -1)
139 TLOG(TLVL_ERROR) <<
"Cannot set outgoing interface, err=" << strerror(errno);
144 if (setsockopt(request_socket_, SOL_SOCKET, SO_REUSEADDR, &yes,
sizeof(yes)) < 0)
146 TLOG(TLVL_ERROR) <<
"Unable to enable port reuse on request socket, err=" << strerror(errno);
149 if (setsockopt(request_socket_, IPPROTO_IP, IP_MULTICAST_LOOP, &yes,
sizeof(yes)) < 0)
151 TLOG(TLVL_ERROR) <<
"Unable to enable multicast loopback on request socket, err=" << strerror(errno);
154 if (setsockopt(request_socket_, SOL_SOCKET, SO_BROADCAST, (
void*)&yes,
sizeof(
int)) == -1)
156 TLOG(TLVL_ERROR) <<
"Cannot set request socket to broadcast, err=" << strerror(errno);
162 void RequestSender::setup_tokens_()
164 if (send_routing_tokens_)
166 TLOG(TLVL_DEBUG) <<
"Creating Routing Token sending socket";
167 auto start_time = std::chrono::steady_clock::now();
168 while (token_socket_ < 0 && TimeUtils::GetElapsedTime(start_time) < 30)
170 token_socket_ =
TCPConnect(token_address_.c_str(), token_port_, 0,
sizeof(detail::RoutingToken));
171 if (token_socket_ < 0)
173 TLOG(TLVL_TRACE) <<
"Waited " << TimeUtils::GetElapsedTime(start_time) <<
" s for Routing Master to open token socket";
177 if (token_socket_ < 0)
179 TLOG(TLVL_ERROR) <<
"I failed to create the socket for sending Routing Tokens! err=" << strerror(errno);
182 TLOG(TLVL_INFO) <<
"Routing Token sending socket created successfully for address " << token_address_;
186 void RequestSender::do_send_request_()
193 if (request_socket_ == -1) setup_requests_();
195 TLOG(TLVL_TRACE) <<
"Waiting for " << request_delay_ <<
" microseconds.";
196 std::this_thread::sleep_for(std::chrono::microseconds(request_delay_));
198 TLOG(TLVL_TRACE) <<
"Creating RequestMessage";
199 detail::RequestMessage message;
200 message.setRank(my_rank);
201 message.setRunNumber(run_number_);
203 std::unique_lock<std::mutex> lk(request_mutex_);
204 for (
auto& req : active_requests_)
206 TLOG(12) <<
"Adding a request with sequence ID " << req.first <<
", timestamp " << req.second <<
" to request message";
207 message.addRequest(req.first, req.second);
210 TLOG(TLVL_TRACE) <<
"Setting mode flag in Message Header to " << request_mode_;
211 message.setMode(request_mode_);
212 char str[INET_ADDRSTRLEN];
213 inet_ntop(AF_INET, &(request_addr_.sin_addr), str, INET_ADDRSTRLEN);
214 std::unique_lock<std::mutex> lk2(request_send_mutex_);
215 TLOG(TLVL_TRACE) <<
"Sending request for " << message.size() <<
" events to multicast group " << str
216 <<
", port " << request_port_ <<
", interface " << multicast_out_addr_;
217 auto buf = message.GetMessage();
218 auto sts = sendto(request_socket_, &buf[0], buf.size(), 0, (
struct sockaddr*)&request_addr_,
sizeof(request_addr_));
219 if (sts < 0 || static_cast<size_t>(sts) != buf.size())
221 TLOG(TLVL_ERROR) <<
"Error sending request message err=" << strerror(errno) <<
"sts=" << sts;
222 request_socket_ = -1;
226 TLOG(TLVL_TRACE) <<
"Done sending request sts=" << sts;
230 void RequestSender::send_routing_token_(
int nSlots,
int run_number)
232 TLOG(TLVL_TRACE) <<
"send_routing_token_ called, send_routing_tokens_=" << std::boolalpha << send_routing_tokens_;
233 if (!send_routing_tokens_)
return;
234 if (token_socket_ == -1) setup_tokens_();
235 detail::RoutingToken token;
236 token.header = TOKEN_MAGIC;
237 token.rank = my_rank;
238 token.new_slots_free = nSlots;
239 token.run_number = run_number;
241 TLOG(TLVL_TRACE) <<
"Sending RoutingToken to " << token_address_ <<
":" << token_port_;
243 while (sts <
sizeof(detail::RoutingToken))
245 auto res = send(token_socket_, reinterpret_cast<uint8_t*>(&token) + sts,
sizeof(detail::RoutingToken) - sts, 0);
248 TLOG(TLVL_WARNING) <<
"Error on token_socket, reconnecting";
249 close(token_socket_);
257 tokens_sent_ += nSlots;
258 TLOG(TLVL_TRACE) <<
"Done sending RoutingToken to " << token_address_ <<
":" << token_port_;
263 while (!initialized_) usleep(1000);
264 if (!send_routing_tokens_)
return;
265 boost::thread token([=] { send_routing_token_(nSlots, run_number); });
272 while (!initialized_) usleep(1000);
274 if (!send_requests_)
return;
277 boost::thread request([=] { do_send_request_(); });
283 while (!initialized_) usleep(1000);
286 std::lock_guard<std::mutex> lk(request_mutex_);
287 if (!active_requests_.count(seqID))
289 TLOG(12) <<
"Adding request for sequence ID " << seqID <<
" and timestamp " << timestamp <<
" to request list.";
290 active_requests_[seqID] = timestamp;
298 while (!initialized_) usleep(1000);
299 std::lock_guard<std::mutex> lk(request_mutex_);
300 TLOG(12) <<
"Removing request for sequence ID " << seqID <<
" from request list.";
301 active_requests_.erase(seqID);
void RemoveRequest(Fragment::sequence_id_t seqID)
Remove a request from the request list.
int ResolveHost(char const *host_in, in_addr &addr)
Convert a string hostname to a in_addr suitable for socket communication.
int TCPConnect(char const *host_in, int dflt_port, long flags=0, int sndbufsiz=0)
Connect to a host on a given port.
End of Run mode (Used to end request processing on receiver)
void SendRoutingToken(int nSlots, int run_number)
Send a RoutingToken message indicating that slots are available.
virtual ~RequestSender()
RequestSender Destructor.
void SendRequest(bool endOfRunOnly=false)
Send a request message containing all current requests.
int GetInterfaceForNetwork(char const *host_in, in_addr &addr)
Convert an IP address to the network address of the interface sharing the subnet mask.
RequestSender()=delete
Default Constructor is deleted.
void SetRequestMode(detail::RequestMessageMode mode)
Set the mode for RequestMessages. Used to indicate when RequestSender should enter "EndOfRun" mode...
RequestMessageMode
Mode used to indicate current run conditions to the request receiver.
void AddRequest(Fragment::sequence_id_t seqID, Fragment::timestamp_t timestamp)
Add a request to the request list.