1 #define TRACE_NAME "RequestSender"
2 #include "artdaq/DAQdata/Globals.hh"
3 #include "artdaq/DAQrate/RequestSender.hh"
12 #include "cetlib_except/exception.h"
13 #include "artdaq-core/Core/StatisticsCollection.hh"
14 #include "artdaq-core/Core/SimpleMemoryReader.hh"
15 #include "artdaq/DAQrate/detail/RoutingPacket.hh"
22 : send_requests_(pset.get<bool>(
"send_requests", false))
24 , request_port_(pset.get<int>(
"request_port", 3001))
25 , request_delay_(pset.get<size_t>(
"request_delay_ms", 10) * 1000)
26 , request_shutdown_timeout_us_(pset.get<size_t>(
"request_shutdown_timeout_us", 100000))
27 , multicast_out_addr_(pset.get<std::string>(
"output_address",
"0.0.0.0"))
28 , request_mode_(detail::RequestMessageMode::Normal)
31 TLOG_DEBUG(
"RequestSender") <<
"RequestSender CONSTRUCTOR" << TLOG_ENDL;
32 setup_requests_(pset.get<std::string>(
"request_address",
"227.128.12.26"));
34 auto rmConfig = pset.get<fhicl::ParameterSet>(
"routing_token_config", fhicl::ParameterSet());
35 send_routing_tokens_ = rmConfig.get<
bool>(
"use_routing_master",
false);
36 token_port_ = rmConfig.get<
int>(
"routing_token_port", 35555);
37 token_address_ = rmConfig.get<std::string>(
"routing_master_hostname",
"localhost");
39 TRACE(12,
"artdaq::RequestSender::RequestSender ctor - reader_thread_ initialized");
45 TLOG_TRACE(
"RequestSender") <<
"Shutting down RequestSender: Waiting for requests to be sent" << TLOG_ENDL;
47 auto start_time = std::chrono::steady_clock::now();
49 while (request_sending_ && request_shutdown_timeout_us_ > TimeUtils::GetElapsedTimeMicroseconds(start_time))
53 TLOG_TRACE(
"RequestSender") <<
"Shutting down RequestSender" << TLOG_ENDL;
54 if (request_socket_ > 0)
56 shutdown(request_socket_, 2);
57 close(request_socket_);
59 if (token_socket_ > 0)
61 shutdown(token_socket_, 2);
74 RequestSender::setup_requests_(std::string request_address)
78 request_socket_ = socket(AF_INET, SOCK_DGRAM, IPPROTO_UDP);
79 if (request_socket_ < 0)
81 TLOG_ERROR(
"RequestSender") <<
"I failed to create the socket for sending Data Requests! err=" << strerror(errno) << TLOG_ENDL;
84 int sts =
ResolveHost(request_address.c_str(), request_port_, request_addr_);
87 TLOG_ERROR(
"RequestSender") <<
"Unable to resolve Data Request address, err=" << strerror(errno) << TLOG_ENDL;
91 if (multicast_out_addr_ ==
"0.0.0.0")
93 multicast_out_addr_.reserve(HOST_NAME_MAX);
94 sts = gethostname(&multicast_out_addr_[0], HOST_NAME_MAX);
97 TLOG_ERROR(
"RequestSender") <<
"Could not get current hostname, err=" << strerror(errno) << TLOG_ENDL;
102 if (multicast_out_addr_ !=
"localhost")
105 sts =
ResolveHost(multicast_out_addr_.c_str(), addr);
108 TLOG_ERROR(
"RequestSender") <<
"Unable to resolve multicast interface address, err=" << strerror(errno) << TLOG_ENDL;
113 if (setsockopt(request_socket_, SOL_SOCKET, SO_REUSEADDR, &yes,
sizeof(yes)) < 0)
115 TLOG_ERROR(
"RequestSender") <<
"Unable to enable port reuse on request socket, err=" << strerror(errno) << TLOG_ENDL;
118 if (setsockopt(request_socket_, IPPROTO_IP, IP_MULTICAST_LOOP, &yes,
sizeof(yes)) < 0)
120 TLOG_ERROR(
"RequestSender") <<
"Unable to enable multicast loopback on request socket, err=" << strerror(errno) << TLOG_ENDL;
123 if (setsockopt(request_socket_, IPPROTO_IP, IP_MULTICAST_IF, &addr,
sizeof(addr)) == -1)
125 TLOG_ERROR(
"RequestSender") <<
"Cannot set outgoing interface, err=" << strerror(errno) << TLOG_ENDL;
130 if (setsockopt(request_socket_, SOL_SOCKET, SO_BROADCAST, (
void*)&yes,
sizeof(
int)) == -1)
132 TLOG_ERROR(
"RequestSender") <<
"Cannot set request socket to broadcast, err=" << strerror(errno) << TLOG_ENDL;
139 RequestSender::setup_tokens_()
141 if (send_routing_tokens_)
143 TLOG_DEBUG(
"RequestSender") <<
"Creating Routing Token sending socket" << TLOG_ENDL;
145 while (retry > 0 && token_socket_ < 0) {
146 token_socket_ =
TCPConnect(token_address_.c_str(), token_port_);
147 if (token_socket_ < 0) usleep(100000);
150 if (token_socket_ < 0)
152 TLOG_ERROR(
"RequestSender") <<
"I failed to create the socket for sending Routing Tokens! err=" << strerror(errno) << TLOG_ENDL;
155 TLOG_DEBUG(
"RequestSender") <<
"Routing Token sending socket created successfully" << TLOG_ENDL;
159 void RequestSender::do_send_request_()
161 if (!send_requests_)
return;
162 TLOG_TRACE(
"RequestSender") <<
"Waiting for " << request_delay_ <<
" microseconds." << TLOG_ENDL;
163 std::this_thread::sleep_for(std::chrono::microseconds(request_delay_));
165 TLOG_TRACE(
"RequestSender") <<
"Creating RequestMessage" << TLOG_ENDL;
166 detail::RequestMessage message;
168 std::lock_guard<std::mutex> lk(request_mutex_);
169 for (
auto& req : active_requests_)
171 message.addRequest(req.first, req.second);
174 TLOG_TRACE(
"RequestSender") <<
"Setting mode flag in Message Header" << TLOG_ENDL;
175 message.header()->mode = request_mode_;
176 char str[INET_ADDRSTRLEN];
177 inet_ntop(AF_INET, &(request_addr_.sin_addr), str, INET_ADDRSTRLEN);
178 std::lock_guard<std::mutex> lk2(request_send_mutex_);
179 TLOG_TRACE(
"RequestSender") <<
"Sending request for " << std::to_string(message.size()) <<
" events to multicast group " << str << TLOG_ENDL;
180 if (sendto(request_socket_, message.header(),
sizeof(detail::RequestHeader), 0, (
struct sockaddr *)&request_addr_,
sizeof(request_addr_)) < 0)
182 TLOG_ERROR(
"RequestSender") <<
"Error sending request message header err=" << strerror(errno) << TLOG_ENDL;
184 if (sendto(request_socket_, message.buffer(),
sizeof(detail::RequestPacket) * message.size(), 0, (
struct sockaddr *)&request_addr_,
sizeof(request_addr_)) < 0)
186 TLOG_ERROR(
"RequestSender") <<
"Error sending request message data err=" << strerror(errno) << TLOG_ENDL;
188 request_sending_ =
false;
191 void RequestSender::send_routing_token_(
int nSlots)
193 TLOG_TRACE(
"RequestSender") <<
"send_routing_token_ called, send_routing_tokens_=" << std::boolalpha << send_routing_tokens_ << TLOG_ENDL;
194 if (!send_routing_tokens_)
return;
195 if (token_socket_ == -1) setup_tokens_();
196 detail::RoutingToken token;
197 token.header = TOKEN_MAGIC;
198 token.rank = my_rank;
199 token.new_slots_free = nSlots;
201 TLOG_TRACE(
"RequestSender") <<
"Sending RoutingToken to " << token_address_ <<
":" << token_port_ << TLOG_ENDL;
203 while (sts <
sizeof(detail::RoutingToken))
205 auto res = send(token_socket_, reinterpret_cast<uint8_t*>(&token) + sts,
sizeof(detail::RoutingToken) - sts, 0);
213 TLOG_TRACE(
"RequestSender") <<
"Done sending RoutingToken to " << token_address_ <<
":" << token_port_ << TLOG_ENDL;
218 if (!send_routing_tokens_)
return;
219 boost::thread token([=] {send_routing_token_(nSlots); });
226 if (!send_requests_)
return;
228 request_sending_ =
true;
229 boost::thread request([=] { do_send_request_(); });
236 std::lock_guard<std::mutex> lk(request_mutex_);
237 if (!active_requests_.count(seqID)) active_requests_[seqID] = timestamp;
244 std::lock_guard<std::mutex> lk(request_mutex_);
245 active_requests_.erase(seqID);
void RemoveRequest(Fragment::sequence_id_t seqID)
Remove a request from the request list.
int ResolveHost(char const *host_in, in_addr &addr)
Convert a string hostname to a in_addr suitable for socket communication.
int TCPConnect(char const *host_in, int dflt_port, long flags=0, int sndbufsiz=0)
Connect to a host on a given port.
End of Run mode (Used to end request processing on receiver)
virtual ~RequestSender()
RequestSender Destructor.
void SendRequest(bool endOfRunOnly=false)
Send a request message containing all current requests.
void SendRoutingToken(int nSlots)
Send a RoutingToken message indicating that slots are available.
RequestSender()=delete
Default Constructor is deleted.
void SetRequestMode(detail::RequestMessageMode mode)
Set the mode for RequestMessages. Used to indicate when RequestSender should enter "EndOfRun" mode...
RequestMessageMode
Mode used to indicate current run conditions to the request receiver.
void AddRequest(Fragment::sequence_id_t seqID, Fragment::timestamp_t timestamp)
Add a request to the request list.