1 #include "TRACE/tracemf.h"
2 #include "artdaq/DAQdata/Globals.hh"
3 #define TRACE_NAME (app_name + "_RequestSender").c_str()
4 #include "artdaq/DAQrate/detail/RequestSender.hh"
8 #include "fhiclcpp/ParameterSet.h"
10 #include <boost/thread.hpp>
24 : send_requests_(pset.get<bool>(
"send_requests", false))
26 , request_address_(pset.get<std::string>(
"request_address",
"227.128.12.26"))
27 , request_port_(pset.get<int>(
"request_port", 3001))
28 , request_delay_(pset.get<size_t>(
"request_delay_ms", 0) * 1000)
29 , request_shutdown_timeout_us_(pset.get<size_t>(
"request_shutdown_timeout_us", 100000))
31 , multicast_out_addr_(pset.get<std::string>(
"multicast_interface_ip", pset.get<std::string>(
"output_address",
"0.0.0.0")))
32 , request_mode_(detail::RequestMessageMode::Normal)
33 , min_request_interval_ms_(pset.get<size_t>(
"min_request_interval_ms", 100))
38 TLOG(TLVL_DEBUG) <<
"RequestSender CONSTRUCTOR pset=" << pset.to_string();
41 TLOG(TLVL_DEBUG + 35) <<
"artdaq::RequestSender::RequestSender ctor - reader_thread_ initialized";
47 TLOG(TLVL_INFO) <<
"Shutting down RequestSender: Waiting for " << request_sending_.load() <<
" requests to be sent (total sent: " << requests_sent_ <<
")";
49 auto start_time = std::chrono::steady_clock::now();
51 while (request_sending_.load() > 0 && request_shutdown_timeout_us_ + request_delay_ > TimeUtils::GetElapsedTimeMicroseconds(start_time))
56 std::lock_guard<std::mutex> lk(request_mutex_);
57 std::lock_guard<std::mutex> lk2(request_send_mutex_);
59 TLOG(TLVL_INFO) <<
"Shutting down RequestSender: request_socket_: " << request_socket_;
60 if (request_socket_ != -1)
62 if (shutdown(request_socket_, 2) != 0 && errno == ENOTSOCK)
64 TLOG(TLVL_ERROR) <<
"Shutdown of request_socket_ resulted in ENOTSOCK. NOT Closing file descriptor!";
68 close(request_socket_);
77 std::lock_guard<std::mutex> lk(request_mutex_);
83 void RequestSender::setup_requests_()
87 request_socket_ = socket(AF_INET, SOCK_DGRAM, IPPROTO_UDP);
88 if (request_socket_ < 0)
90 TLOG(TLVL_ERROR) <<
"I failed to create the socket for sending Data Requests! err=" << strerror(errno);
93 int sts =
ResolveHost(request_address_.c_str(), request_port_, request_addr_);
96 TLOG(TLVL_ERROR) <<
"Unable to resolve Data Request address, err=" << strerror(errno);
113 if (multicast_out_addr_ !=
"localhost" && multicast_out_addr_ !=
"0.0.0.0")
120 TLOG(TLVL_ERROR) <<
"Unable to determine the multicast interface address for " << multicast_out_addr_ <<
", err=" << strerror(errno);
123 char addr_str[INET_ADDRSTRLEN];
124 inet_ntop(AF_INET, &(addr), addr_str, INET_ADDRSTRLEN);
125 TLOG(TLVL_INFO) <<
"Successfully determined the multicast network interface for " << multicast_out_addr_ <<
": " << addr_str;
127 if (setsockopt(request_socket_, IPPROTO_IP, IP_MULTICAST_IF, &addr,
sizeof(addr)) == -1)
129 TLOG(TLVL_ERROR) <<
"Cannot set outgoing interface, err=" << strerror(errno);
134 if (setsockopt(request_socket_, SOL_SOCKET, SO_REUSEADDR, &yes,
sizeof(yes)) < 0)
136 TLOG(TLVL_ERROR) <<
"Unable to enable port reuse on request socket, err=" << strerror(errno);
139 if (setsockopt(request_socket_, IPPROTO_IP, IP_MULTICAST_LOOP, &yes,
sizeof(yes)) < 0)
141 TLOG(TLVL_ERROR) <<
"Unable to enable multicast loopback on request socket, err=" << strerror(errno);
144 if (setsockopt(request_socket_, SOL_SOCKET, SO_BROADCAST, &yes,
sizeof(yes)) == -1)
146 TLOG(TLVL_ERROR) <<
"Cannot set request socket to broadcast, err=" << strerror(errno);
152 void RequestSender::do_send_request_()
159 if (request_socket_ == -1)
164 TLOG(TLVL_DEBUG + 33) <<
"Waiting for " << request_delay_ <<
" microseconds.";
165 std::this_thread::sleep_for(std::chrono::microseconds(request_delay_));
167 TLOG(TLVL_DEBUG + 33) <<
"Creating RequestMessage";
168 detail::RequestMessage message;
169 message.setRank(my_rank);
170 message.setRunNumber(run_number_);
172 std::lock_guard<std::mutex> lk(request_mutex_);
173 for (
auto& req : active_requests_)
175 TLOG(TLVL_DEBUG + 36) <<
"Adding a request with sequence ID " << req.first <<
", timestamp " << req.second <<
" to request message";
176 message.addRequest(req.first, req.second);
178 TLOG(TLVL_DEBUG + 33) <<
"Setting mode flag in Message Header to " <<
static_cast<int>(request_mode_);
179 message.setMode(request_mode_);
181 char str[INET_ADDRSTRLEN];
182 inet_ntop(AF_INET, &(request_addr_.sin_addr), str, INET_ADDRSTRLEN);
183 std::lock_guard<std::mutex> lk2(request_send_mutex_);
184 TLOG(TLVL_DEBUG + 33) <<
"Sending request for " << message.size() <<
" events to multicast group " << str
185 <<
", port " << request_port_ <<
", interface " << multicast_out_addr_;
186 auto buf = message.GetMessage();
187 auto sts = sendto(request_socket_, &buf[0], buf.size(), 0,
reinterpret_cast<struct sockaddr*
>(&request_addr_),
sizeof(request_addr_));
188 if (sts < 0 || static_cast<size_t>(sts) != buf.size())
190 TLOG(TLVL_ERROR) <<
"Error sending request message err=" << strerror(errno) <<
"sts=" << sts;
191 request_socket_ = -1;
195 TLOG(TLVL_DEBUG + 33) <<
"Done sending request sts=" << sts;
202 while (!initialized_)
212 std::lock_guard<std::mutex> lk(request_mutex_);
218 last_request_send_time_ = std::chrono::steady_clock::now();
220 boost::thread request([
this] { do_send_request_(); });
226 while (!initialized_)
232 std::lock_guard<std::mutex> lk(request_mutex_);
233 if (active_requests_.count(seqID) == 0u)
235 TLOG(TLVL_DEBUG + 37) <<
"Adding request for sequence ID " << seqID <<
" and timestamp " << timestamp <<
" to request list.";
236 active_requests_[seqID] = timestamp;
241 TLOG(TLVL_WARNING) <<
"Erasing request with seqID " << active_requests_.begin()->first <<
" due to over-large request list size! (" << active_requests_.size() <<
" / " <<
detail::RequestMessage::max_request_count() <<
")";
242 active_requests_.erase(active_requests_.begin());
245 SendRequest(TimeUtils::GetElapsedTimeMilliseconds(last_request_send_time_) < min_request_interval_ms_);
250 while (!initialized_)
254 std::lock_guard<std::mutex> lk(request_mutex_);
255 TLOG(TLVL_DEBUG + 38) <<
"Removing request for sequence ID " << seqID <<
" from request list.";
256 active_requests_.erase(seqID);
void RemoveRequest(Fragment::sequence_id_t seqID)
Remove a request from the request list.
int ResolveHost(char const *host_in, in_addr &addr)
Convert a string hostname to a in_addr suitable for socket communication.
End of Run mode (Used to end request processing on receiver)
virtual ~RequestSender()
RequestSender Destructor.
void SendRequest(bool endOfRunOnly=false)
Send a request message containing all current requests.
int GetInterfaceForNetwork(char const *host_in, in_addr &addr)
Convert an IP address to the network address of the interface sharing the subnet mask.
static size_t max_request_count()
Get the maximum number of requests that can be sent in a single RequestMessage.
RequestSender()=delete
Default Constructor is deleted.
void SetRequestMode(detail::RequestMessageMode mode)
Set the mode for RequestMessages. Used to indicate when RequestSender should enter "EndOfRun" mode...
RequestMessageMode
Mode used to indicate current run conditions to the request receiver.
void AddRequest(Fragment::sequence_id_t seqID, Fragment::timestamp_t timestamp)
Add a request to the request list.