1 #include "artdaq/DAQdata/Globals.hh"
2 #define TRACE_NAME (app_name + "_TokenReceiver").c_str()
10 #include "artdaq/DAQrate/TokenReceiver.hh"
14 : token_port_(ps.get<int>(
"routing_token_port", 35555))
15 , policy_(std::move(std::move(policy)))
16 , routing_mode_(routing_mode)
17 , number_of_senders_(number_of_senders)
18 , update_interval_msec_(update_interval_msec)
21 , thread_is_running_(false)
22 , reception_is_paused_(false)
23 , shutdown_requested_(false)
25 , statsHelperPtr_(nullptr)
27 receive_token_events_ = std::vector<epoll_event>(policy_->GetReceiverCount() + 1);
32 stopTokenReception(
true);
37 if (token_thread_.joinable())
41 boost::thread::attributes attrs;
42 attrs.set_stack_size(4096 * 2000);
44 reception_is_paused_ =
false;
45 shutdown_requested_ =
false;
47 TLOG(TLVL_INFO) <<
"Starting Token Reception Thread";
50 token_thread_ = boost::thread(attrs, boost::bind(&TokenReceiver::receiveTokensLoop_,
this));
52 snprintf(tname,
sizeof(tname)-1,
"%d-TokenRecv", my_rank);
53 tname[
sizeof(tname)-1] =
'\0';
54 auto handle = token_thread_.native_handle();
55 pthread_setname_np(handle, tname);
57 catch (boost::exception
const& e)
59 TLOG(TLVL_ERROR) <<
"Exception encountered starting Token Reception thread: " << boost::diagnostic_information(e) <<
", errno=" << errno;
60 std::cerr <<
"Exception encountered starting Token Reception thread: " << boost::diagnostic_information(e) <<
", errno=" << errno << std::endl;
63 received_token_count_ = 0;
64 thread_is_running_ =
true;
65 TLOG(TLVL_INFO) <<
"Started Token Reception Thread";
70 shutdown_requested_ =
true;
71 reception_is_paused_ =
false;
72 if (thread_is_running_)
74 if (received_token_count_ == 0 && !force)
76 TLOG(TLVL_DEBUG) <<
"Stop request received by TokenReceiver, but no tokens have ever been received.";
78 TLOG(TLVL_DEBUG) <<
"Joining tokenThread";
81 if (token_thread_.joinable())
90 thread_is_running_ =
false;
93 if (token_socket_ != -1)
101 void artdaq::TokenReceiver::receiveTokensLoop_()
103 while (!shutdown_requested_)
105 TLOG(TLVL_DEBUG) <<
"Receive Token loop start";
106 if (token_socket_ == -1)
108 TLOG(TLVL_DEBUG) <<
"Opening token listener socket";
110 fcntl(token_socket_, F_SETFL, O_NONBLOCK);
112 if (token_epoll_fd_ != -1)
114 close(token_epoll_fd_);
116 struct epoll_event ev;
117 token_epoll_fd_ = epoll_create1(0);
118 ev.events = EPOLLIN | EPOLLPRI;
119 ev.data.fd = token_socket_;
120 if (epoll_ctl(token_epoll_fd_, EPOLL_CTL_ADD, token_socket_, &ev) == -1)
122 TLOG(TLVL_ERROR) <<
"Could not register listen socket to epoll fd";
126 if (token_socket_ == -1 || token_epoll_fd_ == -1)
128 TLOG(TLVL_DEBUG) <<
"One of the listen sockets was not opened successfully.";
132 auto nfds = epoll_wait(token_epoll_fd_, &receive_token_events_[0], receive_token_events_.size(), update_interval_msec_);
135 TLOG(TLVL_ERROR) <<
"Error status received from epoll_wait, exiting with code " << EXIT_FAILURE <<
", errno=" << errno <<
" (" << strerror(errno) <<
")";
136 perror(
"epoll_wait");
140 while (reception_is_paused_ && !shutdown_requested_)
145 TLOG(TLVL_DEBUG) <<
"Received " << nfds <<
" events";
146 for (
auto n = 0; n < nfds; ++n)
148 if (receive_token_events_[n].data.fd == token_socket_)
150 TLOG(TLVL_DEBUG) <<
"Accepting new connection on token_socket";
152 socklen_t arglen =
sizeof(addr);
153 auto conn_sock = accept(token_socket_, reinterpret_cast<struct sockaddr*>(&addr), &arglen);
154 fcntl(conn_sock, F_SETFL, O_NONBLOCK);
158 TLOG(TLVL_ERROR) <<
"Error status received from accept, exiting with code " << EXIT_FAILURE <<
", errno=" << errno <<
" (" << strerror(errno) <<
")";
163 receive_token_addrs_[conn_sock] = std::string(inet_ntoa(addr.sin_addr));
164 TLOG(TLVL_DEBUG) <<
"New fd is " << conn_sock <<
" for data-receiver at " << receive_token_addrs_[conn_sock];
165 struct epoll_event ev;
166 ev.events = EPOLLIN | EPOLLET;
167 ev.data.fd = conn_sock;
168 if (epoll_ctl(token_epoll_fd_, EPOLL_CTL_ADD, conn_sock, &ev) == -1)
170 TLOG(TLVL_ERROR) <<
"Error status received from epoll_ctl, exiting with code " << EXIT_FAILURE <<
", errno=" << errno <<
" (" << strerror(errno) <<
")";
171 perror(
"epoll_ctl: conn_sock");
177 auto startTime = artdaq::MonitoredQuantity::getCurrentTime();
182 detail::RoutingToken buff;
183 sts += read(receive_token_events_[n].data.fd, &buff,
sizeof(detail::RoutingToken) - sts);
186 TLOG(TLVL_INFO) <<
"Received 0-size token from " << receive_token_addrs_[receive_token_events_[n].data.fd];
189 else if (sts < 0 && errno == EAGAIN)
191 TLOG(TLVL_DEBUG) <<
"No more tokens from this rank. Continuing poll loop.";
196 TLOG(TLVL_ERROR) <<
"Error reading from token socket: sts=" << sts <<
", errno=" << errno;
197 receive_token_addrs_.erase(receive_token_events_[n].data.fd);
198 close(receive_token_events_[n].data.fd);
199 epoll_ctl(token_epoll_fd_, EPOLL_CTL_DEL, receive_token_events_[n].data.fd,
nullptr);
202 else if (sts ==
sizeof(detail::RoutingToken) && buff.header != TOKEN_MAGIC)
204 TLOG(TLVL_ERROR) <<
"Received invalid token from " << receive_token_addrs_[receive_token_events_[n].data.fd] <<
" sts=" << sts;
207 else if (sts ==
sizeof(detail::RoutingToken))
210 TLOG(TLVL_DEBUG) <<
"Received token from " << buff.rank <<
" indicating " << buff.new_slots_free <<
" slots are free. (run=" << buff.run_number <<
")";
211 if (buff.run_number != run_number_)
213 TLOG(TLVL_DEBUG) <<
"Received token from a different run number! Current = " << run_number_ <<
", token = " << buff.run_number <<
", ignoring (n=" << buff.new_slots_free <<
")";
217 received_token_count_ += buff.new_slots_free;
220 policy_->AddReceiverToken(buff.rank, buff.new_slots_free);
224 if (received_token_counter_.count(buff.rank) == 0u)
226 received_token_counter_[buff.rank] = 0;
228 received_token_counter_[buff.rank] += buff.new_slots_free;
229 TLOG(TLVL_DEBUG) <<
"RoutingManagerMode is RouteBySendCount. I have " << received_token_counter_[buff.rank] <<
" tokens for rank " << buff.rank <<
" and I need " << number_of_senders_ <<
".";
230 while (received_token_counter_[buff.rank] >= number_of_senders_)
232 TLOG(TLVL_DEBUG) <<
"RoutingManagerMode is RouteBySendCount. I have " << received_token_counter_[buff.rank] <<
" tokens for rank " << buff.rank <<
" and I need " << number_of_senders_
233 <<
"... Sending token to policy";
234 policy_->AddReceiverToken(buff.rank, 1);
235 received_token_counter_[buff.rank] -= number_of_senders_;
241 auto delta_time = artdaq::MonitoredQuantity::getCurrentTime() - startTime;
242 if (statsHelperPtr_ !=
nullptr) { statsHelperPtr_->addSample(tokens_received_stat_key_, delta_time); }
Events should be routed by sequence ID (BR -> EB)
void stopTokenReception(bool force=false)
Stops the reception of event builder tokens.
RoutingManagerMode
Mode indicating whether the RoutingManager is routing events by Sequence ID or by Send Count...
The RoutingToken contains the magic bytes, the rank of the token sender, and the number of slots free...
int TCP_listen_fd(int port, int rcvbuf)
Create a TCP listening socket on the given port and INADDR_ANY, with the given receive buffer...
TokenReceiver(const fhicl::ParameterSet &ps, std::shared_ptr< RoutingManagerPolicy > policy, detail::RoutingManagerMode routing_mode, size_t number_of_senders, size_t update_interval_msec)
TokenReceiver Constructor.
void startTokenReception()
Starts the reception of event builder tokens.
Events should be routed by send count (EB -> Agg)
virtual ~TokenReceiver()
TokenReceiver Destructor.