1 #include "artdaq/DAQdata/Globals.hh"
2 #define TRACE_NAME (app_name + "_TokenReceiver").c_str()
6 #include "artdaq/DAQrate/TokenReceiver.hh"
10 : token_port_(ps.get<int>(
"routing_token_port", 35555))
12 , routing_mode_(routing_mode)
13 , number_of_senders_(number_of_senders)
14 , update_interval_msec_(update_interval_msec)
17 , thread_is_running_(false)
18 , reception_is_paused_(false)
19 , shutdown_requested_(false)
21 , received_token_counter_()
22 , statsHelperPtr_(nullptr)
24 receive_token_events_ = std::vector<epoll_event>(policy_->GetReceiverCount() + 1);
29 stopTokenReception(
true);
34 if (token_thread_.joinable()) token_thread_.join();
35 boost::thread::attributes attrs;
36 attrs.set_stack_size(4096 * 2000);
38 reception_is_paused_ =
false;
39 shutdown_requested_ =
false;
41 TLOG(TLVL_INFO) <<
"Starting Token Reception Thread";
44 token_thread_ = boost::thread(attrs, boost::bind(&TokenReceiver::receiveTokensLoop_,
this));
46 catch (boost::exception
const& e)
48 std::cerr <<
"Exception encountered starting Token Reception thread: " << boost::diagnostic_information(e) <<
", errno=" << errno << std::endl;
51 received_token_count_ = 0;
52 thread_is_running_ =
true;
53 TLOG(TLVL_INFO) <<
"Started Token Reception Thread";
58 shutdown_requested_ =
true;
59 reception_is_paused_ =
false;
60 if (thread_is_running_)
62 if (received_token_count_ == 0 && !force)
64 TLOG(TLVL_DEBUG) <<
"Stop request received by TokenReceiver, but no tokens have ever been received.";
66 TLOG(TLVL_DEBUG) <<
"Joining tokenThread";
67 if (token_thread_.joinable()) token_thread_.join();
68 thread_is_running_ =
false;
71 if (token_socket_ != -1)
79 void artdaq::TokenReceiver::receiveTokensLoop_()
81 while (!shutdown_requested_)
83 TLOG(TLVL_DEBUG) <<
"Receive Token loop start";
84 if (token_socket_ == -1)
86 TLOG(TLVL_DEBUG) <<
"Opening token listener socket";
88 fcntl(token_socket_, F_SETFL, O_NONBLOCK);
90 if (token_epoll_fd_ != -1) close(token_epoll_fd_);
91 struct epoll_event ev;
92 token_epoll_fd_ = epoll_create1(0);
93 ev.events = EPOLLIN | EPOLLPRI;
94 ev.data.fd = token_socket_;
95 if (epoll_ctl(token_epoll_fd_, EPOLL_CTL_ADD, token_socket_, &ev) == -1)
97 TLOG(TLVL_ERROR) <<
"Could not register listen socket to epoll fd";
101 if (token_socket_ == -1 || token_epoll_fd_ == -1)
103 TLOG(TLVL_DEBUG) <<
"One of the listen sockets was not opened successfully.";
107 auto nfds = epoll_wait(token_epoll_fd_, &receive_token_events_[0], receive_token_events_.size(), update_interval_msec_);
110 perror(
"epoll_wait");
114 while (reception_is_paused_ && !shutdown_requested_)
119 TLOG(TLVL_DEBUG) <<
"Received " << nfds <<
" events";
120 for (
auto n = 0; n < nfds; ++n)
122 if (receive_token_events_[n].data.fd == token_socket_)
124 TLOG(TLVL_DEBUG) <<
"Accepting new connection on token_socket";
126 socklen_t arglen =
sizeof(addr);
127 auto conn_sock = accept(token_socket_, (
struct sockaddr*)&addr, &arglen);
128 fcntl(conn_sock, F_SETFL, O_NONBLOCK);
136 receive_token_addrs_[conn_sock] = std::string(inet_ntoa(addr.sin_addr));
137 TLOG(TLVL_DEBUG) <<
"New fd is " << conn_sock <<
" for data-receiver at " << receive_token_addrs_[conn_sock];
138 struct epoll_event ev;
139 ev.events = EPOLLIN | EPOLLET;
140 ev.data.fd = conn_sock;
141 if (epoll_ctl(token_epoll_fd_, EPOLL_CTL_ADD, conn_sock, &ev) == -1)
143 perror(
"epoll_ctl: conn_sock");
149 auto startTime = artdaq::MonitoredQuantity::getCurrentTime();
154 detail::RoutingToken buff;
155 sts += read(receive_token_events_[n].data.fd, &buff,
sizeof(detail::RoutingToken) - sts);
158 TLOG(TLVL_INFO) <<
"Received 0-size token from " << receive_token_addrs_[receive_token_events_[n].data.fd];
161 else if (sts < 0 && errno == EAGAIN)
163 TLOG(TLVL_DEBUG) <<
"No more tokens from this rank. Continuing poll loop.";
168 TLOG(TLVL_ERROR) <<
"Error reading from token socket: sts=" << sts <<
", errno=" << errno;
169 receive_token_addrs_.erase(receive_token_events_[n].data.fd);
170 close(receive_token_events_[n].data.fd);
171 epoll_ctl(token_epoll_fd_, EPOLL_CTL_DEL, receive_token_events_[n].data.fd, NULL);
174 else if (sts ==
sizeof(detail::RoutingToken) && buff.header != TOKEN_MAGIC)
176 TLOG(TLVL_ERROR) <<
"Received invalid token from " << receive_token_addrs_[receive_token_events_[n].data.fd] <<
" sts=" << sts;
179 else if (sts ==
sizeof(detail::RoutingToken))
182 TLOG(TLVL_DEBUG) <<
"Received token from " << buff.rank <<
" indicating " << buff.new_slots_free <<
" slots are free. (run=" << buff.run_number <<
")";
183 if (buff.run_number != run_number_)
185 TLOG(TLVL_DEBUG) <<
"Received token from a different run number! Current = " << run_number_ <<
", token = " << buff.run_number <<
", ignoring (n=" << buff.new_slots_free <<
")";
189 received_token_count_ += buff.new_slots_free;
192 policy_->AddReceiverToken(buff.rank, buff.new_slots_free);
196 if (!received_token_counter_.count(buff.rank)) received_token_counter_[buff.rank] = 0;
197 received_token_counter_[buff.rank] += buff.new_slots_free;
198 TLOG(TLVL_DEBUG) <<
"RoutingMasterMode is RouteBySendCount. I have " << received_token_counter_[buff.rank] <<
" tokens for rank " << buff.rank <<
" and I need " << number_of_senders_ <<
".";
199 while (received_token_counter_[buff.rank] >= number_of_senders_)
201 TLOG(TLVL_DEBUG) <<
"RoutingMasterMode is RouteBySendCount. I have " << received_token_counter_[buff.rank] <<
" tokens for rank " << buff.rank <<
" and I need " << number_of_senders_
202 <<
"... Sending token to policy";
203 policy_->AddReceiverToken(buff.rank, 1);
204 received_token_counter_[buff.rank] -= number_of_senders_;
210 auto delta_time = artdaq::MonitoredQuantity::getCurrentTime() - startTime;
211 if (statsHelperPtr_.get() !=
nullptr) { statsHelperPtr_->addSample(tokens_received_stat_key_, delta_time); }
RoutingMasterMode
Mode indicating whether the RoutingMaster is routing events by Sequence ID or by Send Count...
void stopTokenReception(bool force=false)
Stops the reception of event builder tokens.
TokenReceiver(const fhicl::ParameterSet &ps, std::shared_ptr< RoutingMasterPolicy > policy, detail::RoutingMasterMode routing_mode, size_t number_of_senders, size_t update_interval_msec)
TokenReceiver Constructor.
Events should be routed by sequence ID (BR -> EB)
The RoutingToken contains the magic bytes, the rank of the token sender, and the number of slots free...
int TCP_listen_fd(int port, int rcvbuf)
Create a TCP listening socket on the given port and INADDR_ANY, with the given receive buffer...
void startTokenReception()
Starts the reception of event builder tokens.
virtual ~TokenReceiver()
TokenReceiver Destructor.
Events should be routed by send count (EB -> Agg)