1 #include "artdaq/DAQdata/Globals.hh"
2 #define TRACE_NAME (app_name + "_TokenReceiver").c_str()
4 #include "artdaq/DAQrate/TokenReceiver.hh"
10 : token_port_(ps.get<int>(
"routing_token_port", 35555))
12 , routing_mode_(routing_mode)
13 , number_of_senders_(number_of_senders)
14 , update_interval_msec_(update_interval_msec)
17 , thread_is_running_(false)
18 , reception_is_paused_(false)
19 , shutdown_requested_(false)
21 , received_token_counter_()
22 , statsHelperPtr_(nullptr)
24 receive_token_events_ = std::vector<epoll_event>(policy_->GetReceiverCount() + 1);
29 stopTokenReception(
true);
34 if (token_thread_.joinable()) token_thread_.join();
35 boost::thread::attributes attrs;
36 attrs.set_stack_size(4096 * 2000);
38 reception_is_paused_ =
false;
39 shutdown_requested_ =
false;
41 TLOG(TLVL_INFO) <<
"Starting Token Reception Thread" ;
43 token_thread_ = boost::thread(attrs, boost::bind(&TokenReceiver::receiveTokensLoop_,
this));
45 catch(boost::exception
const& e)
47 std::cerr <<
"Exception encountered starting Token Reception thread: " << boost::diagnostic_information(e) <<
", errno=" << errno << std::endl;
50 received_token_count_ = 0;
51 thread_is_running_ =
true;
52 TLOG(TLVL_INFO) <<
"Started Token Reception Thread";
57 shutdown_requested_ =
true;
58 reception_is_paused_ =
false;
59 if (thread_is_running_) {
60 if (received_token_count_ == 0 && !force)
62 TLOG(TLVL_DEBUG) <<
"Stop request received by TokenReceiver, but no tokens have ever been received.";
64 TLOG(TLVL_DEBUG) <<
"Joining tokenThread";
65 if (token_thread_.joinable()) token_thread_.join();
66 thread_is_running_ =
false;
69 if (token_socket_ != -1)
77 void artdaq::TokenReceiver::receiveTokensLoop_()
79 while (!shutdown_requested_)
81 TLOG(TLVL_DEBUG) <<
"Receive Token loop start" ;
82 if (token_socket_ == -1)
84 TLOG(TLVL_DEBUG) <<
"Opening token listener socket" ;
86 fcntl(token_socket_, F_SETFL, O_NONBLOCK);
88 if (token_epoll_fd_ != -1) close(token_epoll_fd_);
89 struct epoll_event ev;
90 token_epoll_fd_ = epoll_create1(0);
91 ev.events = EPOLLIN | EPOLLPRI;
92 ev.data.fd = token_socket_;
93 if (epoll_ctl(token_epoll_fd_, EPOLL_CTL_ADD, token_socket_, &ev) == -1)
95 TLOG(TLVL_ERROR) <<
"Could not register listen socket to epoll fd" ;
99 if (token_socket_ == -1 || token_epoll_fd_ == -1)
101 TLOG(TLVL_DEBUG) <<
"One of the listen sockets was not opened successfully." ;
105 auto nfds = epoll_wait(token_epoll_fd_, &receive_token_events_[0], receive_token_events_.size(), update_interval_msec_);
108 perror(
"epoll_wait");
112 while (reception_is_paused_ && !shutdown_requested_)
117 TLOG(TLVL_DEBUG) <<
"Received " << nfds <<
" events" ;
118 for (
auto n = 0; n < nfds; ++n)
120 if (receive_token_events_[n].data.fd == token_socket_)
122 TLOG(TLVL_DEBUG) <<
"Accepting new connection on token_socket" ;
124 socklen_t arglen =
sizeof(addr);
125 auto conn_sock = accept(token_socket_, (
struct sockaddr *)&addr, &arglen);
126 fcntl(conn_sock, F_SETFL, O_NONBLOCK);
134 receive_token_addrs_[conn_sock] = std::string(inet_ntoa(addr.sin_addr));
135 TLOG(TLVL_DEBUG) <<
"New fd is " << conn_sock <<
" for data-receiver at " << receive_token_addrs_[conn_sock];
136 struct epoll_event ev;
137 ev.events = EPOLLIN | EPOLLET;
138 ev.data.fd = conn_sock;
139 if (epoll_ctl(token_epoll_fd_, EPOLL_CTL_ADD, conn_sock, &ev) == -1)
141 perror(
"epoll_ctl: conn_sock");
147 auto startTime = artdaq::MonitoredQuantity::getCurrentTime();
152 detail::RoutingToken buff;
153 sts += read(receive_token_events_[n].data.fd, &buff,
sizeof(detail::RoutingToken) - sts);
156 TLOG(TLVL_INFO) <<
"Received 0-size token from " << receive_token_addrs_[receive_token_events_[n].data.fd];
159 else if(sts < 0 && errno == EAGAIN)
161 TLOG(TLVL_DEBUG) <<
"No more tokens from this rank. Continuing poll loop.";
166 TLOG(TLVL_ERROR) <<
"Error reading from token socket: sts=" << sts <<
", errno=" << errno;
167 receive_token_addrs_.erase(receive_token_events_[n].data.fd);
168 close(receive_token_events_[n].data.fd);
169 epoll_ctl(token_epoll_fd_, EPOLL_CTL_DEL, receive_token_events_[n].data.fd, NULL);
172 else if (sts ==
sizeof(detail::RoutingToken) && buff.header != TOKEN_MAGIC)
174 TLOG(TLVL_ERROR) <<
"Received invalid token from " << receive_token_addrs_[receive_token_events_[n].data.fd] <<
" sts=" << sts;
177 else if(sts ==
sizeof(detail::RoutingToken))
180 TLOG(TLVL_DEBUG) <<
"Received token from " << buff.rank <<
" indicating " << buff.new_slots_free <<
" slots are free. (run=" << buff.run_number <<
")" ;
181 if (buff.run_number != run_number_)
183 TLOG(TLVL_DEBUG) <<
"Received token from a different run number! Current = " << run_number_ <<
", token = " << buff.run_number <<
", ignoring (n=" << buff.new_slots_free <<
")";
187 received_token_count_ += buff.new_slots_free;
190 policy_->AddReceiverToken(buff.rank, buff.new_slots_free);
194 if (!received_token_counter_.count(buff.rank)) received_token_counter_[buff.rank] = 0;
195 received_token_counter_[buff.rank] += buff.new_slots_free;
196 TLOG(TLVL_DEBUG) <<
"RoutingMasterMode is RouteBySendCount. I have " << received_token_counter_[buff.rank] <<
" tokens for rank " << buff.rank <<
" and I need " << number_of_senders_ <<
"." ;
197 while (received_token_counter_[buff.rank] >= number_of_senders_)
199 TLOG(TLVL_DEBUG) <<
"RoutingMasterMode is RouteBySendCount. I have " << received_token_counter_[buff.rank] <<
" tokens for rank " << buff.rank <<
" and I need " << number_of_senders_
200 <<
"... Sending token to policy" ;
201 policy_->AddReceiverToken(buff.rank, 1);
202 received_token_counter_[buff.rank] -= number_of_senders_;
208 auto delta_time = artdaq::MonitoredQuantity::getCurrentTime() - startTime;
209 if (statsHelperPtr_.get() !=
nullptr) {statsHelperPtr_->addSample(tokens_received_stat_key_, delta_time);}
RoutingMasterMode
Mode indicating whether the RoutingMaster is routing events by Sequence ID or by Send Count...
void stopTokenReception(bool force=false)
Stops the reception of event builder tokens.
TokenReceiver(const fhicl::ParameterSet &ps, std::shared_ptr< RoutingMasterPolicy > policy, detail::RoutingMasterMode routing_mode, size_t number_of_senders, size_t update_interval_msec)
TokenReceiver Constructor.
Events should be routed by sequence ID (BR -> EB)
The RoutingToken contains the magic bytes, the rank of the token sender, and the number of slots free...
int TCP_listen_fd(int port, int rcvbuf)
Create a TCP listening socket on the given port and INADDR_ANY, with the given receive buffer...
void startTokenReception()
Starts the reception of event builder tokens.
virtual ~TokenReceiver()
TokenReceiver Destructor.
Events should be routed by send count (EB -> Agg)