1 #include "artdaq/DAQdata/Globals.hh"
2 #define TRACE_NAME (app_name + "_TokenReceiver").c_str()
10 #include "artdaq/DAQrate/detail/TokenReceiver.hh"
13 size_t update_interval_msec)
14 : token_port_(ps.get<int>(
"routing_token_port", 35555))
15 , policy_(std::move(std::move(policy)))
16 , update_interval_msec_(update_interval_msec)
19 , thread_is_running_(false)
20 , reception_is_paused_(false)
21 , shutdown_requested_(false)
23 , statsHelperPtr_(nullptr)
25 receive_token_events_ = std::vector<epoll_event>(policy_->GetReceiverCount() + 1);
30 stopTokenReception(
true);
35 if (token_thread_.joinable())
39 boost::thread::attributes attrs;
40 attrs.set_stack_size(4096 * 2000);
42 reception_is_paused_ =
false;
43 shutdown_requested_ =
false;
45 TLOG(TLVL_INFO) <<
"Starting Token Reception Thread";
48 token_thread_ = boost::thread(attrs, boost::bind(&TokenReceiver::receiveTokensLoop_,
this));
50 snprintf(tname,
sizeof(tname) - 1,
"%d-TokenRecv", my_rank);
51 tname[
sizeof(tname) - 1] =
'\0';
52 auto handle = token_thread_.native_handle();
53 pthread_setname_np(handle, tname);
55 catch (boost::exception
const& e)
57 TLOG(TLVL_ERROR) <<
"Exception encountered starting Token Reception thread: " << boost::diagnostic_information(e) <<
", errno=" << errno;
58 std::cerr <<
"Exception encountered starting Token Reception thread: " << boost::diagnostic_information(e) <<
", errno=" << errno << std::endl;
61 received_token_count_ = 0;
62 thread_is_running_ =
true;
63 TLOG(TLVL_INFO) <<
"Started Token Reception Thread";
68 shutdown_requested_ =
true;
69 reception_is_paused_ =
false;
70 if (thread_is_running_)
72 if (received_token_count_ == 0 && !force)
74 TLOG(TLVL_DEBUG + 32) <<
"Stop request received by TokenReceiver, but no tokens have ever been received.";
76 TLOG(TLVL_DEBUG + 32) <<
"Joining tokenThread";
79 if (token_thread_.joinable())
88 thread_is_running_ =
false;
91 if (token_socket_ != -1)
99 void artdaq::TokenReceiver::receiveTokensLoop_()
101 while (!shutdown_requested_)
103 TLOG(TLVL_DEBUG + 33) <<
"Receive Token loop start";
104 if (token_socket_ == -1)
106 TLOG(TLVL_DEBUG + 32) <<
"Opening token listener socket";
109 if (token_epoll_fd_ != -1)
111 close(token_epoll_fd_);
113 struct epoll_event ev;
114 token_epoll_fd_ = epoll_create1(0);
116 ev.data.fd = token_socket_;
117 if (epoll_ctl(token_epoll_fd_, EPOLL_CTL_ADD, token_socket_, &ev) == -1)
119 TLOG(TLVL_ERROR) <<
"Could not register listen socket to epoll fd";
123 if (token_socket_ == -1 || token_epoll_fd_ == -1)
125 TLOG(TLVL_DEBUG + 32) <<
"One of the listen sockets was not opened successfully.";
129 auto nfds = epoll_wait(token_epoll_fd_, &receive_token_events_[0], receive_token_events_.size(), update_interval_msec_);
132 TLOG(TLVL_ERROR) <<
"Error status received from epoll_wait, exiting with code " << EXIT_FAILURE <<
", errno=" << errno <<
" (" << strerror(errno) <<
")";
133 perror(
"epoll_wait");
137 while (reception_is_paused_ && !shutdown_requested_)
142 TLOG(TLVL_DEBUG + 35) <<
"Received " << nfds <<
" events on token sockets";
143 for (
auto n = 0; n < nfds; ++n)
145 if (receive_token_events_[n].data.fd == token_socket_)
147 TLOG(TLVL_DEBUG + 32) <<
"Accepting new connection on token_socket";
149 socklen_t arglen =
sizeof(addr);
150 auto conn_sock = accept(token_socket_, reinterpret_cast<struct sockaddr*>(&addr), &arglen);
151 fcntl(conn_sock, F_SETFL, O_NONBLOCK);
155 TLOG(TLVL_ERROR) <<
"Error status received from accept, exiting with code " << EXIT_FAILURE <<
", errno=" << errno <<
" (" << strerror(errno) <<
")";
160 receive_token_addrs_[conn_sock] = std::string(inet_ntoa(addr.sin_addr));
161 TLOG(TLVL_DEBUG + 32) <<
"New fd is " << conn_sock <<
" for data-receiver at " << receive_token_addrs_[conn_sock];
162 struct epoll_event ev;
164 ev.data.fd = conn_sock;
165 if (epoll_ctl(token_epoll_fd_, EPOLL_CTL_ADD, conn_sock, &ev) == -1)
167 TLOG(TLVL_ERROR) <<
"Error status received from epoll_ctl, exiting with code " << EXIT_FAILURE <<
", errno=" << errno <<
" (" << strerror(errno) <<
")";
168 perror(
"epoll_ctl: conn_sock");
172 else if ((receive_token_events_[n].events & EPOLLIN) != 0)
174 auto startTime = artdaq::MonitoredQuantity::getCurrentTime();
176 detail::RoutingToken buff;
177 int sts = recv(receive_token_events_[n].data.fd, &buff,
sizeof(detail::RoutingToken), MSG_WAITALL);
180 TLOG(TLVL_WARNING) <<
"Received 0-size token from " << receive_token_addrs_[receive_token_events_[n].data.fd] <<
", closing socket";
181 receive_token_addrs_.erase(receive_token_events_[n].data.fd);
182 close(receive_token_events_[n].data.fd);
183 epoll_ctl(token_epoll_fd_, EPOLL_CTL_DEL, receive_token_events_[n].data.fd,
nullptr);
185 else if (sts < 0 && errno == EAGAIN)
187 TLOG(TLVL_DEBUG + 32) <<
"No more tokens from this rank. Continuing poll loop.";
191 TLOG(TLVL_ERROR) <<
"Error reading from token socket: sts=" << sts <<
", errno=" << errno;
192 receive_token_addrs_.erase(receive_token_events_[n].data.fd);
193 close(receive_token_events_[n].data.fd);
194 epoll_ctl(token_epoll_fd_, EPOLL_CTL_DEL, receive_token_events_[n].data.fd,
nullptr);
196 else if (sts ==
sizeof(detail::RoutingToken) && buff.header != TOKEN_MAGIC)
198 TLOG(TLVL_ERROR) <<
"Received invalid token from " << receive_token_addrs_[receive_token_events_[n].data.fd] <<
" sts=" << sts;
200 else if (sts ==
sizeof(detail::RoutingToken))
202 TLOG(TLVL_DEBUG + 32) <<
"Received token from " << buff.rank <<
" indicating " << buff.new_slots_free <<
" slots are free. (run=" << buff.run_number <<
")";
203 if (buff.run_number != run_number_)
205 TLOG(TLVL_DEBUG + 32) <<
"Received token from a different run number! Current = " << run_number_ <<
", token = " << buff.run_number <<
", ignoring (n=" << buff.new_slots_free <<
")";
209 received_token_count_ += buff.new_slots_free;
210 policy_->AddReceiverToken(buff.rank, buff.new_slots_free);
213 auto delta_time = artdaq::MonitoredQuantity::getCurrentTime() - startTime;
214 if (statsHelperPtr_ !=
nullptr) { statsHelperPtr_->addSample(tokens_received_stat_key_, delta_time); }
218 TLOG(TLVL_DEBUG + 32) <<
"Received event mask " << receive_token_events_[n].events <<
" from token fd " << receive_token_events_[n].data.fd;
void stopTokenReception(bool force=false)
Stops the reception of event builder tokens.
The RoutingToken contains the magic bytes, the rank of the token sender, and the number of slots free...
int TCP_listen_fd(int port, int rcvbuf)
Create a TCP listening socket on the given port and INADDR_ANY, with the given receive buffer...
TokenReceiver(const fhicl::ParameterSet &ps, std::shared_ptr< RoutingManagerPolicy > policy, size_t update_interval_msec)
TokenReceiver Constructor.
void startTokenReception()
Starts the reception of event builder tokens.
virtual ~TokenReceiver()
TokenReceiver Destructor.