artdaq  v3_06_00
TokenReceiver.cc
1 #include "artdaq/DAQdata/Globals.hh"
2 #define TRACE_NAME (app_name + "_TokenReceiver").c_str()
3 
4 #include "artdaq/DAQrate/TokenReceiver.hh"
6 #include <arpa/inet.h>
7 
8 artdaq::TokenReceiver::TokenReceiver(const fhicl::ParameterSet& ps, std::shared_ptr<RoutingMasterPolicy> policy,
9  detail::RoutingMasterMode routing_mode, size_t number_of_senders, size_t update_interval_msec)
10  : token_port_(ps.get<int>("routing_token_port", 35555))
11  , policy_(policy)
12  , routing_mode_(routing_mode)
13  , number_of_senders_(number_of_senders)
14  , update_interval_msec_(update_interval_msec)
15  , token_socket_(-1)
16  , token_epoll_fd_(-1)
17  , thread_is_running_(false)
18  , reception_is_paused_(false)
19  , shutdown_requested_(false)
20  , run_number_(0)
21  , received_token_counter_()
22  , statsHelperPtr_(nullptr)
23 {
24  receive_token_events_ = std::vector<epoll_event>(policy_->GetReceiverCount() + 1);
25 }
26 
28 {
29  stopTokenReception(true);
30 }
31 
33 {
34  if (token_thread_.joinable()) token_thread_.join();
35  boost::thread::attributes attrs;
36  attrs.set_stack_size(4096 * 2000); // 8000 KB
37 
38  reception_is_paused_ = false;
39  shutdown_requested_ = false;
40 
41  TLOG(TLVL_INFO) << "Starting Token Reception Thread" ;
42  try {
43  token_thread_ = boost::thread(attrs, boost::bind(&TokenReceiver::receiveTokensLoop_, this));
44  }
45  catch(boost::exception const& e)
46  {
47  std::cerr << "Exception encountered starting Token Reception thread: " << boost::diagnostic_information(e) << ", errno=" << errno << std::endl;
48  exit(3);
49  }
50  received_token_count_ = 0;
51  thread_is_running_ = true;
52  TLOG(TLVL_INFO) << "Started Token Reception Thread";
53 }
54 
56 {
57  shutdown_requested_ = true;
58  reception_is_paused_ = false;
59  if (thread_is_running_) {
60  if (received_token_count_ == 0 && !force)
61  {
62  TLOG(TLVL_DEBUG) << "Stop request received by TokenReceiver, but no tokens have ever been received.";
63  }
64  TLOG(TLVL_DEBUG) << "Joining tokenThread";
65  if (token_thread_.joinable()) token_thread_.join();
66  thread_is_running_ = false;
67  }
68 
69  if (token_socket_ != -1)
70  {
71  close(token_socket_);
72  token_socket_ = -1;
73  token_epoll_fd_ = -1;
74  }
75 }
76 
77 void artdaq::TokenReceiver::receiveTokensLoop_()
78 {
79  while (!shutdown_requested_)
80  {
81  TLOG(TLVL_DEBUG) << "Receive Token loop start" ;
82  if (token_socket_ == -1)
83  {
84  TLOG(TLVL_DEBUG) << "Opening token listener socket" ;
85  token_socket_ = TCP_listen_fd(token_port_, 3 * sizeof(detail::RoutingToken));
86  fcntl(token_socket_, F_SETFL, O_NONBLOCK); // set O_NONBLOCK
87 
88  if (token_epoll_fd_ != -1) close(token_epoll_fd_);
89  struct epoll_event ev;
90  token_epoll_fd_ = epoll_create1(0);
91  ev.events = EPOLLIN | EPOLLPRI;
92  ev.data.fd = token_socket_;
93  if (epoll_ctl(token_epoll_fd_, EPOLL_CTL_ADD, token_socket_, &ev) == -1)
94  {
95  TLOG(TLVL_ERROR) << "Could not register listen socket to epoll fd" ;
96  exit(3);
97  }
98  }
99  if (token_socket_ == -1 || token_epoll_fd_ == -1)
100  {
101  TLOG(TLVL_DEBUG) << "One of the listen sockets was not opened successfully." ;
102  return;
103  }
104 
105  auto nfds = epoll_wait(token_epoll_fd_, &receive_token_events_[0], receive_token_events_.size(), update_interval_msec_);
106  if (nfds == -1)
107  {
108  perror("epoll_wait");
109  exit(EXIT_FAILURE);
110  }
111 
112  while (reception_is_paused_ && !shutdown_requested_)
113  {
114  usleep(10000);
115  }
116 
117  TLOG(TLVL_DEBUG) << "Received " << nfds << " events" ;
118  for (auto n = 0; n < nfds; ++n)
119  {
120  if (receive_token_events_[n].data.fd == token_socket_)
121  {
122  TLOG(TLVL_DEBUG) << "Accepting new connection on token_socket" ;
123  sockaddr_in addr;
124  socklen_t arglen = sizeof(addr);
125  auto conn_sock = accept(token_socket_, (struct sockaddr *)&addr, &arglen);
126  fcntl(conn_sock, F_SETFL, O_NONBLOCK); // set O_NONBLOCK
127 
128  if (conn_sock == -1)
129  {
130  perror("accept");
131  exit(EXIT_FAILURE);
132  }
133 
134  receive_token_addrs_[conn_sock] = std::string(inet_ntoa(addr.sin_addr));
135  TLOG(TLVL_DEBUG) << "New fd is " << conn_sock << " for data-receiver at " << receive_token_addrs_[conn_sock];
136  struct epoll_event ev;
137  ev.events = EPOLLIN | EPOLLET;
138  ev.data.fd = conn_sock;
139  if (epoll_ctl(token_epoll_fd_, EPOLL_CTL_ADD, conn_sock, &ev) == -1)
140  {
141  perror("epoll_ctl: conn_sock");
142  exit(EXIT_FAILURE);
143  }
144  }
145  else
146  {
147  auto startTime = artdaq::MonitoredQuantity::getCurrentTime();
148  bool reading = true;
149  int sts = 0;
150  while(reading)
151  {
152  detail::RoutingToken buff;
153  sts += read(receive_token_events_[n].data.fd, &buff, sizeof(detail::RoutingToken) - sts);
154  if (sts == 0)
155  {
156  TLOG(TLVL_INFO) << "Received 0-size token from " << receive_token_addrs_[receive_token_events_[n].data.fd];
157  reading = false;
158  }
159  else if(sts < 0 && errno == EAGAIN)
160  {
161  TLOG(TLVL_DEBUG) << "No more tokens from this rank. Continuing poll loop.";
162  reading = false;
163  }
164  else if(sts < 0)
165  {
166  TLOG(TLVL_ERROR) << "Error reading from token socket: sts=" << sts << ", errno=" << errno;
167  receive_token_addrs_.erase(receive_token_events_[n].data.fd);
168  close(receive_token_events_[n].data.fd);
169  epoll_ctl(token_epoll_fd_, EPOLL_CTL_DEL, receive_token_events_[n].data.fd, NULL);
170  reading = false;
171  }
172  else if (sts == sizeof(detail::RoutingToken) && buff.header != TOKEN_MAGIC)
173  {
174  TLOG(TLVL_ERROR) << "Received invalid token from " << receive_token_addrs_[receive_token_events_[n].data.fd] << " sts=" << sts;
175  reading = false;
176  }
177  else if(sts == sizeof(detail::RoutingToken))
178  {
179  sts = 0;
180  TLOG(TLVL_DEBUG) << "Received token from " << buff.rank << " indicating " << buff.new_slots_free << " slots are free. (run=" << buff.run_number << ")" ;
181  if (buff.run_number != run_number_)
182  {
183  TLOG(TLVL_DEBUG) << "Received token from a different run number! Current = " << run_number_ << ", token = " << buff.run_number << ", ignoring (n=" << buff.new_slots_free << ")";
184  }
185  else
186  {
187  received_token_count_ += buff.new_slots_free;
189  {
190  policy_->AddReceiverToken(buff.rank, buff.new_slots_free);
191  }
192  else if (routing_mode_ == detail::RoutingMasterMode::RouteBySendCount)
193  {
194  if (!received_token_counter_.count(buff.rank)) received_token_counter_[buff.rank] = 0;
195  received_token_counter_[buff.rank] += buff.new_slots_free;
196  TLOG(TLVL_DEBUG) << "RoutingMasterMode is RouteBySendCount. I have " << received_token_counter_[buff.rank] << " tokens for rank " << buff.rank << " and I need " << number_of_senders_ << "." ;
197  while (received_token_counter_[buff.rank] >= number_of_senders_)
198  {
199  TLOG(TLVL_DEBUG) << "RoutingMasterMode is RouteBySendCount. I have " << received_token_counter_[buff.rank] << " tokens for rank " << buff.rank << " and I need " << number_of_senders_
200  << "... Sending token to policy" ;
201  policy_->AddReceiverToken(buff.rank, 1);
202  received_token_counter_[buff.rank] -= number_of_senders_;
203  }
204  }
205  }
206  }
207  }
208  auto delta_time = artdaq::MonitoredQuantity::getCurrentTime() - startTime;
209  if (statsHelperPtr_.get() != nullptr) {statsHelperPtr_->addSample(tokens_received_stat_key_, delta_time);}
210  }
211  }
212  }
213 }
RoutingMasterMode
Mode indicating whether the RoutingMaster is routing events by Sequence ID or by Send Count...
void stopTokenReception(bool force=false)
Stops the reception of event builder tokens.
TokenReceiver(const fhicl::ParameterSet &ps, std::shared_ptr< RoutingMasterPolicy > policy, detail::RoutingMasterMode routing_mode, size_t number_of_senders, size_t update_interval_msec)
TokenReceiver Constructor.
Definition: TokenReceiver.cc:8
Events should be routed by sequence ID (BR -&gt; EB)
The RoutingToken contains the magic bytes, the rank of the token sender, and the number of slots free...
int TCP_listen_fd(int port, int rcvbuf)
Create a TCP listening socket on the given port and INADDR_ANY, with the given receive buffer...
void startTokenReception()
Starts the reception of event builder tokens.
virtual ~TokenReceiver()
TokenReceiver Destructor.
Events should be routed by send count (EB -&gt; Agg)