artdaq  v3_07_02
TokenReceiver.cc
1 #include "artdaq/DAQdata/Globals.hh"
2 #define TRACE_NAME (app_name + "_TokenReceiver").c_str()
3 
4 #include <arpa/inet.h>
6 #include "artdaq/DAQrate/TokenReceiver.hh"
7 
8 artdaq::TokenReceiver::TokenReceiver(const fhicl::ParameterSet& ps, std::shared_ptr<RoutingMasterPolicy> policy,
9  detail::RoutingMasterMode routing_mode, size_t number_of_senders, size_t update_interval_msec)
10  : token_port_(ps.get<int>("routing_token_port", 35555))
11  , policy_(policy)
12  , routing_mode_(routing_mode)
13  , number_of_senders_(number_of_senders)
14  , update_interval_msec_(update_interval_msec)
15  , token_socket_(-1)
16  , token_epoll_fd_(-1)
17  , thread_is_running_(false)
18  , reception_is_paused_(false)
19  , shutdown_requested_(false)
20  , run_number_(0)
21  , received_token_counter_()
22  , statsHelperPtr_(nullptr)
23 {
24  receive_token_events_ = std::vector<epoll_event>(policy_->GetReceiverCount() + 1);
25 }
26 
28 {
29  stopTokenReception(true);
30 }
31 
33 {
34  if (token_thread_.joinable()) token_thread_.join();
35  boost::thread::attributes attrs;
36  attrs.set_stack_size(4096 * 2000); // 8000 KB
37 
38  reception_is_paused_ = false;
39  shutdown_requested_ = false;
40 
41  TLOG(TLVL_INFO) << "Starting Token Reception Thread";
42  try
43  {
44  token_thread_ = boost::thread(attrs, boost::bind(&TokenReceiver::receiveTokensLoop_, this));
45  }
46  catch (boost::exception const& e)
47  {
48  std::cerr << "Exception encountered starting Token Reception thread: " << boost::diagnostic_information(e) << ", errno=" << errno << std::endl;
49  exit(3);
50  }
51  received_token_count_ = 0;
52  thread_is_running_ = true;
53  TLOG(TLVL_INFO) << "Started Token Reception Thread";
54 }
55 
57 {
58  shutdown_requested_ = true;
59  reception_is_paused_ = false;
60  if (thread_is_running_)
61  {
62  if (received_token_count_ == 0 && !force)
63  {
64  TLOG(TLVL_DEBUG) << "Stop request received by TokenReceiver, but no tokens have ever been received.";
65  }
66  TLOG(TLVL_DEBUG) << "Joining tokenThread";
67  if (token_thread_.joinable()) token_thread_.join();
68  thread_is_running_ = false;
69  }
70 
71  if (token_socket_ != -1)
72  {
73  close(token_socket_);
74  token_socket_ = -1;
75  token_epoll_fd_ = -1;
76  }
77 }
78 
79 void artdaq::TokenReceiver::receiveTokensLoop_()
80 {
81  while (!shutdown_requested_)
82  {
83  TLOG(TLVL_DEBUG) << "Receive Token loop start";
84  if (token_socket_ == -1)
85  {
86  TLOG(TLVL_DEBUG) << "Opening token listener socket";
87  token_socket_ = TCP_listen_fd(token_port_, 3 * sizeof(detail::RoutingToken));
88  fcntl(token_socket_, F_SETFL, O_NONBLOCK); // set O_NONBLOCK
89 
90  if (token_epoll_fd_ != -1) close(token_epoll_fd_);
91  struct epoll_event ev;
92  token_epoll_fd_ = epoll_create1(0);
93  ev.events = EPOLLIN | EPOLLPRI;
94  ev.data.fd = token_socket_;
95  if (epoll_ctl(token_epoll_fd_, EPOLL_CTL_ADD, token_socket_, &ev) == -1)
96  {
97  TLOG(TLVL_ERROR) << "Could not register listen socket to epoll fd";
98  exit(3);
99  }
100  }
101  if (token_socket_ == -1 || token_epoll_fd_ == -1)
102  {
103  TLOG(TLVL_DEBUG) << "One of the listen sockets was not opened successfully.";
104  return;
105  }
106 
107  auto nfds = epoll_wait(token_epoll_fd_, &receive_token_events_[0], receive_token_events_.size(), update_interval_msec_);
108  if (nfds == -1)
109  {
110  perror("epoll_wait");
111  exit(EXIT_FAILURE);
112  }
113 
114  while (reception_is_paused_ && !shutdown_requested_)
115  {
116  usleep(10000);
117  }
118 
119  TLOG(TLVL_DEBUG) << "Received " << nfds << " events";
120  for (auto n = 0; n < nfds; ++n)
121  {
122  if (receive_token_events_[n].data.fd == token_socket_)
123  {
124  TLOG(TLVL_DEBUG) << "Accepting new connection on token_socket";
125  sockaddr_in addr;
126  socklen_t arglen = sizeof(addr);
127  auto conn_sock = accept(token_socket_, (struct sockaddr*)&addr, &arglen);
128  fcntl(conn_sock, F_SETFL, O_NONBLOCK); // set O_NONBLOCK
129 
130  if (conn_sock == -1)
131  {
132  perror("accept");
133  exit(EXIT_FAILURE);
134  }
135 
136  receive_token_addrs_[conn_sock] = std::string(inet_ntoa(addr.sin_addr));
137  TLOG(TLVL_DEBUG) << "New fd is " << conn_sock << " for data-receiver at " << receive_token_addrs_[conn_sock];
138  struct epoll_event ev;
139  ev.events = EPOLLIN | EPOLLET;
140  ev.data.fd = conn_sock;
141  if (epoll_ctl(token_epoll_fd_, EPOLL_CTL_ADD, conn_sock, &ev) == -1)
142  {
143  perror("epoll_ctl: conn_sock");
144  exit(EXIT_FAILURE);
145  }
146  }
147  else
148  {
149  auto startTime = artdaq::MonitoredQuantity::getCurrentTime();
150  bool reading = true;
151  int sts = 0;
152  while (reading)
153  {
154  detail::RoutingToken buff;
155  sts += read(receive_token_events_[n].data.fd, &buff, sizeof(detail::RoutingToken) - sts);
156  if (sts == 0)
157  {
158  TLOG(TLVL_INFO) << "Received 0-size token from " << receive_token_addrs_[receive_token_events_[n].data.fd];
159  reading = false;
160  }
161  else if (sts < 0 && errno == EAGAIN)
162  {
163  TLOG(TLVL_DEBUG) << "No more tokens from this rank. Continuing poll loop.";
164  reading = false;
165  }
166  else if (sts < 0)
167  {
168  TLOG(TLVL_ERROR) << "Error reading from token socket: sts=" << sts << ", errno=" << errno;
169  receive_token_addrs_.erase(receive_token_events_[n].data.fd);
170  close(receive_token_events_[n].data.fd);
171  epoll_ctl(token_epoll_fd_, EPOLL_CTL_DEL, receive_token_events_[n].data.fd, NULL);
172  reading = false;
173  }
174  else if (sts == sizeof(detail::RoutingToken) && buff.header != TOKEN_MAGIC)
175  {
176  TLOG(TLVL_ERROR) << "Received invalid token from " << receive_token_addrs_[receive_token_events_[n].data.fd] << " sts=" << sts;
177  reading = false;
178  }
179  else if (sts == sizeof(detail::RoutingToken))
180  {
181  sts = 0;
182  TLOG(TLVL_DEBUG) << "Received token from " << buff.rank << " indicating " << buff.new_slots_free << " slots are free. (run=" << buff.run_number << ")";
183  if (buff.run_number != run_number_)
184  {
185  TLOG(TLVL_DEBUG) << "Received token from a different run number! Current = " << run_number_ << ", token = " << buff.run_number << ", ignoring (n=" << buff.new_slots_free << ")";
186  }
187  else
188  {
189  received_token_count_ += buff.new_slots_free;
191  {
192  policy_->AddReceiverToken(buff.rank, buff.new_slots_free);
193  }
194  else if (routing_mode_ == detail::RoutingMasterMode::RouteBySendCount)
195  {
196  if (!received_token_counter_.count(buff.rank)) received_token_counter_[buff.rank] = 0;
197  received_token_counter_[buff.rank] += buff.new_slots_free;
198  TLOG(TLVL_DEBUG) << "RoutingMasterMode is RouteBySendCount. I have " << received_token_counter_[buff.rank] << " tokens for rank " << buff.rank << " and I need " << number_of_senders_ << ".";
199  while (received_token_counter_[buff.rank] >= number_of_senders_)
200  {
201  TLOG(TLVL_DEBUG) << "RoutingMasterMode is RouteBySendCount. I have " << received_token_counter_[buff.rank] << " tokens for rank " << buff.rank << " and I need " << number_of_senders_
202  << "... Sending token to policy";
203  policy_->AddReceiverToken(buff.rank, 1);
204  received_token_counter_[buff.rank] -= number_of_senders_;
205  }
206  }
207  }
208  }
209  }
210  auto delta_time = artdaq::MonitoredQuantity::getCurrentTime() - startTime;
211  if (statsHelperPtr_.get() != nullptr) { statsHelperPtr_->addSample(tokens_received_stat_key_, delta_time); }
212  }
213  }
214  }
215 }
RoutingMasterMode
Mode indicating whether the RoutingMaster is routing events by Sequence ID or by Send Count...
void stopTokenReception(bool force=false)
Stops the reception of event builder tokens.
TokenReceiver(const fhicl::ParameterSet &ps, std::shared_ptr< RoutingMasterPolicy > policy, detail::RoutingMasterMode routing_mode, size_t number_of_senders, size_t update_interval_msec)
TokenReceiver Constructor.
Definition: TokenReceiver.cc:8
Events should be routed by sequence ID (BR -&gt; EB)
The RoutingToken contains the magic bytes, the rank of the token sender, and the number of slots free...
int TCP_listen_fd(int port, int rcvbuf)
Create a TCP listening socket on the given port and INADDR_ANY, with the given receive buffer...
void startTokenReception()
Starts the reception of event builder tokens.
virtual ~TokenReceiver()
TokenReceiver Destructor.
Events should be routed by send count (EB -&gt; Agg)