artdaq  v3_12_02
TokenSender.cc
1 #include "artdaq/DAQdata/Globals.hh" // Before trace.h gets included in ConcurrentQueue (from GlobalQueue)
2 #define TRACE_NAME (app_name + "_TokenSender").c_str()
3 #include <dlfcn.h>
4 #include <chrono>
5 #include <cstring>
6 #include <fstream>
7 #include <iomanip>
8 #include <sstream>
9 #include <utility>
10 #include "artdaq/DAQrate/detail/TokenSender.hh"
11 
12 #include "artdaq-core/Core/StatisticsCollection.hh"
14 #include "artdaq/DAQrate/detail/RoutingPacket.hh"
15 #include "cetlib_except/exception.h"
16 
17 namespace artdaq {
18 TokenSender::TokenSender(const fhicl::ParameterSet& pset)
19  : initialized_(false)
20  , send_routing_tokens_(pset.get<bool>("use_routing_manager", false))
21  , token_port_(pset.get<int>("routing_token_port", 35555))
22  , token_socket_(-1)
23  , token_address_(pset.get<std::string>("routing_manager_hostname", "localhost"))
24  , tokens_sent_(0)
25  , run_number_(0)
26 {
27  TLOG(TLVL_DEBUG + 32) << "TokenSender CONSTRUCTOR";
28 
29  setup_tokens_();
30  TLOG(TLVL_DEBUG + 35) << "artdaq::TokenSender::TokenSender ctor - reader_thread_ initialized";
31  initialized_ = true;
32 }
33 
35 {
36  TLOG(TLVL_INFO) << "Shutting down TokenSender, token_socket_: " << token_socket_;
37 
38  if (token_socket_ != -1)
39  {
40  if (shutdown(token_socket_, 2) != 0 && errno == ENOTSOCK)
41  {
42  TLOG(TLVL_ERROR) << "Shutdown of token_socket_ resulted in ENOTSOCK. NOT Closing file descriptor!";
43  }
44  else
45  {
46  close(token_socket_);
47  }
48  token_socket_ = -1;
49  }
50 }
51 
52 void TokenSender::setup_tokens_()
53 {
54  if (send_routing_tokens_)
55  {
56  TLOG(TLVL_DEBUG + 32) << "Creating Routing Token sending socket";
57  auto start_time = std::chrono::steady_clock::now();
58  while (token_socket_ < 0 && TimeUtils::GetElapsedTime(start_time) < 30)
59  {
60  token_socket_ = TCPConnect(token_address_.c_str(), token_port_, 0, sizeof(detail::RoutingToken));
61  if (token_socket_ < 0)
62  {
63  TLOG(TLVL_DEBUG + 33) << "Waited " << TimeUtils::GetElapsedTime(start_time) << " s for Routing Manager to open token socket";
64  usleep(100000);
65  }
66  }
67  if (token_socket_ < 0)
68  {
69  TLOG(TLVL_ERROR) << "I failed to create the socket for sending Routing Tokens! err=" << strerror(errno);
70  exit(1);
71  }
72  TLOG(TLVL_INFO) << "Routing Token sending socket created successfully for address " << token_address_;
73  }
74 }
75 
76 void TokenSender::send_routing_token_(int nSlots, int run_number, int rank)
77 {
78  TLOG(TLVL_DEBUG + 33) << "send_routing_token_ called, send_routing_tokens_=" << std::boolalpha << send_routing_tokens_;
79  if (!send_routing_tokens_)
80  {
81  return;
82  }
83  if (token_socket_ == -1)
84  {
85  setup_tokens_();
86  }
87  detail::RoutingToken token;
88  token.header = TOKEN_MAGIC;
89  token.rank = rank;
90  token.new_slots_free = nSlots;
91  token.run_number = run_number;
92 
93  TLOG(TLVL_DEBUG + 33) << "Sending RoutingToken to " << token_address_ << ":" << token_port_;
94  size_t sts = 0;
95  while (sts < sizeof(detail::RoutingToken))
96  {
97  auto res = send(token_socket_, reinterpret_cast<uint8_t*>(&token) + sts, sizeof(detail::RoutingToken) - sts, 0); // NOLINT(cppcoreguidelines-pro-type-reinterpret-cast,cppcoreguidelines-pro-bounds-pointer-arithmetic)
98  if (res < 0)
99  {
100  TLOG(TLVL_WARNING) << "Error on token_socket, reconnecting";
101  close(token_socket_);
102  token_socket_ = -1;
103  sts = 0;
104  setup_tokens_();
105  continue;
106  }
107  sts += res;
108  }
109  tokens_sent_ += nSlots;
110  TLOG(TLVL_DEBUG + 33) << "Done sending RoutingToken to " << token_address_ << ":" << token_port_;
111 }
112 
113 void TokenSender::SendRoutingToken(int nSlots, int run_number, int rank)
114 {
115  while (!initialized_)
116  {
117  usleep(1000);
118  }
119  if (!send_routing_tokens_)
120  {
121  return;
122  }
123  boost::thread token([=] { send_routing_token_(nSlots, run_number, rank); });
124  token.detach();
125  usleep(0); // Give up time slice
126 }
127 
128 } // namespace artdaq
int TCPConnect(char const *host_in, int dflt_port, int64_t flags=0, int sndbufsiz=0)
Connect to a host on a given port.
Definition: TCPConnect.cc:377
void SendRoutingToken(int nSlots, int run_number, int rank=my_rank)
Send a RoutingToken message indicating that slots are available.
Definition: TokenSender.cc:113
The RoutingToken contains the magic bytes, the rank of the token sender, and the number of slots free...
virtual ~TokenSender()
TokenSender Destructor.
Definition: TokenSender.cc:34
TokenSender()=delete
Default Constructor is deleted.