artdaq  v3_00_03
RequestReceiver.cc
1 #define TRACE_NAME "RequestReceiver"
2 
3 #include "artdaq/DAQrate/RequestReceiver.hh"
4 #include "artdaq/DAQdata/Globals.hh"
5 #include "artdaq/DAQrate/detail/RequestMessage.hh"
6 
7 #include <boost/exception/all.hpp>
8 #include <boost/throw_exception.hpp>
9 
10 #include <limits>
11 #include <iterator>
12 
13 #include "canvas/Utilities/Exception.h"
14 #include "cetlib_except/exception.h"
15 #include "fhiclcpp/ParameterSet.h"
16 
17 #include "artdaq-core/Utilities/SimpleLookupPolicy.hh"
18 #include "artdaq-core/Data/Fragment.hh"
19 #include "artdaq-core/Data/ContainerFragmentLoader.hh"
20 #include "artdaq-core/Utilities/ExceptionHandler.hh"
21 #include "artdaq-core/Utilities/TimeUtils.hh"
22 
23 #include <fstream>
24 #include <iomanip>
25 #include <iterator>
26 #include <iostream>
27 #include <iomanip>
28 #include <algorithm>
29 #include <sys/poll.h>
31 
32 artdaq::RequestReceiver::RequestReceiver()
33  : request_port_(3001)
34  , request_addr_("227.128.12.26")
35  , running_(false)
36  , requests_()
37  , request_stop_requested_(false)
38  , request_received_(false)
39  , end_of_run_timeout_ms_(1000)
40  , should_stop_(false)
41  , highest_seen_request_(0)
42 {}
43 
44 artdaq::RequestReceiver::RequestReceiver(const fhicl::ParameterSet& ps)
45  : request_port_(ps.get<int>("request_port", 3001))
46  , request_addr_(ps.get<std::string>("request_address", "227.128.12.26"))
47  , running_(false)
48  , requests_()
49  , request_stop_requested_(false)
50  , request_received_(false)
51  , end_of_run_timeout_ms_(ps.get<size_t>("end_of_run_quiet_timeout_ms", 1000))
52  , should_stop_(false)
53  , highest_seen_request_(0)
54 {
55  setupRequestListener();
56 }
57 
59 {
60  TLOG_INFO("RequestReceiver") << "Setting up request listen socket, rank=" << my_rank << ", address=" << request_addr_ << ":" << request_port_ << TLOG_ENDL;
61  request_socket_ = socket(AF_INET, SOCK_DGRAM, IPPROTO_UDP);
62  if (request_socket_ < 0)
63  {
64  TLOG_ERROR("RequestReceiver") << "Error creating socket for receiving data requests! err=" << strerror(errno) << TLOG_ENDL;
65  exit(1);
66  }
67 
68  struct sockaddr_in si_me_request;
69 
70  int yes = 1;
71  if (setsockopt(request_socket_, SOL_SOCKET, SO_REUSEADDR, &yes, sizeof(yes)) < 0)
72  {
73  TLOG_ERROR("RequestReceiver") << "Unable to enable port reuse on request socket, err=" << strerror(errno) << TLOG_ENDL;
74  exit(1);
75  }
76  memset(&si_me_request, 0, sizeof(si_me_request));
77  si_me_request.sin_family = AF_INET;
78  si_me_request.sin_port = htons(request_port_);
79  si_me_request.sin_addr.s_addr = htonl(INADDR_ANY);
80  if (bind(request_socket_, (struct sockaddr *)&si_me_request, sizeof(si_me_request)) == -1)
81  {
82  TLOG_ERROR("RequestReceiver") << "Cannot bind request socket to port " << request_port_ << ", err=" << strerror(errno) << TLOG_ENDL;
83  exit(1);
84  }
85 
86  if (request_addr_ != "localhost")
87  {
88  struct ip_mreq mreq;
89  int sts = ResolveHost(request_addr_.c_str(), mreq.imr_multiaddr);
90  if (sts == -1)
91  {
92  TLOG_ERROR("RequestReceiver") << "Unable to resolve multicast request address, err=" << strerror(errno) << TLOG_ENDL;
93  exit(1);
94  }
95  mreq.imr_interface.s_addr = htonl(INADDR_ANY);
96  if (setsockopt(request_socket_, IPPROTO_IP, IP_ADD_MEMBERSHIP, &mreq, sizeof(mreq)) < 0)
97  {
98  TLOG_ERROR("RequestReceiver") << "Unable to join multicast group, err=" << strerror(errno) << TLOG_ENDL;
99  exit(1);
100  }
101  }
102  TLOG_INFO("RequestReceiver") << "Done setting up request socket, rank=" << my_rank << TLOG_ENDL;
103 }
104 
105 artdaq::RequestReceiver::~RequestReceiver()
106 {
107  if (!request_received_)
108  {
109  TLOG_ERROR("RequestReceiver") << "Stop request received by RequestReceiver, but no requests have ever been received." << std::endl
110  << "Check that UDP port " << request_port_ << " is open in the firewall config." << TLOG_ENDL;
111  }
112  should_stop_ = true;
113  TLOG_DEBUG("RequestReceiver") << "Joining requestThread" << TLOG_ENDL;
114  if (requestThread_.joinable()) requestThread_.join();
115  if (request_socket_ != -1) close(request_socket_);
116 }
117 
119 {
120  if (requestThread_.joinable()) requestThread_.join();
121  TLOG_INFO("RequestReceiver") << "Starting Request Reception Thread" << TLOG_ENDL;
122  requestThread_ = boost::thread(&RequestReceiver::receiveRequestsLoop, this);
123  running_ = true;
124 }
125 
127 {
128  while (!should_stop_)
129  {
130  TLOG_ARB(16, "RequestReceiver") << "receiveRequestsLoop: Polling Request socket for new requests" << TLOG_ENDL;
131 
132  int ms_to_wait = 100;
133  struct pollfd ufds[1];
134  ufds[0].fd = request_socket_;
135  ufds[0].events = POLLIN | POLLPRI;
136  int rv = poll(ufds, 1, ms_to_wait);
137 
138  // Continue loop if no message received or message does not have correct event ID
139  if (rv <= 0 || (ufds[0].revents != POLLIN && ufds[0].revents != POLLPRI))
140  {
141  if (request_stop_requested_ && TimeUtils::GetElapsedTimeMilliseconds(request_stop_timeout_) > end_of_run_timeout_ms_)
142  {
143  break;
144  }
145  continue;
146  }
147 
148  TLOG_ARB(11, "RequestReceiver") << "Recieved packet on Request channel" << TLOG_ENDL;
150  recv(request_socket_, &hdr_buffer, sizeof(hdr_buffer), 0);
151  TLOG_ARB(11, "RequestReceiver") << "Request header word: 0x" << std::hex << hdr_buffer.header << TLOG_ENDL;
152  if (!hdr_buffer.isValid()) continue;
153 
154  request_received_ = true;
156  {
157  TLOG_INFO("RequestReceiver") << "Received Request Message with the EndOfRun marker. (Re)Starting 1-second timeout for receiving all outstanding requests..." << TLOG_ENDL;
158  request_stop_timeout_ = std::chrono::steady_clock::now();
159  request_stop_requested_ = true;
160  }
161 
162  std::vector<artdaq::detail::RequestPacket> pkt_buffer(hdr_buffer.packet_count);
163  recv(request_socket_, &pkt_buffer[0], sizeof(artdaq::detail::RequestPacket) * hdr_buffer.packet_count, 0);
164  bool anyNew = false;
165  for (auto& buffer : pkt_buffer)
166  {
167  if (!buffer.isValid()) continue;
168  if (requests_.count(buffer.sequence_id) && requests_[buffer.sequence_id] != buffer.timestamp)
169  {
170  TLOG_ERROR("RequestReceiver") << "Received conflicting request for SeqID "
171  << std::to_string(buffer.sequence_id) << "!"
172  << " Old ts=" << std::to_string(requests_[buffer.sequence_id])
173  << ", new ts=" << std::to_string(buffer.timestamp) << ". Keeping OLD!" << TLOG_ENDL;
174  }
175  else if (!requests_.count(buffer.sequence_id))
176  {
177  int delta = buffer.sequence_id - highest_seen_request_;
178  TLOG_ARB(11, "RequestReceiver") << "Recieved request for sequence ID " << std::to_string(buffer.sequence_id)
179  << " and timestamp " << std::to_string(buffer.timestamp) << " (delta: " << delta << ")" << TLOG_ENDL;
180  if (delta < 0)
181  {
182  TLOG_ARB(11, "RequestReceiver") << "Already serviced this request! Ignoring..." << TLOG_ENDL;
183  }
184  else
185  {
186  std::unique_lock<std::mutex> tlk(request_mutex_);
187  requests_[buffer.sequence_id] = buffer.timestamp;
188  anyNew = true;
189  }
190  }
191  }
192  if (anyNew) {
193  request_cv_.notify_all();
194  }
195  }
196  running_ = false;
197 }
198 
199 void artdaq::RequestReceiver::RemoveRequest(artdaq::Fragment::sequence_id_t reqID)
200 {
201  std::unique_lock<std::mutex> lk(request_mutex_);
202  requests_.erase(reqID);
203  if (reqID > highest_seen_request_) highest_seen_request_ = reqID;
204 }
int ResolveHost(char const *host_in, in_addr &addr)
Convert a string hostname to a in_addr suitable for socket communication.
Definition: TCPConnect.cc:27
End of Run mode (Used to end request processing on receiver)
RequestMessageMode mode
Communicates additional information to the Request receiver.
void startRequestReceiverThread()
Function that launches the data request receiver thread (receiveRequestsLoop())
bool isValid() const
Check the magic bytes of the packet.
Header of a RequestMessage. Contains magic bytes for validation and a count of expected RequestPacket...
uint32_t packet_count
The number of RequestPackets in this Request message.
void setupRequestListener()
Opens the socket used to listen for data requests.
void receiveRequestsLoop()
This function receives data request packets, adding new requests to the request list.
The RequestPacket contains information about a single data request.