artdaq  v3_12_02
TCPSocketTransfer.cc
1 // Sep 14, 2016. "TERMS AND CONDITIONS" governing this file are in the README
2 // or COPYING file. If you do not have such a file, one can be obtained by
3 // contacting Ron or Fermi Lab in Batavia IL, 60510, phone: 630-840-3000.
4 // $RCSfile: .emacs.gnu,v $
5 // rev="$Revision: 1.30 $$Date: 2016/03/01 14:27:27 $";
6 
7 // C Includes
8 #include <arpa/inet.h> // ntohl, ntohs
9 #include <poll.h> // struct pollfd
10 #include <sys/socket.h> // socket, socklen_t
11 #include <sys/types.h> // size_t
12 #include <sys/un.h> // sockaddr_un
13 #include <cstdlib> // atoi, strtoul
14 
15 // C++ Includes
16 #include <atomic>
17 #include <fstream>
18 #include <memory>
19 #include <mutex>
20 #include <stdexcept>
21 #include <string>
22 
23 // product Includes
24 #include "artdaq/DAQdata/Globals.hh"
25 #define TRACE_NAME (app_name + "_TCPSocketTransfer").c_str()
26 
27 // artdaq Includes
28 #include <iomanip>
29 #include "artdaq-core/Data/Fragment.hh"
30 #include "artdaq-core/Utilities/TimeUtils.hh"
33 #include "artdaq/TransferPlugins/TCPSocketTransfer.hh"
34 #include "artdaq/TransferPlugins/detail/SRSockets.hh"
35 #include "artdaq/TransferPlugins/detail/Timeout.hh"
36 
37 #define USE_SENDMSG 1
38 
39 std::atomic<int> artdaq::TCPSocketTransfer::listen_thread_refcount_(0);
40 std::unique_ptr<boost::thread> artdaq::TCPSocketTransfer::listen_thread_ = nullptr;
41 std::map<int, std::set<int>> artdaq::TCPSocketTransfer::connected_fds_ = std::map<int, std::set<int>>();
42 std::mutex artdaq::TCPSocketTransfer::listen_thread_mutex_;
43 std::mutex artdaq::TCPSocketTransfer::fd_mutex_;
44 
46  TCPSocketTransfer(fhicl::ParameterSet const& pset, TransferInterface::Role role)
47  : TransferInterface(pset, role)
48  , send_fd_(-1)
49  , rcvbuf_(pset.get<size_t>("tcp_receive_buffer_size", 0))
50  , sndbuf_(pset.get<size_t>("tcp_send_buffer_size", max_fragment_size_words_ * sizeof(artdaq::RawDataType) * buffer_count_))
51  , send_retry_timeout_us_(pset.get<size_t>("send_retry_timeout_us", 1000000))
52  , timeoutMessageArmed_(true)
53  , receive_disconnected_wait_s_(pset.get<double>("receive_socket_disconnected_wait_s", 10.0))
54  , receive_err_wait_us_(pset.get<size_t>("receive_socket_disconnected_wait_us", 10000))
55  , receive_socket_has_been_connected_(false)
56  , send_ack_diff_(0)
57 {
58  TLOG(TLVL_DEBUG + 32) << GetTraceName() << " Constructor: pset=" << pset.to_string() << ", role=" << (role == TransferInterface::Role::kReceive ? "kReceive" : "kSend");
59  connection_was_lost_ = false;
60 
62  {
63  // Wait for sender to connect...
64  TLOG(TLVL_DEBUG + 32) << GetTraceName() << "Listening for connections";
65  start_listen_thread_();
66  TLOG(TLVL_DEBUG + 32) << GetTraceName() << "Done Listening";
67  }
68  else
69  {
70  hostMap_ = MakeHostMap(pset);
71  TLOG(TLVL_DEBUG + 32) << GetTraceName() << "Connecting to destination";
72  connect_();
73  TLOG(TLVL_DEBUG + 32) << GetTraceName() << "Done Connecting";
74  }
75  TLOG(TLVL_DEBUG + 32) << GetTraceName() << "End of Constructor";
76 }
77 
78 artdaq::TCPSocketTransfer::~TCPSocketTransfer() noexcept
79 {
80  TLOG(TLVL_DEBUG + 32) << GetTraceName() << "Shutting down TCPSocketTransfer";
81 
82  if (role() == TransferInterface::Role::kSend)
83  {
84  // close all open connections (send stop_v0) first
85  MessHead mh = {0, MessHead::stop_v0, htons(TransferInterface::source_rank()), {0}};
86  if (send_fd_ != -1)
87  {
88  // should be blocking with modest timeo
89  timeval tv = {0, 100000};
90  socklen_t len = sizeof(tv);
91  setsockopt(send_fd_, SOL_SOCKET, SO_SNDTIMEO, &tv, len);
92  write(send_fd_, &mh, sizeof(mh));
93  }
94  close(send_fd_);
95  send_fd_ = -1;
96  }
97  else
98  {
100  try
101  {
102  if (ack_listen_thread_ && ack_listen_thread_->joinable())
103  {
104  ack_listen_thread_->join();
105  }
106  }
107  catch (...)
108  {
109  // IGNORED
110  }
111 
112  std::lock_guard<std::mutex> lk(listen_thread_mutex_);
113  listen_thread_refcount_--;
114  try
115  {
116  if (listen_thread_refcount_ <= 0 && listen_thread_ && listen_thread_->joinable())
117  {
118  listen_thread_->join();
119  }
120  }
121  catch (...)
122  {
123  // IGNORED
124  }
125  }
126 
127  TLOG(TLVL_DEBUG + 32) << GetTraceName() << "End of Destructor";
128 }
129 
130 int artdaq::TCPSocketTransfer::receiveFragmentHeader(detail::RawFragmentHeader& header, size_t timeout_usec)
131 {
132  TLOG(TLVL_DEBUG + 34) << GetTraceName() << "receiveFragmentHeader: BEGIN";
133  int ret_rank = RECV_TIMEOUT;
134 
135  // Don't bomb out until received at least one connection...
136  if (getConnectedFDCount_(source_rank()) == 0)
137  { // what if just listen_fd???
138  // if (receive_socket_has_been_connected_ && TimeUtils::GetElapsedTime(last_recv_time_) > receive_disconnected_wait_s_)
139  // {
140  // TLOG(TLVL_ERROR) << GetTraceName() << "receiveFragmentHeader: senders have been disconnected for "
141  // << TimeUtils::GetElapsedTime(last_recv_time_) << " s (receive_socket_disconnected_wait_s = " << receive_disconnected_wait_s_ << " s). RETURNING DATA_END!";
142  // return DATA_END;
143  // }
144  // if (++not_connected_count_ > receive_err_threshold_) { return DATA_END; }
145  TLOG(TLVL_DEBUG + 36) << GetTraceName() << "receiveFragmentHeader: Receive socket not connected, returning RECV_TIMEOUT";
146  usleep(receive_err_wait_us_);
147  return RECV_TIMEOUT;
148  }
149  receive_socket_has_been_connected_ = true;
150  last_recv_time_ = std::chrono::steady_clock::now();
151 
152  TLOG(TLVL_DEBUG + 34) << GetTraceName() << "receiveFragmentHeader timeout_usec=" << timeout_usec;
153  // void* buff=alloca(max_fragment_size_words_*8);
154  size_t byte_cnt = 0;
155  int sts;
156  int offset = 0;
157  SocketState state = SocketState::Metadata;
158  int target_bytes = sizeof(MessHead);
159  uint64_t start_time_us = TimeUtils::gettimeofday_us();
160 
161  // while (active_receive_fd_ != -1)
162  //{
163  // TLOG(TLVL_DEBUG + 33) << GetTraceName() << "Currently receiving from fd " << active_receive_fd_ << ", waiting!";
164  // usleep(1000);
165  // }
166 
167  uint8_t* buff;
168 
169  int timeout_ms;
170  if (timeout_usec == 0)
171  {
172  timeout_ms = 0;
173  }
174  else
175  {
176  timeout_ms = (timeout_usec + 999) / 1000; // want at least 1 ms
177  }
178 
179  bool done = false;
180  bool noDataWarningSent = false;
181  int loop_guard = 0;
182 
183  while (!done && getConnectedFDCount_(source_rank()) > 0)
184  {
185  if (getActiveFD_(source_rank()) == -1)
186  {
187  loop_guard = 0;
188  size_t fd_count = 0;
189  std::vector<pollfd> pollfds;
190  {
191  std::lock_guard<std::mutex> lk(fd_mutex_);
192  fd_count = connected_fds_[source_rank()].size();
193  pollfds.resize(fd_count);
194  auto iter = connected_fds_[source_rank()].begin();
195  for (size_t ii = 0; ii < fd_count; ++ii)
196  {
197  pollfds[ii].events = POLLIN | POLLPRI | POLLERR;
198  pollfds[ii].fd = *iter;
199  ++iter;
200  }
201  }
202  // TLOG(TLVL_DEBUG + 32) << GetTraceName() << "receiveFragment: Polling fd to see if there's data" ;
203  int num_fds_ready = poll(&pollfds[0], fd_count, timeout_ms);
204  if (num_fds_ready <= 0)
205  {
206  TLOG(TLVL_DEBUG + 34) << GetTraceName() << "receiveFragmentHeader: No data on receive socket, returning RECV_TIMEOUT";
207  return RECV_TIMEOUT;
208  }
209 
210  size_t index = 0;
211  if (getLastActiveFD_(source_rank()) != -1)
212  {
213  for (auto& pollfd : pollfds)
214  {
215  index++;
216  if (pollfd.fd == getLastActiveFD_(source_rank()))
217  {
218  break;
219  }
220  }
221  }
222 
223  int active_index = -1;
224  int16_t anomolous_events = 0;
225  for (size_t ii = index; ii < index + pollfds.size(); ++ii)
226  {
227  auto pollfd_index = (ii + index) % pollfds.size();
228  setActiveFD_(source_rank(), pollfds[pollfd_index].fd);
229  if ((pollfds[pollfd_index].revents & (POLLIN | POLLPRI)) != 0)
230  {
231  active_index = pollfd_index;
232  break;
233  }
234  if ((pollfds[pollfd_index].revents & (POLLHUP | POLLERR)) != 0)
235  {
236  disconnect_receive_socket_("Poll returned POLLHUP or POLLERR, indicating problems with the sender.");
237  continue;
238  }
239  else if ((pollfds[pollfd_index].revents & (POLLNVAL)) != 0)
240  {
241  TLOG(TLVL_DEBUG + 32) << GetTraceName() << "receiveFragmentHeader: FD is closed, most likely because the peer went away. Removing from fd list.";
242  disconnect_receive_socket_("FD is closed, most likely because the peer went away.");
243  continue;
244  }
245  else if (pollfds[pollfd_index].revents != 0)
246  {
247  anomolous_events |= pollfds[pollfd_index].revents;
248  }
249  }
250 
251  if (active_index == -1)
252  {
253  if (anomolous_events != 0)
254  {
255  TLOG(TLVL_DEBUG + 32) << GetTraceName() << "receiveFragmentHeader: Wrong event received from a pollfd. Mask: " << static_cast<int>(anomolous_events);
256  }
257  setActiveFD_(source_rank(), -1);
258  continue;
259  }
260 
261  if (!done && timeout_usec > 0)
262  {
263  // calc next timeout_ms (unless timed out)
264  size_t delta_us = TimeUtils::gettimeofday_us() - start_time_us;
265  if (delta_us > timeout_usec)
266  {
267  return RECV_TIMEOUT;
268  }
269  timeout_ms = ((timeout_usec - delta_us) + 999) / 1000; // want at least 1 ms
270  }
271  }
272  if (loop_guard > 10) { usleep(1000); }
273  if (++loop_guard > 10010)
274  {
275  TLOG(TLVL_WARNING) << GetTraceName() << "receiveFragmentHeader: loop guard triggered, returning RECV_TIMEOUT";
276  usleep(receive_err_wait_us_);
277  setActiveFD_(source_rank(), -1);
278  return RECV_TIMEOUT;
279  }
280 
281  if (state == SocketState::Metadata)
282  {
283  // TLOG(TLVL_DEBUG + 32) << GetTraceName() << "receiveFragmentHeader: Reading Message Header" ;
284  buff = &(mha[offset]); // NOLINT(cppcoreguidelines-pro-bounds-constant-array-index)
285  byte_cnt = sizeof(MessHead) - offset;
286  }
287  else
288  {
289  // TLOG(TLVL_DEBUG + 32) << GetTraceName() << "receiveFragmentHeader: Reading data" ;
290  buff = reinterpret_cast<uint8_t*>(&header) + offset; // NOLINT(cppcoreguidelines-pro-type-reinterpret-cast,cppcoreguidelines-pro-bounds-pointer-arithmetic)
291  byte_cnt = target_bytes - offset;
292  }
293  // if (byte_cnt > sizeof(MessHead))
294  // {
295  // TLOG(TLVL_ERROR) << "Invalid byte count for read (count=" << byte_cnt
296  // << ",offset=" << offset << ",mh.byte_count=" << mh.byte_count
297  // << "), skipping read and returning RECV_TIMEOUT";
298  // return RECV_TIMEOUT;
299  // }
300 
301  auto fd = getActiveFD_(source_rank());
302  if (byte_cnt > 0)
303  {
304  TLOG(TLVL_DEBUG + 35) << GetTraceName() << "receiveFragmentHeader: Reading " << byte_cnt << " bytes from socket " << fd;
305  sts = read(fd, buff, byte_cnt);
306  TLOG(TLVL_DEBUG + 35) << GetTraceName() << "receiveFragmentHeader: Done with read";
307  }
308  if (sts > 0)
309  {
310  loop_guard = 0;
311  last_recv_time_ = std::chrono::steady_clock::now();
312  }
313 
314  TLOG(TLVL_DEBUG + 36) << GetTraceName() << "receiveFragmentHeader state=" << static_cast<int>(state) << " read=" << sts;
315  if (sts < 0 && errno != EAGAIN)
316  {
317  TLOG(TLVL_WARNING) << GetTraceName() << "receiveFragmentHeader: Error on receive, closing socket " << fd
318  << " (errno=" << errno << ": " << strerror(errno) << ")";
319  disconnect_receive_socket_("Error on receive");
320  }
321  else if (sts == 0 || errno == EAGAIN)
322  {
323  if (!noDataWarningSent)
324  {
325  TLOG(TLVL_WARNING) << GetTraceName() << "receiveFragmentHeader: No data received, is the sender still sending?!?";
326  noDataWarningSent = true;
327  }
328  if (TimeUtils::GetElapsedTime(last_recv_time_) > receive_disconnected_wait_s_)
329  {
330  TLOG(TLVL_ERROR) << GetTraceName() << "receiveFragmentHeader: No data received within timeout, aborting!";
331  return RECV_TIMEOUT;
332  }
333  }
334  else
335  {
336  // see if we're done (with this state)
337  sts = offset += sts;
338  if (sts >= target_bytes)
339  {
340  TLOG(TLVL_DEBUG + 36) << GetTraceName() << "receiveFragmentHeader: Target read bytes reached. Changing state";
341  offset = 0;
342  if (state == SocketState::Metadata)
343  {
344  state = SocketState::Data;
345  mh.byte_count = ntohl(mh.byte_count);
346  mh.source_id = ntohs(mh.source_id);
347  target_bytes = mh.byte_count;
348  TLOG(TLVL_DEBUG + 36) << GetTraceName() << "receiveFragmentHeader: Expected header size = " << target_bytes << ", sizeof(RawFragmentHeader) = " << sizeof(artdaq::detail::RawFragmentHeader);
349  // assert(target_bytes == sizeof(artdaq::detail::RawFragmentHeader) || target_bytes == 0);
350 
351  if (mh.message_type == MessHead::stop_v0)
352  {
353  disconnect_receive_socket_("Stop Message received.");
354  }
355  else if (mh.message_type == MessHead::data_v0 || mh.message_type == MessHead::data_more_v0)
356  {
357  TLOG(TLVL_WARNING) << GetTraceName() << "receiveFragmentHeader: Message header indicates that Fragment data follows when I was expecting a Fragment header!";
358  disconnect_receive_socket_("Desync detected");
359  }
360 
361  if (target_bytes == 0)
362  {
363  // Probably a stop_v0, return timeout so we can try again.
364  return RECV_TIMEOUT;
365  }
366  }
367  else
368  {
369  ret_rank = source_rank();
370  TLOG(TLVL_DEBUG + 35) << GetTraceName() << "receiveFragmentHeader done sts=" << sts << " src=" << ret_rank;
371  TLOG(TLVL_DEBUG + 36) << GetTraceName() << "receiveFragmentHeader: Done receiving fragment header. Moving into output.";
372 
373  done = true; // no more polls
374  // break; // no more read of ready fds
375  }
376  }
377  }
378 
379  } // while(!done)...poll
380 
381  TLOG(TLVL_DEBUG + 34) << GetTraceName() << "receiveFragmentHeader: Returning " << ret_rank;
382  return ret_rank;
383 }
384 
385 void artdaq::TCPSocketTransfer::disconnect_receive_socket_(const std::string& msg)
386 {
387  std::lock_guard<std::mutex> lk(fd_mutex_);
388  auto fd = active_receive_fds_[source_rank()];
389  TLOG(TLVL_WARNING) << GetTraceName() << "disconnect_receive_socket_: " << msg << " Closing socket " << fd << " for rank " << source_rank();
390  close(fd);
391  if (connected_fds_.count(source_rank()) != 0u)
392  {
393  connected_fds_[source_rank()].erase(fd);
394  }
395  active_receive_fds_[source_rank()] = -1;
396  TLOG(TLVL_DEBUG + 32) << GetTraceName() << "disconnect_receive_socket_: There are now " << connected_fds_[source_rank()].size() << " active senders.";
397 }
398 
399 int artdaq::TCPSocketTransfer::receiveFragmentData(RawDataType* destination, size_t /*wordCount*/)
400 {
401  TLOG(TLVL_DEBUG + 39) << GetTraceName() << "receiveFragmentData: BEGIN";
402  int ret_rank = RECV_TIMEOUT;
403  if (getActiveFD_(source_rank()) == -1)
404  { // what if just listen_fd???
405  TLOG(TLVL_ERROR) << GetTraceName() << "receiveFragmentData: Receive socket not connected, returning RECV_TIMEOUT (Will result in \"Unexpected return code error\")";
406  return RECV_TIMEOUT;
407  }
408 
409  // void* buff=alloca(max_fragment_size_words_*8);
410  uint8_t* buff;
411  size_t byte_cnt = 0;
412  int sts;
413  int offset = 0;
414  SocketState state = SocketState::Metadata;
415  int target_bytes = sizeof(MessHead);
416 
417  pollfd pollfd_s;
418  pollfd_s.events = POLLIN | POLLPRI | POLLERR;
419  pollfd_s.fd = getActiveFD_(source_rank());
420 
421  int loop_guard = 0;
422  bool done = false;
423  bool noDataWarningSent = false;
424  last_recv_time_ = std::chrono::steady_clock::now();
425  while (!done)
426  {
427  TLOG(TLVL_DEBUG + 33) << GetTraceName() << "receiveFragmentData: Polling fd to see if there's data";
428  int num_fds_ready = poll(&pollfd_s, 1, 1000);
429  TLOG(TLVL_DEBUG + 33) << GetTraceName() << "receiveFragmentData: Polled fd to see if there's data"
430  << ", num_fds_ready = " << num_fds_ready;
431  if (num_fds_ready <= 0)
432  {
433  if (num_fds_ready == 0)
434  {
435  TLOG(TLVL_WARNING) << GetTraceName() << "receiveFragmentData: No data from " << source_rank() << " in " << TimeUtils::GetElapsedTimeMilliseconds(last_recv_time_) << " ms!"
436  << " State = " << (state == SocketState::Metadata ? "Metadata" : "Data") << ", recvd/total=" << offset << "/" << target_bytes << " (delta=" << target_bytes - offset << ")";
437 
438  if (TimeUtils::GetElapsedTime(last_recv_time_) > receive_disconnected_wait_s_)
439  {
440  TLOG(TLVL_WARNING) << GetTraceName() << "receiveFragmentData: No data received within timeout (" << TimeUtils::GetElapsedTime(last_recv_time_) << " / " << receive_disconnected_wait_s_ << " ), returning RECV_TIMEOUT";
441  disconnect_receive_socket_("No data on this socket within timeout");
442  return RECV_TIMEOUT;
443  }
444  continue;
445  }
446 
447  TLOG(TLVL_ERROR) << "Error in poll: errno=" << errno;
448  break;
449  }
450 
451  last_recv_time_ = std::chrono::steady_clock::now();
452 
453  if ((pollfd_s.revents & (POLLIN | POLLPRI)) != 0)
454  {
455  // Expected, don't have to check revents any further
456  }
457  else if ((pollfd_s.revents & (POLLNVAL)) != 0)
458  {
459  disconnect_receive_socket_("FD is closed, most likely because the peer went away.");
460  break;
461  }
462  else if ((pollfd_s.revents & (POLLHUP | POLLERR)) != 0)
463  {
464  disconnect_receive_socket_("Poll returned POLLHUP or POLLERR, indicating problems with the sender.");
465  break;
466  }
467  else
468  {
469  TLOG(TLVL_WARNING) << GetTraceName() << "receiveFragmentData: Wrong event received from pollfd: " << pollfd_s.revents;
470  disconnect_receive_socket_("Wrong event received from pollfd.");
471  break;
472  }
473 
474  if (state == SocketState::Metadata)
475  {
476  // TLOG(TLVL_DEBUG + 32) << GetTraceName() << "receiveFragmentData: Reading Message Header" ;
477  buff = &(mha[offset]); // NOLINT(cppcoreguidelines-pro-bounds-constant-array-index)
478  byte_cnt = sizeof(MessHead) - offset;
479  }
480  else
481  {
482  // TLOG(TLVL_DEBUG + 32) << GetTraceName() << "receiveFragmentData: Reading data" ;
483  buff = reinterpret_cast<uint8_t*>(destination) + offset; // NOLINT(cppcoreguidelines-pro-bounds-pointer-arithmetic,cppcoreguidelines-pro-type-reinterpret-cast)
484  byte_cnt = mh.byte_count - offset;
485  }
486 
487  TLOG(TLVL_DEBUG + 38) << GetTraceName() << "receiveFragmentData: Reading " << byte_cnt << " bytes from socket into " << static_cast<void*>(buff);
488  sts = read(getActiveFD_(source_rank()), buff, byte_cnt);
489  // TLOG(TLVL_DEBUG + 32) << GetTraceName() << "receiveFragmentData: Done with read" ;
490 
491  TLOG(TLVL_DEBUG + 38) << GetTraceName() << "recvFragment state=" << static_cast<int>(state) << " read=" << sts;
492 
493  if (sts == 0 || (sts < 0 && errno == EAGAIN))
494  {
495  sts = 0; // Treat EAGAIN as receiving no data
496  if (loop_guard > 10) { usleep(1000); }
497  if (++loop_guard > 10010)
498  {
499  TLOG(TLVL_WARNING) << GetTraceName() << "receiveFragmentData: loop guard triggered, returning RECV_TIMEOUT";
500  setActiveFD_(source_rank(), -1);
501  return RECV_TIMEOUT;
502  }
503  }
504  else if (sts > 0)
505  {
506  loop_guard = 0;
507  last_recv_time_ = std::chrono::steady_clock::now();
508  }
509 
510  if (sts < 0)
511  {
512  TLOG(TLVL_WARNING) << GetTraceName() << "receiveFragmentData: Error on receive, closing socket"
513  << " (errno=" << errno << ": " << strerror(errno) << ")";
514  disconnect_receive_socket_("Error on receive");
515  }
516  else if (sts == 0)
517  {
518  if (!noDataWarningSent)
519  {
520  TLOG(TLVL_WARNING) << GetTraceName() << "receiveFragmentData: No data received, is the sender still sending?!?";
521  noDataWarningSent = true;
522  }
523  if (TimeUtils::GetElapsedTime(last_recv_time_) > receive_disconnected_wait_s_)
524  {
525  TLOG(TLVL_ERROR) << GetTraceName() << "receiveFragmentData: No data received within timeout, aborting!";
526  return RECV_TIMEOUT;
527  }
528  }
529  else
530  {
531  // see if we're done (with this state)
532  sts = offset += sts;
533  if (sts >= target_bytes)
534  {
535  TLOG(TLVL_DEBUG + 42) << GetTraceName() << "receiveFragmentData: Target read bytes reached. Changing state";
536  offset = 0;
537  if (state == SocketState::Metadata)
538  {
539  state = SocketState::Data;
540  mh.byte_count = ntohl(mh.byte_count);
541  mh.source_id = ntohs(mh.source_id);
542  target_bytes = mh.byte_count;
543 
544  if (mh.message_type == MessHead::header_v0)
545  {
546  TLOG(TLVL_WARNING) << GetTraceName() << "receiveFragmentData: Message header indicates that a Fragment header follows when I was expecting Fragment data!";
547  disconnect_receive_socket_("Desync detected");
548  }
549  }
550  else
551  {
552  ret_rank = source_rank();
553  TLOG(TLVL_DEBUG + 41) << GetTraceName() << "receiveFragmentData done sts=" << sts << " src=" << ret_rank;
554  TLOG(TLVL_DEBUG + 39) << GetTraceName() << "receiveFragmentData: Done receiving fragment. Moving into output.";
555 
556 #if USE_ACKS
557  send_ack_(active_receive_fd_);
558 #endif
559 
560  done = true; // no more polls
561  // break; // no more read of ready fds
562  }
563  }
564  }
565 
566  // Check if we were asked to do a 0-size receive
567  if (target_bytes == 0 && state == SocketState::Data)
568  {
569  ret_rank = source_rank();
570  TLOG(TLVL_DEBUG + 41) << GetTraceName() << "receiveFragmentData done sts=" << sts << " src=" << ret_rank;
571  TLOG(TLVL_DEBUG + 39) << GetTraceName() << "receiveFragmentData: Done receiving fragment. Moving into output.";
572 
573 #if USE_ACKS
574  send_ack_(active_receive_fd_);
575 #endif
576 
577  done = true; // no more polls
578  }
579 
580  } // while(!done)...poll
581 
582  setLastActiveFD_(source_rank(), getActiveFD_(source_rank()));
583  setActiveFD_(source_rank(), -1);
584 
585  TLOG(TLVL_DEBUG + 39) << GetTraceName() << "receiveFragmentData: Returning rank " << ret_rank;
586  return ret_rank;
587 }
588 
590 {
591  switch (role())
592  {
594  return send_fd_ != -1;
596  auto count = getConnectedFDCount_(source_rank());
597  TLOG(TLVL_DEBUG + 32) << GetTraceName() << "isRunning: There are " << count << " fds connected.";
598  return count > 0;
599  }
600  return false;
601 }
602 
604 {
605  std::set<int> fds;
606  auto rank = TransferInterface::source_rank();
607  {
608  std::lock_guard<std::mutex> lk(fd_mutex_);
609  if (connected_fds_.count(rank) != 0)
610  {
611  fds = connected_fds_[rank];
612  connected_fds_.erase(rank);
613  }
614  }
615 
616  char discard_buf[0x1000];
617  for (auto& fd : fds)
618  {
619  TLOG(TLVL_INFO) << GetTraceName() << "flush_buffers: Checking for data in socket " << fd << " for rank " << rank;
620  size_t bytes_read = 0;
621  while (int sts = static_cast<int>(read(fd, discard_buf, sizeof(discard_buf)) > 0))
622  {
623  bytes_read += sts;
624  }
625  if (bytes_read > 0)
626  {
627  TLOG(TLVL_WARNING) << GetTraceName() << "flush_buffers: Flushed " << bytes_read << " bytes from socket " << fd << " for rank " << rank;
628  }
629  TLOG(TLVL_INFO) << GetTraceName() << "flush_buffers: Closing socket " << fd << " for rank " << rank;
630  close(fd);
631  }
632  active_receive_fds_[rank] = -1;
633  last_active_receive_fds_[rank] = -1;
634 }
635 
636 // Send the given Fragment. Return the rank of the destination to which
637 // the Fragment was sent OR -1 if to none.
638 artdaq::TransferInterface::CopyStatus artdaq::TCPSocketTransfer::sendFragment_(Fragment&& frag, size_t send_timeout_usec)
639 {
640  TLOG(TLVL_DEBUG + 42) << GetTraceName() << "sendFragment begin send of fragment with sequenceID=" << frag.sequenceID();
641  artdaq::Fragment grab_ownership_frag = std::move(frag);
642 
643  reconnect_();
644  if (send_fd_ == -1 && connection_was_lost_)
645  {
646  TLOG(TLVL_INFO) << GetTraceName() << "reconnection attempt failed, returning quickly.";
648  }
649 
650  // Send Fragment Header
651 
652 #if USE_ACKS
653  // Wait for fragments to be received
654  while (static_cast<size_t>(send_ack_diff_) > buffer_count_) usleep(10000);
655 #endif
656 
657  iovec iov = {static_cast<void*>(grab_ownership_frag.headerAddress()),
658  detail::RawFragmentHeader::num_words() * sizeof(RawDataType)};
659 
660  auto sts = sendData_(&iov, 1, send_retry_timeout_us_, true);
661  auto start_time = std::chrono::steady_clock::now();
662  // If it takes more than 10 seconds to write a Fragment header, give up
663  while (sts == CopyStatus::kTimeout && (send_timeout_usec == 0 || TimeUtils::GetElapsedTimeMicroseconds(start_time) < send_timeout_usec) && TimeUtils::GetElapsedTimeMicroseconds(start_time) < 10000000)
664  {
665  TLOG(TLVL_DEBUG + 43) << GetTraceName() << "sendFragment: Timeout sending fragment";
666  sts = sendData_(&iov, 1, send_retry_timeout_us_, true);
667  usleep(1000);
668  }
669  if (sts != CopyStatus::kSuccess)
670  {
671  return sts;
672  }
673 
674  // Send Fragment Data
675 
676  iov = {static_cast<void*>(grab_ownership_frag.headerAddress() + detail::RawFragmentHeader::num_words()), // NOLINT(cppcoreguidelines-pro-bounds-pointer-arithmetic)
677  grab_ownership_frag.sizeBytes() - detail::RawFragmentHeader::num_words() * sizeof(RawDataType)};
678  sts = sendData_(&iov, 1, send_retry_timeout_us_);
679  start_time = std::chrono::steady_clock::now();
680  while (sts == CopyStatus::kTimeout && (send_timeout_usec == 0 || TimeUtils::GetElapsedTimeMicroseconds(start_time) < send_timeout_usec) && TimeUtils::GetElapsedTimeMicroseconds(start_time) < 10000000)
681  {
682  TLOG(TLVL_DEBUG + 43) << GetTraceName() << "sendFragment: Timeout sending fragment";
683  sts = sendData_(&iov, 1, send_retry_timeout_us_);
684  usleep(1000);
685  }
686 
687 #if USE_ACKS
688  send_ack_diff_++;
689 #endif
690 
691  TLOG(TLVL_DEBUG + 42) << GetTraceName() << "sendFragment returning " << CopyStatusToString(sts);
692  return sts;
693 }
694 
695 artdaq::TransferInterface::CopyStatus artdaq::TCPSocketTransfer::sendData_(const void* buf, size_t bytes, size_t send_timeout_usec, bool isHeader)
696 {
697  TLOG(TLVL_DEBUG + 32) << GetTraceName() << "sendData_ Converting buf to iovec";
698  iovec iov = {const_cast<void*>(buf), bytes}; // NOLINT(cppcoreguidelines-pro-type-const-cast)
699  return sendData_(&iov, 1, send_timeout_usec, isHeader);
700 }
701 
702 artdaq::TransferInterface::CopyStatus artdaq::TCPSocketTransfer::sendData_(const struct iovec* iov, int iovcnt, size_t send_timeout_usec, bool isHeader)
703 {
704  // check all connected??? -- currently just check fd!=-1
705  if (send_fd_ == -1)
706  {
707  if (timeoutMessageArmed_)
708  {
709  TLOG(TLVL_DEBUG + 32) << GetTraceName() << "sendData_: Send fd is not open. Returning kTimeout";
710  timeoutMessageArmed_ = false;
711  }
712  return CopyStatus::kTimeout;
713  }
714  timeoutMessageArmed_ = true;
715  TLOG(TLVL_DEBUG + 44) << GetTraceName() << "send_timeout_usec is " << send_timeout_usec << ", currently unused.";
716 
717  // TLOG(TLVL_DEBUG + 32) << GetTraceName() << "sendData_: Determining write size" ;
718  uint32_t total_to_write_bytes = 0;
719  std::vector<iovec> iov_in(iovcnt + 1); // need contiguous (for the unlike case that only partial MH
720  std::vector<iovec> iovv(iovcnt + 2); // 1 more for mh and another one for any partial
721  int ii;
722  for (ii = 0; ii < iovcnt; ++ii)
723  {
724  iov_in[ii + 1] = iov[ii]; // NOLINT(cppcoreguidelines-pro-bounds-pointer-arithmetic)
725  total_to_write_bytes += iov[ii].iov_len; // NOLINT(cppcoreguidelines-pro-bounds-pointer-arithmetic)
726  }
727  // TLOG(TLVL_DEBUG + 32) << GetTraceName() << "sendData_: Constructing Message Header" ;
728  MessHead mh = {0, isHeader ? MessHead::header_v0 : MessHead::data_v0, htons(source_rank()), {htonl(total_to_write_bytes)}};
729  iov_in[0].iov_base = &mh;
730  iov_in[0].iov_len = sizeof(mh);
731  total_to_write_bytes += sizeof(mh);
732 
733  ssize_t sts = 0;
734  ssize_t total_written_bytes = 0;
735  ssize_t per_write_max_bytes = (32 * 1024);
736 
737  size_t in_iov_idx = 0; // only increment this when we know the associated data has been xferred
738  size_t out_iov_idx = 0;
739  ssize_t this_write_bytes = 0;
740 
741  do
742  {
743  // The first out_iov may be set at the end of the previous loop.
744  // iov looping from below (b/c of the latter, we need to check this_write_bytes)
745  for (;
746  (in_iov_idx + out_iov_idx) < iov_in.size() && this_write_bytes < per_write_max_bytes;
747  ++out_iov_idx)
748  {
749  this_write_bytes += iov_in[in_iov_idx + out_iov_idx].iov_len;
750  iovv[out_iov_idx] = iov_in[in_iov_idx + out_iov_idx];
751  }
752  if (this_write_bytes > per_write_max_bytes)
753  {
754  iovv[out_iov_idx - 1].iov_len -= this_write_bytes - per_write_max_bytes;
755  this_write_bytes = per_write_max_bytes;
756  }
757 
758  // need to do blocking algorithm -- including throttled block notifications
759  do_again:
760 #ifndef __OPTIMIZE__ // This can be an expensive TRACE call (even if disabled) due to multiplicity of calls
761  TLOG(TLVL_DEBUG + 44) << GetTraceName() << "sendFragment b4 writev " << std::setw(7) << total_written_bytes << " total_written_bytes send_fd_=" << send_fd_ << " in_idx=" << in_iov_idx
762  << " iovcnt=" << out_iov_idx << " 1st.len=" << iovv[0].iov_len;
763 #endif
764 // TLOG(TLVL_DEBUG + 32) << GetTraceName() << " calling writev" ;
765 #if USE_SENDMSG
766  msghdr msg;
767  memset(&msg, 0, sizeof(msghdr));
768  msg.msg_iov = &(iovv[0]);
769  msg.msg_iovlen = out_iov_idx; // at this point out_iov_idx is really the count (not an idx per se)
770  sts = sendmsg(send_fd_, &msg, MSG_NOSIGNAL | (blocking != 0u ? 0 : MSG_DONTWAIT));
771 #else
772  sts = writev(send_fd_, &(iovv[0]), out_iov_idx); // SIGPIPE may occur -- need signal handler or mask/ignore
773 #endif
774  // TLOG(TLVL_DEBUG + 32) << GetTraceName() << " done with writev" ;
775 
776  if (sts == -1)
777  {
778  if (errno == EAGAIN /* same as EWOULDBLOCK */)
779  {
780  TLOG(TLVL_DEBUG + 32) << GetTraceName() << "sendFragment EWOULDBLOCK";
781  blocking = 1u;
782 
783  fcntl(send_fd_, F_SETFL, 0); // clear O_NONBLOCK
784 
785  // NOTE: YES -- could drop here
786  goto do_again;
787  }
788  TLOG(TLVL_WARNING) << GetTraceName() << "sendFragment_: WRITE ERROR " << errno << ": " << strerror(errno);
789  connect_state = 0; // any write error closes
790  close(send_fd_);
791  send_fd_ = -1;
792  connection_was_lost_ = true;
794  }
795  if (sts != this_write_bytes)
796  {
797  // we'll loop around -- with
798  TLOG(TLVL_DEBUG + 32) << GetTraceName() << "sendFragment writev sts(" << sts << ")!=requested_send_bytes(" << this_write_bytes << ")";
799  total_written_bytes += sts; // add sts to total_written_bytes now as sts is adjusted next
800  // find which iovs are done
801  for (ii = 0; static_cast<size_t>(sts) >= iovv[ii].iov_len; ++ii)
802  {
803  sts -= iovv[ii].iov_len;
804  }
805  in_iov_idx += ii; // done with these in_iovs
806  iovv[ii].iov_len -= sts; // adjust partial iov
807  iovv[ii].iov_base = static_cast<uint8_t*>(iovv[ii].iov_base) + sts; // adjust partial iov // NOLINT(cppcoreguidelines-pro-bounds-pointer-arithmetic)
808 
809  // add more to get up to per_write_max_bytes
810  out_iov_idx = 0;
811  if (ii != 0)
812  {
813  iovv[out_iov_idx] = iovv[ii];
814  }
815  // starting over
816  this_write_bytes = iovv[out_iov_idx].iov_len;
817  // add any left over from appropriate in_iov_idx --
818  // i.e. match this out_iov with the in_iov that was used to
819  // initialize it; see how close the out base+len is to in base+len
820  // check !>per_write_max_bytes
821  auto additional = (reinterpret_cast<uintptr_t>(iov_in[in_iov_idx].iov_base) + iov_in[in_iov_idx].iov_len) - (reinterpret_cast<uintptr_t>(iovv[out_iov_idx].iov_base) + iovv[out_iov_idx].iov_len); // NOLINT(cppcoreguidelines-pro-type-reinterpret-cast)
822  if (additional != 0u)
823  {
824  iovv[out_iov_idx].iov_len += additional;
825  this_write_bytes += additional;
826  if (this_write_bytes > per_write_max_bytes)
827  {
828  iovv[out_iov_idx].iov_len -= this_write_bytes - per_write_max_bytes;
829  this_write_bytes = per_write_max_bytes;
830  }
831  }
832  ++out_iov_idx; // done with
833  TLOG(TLVL_DEBUG + 33) << GetTraceName() << "sendFragment writev sts!=: this_write_bytes=" << this_write_bytes
834  << " out_iov_idx=" << out_iov_idx
835  << " additional=" << additional
836  << " ii=" << ii;
837  }
838  else
839  {
840 #ifndef __OPTIMIZE__ // This can be an expensive TRACE call (even if disabled) due to multiplicity of calls
841  TLOG(TLVL_DEBUG + 33) << GetTraceName() << "sendFragment writev sts(" << sts << ")==requested_send_bytes(" << this_write_bytes << ")";
842 #endif
843  total_written_bytes += sts;
844  --out_iov_idx; // make it the index of the last iovv
845  iovv[out_iov_idx].iov_base = static_cast<uint8_t*>(iovv[out_iov_idx].iov_base) + iovv[out_iov_idx].iov_len; // NOLINT(cppcoreguidelines-pro-bounds-pointer-arithmetic)
846  iovv[out_iov_idx].iov_len = 0;
847  in_iov_idx += out_iov_idx; // at least this many complete (one more if "last iovv" is complete
848  this_write_bytes = 0;
849  // need to check last iovv against appropriate iov_in
850  auto additional = (reinterpret_cast<uintptr_t>(iov_in[in_iov_idx].iov_base) + iov_in[in_iov_idx].iov_len) - (reinterpret_cast<uintptr_t>(iovv[out_iov_idx].iov_base) + iovv[out_iov_idx].iov_len); // NOLINT(cppcoreguidelines-pro-type-reinterpret-cast)
851  if (additional != 0u)
852  {
853  iovv[out_iov_idx].iov_len += additional;
854  this_write_bytes += additional;
855  if (this_write_bytes > per_write_max_bytes)
856  {
857  iovv[out_iov_idx].iov_len -= this_write_bytes - per_write_max_bytes;
858  this_write_bytes = per_write_max_bytes;
859  }
860  if (out_iov_idx != 0)
861  {
862  iovv[0] = iovv[out_iov_idx];
863  }
864  out_iov_idx = 1;
865  }
866  else
867  {
868  ++in_iov_idx;
869  out_iov_idx = 0;
870  }
871  }
872  } while (total_written_bytes < total_to_write_bytes);
873  if (total_written_bytes > total_to_write_bytes)
874  {
875  TLOG(TLVL_ERROR) << GetTraceName() << "sendFragment program error: too many bytes transferred";
876  }
877 
878  if (blocking != 0u)
879  {
880  blocking = 0u;
881  fcntl(send_fd_, F_SETFL, O_NONBLOCK); // set O_NONBLOCK
882  }
883  sts = total_written_bytes - sizeof(MessHead);
884 
885  TLOG(TLVL_DEBUG + 44) << GetTraceName() << "sendFragment sts=" << sts;
887 }
888 
889 void artdaq::TCPSocketTransfer::connect_()
890 {
891  auto start_time = std::chrono::steady_clock::now();
892 
893  // Retry a few times if we can't connect
894  while (send_fd_ == -1 && TimeUtils::GetElapsedTimeMicroseconds(start_time) < send_retry_timeout_us_ * 10)
895  {
896  TLOG(TLVL_DEBUG + 32) << GetTraceName() << "Connecting sender socket";
897  int sndbuf_bytes = static_cast<int>(sndbuf_);
898  if (sndbuf_ > INT_MAX)
899  {
900  sndbuf_bytes = INT_MAX;
901  TLOG(TLVL_WARNING) << "Requested SNDBUF " << sndbuf_ << " too large, setting to INT_MAX: " << INT_MAX;
902  }
903  TLOG(TLVL_DEBUG + 32) << "Requested SNDBUF is " << sndbuf_bytes;
904 
905  send_fd_ = TCPConnect(hostMap_[destination_rank()].c_str(), portMan->GetTCPSocketTransferPort(destination_rank()), O_NONBLOCK, sndbuf_bytes);
906  if (send_fd_ == -1)
907  {
908  if (connection_was_lost_) { break; }
909 
910  usleep(send_retry_timeout_us_);
911  }
912  }
913  connect_state = 0;
914  blocking = 0;
915  TLOG(TLVL_DEBUG + 32) << GetTraceName() << "connect_ " + hostMap_[destination_rank()] + ":" << portMan->GetTCPSocketTransferPort(destination_rank()) << " send_fd_=" << send_fd_;
916  if (send_fd_ != -1)
917  {
918  // write connect msg
919  TLOG(TLVL_DEBUG + 32) << GetTraceName() << "connect_: Writing connect message";
920  MessHead mh = {0, MessHead::connect_v0, htons(source_rank()), {htonl(CONN_MAGIC)}};
921  ssize_t sts = write(send_fd_, &mh, sizeof(mh));
922  if (sts == -1)
923  {
924  TLOG(TLVL_ERROR) << GetTraceName() << "connect_: Error writing connect message!";
925  // a write error here is completely unexpected!
926  connect_state = 0;
927  close(send_fd_);
928  send_fd_ = -1;
929  }
930  else
931  {
932  TLOG(TLVL_INFO) << GetTraceName() << "connect_: Successfully connected";
933  // consider it all connected/established
934  connect_state = 1;
935  connection_was_lost_ = false;
936  }
937 
938 #if USE_ACKS
939  if (ack_listen_thread_ && ack_listen_thread_->joinable()) ack_listen_thread_->join();
940  TLOG(TLVL_INFO) << GetTraceName() << "Starting Ack Listener Thread";
941 
942  try
943  {
944  ack_listen_thread_ = std::make_unique<boost::thread>(&TCPSocketTransfer::receive_acks_, this);
945  char tname[16]; // Size 16 - see man page pthread_setname_np(3) and/or prctl(2)
946  snprintf(tname, sizeof(tname) - 1, "%d-AckListen", my_rank); // NOLINT
947  tname[sizeof(tname) - 1] = '\0'; // assure term. snprintf is not too evil :)
948  auto handle = ack_listen_thread_.native_handle();
949  pthread_setname_np(handle, tname);
950  }
951  catch (const boost::exception& e)
952  {
953  TLOG(TLVL_ERROR) << "Caught boost::exception starting TCP Socket Ack Listen thread: " << boost::diagnostic_information(e) << ", errno=" << errno;
954  std::cerr << "Caught boost::exception starting TCP Socket Ack Listen thread: " << boost::diagnostic_information(e) << ", errno=" << errno << std::endl;
955  exit(5);
956  }
957 #endif
958  }
959 }
960 
961 void artdaq::TCPSocketTransfer::reconnect_()
962 {
963  if (send_fd_ == -1 && role() == TransferInterface::Role::kSend)
964  {
965  TLOG(TLVL_DEBUG + 33) << GetTraceName() << "check/reconnect";
966  return connect_();
967  }
968 }
969 
970 void artdaq::TCPSocketTransfer::start_listen_thread_()
971 {
972  std::lock_guard<std::mutex> start_lock(listen_thread_mutex_);
973  if (listen_thread_refcount_ == 0)
974  {
975  if (listen_thread_ && listen_thread_->joinable())
976  {
977  listen_thread_->join();
978  }
979  listen_thread_refcount_ = 1;
980  TLOG(TLVL_INFO) << GetTraceName() << "Starting Listener Thread";
981 
982  try
983  {
984  listen_thread_ = std::make_unique<boost::thread>(&TCPSocketTransfer::listen_, portMan->GetTCPSocketTransferPort(destination_rank()), rcvbuf_);
985  char tname[16]; // Size 16 - see man page pthread_setname_np(3) and/or prctl(2)
986  snprintf(tname, sizeof(tname) - 1, "%d-Listen", my_rank); // NOLINT
987  tname[sizeof(tname) - 1] = '\0'; // assure term. snprintf is not too evil :)
988  auto handle = listen_thread_->native_handle();
989  pthread_setname_np(handle, tname);
990  }
991  catch (const boost::exception& e)
992  {
993  TLOG(TLVL_ERROR) << "Caught boost::exception starting TCP Socket Listen thread: " << boost::diagnostic_information(e) << ", errno=" << errno;
994  std::cerr << "Caught boost::exception starting TCP Socket Listen thread: " << boost::diagnostic_information(e) << ", errno=" << errno << std::endl;
995  exit(5);
996  }
997  }
998  else
999  {
1000  listen_thread_refcount_++;
1001  }
1002 }
1003 
1004 #if USE_ACKS
1005 void artdaq::TCPSocketTransfer::receive_acks_()
1006 {
1007  while (send_fd_ >= 0)
1008  {
1009  pollfd pollfd_s;
1010  pollfd_s.events = POLLIN | POLLPRI;
1011  pollfd_s.fd = send_fd_;
1012 
1013  TLOG(TLVL_DEBUG + 48) << GetTraceName() << "receive_acks_: Polling fd to see if there's data";
1014  int num_fds_ready = poll(&pollfd_s, 1, 1000);
1015  if (num_fds_ready <= 0)
1016  {
1017  if (num_fds_ready == 0)
1018  {
1019  TLOG(TLVL_DEBUG + 48) << GetTraceName() << "receive_acks_: No data on receive socket";
1020  continue;
1021  }
1022 
1023  TLOG(TLVL_ERROR) << "Error in poll: errno=" << errno;
1024  break;
1025  }
1026 
1027  if (pollfd_s.revents & (POLLIN | POLLPRI))
1028  {
1029  // Expected, don't have to check revents any further
1030  }
1031  else
1032  {
1033  TLOG(TLVL_DEBUG + 32) << GetTraceName() << "receive_acks_: Wrong event received from pollfd: " << pollfd_s.revents;
1034  break;
1035  }
1036 
1037  MessHead mh;
1038  auto sts = read(send_fd_, &mh, sizeof(mh));
1039 
1040  if (sts != sizeof(mh))
1041  {
1042  TLOG(TLVL_ERROR) << GetTraceName() << "receive_ack_: Wrong message header length received! (actual " << sts << " != " << sizeof(mh) << " expected)";
1043  continue;
1044  }
1045 
1046  // check for "magic" and valid source_id(aka rank)
1047  mh.source_id = ntohs(mh.source_id); // convert here as it is reference several times
1048  if (mh.source_id != my_rank)
1049  {
1050  TLOG(TLVL_ERROR) << GetTraceName() << "receive_ack_: Received ack for different sender! Rank=" << my_rank << ", hdr=" << mh.source_id;
1051  continue;
1052  }
1053  if (ntohl(mh.conn_magic) != ACK_MAGIC || !(mh.message_type == MessHead::ack_v0)) // Allow for future connect message versions
1054  {
1055  TLOG(TLVL_ERROR) << GetTraceName() << "receive_ack_: Wrong magic bytes in header!";
1056  continue;
1057  }
1058 
1059  TLOG(TLVL_DEBUG + 47) << GetTraceName() << "receive_acks_: Received ack message, diff is now " << (send_ack_diff_.load() - 1);
1060  send_ack_diff_--;
1061  }
1062 }
1063 
1064 void artdaq::TCPSocketTransfer::send_ack_(int fd)
1065 {
1066  MessHead mh = {0, MessHead::ack_v0, htons(source_rank()), {htonl(ACK_MAGIC)}};
1067  write(fd, &mh, sizeof(mh));
1068 }
1069 #endif
1070 
1071 void artdaq::TCPSocketTransfer::listen_(int port, size_t rcvbuf)
1072 {
1073  int listen_fd = -1;
1074  while (listen_thread_refcount_ > 0)
1075  {
1076  TLOG(TLVL_DEBUG + 33) << "listen_: Listening/accepting new connections on port " << port;
1077  if (listen_fd == -1)
1078  {
1079  TLOG(TLVL_DEBUG + 32) << "listen_: Opening listener";
1080  listen_fd = TCP_listen_fd(port, rcvbuf);
1081  }
1082  if (listen_fd == -1)
1083  {
1084  TLOG(TLVL_DEBUG + 32) << "listen_: Error creating listen_fd!";
1085  break;
1086  }
1087 
1088  int res;
1089  timeval tv = {2, 0}; // maybe increase of some global "debugging" flag set???
1090  fd_set rfds;
1091  FD_ZERO(&rfds);
1092  FD_SET(listen_fd, &rfds); // NOLINT
1093 
1094  res = select(listen_fd + 1, &rfds, static_cast<fd_set*>(nullptr), static_cast<fd_set*>(nullptr), &tv);
1095  if (res > 0)
1096  {
1097  int sts;
1098  sockaddr_un un;
1099  socklen_t arglen = sizeof(un);
1100  int fd;
1101  TLOG(TLVL_DEBUG + 32) << "listen_: Calling accept";
1102  fd = accept(listen_fd, reinterpret_cast<sockaddr*>(&un), &arglen); // NOLINT(cppcoreguidelines-pro-type-reinterpret-cast)
1103  TLOG(TLVL_DEBUG + 32) << "listen_: Done with accept";
1104 
1105  TLOG(TLVL_DEBUG + 32) << "listen_: Reading connect message";
1106  socklen_t lenlen = sizeof(tv);
1107  /*sts=*/
1108  setsockopt(fd, SOL_SOCKET, SO_RCVTIMEO, &tv, lenlen); // see man 7 socket.
1109  MessHead mh;
1110  uint64_t mark_us = TimeUtils::gettimeofday_us();
1111  sts = read(fd, &mh, sizeof(mh));
1112  uint64_t delta_us = TimeUtils::gettimeofday_us() - mark_us;
1113  TLOG(TLVL_DEBUG + 32) << "listen_: Read of connect message took " << delta_us << " microseconds.";
1114  if (sts != sizeof(mh))
1115  {
1116  TLOG(TLVL_DEBUG + 32) << "listen_: Wrong message header length received!";
1117  close(fd);
1118  continue;
1119  }
1120 
1121  // check for "magic" and valid source_id(aka rank)
1122  mh.source_id = ntohs(mh.source_id); // convert here as it is reference several times
1123  if (ntohl(mh.conn_magic) != CONN_MAGIC || !(mh.message_type == MessHead::connect_v0)) // Allow for future connect message versions
1124  {
1125  TLOG(TLVL_DEBUG + 32) << "listen_: Wrong magic bytes in header!";
1126  close(fd);
1127  continue;
1128  }
1129 
1130  // now add (new) connection
1131  std::lock_guard<std::mutex> lk(fd_mutex_);
1132  connected_fds_[mh.source_id].insert(fd);
1133 
1134  TLOG(TLVL_INFO) << "listen_: New fd is " << fd << " for source rank " << mh.source_id;
1135  }
1136  else
1137  {
1138  TLOG(TLVL_DEBUG + 46) << "listen_: No connections in timeout interval!";
1139  }
1140  }
1141 
1142  TLOG(TLVL_INFO) << "listen_: Shutting down connection listener";
1143  if (listen_fd != -1)
1144  {
1145  close(listen_fd);
1146  }
1147  std::lock_guard<std::mutex> lk(fd_mutex_);
1148  auto it = connected_fds_.begin();
1149  while (it != connected_fds_.end())
1150  {
1151  auto& fd_set = it->second;
1152  auto rank_it = fd_set.begin();
1153  while (rank_it != fd_set.end())
1154  {
1155  close(*rank_it);
1156  rank_it = fd_set.erase(rank_it);
1157  }
1158  it = connected_fds_.erase(it);
1159  }
1160 
1161 } // do_connect_
1162 
1163 size_t artdaq::TCPSocketTransfer::getConnectedFDCount_(int source_rank)
1164 {
1165  std::lock_guard<std::mutex> lk(fd_mutex_);
1166 #ifndef __OPTIMIZE__
1167  TLOG(TLVL_DEBUG + 45) << GetTraceName() << "getConnectedFDCount_: count is " << (connected_fds_.count(source_rank) != 0u ? connected_fds_[source_rank].size() : 0);
1168 #endif
1169  return connected_fds_.count(source_rank) != 0u ? connected_fds_[source_rank].size() : 0;
1170 }
1171 
1172 int artdaq::TCPSocketTransfer::getActiveFD_(int source_rank)
1173 {
1174  std::lock_guard<std::mutex> lk(fd_mutex_);
1175 #ifndef __OPTIMIZE__
1176  TLOG(TLVL_DEBUG + 45) << GetTraceName() << "getActiveFD_: fd is " << (active_receive_fds_.count(source_rank) != 0u ? active_receive_fds_[source_rank] : -1);
1177 #endif
1178  return active_receive_fds_.count(source_rank) != 0u ? active_receive_fds_[source_rank] : -1;
1179 }
1180 void artdaq::TCPSocketTransfer::setActiveFD_(int source_rank, int fd)
1181 {
1182  std::lock_guard<std::mutex> lk(fd_mutex_);
1183 #ifndef __OPTIMIZE__
1184  TLOG(TLVL_DEBUG + 45) << GetTraceName() << "setActiveFD_: setting active fd for rank " << source_rank << " to " << fd;
1185 #endif
1186  active_receive_fds_[source_rank] = fd;
1187 }
1188 int artdaq::TCPSocketTransfer::getLastActiveFD_(int source_rank)
1189 {
1190  std::lock_guard<std::mutex> lk(fd_mutex_);
1191 #ifndef __OPTIMIZE__
1192  TLOG(TLVL_DEBUG + 45) << GetTraceName() << "getLastActiveFD_: fd is " << (last_active_receive_fds_.count(source_rank) != 0u ? last_active_receive_fds_[source_rank] : -1);
1193 #endif
1194  return last_active_receive_fds_.count(source_rank) != 0u ? last_active_receive_fds_[source_rank] : -1;
1195 }
1196 void artdaq::TCPSocketTransfer::setLastActiveFD_(int source_rank, int fd)
1197 {
1198  std::lock_guard<std::mutex> lk(fd_mutex_);
1199 #ifndef __OPTIMIZE__
1200  TLOG(TLVL_DEBUG + 45) << GetTraceName() << "setLastActiveFD_: setting last active fd for rank " << source_rank << " to " << fd;
1201 #endif
1202  last_active_receive_fds_[source_rank] = fd;
1203 }
bool isRunning() override
Determine whether the TransferInterface plugin is able to send/receive data.
virtual int source_rank() const
Get the source rank for this TransferInterface instance.
uint32_t conn_magic
unsigned first is better for MessHead initializer: {0,0,my_node_idx_,CONN_MAGIC}
Definition: SRSockets.hh:40
int receiveFragmentHeader(detail::RawFragmentHeader &header, size_t timeout_usec) override
Receive a Fragment Header from the transport mechanism.
int TCPConnect(char const *host_in, int dflt_port, int64_t flags=0, int sndbufsiz=0)
Connect to a host on a given port.
Definition: TCPConnect.cc:377
This TransferInterface is a Receiver.
int receiveFragmentData(RawDataType *destination, size_t wordCount) override
Receive the body of a Fragment to the given destination pointer.
int TCP_listen_fd(int port, int rcvbuf)
Create a TCP listening socket on the given port and INADDR_ANY, with the given receive buffer...
TCPSocketTransfer(fhicl::ParameterSet const &ps, Role role)
TCPSocketTransfer Constructor.
This TransferInterface is a Sender.
void flush_buffers() override
Flush any in-flight data. This should be used by the receiver after the receive loop has ended...
int32_t byte_count
use CONN_MAGIC for connect_v0, data that follow for data_v0 (and 0 lenght data)
Definition: SRSockets.hh:41
Some error occurred, but no exception was thrown.
Role
Used to determine if a TransferInterface is a Sender or Receiver.
int64_t source_id
Rank of the source.
Definition: SRSockets.hh:36
MessType message_type
Message Type.
Definition: SRSockets.hh:35
The send operation completed successfully.
This interface defines the functions used to transfer data between artdaq applications.
This header is sent by the TCPSocket_transfer to allow for more efficient writev calls.
Definition: SRSockets.hh:15
hostMap_t MakeHostMap(fhicl::ParameterSet const &pset, hostMap_t map=hostMap_t())
Make a hostMap_t from a HostMap::Config ParameterSet
Definition: HostMap.hh:68
CopyStatus
Returned from the send functions, this enumeration describes the possible return codes. If an exception occurs, it will be thrown and should be handled normally.