artdaq  v3_06_02
TCPSocket_transfer.cc
1 // Sep 14, 2016. "TERMS AND CONDITIONS" governing this file are in the README
2 // or COPYING file. If you do not have such a file, one can be obtained by
3 // contacting Ron or Fermi Lab in Batavia IL, 60510, phone: 630-840-3000.
4 // $RCSfile: .emacs.gnu,v $
5 // rev="$Revision: 1.30 $$Date: 2016/03/01 14:27:27 $";
6 
7 // C Includes
8 #include <arpa/inet.h> // ntohl, ntohs
9 #include <poll.h> // struct pollfd
10 #include <stdlib.h> // atoi, strtoul
11 #include <sys/socket.h> // socket, socklen_t
12 #include <sys/types.h> // size_t
13 #include <sys/un.h> // sockaddr_un
14 
15 // C++ Includes
16 #include <fstream>
17 #include <stdexcept>
18 #include <string>
19 
20 // product Includes
21 #include "artdaq/DAQdata/Globals.hh"
22 #define TRACE_NAME (app_name + "_TCPSocketTransfer").c_str()
23 
24 // artdaq Includes
25 #include <iomanip>
26 #include "artdaq-core/Data/Fragment.hh"
27 #include "artdaq-core/Utilities/TimeUtils.hh"
30 #include "artdaq/TransferPlugins/TCPSocketTransfer.hh"
31 #include "artdaq/TransferPlugins/detail/SRSockets.hh"
32 #include "artdaq/TransferPlugins/detail/Timeout.hh"
33 
34 std::atomic<int> artdaq::TCPSocketTransfer::listen_thread_refcount_(0);
35 std::unique_ptr<boost::thread> artdaq::TCPSocketTransfer::listen_thread_ = nullptr;
36 std::map<int, std::set<int>> artdaq::TCPSocketTransfer::connected_fds_ = std::map<int, std::set<int>>();
37 std::mutex artdaq::TCPSocketTransfer::listen_thread_mutex_;
38 std::mutex artdaq::TCPSocketTransfer::connected_fd_mutex_;
39 
41  TCPSocketTransfer(fhicl::ParameterSet const& pset, TransferInterface::Role role)
42  : TransferInterface(pset, role)
43  , send_fd_(-1)
44  , active_receive_fd_(-1)
45  , last_active_receive_fd_(-1)
46  , rcvbuf_(pset.get<size_t>("tcp_receive_buffer_size", 0))
47  , sndbuf_(pset.get<size_t>("tcp_send_buffer_size", max_fragment_size_words_ * sizeof(artdaq::RawDataType) * buffer_count_))
48  , send_retry_timeout_us_(pset.get<size_t>("send_retry_timeout_us", 1000000))
49  , timeoutMessageArmed_(true)
50  , last_recv_time_()
51  , receive_disconnected_wait_s_(pset.get<double>("receive_socket_disconnected_wait_s", 10.0))
52  , receive_err_wait_us_(pset.get<size_t>("receive_socket_disconnected_wait_us", 10000))
53  , receive_socket_has_been_connected_(false)
54  , send_ack_diff_(0)
55 {
56  TLOG(TLVL_DEBUG) << GetTraceName() << " Constructor: pset=" << pset.to_string() << ", role=" << (role == TransferInterface::Role::kReceive ? "kReceive" : "kSend");
57  connection_was_lost_ = false;
58 
60  {
61  // Wait for sender to connect...
62  TLOG(TLVL_DEBUG) << GetTraceName() << ": Listening for connections";
63  start_listen_thread_();
64  TLOG(TLVL_DEBUG) << GetTraceName() << ": Done Listening";
65  }
66  else
67  {
68  hostMap_ = MakeHostMap(pset);
69  TLOG(TLVL_DEBUG) << GetTraceName() << ": Connecting to destination";
70  connect_();
71  TLOG(TLVL_DEBUG) << GetTraceName() << ": Done Connecting";
72  }
73  TLOG(TLVL_DEBUG) << GetTraceName() << ": End of Constructor";
74 }
75 
76 artdaq::TCPSocketTransfer::~TCPSocketTransfer() noexcept
77 {
78  TLOG(TLVL_DEBUG) << GetTraceName() << ": Shutting down TCPSocketTransfer";
79 
80  if (role() == TransferInterface::Role::kSend)
81  {
82  // close all open connections (send stop_v0) first
83  MessHead mh = {0, MessHead::stop_v0, htons(TransferInterface::source_rank()), {0}};
84  if (send_fd_ != -1)
85  {
86  // should be blocking with modest timeo
87  timeval tv = {0, 100000};
88  socklen_t len = sizeof(tv);
89  setsockopt(send_fd_, SOL_SOCKET, SO_SNDTIMEO, &tv, len);
90  write(send_fd_, &mh, sizeof(mh));
91  }
92  close(send_fd_);
93  send_fd_ = -1;
94  }
95  else
96  {
97  {
98  std::unique_lock<std::mutex> fd_lock(connected_fd_mutex_);
99  if (connected_fds_.count(source_rank()))
100  {
101  auto it = connected_fds_[source_rank()].begin();
102  while (it != connected_fds_[source_rank()].end())
103  {
104  close(*it);
105  it = connected_fds_[source_rank()].erase(it);
106  }
107  connected_fds_.erase(source_rank());
108  }
109  if (ack_listen_thread_ && ack_listen_thread_->joinable()) ack_listen_thread_->join();
110  }
111 
112  std::unique_lock<std::mutex> lk(listen_thread_mutex_);
113  listen_thread_refcount_--;
114  if (listen_thread_refcount_ <= 0 && listen_thread_ && listen_thread_->joinable())
115  {
116  listen_thread_->join();
117  }
118  }
119 
120  TLOG(TLVL_DEBUG) << GetTraceName() << ": End of Destructor";
121 }
122 
123 int artdaq::TCPSocketTransfer::receiveFragmentHeader(detail::RawFragmentHeader& header, size_t timeout_usec)
124 {
125  TLOG(5) << GetTraceName() << ": receiveFragmentHeader: BEGIN";
126  int ret_rank = RECV_TIMEOUT;
127 
128  // Don't bomb out until received at least one connection...
129  if (getConnectedFDCount(source_rank()) == 0)
130  { // what if just listen_fd???
131  // if (receive_socket_has_been_connected_ && TimeUtils::GetElapsedTime(last_recv_time_) > receive_disconnected_wait_s_)
132  // {
133  // TLOG(TLVL_ERROR) << GetTraceName() << ": receiveFragmentHeader: senders have been disconnected for "
134  // << TimeUtils::GetElapsedTime(last_recv_time_) << " s (receive_socket_disconnected_wait_s = " << receive_disconnected_wait_s_ << " s). RETURNING DATA_END!";
135  // return DATA_END;
136  // }
137  //if (++not_connected_count_ > receive_err_threshold_) { return DATA_END; }
138  TLOG(7) << GetTraceName() << ": receiveFragmentHeader: Receive socket not connected, returning RECV_TIMEOUT";
139  usleep(receive_err_wait_us_);
140  return RECV_TIMEOUT;
141  }
142  receive_socket_has_been_connected_ = true;
143  last_recv_time_ = std::chrono::steady_clock::now();
144 
145  TLOG(5) << GetTraceName() << ": receiveFragmentHeader timeout_usec=" << timeout_usec;
146  //void* buff=alloca(max_fragment_size_words_*8);
147  size_t byte_cnt = 0;
148  int sts;
149  int offset = 0;
150  SocketState state = SocketState::Metadata;
151  int target_bytes = sizeof(MessHead);
152  uint64_t start_time_us = TimeUtils::gettimeofday_us();
153 
154  //while (active_receive_fd_ != -1)
155  //{
156  // TLOG(TLVL_TRACE) << GetTraceName() << ": Currently receiving from fd " << active_receive_fd_ << ", waiting!";
157  // usleep(1000);
158  //}
159 
160  uint8_t* buff;
161 
162  int timeout_ms;
163  if (timeout_usec == 0)
164  timeout_ms = 0;
165  else
166  timeout_ms = (timeout_usec + 999) / 1000; // want at least 1 ms
167 
168  bool done = false;
169  bool noDataWarningSent = false;
170  int loop_guard = 0;
171 
172  while (!done && getConnectedFDCount(source_rank()) > 0)
173  {
174  if (active_receive_fd_ == -1)
175  {
176  loop_guard = 0;
177  size_t fd_count = 0;
178  std::vector<pollfd> pollfds;
179  {
180  std::unique_lock<std::mutex> lk(connected_fd_mutex_);
181  fd_count = connected_fds_[source_rank()].size();
182  pollfds.resize(fd_count);
183  auto iter = connected_fds_[source_rank()].begin();
184  for (size_t ii = 0; ii < fd_count; ++ii)
185  {
186  pollfds[ii].events = POLLIN | POLLPRI | POLLERR;
187  pollfds[ii].fd = *iter;
188  ++iter;
189  }
190  }
191  //TLOG(TLVL_DEBUG) << GetTraceName() << ": receiveFragment: Polling fd to see if there's data" ;
192  int num_fds_ready = poll(&pollfds[0], fd_count, timeout_ms);
193  if (num_fds_ready <= 0)
194  {
195  TLOG(5) << GetTraceName() << ": receiveFragmentHeader: No data on receive socket, returning RECV_TIMEOUT";
196  return RECV_TIMEOUT;
197  }
198 
199  size_t index = 0;
200  if (last_active_receive_fd_ != -1)
201  {
202  for (auto& pollfd : pollfds)
203  {
204  index++;
205  if (pollfd.fd == last_active_receive_fd_)
206  {
207  break;
208  }
209  }
210  }
211 
212  int active_index = -1;
213  short anomolous_events = 0;
214  for (size_t ii = index; ii < index + pollfds.size(); ++ii)
215  {
216  auto pollfd_index = (ii + index) % pollfds.size();
217  if (pollfds[pollfd_index].revents & (POLLIN | POLLPRI))
218  {
219  active_index = pollfd_index;
220  active_receive_fd_ = pollfds[active_index].fd;
221  active_revents_ = pollfds[active_index].revents;
222  ;
223  break;
224  }
225  else if (pollfds[pollfd_index].revents & (POLLHUP | POLLERR))
226  {
227  disconnect_receive_socket_(pollfds[pollfd_index].fd, "Poll returned POLLHUP or POLLERR, indicating problems with the sender.");
228  continue;
229  }
230  else if (pollfds[pollfd_index].revents & (POLLNVAL))
231  {
232  TLOG(TLVL_DEBUG) << GetTraceName() << ": receiveFragmentHeader: FD is closed, most likely because the peer went away. Removing from fd list.";
233  disconnect_receive_socket_(pollfds[pollfd_index].fd, "FD is closed, most likely because the peer went away.");
234  continue;
235  }
236  else if (pollfds[pollfd_index].revents)
237  {
238  anomolous_events |= pollfds[pollfd_index].revents;
239  }
240  }
241 
242  if (active_index == -1)
243  {
244  if (anomolous_events)
245  TLOG(TLVL_DEBUG) << GetTraceName() << ": receiveFragmentHeader: Wrong event received from a pollfd. Mask: " << static_cast<int>(anomolous_events);
246  active_receive_fd_ = -1;
247  continue;
248  }
249 
250  if (!done && timeout_usec > 0)
251  {
252  // calc next timeout_ms (unless timed out)
253  size_t delta_us = TimeUtils::gettimeofday_us() - start_time_us;
254  if (delta_us > timeout_usec)
255  {
256  return RECV_TIMEOUT;
257  }
258  timeout_ms = ((timeout_usec - delta_us) + 999) / 1000; // want at least 1 ms
259  }
260  }
261  if (loop_guard > 10) { usleep(1000); }
262  if (++loop_guard > 10010)
263  {
264  TLOG(TLVL_WARNING) << GetTraceName() << ": receiveFragmentHeader: loop guard triggered, returning RECV_TIMEOUT";
265  usleep(receive_err_wait_us_);
266  active_receive_fd_ = -1;
267  return RECV_TIMEOUT;
268  }
269 
270  if (state == SocketState::Metadata)
271  {
272  //TLOG(TLVL_DEBUG) << GetTraceName() << ": receiveFragmentHeader: Reading Message Header" ;
273  buff = &(mha[offset]);
274  byte_cnt = sizeof(MessHead) - offset;
275  }
276  else
277  {
278  //TLOG(TLVL_DEBUG) << GetTraceName() << ": receiveFragmentHeader: Reading data" ;
279  buff = reinterpret_cast<uint8_t*>(&header) + offset;
280  byte_cnt = target_bytes - offset;
281  }
282  //if (byte_cnt > sizeof(MessHead))
283  // {
284  // TLOG(TLVL_ERROR) << "Invalid byte count for read (count=" << byte_cnt
285  // << ",offset=" << offset << ",mh.byte_count=" << mh.byte_count
286  // << "), skipping read and returning RECV_TIMEOUT";
287  // return RECV_TIMEOUT;
288  //}
289 
290  if (byte_cnt > 0)
291  {
292  TLOG(6) << GetTraceName() << ": receiveFragmentHeader: Reading " << byte_cnt << " bytes from socket";
293  sts = read(active_receive_fd_, buff, byte_cnt);
294  TLOG(6) << GetTraceName() << ": receiveFragmentHeader: Done with read";
295  }
296  if (sts > 0)
297  {
298  loop_guard = 0;
299  last_recv_time_ = std::chrono::steady_clock::now();
300  }
301 
302  TLOG(7) << GetTraceName() << ": receiveFragmentHeader state=" << static_cast<int>(state) << " read=" << sts;
303  if (sts < 0)
304  {
305  TLOG(TLVL_WARNING) << GetTraceName() << ": receiveFragmentHeader: Error on receive, closing socket "
306  << " (errno=" << errno << ": " << strerror(errno) << ")";
307  active_receive_fd_ = disconnect_receive_socket_(active_receive_fd_);
308  }
309  else if (sts == 0)
310  {
311  if (!noDataWarningSent)
312  {
313  TLOG(TLVL_WARNING) << GetTraceName() << ": receiveFragmentHeader: No data received, is the sender still sending?!?";
314  noDataWarningSent = true;
315  }
316  if (TimeUtils::GetElapsedTime(last_recv_time_) > receive_disconnected_wait_s_)
317  {
318  TLOG(TLVL_ERROR) << GetTraceName() << ": receiveFragmentHeader: No data received within timeout, aborting!";
319  return RECV_TIMEOUT;
320  }
321  }
322  else
323  {
324  // see if we're done (with this state)
325  sts = offset += sts;
326  if (sts >= target_bytes)
327  {
328  TLOG(7) << GetTraceName() << ": receiveFragmentHeader: Target read bytes reached. Changing state";
329  offset = 0;
330  if (state == SocketState::Metadata)
331  {
332  state = SocketState::Data;
333  mh.byte_count = ntohl(mh.byte_count);
334  mh.source_id = ntohs(mh.source_id);
335  target_bytes = mh.byte_count;
336  TLOG(7) << GetTraceName() << ": receiveFragmentHeader: Expected header size = " << target_bytes << ", sizeof(RawFragmentHeader) = " << sizeof(artdaq::detail::RawFragmentHeader);
337  //assert(target_bytes == sizeof(artdaq::detail::RawFragmentHeader) || target_bytes == 0);
338 
339  if (mh.message_type == MessHead::stop_v0)
340  {
341  active_receive_fd_ = disconnect_receive_socket_(active_receive_fd_, "Stop Message received.");
342  }
343  else if (mh.message_type == MessHead::data_v0 || mh.message_type == MessHead::data_more_v0)
344  {
345  TLOG(TLVL_WARNING) << GetTraceName() << ": receiveFragmentHeader: Message header indicates that Fragment data follows when I was expecting a Fragment header!";
346  active_receive_fd_ = disconnect_receive_socket_(active_receive_fd_, "Desync detected");
347  }
348 
349  if (target_bytes == 0)
350  {
351  //Probably a stop_v0, return timeout so we can try again.
352  return RECV_TIMEOUT;
353  }
354  }
355  else
356  {
357  ret_rank = source_rank();
358  TLOG(8) << GetTraceName() << ": receiveFragmentHeader done sts=" << sts << " src=" << ret_rank;
359  TLOG(7) << GetTraceName() << ": receiveFragmentHeader: Done receiving fragment header. Moving into output.";
360 
361  done = true; // no more polls
362  //break; // no more read of ready fds
363  }
364  }
365  }
366 
367  } // while(!done)...poll
368 
369  TLOG(5) << GetTraceName() << ": receiveFragmentHeader: Returning " << ret_rank;
370  return ret_rank;
371 }
372 
373 int artdaq::TCPSocketTransfer::disconnect_receive_socket_(int fd, std::string msg)
374 {
375  TLOG(TLVL_WARNING) << GetTraceName() << ": disconnect_receive_socket_: " << msg << " Closing socket " << fd;
376  close(fd);
377  std::unique_lock<std::mutex> lk(connected_fd_mutex_);
378  if (connected_fds_.count(source_rank()))
379  connected_fds_[source_rank()].erase(fd);
380  fd = -1;
381  TLOG(TLVL_DEBUG) << GetTraceName() << ": disconnect_receive_socket_: There are now " << connected_fds_[source_rank()].size() << " active senders.";
382  return fd;
383 }
384 
385 int artdaq::TCPSocketTransfer::receiveFragmentData(RawDataType* destination, size_t)
386 {
387  TLOG(19) << GetTraceName() << ": receiveFragmentData: BEGIN";
388  int ret_rank = RECV_TIMEOUT;
389  if (active_receive_fd_ == -1)
390  { // what if just listen_fd???
391  TLOG(TLVL_ERROR) << GetTraceName() << ": receiveFragmentData: Receive socket not connected, returning RECV_TIMEOUT (Will result in \"Unexpected return code error\")";
392  return RECV_TIMEOUT;
393  }
394 
395  //void* buff=alloca(max_fragment_size_words_*8);
396  uint8_t* buff;
397  size_t byte_cnt = 0;
398  int sts;
399  int offset = 0;
400  SocketState state = SocketState::Metadata;
401  int target_bytes = sizeof(MessHead);
402 
403  pollfd pollfd_s;
404  pollfd_s.events = POLLIN | POLLPRI | POLLERR;
405  pollfd_s.fd = active_receive_fd_;
406 
407  int loop_guard = 0;
408  bool done = false;
409  bool noDataWarningSent = false;
410  last_recv_time_ = std::chrono::steady_clock::now();
411  while (!done)
412  {
413  TLOG(9) << GetTraceName() << ": receiveFragmentData: Polling fd to see if there's data";
414  int num_fds_ready = poll(&pollfd_s, 1, 1000);
415  TLOG(TLVL_TRACE) << GetTraceName() << ": receiveFragmentData: Polled fd to see if there's data"
416  << ", num_fds_ready = " << num_fds_ready;
417  if (num_fds_ready <= 0)
418  {
419  if (num_fds_ready == 0)
420  {
421  TLOG(TLVL_WARNING) << GetTraceName() << ": receiveFragmentData: No data from " << source_rank() << " in " << TimeUtils::GetElapsedTimeMilliseconds(last_recv_time_) << " ms!"
422  << " State = " << (state == SocketState::Metadata ? "Metadata" : "Data") << ", recvd/total=" << offset << "/" << target_bytes << " (delta=" << target_bytes - offset << ")";
423 
424  if (TimeUtils::GetElapsedTime(last_recv_time_) > receive_disconnected_wait_s_)
425  {
426  TLOG(TLVL_WARNING) << GetTraceName() << ": receiveFragmentData: No data received within timeout (" << TimeUtils::GetElapsedTime(last_recv_time_) << " / " << receive_disconnected_wait_s_ << " ), returning RECV_TIMEOUT";
427  disconnect_receive_socket_(active_receive_fd_, "No data on this socket within timeout");
428  active_receive_fd_ = -1;
429  return RECV_TIMEOUT;
430  }
431  continue;
432  }
433 
434  TLOG(TLVL_ERROR) << "Error in poll: errno=" << errno;
435  active_receive_fd_ = -1;
436  break;
437  }
438  else
439  {
440  last_recv_time_ = std::chrono::steady_clock::now();
441  }
442 
443  if (pollfd_s.revents & (POLLIN | POLLPRI))
444  {
445  // Expected, don't have to check revents any further
446  }
447  else if (pollfd_s.revents & (POLLNVAL))
448  {
449  disconnect_receive_socket_(pollfd_s.fd, "FD is closed, most likely because the peer went away.");
450  break;
451  }
452  else if (pollfd_s.revents & (POLLHUP | POLLERR))
453  {
454  disconnect_receive_socket_(pollfd_s.fd, "Poll returned POLLHUP or POLLERR, indicating problems with the sender.");
455  break;
456  }
457  else
458  {
459  TLOG(TLVL_WARNING) << GetTraceName() << ": receiveFragmentData: Wrong event received from pollfd: " << pollfd_s.revents;
460  disconnect_receive_socket_(pollfd_s.fd);
461  break;
462  }
463 
464  if (state == SocketState::Metadata)
465  {
466  //TLOG(TLVL_DEBUG) << GetTraceName() << ": receiveFragmentData: Reading Message Header" ;
467  buff = &(mha[offset]);
468  byte_cnt = sizeof(MessHead) - offset;
469  }
470  else
471  {
472  //TLOG(TLVL_DEBUG) << GetTraceName() << ": receiveFragmentData: Reading data" ;
473  buff = reinterpret_cast<uint8_t*>(destination) + offset;
474  byte_cnt = mh.byte_count - offset;
475  }
476 
477  TLOG(10) << GetTraceName() << ": receiveFragmentData: Reading " << byte_cnt << " bytes from socket into " << (void*)buff;
478  sts = read(active_receive_fd_, buff, byte_cnt);
479  //TLOG(TLVL_DEBUG) << GetTraceName() << ": receiveFragmentData: Done with read" ;
480 
481  TLOG(10) << GetTraceName() << ": recvFragment state=" << static_cast<int>(state) << " read=" << sts;
482 
483  if (sts == 0)
484  {
485  if (loop_guard > 10) { usleep(1000); }
486  if (++loop_guard > 10010)
487  {
488  TLOG(TLVL_WARNING) << GetTraceName() << ": receiveFragmentData: loop guard triggered, returning RECV_TIMEOUT";
489  active_receive_fd_ = -1;
490  return RECV_TIMEOUT;
491  }
492  }
493  else if (sts > 0)
494  {
495  loop_guard = 0;
496  last_recv_time_ = std::chrono::steady_clock::now();
497  }
498 
499  if (sts < 0)
500  {
501  TLOG(TLVL_WARNING) << GetTraceName() << ": receiveFragmentData: Error on receive, closing socket"
502  << " (errno=" << errno << ": " << strerror(errno) << ")";
503  disconnect_receive_socket_(pollfd_s.fd);
504  }
505  else if (sts == 0)
506  {
507  if (!noDataWarningSent)
508  {
509  TLOG(TLVL_WARNING) << GetTraceName() << ": receiveFragmentData: No data received, is the sender still sending?!?";
510  noDataWarningSent = true;
511  }
512  if (TimeUtils::GetElapsedTime(last_recv_time_) > receive_disconnected_wait_s_)
513  {
514  TLOG(TLVL_ERROR) << GetTraceName() << ": receiveFragmentData: No data received within timeout, aborting!";
515  return RECV_TIMEOUT;
516  }
517  }
518  else
519  {
520  // see if we're done (with this state)
521  sts = offset += sts;
522  if (sts >= target_bytes)
523  {
524  TLOG(9) << GetTraceName() << ": receiveFragmentData: Target read bytes reached. Changing state";
525  offset = 0;
526  if (state == SocketState::Metadata)
527  {
528  state = SocketState::Data;
529  mh.byte_count = ntohl(mh.byte_count);
530  mh.source_id = ntohs(mh.source_id);
531  target_bytes = mh.byte_count;
532 
533  if (mh.message_type == MessHead::header_v0)
534  {
535  TLOG(TLVL_WARNING) << GetTraceName() << ": receiveFragmentData: Message header indicates that a Fragment header follows when I was expecting Fragment data!";
536  active_receive_fd_ = disconnect_receive_socket_(active_receive_fd_, "Desync detected");
537  }
538  }
539  else
540  {
541  ret_rank = source_rank();
542  TLOG(11) << GetTraceName() << ": receiveFragmentData done sts=" << sts << " src=" << ret_rank;
543  TLOG(9) << GetTraceName() << ": receiveFragmentData: Done receiving fragment. Moving into output.";
544 
545 #if USE_ACKS
546  send_ack_(active_receive_fd_);
547 #endif
548 
549  done = true; // no more polls
550  //break; // no more read of ready fds
551  }
552  }
553  }
554 
555  // Check if we were asked to do a 0-size receive
556  if (target_bytes == 0 && state == SocketState::Data)
557  {
558  ret_rank = source_rank();
559  TLOG(11) << GetTraceName() << ": receiveFragmentData done sts=" << sts << " src=" << ret_rank;
560  TLOG(9) << GetTraceName() << ": receiveFragmentData: Done receiving fragment. Moving into output.";
561 
562 #if USE_ACKS
563  send_ack_(active_receive_fd_);
564 #endif
565 
566  done = true; // no more polls
567  }
568 
569  } // while(!done)...poll
570 
571  last_active_receive_fd_ = active_receive_fd_;
572  active_receive_fd_ = -1;
573 
574  TLOG(9) << GetTraceName() << ": receiveFragmentData: Returning rank " << ret_rank;
575  return ret_rank;
576 }
577 
579 {
580  switch (role())
581  {
583  return send_fd_ != -1;
585  TLOG(TLVL_DEBUG) << GetTraceName() << ": isRunning: There are " << getConnectedFDCount(source_rank()) << " fds connected.";
586  return getConnectedFDCount(source_rank()) > 0;
587  }
588  return false;
589 }
590 
592 {
593  while (connected_fds_[source_rank()].size())
594  {
595  disconnect_receive_socket_(*connected_fds_[source_rank()].begin(), "Flushing connections");
596  }
597 }
598 
599 // Send the given Fragment. Return the rank of the destination to which
600 // the Fragment was sent OR -1 if to none.
601 artdaq::TransferInterface::CopyStatus artdaq::TCPSocketTransfer::sendFragment_(Fragment&& frag, size_t send_timeout_usec)
602 {
603  TLOG(12) << GetTraceName() << ": sendFragment begin send of fragment with sequenceID=" << frag.sequenceID();
604  artdaq::Fragment grab_ownership_frag = std::move(frag);
605 
606  reconnect_();
607  if (send_fd_ == -1 && connection_was_lost_)
608  {
609  TLOG(TLVL_INFO) << GetTraceName() << ": reconnection attempt failed, returning quickly.";
611  }
612 
613  // Send Fragment Header
614 
615 #if USE_ACKS
616  // Wait for fragments to be received
617  while (static_cast<size_t>(send_ack_diff_) > buffer_count_) usleep(10000);
618 #endif
619 
620  iovec iov = {reinterpret_cast<void*>(grab_ownership_frag.headerAddress()),
621  detail::RawFragmentHeader::num_words() * sizeof(RawDataType)};
622 
623  auto sts = sendData_(&iov, 1, send_retry_timeout_us_, true);
624  auto start_time = std::chrono::steady_clock::now();
625  //If it takes more than 10 seconds to write a Fragment header, give up
626  while (sts == CopyStatus::kTimeout && (send_timeout_usec == 0 || TimeUtils::GetElapsedTimeMicroseconds(start_time) < send_timeout_usec) && TimeUtils::GetElapsedTimeMicroseconds(start_time) < 10000000)
627  {
628  TLOG(13) << GetTraceName() << ": sendFragment: Timeout sending fragment";
629  sts = sendData_(&iov, 1, send_retry_timeout_us_, true);
630  usleep(1000);
631  }
632  if (sts != CopyStatus::kSuccess) return sts;
633 
634  // Send Fragment Data
635 
636  iov = {reinterpret_cast<void*>(grab_ownership_frag.headerAddress() + detail::RawFragmentHeader::num_words()),
637  grab_ownership_frag.sizeBytes() - detail::RawFragmentHeader::num_words() * sizeof(RawDataType)};
638  sts = sendData_(&iov, 1, send_retry_timeout_us_);
639  start_time = std::chrono::steady_clock::now();
640  while (sts == CopyStatus::kTimeout && (send_timeout_usec == 0 || TimeUtils::GetElapsedTimeMicroseconds(start_time) < send_timeout_usec) && TimeUtils::GetElapsedTimeMicroseconds(start_time) < 10000000)
641  {
642  TLOG(13) << GetTraceName() << ": sendFragment: Timeout sending fragment";
643  sts = sendData_(&iov, 1, send_retry_timeout_us_);
644  usleep(1000);
645  }
646 
647 #if USE_ACKS
648  send_ack_diff_++;
649 #endif
650 
651  TLOG(12) << GetTraceName() << ": sendFragment returning " << CopyStatusToString(sts);
652  return sts;
653 }
654 
655 artdaq::TransferInterface::CopyStatus artdaq::TCPSocketTransfer::sendData_(const void* buf, size_t bytes, size_t send_timeout_usec, bool isHeader)
656 {
657  TLOG(TLVL_DEBUG) << GetTraceName() << ": sendData_ Converting buf to iovec";
658  iovec iov = {(void*)buf, bytes};
659  return sendData_(&iov, 1, send_timeout_usec, isHeader);
660 }
661 
662 artdaq::TransferInterface::CopyStatus artdaq::TCPSocketTransfer::sendData_(const struct iovec* iov, int iovcnt, size_t send_timeout_usec, bool isHeader)
663 {
664  // check all connected??? -- currently just check fd!=-1
665  if (send_fd_ == -1)
666  {
667  if (timeoutMessageArmed_)
668  {
669  TLOG(TLVL_DEBUG) << GetTraceName() << ": sendData_: Send fd is not open. Returning kTimeout";
670  timeoutMessageArmed_ = false;
671  }
672  return CopyStatus::kTimeout;
673  }
674  timeoutMessageArmed_ = true;
675  TLOG(14) << GetTraceName() << ": send_timeout_usec is " << send_timeout_usec << ", currently unused.";
676 
677  //TLOG(TLVL_DEBUG) << GetTraceName() << ": sendData_: Determining write size" ;
678  uint32_t total_to_write_bytes = 0;
679  std::vector<iovec> iov_in(iovcnt + 1); // need contiguous (for the unlike case that only partial MH
680  std::vector<iovec> iovv(iovcnt + 2); // 1 more for mh and another one for any partial
681  int ii;
682  for (ii = 0; ii < iovcnt; ++ii)
683  {
684  iov_in[ii + 1] = iov[ii];
685  total_to_write_bytes += iov[ii].iov_len;
686  }
687  //TLOG(TLVL_DEBUG) << GetTraceName() << ": sendData_: Constructing Message Header" ;
688  MessHead mh = {0, isHeader ? MessHead::header_v0 : MessHead::data_v0, htons(source_rank()), {htonl(total_to_write_bytes)}};
689  iov_in[0].iov_base = &mh;
690  iov_in[0].iov_len = sizeof(mh);
691  total_to_write_bytes += sizeof(mh);
692 
693  ssize_t sts = 0;
694  ssize_t total_written_bytes = 0;
695  ssize_t per_write_max_bytes = (32 * 1024);
696 
697  size_t in_iov_idx = 0; // only increment this when we know the associated data has been xferred
698  size_t out_iov_idx = 0;
699  ssize_t this_write_bytes = 0;
700 
701  do
702  {
703  // The first out_iov may be set at the end of the previous loop.
704  // iov looping from below (b/c of the latter, we need to check this_write_bytes)
705  for (;
706  (in_iov_idx + out_iov_idx) < iov_in.size() && this_write_bytes < per_write_max_bytes;
707  ++out_iov_idx)
708  {
709  this_write_bytes += iov_in[in_iov_idx + out_iov_idx].iov_len;
710  iovv[out_iov_idx] = iov_in[in_iov_idx + out_iov_idx];
711  }
712  if (this_write_bytes > per_write_max_bytes)
713  {
714  iovv[out_iov_idx - 1].iov_len -= this_write_bytes - per_write_max_bytes;
715  this_write_bytes = per_write_max_bytes;
716  }
717 
718  // need to do blocking algorithm -- including throttled block notifications
719  do_again:
720 #ifndef __OPTIMIZE__ // This can be an expensive TRACE call (even if disabled) due to multiplicity of calls
721  TLOG(14) << GetTraceName() << ": sendFragment b4 writev " << std::setw(7) << total_written_bytes << " total_written_bytes send_fd_=" << send_fd_ << " in_idx=" << in_iov_idx
722  << " iovcnt=" << out_iov_idx << " 1st.len=" << iovv[0].iov_len;
723 #endif
724  //TLOG(TLVL_DEBUG) << GetTraceName() << " calling writev" ;
725  sts = writev(send_fd_, &(iovv[0]), out_iov_idx);
726  //TLOG(TLVL_DEBUG) << GetTraceName() << " done with writev" ;
727 
728  if (sts == -1)
729  {
730  if (errno == EAGAIN /* same as EWOULDBLOCK */)
731  {
732  TLOG(TLVL_DEBUG) << GetTraceName() << ": sendFragment EWOULDBLOCK";
733  fcntl(send_fd_, F_SETFL, 0); // clear O_NONBLOCK
734  blocking = true;
735  // NOTE: YES -- could drop here
736  goto do_again;
737  }
738  TLOG(TLVL_WARNING) << GetTraceName() << ": sendFragment_: WRITE ERROR: " << strerror(errno);
739  connect_state = 0; // any write error closes
740  close(send_fd_);
741  send_fd_ = -1;
742  connection_was_lost_ = true;
744  }
745  else if (sts != this_write_bytes)
746  {
747  // we'll loop around -- with
748  TLOG(TLVL_DEBUG) << GetTraceName() << ": sendFragment writev sts(" << sts << ")!=requested_send_bytes(" << this_write_bytes << ")";
749  total_written_bytes += sts; // add sts to total_written_bytes now as sts is adjusted next
750  // find which iovs are done
751  for (ii = 0; (size_t)sts >= iovv[ii].iov_len; ++ii)
752  sts -= iovv[ii].iov_len;
753  in_iov_idx += ii; // done with these in_iovs
754  iovv[ii].iov_len -= sts; // adjust partial iov
755  iovv[ii].iov_base = (uint8_t*)(iovv[ii].iov_base) + sts; // adjust partial iov
756 
757  // add more to get up to per_write_max_bytes
758  out_iov_idx = 0;
759  if (ii != 0)
760  iovv[out_iov_idx] = iovv[ii];
761  // starting over
762  this_write_bytes = iovv[out_iov_idx].iov_len;
763  // add any left over from appropriate in_iov_idx --
764  // i.e. match this out_iov with the in_iov that was used to
765  // initialize it; see how close the out base+len is to in base+len
766  // check !>per_write_max_bytes
767  unsigned long additional = ((unsigned long)iov_in[in_iov_idx].iov_base + iov_in[in_iov_idx].iov_len) - ((unsigned long)iovv[out_iov_idx].iov_base + iovv[out_iov_idx].iov_len);
768  if (additional)
769  {
770  iovv[out_iov_idx].iov_len += additional;
771  this_write_bytes += additional;
772  if (this_write_bytes > per_write_max_bytes)
773  {
774  iovv[out_iov_idx].iov_len -= this_write_bytes - per_write_max_bytes;
775  this_write_bytes = per_write_max_bytes;
776  }
777  }
778  ++out_iov_idx; // done with
779  TLOG(TLVL_TRACE) << GetTraceName() << ": sendFragment writev sts!=: this_write_bytes=" << this_write_bytes
780  << " out_iov_idx=" << out_iov_idx
781  << " additional=" << additional
782  << " ii=" << ii;
783  }
784  else
785  {
786 #ifndef __OPTIMIZE__ // This can be an expensive TRACE call (even if disabled) due to multiplicity of calls
787  TLOG(TLVL_TRACE) << GetTraceName() << ": sendFragment writev sts(" << sts << ")==requested_send_bytes(" << this_write_bytes << ")";
788 #endif
789  total_written_bytes += sts;
790  --out_iov_idx; // make it the index of the last iovv
791  iovv[out_iov_idx].iov_base = (uint8_t*)(iovv[out_iov_idx].iov_base) + iovv[out_iov_idx].iov_len;
792  iovv[out_iov_idx].iov_len = 0;
793  in_iov_idx += out_iov_idx; // at least this many complete (one more if "last iovv" is complete
794  this_write_bytes = 0;
795  // need to check last iovv against appropriate iov_in
796  unsigned long additional = ((unsigned long)iov_in[in_iov_idx].iov_base + iov_in[in_iov_idx].iov_len) - ((unsigned long)iovv[out_iov_idx].iov_base + iovv[out_iov_idx].iov_len);
797  if (additional)
798  {
799  iovv[out_iov_idx].iov_len += additional;
800  this_write_bytes += additional;
801  if (this_write_bytes > per_write_max_bytes)
802  {
803  iovv[out_iov_idx].iov_len -= this_write_bytes - per_write_max_bytes;
804  this_write_bytes = per_write_max_bytes;
805  }
806  if (out_iov_idx != 0)
807  iovv[0] = iovv[out_iov_idx];
808  out_iov_idx = 1;
809  }
810  else
811  {
812  ++in_iov_idx;
813  out_iov_idx = 0;
814  }
815  }
816  } while (total_written_bytes < total_to_write_bytes);
817  if (total_written_bytes > total_to_write_bytes)
818  TLOG(TLVL_ERROR) << GetTraceName() << ": sendFragment program error: too many bytes transferred";
819 
820  if (blocking)
821  {
822  blocking = false;
823  fcntl(send_fd_, F_SETFL, O_NONBLOCK); // set O_NONBLOCK
824  }
825  sts = total_written_bytes - sizeof(MessHead);
826 
827  TLOG(14) << GetTraceName() << ": sendFragment sts=" << sts;
829 }
830 
831 void artdaq::TCPSocketTransfer::connect_()
832 {
833  auto start_time = std::chrono::steady_clock::now();
834 
835  // Retry a few times if we can't connect
836  while (send_fd_ == -1 && TimeUtils::GetElapsedTimeMicroseconds(start_time) < send_retry_timeout_us_ * 10)
837  {
838  TLOG(TLVL_DEBUG) << GetTraceName() << ": Connecting sender socket";
839  int sndbuf_bytes = static_cast<int>(sndbuf_);
840  if (sndbuf_ > INT_MAX)
841  {
842  sndbuf_bytes = INT_MAX;
843  TLOG(TLVL_WARNING) << "Requested SNDBUF " << sndbuf_ << " too large, setting to INT_MAX: " << INT_MAX;
844  }
845  TLOG(TLVL_DEBUG) << "Requested SNDBUF is " << sndbuf_bytes;
846 
847  send_fd_ = TCPConnect(hostMap_[destination_rank()].c_str(), portMan->GetTCPSocketTransferPort(destination_rank()), O_NONBLOCK, sndbuf_bytes);
848  if (send_fd_ == -1)
849  {
850  if (connection_was_lost_) { break; }
851  else
852  {
853  usleep(send_retry_timeout_us_);
854  }
855  }
856  }
857  connect_state = 0;
858  blocking = 0;
859  TLOG(TLVL_DEBUG) << GetTraceName() << ": connect_ " + hostMap_[destination_rank()] + ":" << portMan->GetTCPSocketTransferPort(destination_rank()) << " send_fd_=" << send_fd_;
860  if (send_fd_ != -1)
861  {
862  // write connect msg
863  TLOG(TLVL_DEBUG) << GetTraceName() << ": connect_: Writing connect message";
864  MessHead mh = {0, MessHead::connect_v0, htons(source_rank()), {htonl(CONN_MAGIC)}};
865  ssize_t sts = write(send_fd_, &mh, sizeof(mh));
866  if (sts == -1)
867  {
868  TLOG(TLVL_ERROR) << GetTraceName() << ": connect_: Error writing connect message!";
869  // a write error here is completely unexpected!
870  connect_state = 0;
871  close(send_fd_);
872  send_fd_ = -1;
873  }
874  else
875  {
876  TLOG(TLVL_INFO) << GetTraceName() << ": connect_: Successfully connected";
877  // consider it all connected/established
878  connect_state = 1;
879  connection_was_lost_ = false;
880  }
881 
882 #if USE_ACKS
883  if (ack_listen_thread_ && ack_listen_thread_->joinable()) ack_listen_thread_->join();
884  TLOG(TLVL_INFO) << GetTraceName() << ": Starting Ack Listener Thread";
885 
886  try
887  {
888  ack_listen_thread_ = std::make_unique<boost::thread>(&TCPSocketTransfer::receive_acks_, this);
889  }
890  catch (const boost::exception& e)
891  {
892  TLOG(TLVL_ERROR) << "Caught boost::exception starting TCP Socket Ack Listen thread: " << boost::diagnostic_information(e) << ", errno=" << errno;
893  std::cerr << "Caught boost::exception starting TCP Socket Ack Listen thread: " << boost::diagnostic_information(e) << ", errno=" << errno << std::endl;
894  exit(5);
895  }
896 #endif
897  }
898 }
899 
900 void artdaq::TCPSocketTransfer::reconnect_()
901 {
902  if (send_fd_ == -1 && role() == TransferInterface::Role::kSend)
903  {
904  TLOG(TLVL_TRACE) << GetTraceName() << ": check/reconnect";
905  return connect_();
906  }
907 }
908 
909 void artdaq::TCPSocketTransfer::start_listen_thread_()
910 {
911  std::unique_lock<std::mutex> start_lock(listen_thread_mutex_);
912  if (listen_thread_refcount_ == 0)
913  {
914  if (listen_thread_ && listen_thread_->joinable()) listen_thread_->join();
915  listen_thread_refcount_ = 1;
916  TLOG(TLVL_INFO) << GetTraceName() << ": Starting Listener Thread";
917 
918  try
919  {
920  listen_thread_ = std::make_unique<boost::thread>(&TCPSocketTransfer::listen_, portMan->GetTCPSocketTransferPort(destination_rank()), rcvbuf_);
921  }
922  catch (const boost::exception& e)
923  {
924  TLOG(TLVL_ERROR) << "Caught boost::exception starting TCP Socket Listen thread: " << boost::diagnostic_information(e) << ", errno=" << errno;
925  std::cerr << "Caught boost::exception starting TCP Socket Listen thread: " << boost::diagnostic_information(e) << ", errno=" << errno << std::endl;
926  exit(5);
927  }
928  }
929  else
930  {
931  listen_thread_refcount_++;
932  }
933 }
934 
935 #if USE_ACKS
936 void artdaq::TCPSocketTransfer::receive_acks_()
937 {
938  while (send_fd_ >= 0)
939  {
940  pollfd pollfd_s;
941  pollfd_s.events = POLLIN | POLLPRI;
942  pollfd_s.fd = send_fd_;
943 
944  TLOG(18) << GetTraceName() << ": receive_acks_: Polling fd to see if there's data";
945  int num_fds_ready = poll(&pollfd_s, 1, 1000);
946  if (num_fds_ready <= 0)
947  {
948  if (num_fds_ready == 0)
949  {
950  TLOG(18) << GetTraceName() << ": receive_acks_: No data on receive socket";
951  continue;
952  }
953 
954  TLOG(TLVL_ERROR) << "Error in poll: errno=" << errno;
955  break;
956  }
957 
958  if (pollfd_s.revents & (POLLIN | POLLPRI))
959  {
960  // Expected, don't have to check revents any further
961  }
962  else
963  {
964  TLOG(TLVL_DEBUG) << GetTraceName() << ": receive_acks_: Wrong event received from pollfd: " << pollfd_s.revents;
965  break;
966  }
967 
968  MessHead mh;
969  auto sts = read(send_fd_, &mh, sizeof(mh));
970 
971  if (sts != sizeof(mh))
972  {
973  TLOG(TLVL_ERROR) << GetTraceName() << ": receive_ack_: Wrong message header length received! (actual " << sts << " != " << sizeof(mh) << " expected)";
974  continue;
975  }
976 
977  // check for "magic" and valid source_id(aka rank)
978  mh.source_id = ntohs(mh.source_id); // convert here as it is reference several times
979  if (mh.source_id != my_rank)
980  {
981  TLOG(TLVL_ERROR) << GetTraceName() << ": receive_ack_: Received ack for different sender! Rank=" << my_rank << ", hdr=" << mh.source_id;
982  continue;
983  }
984  if (ntohl(mh.conn_magic) != ACK_MAGIC || !(mh.message_type == MessHead::ack_v0)) // Allow for future connect message versions
985  {
986  TLOG(TLVL_ERROR) << GetTraceName() << ": receive_ack_: Wrong magic bytes in header!";
987  continue;
988  }
989 
990  TLOG(17) << GetTraceName() << ": receive_acks_: Received ack message, diff is now " << (send_ack_diff_.load() - 1);
991  send_ack_diff_--;
992  }
993 }
994 
995 void artdaq::TCPSocketTransfer::send_ack_(int fd)
996 {
997  MessHead mh = {0, MessHead::ack_v0, htons(source_rank()), {htonl(ACK_MAGIC)}};
998  write(fd, &mh, sizeof(mh));
999 }
1000 #endif
1001 
1002 void artdaq::TCPSocketTransfer::listen_(int port, size_t rcvbuf)
1003 {
1004  int listen_fd = -1;
1005  while (listen_thread_refcount_ > 0)
1006  {
1007  TLOG(TLVL_TRACE) << "listen_: Listening/accepting new connections on port " << port;
1008  if (listen_fd == -1)
1009  {
1010  TLOG(TLVL_DEBUG) << "listen_: Opening listener";
1011  listen_fd = TCP_listen_fd(port, rcvbuf);
1012  }
1013  if (listen_fd == -1)
1014  {
1015  TLOG(TLVL_DEBUG) << "listen_: Error creating listen_fd!";
1016  break;
1017  }
1018 
1019  int res;
1020  timeval tv = {2, 0}; // maybe increase of some global "debugging" flag set???
1021  fd_set rfds;
1022  FD_ZERO(&rfds);
1023  FD_SET(listen_fd, &rfds);
1024 
1025  res = select(listen_fd + 1, &rfds, (fd_set*)0, (fd_set*)0, &tv);
1026  if (res > 0)
1027  {
1028  int sts;
1029  sockaddr_un un;
1030  socklen_t arglen = sizeof(un);
1031  int fd;
1032  TLOG(TLVL_DEBUG) << "listen_: Calling accept";
1033  fd = accept(listen_fd, (sockaddr*)&un, &arglen);
1034  TLOG(TLVL_DEBUG) << "listen_: Done with accept";
1035 
1036  TLOG(TLVL_DEBUG) << "listen_: Reading connect message";
1037  socklen_t lenlen = sizeof(tv);
1038  /*sts=*/
1039  setsockopt(fd, SOL_SOCKET, SO_RCVTIMEO, &tv, lenlen); // see man 7 socket.
1040  MessHead mh;
1041  uint64_t mark_us = TimeUtils::gettimeofday_us();
1042  sts = read(fd, &mh, sizeof(mh));
1043  uint64_t delta_us = TimeUtils::gettimeofday_us() - mark_us;
1044  TLOG(TLVL_DEBUG) << "listen_: Read of connect message took " << delta_us << " microseconds.";
1045  if (sts != sizeof(mh))
1046  {
1047  TLOG(TLVL_DEBUG) << "listen_: Wrong message header length received!";
1048  close(fd);
1049  continue;
1050  }
1051 
1052  // check for "magic" and valid source_id(aka rank)
1053  mh.source_id = ntohs(mh.source_id); // convert here as it is reference several times
1054  if (ntohl(mh.conn_magic) != CONN_MAGIC || !(mh.message_type == MessHead::connect_v0)) // Allow for future connect message versions
1055  {
1056  TLOG(TLVL_DEBUG) << "listen_: Wrong magic bytes in header!";
1057  close(fd);
1058  continue;
1059  }
1060 
1061  // now add (new) connection
1062  std::unique_lock<std::mutex> lk(connected_fd_mutex_);
1063  connected_fds_[mh.source_id].insert(fd);
1064 
1065  TLOG(TLVL_INFO) << "listen_: New fd is " << fd << " for source rank " << mh.source_id;
1066  }
1067  else
1068  {
1069  TLOG(16) << "listen_: No connections in timeout interval!";
1070  }
1071  }
1072 
1073  TLOG(TLVL_INFO) << "listen_: Shutting down connection listener";
1074  if (listen_fd != -1) close(listen_fd);
1075  std::unique_lock<std::mutex> lk(connected_fd_mutex_);
1076  auto it = connected_fds_.begin();
1077  while (it != connected_fds_.end())
1078  {
1079  auto& fd_set = it->second;
1080  auto rank_it = fd_set.begin();
1081  while (rank_it != fd_set.end())
1082  {
1083  close(*rank_it);
1084  rank_it = fd_set.erase(rank_it);
1085  }
1086  it = connected_fds_.erase(it);
1087  }
1088 
1089 } // do_connect_
1090 
1091 DEFINE_ARTDAQ_TRANSFER(artdaq::TCPSocketTransfer)
bool isRunning() override
Determine whether the TransferInterface plugin is able to send/receive data.
virtual int source_rank() const
Get the source rank for this TransferInterface instance.
int TCPConnect(char const *host_in, int dflt_port, long flags=0, int sndbufsiz=0)
Connect to a host on a given port.
Definition: TCPConnect.cc:358
uint32_t conn_magic
unsigned first is better for MessHead initializer: {0,0,my_node_idx_,CONN_MAGIC}
Definition: SRSockets.hh:40
hostMap_t MakeHostMap(fhicl::ParameterSet pset, hostMap_t map=hostMap_t())
Make a hostMap_t from a HostMap::Config ParameterSet
Definition: HostMap.hh:66
This TransferInterface is a Receiver.
int receiveFragmentData(RawDataType *destination, size_t wordCount) override
Receive the body of a Fragment to the given destination pointer.
int TCP_listen_fd(int port, int rcvbuf)
Create a TCP listening socket on the given port and INADDR_ANY, with the given receive buffer...
TCPSocketTransfer(fhicl::ParameterSet const &ps, Role role)
TCPSocketTransfer Constructor.
int receiveFragmentHeader(detail::RawFragmentHeader &header, size_t receiveTimeout) override
Receive a Fragment Header from the transport mechanism.
This TransferInterface is a Sender.
void flush_buffers() override
Flush any in-flight data. This should be used by the receiver after the receive loop has ended...
int32_t byte_count
use CONN_MAGIC for connect_v0, data that follow for data_v0 (and 0 lenght data)
Definition: SRSockets.hh:41
Some error occurred, but no exception was thrown.
Role
Used to determine if a TransferInterface is a Sender or Receiver.
int64_t source_id
Rank of the source.
Definition: SRSockets.hh:36
MessType message_type
Message Type.
Definition: SRSockets.hh:35
The send operation completed successfully.
This interface defines the functions used to transfer data between artdaq applications.
TransferInterface implementation plugin that sends data using TCP sockets.
This header is sent by the TCPSocket_transfer to allow for more efficient writev calls.
Definition: SRSockets.hh:15
CopyStatus
Returned from the send functions, this enumeration describes the possible return codes. If an exception occurs, it will be thrown and should be handled normally.