artdaq  v3_04_00
TCPSocket_transfer.cc
1 // Sep 14, 2016. "TERMS AND CONDITIONS" governing this file are in the README
2 // or COPYING file. If you do not have such a file, one can be obtained by
3 // contacting Ron or Fermi Lab in Batavia IL, 60510, phone: 630-840-3000.
4 // $RCSfile: .emacs.gnu,v $
5 // rev="$Revision: 1.30 $$Date: 2016/03/01 14:27:27 $";
6 
7 // C Includes
8 #include <stdlib.h> // atoi, strtoul
9 #include <sys/socket.h> // socket, socklen_t
10 #include <sys/un.h> // sockaddr_un
11 #include <arpa/inet.h> // ntohl, ntohs
12 #include <sys/types.h> // size_t
13 #include <poll.h> // struct pollfd
14 
15 // C++ Includes
16 #include <string>
17 #include <fstream>
18 #include <stdexcept>
19 
20 // product Includes
21 #define TRACE_NAME (app_name + "_TCPSocketTransfer").c_str()
22 #include "artdaq/DAQdata/Globals.hh"
23 
24 // artdaq Includes
25 #include "artdaq/TransferPlugins/TCPSocketTransfer.hh"
28 #include "artdaq/TransferPlugins/detail/Timeout.hh"
29 #include "artdaq/TransferPlugins/detail/SRSockets.hh"
30 #include "artdaq-core/Data/Fragment.hh"
31 #include "artdaq-core/Utilities/TimeUtils.hh"
32 #include <iomanip>
33 
34 std::atomic<int> artdaq::TCPSocketTransfer::listen_thread_refcount_(0);
35 std::unique_ptr<boost::thread> artdaq::TCPSocketTransfer::listen_thread_ = nullptr;
36 std::map<int, std::set<int>> artdaq::TCPSocketTransfer::connected_fds_ = std::map<int, std::set<int>>();
37 std::mutex artdaq::TCPSocketTransfer::listen_thread_mutex_;
38 std::mutex artdaq::TCPSocketTransfer::connected_fd_mutex_;
39 
41 TCPSocketTransfer(fhicl::ParameterSet const& pset, TransferInterface::Role role)
42  : TransferInterface(pset, role)
43  , send_fd_(-1)
44  , active_receive_fd_(-1)
45  , last_active_receive_fd_(-1)
46  , rcvbuf_(pset.get<size_t>("tcp_receive_buffer_size", 0))
47  , sndbuf_(pset.get<size_t>("tcp_send_buffer_size", max_fragment_size_words_ * sizeof(artdaq::RawDataType) * buffer_count_))
48  , send_retry_timeout_us_(pset.get<size_t>("send_retry_timeout_us", 1000000))
49  , timeoutMessageArmed_(true)
50  , last_recv_time_()
51  , receive_disconnected_wait_s_(pset.get<double>("receive_socket_disconnected_wait_s", 10.0))
52  , receive_err_wait_us_(pset.get<size_t>("receive_socket_disconnected_wait_us", 10000))
53  , receive_socket_has_been_connected_(false)
54  , send_ack_diff_(0)
55 {
56  TLOG(TLVL_DEBUG) << GetTraceName() << " Constructor: pset=" << pset.to_string() << ", role=" << (role == TransferInterface::Role::kReceive ? "kReceive" : "kSend");
57 
59  {
60  // Wait for sender to connect...
61  TLOG(TLVL_DEBUG) << GetTraceName() << ": Listening for connections";
62  start_listen_thread_();
63  TLOG(TLVL_DEBUG) << GetTraceName() << ": Done Listening";
64  }
65  else
66  {
67  hostMap_ = MakeHostMap(pset);
68  TLOG(TLVL_DEBUG) << GetTraceName() << ": Connecting to destination";
69  connect_();
70  TLOG(TLVL_DEBUG) << GetTraceName() << ": Done Connecting";
71  }
72  TLOG(TLVL_DEBUG) << GetTraceName() << ": End of Constructor";
73 }
74 
75 artdaq::TCPSocketTransfer::~TCPSocketTransfer() noexcept
76 {
77  TLOG(TLVL_DEBUG) << GetTraceName() << ": Shutting down TCPSocketTransfer";
78 
79  if (role() == TransferInterface::Role::kSend)
80  {
81  // close all open connections (send stop_v0) first
82  MessHead mh = { 0,MessHead::stop_v0,htons(TransferInterface::source_rank()),{0} };
83  if (send_fd_ != -1)
84  {
85  // should be blocking with modest timeo
86  timeval tv = { 0,100000 };
87  socklen_t len = sizeof(tv);
88  setsockopt(send_fd_, SOL_SOCKET, SO_SNDTIMEO, &tv, len);
89  write(send_fd_, &mh, sizeof(mh));
90  }
91  close(send_fd_);
92  send_fd_ = -1;
93  }
94  else
95  {
96  {
97  std::unique_lock<std::mutex> fd_lock(connected_fd_mutex_);
98  if (connected_fds_.count(source_rank()))
99  {
100  auto it = connected_fds_[source_rank()].begin();
101  while (it != connected_fds_[source_rank()].end())
102  {
103  close(*it);
104  it = connected_fds_[source_rank()].erase(it);
105  }
106  connected_fds_.erase(source_rank());
107  }
108  if (ack_listen_thread_ && ack_listen_thread_->joinable()) ack_listen_thread_->join();
109  }
110 
111  std::unique_lock<std::mutex> lk(listen_thread_mutex_);
112  listen_thread_refcount_--;
113  if (listen_thread_refcount_ <= 0 && listen_thread_ && listen_thread_->joinable())
114  {
115  listen_thread_->join();
116  }
117  }
118 
119  TLOG(TLVL_DEBUG) << GetTraceName() << ": End of Destructor";
120 }
121 
122 int artdaq::TCPSocketTransfer::receiveFragmentHeader(detail::RawFragmentHeader& header, size_t timeout_usec)
123 {
124  TLOG(5) << GetTraceName() << ": receiveFragmentHeader: BEGIN";
125  int ret_rank = RECV_TIMEOUT;
126 
127  // Don't bomb out until received at least one connection...
128  if (getConnectedFDCount(source_rank()) == 0)
129  { // what if just listen_fd???
130  // if (receive_socket_has_been_connected_ && TimeUtils::GetElapsedTime(last_recv_time_) > receive_disconnected_wait_s_)
131  // {
132  // TLOG(TLVL_ERROR) << GetTraceName() << ": receiveFragmentHeader: senders have been disconnected for "
133  // << TimeUtils::GetElapsedTime(last_recv_time_) << " s (receive_socket_disconnected_wait_s = " << receive_disconnected_wait_s_ << " s). RETURNING DATA_END!";
134  // return DATA_END;
135  // }
136  //if (++not_connected_count_ > receive_err_threshold_) { return DATA_END; }
137  TLOG(7) << GetTraceName() << ": receiveFragmentHeader: Receive socket not connected, returning RECV_TIMEOUT";
138  usleep(receive_err_wait_us_);
139  return RECV_TIMEOUT;
140  }
141  receive_socket_has_been_connected_ = true;
142  last_recv_time_ = std::chrono::steady_clock::now();
143 
144  TLOG(5) << GetTraceName() << ": receiveFragmentHeader timeout_usec=" << timeout_usec;
145  //void* buff=alloca(max_fragment_size_words_*8);
146  size_t byte_cnt = 0;
147  int sts;
148  int offset = 0;
149  SocketState state = SocketState::Metadata;
150  int target_bytes = sizeof(MessHead);
151  uint64_t start_time_us = TimeUtils::gettimeofday_us();
152 
153  //while (active_receive_fd_ != -1)
154  //{
155  // TLOG(TLVL_TRACE) << GetTraceName() << ": Currently receiving from fd " << active_receive_fd_ << ", waiting!";
156  // usleep(1000);
157  //}
158 
159 
160  uint8_t* buff;
161 
162  int timeout_ms;
163  if (timeout_usec == 0)
164  timeout_ms = 0;
165  else
166  timeout_ms = (timeout_usec + 999) / 1000; // want at least 1 ms
167 
168  bool done = false;
169  bool noDataWarningSent = false;
170  int loop_guard = 0;
171 
172  while (!done && getConnectedFDCount(source_rank()) > 0)
173  {
174  if (active_receive_fd_ == -1)
175  {
176  loop_guard = 0;
177  size_t fd_count = 0;
178  std::vector<pollfd> pollfds;
179  {
180  std::unique_lock<std::mutex> lk(connected_fd_mutex_);
181  fd_count = connected_fds_[source_rank()].size();
182  pollfds.resize(fd_count);
183  auto iter = connected_fds_[source_rank()].begin();
184  for (size_t ii = 0; ii < fd_count; ++ii)
185  {
186  pollfds[ii].events = POLLIN | POLLPRI | POLLERR;
187  pollfds[ii].fd = *iter;
188  ++iter;
189  }
190  }
191  //TLOG(TLVL_DEBUG) << GetTraceName() << ": receiveFragment: Polling fd to see if there's data" ;
192  int num_fds_ready = poll(&pollfds[0], fd_count, timeout_ms);
193  if (num_fds_ready <= 0)
194  {
195  TLOG(5) << GetTraceName() << ": receiveFragmentHeader: No data on receive socket, returning RECV_TIMEOUT";
196  return RECV_TIMEOUT;
197  }
198 
199  size_t index = 0;
200  if (last_active_receive_fd_ != -1)
201  {
202  for (auto& pollfd : pollfds)
203  {
204  index++;
205  if (pollfd.fd == last_active_receive_fd_)
206  {
207  break;
208  }
209  }
210  }
211 
212  int active_index = -1;
213  short anomolous_events = 0;
214  for (size_t ii = index; ii < index + pollfds.size(); ++ii)
215  {
216  auto pollfd_index = (ii + index) % pollfds.size();
217  if (pollfds[pollfd_index].revents & (POLLIN | POLLPRI))
218  {
219  active_index = pollfd_index;
220  active_receive_fd_ = pollfds[active_index].fd;
221  active_revents_ = pollfds[active_index].revents;;
222  break;
223  }
224  else if (pollfds[pollfd_index].revents & (POLLHUP | POLLERR))
225  {
226  disconnect_receive_socket_(pollfds[pollfd_index].fd, "Poll returned POLLHUP or POLLERR, indicating problems with the sender.");
227  continue;
228  }
229  else if (pollfds[pollfd_index].revents & (POLLNVAL))
230  {
231  TLOG(TLVL_DEBUG) << GetTraceName() << ": receiveFragmentHeader: FD is closed, most likely because the peer went away. Removing from fd list.";
232  disconnect_receive_socket_(pollfds[pollfd_index].fd, "FD is closed, most likely because the peer went away.");
233  continue;
234  }
235  else if (pollfds[pollfd_index].revents)
236  {
237  anomolous_events |= pollfds[pollfd_index].revents;
238  }
239  }
240 
241  if (active_index == -1)
242  {
243  if (anomolous_events)
244  TLOG(TLVL_DEBUG) << GetTraceName() << ": receiveFragmentHeader: Wrong event received from a pollfd. Mask: " << static_cast<int>(anomolous_events);
245  active_receive_fd_ = -1;
246  continue;
247  }
248 
249  if (!done && timeout_usec > 0)
250  {
251  // calc next timeout_ms (unless timed out)
252  size_t delta_us = TimeUtils::gettimeofday_us() - start_time_us;
253  if (delta_us > timeout_usec)
254  {
255  return RECV_TIMEOUT;
256  }
257  timeout_ms = ((timeout_usec - delta_us) + 999) / 1000; // want at least 1 ms
258  }
259  }
260  if (loop_guard > 10) { usleep(1000); }
261  if (++loop_guard > 10010)
262  {
263  TLOG(TLVL_WARNING) << GetTraceName() << ": receiveFragmentHeader: loop guard triggered, returning RECV_TIMEOUT";
264  usleep(receive_err_wait_us_);
265  active_receive_fd_ = -1;
266  return RECV_TIMEOUT;
267  }
268 
269  if (state == SocketState::Metadata)
270  {
271  //TLOG(TLVL_DEBUG) << GetTraceName() << ": receiveFragmentHeader: Reading Message Header" ;
272  buff = &(mha[offset]);
273  byte_cnt = sizeof(MessHead) - offset;
274  }
275  else
276  {
277  //TLOG(TLVL_DEBUG) << GetTraceName() << ": receiveFragmentHeader: Reading data" ;
278  buff = reinterpret_cast<uint8_t*>(&header) + offset;
279  byte_cnt = target_bytes - offset;
280  }
281  //if (byte_cnt > sizeof(MessHead))
282  // {
283  // TLOG(TLVL_ERROR) << "Invalid byte count for read (count=" << byte_cnt
284  // << ",offset=" << offset << ",mh.byte_count=" << mh.byte_count
285  // << "), skipping read and returning RECV_TIMEOUT";
286  // return RECV_TIMEOUT;
287  //}
288 
289  if (byte_cnt > 0)
290  {
291  TLOG(6) << GetTraceName() << ": receiveFragmentHeader: Reading " << byte_cnt << " bytes from socket";
292  sts = read(active_receive_fd_, buff, byte_cnt);
293  TLOG(6) << GetTraceName() << ": receiveFragmentHeader: Done with read";
294  }
295  if (sts > 0) {
296  loop_guard = 0;
297  last_recv_time_ = std::chrono::steady_clock::now();
298  }
299 
300  TLOG(7) << GetTraceName() << ": receiveFragmentHeader state=" << static_cast<int>(state) << " read=" << sts;
301  if (sts < 0)
302  {
303  TLOG(TLVL_WARNING) << GetTraceName() << ": receiveFragmentHeader: Error on receive, closing socket " << " (errno=" << errno << ": " << strerror(errno) << ")";
304  active_receive_fd_ = disconnect_receive_socket_(active_receive_fd_);
305  }
306  else if (sts == 0)
307  {
308  if (!noDataWarningSent) {
309  TLOG(TLVL_WARNING) << GetTraceName() << ": receiveFragmentHeader: No data received, is the sender still sending?!?";
310  noDataWarningSent = true;
311  }
312  if (TimeUtils::GetElapsedTime(last_recv_time_) > receive_disconnected_wait_s_)
313  {
314  TLOG(TLVL_ERROR) << GetTraceName() << ": receiveFragmentHeader: No data received within timeout, aborting!";
315  return RECV_TIMEOUT;
316  }
317  }
318  else
319  {
320  // see if we're done (with this state)
321  sts = offset += sts;
322  if (sts >= target_bytes)
323  {
324  TLOG(7) << GetTraceName() << ": receiveFragmentHeader: Target read bytes reached. Changing state";
325  offset = 0;
326  if (state == SocketState::Metadata)
327  {
328  state = SocketState::Data;
329  mh.byte_count = ntohl(mh.byte_count);
330  mh.source_id = ntohs(mh.source_id);
331  target_bytes = mh.byte_count;
332  TLOG(7) << GetTraceName() << ": receiveFragmentHeader: Expected header size = " << target_bytes << ", sizeof(RawFragmentHeader) = " << sizeof(artdaq::detail::RawFragmentHeader);
333  //assert(target_bytes == sizeof(artdaq::detail::RawFragmentHeader) || target_bytes == 0);
334 
335  if (mh.message_type == MessHead::stop_v0)
336  {
337  active_receive_fd_ = disconnect_receive_socket_(active_receive_fd_, "Stop Message received.");
338  }
339  else if (mh.message_type == MessHead::data_v0 || mh.message_type == MessHead::data_more_v0)
340  {
341  TLOG(TLVL_WARNING) << GetTraceName() << ": receiveFragmentHeader: Message header indicates that Fragment data follows when I was expecting a Fragment header!";
342  active_receive_fd_ = disconnect_receive_socket_(active_receive_fd_, "Desync detected");
343  }
344 
345  if (target_bytes == 0)
346  {
347  //Probably a stop_v0, return timeout so we can try again.
348  return RECV_TIMEOUT;
349  }
350  }
351  else
352  {
353  ret_rank = source_rank();
354  TLOG(8) << GetTraceName() << ": receiveFragmentHeader done sts=" << sts << " src=" << ret_rank;
355  TLOG(7) << GetTraceName() << ": receiveFragmentHeader: Done receiving fragment header. Moving into output.";
356 
357  done = true; // no more polls
358  //break; // no more read of ready fds
359  }
360  }
361  }
362 
363  } // while(!done)...poll
364 
365  TLOG(5) << GetTraceName() << ": receiveFragmentHeader: Returning " << ret_rank;
366  return ret_rank;
367 }
368 
369 int artdaq::TCPSocketTransfer::disconnect_receive_socket_(int fd, std::string msg)
370 {
371  TLOG(TLVL_WARNING) << GetTraceName() << ": disconnect_receive_socket_: " << msg << " Closing socket " << fd;
372  close(fd);
373  std::unique_lock<std::mutex> lk(connected_fd_mutex_);
374  if (connected_fds_.count(source_rank()))
375  connected_fds_[source_rank()].erase(fd);
376  fd = -1;
377  TLOG(TLVL_DEBUG) << GetTraceName() << ": disconnect_receive_socket_: There are now " << connected_fds_[source_rank()].size() << " active senders.";
378  return fd;
379 }
380 
381 int artdaq::TCPSocketTransfer::receiveFragmentData(RawDataType* destination, size_t)
382 {
383  TLOG(19) << GetTraceName() << ": receiveFragmentData: BEGIN";
384  int ret_rank = RECV_TIMEOUT;
385  if (active_receive_fd_ == -1)
386  { // what if just listen_fd???
387  TLOG(TLVL_ERROR) << GetTraceName() << ": receiveFragmentData: Receive socket not connected, returning RECV_TIMEOUT (Will result in \"Unexpected return code error\")";
388  return RECV_TIMEOUT;
389  }
390 
391  //void* buff=alloca(max_fragment_size_words_*8);
392  uint8_t* buff;
393  size_t byte_cnt = 0;
394  int sts;
395  int offset = 0;
396  SocketState state = SocketState::Metadata;
397  int target_bytes = sizeof(MessHead);
398 
399  pollfd pollfd_s;
400  pollfd_s.events = POLLIN | POLLPRI | POLLERR;
401  pollfd_s.fd = active_receive_fd_;
402 
403  int loop_guard = 0;
404  bool done = false;
405  bool noDataWarningSent = false;
406  last_recv_time_ = std::chrono::steady_clock::now();
407  while (!done)
408  {
409  TLOG(9) << GetTraceName() << ": receiveFragmentData: Polling fd to see if there's data";
410  int num_fds_ready = poll(&pollfd_s, 1, 1000);
411  TLOG(TLVL_TRACE) << GetTraceName() << ": receiveFragmentData: Polled fd to see if there's data"
412  << ", num_fds_ready = " << num_fds_ready;
413  if (num_fds_ready <= 0)
414  {
415  if (num_fds_ready == 0)
416  {
417  TLOG(TLVL_WARNING) << GetTraceName() << ": receiveFragmentData: No data from " << source_rank() << " in " << TimeUtils::GetElapsedTimeMilliseconds(last_recv_time_) << " ms!"
418  << " State = " << (state == SocketState::Metadata ? "Metadata" : "Data") << ", recvd/total=" << offset << "/" << target_bytes << " (delta=" << target_bytes - offset << ")";
419 
420  if (TimeUtils::GetElapsedTime(last_recv_time_) > receive_disconnected_wait_s_)
421  {
422  TLOG(TLVL_WARNING) << GetTraceName() << ": receiveFragmentData: No data received within timeout (" << TimeUtils::GetElapsedTime(last_recv_time_) << " / " << receive_disconnected_wait_s_ << " ), returning RECV_TIMEOUT";
423  disconnect_receive_socket_(active_receive_fd_, "No data on this socket within timeout");
424  active_receive_fd_ = -1;
425  return RECV_TIMEOUT;
426  }
427  continue;
428  }
429 
430  TLOG(TLVL_ERROR) << "Error in poll: errno=" << errno;
431  active_receive_fd_ = -1;
432  break;
433  }
434  else { last_recv_time_ = std::chrono::steady_clock::now(); }
435 
436  if (pollfd_s.revents & (POLLIN | POLLPRI))
437  {
438  // Expected, don't have to check revents any further
439  }
440  else if (pollfd_s.revents & (POLLNVAL))
441  {
442  disconnect_receive_socket_(pollfd_s.fd, "FD is closed, most likely because the peer went away.");
443  break;
444  }
445  else if (pollfd_s.revents & (POLLHUP | POLLERR))
446  {
447  disconnect_receive_socket_(pollfd_s.fd, "Poll returned POLLHUP or POLLERR, indicating problems with the sender.");
448  break;
449  }
450  else
451  {
452  TLOG(TLVL_WARNING) << GetTraceName() << ": receiveFragmentData: Wrong event received from pollfd: " << pollfd_s.revents;
453  disconnect_receive_socket_(pollfd_s.fd);
454  break;
455  }
456 
457  if (state == SocketState::Metadata)
458  {
459  //TLOG(TLVL_DEBUG) << GetTraceName() << ": receiveFragmentData: Reading Message Header" ;
460  buff = &(mha[offset]);
461  byte_cnt = sizeof(MessHead) - offset;
462  }
463  else
464  {
465  //TLOG(TLVL_DEBUG) << GetTraceName() << ": receiveFragmentData: Reading data" ;
466  buff = reinterpret_cast<uint8_t*>(destination) + offset;
467  byte_cnt = mh.byte_count - offset;
468  }
469 
470  TLOG(10) << GetTraceName() << ": receiveFragmentData: Reading " << byte_cnt << " bytes from socket into " << (void*)buff;
471  sts = read(active_receive_fd_, buff, byte_cnt);
472  //TLOG(TLVL_DEBUG) << GetTraceName() << ": receiveFragmentData: Done with read" ;
473 
474  TLOG(10) << GetTraceName() << ": recvFragment state=" << static_cast<int>(state) << " read=" << sts;
475 
476  if (sts == 0)
477  {
478  if (loop_guard > 10) { usleep(1000); }
479  if (++loop_guard > 10010)
480  {
481  TLOG(TLVL_WARNING) << GetTraceName() << ": receiveFragmentData: loop guard triggered, returning RECV_TIMEOUT";
482  active_receive_fd_ = -1;
483  return RECV_TIMEOUT;
484  }
485  }
486  else if(sts > 0)
487  {
488  loop_guard = 0;
489  last_recv_time_ = std::chrono::steady_clock::now();
490  }
491 
492  if (sts < 0)
493  {
494  TLOG(TLVL_WARNING) << GetTraceName() << ": receiveFragmentData: Error on receive, closing socket"
495  << " (errno=" << errno << ": " << strerror(errno) << ")";
496  disconnect_receive_socket_(pollfd_s.fd);
497  }
498  else if (sts == 0)
499  {
500  if (!noDataWarningSent) {
501  TLOG(TLVL_WARNING) << GetTraceName() << ": receiveFragmentData: No data received, is the sender still sending?!?";
502  noDataWarningSent = true;
503  }
504  if (TimeUtils::GetElapsedTime(last_recv_time_) > receive_disconnected_wait_s_)
505  {
506  TLOG(TLVL_ERROR) << GetTraceName() << ": receiveFragmentData: No data received within timeout, aborting!";
507  return RECV_TIMEOUT;
508  }
509  }
510  else
511  {
512  // see if we're done (with this state)
513  sts = offset += sts;
514  if (sts >= target_bytes)
515  {
516  TLOG(9) << GetTraceName() << ": receiveFragmentData: Target read bytes reached. Changing state";
517  offset = 0;
518  if (state == SocketState::Metadata)
519  {
520  state = SocketState::Data;
521  mh.byte_count = ntohl(mh.byte_count);
522  mh.source_id = ntohs(mh.source_id);
523  target_bytes = mh.byte_count;
524 
525 
526  if (mh.message_type == MessHead::header_v0)
527  {
528  TLOG(TLVL_WARNING) << GetTraceName() << ": receiveFragmentData: Message header indicates that a Fragment header follows when I was expecting Fragment data!";
529  active_receive_fd_ = disconnect_receive_socket_(active_receive_fd_, "Desync detected");
530  }
531  }
532  else
533  {
534  ret_rank = source_rank();
535  TLOG(11) << GetTraceName() << ": receiveFragmentData done sts=" << sts << " src=" << ret_rank;
536  TLOG(9) << GetTraceName() << ": receiveFragmentData: Done receiving fragment. Moving into output.";
537 
538 #if USE_ACKS
539  send_ack_(active_receive_fd_);
540 #endif
541 
542  done = true; // no more polls
543  //break; // no more read of ready fds
544  }
545  }
546  }
547 
548  // Check if we were asked to do a 0-size receive
549  if (target_bytes == 0 && state == SocketState::Data)
550  {
551  ret_rank = source_rank();
552  TLOG(11) << GetTraceName() << ": receiveFragmentData done sts=" << sts << " src=" << ret_rank;
553  TLOG(9) << GetTraceName() << ": receiveFragmentData: Done receiving fragment. Moving into output.";
554 
555 #if USE_ACKS
556  send_ack_(active_receive_fd_);
557 #endif
558 
559  done = true; // no more polls
560  }
561 
562  } // while(!done)...poll
563 
564  last_active_receive_fd_ = active_receive_fd_;
565  active_receive_fd_ = -1;
566 
567  TLOG(9) << GetTraceName() << ": receiveFragmentData: Returning rank " << ret_rank;
568  return ret_rank;
569 }
570 
572 {
573  switch (role())
574  {
576  return send_fd_ != -1;
578  TLOG(TLVL_DEBUG) << GetTraceName() << ": isRunning: There are " << getConnectedFDCount(source_rank()) << " fds connected.";
579  return getConnectedFDCount(source_rank()) > 0;
580  }
581  return false;
582 }
583 
585 {
586  while(connected_fds_[source_rank()].size()) {
587  disconnect_receive_socket_(*connected_fds_[source_rank()].begin(), "Flushing connections");
588  }
589 }
590 
591 // Send the given Fragment. Return the rank of the destination to which
592 // the Fragment was sent OR -1 if to none.
593 artdaq::TransferInterface::CopyStatus artdaq::TCPSocketTransfer::sendFragment_(Fragment&& frag, size_t send_timeout_usec)
594 {
595  TLOG(12) << GetTraceName() << ": sendFragment begin send of fragment with sequenceID="<<frag.sequenceID();
596  artdaq::Fragment grab_ownership_frag = std::move(frag);
597 
598  reconnect_();
599  // Send Fragment Header
600 
601 #if USE_ACKS
602  // Wait for fragments to be received
603  while (static_cast<size_t>(send_ack_diff_) > buffer_count_) usleep(10000);
604 #endif
605 
606  iovec iov = { reinterpret_cast<void*>(grab_ownership_frag.headerAddress()),
607  detail::RawFragmentHeader::num_words() * sizeof(RawDataType) };
608 
609  auto sts = sendData_(&iov, 1, send_retry_timeout_us_, true);
610  auto start_time = std::chrono::steady_clock::now();
611  //If it takes more than 10 seconds to write a Fragment header, give up
612  while (sts == CopyStatus::kTimeout && (send_timeout_usec == 0 || TimeUtils::GetElapsedTimeMicroseconds(start_time) < send_timeout_usec) && TimeUtils::GetElapsedTimeMicroseconds(start_time) < 10000000)
613  {
614  TLOG(13) << GetTraceName() << ": sendFragment: Timeout sending fragment";
615  sts = sendData_(&iov, 1, send_retry_timeout_us_, true);
616  usleep(1000);
617  }
618  if (sts != CopyStatus::kSuccess) return sts;
619 
620  // Send Fragment Data
621 
622  iov = { reinterpret_cast<void*>(grab_ownership_frag.headerAddress() + detail::RawFragmentHeader::num_words()),
623  grab_ownership_frag.sizeBytes() - detail::RawFragmentHeader::num_words() * sizeof(RawDataType)
624 };
625  sts = sendData_(&iov, 1, send_retry_timeout_us_);
626  start_time = std::chrono::steady_clock::now();
627  while (sts == CopyStatus::kTimeout && (send_timeout_usec == 0 || TimeUtils::GetElapsedTimeMicroseconds(start_time) < send_timeout_usec) && TimeUtils::GetElapsedTimeMicroseconds(start_time) < 10000000)
628  {
629  TLOG(13) << GetTraceName() << ": sendFragment: Timeout sending fragment";
630  sts = sendData_(&iov, 1, send_retry_timeout_us_);
631  usleep(1000);
632  }
633 
634 #if USE_ACKS
635  send_ack_diff_++;
636 #endif
637 
638  TLOG(12) << GetTraceName() << ": sendFragment returning " << CopyStatusToString(sts);
639  return sts;
640 }
641 
642 artdaq::TransferInterface::CopyStatus artdaq::TCPSocketTransfer::sendData_(const void* buf, size_t bytes, size_t send_timeout_usec, bool isHeader)
643 {
644  TLOG(TLVL_DEBUG) << GetTraceName() << ": sendData_ Converting buf to iovec";
645  iovec iov = { (void*)buf, bytes };
646  return sendData_(&iov, 1, send_timeout_usec, isHeader);
647 }
648 
649 artdaq::TransferInterface::CopyStatus artdaq::TCPSocketTransfer::sendData_(const struct iovec* iov, int iovcnt, size_t send_timeout_usec, bool isHeader)
650 {
651  // check all connected??? -- currently just check fd!=-1
652  if (send_fd_ == -1)
653  {
654  if (timeoutMessageArmed_)
655  {
656  TLOG(TLVL_DEBUG) << GetTraceName() << ": sendData_: Send fd is not open. Returning kTimeout";
657  timeoutMessageArmed_ = false;
658  }
659  return CopyStatus::kTimeout;
660  }
661  timeoutMessageArmed_ = true;
662  TLOG(14) << GetTraceName() << ": send_timeout_usec is " << send_timeout_usec << ", currently unused.";
663 
664  //TLOG(TLVL_DEBUG) << GetTraceName() << ": sendData_: Determining write size" ;
665  uint32_t total_to_write_bytes = 0;
666  std::vector<iovec> iov_in(iovcnt + 1); // need contiguous (for the unlike case that only partial MH
667  std::vector<iovec> iovv(iovcnt + 2); // 1 more for mh and another one for any partial
668  int ii;
669  for (ii = 0; ii < iovcnt; ++ii)
670  {
671  iov_in[ii + 1] = iov[ii];
672  total_to_write_bytes += iov[ii].iov_len;
673  }
674  //TLOG(TLVL_DEBUG) << GetTraceName() << ": sendData_: Constructing Message Header" ;
675  MessHead mh = { 0,isHeader ? MessHead::header_v0 : MessHead::data_v0,htons(source_rank()),{htonl(total_to_write_bytes)} };
676  iov_in[0].iov_base = &mh;
677  iov_in[0].iov_len = sizeof(mh);
678  total_to_write_bytes += sizeof(mh);
679 
680  ssize_t sts = 0;
681  ssize_t total_written_bytes = 0;
682  ssize_t per_write_max_bytes = (32 * 1024);
683 
684  size_t in_iov_idx = 0; // only increment this when we know the associated data has been xferred
685  size_t out_iov_idx = 0;
686  ssize_t this_write_bytes = 0;
687 
688  do
689  {
690  // The first out_iov may be set at the end of the previous loop.
691  // iov looping from below (b/c of the latter, we need to check this_write_bytes)
692  for (;
693  (in_iov_idx + out_iov_idx) < iov_in.size() && this_write_bytes < per_write_max_bytes;
694  ++out_iov_idx)
695  {
696  this_write_bytes += iov_in[in_iov_idx + out_iov_idx].iov_len;
697  iovv[out_iov_idx] = iov_in[in_iov_idx + out_iov_idx];
698  }
699  if (this_write_bytes > per_write_max_bytes)
700  {
701  iovv[out_iov_idx - 1].iov_len -= this_write_bytes - per_write_max_bytes;
702  this_write_bytes = per_write_max_bytes;
703  }
704 
705  // need to do blocking algorithm -- including throttled block notifications
706  do_again:
707 #ifndef __OPTIMIZE__ // This can be an expensive TRACE call (even if disabled) due to multiplicity of calls
708  TLOG(14) << GetTraceName() << ": sendFragment b4 writev " << std::setw(7) << total_written_bytes << " total_written_bytes send_fd_=" << send_fd_ << " in_idx=" << in_iov_idx
709  << " iovcnt=" << out_iov_idx << " 1st.len=" << iovv[0].iov_len;
710 #endif
711  //TLOG(TLVL_DEBUG) << GetTraceName() << " calling writev" ;
712  sts = writev(send_fd_, &(iovv[0]), out_iov_idx);
713  //TLOG(TLVL_DEBUG) << GetTraceName() << " done with writev" ;
714 
715  if (sts == -1)
716  {
717  if (errno == EAGAIN /* same as EWOULDBLOCK */)
718  {
719  TLOG(TLVL_DEBUG) << GetTraceName() << ": sendFragment EWOULDBLOCK";
720  fcntl(send_fd_, F_SETFL, 0); // clear O_NONBLOCK
721  blocking = true;
722  // NOTE: YES -- could drop here
723  goto do_again;
724  }
725  TLOG(TLVL_WARNING) << GetTraceName() << ": sendFragment_: WRITE ERROR: " << strerror(errno);
726  connect_state = 0; // any write error closes
727  close(send_fd_);
728  send_fd_ = -1;
730  }
731  else if (sts != this_write_bytes)
732  {
733  // we'll loop around -- with
734  TLOG(TLVL_DEBUG) << GetTraceName() << ": sendFragment writev sts(" << sts << ")!=requested_send_bytes(" << this_write_bytes << ")";
735  total_written_bytes += sts; // add sts to total_written_bytes now as sts is adjusted next
736  // find which iovs are done
737  for (ii = 0; (size_t)sts >= iovv[ii].iov_len; ++ii)
738  sts -= iovv[ii].iov_len;
739  in_iov_idx += ii; // done with these in_iovs
740  iovv[ii].iov_len -= sts; // adjust partial iov
741  iovv[ii].iov_base = (uint8_t*)(iovv[ii].iov_base) + sts; // adjust partial iov
742 
743  // add more to get up to per_write_max_bytes
744  out_iov_idx = 0;
745  if (ii != 0)
746  iovv[out_iov_idx] = iovv[ii];
747  // starting over
748  this_write_bytes = iovv[out_iov_idx].iov_len;
749  // add any left over from appropriate in_iov_idx --
750  // i.e. match this out_iov with the in_iov that was used to
751  // initialize it; see how close the out base+len is to in base+len
752  // check !>per_write_max_bytes
753  unsigned long additional = ((unsigned long)iov_in[in_iov_idx].iov_base + iov_in[in_iov_idx].iov_len)
754  - ((unsigned long)iovv[out_iov_idx].iov_base + iovv[out_iov_idx].iov_len);
755  if (additional)
756  {
757  iovv[out_iov_idx].iov_len += additional;
758  this_write_bytes += additional;
759  if (this_write_bytes > per_write_max_bytes)
760  {
761  iovv[out_iov_idx].iov_len -= this_write_bytes - per_write_max_bytes;
762  this_write_bytes = per_write_max_bytes;
763  }
764  }
765  ++out_iov_idx; // done with
766  TLOG(TLVL_TRACE) << GetTraceName() << ": sendFragment writev sts!=: this_write_bytes=" << this_write_bytes
767  << " out_iov_idx=" << out_iov_idx
768  << " additional=" << additional
769  << " ii=" << ii;
770  }
771  else
772  {
773 #ifndef __OPTIMIZE__ // This can be an expensive TRACE call (even if disabled) due to multiplicity of calls
774  TLOG(TLVL_TRACE) << GetTraceName() << ": sendFragment writev sts(" << sts << ")==requested_send_bytes(" << this_write_bytes << ")";
775 #endif
776  total_written_bytes += sts;
777  --out_iov_idx; // make it the index of the last iovv
778  iovv[out_iov_idx].iov_base = (uint8_t*)(iovv[out_iov_idx].iov_base) + iovv[out_iov_idx].iov_len;
779  iovv[out_iov_idx].iov_len = 0;
780  in_iov_idx += out_iov_idx; // at least this many complete (one more if "last iovv" is complete
781  this_write_bytes = 0;
782  // need to check last iovv against appropriate iov_in
783  unsigned long additional = ((unsigned long)iov_in[in_iov_idx].iov_base + iov_in[in_iov_idx].iov_len)
784  - ((unsigned long)iovv[out_iov_idx].iov_base + iovv[out_iov_idx].iov_len);
785  if (additional)
786  {
787  iovv[out_iov_idx].iov_len += additional;
788  this_write_bytes += additional;
789  if (this_write_bytes > per_write_max_bytes)
790  {
791  iovv[out_iov_idx].iov_len -= this_write_bytes - per_write_max_bytes;
792  this_write_bytes = per_write_max_bytes;
793  }
794  if (out_iov_idx != 0)
795  iovv[0] = iovv[out_iov_idx];
796  out_iov_idx = 1;
797  }
798  else
799  {
800  ++in_iov_idx;
801  out_iov_idx = 0;
802  }
803  }
804  } while (total_written_bytes < total_to_write_bytes);
805  if (total_written_bytes > total_to_write_bytes)
806  TLOG(TLVL_ERROR) << GetTraceName() << ": sendFragment program error: too many bytes transferred";
807 
808  if (blocking)
809  {
810  blocking = false;
811  fcntl(send_fd_, F_SETFL, O_NONBLOCK); // set O_NONBLOCK
812  }
813  sts = total_written_bytes - sizeof(MessHead);
814 
815  TLOG(14) << GetTraceName() << ": sendFragment sts=" << sts;
817 }
818 
819 void artdaq::TCPSocketTransfer::connect_()
820 {
821  auto start_time = std::chrono::steady_clock::now();
822 
823  // Retry a few times if we can't connect
824  while (send_fd_ == -1 && TimeUtils::GetElapsedTimeMicroseconds(start_time) < send_retry_timeout_us_ * 10)
825  {
826  TLOG(TLVL_DEBUG) << GetTraceName() << ": Connecting sender socket";
827  int sndbuf_bytes = static_cast<int>(sndbuf_);
828  if (sndbuf_ > INT_MAX)
829  {
830  sndbuf_bytes = INT_MAX;
831  TLOG(TLVL_WARNING) << "Requested SNDBUF " << sndbuf_ << " too large, setting to INT_MAX: " << INT_MAX;
832  }
833  TLOG(TLVL_DEBUG) << "Requested SNDBUF is " << sndbuf_bytes;
834 
835  send_fd_ = TCPConnect(hostMap_[destination_rank()].c_str()
836  , portMan->GetTCPSocketTransferPort(destination_rank())
837  , O_NONBLOCK
838  , sndbuf_bytes);
839  if (send_fd_ == -1)
840  usleep(send_retry_timeout_us_);
841  }
842  connect_state = 0;
843  blocking = 0;
844  TLOG(TLVL_DEBUG) << GetTraceName() << ": connect_ " + hostMap_[destination_rank()] + ":" << portMan->GetTCPSocketTransferPort(destination_rank()) << " send_fd_=" << send_fd_;
845  if (send_fd_ != -1)
846  {
847  // write connect msg
848  TLOG(TLVL_DEBUG) << GetTraceName() << ": connect_: Writing connect message";
849  MessHead mh = { 0,MessHead::connect_v0,htons(source_rank()),{htonl(CONN_MAGIC)} };
850  ssize_t sts = write(send_fd_, &mh, sizeof(mh));
851  if (sts == -1)
852  {
853  TLOG(TLVL_ERROR) << GetTraceName() << ": connect_: Error writing connect message!";
854  // a write error here is completely unexpected!
855  connect_state = 0;
856  close(send_fd_);
857  send_fd_ = -1;
858  }
859  else
860  {
861  TLOG(TLVL_INFO) << GetTraceName() << ": connect_: Successfully connected";
862  // consider it all connected/established
863  connect_state = 1;
864  }
865 
866 #if USE_ACKS
867  if (ack_listen_thread_ && ack_listen_thread_->joinable()) ack_listen_thread_->join();
868  TLOG(TLVL_INFO) << GetTraceName() << ": Starting Ack Listener Thread";
869 
870  try {
871  ack_listen_thread_ = std::make_unique<boost::thread>(&TCPSocketTransfer::receive_acks_, this);
872  }
873  catch (const boost::exception& e)
874  {
875  TLOG(TLVL_ERROR) << "Caught boost::exception starting TCP Socket Ack Listen thread: " << boost::diagnostic_information(e) << ", errno=" << errno;
876  std::cerr << "Caught boost::exception starting TCP Socket Ack Listen thread: " << boost::diagnostic_information(e) << ", errno=" << errno << std::endl;
877  exit(5);
878  }
879 #endif
880  }
881  }
882 
883 void artdaq::TCPSocketTransfer::reconnect_()
884 {
885  if (send_fd_ == -1 && role() == TransferInterface::Role::kSend)
886  {
887  TLOG(TLVL_TRACE) << GetTraceName() << ": check/reconnect";
888  return connect_();
889  }
890 }
891 
892 void artdaq::TCPSocketTransfer::start_listen_thread_()
893 {
894  std::unique_lock<std::mutex> start_lock(listen_thread_mutex_);
895  if (listen_thread_refcount_ == 0)
896  {
897  if (listen_thread_ && listen_thread_->joinable()) listen_thread_->join();
898  listen_thread_refcount_ = 1;
899  TLOG(TLVL_INFO) << GetTraceName() << ": Starting Listener Thread";
900 
901  try {
902  listen_thread_ = std::make_unique<boost::thread>(&TCPSocketTransfer::listen_, portMan->GetTCPSocketTransferPort(destination_rank()), rcvbuf_);
903  }
904  catch (const boost::exception& e)
905  {
906  TLOG(TLVL_ERROR) << "Caught boost::exception starting TCP Socket Listen thread: " << boost::diagnostic_information(e) << ", errno=" << errno;
907  std::cerr << "Caught boost::exception starting TCP Socket Listen thread: " << boost::diagnostic_information(e) << ", errno=" << errno << std::endl;
908  exit(5);
909  }
910  }
911  else
912  {
913  listen_thread_refcount_++;
914  }
915 }
916 
917 #if USE_ACKS
918 void artdaq::TCPSocketTransfer::receive_acks_()
919 {
920  while (send_fd_ >= 0)
921  {
922  pollfd pollfd_s;
923  pollfd_s.events = POLLIN | POLLPRI;
924  pollfd_s.fd = send_fd_;
925 
926  TLOG(18) << GetTraceName() << ": receive_acks_: Polling fd to see if there's data";
927  int num_fds_ready = poll(&pollfd_s, 1, 1000);
928  if (num_fds_ready <= 0)
929  {
930  if (num_fds_ready == 0)
931  {
932  TLOG(18) << GetTraceName() << ": receive_acks_: No data on receive socket";
933  continue;
934  }
935 
936  TLOG(TLVL_ERROR) << "Error in poll: errno=" << errno;
937  break;
938  }
939 
940  if (pollfd_s.revents & (POLLIN | POLLPRI))
941  {
942  // Expected, don't have to check revents any further
943  }
944  else
945  {
946  TLOG(TLVL_DEBUG) << GetTraceName() << ": receive_acks_: Wrong event received from pollfd: " << pollfd_s.revents;
947  break;
948  }
949 
950  MessHead mh;
951  auto sts = read(send_fd_, &mh, sizeof(mh));
952 
953  if (sts != sizeof(mh))
954  {
955  TLOG(TLVL_ERROR) << GetTraceName() << ": receive_ack_: Wrong message header length received! (actual " << sts << " != " << sizeof(mh) << " expected)";
956  continue;
957  }
958 
959  // check for "magic" and valid source_id(aka rank)
960  mh.source_id = ntohs(mh.source_id); // convert here as it is reference several times
961  if (mh.source_id != my_rank)
962  {
963  TLOG(TLVL_ERROR) << GetTraceName() << ": receive_ack_: Received ack for different sender! Rank=" << my_rank << ", hdr=" << mh.source_id;
964  continue;
965  }
966  if (ntohl(mh.conn_magic) != ACK_MAGIC || !(mh.message_type == MessHead::ack_v0)) // Allow for future connect message versions
967  {
968  TLOG(TLVL_ERROR) << GetTraceName() << ": receive_ack_: Wrong magic bytes in header!";
969  continue;
970  }
971 
972  TLOG(17) << GetTraceName() << ": receive_acks_: Received ack message, diff is now " << (send_ack_diff_.load() - 1);
973  send_ack_diff_--;
974 }
975 }
976 
977 void artdaq::TCPSocketTransfer::send_ack_(int fd)
978 {
979  MessHead mh = { 0,MessHead::ack_v0,htons(source_rank()),{ htonl(ACK_MAGIC) } };
980  write(fd, &mh, sizeof(mh));
981 }
982 #endif
983 
984 void artdaq::TCPSocketTransfer::listen_(int port, size_t rcvbuf)
985 {
986  int listen_fd = -1;
987  while (listen_thread_refcount_ > 0)
988  {
989  TLOG(TLVL_TRACE) << "listen_: Listening/accepting new connections on port " << port;
990  if (listen_fd == -1)
991  {
992  TLOG(TLVL_DEBUG) << "listen_: Opening listener";
993  listen_fd = TCP_listen_fd(port, rcvbuf);
994  }
995  if (listen_fd == -1)
996  {
997  TLOG(TLVL_DEBUG) << "listen_: Error creating listen_fd!";
998  break;
999  }
1000 
1001  int res;
1002  timeval tv = { 2,0 }; // maybe increase of some global "debugging" flag set???
1003  fd_set rfds;
1004  FD_ZERO(&rfds);
1005  FD_SET(listen_fd, &rfds);
1006 
1007  res = select(listen_fd + 1, &rfds, (fd_set *)0, (fd_set *)0, &tv);
1008  if (res > 0)
1009  {
1010  int sts;
1011  sockaddr_un un;
1012  socklen_t arglen = sizeof(un);
1013  int fd;
1014  TLOG(TLVL_DEBUG) << "listen_: Calling accept";
1015  fd = accept(listen_fd, (sockaddr *)&un, &arglen);
1016  TLOG(TLVL_DEBUG) << "listen_: Done with accept";
1017 
1018  TLOG(TLVL_DEBUG) << "listen_: Reading connect message";
1019  socklen_t lenlen = sizeof(tv);
1020  /*sts=*/
1021  setsockopt(fd, SOL_SOCKET, SO_RCVTIMEO, &tv, lenlen); // see man 7 socket.
1022  MessHead mh;
1023  uint64_t mark_us = TimeUtils::gettimeofday_us();
1024  sts = read(fd, &mh, sizeof(mh));
1025  uint64_t delta_us = TimeUtils::gettimeofday_us() - mark_us;
1026  TLOG(TLVL_DEBUG) << "listen_: Read of connect message took " << delta_us << " microseconds.";
1027  if (sts != sizeof(mh))
1028  {
1029  TLOG(TLVL_DEBUG) << "listen_: Wrong message header length received!";
1030  close(fd);
1031  continue;
1032  }
1033 
1034  // check for "magic" and valid source_id(aka rank)
1035  mh.source_id = ntohs(mh.source_id); // convert here as it is reference several times
1036  if (ntohl(mh.conn_magic) != CONN_MAGIC || !(mh.message_type == MessHead::connect_v0)) // Allow for future connect message versions
1037  {
1038  TLOG(TLVL_DEBUG) << "listen_: Wrong magic bytes in header!";
1039  close(fd);
1040  continue;
1041  }
1042 
1043  // now add (new) connection
1044  std::unique_lock<std::mutex> lk(connected_fd_mutex_);
1045  connected_fds_[mh.source_id].insert(fd);
1046 
1047  TLOG(TLVL_INFO) << "listen_: New fd is " << fd << " for source rank " << mh.source_id;
1048  }
1049  else
1050  {
1051  TLOG(16) << "listen_: No connections in timeout interval!";
1052  }
1053  }
1054 
1055  TLOG(TLVL_INFO) << "listen_: Shutting down connection listener";
1056  if (listen_fd != -1) close(listen_fd);
1057  std::unique_lock<std::mutex> lk(connected_fd_mutex_);
1058  auto it = connected_fds_.begin();
1059  while (it != connected_fds_.end())
1060  {
1061  auto& fd_set = it->second;
1062  auto rank_it = fd_set.begin();
1063  while (rank_it != fd_set.end())
1064  {
1065  close(*rank_it);
1066  rank_it = fd_set.erase(rank_it);
1067  }
1068  it = connected_fds_.erase(it);
1069  }
1070 
1071 } // do_connect_
1072 
1073 DEFINE_ARTDAQ_TRANSFER(artdaq::TCPSocketTransfer)
bool isRunning() override
Determine whether the TransferInterface plugin is able to send/receive data.
virtual int source_rank() const
Get the source rank for this TransferInterface instance.
int TCPConnect(char const *host_in, int dflt_port, long flags=0, int sndbufsiz=0)
Connect to a host on a given port.
Definition: TCPConnect.cc:358
uint32_t conn_magic
unsigned first is better for MessHead initializer: {0,0,my_node_idx_,CONN_MAGIC}
Definition: SRSockets.hh:40
hostMap_t MakeHostMap(fhicl::ParameterSet pset, hostMap_t map=hostMap_t())
Make a hostMap_t from a HostMap::Config ParameterSet
Definition: HostMap.hh:66
This TransferInterface is a Receiver.
int receiveFragmentData(RawDataType *destination, size_t wordCount) override
Receive the body of a Fragment to the given destination pointer.
int TCP_listen_fd(int port, int rcvbuf)
Create a TCP listening socket on the given port and INADDR_ANY, with the given receive buffer...
TCPSocketTransfer(fhicl::ParameterSet const &ps, Role role)
TCPSocketTransfer Constructor.
int receiveFragmentHeader(detail::RawFragmentHeader &header, size_t receiveTimeout) override
Receive a Fragment Header from the transport mechanism.
This TransferInterface is a Sender.
void flush_buffers() override
Flush any in-flight data. This should be used by the receiver after the receive loop has ended...
int32_t byte_count
use CONN_MAGIC for connect_v0, data that follow for data_v0 (and 0 lenght data)
Definition: SRSockets.hh:41
Some error occurred, but no exception was thrown.
Role
Used to determine if a TransferInterface is a Sender or Receiver.
int64_t source_id
Rank of the source.
Definition: SRSockets.hh:36
MessType message_type
Message Type.
Definition: SRSockets.hh:35
The send operation completed successfully.
This interface defines the functions used to transfer data between artdaq applications.
TransferInterface implementation plugin that sends data using TCP sockets.
This header is sent by the TCPSocket_transfer to allow for more efficient writev calls.
Definition: SRSockets.hh:15
CopyStatus
Returned from the send functions, this enumeration describes the possible return codes. If an exception occurs, it will be thrown and should be handled normally.