artdaq  v3_03_00
TCPSocket_transfer.cc
1 // Sep 14, 2016. "TERMS AND CONDITIONS" governing this file are in the README
2 // or COPYING file. If you do not have such a file, one can be obtained by
3 // contacting Ron or Fermi Lab in Batavia IL, 60510, phone: 630-840-3000.
4 // $RCSfile: .emacs.gnu,v $
5 // rev="$Revision: 1.30 $$Date: 2016/03/01 14:27:27 $";
6 
7 // C Includes
8 #include <stdlib.h> // atoi, strtoul
9 #include <sys/socket.h> // socket, socklen_t
10 #include <sys/un.h> // sockaddr_un
11 #include <arpa/inet.h> // ntohl, ntohs
12 #include <sys/types.h> // size_t
13 #include <poll.h> // struct pollfd
14 
15 // C++ Includes
16 #include <string>
17 #include <fstream>
18 #include <stdexcept>
19 
20 // product Includes
21 #define TRACE_NAME (app_name + "_TCPSocketTransfer").c_str()
22 #include "artdaq/DAQdata/Globals.hh"
23 
24 // artdaq Includes
25 #include "artdaq/TransferPlugins/TCPSocketTransfer.hh"
28 #include "artdaq/TransferPlugins/detail/Timeout.hh"
29 #include "artdaq/TransferPlugins/detail/SRSockets.hh"
30 #include "artdaq-core/Data/Fragment.hh"
31 #include "artdaq-core/Utilities/TimeUtils.hh"
32 #include <iomanip>
33 
34 std::atomic<int> artdaq::TCPSocketTransfer::listen_thread_refcount_(0);
35 std::unique_ptr<boost::thread> artdaq::TCPSocketTransfer::listen_thread_ = nullptr;
36 std::map<int, std::set<int>> artdaq::TCPSocketTransfer::connected_fds_ = std::map<int, std::set<int>>();
37 std::mutex artdaq::TCPSocketTransfer::listen_thread_mutex_;
38 std::mutex artdaq::TCPSocketTransfer::connected_fd_mutex_;
39 
41 TCPSocketTransfer(fhicl::ParameterSet const& pset, TransferInterface::Role role)
42  : TransferInterface(pset, role)
43  , send_fd_(-1)
44  , active_receive_fd_(-1)
45  , last_active_receive_fd_(-1)
46  , rcvbuf_(pset.get<size_t>("tcp_receive_buffer_size", 0))
47  , sndbuf_(pset.get<size_t>("tcp_send_buffer_size", max_fragment_size_words_ * sizeof(artdaq::RawDataType) * buffer_count_))
48  , send_retry_timeout_us_(pset.get<size_t>("send_retry_timeout_us", 1000000))
49  , timeoutMessageArmed_(true)
50  , last_recv_time_()
51  , receive_disconnected_wait_s_(pset.get<double>("receive_socket_disconnected_wait_s", 10.0))
52  , receive_err_wait_us_(pset.get<size_t>("receive_socket_disconnected_wait_us", 10000))
53  , receive_socket_has_been_connected_(false)
54  , send_ack_diff_(0)
55 {
56  TLOG(TLVL_DEBUG) << GetTraceName() << " Constructor: pset=" << pset.to_string() << ", role=" << (role == TransferInterface::Role::kReceive ? "kReceive" : "kSend");
57 
59  {
60  // Wait for sender to connect...
61  TLOG(TLVL_DEBUG) << GetTraceName() << ": Listening for connections";
62  start_listen_thread_();
63  TLOG(TLVL_DEBUG) << GetTraceName() << ": Done Listening";
64  }
65  else
66  {
67  hostMap_ = MakeHostMap(pset);
68  TLOG(TLVL_DEBUG) << GetTraceName() << ": Connecting to destination";
69  connect_();
70  TLOG(TLVL_DEBUG) << GetTraceName() << ": Done Connecting";
71  }
72  TLOG(TLVL_DEBUG) << GetTraceName() << ": End of Constructor";
73 }
74 
75 artdaq::TCPSocketTransfer::~TCPSocketTransfer() noexcept
76 {
77  TLOG(TLVL_DEBUG) << GetTraceName() << ": Shutting down TCPSocketTransfer";
78 
79  if (role() == TransferInterface::Role::kSend)
80  {
81  // close all open connections (send stop_v0) first
82  MessHead mh = { 0,MessHead::stop_v0,htons(TransferInterface::source_rank()),{0} };
83  if (send_fd_ != -1)
84  {
85  // should be blocking with modest timeo
86  timeval tv = { 0,100000 };
87  socklen_t len = sizeof(tv);
88  setsockopt(send_fd_, SOL_SOCKET, SO_SNDTIMEO, &tv, len);
89  write(send_fd_, &mh, sizeof(mh));
90  }
91  close(send_fd_);
92  send_fd_ = -1;
93  }
94  else
95  {
96  {
97  std::unique_lock<std::mutex> fd_lock(connected_fd_mutex_);
98  if (connected_fds_.count(source_rank()))
99  {
100  auto it = connected_fds_[source_rank()].begin();
101  while (it != connected_fds_[source_rank()].end())
102  {
103  close(*it);
104  it = connected_fds_[source_rank()].erase(it);
105  }
106  connected_fds_.erase(source_rank());
107  }
108  if (ack_listen_thread_ && ack_listen_thread_->joinable()) ack_listen_thread_->join();
109  }
110 
111  std::unique_lock<std::mutex> lk(listen_thread_mutex_);
112  listen_thread_refcount_--;
113  if (listen_thread_refcount_ <= 0 && listen_thread_ && listen_thread_->joinable())
114  {
115  listen_thread_->join();
116  }
117  }
118 
119  TLOG(TLVL_DEBUG) << GetTraceName() << ": End of Destructor";
120 }
121 
122 int artdaq::TCPSocketTransfer::receiveFragmentHeader(detail::RawFragmentHeader& header, size_t timeout_usec)
123 {
124  TLOG(5) << GetTraceName() << ": receiveFragmentHeader: BEGIN";
125  int ret_rank = RECV_TIMEOUT;
126 
127  // Don't bomb out until received at least one connection...
128  if (getConnectedFDCount(source_rank()) == 0)
129  { // what if just listen_fd???
130  // if (receive_socket_has_been_connected_ && TimeUtils::GetElapsedTime(last_recv_time_) > receive_disconnected_wait_s_)
131  // {
132  // TLOG(TLVL_ERROR) << GetTraceName() << ": receiveFragmentHeader: senders have been disconnected for "
133  // << TimeUtils::GetElapsedTime(last_recv_time_) << " s (receive_socket_disconnected_wait_s = " << receive_disconnected_wait_s_ << " s). RETURNING DATA_END!";
134  // return DATA_END;
135  // }
136  //if (++not_connected_count_ > receive_err_threshold_) { return DATA_END; }
137  TLOG(7) << GetTraceName() << ": receiveFragmentHeader: Receive socket not connected, returning RECV_TIMEOUT";
138  usleep(receive_err_wait_us_);
139  return RECV_TIMEOUT;
140  }
141  receive_socket_has_been_connected_ = true;
142  last_recv_time_ = std::chrono::steady_clock::now();
143 
144  TLOG(5) << GetTraceName() << ": receiveFragmentHeader timeout_usec=" << timeout_usec;
145  //void* buff=alloca(max_fragment_size_words_*8);
146  size_t byte_cnt = 0;
147  int sts;
148  int offset = 0;
149  SocketState state = SocketState::Metadata;
150  int target_bytes = sizeof(MessHead);
151  uint64_t start_time_us = TimeUtils::gettimeofday_us();
152 
153  //while (active_receive_fd_ != -1)
154  //{
155  // TLOG(TLVL_TRACE) << GetTraceName() << ": Currently receiving from fd " << active_receive_fd_ << ", waiting!";
156  // usleep(1000);
157  //}
158 
159 
160  uint8_t* buff;
161 
162  int timeout_ms;
163  if (timeout_usec == 0)
164  timeout_ms = 0;
165  else
166  timeout_ms = (timeout_usec + 999) / 1000; // want at least 1 ms
167 
168  bool done = false;
169  bool noDataWarningSent = false;
170  int loop_guard = 0;
171 
172  while (!done && getConnectedFDCount(source_rank()) > 0)
173  {
174  if (active_receive_fd_ == -1)
175  {
176  loop_guard = 0;
177  size_t fd_count = 0;
178  std::vector<pollfd> pollfds;
179  {
180  std::unique_lock<std::mutex> lk(connected_fd_mutex_);
181  fd_count = connected_fds_[source_rank()].size();
182  pollfds.resize(fd_count);
183  auto iter = connected_fds_[source_rank()].begin();
184  for (size_t ii = 0; ii < fd_count; ++ii)
185  {
186  pollfds[ii].events = POLLIN | POLLPRI | POLLERR;
187  pollfds[ii].fd = *iter;
188  ++iter;
189  }
190  }
191  //TLOG(TLVL_DEBUG) << GetTraceName() << ": receiveFragment: Polling fd to see if there's data" ;
192  int num_fds_ready = poll(&pollfds[0], fd_count, timeout_ms);
193  if (num_fds_ready <= 0)
194  {
195  TLOG(5) << GetTraceName() << ": receiveFragmentHeader: No data on receive socket, returning RECV_TIMEOUT";
196  return RECV_TIMEOUT;
197  }
198 
199  size_t index = 0;
200  if (last_active_receive_fd_ != -1)
201  {
202  for (auto& pollfd : pollfds)
203  {
204  index++;
205  if (pollfd.fd == last_active_receive_fd_)
206  {
207  break;
208  }
209  }
210  }
211 
212  int active_index = -1;
213  short anomolous_events = 0;
214  for (size_t ii = index; ii < index + pollfds.size(); ++ii)
215  {
216  auto pollfd_index = (ii + index) % pollfds.size();
217  if (pollfds[pollfd_index].revents & (POLLIN | POLLPRI))
218  {
219  active_index = pollfd_index;
220  active_receive_fd_ = pollfds[active_index].fd;
221  active_revents_ = pollfds[active_index].revents;;
222  break;
223  }
224  else if (pollfds[pollfd_index].revents & (POLLHUP | POLLERR))
225  {
226  disconnect_receive_socket_(pollfds[pollfd_index].fd, "Poll returned POLLHUP or POLLERR, indicating problems with the sender.");
227  continue;
228  }
229  else if (pollfds[pollfd_index].revents & (POLLNVAL))
230  {
231  TLOG(TLVL_DEBUG) << GetTraceName() << ": receiveFragmentHeader: FD is closed, most likely because the peer went away. Removing from fd list.";
232  disconnect_receive_socket_(pollfds[pollfd_index].fd, "FD is closed, most likely because the peer went away.");
233  continue;
234  }
235  else if (pollfds[pollfd_index].revents)
236  {
237  anomolous_events |= pollfds[pollfd_index].revents;
238  }
239  }
240 
241  if (active_index == -1)
242  {
243  if (anomolous_events)
244  TLOG(TLVL_DEBUG) << GetTraceName() << ": receiveFragmentHeader: Wrong event received from a pollfd. Mask: " << static_cast<int>(anomolous_events);
245  active_receive_fd_ = -1;
246  continue;
247  }
248 
249  if (!done && timeout_usec > 0)
250  {
251  // calc next timeout_ms (unless timed out)
252  size_t delta_us = TimeUtils::gettimeofday_us() - start_time_us;
253  if (delta_us > timeout_usec)
254  {
255  return RECV_TIMEOUT;
256  }
257  timeout_ms = ((timeout_usec - delta_us) + 999) / 1000; // want at least 1 ms
258  }
259  }
260  if (loop_guard > 10) { usleep(1000); }
261  if (++loop_guard > 10010)
262  {
263  TLOG(TLVL_WARNING) << GetTraceName() << ": receiveFragmentHeader: loop guard triggered, returning RECV_TIMEOUT";
264  usleep(receive_err_wait_us_);
265  active_receive_fd_ = -1;
266  return RECV_TIMEOUT;
267  }
268 
269  if (state == SocketState::Metadata)
270  {
271  //TLOG(TLVL_DEBUG) << GetTraceName() << ": receiveFragmentHeader: Reading Message Header" ;
272  buff = &(mha[offset]);
273  byte_cnt = sizeof(MessHead) - offset;
274  }
275  else
276  {
277  //TLOG(TLVL_DEBUG) << GetTraceName() << ": receiveFragmentHeader: Reading data" ;
278  buff = reinterpret_cast<uint8_t*>(&header) + offset;
279  byte_cnt = target_bytes - offset;
280  }
281  //if (byte_cnt > sizeof(MessHead))
282  // {
283  // TLOG(TLVL_ERROR) << "Invalid byte count for read (count=" << byte_cnt
284  // << ",offset=" << offset << ",mh.byte_count=" << mh.byte_count
285  // << "), skipping read and returning RECV_TIMEOUT";
286  // return RECV_TIMEOUT;
287  //}
288 
289  if (byte_cnt > 0)
290  {
291  TLOG(6) << GetTraceName() << ": receiveFragmentHeader: Reading " << byte_cnt << " bytes from socket";
292  sts = read(active_receive_fd_, buff, byte_cnt);
293  TLOG(6) << GetTraceName() << ": receiveFragmentHeader: Done with read";
294  }
295  if (sts > 0) {
296  loop_guard = 0;
297  last_recv_time_ = std::chrono::steady_clock::now();
298  }
299 
300  TLOG(7) << GetTraceName() << ": receiveFragmentHeader state=" << static_cast<int>(state) << " read=" << sts;
301  if (sts < 0)
302  {
303  TLOG(TLVL_WARNING) << GetTraceName() << ": receiveFragmentHeader: Error on receive, closing socket " << " (errno=" << errno << ": " << strerror(errno) << ")";
304  active_receive_fd_ = disconnect_receive_socket_(active_receive_fd_);
305  }
306  else if (sts == 0)
307  {
308  if (!noDataWarningSent) {
309  TLOG(TLVL_WARNING) << GetTraceName() << ": receiveFragmentHeader: No data received, is the sender still sending?!?";
310  noDataWarningSent = true;
311  }
312  if (TimeUtils::GetElapsedTime(last_recv_time_) > receive_disconnected_wait_s_)
313  {
314  TLOG(TLVL_ERROR) << GetTraceName() << ": receiveFragmentHeader: No data received within timeout, aborting!";
315  return RECV_TIMEOUT;
316  }
317  }
318  else
319  {
320  // see if we're done (with this state)
321  sts = offset += sts;
322  if (sts >= target_bytes)
323  {
324  TLOG(7) << GetTraceName() << ": receiveFragmentHeader: Target read bytes reached. Changing state";
325  offset = 0;
326  if (state == SocketState::Metadata)
327  {
328  state = SocketState::Data;
329  mh.byte_count = ntohl(mh.byte_count);
330  mh.source_id = ntohs(mh.source_id);
331  target_bytes = mh.byte_count;
332  TLOG(7) << GetTraceName() << ": receiveFragmentHeader: Expected header size = " << target_bytes << ", sizeof(RawFragmentHeader) = " << sizeof(artdaq::detail::RawFragmentHeader);
333  //assert(target_bytes == sizeof(artdaq::detail::RawFragmentHeader) || target_bytes == 0);
334 
335  if (mh.message_type == MessHead::stop_v0)
336  {
337  active_receive_fd_ = disconnect_receive_socket_(active_receive_fd_, "Stop Message received.");
338  }
339  else if (mh.message_type == MessHead::data_v0 || mh.message_type == MessHead::data_more_v0)
340  {
341  TLOG(TLVL_WARNING) << GetTraceName() << ": receiveFragmentHeader: Message header indicates that Fragment data follows when I was expecting a Fragment header!";
342  active_receive_fd_ = disconnect_receive_socket_(active_receive_fd_, "Desync detected");
343  }
344 
345  if (target_bytes == 0)
346  {
347  //Probably a stop_v0, return timeout so we can try again.
348  return RECV_TIMEOUT;
349  }
350  }
351  else
352  {
353  ret_rank = source_rank();
354  TLOG(8) << GetTraceName() << ": receiveFragmentHeader done sts=" << sts << " src=" << ret_rank;
355  TLOG(7) << GetTraceName() << ": receiveFragmentHeader: Done receiving fragment header. Moving into output.";
356 
357  done = true; // no more polls
358  //break; // no more read of ready fds
359  }
360  }
361  }
362 
363  } // while(!done)...poll
364 
365  TLOG(5) << GetTraceName() << ": receiveFragmentHeader: Returning " << ret_rank;
366  return ret_rank;
367 }
368 
369 int artdaq::TCPSocketTransfer::disconnect_receive_socket_(int fd, std::string msg)
370 {
371  TLOG(TLVL_WARNING) << GetTraceName() << ": disconnect_receive_socket_: " << msg << " Closing socket " << fd;
372  close(fd);
373  std::unique_lock<std::mutex> lk(connected_fd_mutex_);
374  if (connected_fds_.count(source_rank()))
375  connected_fds_[source_rank()].erase(fd);
376  fd = -1;
377  TLOG(TLVL_DEBUG) << GetTraceName() << ": disconnect_receive_socket_: There are now " << connected_fds_[source_rank()].size() << " active senders.";
378  return fd;
379 }
380 
381 int artdaq::TCPSocketTransfer::receiveFragmentData(RawDataType* destination, size_t)
382 {
383  TLOG(9) << GetTraceName() << ": receiveFragmentData: BEGIN";
384  int ret_rank = RECV_TIMEOUT;
385  if (active_receive_fd_ == -1)
386  { // what if just listen_fd???
387  TLOG(TLVL_ERROR) << GetTraceName() << ": receiveFragmentData: Receive socket not connected, returning RECV_TIMEOUT (Will result in \"Unexpected return code error\")";
388  return RECV_TIMEOUT;
389  }
390 
391  //void* buff=alloca(max_fragment_size_words_*8);
392  uint8_t* buff;
393  size_t byte_cnt = 0;
394  int sts;
395  int offset = 0;
396  SocketState state = SocketState::Metadata;
397  int target_bytes = sizeof(MessHead);
398 
399  pollfd pollfd_s;
400  pollfd_s.events = POLLIN | POLLPRI | POLLERR;
401  pollfd_s.fd = active_receive_fd_;
402 
403  int loop_guard = 0;
404  bool done = false;
405  bool noDataWarningSent = false;
406  last_recv_time_ = std::chrono::steady_clock::now();
407  while (!done)
408  {
409  TLOG(9) << GetTraceName() << ": receiveFragmentData: Polling fd to see if there's data";
410  int num_fds_ready = poll(&pollfd_s, 1, 1000);
411  TLOG(TLVL_TRACE) << GetTraceName() << ": receiveFragmentData: Polled fd to see if there's data"
412  << ", num_fds_ready = " << num_fds_ready;
413  if (num_fds_ready <= 0)
414  {
415  if (num_fds_ready == 0)
416  {
417  TLOG(TLVL_WARNING) << GetTraceName() << ": receiveFragmentData: No data from " << source_rank() << " in " << TimeUtils::GetElapsedTimeMilliseconds(last_recv_time_) << " ms!"
418  << " State = " << (state == SocketState::Metadata ? "Metadata" : "Data") << ", recvd/total=" << offset << "/" << target_bytes << " (delta=" << target_bytes - offset << ")";
419 
420  if (TimeUtils::GetElapsedTime(last_recv_time_) > receive_disconnected_wait_s_)
421  {
422  TLOG(TLVL_WARNING) << GetTraceName() << ": receiveFragmentData: No data received within timeout (" << TimeUtils::GetElapsedTime(last_recv_time_) << " / " << receive_disconnected_wait_s_ << " ), returning RECV_TIMEOUT";
423  disconnect_receive_socket_(active_receive_fd_, "No data on this socket within timeout");
424  active_receive_fd_ = -1;
425  return RECV_TIMEOUT;
426  }
427  continue;
428  }
429 
430  TLOG(TLVL_ERROR) << "Error in poll: errno=" << errno;
431  active_receive_fd_ = -1;
432  break;
433  }
434  else { last_recv_time_ = std::chrono::steady_clock::now(); }
435 
436  if (pollfd_s.revents & (POLLIN | POLLPRI))
437  {
438  // Expected, don't have to check revents any further
439  }
440  else if (pollfd_s.revents & (POLLNVAL))
441  {
442  disconnect_receive_socket_(pollfd_s.fd, "FD is closed, most likely because the peer went away.");
443  break;
444  }
445  else if (pollfd_s.revents & (POLLHUP | POLLERR))
446  {
447  disconnect_receive_socket_(pollfd_s.fd, "Poll returned POLLHUP or POLLERR, indicating problems with the sender.");
448  break;
449  }
450  else
451  {
452  TLOG(TLVL_WARNING) << GetTraceName() << ": receiveFragmentData: Wrong event received from pollfd: " << pollfd_s.revents;
453  disconnect_receive_socket_(pollfd_s.fd);
454  break;
455  }
456 
457  if (state == SocketState::Metadata)
458  {
459  //TLOG(TLVL_DEBUG) << GetTraceName() << ": receiveFragmentData: Reading Message Header" ;
460  buff = &(mha[offset]);
461  byte_cnt = sizeof(MessHead) - offset;
462  }
463  else
464  {
465  //TLOG(TLVL_DEBUG) << GetTraceName() << ": receiveFragmentData: Reading data" ;
466  buff = reinterpret_cast<uint8_t*>(destination) + offset;
467  byte_cnt = mh.byte_count - offset;
468  }
469 
470  TLOG(10) << GetTraceName() << ": receiveFragmentData: Reading " << byte_cnt << " bytes from socket into " << (void*)buff;
471  sts = read(active_receive_fd_, buff, byte_cnt);
472  //TLOG(TLVL_DEBUG) << GetTraceName() << ": receiveFragmentData: Done with read" ;
473 
474  TLOG(10) << GetTraceName() << ": recvFragment state=" << static_cast<int>(state) << " read=" << sts;
475 
476  if (sts == 0)
477  {
478  if (loop_guard > 10) { usleep(1000); }
479  if (++loop_guard > 10010)
480  {
481  TLOG(TLVL_WARNING) << GetTraceName() << ": receiveFragmentData: loop guard triggered, returning RECV_TIMEOUT";
482  active_receive_fd_ = -1;
483  return RECV_TIMEOUT;
484  }
485  }
486  else if(sts > 0)
487  {
488  loop_guard = 0;
489  last_recv_time_ = std::chrono::steady_clock::now();
490  }
491 
492  if (sts < 0)
493  {
494  TLOG(TLVL_WARNING) << GetTraceName() << ": receiveFragmentData: Error on receive, closing socket"
495  << " (errno=" << errno << ": " << strerror(errno) << ")";
496  disconnect_receive_socket_(pollfd_s.fd);
497  }
498  else if (sts == 0)
499  {
500  if (!noDataWarningSent) {
501  TLOG(TLVL_WARNING) << GetTraceName() << ": receiveFragmentData: No data received, is the sender still sending?!?";
502  noDataWarningSent = true;
503  }
504  if (TimeUtils::GetElapsedTime(last_recv_time_) > receive_disconnected_wait_s_)
505  {
506  TLOG(TLVL_ERROR) << GetTraceName() << ": receiveFragmentData: No data received within timeout, aborting!";
507  return RECV_TIMEOUT;
508  }
509  }
510  else
511  {
512  // see if we're done (with this state)
513  sts = offset += sts;
514  if (sts >= target_bytes)
515  {
516  TLOG(9) << GetTraceName() << ": receiveFragmentData: Target read bytes reached. Changing state";
517  offset = 0;
518  if (state == SocketState::Metadata)
519  {
520  state = SocketState::Data;
521  mh.byte_count = ntohl(mh.byte_count);
522  mh.source_id = ntohs(mh.source_id);
523  target_bytes = mh.byte_count;
524 
525 
526  if (mh.message_type == MessHead::header_v0)
527  {
528  TLOG(TLVL_WARNING) << GetTraceName() << ": receiveFragmentData: Message header indicates that a Fragment header follows when I was expecting Fragment data!";
529  active_receive_fd_ = disconnect_receive_socket_(active_receive_fd_, "Desync detected");
530  }
531  }
532  else
533  {
534  ret_rank = source_rank();
535  TLOG(11) << GetTraceName() << ": receiveFragmentData done sts=" << sts << " src=" << ret_rank;
536  TLOG(9) << GetTraceName() << ": receiveFragmentData: Done receiving fragment. Moving into output.";
537 
538 #if USE_ACKS
539  send_ack_(active_receive_fd_);
540 #endif
541 
542  done = true; // no more polls
543  //break; // no more read of ready fds
544  }
545  }
546  }
547 
548  // Check if we were asked to do a 0-size receive
549  if (target_bytes == 0 && state == SocketState::Data)
550  {
551  ret_rank = source_rank();
552  TLOG(11) << GetTraceName() << ": receiveFragmentData done sts=" << sts << " src=" << ret_rank;
553  TLOG(9) << GetTraceName() << ": receiveFragmentData: Done receiving fragment. Moving into output.";
554 
555 #if USE_ACKS
556  send_ack_(active_receive_fd_);
557 #endif
558 
559  done = true; // no more polls
560  }
561 
562  } // while(!done)...poll
563 
564  last_active_receive_fd_ = active_receive_fd_;
565  active_receive_fd_ = -1;
566 
567  TLOG(9) << GetTraceName() << ": receiveFragmentData: Returning " << ret_rank;
568  return ret_rank;
569 }
570 
572 {
573  switch (role())
574  {
576  return send_fd_ != -1;
578  TLOG(TLVL_DEBUG) << GetTraceName() << ": isRunning: There are " << getConnectedFDCount(source_rank()) << " fds connected.";
579  return getConnectedFDCount(source_rank()) > 0;
580  }
581  return false;
582 }
583 
584 // Send the given Fragment. Return the rank of the destination to which
585 // the Fragment was sent OR -1 if to none.
586 artdaq::TransferInterface::CopyStatus artdaq::TCPSocketTransfer::sendFragment_(Fragment&& frag, size_t send_timeout_usec)
587 {
588  TLOG(12) << GetTraceName() << ": sendFragment begin send of fragment with sequenceID="<<frag.sequenceID();
589  artdaq::Fragment grab_ownership_frag = std::move(frag);
590 
591  reconnect_();
592  // Send Fragment Header
593 
594 #if USE_ACKS
595  // Wait for fragments to be received
596  while (static_cast<size_t>(send_ack_diff_) > buffer_count_) usleep(10000);
597 #endif
598 
599  iovec iov = { reinterpret_cast<void*>(grab_ownership_frag.headerAddress()),
600  detail::RawFragmentHeader::num_words() * sizeof(RawDataType) };
601 
602  auto sts = sendData_(&iov, 1, send_retry_timeout_us_, true);
603  auto start_time = std::chrono::steady_clock::now();
604  //If it takes more than 10 seconds to write a Fragment header, give up
605  while (sts == CopyStatus::kTimeout && (send_timeout_usec == 0 || TimeUtils::GetElapsedTimeMicroseconds(start_time) < send_timeout_usec) && TimeUtils::GetElapsedTimeMicroseconds(start_time) < 10000000)
606  {
607  TLOG(13) << GetTraceName() << ": sendFragment: Timeout sending fragment";
608  sts = sendData_(&iov, 1, send_retry_timeout_us_, true);
609  usleep(1000);
610  }
611  if (sts != CopyStatus::kSuccess) return sts;
612 
613  // Send Fragment Data
614 
615  iov = { reinterpret_cast<void*>(grab_ownership_frag.headerAddress() + detail::RawFragmentHeader::num_words()),
616  grab_ownership_frag.sizeBytes() - detail::RawFragmentHeader::num_words() * sizeof(RawDataType)
617 };
618  sts = sendData_(&iov, 1, send_retry_timeout_us_);
619  start_time = std::chrono::steady_clock::now();
620  while (sts == CopyStatus::kTimeout && (send_timeout_usec == 0 || TimeUtils::GetElapsedTimeMicroseconds(start_time) < send_timeout_usec) && TimeUtils::GetElapsedTimeMicroseconds(start_time) < 10000000)
621  {
622  TLOG(13) << GetTraceName() << ": sendFragment: Timeout sending fragment";
623  sts = sendData_(&iov, 1, send_retry_timeout_us_);
624  usleep(1000);
625  }
626 
627 #if USE_ACKS
628  send_ack_diff_++;
629 #endif
630 
631  TLOG(12) << GetTraceName() << ": sendFragment returning " << CopyStatusToString(sts);
632  return sts;
633 }
634 
635 artdaq::TransferInterface::CopyStatus artdaq::TCPSocketTransfer::sendData_(const void* buf, size_t bytes, size_t send_timeout_usec, bool isHeader)
636 {
637  TLOG(TLVL_DEBUG) << GetTraceName() << ": sendData_ Converting buf to iovec";
638  iovec iov = { (void*)buf, bytes };
639  return sendData_(&iov, 1, send_timeout_usec, isHeader);
640 }
641 
642 artdaq::TransferInterface::CopyStatus artdaq::TCPSocketTransfer::sendData_(const struct iovec* iov, int iovcnt, size_t send_timeout_usec, bool isHeader)
643 {
644  // check all connected??? -- currently just check fd!=-1
645  if (send_fd_ == -1)
646  {
647  if (timeoutMessageArmed_)
648  {
649  TLOG(TLVL_DEBUG) << GetTraceName() << ": sendData_: Send fd is not open. Returning kTimeout";
650  timeoutMessageArmed_ = false;
651  }
652  return CopyStatus::kTimeout;
653  }
654  timeoutMessageArmed_ = true;
655  TLOG(14) << GetTraceName() << ": send_timeout_usec is " << send_timeout_usec << ", currently unused.";
656 
657  //TLOG(TLVL_DEBUG) << GetTraceName() << ": sendData_: Determining write size" ;
658  uint32_t total_to_write_bytes = 0;
659  std::vector<iovec> iov_in(iovcnt + 1); // need contiguous (for the unlike case that only partial MH
660  std::vector<iovec> iovv(iovcnt + 2); // 1 more for mh and another one for any partial
661  int ii;
662  for (ii = 0; ii < iovcnt; ++ii)
663  {
664  iov_in[ii + 1] = iov[ii];
665  total_to_write_bytes += iov[ii].iov_len;
666  }
667  //TLOG(TLVL_DEBUG) << GetTraceName() << ": sendData_: Constructing Message Header" ;
668  MessHead mh = { 0,isHeader ? MessHead::header_v0 : MessHead::data_v0,htons(source_rank()),{htonl(total_to_write_bytes)} };
669  iov_in[0].iov_base = &mh;
670  iov_in[0].iov_len = sizeof(mh);
671  total_to_write_bytes += sizeof(mh);
672 
673  ssize_t sts = 0;
674  ssize_t total_written_bytes = 0;
675  ssize_t per_write_max_bytes = (32 * 1024);
676 
677  size_t in_iov_idx = 0; // only increment this when we know the associated data has been xferred
678  size_t out_iov_idx = 0;
679  ssize_t this_write_bytes = 0;
680 
681  do
682  {
683  // The first out_iov may be set at the end of the previous loop.
684  // iov looping from below (b/c of the latter, we need to check this_write_bytes)
685  for (;
686  (in_iov_idx + out_iov_idx) < iov_in.size() && this_write_bytes < per_write_max_bytes;
687  ++out_iov_idx)
688  {
689  this_write_bytes += iov_in[in_iov_idx + out_iov_idx].iov_len;
690  iovv[out_iov_idx] = iov_in[in_iov_idx + out_iov_idx];
691  }
692  if (this_write_bytes > per_write_max_bytes)
693  {
694  iovv[out_iov_idx - 1].iov_len -= this_write_bytes - per_write_max_bytes;
695  this_write_bytes = per_write_max_bytes;
696  }
697 
698  // need to do blocking algorithm -- including throttled block notifications
699  do_again:
700 #ifndef __OPTIMIZE__ // This can be an expensive TRACE call (even if disabled) due to multiplicity of calls
701  TLOG(14) << GetTraceName() << ": sendFragment b4 writev " << std::setw(7) << total_written_bytes << " total_written_bytes send_fd_=" << send_fd_ << " in_idx=" << in_iov_idx
702  << " iovcnt=" << out_iov_idx << " 1st.len=" << iovv[0].iov_len;
703 #endif
704  //TLOG(TLVL_DEBUG) << GetTraceName() << " calling writev" ;
705  sts = writev(send_fd_, &(iovv[0]), out_iov_idx);
706  //TLOG(TLVL_DEBUG) << GetTraceName() << " done with writev" ;
707 
708  if (sts == -1)
709  {
710  if (errno == EAGAIN /* same as EWOULDBLOCK */)
711  {
712  TLOG(TLVL_DEBUG) << GetTraceName() << ": sendFragment EWOULDBLOCK";
713  fcntl(send_fd_, F_SETFL, 0); // clear O_NONBLOCK
714  blocking = true;
715  // NOTE: YES -- could drop here
716  goto do_again;
717  }
718  TLOG(TLVL_WARNING) << GetTraceName() << ": sendFragment_: WRITE ERROR: " << strerror(errno);
719  connect_state = 0; // any write error closes
720  close(send_fd_);
721  send_fd_ = -1;
723  }
724  else if (sts != this_write_bytes)
725  {
726  // we'll loop around -- with
727  TLOG(TLVL_DEBUG) << GetTraceName() << ": sendFragment writev sts(" << sts << ")!=requested_send_bytes(" << this_write_bytes << ")";
728  total_written_bytes += sts; // add sts to total_written_bytes now as sts is adjusted next
729  // find which iovs are done
730  for (ii = 0; (size_t)sts >= iovv[ii].iov_len; ++ii)
731  sts -= iovv[ii].iov_len;
732  in_iov_idx += ii; // done with these in_iovs
733  iovv[ii].iov_len -= sts; // adjust partial iov
734  iovv[ii].iov_base = (uint8_t*)(iovv[ii].iov_base) + sts; // adjust partial iov
735 
736  // add more to get up to per_write_max_bytes
737  out_iov_idx = 0;
738  if (ii != 0)
739  iovv[out_iov_idx] = iovv[ii];
740  // starting over
741  this_write_bytes = iovv[out_iov_idx].iov_len;
742  // add any left over from appropriate in_iov_idx --
743  // i.e. match this out_iov with the in_iov that was used to
744  // initialize it; see how close the out base+len is to in base+len
745  // check !>per_write_max_bytes
746  unsigned long additional = ((unsigned long)iov_in[in_iov_idx].iov_base + iov_in[in_iov_idx].iov_len)
747  - ((unsigned long)iovv[out_iov_idx].iov_base + iovv[out_iov_idx].iov_len);
748  if (additional)
749  {
750  iovv[out_iov_idx].iov_len += additional;
751  this_write_bytes += additional;
752  if (this_write_bytes > per_write_max_bytes)
753  {
754  iovv[out_iov_idx].iov_len -= this_write_bytes - per_write_max_bytes;
755  this_write_bytes = per_write_max_bytes;
756  }
757  }
758  ++out_iov_idx; // done with
759  TLOG(TLVL_TRACE) << GetTraceName() << ": sendFragment writev sts!=: this_write_bytes=" << this_write_bytes
760  << " out_iov_idx=" << out_iov_idx
761  << " additional=" << additional
762  << " ii=" << ii;
763  }
764  else
765  {
766 #ifndef __OPTIMIZE__ // This can be an expensive TRACE call (even if disabled) due to multiplicity of calls
767  TLOG(TLVL_TRACE) << GetTraceName() << ": sendFragment writev sts(" << sts << ")==requested_send_bytes(" << this_write_bytes << ")";
768 #endif
769  total_written_bytes += sts;
770  --out_iov_idx; // make it the index of the last iovv
771  iovv[out_iov_idx].iov_base = (uint8_t*)(iovv[out_iov_idx].iov_base) + iovv[out_iov_idx].iov_len;
772  iovv[out_iov_idx].iov_len = 0;
773  in_iov_idx += out_iov_idx; // at least this many complete (one more if "last iovv" is complete
774  this_write_bytes = 0;
775  // need to check last iovv against appropriate iov_in
776  unsigned long additional = ((unsigned long)iov_in[in_iov_idx].iov_base + iov_in[in_iov_idx].iov_len)
777  - ((unsigned long)iovv[out_iov_idx].iov_base + iovv[out_iov_idx].iov_len);
778  if (additional)
779  {
780  iovv[out_iov_idx].iov_len += additional;
781  this_write_bytes += additional;
782  if (this_write_bytes > per_write_max_bytes)
783  {
784  iovv[out_iov_idx].iov_len -= this_write_bytes - per_write_max_bytes;
785  this_write_bytes = per_write_max_bytes;
786  }
787  if (out_iov_idx != 0)
788  iovv[0] = iovv[out_iov_idx];
789  out_iov_idx = 1;
790  }
791  else
792  {
793  ++in_iov_idx;
794  out_iov_idx = 0;
795  }
796  }
797  } while (total_written_bytes < total_to_write_bytes);
798  if (total_written_bytes > total_to_write_bytes)
799  TLOG(TLVL_ERROR) << GetTraceName() << ": sendFragment program error: too many bytes transferred";
800 
801  if (blocking)
802  {
803  blocking = false;
804  fcntl(send_fd_, F_SETFL, O_NONBLOCK); // set O_NONBLOCK
805  }
806  sts = total_written_bytes - sizeof(MessHead);
807 
808  TLOG(14) << GetTraceName() << ": sendFragment sts=" << sts;
810 }
811 
812 void artdaq::TCPSocketTransfer::connect_()
813 {
814  auto start_time = std::chrono::steady_clock::now();
815 
816  // Retry a few times if we can't connect
817  while (send_fd_ == -1 && TimeUtils::GetElapsedTimeMicroseconds(start_time) < send_retry_timeout_us_ * 10)
818  {
819  TLOG(TLVL_DEBUG) << GetTraceName() << ": Connecting sender socket";
820  int sndbuf_bytes = static_cast<int>(sndbuf_);
821  if (sndbuf_ > INT_MAX)
822  {
823  sndbuf_bytes = INT_MAX;
824  TLOG(TLVL_WARNING) << "Requested SNDBUF " << sndbuf_ << " too large, setting to INT_MAX: " << INT_MAX;
825  }
826  TLOG(TLVL_DEBUG) << "Requested SNDBUF is " << sndbuf_bytes;
827 
828  send_fd_ = TCPConnect(hostMap_[destination_rank()].c_str()
829  , portMan->GetTCPSocketTransferPort(destination_rank())
830  , O_NONBLOCK
831  , sndbuf_bytes);
832  if (send_fd_ == -1)
833  usleep(send_retry_timeout_us_);
834  }
835  connect_state = 0;
836  blocking = 0;
837  TLOG(TLVL_DEBUG) << GetTraceName() << ": connect_ " + hostMap_[destination_rank()] + ":" << portMan->GetTCPSocketTransferPort(destination_rank()) << " send_fd_=" << send_fd_;
838  if (send_fd_ != -1)
839  {
840  // write connect msg
841  TLOG(TLVL_DEBUG) << GetTraceName() << ": connect_: Writing connect message";
842  MessHead mh = { 0,MessHead::connect_v0,htons(source_rank()),{htonl(CONN_MAGIC)} };
843  ssize_t sts = write(send_fd_, &mh, sizeof(mh));
844  if (sts == -1)
845  {
846  TLOG(TLVL_ERROR) << GetTraceName() << ": connect_: Error writing connect message!";
847  // a write error here is completely unexpected!
848  connect_state = 0;
849  close(send_fd_);
850  send_fd_ = -1;
851  }
852  else
853  {
854  TLOG(TLVL_INFO) << GetTraceName() << ": connect_: Successfully connected";
855  // consider it all connected/established
856  connect_state = 1;
857  }
858 
859 #if USE_ACKS
860  if (ack_listen_thread_ && ack_listen_thread_->joinable()) ack_listen_thread_->join();
861  TLOG(TLVL_INFO) << GetTraceName() << ": Starting Ack Listener Thread";
862 
863  try {
864  ack_listen_thread_ = std::make_unique<boost::thread>(&TCPSocketTransfer::receive_acks_, this);
865  }
866  catch (const boost::exception& e)
867  {
868  TLOG(TLVL_ERROR) << "Caught boost::exception starting TCP Socket Ack Listen thread: " << boost::diagnostic_information(e) << ", errno=" << errno;
869  std::cerr << "Caught boost::exception starting TCP Socket Ack Listen thread: " << boost::diagnostic_information(e) << ", errno=" << errno << std::endl;
870  exit(5);
871  }
872 #endif
873  }
874  }
875 
876 void artdaq::TCPSocketTransfer::reconnect_()
877 {
878  if (send_fd_ == -1 && role() == TransferInterface::Role::kSend)
879  {
880  TLOG(TLVL_TRACE) << GetTraceName() << ": check/reconnect";
881  return connect_();
882  }
883 }
884 
885 void artdaq::TCPSocketTransfer::start_listen_thread_()
886 {
887  std::unique_lock<std::mutex> start_lock(listen_thread_mutex_);
888  if (listen_thread_refcount_ == 0)
889  {
890  if (listen_thread_ && listen_thread_->joinable()) listen_thread_->join();
891  listen_thread_refcount_ = 1;
892  TLOG(TLVL_INFO) << GetTraceName() << ": Starting Listener Thread";
893 
894  try {
895  listen_thread_ = std::make_unique<boost::thread>(&TCPSocketTransfer::listen_, portMan->GetTCPSocketTransferPort(destination_rank()), rcvbuf_);
896  }
897  catch (const boost::exception& e)
898  {
899  TLOG(TLVL_ERROR) << "Caught boost::exception starting TCP Socket Listen thread: " << boost::diagnostic_information(e) << ", errno=" << errno;
900  std::cerr << "Caught boost::exception starting TCP Socket Listen thread: " << boost::diagnostic_information(e) << ", errno=" << errno << std::endl;
901  exit(5);
902  }
903  }
904  else
905  {
906  listen_thread_refcount_++;
907  }
908 }
909 
910 #if USE_ACKS
911 void artdaq::TCPSocketTransfer::receive_acks_()
912 {
913  while (send_fd_ >= 0)
914  {
915  pollfd pollfd_s;
916  pollfd_s.events = POLLIN | POLLPRI;
917  pollfd_s.fd = send_fd_;
918 
919  TLOG(18) << GetTraceName() << ": receive_acks_: Polling fd to see if there's data";
920  int num_fds_ready = poll(&pollfd_s, 1, 1000);
921  if (num_fds_ready <= 0)
922  {
923  if (num_fds_ready == 0)
924  {
925  TLOG(18) << GetTraceName() << ": receive_acks_: No data on receive socket";
926  continue;
927  }
928 
929  TLOG(TLVL_ERROR) << "Error in poll: errno=" << errno;
930  break;
931  }
932 
933  if (pollfd_s.revents & (POLLIN | POLLPRI))
934  {
935  // Expected, don't have to check revents any further
936  }
937  else
938  {
939  TLOG(TLVL_DEBUG) << GetTraceName() << ": receive_acks_: Wrong event received from pollfd: " << pollfd_s.revents;
940  break;
941  }
942 
943  MessHead mh;
944  auto sts = read(send_fd_, &mh, sizeof(mh));
945 
946  if (sts != sizeof(mh))
947  {
948  TLOG(TLVL_ERROR) << GetTraceName() << ": receive_ack_: Wrong message header length received! (actual " << sts << " != " << sizeof(mh) << " expected)";
949  continue;
950  }
951 
952  // check for "magic" and valid source_id(aka rank)
953  mh.source_id = ntohs(mh.source_id); // convert here as it is reference several times
954  if (mh.source_id != my_rank)
955  {
956  TLOG(TLVL_ERROR) << GetTraceName() << ": receive_ack_: Received ack for different sender! Rank=" << my_rank << ", hdr=" << mh.source_id;
957  continue;
958  }
959  if (ntohl(mh.conn_magic) != ACK_MAGIC || !(mh.message_type == MessHead::ack_v0)) // Allow for future connect message versions
960  {
961  TLOG(TLVL_ERROR) << GetTraceName() << ": receive_ack_: Wrong magic bytes in header!";
962  continue;
963  }
964 
965  TLOG(17) << GetTraceName() << ": receive_acks_: Received ack message, diff is now " << (send_ack_diff_.load() - 1);
966  send_ack_diff_--;
967 }
968 }
969 
970 void artdaq::TCPSocketTransfer::send_ack_(int fd)
971 {
972  MessHead mh = { 0,MessHead::ack_v0,htons(source_rank()),{ htonl(ACK_MAGIC) } };
973  write(fd, &mh, sizeof(mh));
974 }
975 #endif
976 
977 void artdaq::TCPSocketTransfer::listen_(int port, size_t rcvbuf)
978 {
979  int listen_fd = -1;
980  while (listen_thread_refcount_ > 0)
981  {
982  TLOG(TLVL_TRACE) << "listen_: Listening/accepting new connections on port " << port;
983  if (listen_fd == -1)
984  {
985  TLOG(TLVL_DEBUG) << "listen_: Opening listener";
986  listen_fd = TCP_listen_fd(port, rcvbuf);
987  }
988  if (listen_fd == -1)
989  {
990  TLOG(TLVL_DEBUG) << "listen_: Error creating listen_fd!";
991  break;
992  }
993 
994  int res;
995  timeval tv = { 2,0 }; // maybe increase of some global "debugging" flag set???
996  fd_set rfds;
997  FD_ZERO(&rfds);
998  FD_SET(listen_fd, &rfds);
999 
1000  res = select(listen_fd + 1, &rfds, (fd_set *)0, (fd_set *)0, &tv);
1001  if (res > 0)
1002  {
1003  int sts;
1004  sockaddr_un un;
1005  socklen_t arglen = sizeof(un);
1006  int fd;
1007  TLOG(TLVL_DEBUG) << "listen_: Calling accept";
1008  fd = accept(listen_fd, (sockaddr *)&un, &arglen);
1009  TLOG(TLVL_DEBUG) << "listen_: Done with accept";
1010 
1011  TLOG(TLVL_DEBUG) << "listen_: Reading connect message";
1012  socklen_t lenlen = sizeof(tv);
1013  /*sts=*/
1014  setsockopt(fd, SOL_SOCKET, SO_RCVTIMEO, &tv, lenlen); // see man 7 socket.
1015  MessHead mh;
1016  uint64_t mark_us = TimeUtils::gettimeofday_us();
1017  sts = read(fd, &mh, sizeof(mh));
1018  uint64_t delta_us = TimeUtils::gettimeofday_us() - mark_us;
1019  TLOG(TLVL_DEBUG) << "listen_: Read of connect message took " << delta_us << " microseconds.";
1020  if (sts != sizeof(mh))
1021  {
1022  TLOG(TLVL_DEBUG) << "listen_: Wrong message header length received!";
1023  close(fd);
1024  continue;
1025  }
1026 
1027  // check for "magic" and valid source_id(aka rank)
1028  mh.source_id = ntohs(mh.source_id); // convert here as it is reference several times
1029  if (ntohl(mh.conn_magic) != CONN_MAGIC || !(mh.message_type == MessHead::connect_v0)) // Allow for future connect message versions
1030  {
1031  TLOG(TLVL_DEBUG) << "listen_: Wrong magic bytes in header!";
1032  close(fd);
1033  continue;
1034  }
1035 
1036  // now add (new) connection
1037  std::unique_lock<std::mutex> lk(connected_fd_mutex_);
1038  connected_fds_[mh.source_id].insert(fd);
1039 
1040  TLOG(TLVL_INFO) << "listen_: New fd is " << fd << " for source rank " << mh.source_id;
1041  }
1042  else
1043  {
1044  TLOG(16) << "listen_: No connections in timeout interval!";
1045  }
1046  }
1047 
1048  TLOG(TLVL_INFO) << "listen_: Shutting down connection listener";
1049  if (listen_fd != -1) close(listen_fd);
1050  std::unique_lock<std::mutex> lk(connected_fd_mutex_);
1051  auto it = connected_fds_.begin();
1052  while (it != connected_fds_.end())
1053  {
1054  auto& fd_set = it->second;
1055  auto rank_it = fd_set.begin();
1056  while (rank_it != fd_set.end())
1057  {
1058  close(*rank_it);
1059  rank_it = fd_set.erase(rank_it);
1060  }
1061  it = connected_fds_.erase(it);
1062  }
1063 
1064 } // do_connect_
1065 
1066 DEFINE_ARTDAQ_TRANSFER(artdaq::TCPSocketTransfer)
bool isRunning() override
Determine whether the TransferInterface plugin is able to send/receive data.
virtual int source_rank() const
Get the source rank for this TransferInterface instance.
int TCPConnect(char const *host_in, int dflt_port, long flags=0, int sndbufsiz=0)
Connect to a host on a given port.
Definition: TCPConnect.cc:358
uint32_t conn_magic
unsigned first is better for MessHead initializer: {0,0,my_node_idx_,CONN_MAGIC}
Definition: SRSockets.hh:40
hostMap_t MakeHostMap(fhicl::ParameterSet pset, hostMap_t map=hostMap_t())
Make a hostMap_t from a HostMap::Config ParameterSet
Definition: HostMap.hh:66
This TransferInterface is a Receiver.
int receiveFragmentData(RawDataType *destination, size_t wordCount) override
Receive the body of a Fragment to the given destination pointer.
int TCP_listen_fd(int port, int rcvbuf)
Create a TCP listening socket on the given port and INADDR_ANY, with the given receive buffer...
TCPSocketTransfer(fhicl::ParameterSet const &ps, Role role)
TCPSocketTransfer Constructor.
int receiveFragmentHeader(detail::RawFragmentHeader &header, size_t receiveTimeout) override
Receive a Fragment Header from the transport mechanism.
This TransferInterface is a Sender.
int32_t byte_count
use CONN_MAGIC for connect_v0, data that follow for data_v0 (and 0 lenght data)
Definition: SRSockets.hh:41
Some error occurred, but no exception was thrown.
Role
Used to determine if a TransferInterface is a Sender or Receiver.
int64_t source_id
Rank of the source.
Definition: SRSockets.hh:36
MessType message_type
Message Type.
Definition: SRSockets.hh:35
The send operation completed successfully.
This interface defines the functions used to transfer data between artdaq applications.
TransferInterface implementation plugin that sends data using TCP sockets.
This header is sent by the TCPSocket_transfer to allow for more efficient writev calls.
Definition: SRSockets.hh:15
CopyStatus
Returned from the send functions, this enumeration describes the possible return codes. If an exception occurs, it will be thrown and should be handled normally.