artdaq  v3_08_00
TCPSocket_transfer.cc
1 // Sep 14, 2016. "TERMS AND CONDITIONS" governing this file are in the README
2 // or COPYING file. If you do not have such a file, one can be obtained by
3 // contacting Ron or Fermi Lab in Batavia IL, 60510, phone: 630-840-3000.
4 // $RCSfile: .emacs.gnu,v $
5 // rev="$Revision: 1.30 $$Date: 2016/03/01 14:27:27 $";
6 
7 // C Includes
8 #include <arpa/inet.h> // ntohl, ntohs
9 #include <poll.h> // struct pollfd
10 #include <stdlib.h> // atoi, strtoul
11 #include <sys/socket.h> // socket, socklen_t
12 #include <sys/types.h> // size_t
13 #include <sys/un.h> // sockaddr_un
14 
15 // C++ Includes
16 #include <fstream>
17 #include <stdexcept>
18 #include <string>
19 
20 // product Includes
21 #include "artdaq/DAQdata/Globals.hh"
22 #define TRACE_NAME (app_name + "_TCPSocketTransfer").c_str()
23 
24 // artdaq Includes
25 #include <iomanip>
26 #include "artdaq-core/Data/Fragment.hh"
27 #include "artdaq-core/Utilities/TimeUtils.hh"
30 #include "artdaq/TransferPlugins/TCPSocketTransfer.hh"
31 #include "artdaq/TransferPlugins/detail/SRSockets.hh"
32 #include "artdaq/TransferPlugins/detail/Timeout.hh"
33 
34 #define USE_SENDMSG 1
35 
36 std::atomic<int> artdaq::TCPSocketTransfer::listen_thread_refcount_(0);
37 std::unique_ptr<boost::thread> artdaq::TCPSocketTransfer::listen_thread_ = nullptr;
38 std::map<int, std::set<int>> artdaq::TCPSocketTransfer::connected_fds_ = std::map<int, std::set<int>>();
39 std::mutex artdaq::TCPSocketTransfer::listen_thread_mutex_;
40 std::mutex artdaq::TCPSocketTransfer::connected_fd_mutex_;
41 
43  TCPSocketTransfer(fhicl::ParameterSet const& pset, TransferInterface::Role role)
44  : TransferInterface(pset, role)
45  , send_fd_(-1)
46  , active_receive_fd_(-1)
47  , last_active_receive_fd_(-1)
48  , rcvbuf_(pset.get<size_t>("tcp_receive_buffer_size", 0))
49  , sndbuf_(pset.get<size_t>("tcp_send_buffer_size", max_fragment_size_words_ * sizeof(artdaq::RawDataType) * buffer_count_))
50  , send_retry_timeout_us_(pset.get<size_t>("send_retry_timeout_us", 1000000))
51  , timeoutMessageArmed_(true)
52  , last_recv_time_()
53  , receive_disconnected_wait_s_(pset.get<double>("receive_socket_disconnected_wait_s", 10.0))
54  , receive_err_wait_us_(pset.get<size_t>("receive_socket_disconnected_wait_us", 10000))
55  , receive_socket_has_been_connected_(false)
56  , send_ack_diff_(0)
57 {
58  TLOG(TLVL_DEBUG) << GetTraceName() << " Constructor: pset=" << pset.to_string() << ", role=" << (role == TransferInterface::Role::kReceive ? "kReceive" : "kSend");
59  connection_was_lost_ = false;
60 
62  {
63  // Wait for sender to connect...
64  TLOG(TLVL_DEBUG) << GetTraceName() << ": Listening for connections";
65  start_listen_thread_();
66  TLOG(TLVL_DEBUG) << GetTraceName() << ": Done Listening";
67  }
68  else
69  {
70  hostMap_ = MakeHostMap(pset);
71  TLOG(TLVL_DEBUG) << GetTraceName() << ": Connecting to destination";
72  connect_();
73  TLOG(TLVL_DEBUG) << GetTraceName() << ": Done Connecting";
74  }
75  TLOG(TLVL_DEBUG) << GetTraceName() << ": End of Constructor";
76 }
77 
78 artdaq::TCPSocketTransfer::~TCPSocketTransfer() noexcept
79 {
80  TLOG(TLVL_DEBUG) << GetTraceName() << ": Shutting down TCPSocketTransfer";
81 
82  if (role() == TransferInterface::Role::kSend)
83  {
84  // close all open connections (send stop_v0) first
85  MessHead mh = {0, MessHead::stop_v0, htons(TransferInterface::source_rank()), {0}};
86  if (send_fd_ != -1)
87  {
88  // should be blocking with modest timeo
89  timeval tv = {0, 100000};
90  socklen_t len = sizeof(tv);
91  setsockopt(send_fd_, SOL_SOCKET, SO_SNDTIMEO, &tv, len);
92  write(send_fd_, &mh, sizeof(mh));
93  }
94  close(send_fd_);
95  send_fd_ = -1;
96  }
97  else
98  {
99  {
100  std::unique_lock<std::mutex> fd_lock(connected_fd_mutex_);
101  if (connected_fds_.count(source_rank()))
102  {
103  auto it = connected_fds_[source_rank()].begin();
104  while (it != connected_fds_[source_rank()].end())
105  {
106  close(*it);
107  it = connected_fds_[source_rank()].erase(it);
108  }
109  connected_fds_.erase(source_rank());
110  }
111  if (ack_listen_thread_ && ack_listen_thread_->joinable()) ack_listen_thread_->join();
112  }
113 
114  std::unique_lock<std::mutex> lk(listen_thread_mutex_);
115  listen_thread_refcount_--;
116  if (listen_thread_refcount_ <= 0 && listen_thread_ && listen_thread_->joinable())
117  {
118  listen_thread_->join();
119  }
120  }
121 
122  TLOG(TLVL_DEBUG) << GetTraceName() << ": End of Destructor";
123 }
124 
125 int artdaq::TCPSocketTransfer::receiveFragmentHeader(detail::RawFragmentHeader& header, size_t timeout_usec)
126 {
127  TLOG(5) << GetTraceName() << ": receiveFragmentHeader: BEGIN";
128  int ret_rank = RECV_TIMEOUT;
129 
130  // Don't bomb out until received at least one connection...
131  if (getConnectedFDCount(source_rank()) == 0)
132  { // what if just listen_fd???
133  // if (receive_socket_has_been_connected_ && TimeUtils::GetElapsedTime(last_recv_time_) > receive_disconnected_wait_s_)
134  // {
135  // TLOG(TLVL_ERROR) << GetTraceName() << ": receiveFragmentHeader: senders have been disconnected for "
136  // << TimeUtils::GetElapsedTime(last_recv_time_) << " s (receive_socket_disconnected_wait_s = " << receive_disconnected_wait_s_ << " s). RETURNING DATA_END!";
137  // return DATA_END;
138  // }
139  //if (++not_connected_count_ > receive_err_threshold_) { return DATA_END; }
140  TLOG(7) << GetTraceName() << ": receiveFragmentHeader: Receive socket not connected, returning RECV_TIMEOUT";
141  usleep(receive_err_wait_us_);
142  return RECV_TIMEOUT;
143  }
144  receive_socket_has_been_connected_ = true;
145  last_recv_time_ = std::chrono::steady_clock::now();
146 
147  TLOG(5) << GetTraceName() << ": receiveFragmentHeader timeout_usec=" << timeout_usec;
148  //void* buff=alloca(max_fragment_size_words_*8);
149  size_t byte_cnt = 0;
150  int sts;
151  int offset = 0;
152  SocketState state = SocketState::Metadata;
153  int target_bytes = sizeof(MessHead);
154  uint64_t start_time_us = TimeUtils::gettimeofday_us();
155 
156  //while (active_receive_fd_ != -1)
157  //{
158  // TLOG(TLVL_TRACE) << GetTraceName() << ": Currently receiving from fd " << active_receive_fd_ << ", waiting!";
159  // usleep(1000);
160  //}
161 
162  uint8_t* buff;
163 
164  int timeout_ms;
165  if (timeout_usec == 0)
166  timeout_ms = 0;
167  else
168  timeout_ms = (timeout_usec + 999) / 1000; // want at least 1 ms
169 
170  bool done = false;
171  bool noDataWarningSent = false;
172  int loop_guard = 0;
173 
174  while (!done && getConnectedFDCount(source_rank()) > 0)
175  {
176  if (active_receive_fd_ == -1)
177  {
178  loop_guard = 0;
179  size_t fd_count = 0;
180  std::vector<pollfd> pollfds;
181  {
182  std::unique_lock<std::mutex> lk(connected_fd_mutex_);
183  fd_count = connected_fds_[source_rank()].size();
184  pollfds.resize(fd_count);
185  auto iter = connected_fds_[source_rank()].begin();
186  for (size_t ii = 0; ii < fd_count; ++ii)
187  {
188  pollfds[ii].events = POLLIN | POLLPRI | POLLERR;
189  pollfds[ii].fd = *iter;
190  ++iter;
191  }
192  }
193  //TLOG(TLVL_DEBUG) << GetTraceName() << ": receiveFragment: Polling fd to see if there's data" ;
194  int num_fds_ready = poll(&pollfds[0], fd_count, timeout_ms);
195  if (num_fds_ready <= 0)
196  {
197  TLOG(5) << GetTraceName() << ": receiveFragmentHeader: No data on receive socket, returning RECV_TIMEOUT";
198  return RECV_TIMEOUT;
199  }
200 
201  size_t index = 0;
202  if (last_active_receive_fd_ != -1)
203  {
204  for (auto& pollfd : pollfds)
205  {
206  index++;
207  if (pollfd.fd == last_active_receive_fd_)
208  {
209  break;
210  }
211  }
212  }
213 
214  int active_index = -1;
215  short anomolous_events = 0;
216  for (size_t ii = index; ii < index + pollfds.size(); ++ii)
217  {
218  auto pollfd_index = (ii + index) % pollfds.size();
219  if (pollfds[pollfd_index].revents & (POLLIN | POLLPRI))
220  {
221  active_index = pollfd_index;
222  active_receive_fd_ = pollfds[active_index].fd;
223  active_revents_ = pollfds[active_index].revents;
224  ;
225  break;
226  }
227  else if (pollfds[pollfd_index].revents & (POLLHUP | POLLERR))
228  {
229  disconnect_receive_socket_(pollfds[pollfd_index].fd, "Poll returned POLLHUP or POLLERR, indicating problems with the sender.");
230  continue;
231  }
232  else if (pollfds[pollfd_index].revents & (POLLNVAL))
233  {
234  TLOG(TLVL_DEBUG) << GetTraceName() << ": receiveFragmentHeader: FD is closed, most likely because the peer went away. Removing from fd list.";
235  disconnect_receive_socket_(pollfds[pollfd_index].fd, "FD is closed, most likely because the peer went away.");
236  continue;
237  }
238  else if (pollfds[pollfd_index].revents)
239  {
240  anomolous_events |= pollfds[pollfd_index].revents;
241  }
242  }
243 
244  if (active_index == -1)
245  {
246  if (anomolous_events)
247  TLOG(TLVL_DEBUG) << GetTraceName() << ": receiveFragmentHeader: Wrong event received from a pollfd. Mask: " << static_cast<int>(anomolous_events);
248  active_receive_fd_ = -1;
249  continue;
250  }
251 
252  if (!done && timeout_usec > 0)
253  {
254  // calc next timeout_ms (unless timed out)
255  size_t delta_us = TimeUtils::gettimeofday_us() - start_time_us;
256  if (delta_us > timeout_usec)
257  {
258  return RECV_TIMEOUT;
259  }
260  timeout_ms = ((timeout_usec - delta_us) + 999) / 1000; // want at least 1 ms
261  }
262  }
263  if (loop_guard > 10) { usleep(1000); }
264  if (++loop_guard > 10010)
265  {
266  TLOG(TLVL_WARNING) << GetTraceName() << ": receiveFragmentHeader: loop guard triggered, returning RECV_TIMEOUT";
267  usleep(receive_err_wait_us_);
268  active_receive_fd_ = -1;
269  return RECV_TIMEOUT;
270  }
271 
272  if (state == SocketState::Metadata)
273  {
274  //TLOG(TLVL_DEBUG) << GetTraceName() << ": receiveFragmentHeader: Reading Message Header" ;
275  buff = &(mha[offset]);
276  byte_cnt = sizeof(MessHead) - offset;
277  }
278  else
279  {
280  //TLOG(TLVL_DEBUG) << GetTraceName() << ": receiveFragmentHeader: Reading data" ;
281  buff = reinterpret_cast<uint8_t*>(&header) + offset;
282  byte_cnt = target_bytes - offset;
283  }
284  //if (byte_cnt > sizeof(MessHead))
285  // {
286  // TLOG(TLVL_ERROR) << "Invalid byte count for read (count=" << byte_cnt
287  // << ",offset=" << offset << ",mh.byte_count=" << mh.byte_count
288  // << "), skipping read and returning RECV_TIMEOUT";
289  // return RECV_TIMEOUT;
290  //}
291 
292  if (byte_cnt > 0)
293  {
294  TLOG(6) << GetTraceName() << ": receiveFragmentHeader: Reading " << byte_cnt << " bytes from socket";
295  sts = read(active_receive_fd_, buff, byte_cnt);
296  TLOG(6) << GetTraceName() << ": receiveFragmentHeader: Done with read";
297  }
298  if (sts > 0)
299  {
300  loop_guard = 0;
301  last_recv_time_ = std::chrono::steady_clock::now();
302  }
303 
304  TLOG(7) << GetTraceName() << ": receiveFragmentHeader state=" << static_cast<int>(state) << " read=" << sts;
305  if (sts < 0)
306  {
307  TLOG(TLVL_WARNING) << GetTraceName() << ": receiveFragmentHeader: Error on receive, closing socket "
308  << " (errno=" << errno << ": " << strerror(errno) << ")";
309  active_receive_fd_ = disconnect_receive_socket_(active_receive_fd_);
310  }
311  else if (sts == 0)
312  {
313  if (!noDataWarningSent)
314  {
315  TLOG(TLVL_WARNING) << GetTraceName() << ": receiveFragmentHeader: No data received, is the sender still sending?!?";
316  noDataWarningSent = true;
317  }
318  if (TimeUtils::GetElapsedTime(last_recv_time_) > receive_disconnected_wait_s_)
319  {
320  TLOG(TLVL_ERROR) << GetTraceName() << ": receiveFragmentHeader: No data received within timeout, aborting!";
321  return RECV_TIMEOUT;
322  }
323  }
324  else
325  {
326  // see if we're done (with this state)
327  sts = offset += sts;
328  if (sts >= target_bytes)
329  {
330  TLOG(7) << GetTraceName() << ": receiveFragmentHeader: Target read bytes reached. Changing state";
331  offset = 0;
332  if (state == SocketState::Metadata)
333  {
334  state = SocketState::Data;
335  mh.byte_count = ntohl(mh.byte_count);
336  mh.source_id = ntohs(mh.source_id);
337  target_bytes = mh.byte_count;
338  TLOG(7) << GetTraceName() << ": receiveFragmentHeader: Expected header size = " << target_bytes << ", sizeof(RawFragmentHeader) = " << sizeof(artdaq::detail::RawFragmentHeader);
339  //assert(target_bytes == sizeof(artdaq::detail::RawFragmentHeader) || target_bytes == 0);
340 
341  if (mh.message_type == MessHead::stop_v0)
342  {
343  active_receive_fd_ = disconnect_receive_socket_(active_receive_fd_, "Stop Message received.");
344  }
345  else if (mh.message_type == MessHead::data_v0 || mh.message_type == MessHead::data_more_v0)
346  {
347  TLOG(TLVL_WARNING) << GetTraceName() << ": receiveFragmentHeader: Message header indicates that Fragment data follows when I was expecting a Fragment header!";
348  active_receive_fd_ = disconnect_receive_socket_(active_receive_fd_, "Desync detected");
349  }
350 
351  if (target_bytes == 0)
352  {
353  //Probably a stop_v0, return timeout so we can try again.
354  return RECV_TIMEOUT;
355  }
356  }
357  else
358  {
359  ret_rank = source_rank();
360  TLOG(8) << GetTraceName() << ": receiveFragmentHeader done sts=" << sts << " src=" << ret_rank;
361  TLOG(7) << GetTraceName() << ": receiveFragmentHeader: Done receiving fragment header. Moving into output.";
362 
363  done = true; // no more polls
364  //break; // no more read of ready fds
365  }
366  }
367  }
368 
369  } // while(!done)...poll
370 
371  TLOG(5) << GetTraceName() << ": receiveFragmentHeader: Returning " << ret_rank;
372  return ret_rank;
373 }
374 
375 int artdaq::TCPSocketTransfer::disconnect_receive_socket_(int fd, std::string msg)
376 {
377  TLOG(TLVL_WARNING) << GetTraceName() << ": disconnect_receive_socket_: " << msg << " Closing socket " << fd;
378  close(fd);
379  std::unique_lock<std::mutex> lk(connected_fd_mutex_);
380  if (connected_fds_.count(source_rank()))
381  connected_fds_[source_rank()].erase(fd);
382  fd = -1;
383  TLOG(TLVL_DEBUG) << GetTraceName() << ": disconnect_receive_socket_: There are now " << connected_fds_[source_rank()].size() << " active senders.";
384  return fd;
385 }
386 
387 int artdaq::TCPSocketTransfer::receiveFragmentData(RawDataType* destination, size_t)
388 {
389  TLOG(19) << GetTraceName() << ": receiveFragmentData: BEGIN";
390  int ret_rank = RECV_TIMEOUT;
391  if (active_receive_fd_ == -1)
392  { // what if just listen_fd???
393  TLOG(TLVL_ERROR) << GetTraceName() << ": receiveFragmentData: Receive socket not connected, returning RECV_TIMEOUT (Will result in \"Unexpected return code error\")";
394  return RECV_TIMEOUT;
395  }
396 
397  //void* buff=alloca(max_fragment_size_words_*8);
398  uint8_t* buff;
399  size_t byte_cnt = 0;
400  int sts;
401  int offset = 0;
402  SocketState state = SocketState::Metadata;
403  int target_bytes = sizeof(MessHead);
404 
405  pollfd pollfd_s;
406  pollfd_s.events = POLLIN | POLLPRI | POLLERR;
407  pollfd_s.fd = active_receive_fd_;
408 
409  int loop_guard = 0;
410  bool done = false;
411  bool noDataWarningSent = false;
412  last_recv_time_ = std::chrono::steady_clock::now();
413  while (!done)
414  {
415  TLOG(9) << GetTraceName() << ": receiveFragmentData: Polling fd to see if there's data";
416  int num_fds_ready = poll(&pollfd_s, 1, 1000);
417  TLOG(TLVL_TRACE) << GetTraceName() << ": receiveFragmentData: Polled fd to see if there's data"
418  << ", num_fds_ready = " << num_fds_ready;
419  if (num_fds_ready <= 0)
420  {
421  if (num_fds_ready == 0)
422  {
423  TLOG(TLVL_WARNING) << GetTraceName() << ": receiveFragmentData: No data from " << source_rank() << " in " << TimeUtils::GetElapsedTimeMilliseconds(last_recv_time_) << " ms!"
424  << " State = " << (state == SocketState::Metadata ? "Metadata" : "Data") << ", recvd/total=" << offset << "/" << target_bytes << " (delta=" << target_bytes - offset << ")";
425 
426  if (TimeUtils::GetElapsedTime(last_recv_time_) > receive_disconnected_wait_s_)
427  {
428  TLOG(TLVL_WARNING) << GetTraceName() << ": receiveFragmentData: No data received within timeout (" << TimeUtils::GetElapsedTime(last_recv_time_) << " / " << receive_disconnected_wait_s_ << " ), returning RECV_TIMEOUT";
429  disconnect_receive_socket_(active_receive_fd_, "No data on this socket within timeout");
430  active_receive_fd_ = -1;
431  return RECV_TIMEOUT;
432  }
433  continue;
434  }
435 
436  TLOG(TLVL_ERROR) << "Error in poll: errno=" << errno;
437  active_receive_fd_ = -1;
438  break;
439  }
440  else
441  {
442  last_recv_time_ = std::chrono::steady_clock::now();
443  }
444 
445  if (pollfd_s.revents & (POLLIN | POLLPRI))
446  {
447  // Expected, don't have to check revents any further
448  }
449  else if (pollfd_s.revents & (POLLNVAL))
450  {
451  disconnect_receive_socket_(pollfd_s.fd, "FD is closed, most likely because the peer went away.");
452  break;
453  }
454  else if (pollfd_s.revents & (POLLHUP | POLLERR))
455  {
456  disconnect_receive_socket_(pollfd_s.fd, "Poll returned POLLHUP or POLLERR, indicating problems with the sender.");
457  break;
458  }
459  else
460  {
461  TLOG(TLVL_WARNING) << GetTraceName() << ": receiveFragmentData: Wrong event received from pollfd: " << pollfd_s.revents;
462  disconnect_receive_socket_(pollfd_s.fd);
463  break;
464  }
465 
466  if (state == SocketState::Metadata)
467  {
468  //TLOG(TLVL_DEBUG) << GetTraceName() << ": receiveFragmentData: Reading Message Header" ;
469  buff = &(mha[offset]);
470  byte_cnt = sizeof(MessHead) - offset;
471  }
472  else
473  {
474  //TLOG(TLVL_DEBUG) << GetTraceName() << ": receiveFragmentData: Reading data" ;
475  buff = reinterpret_cast<uint8_t*>(destination) + offset;
476  byte_cnt = mh.byte_count - offset;
477  }
478 
479  TLOG(10) << GetTraceName() << ": receiveFragmentData: Reading " << byte_cnt << " bytes from socket into " << (void*)buff;
480  sts = read(active_receive_fd_, buff, byte_cnt);
481  //TLOG(TLVL_DEBUG) << GetTraceName() << ": receiveFragmentData: Done with read" ;
482 
483  TLOG(10) << GetTraceName() << ": recvFragment state=" << static_cast<int>(state) << " read=" << sts;
484 
485  if (sts == 0)
486  {
487  if (loop_guard > 10) { usleep(1000); }
488  if (++loop_guard > 10010)
489  {
490  TLOG(TLVL_WARNING) << GetTraceName() << ": receiveFragmentData: loop guard triggered, returning RECV_TIMEOUT";
491  active_receive_fd_ = -1;
492  return RECV_TIMEOUT;
493  }
494  }
495  else if (sts > 0)
496  {
497  loop_guard = 0;
498  last_recv_time_ = std::chrono::steady_clock::now();
499  }
500 
501  if (sts < 0)
502  {
503  TLOG(TLVL_WARNING) << GetTraceName() << ": receiveFragmentData: Error on receive, closing socket"
504  << " (errno=" << errno << ": " << strerror(errno) << ")";
505  disconnect_receive_socket_(pollfd_s.fd);
506  }
507  else if (sts == 0)
508  {
509  if (!noDataWarningSent)
510  {
511  TLOG(TLVL_WARNING) << GetTraceName() << ": receiveFragmentData: No data received, is the sender still sending?!?";
512  noDataWarningSent = true;
513  }
514  if (TimeUtils::GetElapsedTime(last_recv_time_) > receive_disconnected_wait_s_)
515  {
516  TLOG(TLVL_ERROR) << GetTraceName() << ": receiveFragmentData: No data received within timeout, aborting!";
517  return RECV_TIMEOUT;
518  }
519  }
520  else
521  {
522  // see if we're done (with this state)
523  sts = offset += sts;
524  if (sts >= target_bytes)
525  {
526  TLOG(9) << GetTraceName() << ": receiveFragmentData: Target read bytes reached. Changing state";
527  offset = 0;
528  if (state == SocketState::Metadata)
529  {
530  state = SocketState::Data;
531  mh.byte_count = ntohl(mh.byte_count);
532  mh.source_id = ntohs(mh.source_id);
533  target_bytes = mh.byte_count;
534 
535  if (mh.message_type == MessHead::header_v0)
536  {
537  TLOG(TLVL_WARNING) << GetTraceName() << ": receiveFragmentData: Message header indicates that a Fragment header follows when I was expecting Fragment data!";
538  active_receive_fd_ = disconnect_receive_socket_(active_receive_fd_, "Desync detected");
539  }
540  }
541  else
542  {
543  ret_rank = source_rank();
544  TLOG(11) << GetTraceName() << ": receiveFragmentData done sts=" << sts << " src=" << ret_rank;
545  TLOG(9) << GetTraceName() << ": receiveFragmentData: Done receiving fragment. Moving into output.";
546 
547 #if USE_ACKS
548  send_ack_(active_receive_fd_);
549 #endif
550 
551  done = true; // no more polls
552  //break; // no more read of ready fds
553  }
554  }
555  }
556 
557  // Check if we were asked to do a 0-size receive
558  if (target_bytes == 0 && state == SocketState::Data)
559  {
560  ret_rank = source_rank();
561  TLOG(11) << GetTraceName() << ": receiveFragmentData done sts=" << sts << " src=" << ret_rank;
562  TLOG(9) << GetTraceName() << ": receiveFragmentData: Done receiving fragment. Moving into output.";
563 
564 #if USE_ACKS
565  send_ack_(active_receive_fd_);
566 #endif
567 
568  done = true; // no more polls
569  }
570 
571  } // while(!done)...poll
572 
573  last_active_receive_fd_ = active_receive_fd_;
574  active_receive_fd_ = -1;
575 
576  TLOG(9) << GetTraceName() << ": receiveFragmentData: Returning rank " << ret_rank;
577  return ret_rank;
578 }
579 
581 {
582  switch (role())
583  {
585  return send_fd_ != -1;
587  TLOG(TLVL_DEBUG) << GetTraceName() << ": isRunning: There are " << getConnectedFDCount(source_rank()) << " fds connected.";
588  return getConnectedFDCount(source_rank()) > 0;
589  }
590  return false;
591 }
592 
594 {
595  while (connected_fds_[source_rank()].size())
596  {
597  disconnect_receive_socket_(*connected_fds_[source_rank()].begin(), "Flushing connections");
598  }
599 }
600 
601 // Send the given Fragment. Return the rank of the destination to which
602 // the Fragment was sent OR -1 if to none.
603 artdaq::TransferInterface::CopyStatus artdaq::TCPSocketTransfer::sendFragment_(Fragment&& frag, size_t send_timeout_usec)
604 {
605  TLOG(12) << GetTraceName() << ": sendFragment begin send of fragment with sequenceID=" << frag.sequenceID();
606  artdaq::Fragment grab_ownership_frag = std::move(frag);
607 
608  reconnect_();
609  if (send_fd_ == -1 && connection_was_lost_)
610  {
611  TLOG(TLVL_INFO) << GetTraceName() << ": reconnection attempt failed, returning quickly.";
613  }
614 
615  // Send Fragment Header
616 
617 #if USE_ACKS
618  // Wait for fragments to be received
619  while (static_cast<size_t>(send_ack_diff_) > buffer_count_) usleep(10000);
620 #endif
621 
622  iovec iov = {reinterpret_cast<void*>(grab_ownership_frag.headerAddress()),
623  detail::RawFragmentHeader::num_words() * sizeof(RawDataType)};
624 
625  auto sts = sendData_(&iov, 1, send_retry_timeout_us_, true);
626  auto start_time = std::chrono::steady_clock::now();
627  //If it takes more than 10 seconds to write a Fragment header, give up
628  while (sts == CopyStatus::kTimeout && (send_timeout_usec == 0 || TimeUtils::GetElapsedTimeMicroseconds(start_time) < send_timeout_usec) && TimeUtils::GetElapsedTimeMicroseconds(start_time) < 10000000)
629  {
630  TLOG(13) << GetTraceName() << ": sendFragment: Timeout sending fragment";
631  sts = sendData_(&iov, 1, send_retry_timeout_us_, true);
632  usleep(1000);
633  }
634  if (sts != CopyStatus::kSuccess) return sts;
635 
636  // Send Fragment Data
637 
638  iov = {reinterpret_cast<void*>(grab_ownership_frag.headerAddress() + detail::RawFragmentHeader::num_words()),
639  grab_ownership_frag.sizeBytes() - detail::RawFragmentHeader::num_words() * sizeof(RawDataType)};
640  sts = sendData_(&iov, 1, send_retry_timeout_us_);
641  start_time = std::chrono::steady_clock::now();
642  while (sts == CopyStatus::kTimeout && (send_timeout_usec == 0 || TimeUtils::GetElapsedTimeMicroseconds(start_time) < send_timeout_usec) && TimeUtils::GetElapsedTimeMicroseconds(start_time) < 10000000)
643  {
644  TLOG(13) << GetTraceName() << ": sendFragment: Timeout sending fragment";
645  sts = sendData_(&iov, 1, send_retry_timeout_us_);
646  usleep(1000);
647  }
648 
649 #if USE_ACKS
650  send_ack_diff_++;
651 #endif
652 
653  TLOG(12) << GetTraceName() << ": sendFragment returning " << CopyStatusToString(sts);
654  return sts;
655 }
656 
657 artdaq::TransferInterface::CopyStatus artdaq::TCPSocketTransfer::sendData_(const void* buf, size_t bytes, size_t send_timeout_usec, bool isHeader)
658 {
659  TLOG(TLVL_DEBUG) << GetTraceName() << ": sendData_ Converting buf to iovec";
660  iovec iov = {(void*)buf, bytes};
661  return sendData_(&iov, 1, send_timeout_usec, isHeader);
662 }
663 
664 artdaq::TransferInterface::CopyStatus artdaq::TCPSocketTransfer::sendData_(const struct iovec* iov, int iovcnt, size_t send_timeout_usec, bool isHeader)
665 {
666  // check all connected??? -- currently just check fd!=-1
667  if (send_fd_ == -1)
668  {
669  if (timeoutMessageArmed_)
670  {
671  TLOG(TLVL_DEBUG) << GetTraceName() << ": sendData_: Send fd is not open. Returning kTimeout";
672  timeoutMessageArmed_ = false;
673  }
674  return CopyStatus::kTimeout;
675  }
676  timeoutMessageArmed_ = true;
677  TLOG(14) << GetTraceName() << ": send_timeout_usec is " << send_timeout_usec << ", currently unused.";
678 
679  //TLOG(TLVL_DEBUG) << GetTraceName() << ": sendData_: Determining write size" ;
680  uint32_t total_to_write_bytes = 0;
681  std::vector<iovec> iov_in(iovcnt + 1); // need contiguous (for the unlike case that only partial MH
682  std::vector<iovec> iovv(iovcnt + 2); // 1 more for mh and another one for any partial
683  int ii;
684  for (ii = 0; ii < iovcnt; ++ii)
685  {
686  iov_in[ii + 1] = iov[ii];
687  total_to_write_bytes += iov[ii].iov_len;
688  }
689  //TLOG(TLVL_DEBUG) << GetTraceName() << ": sendData_: Constructing Message Header" ;
690  MessHead mh = {0, isHeader ? MessHead::header_v0 : MessHead::data_v0, htons(source_rank()), {htonl(total_to_write_bytes)}};
691  iov_in[0].iov_base = &mh;
692  iov_in[0].iov_len = sizeof(mh);
693  total_to_write_bytes += sizeof(mh);
694 
695  ssize_t sts = 0;
696  ssize_t total_written_bytes = 0;
697  ssize_t per_write_max_bytes = (32 * 1024);
698 
699  size_t in_iov_idx = 0; // only increment this when we know the associated data has been xferred
700  size_t out_iov_idx = 0;
701  ssize_t this_write_bytes = 0;
702 
703  do
704  {
705  // The first out_iov may be set at the end of the previous loop.
706  // iov looping from below (b/c of the latter, we need to check this_write_bytes)
707  for (;
708  (in_iov_idx + out_iov_idx) < iov_in.size() && this_write_bytes < per_write_max_bytes;
709  ++out_iov_idx)
710  {
711  this_write_bytes += iov_in[in_iov_idx + out_iov_idx].iov_len;
712  iovv[out_iov_idx] = iov_in[in_iov_idx + out_iov_idx];
713  }
714  if (this_write_bytes > per_write_max_bytes)
715  {
716  iovv[out_iov_idx - 1].iov_len -= this_write_bytes - per_write_max_bytes;
717  this_write_bytes = per_write_max_bytes;
718  }
719 
720  // need to do blocking algorithm -- including throttled block notifications
721  do_again:
722 #ifndef __OPTIMIZE__ // This can be an expensive TRACE call (even if disabled) due to multiplicity of calls
723  TLOG(14) << GetTraceName() << ": sendFragment b4 writev " << std::setw(7) << total_written_bytes << " total_written_bytes send_fd_=" << send_fd_ << " in_idx=" << in_iov_idx
724  << " iovcnt=" << out_iov_idx << " 1st.len=" << iovv[0].iov_len;
725 #endif
726 //TLOG(TLVL_DEBUG) << GetTraceName() << " calling writev" ;
727 #if USE_SENDMSG
728  msghdr msg;
729  memset(&msg, 0, sizeof(msghdr));
730  msg.msg_iov = &(iovv[0]);
731  msg.msg_iovlen = out_iov_idx; // at this point out_iov_idx is really the count (not an idx per se)
732  sts = sendmsg(send_fd_, &msg, MSG_NOSIGNAL | (blocking ? 0 : MSG_DONTWAIT));
733 #else
734  sts = writev(send_fd_, &(iovv[0]), out_iov_idx); // SIGPIPE may occur -- need signal handler or mask/ignore
735 #endif
736  //TLOG(TLVL_DEBUG) << GetTraceName() << " done with writev" ;
737 
738  if (sts == -1)
739  {
740  if (errno == EAGAIN /* same as EWOULDBLOCK */)
741  {
742  TLOG(TLVL_DEBUG) << GetTraceName() << ": sendFragment EWOULDBLOCK";
743  blocking = true;
744 
745 #if !USE_SENDMSG
746  fcntl(send_fd_, F_SETFL, 0); // clear O_NONBLOCK
747 #endif
748  // NOTE: YES -- could drop here
749  goto do_again;
750  }
751  TLOG(TLVL_WARNING) << GetTraceName() << ": sendFragment_: WRITE ERROR " << errno << ": " << strerror(errno);
752  connect_state = 0; // any write error closes
753  close(send_fd_);
754  send_fd_ = -1;
755  connection_was_lost_ = true;
757  }
758  else if (sts != this_write_bytes)
759  {
760  // we'll loop around -- with
761  TLOG(TLVL_DEBUG) << GetTraceName() << ": sendFragment writev sts(" << sts << ")!=requested_send_bytes(" << this_write_bytes << ")";
762  total_written_bytes += sts; // add sts to total_written_bytes now as sts is adjusted next
763  // find which iovs are done
764  for (ii = 0; (size_t)sts >= iovv[ii].iov_len; ++ii)
765  sts -= iovv[ii].iov_len;
766  in_iov_idx += ii; // done with these in_iovs
767  iovv[ii].iov_len -= sts; // adjust partial iov
768  iovv[ii].iov_base = (uint8_t*)(iovv[ii].iov_base) + sts; // adjust partial iov
769 
770  // add more to get up to per_write_max_bytes
771  out_iov_idx = 0;
772  if (ii != 0)
773  iovv[out_iov_idx] = iovv[ii];
774  // starting over
775  this_write_bytes = iovv[out_iov_idx].iov_len;
776  // add any left over from appropriate in_iov_idx --
777  // i.e. match this out_iov with the in_iov that was used to
778  // initialize it; see how close the out base+len is to in base+len
779  // check !>per_write_max_bytes
780  unsigned long additional = ((unsigned long)iov_in[in_iov_idx].iov_base + iov_in[in_iov_idx].iov_len) - ((unsigned long)iovv[out_iov_idx].iov_base + iovv[out_iov_idx].iov_len);
781  if (additional)
782  {
783  iovv[out_iov_idx].iov_len += additional;
784  this_write_bytes += additional;
785  if (this_write_bytes > per_write_max_bytes)
786  {
787  iovv[out_iov_idx].iov_len -= this_write_bytes - per_write_max_bytes;
788  this_write_bytes = per_write_max_bytes;
789  }
790  }
791  ++out_iov_idx; // done with
792  TLOG(TLVL_TRACE) << GetTraceName() << ": sendFragment writev sts!=: this_write_bytes=" << this_write_bytes
793  << " out_iov_idx=" << out_iov_idx
794  << " additional=" << additional
795  << " ii=" << ii;
796  }
797  else
798  {
799 #ifndef __OPTIMIZE__ // This can be an expensive TRACE call (even if disabled) due to multiplicity of calls
800  TLOG(TLVL_TRACE) << GetTraceName() << ": sendFragment writev sts(" << sts << ")==requested_send_bytes(" << this_write_bytes << ")";
801 #endif
802  total_written_bytes += sts;
803  --out_iov_idx; // make it the index of the last iovv
804  iovv[out_iov_idx].iov_base = (uint8_t*)(iovv[out_iov_idx].iov_base) + iovv[out_iov_idx].iov_len;
805  iovv[out_iov_idx].iov_len = 0;
806  in_iov_idx += out_iov_idx; // at least this many complete (one more if "last iovv" is complete
807  this_write_bytes = 0;
808  // need to check last iovv against appropriate iov_in
809  unsigned long additional = ((unsigned long)iov_in[in_iov_idx].iov_base + iov_in[in_iov_idx].iov_len) - ((unsigned long)iovv[out_iov_idx].iov_base + iovv[out_iov_idx].iov_len);
810  if (additional)
811  {
812  iovv[out_iov_idx].iov_len += additional;
813  this_write_bytes += additional;
814  if (this_write_bytes > per_write_max_bytes)
815  {
816  iovv[out_iov_idx].iov_len -= this_write_bytes - per_write_max_bytes;
817  this_write_bytes = per_write_max_bytes;
818  }
819  if (out_iov_idx != 0)
820  iovv[0] = iovv[out_iov_idx];
821  out_iov_idx = 1;
822  }
823  else
824  {
825  ++in_iov_idx;
826  out_iov_idx = 0;
827  }
828  }
829  } while (total_written_bytes < total_to_write_bytes);
830  if (total_written_bytes > total_to_write_bytes)
831  TLOG(TLVL_ERROR) << GetTraceName() << ": sendFragment program error: too many bytes transferred";
832 
833  if (blocking)
834  {
835  blocking = false;
836 #if !USE_SENDMSG
837  fcntl(send_fd_, F_SETFL, O_NONBLOCK); // set O_NONBLOCK
838 #endif
839  }
840  sts = total_written_bytes - sizeof(MessHead);
841 
842  TLOG(14) << GetTraceName() << ": sendFragment sts=" << sts;
844 }
845 
846 void artdaq::TCPSocketTransfer::connect_()
847 {
848  auto start_time = std::chrono::steady_clock::now();
849 
850  // Retry a few times if we can't connect
851  while (send_fd_ == -1 && TimeUtils::GetElapsedTimeMicroseconds(start_time) < send_retry_timeout_us_ * 10)
852  {
853  TLOG(TLVL_DEBUG) << GetTraceName() << ": Connecting sender socket";
854  int sndbuf_bytes = static_cast<int>(sndbuf_);
855  if (sndbuf_ > INT_MAX)
856  {
857  sndbuf_bytes = INT_MAX;
858  TLOG(TLVL_WARNING) << "Requested SNDBUF " << sndbuf_ << " too large, setting to INT_MAX: " << INT_MAX;
859  }
860  TLOG(TLVL_DEBUG) << "Requested SNDBUF is " << sndbuf_bytes;
861 
862  send_fd_ = TCPConnect(hostMap_[destination_rank()].c_str(), portMan->GetTCPSocketTransferPort(destination_rank()), O_NONBLOCK, sndbuf_bytes);
863  if (send_fd_ == -1)
864  {
865  if (connection_was_lost_) { break; }
866  else
867  {
868  usleep(send_retry_timeout_us_);
869  }
870  }
871  }
872  connect_state = 0;
873  blocking = 0;
874  TLOG(TLVL_DEBUG) << GetTraceName() << ": connect_ " + hostMap_[destination_rank()] + ":" << portMan->GetTCPSocketTransferPort(destination_rank()) << " send_fd_=" << send_fd_;
875  if (send_fd_ != -1)
876  {
877  // write connect msg
878  TLOG(TLVL_DEBUG) << GetTraceName() << ": connect_: Writing connect message";
879  MessHead mh = {0, MessHead::connect_v0, htons(source_rank()), {htonl(CONN_MAGIC)}};
880  ssize_t sts = write(send_fd_, &mh, sizeof(mh));
881  if (sts == -1)
882  {
883  TLOG(TLVL_ERROR) << GetTraceName() << ": connect_: Error writing connect message!";
884  // a write error here is completely unexpected!
885  connect_state = 0;
886  close(send_fd_);
887  send_fd_ = -1;
888  }
889  else
890  {
891  TLOG(TLVL_INFO) << GetTraceName() << ": connect_: Successfully connected";
892  // consider it all connected/established
893  connect_state = 1;
894  connection_was_lost_ = false;
895  }
896 
897 #if USE_ACKS
898  if (ack_listen_thread_ && ack_listen_thread_->joinable()) ack_listen_thread_->join();
899  TLOG(TLVL_INFO) << GetTraceName() << ": Starting Ack Listener Thread";
900 
901  try
902  {
903  ack_listen_thread_ = std::make_unique<boost::thread>(&TCPSocketTransfer::receive_acks_, this);
904  }
905  catch (const boost::exception& e)
906  {
907  TLOG(TLVL_ERROR) << "Caught boost::exception starting TCP Socket Ack Listen thread: " << boost::diagnostic_information(e) << ", errno=" << errno;
908  std::cerr << "Caught boost::exception starting TCP Socket Ack Listen thread: " << boost::diagnostic_information(e) << ", errno=" << errno << std::endl;
909  exit(5);
910  }
911 #endif
912  }
913 }
914 
915 void artdaq::TCPSocketTransfer::reconnect_()
916 {
917  if (send_fd_ == -1 && role() == TransferInterface::Role::kSend)
918  {
919  TLOG(TLVL_TRACE) << GetTraceName() << ": check/reconnect";
920  return connect_();
921  }
922 }
923 
924 void artdaq::TCPSocketTransfer::start_listen_thread_()
925 {
926  std::unique_lock<std::mutex> start_lock(listen_thread_mutex_);
927  if (listen_thread_refcount_ == 0)
928  {
929  if (listen_thread_ && listen_thread_->joinable()) listen_thread_->join();
930  listen_thread_refcount_ = 1;
931  TLOG(TLVL_INFO) << GetTraceName() << ": Starting Listener Thread";
932 
933  try
934  {
935  listen_thread_ = std::make_unique<boost::thread>(&TCPSocketTransfer::listen_, portMan->GetTCPSocketTransferPort(destination_rank()), rcvbuf_);
936  }
937  catch (const boost::exception& e)
938  {
939  TLOG(TLVL_ERROR) << "Caught boost::exception starting TCP Socket Listen thread: " << boost::diagnostic_information(e) << ", errno=" << errno;
940  std::cerr << "Caught boost::exception starting TCP Socket Listen thread: " << boost::diagnostic_information(e) << ", errno=" << errno << std::endl;
941  exit(5);
942  }
943  }
944  else
945  {
946  listen_thread_refcount_++;
947  }
948 }
949 
950 #if USE_ACKS
951 void artdaq::TCPSocketTransfer::receive_acks_()
952 {
953  while (send_fd_ >= 0)
954  {
955  pollfd pollfd_s;
956  pollfd_s.events = POLLIN | POLLPRI;
957  pollfd_s.fd = send_fd_;
958 
959  TLOG(18) << GetTraceName() << ": receive_acks_: Polling fd to see if there's data";
960  int num_fds_ready = poll(&pollfd_s, 1, 1000);
961  if (num_fds_ready <= 0)
962  {
963  if (num_fds_ready == 0)
964  {
965  TLOG(18) << GetTraceName() << ": receive_acks_: No data on receive socket";
966  continue;
967  }
968 
969  TLOG(TLVL_ERROR) << "Error in poll: errno=" << errno;
970  break;
971  }
972 
973  if (pollfd_s.revents & (POLLIN | POLLPRI))
974  {
975  // Expected, don't have to check revents any further
976  }
977  else
978  {
979  TLOG(TLVL_DEBUG) << GetTraceName() << ": receive_acks_: Wrong event received from pollfd: " << pollfd_s.revents;
980  break;
981  }
982 
983  MessHead mh;
984  auto sts = read(send_fd_, &mh, sizeof(mh));
985 
986  if (sts != sizeof(mh))
987  {
988  TLOG(TLVL_ERROR) << GetTraceName() << ": receive_ack_: Wrong message header length received! (actual " << sts << " != " << sizeof(mh) << " expected)";
989  continue;
990  }
991 
992  // check for "magic" and valid source_id(aka rank)
993  mh.source_id = ntohs(mh.source_id); // convert here as it is reference several times
994  if (mh.source_id != my_rank)
995  {
996  TLOG(TLVL_ERROR) << GetTraceName() << ": receive_ack_: Received ack for different sender! Rank=" << my_rank << ", hdr=" << mh.source_id;
997  continue;
998  }
999  if (ntohl(mh.conn_magic) != ACK_MAGIC || !(mh.message_type == MessHead::ack_v0)) // Allow for future connect message versions
1000  {
1001  TLOG(TLVL_ERROR) << GetTraceName() << ": receive_ack_: Wrong magic bytes in header!";
1002  continue;
1003  }
1004 
1005  TLOG(17) << GetTraceName() << ": receive_acks_: Received ack message, diff is now " << (send_ack_diff_.load() - 1);
1006  send_ack_diff_--;
1007  }
1008 }
1009 
1010 void artdaq::TCPSocketTransfer::send_ack_(int fd)
1011 {
1012  MessHead mh = {0, MessHead::ack_v0, htons(source_rank()), {htonl(ACK_MAGIC)}};
1013  write(fd, &mh, sizeof(mh));
1014 }
1015 #endif
1016 
1017 void artdaq::TCPSocketTransfer::listen_(int port, size_t rcvbuf)
1018 {
1019  int listen_fd = -1;
1020  while (listen_thread_refcount_ > 0)
1021  {
1022  TLOG(TLVL_TRACE) << "listen_: Listening/accepting new connections on port " << port;
1023  if (listen_fd == -1)
1024  {
1025  TLOG(TLVL_DEBUG) << "listen_: Opening listener";
1026  listen_fd = TCP_listen_fd(port, rcvbuf);
1027  }
1028  if (listen_fd == -1)
1029  {
1030  TLOG(TLVL_DEBUG) << "listen_: Error creating listen_fd!";
1031  break;
1032  }
1033 
1034  int res;
1035  timeval tv = {2, 0}; // maybe increase of some global "debugging" flag set???
1036  fd_set rfds;
1037  FD_ZERO(&rfds);
1038  FD_SET(listen_fd, &rfds);
1039 
1040  res = select(listen_fd + 1, &rfds, (fd_set*)0, (fd_set*)0, &tv);
1041  if (res > 0)
1042  {
1043  int sts;
1044  sockaddr_un un;
1045  socklen_t arglen = sizeof(un);
1046  int fd;
1047  TLOG(TLVL_DEBUG) << "listen_: Calling accept";
1048  fd = accept(listen_fd, (sockaddr*)&un, &arglen);
1049  TLOG(TLVL_DEBUG) << "listen_: Done with accept";
1050 
1051  TLOG(TLVL_DEBUG) << "listen_: Reading connect message";
1052  socklen_t lenlen = sizeof(tv);
1053  /*sts=*/
1054  setsockopt(fd, SOL_SOCKET, SO_RCVTIMEO, &tv, lenlen); // see man 7 socket.
1055  MessHead mh;
1056  uint64_t mark_us = TimeUtils::gettimeofday_us();
1057  sts = read(fd, &mh, sizeof(mh));
1058  uint64_t delta_us = TimeUtils::gettimeofday_us() - mark_us;
1059  TLOG(TLVL_DEBUG) << "listen_: Read of connect message took " << delta_us << " microseconds.";
1060  if (sts != sizeof(mh))
1061  {
1062  TLOG(TLVL_DEBUG) << "listen_: Wrong message header length received!";
1063  close(fd);
1064  continue;
1065  }
1066 
1067  // check for "magic" and valid source_id(aka rank)
1068  mh.source_id = ntohs(mh.source_id); // convert here as it is reference several times
1069  if (ntohl(mh.conn_magic) != CONN_MAGIC || !(mh.message_type == MessHead::connect_v0)) // Allow for future connect message versions
1070  {
1071  TLOG(TLVL_DEBUG) << "listen_: Wrong magic bytes in header!";
1072  close(fd);
1073  continue;
1074  }
1075 
1076  // now add (new) connection
1077  std::unique_lock<std::mutex> lk(connected_fd_mutex_);
1078  connected_fds_[mh.source_id].insert(fd);
1079 
1080  TLOG(TLVL_INFO) << "listen_: New fd is " << fd << " for source rank " << mh.source_id;
1081  }
1082  else
1083  {
1084  TLOG(16) << "listen_: No connections in timeout interval!";
1085  }
1086  }
1087 
1088  TLOG(TLVL_INFO) << "listen_: Shutting down connection listener";
1089  if (listen_fd != -1) close(listen_fd);
1090  std::unique_lock<std::mutex> lk(connected_fd_mutex_);
1091  auto it = connected_fds_.begin();
1092  while (it != connected_fds_.end())
1093  {
1094  auto& fd_set = it->second;
1095  auto rank_it = fd_set.begin();
1096  while (rank_it != fd_set.end())
1097  {
1098  close(*rank_it);
1099  rank_it = fd_set.erase(rank_it);
1100  }
1101  it = connected_fds_.erase(it);
1102  }
1103 
1104 } // do_connect_
1105 
1106 DEFINE_ARTDAQ_TRANSFER(artdaq::TCPSocketTransfer)
bool isRunning() override
Determine whether the TransferInterface plugin is able to send/receive data.
virtual int source_rank() const
Get the source rank for this TransferInterface instance.
int TCPConnect(char const *host_in, int dflt_port, long flags=0, int sndbufsiz=0)
Connect to a host on a given port.
Definition: TCPConnect.cc:361
uint32_t conn_magic
unsigned first is better for MessHead initializer: {0,0,my_node_idx_,CONN_MAGIC}
Definition: SRSockets.hh:40
hostMap_t MakeHostMap(fhicl::ParameterSet pset, hostMap_t map=hostMap_t())
Make a hostMap_t from a HostMap::Config ParameterSet
Definition: HostMap.hh:66
This TransferInterface is a Receiver.
int receiveFragmentData(RawDataType *destination, size_t wordCount) override
Receive the body of a Fragment to the given destination pointer.
int TCP_listen_fd(int port, int rcvbuf)
Create a TCP listening socket on the given port and INADDR_ANY, with the given receive buffer...
TCPSocketTransfer(fhicl::ParameterSet const &ps, Role role)
TCPSocketTransfer Constructor.
int receiveFragmentHeader(detail::RawFragmentHeader &header, size_t receiveTimeout) override
Receive a Fragment Header from the transport mechanism.
This TransferInterface is a Sender.
void flush_buffers() override
Flush any in-flight data. This should be used by the receiver after the receive loop has ended...
int32_t byte_count
use CONN_MAGIC for connect_v0, data that follow for data_v0 (and 0 lenght data)
Definition: SRSockets.hh:41
Some error occurred, but no exception was thrown.
Role
Used to determine if a TransferInterface is a Sender or Receiver.
int64_t source_id
Rank of the source.
Definition: SRSockets.hh:36
MessType message_type
Message Type.
Definition: SRSockets.hh:35
The send operation completed successfully.
This interface defines the functions used to transfer data between artdaq applications.
TransferInterface implementation plugin that sends data using TCP sockets.
This header is sent by the TCPSocket_transfer to allow for more efficient writev calls.
Definition: SRSockets.hh:15
CopyStatus
Returned from the send functions, this enumeration describes the possible return codes. If an exception occurs, it will be thrown and should be handled normally.