artdaq  v3_06_00
TCPSocket_transfer.cc
1 // Sep 14, 2016. "TERMS AND CONDITIONS" governing this file are in the README
2 // or COPYING file. If you do not have such a file, one can be obtained by
3 // contacting Ron or Fermi Lab in Batavia IL, 60510, phone: 630-840-3000.
4 // $RCSfile: .emacs.gnu,v $
5 // rev="$Revision: 1.30 $$Date: 2016/03/01 14:27:27 $";
6 
7 // C Includes
8 #include <stdlib.h> // atoi, strtoul
9 #include <sys/socket.h> // socket, socklen_t
10 #include <sys/un.h> // sockaddr_un
11 #include <arpa/inet.h> // ntohl, ntohs
12 #include <sys/types.h> // size_t
13 #include <poll.h> // struct pollfd
14 
15 // C++ Includes
16 #include <string>
17 #include <fstream>
18 #include <stdexcept>
19 
20 // product Includes
21 #include "artdaq/DAQdata/Globals.hh"
22 #define TRACE_NAME (app_name + "_TCPSocketTransfer").c_str()
23 
24 // artdaq Includes
25 #include "artdaq/TransferPlugins/TCPSocketTransfer.hh"
28 #include "artdaq/TransferPlugins/detail/Timeout.hh"
29 #include "artdaq/TransferPlugins/detail/SRSockets.hh"
30 #include "artdaq-core/Data/Fragment.hh"
31 #include "artdaq-core/Utilities/TimeUtils.hh"
32 #include <iomanip>
33 
34 std::atomic<int> artdaq::TCPSocketTransfer::listen_thread_refcount_(0);
35 std::unique_ptr<boost::thread> artdaq::TCPSocketTransfer::listen_thread_ = nullptr;
36 std::map<int, std::set<int>> artdaq::TCPSocketTransfer::connected_fds_ = std::map<int, std::set<int>>();
37 std::mutex artdaq::TCPSocketTransfer::listen_thread_mutex_;
38 std::mutex artdaq::TCPSocketTransfer::connected_fd_mutex_;
39 
41 TCPSocketTransfer(fhicl::ParameterSet const& pset, TransferInterface::Role role)
42  : TransferInterface(pset, role)
43  , send_fd_(-1)
44  , active_receive_fd_(-1)
45  , last_active_receive_fd_(-1)
46  , rcvbuf_(pset.get<size_t>("tcp_receive_buffer_size", 0))
47  , sndbuf_(pset.get<size_t>("tcp_send_buffer_size", max_fragment_size_words_ * sizeof(artdaq::RawDataType) * buffer_count_))
48  , send_retry_timeout_us_(pset.get<size_t>("send_retry_timeout_us", 1000000))
49  , timeoutMessageArmed_(true)
50  , last_recv_time_()
51  , receive_disconnected_wait_s_(pset.get<double>("receive_socket_disconnected_wait_s", 10.0))
52  , receive_err_wait_us_(pset.get<size_t>("receive_socket_disconnected_wait_us", 10000))
53  , receive_socket_has_been_connected_(false)
54  , send_ack_diff_(0)
55 {
56  TLOG(TLVL_DEBUG) << GetTraceName() << " Constructor: pset=" << pset.to_string() << ", role=" << (role == TransferInterface::Role::kReceive ? "kReceive" : "kSend");
57  connection_was_lost_ = false;
58 
60  {
61  // Wait for sender to connect...
62  TLOG(TLVL_DEBUG) << GetTraceName() << ": Listening for connections";
63  start_listen_thread_();
64  TLOG(TLVL_DEBUG) << GetTraceName() << ": Done Listening";
65  }
66  else
67  {
68  hostMap_ = MakeHostMap(pset);
69  TLOG(TLVL_DEBUG) << GetTraceName() << ": Connecting to destination";
70  connect_();
71  TLOG(TLVL_DEBUG) << GetTraceName() << ": Done Connecting";
72  }
73  TLOG(TLVL_DEBUG) << GetTraceName() << ": End of Constructor";
74 }
75 
76 artdaq::TCPSocketTransfer::~TCPSocketTransfer() noexcept
77 {
78  TLOG(TLVL_DEBUG) << GetTraceName() << ": Shutting down TCPSocketTransfer";
79 
80  if (role() == TransferInterface::Role::kSend)
81  {
82  // close all open connections (send stop_v0) first
83  MessHead mh = { 0,MessHead::stop_v0,htons(TransferInterface::source_rank()),{0} };
84  if (send_fd_ != -1)
85  {
86  // should be blocking with modest timeo
87  timeval tv = { 0,100000 };
88  socklen_t len = sizeof(tv);
89  setsockopt(send_fd_, SOL_SOCKET, SO_SNDTIMEO, &tv, len);
90  write(send_fd_, &mh, sizeof(mh));
91  }
92  close(send_fd_);
93  send_fd_ = -1;
94  }
95  else
96  {
97  {
98  std::unique_lock<std::mutex> fd_lock(connected_fd_mutex_);
99  if (connected_fds_.count(source_rank()))
100  {
101  auto it = connected_fds_[source_rank()].begin();
102  while (it != connected_fds_[source_rank()].end())
103  {
104  close(*it);
105  it = connected_fds_[source_rank()].erase(it);
106  }
107  connected_fds_.erase(source_rank());
108  }
109  if (ack_listen_thread_ && ack_listen_thread_->joinable()) ack_listen_thread_->join();
110  }
111 
112  std::unique_lock<std::mutex> lk(listen_thread_mutex_);
113  listen_thread_refcount_--;
114  if (listen_thread_refcount_ <= 0 && listen_thread_ && listen_thread_->joinable())
115  {
116  listen_thread_->join();
117  }
118  }
119 
120  TLOG(TLVL_DEBUG) << GetTraceName() << ": End of Destructor";
121 }
122 
123 int artdaq::TCPSocketTransfer::receiveFragmentHeader(detail::RawFragmentHeader& header, size_t timeout_usec)
124 {
125  TLOG(5) << GetTraceName() << ": receiveFragmentHeader: BEGIN";
126  int ret_rank = RECV_TIMEOUT;
127 
128  // Don't bomb out until received at least one connection...
129  if (getConnectedFDCount(source_rank()) == 0)
130  { // what if just listen_fd???
131  // if (receive_socket_has_been_connected_ && TimeUtils::GetElapsedTime(last_recv_time_) > receive_disconnected_wait_s_)
132  // {
133  // TLOG(TLVL_ERROR) << GetTraceName() << ": receiveFragmentHeader: senders have been disconnected for "
134  // << TimeUtils::GetElapsedTime(last_recv_time_) << " s (receive_socket_disconnected_wait_s = " << receive_disconnected_wait_s_ << " s). RETURNING DATA_END!";
135  // return DATA_END;
136  // }
137  //if (++not_connected_count_ > receive_err_threshold_) { return DATA_END; }
138  TLOG(7) << GetTraceName() << ": receiveFragmentHeader: Receive socket not connected, returning RECV_TIMEOUT";
139  usleep(receive_err_wait_us_);
140  return RECV_TIMEOUT;
141  }
142  receive_socket_has_been_connected_ = true;
143  last_recv_time_ = std::chrono::steady_clock::now();
144 
145  TLOG(5) << GetTraceName() << ": receiveFragmentHeader timeout_usec=" << timeout_usec;
146  //void* buff=alloca(max_fragment_size_words_*8);
147  size_t byte_cnt = 0;
148  int sts;
149  int offset = 0;
150  SocketState state = SocketState::Metadata;
151  int target_bytes = sizeof(MessHead);
152  uint64_t start_time_us = TimeUtils::gettimeofday_us();
153 
154  //while (active_receive_fd_ != -1)
155  //{
156  // TLOG(TLVL_TRACE) << GetTraceName() << ": Currently receiving from fd " << active_receive_fd_ << ", waiting!";
157  // usleep(1000);
158  //}
159 
160 
161  uint8_t* buff;
162 
163  int timeout_ms;
164  if (timeout_usec == 0)
165  timeout_ms = 0;
166  else
167  timeout_ms = (timeout_usec + 999) / 1000; // want at least 1 ms
168 
169  bool done = false;
170  bool noDataWarningSent = false;
171  int loop_guard = 0;
172 
173  while (!done && getConnectedFDCount(source_rank()) > 0)
174  {
175  if (active_receive_fd_ == -1)
176  {
177  loop_guard = 0;
178  size_t fd_count = 0;
179  std::vector<pollfd> pollfds;
180  {
181  std::unique_lock<std::mutex> lk(connected_fd_mutex_);
182  fd_count = connected_fds_[source_rank()].size();
183  pollfds.resize(fd_count);
184  auto iter = connected_fds_[source_rank()].begin();
185  for (size_t ii = 0; ii < fd_count; ++ii)
186  {
187  pollfds[ii].events = POLLIN | POLLPRI | POLLERR;
188  pollfds[ii].fd = *iter;
189  ++iter;
190  }
191  }
192  //TLOG(TLVL_DEBUG) << GetTraceName() << ": receiveFragment: Polling fd to see if there's data" ;
193  int num_fds_ready = poll(&pollfds[0], fd_count, timeout_ms);
194  if (num_fds_ready <= 0)
195  {
196  TLOG(5) << GetTraceName() << ": receiveFragmentHeader: No data on receive socket, returning RECV_TIMEOUT";
197  return RECV_TIMEOUT;
198  }
199 
200  size_t index = 0;
201  if (last_active_receive_fd_ != -1)
202  {
203  for (auto& pollfd : pollfds)
204  {
205  index++;
206  if (pollfd.fd == last_active_receive_fd_)
207  {
208  break;
209  }
210  }
211  }
212 
213  int active_index = -1;
214  short anomolous_events = 0;
215  for (size_t ii = index; ii < index + pollfds.size(); ++ii)
216  {
217  auto pollfd_index = (ii + index) % pollfds.size();
218  if (pollfds[pollfd_index].revents & (POLLIN | POLLPRI))
219  {
220  active_index = pollfd_index;
221  active_receive_fd_ = pollfds[active_index].fd;
222  active_revents_ = pollfds[active_index].revents;;
223  break;
224  }
225  else if (pollfds[pollfd_index].revents & (POLLHUP | POLLERR))
226  {
227  disconnect_receive_socket_(pollfds[pollfd_index].fd, "Poll returned POLLHUP or POLLERR, indicating problems with the sender.");
228  continue;
229  }
230  else if (pollfds[pollfd_index].revents & (POLLNVAL))
231  {
232  TLOG(TLVL_DEBUG) << GetTraceName() << ": receiveFragmentHeader: FD is closed, most likely because the peer went away. Removing from fd list.";
233  disconnect_receive_socket_(pollfds[pollfd_index].fd, "FD is closed, most likely because the peer went away.");
234  continue;
235  }
236  else if (pollfds[pollfd_index].revents)
237  {
238  anomolous_events |= pollfds[pollfd_index].revents;
239  }
240  }
241 
242  if (active_index == -1)
243  {
244  if (anomolous_events)
245  TLOG(TLVL_DEBUG) << GetTraceName() << ": receiveFragmentHeader: Wrong event received from a pollfd. Mask: " << static_cast<int>(anomolous_events);
246  active_receive_fd_ = -1;
247  continue;
248  }
249 
250  if (!done && timeout_usec > 0)
251  {
252  // calc next timeout_ms (unless timed out)
253  size_t delta_us = TimeUtils::gettimeofday_us() - start_time_us;
254  if (delta_us > timeout_usec)
255  {
256  return RECV_TIMEOUT;
257  }
258  timeout_ms = ((timeout_usec - delta_us) + 999) / 1000; // want at least 1 ms
259  }
260  }
261  if (loop_guard > 10) { usleep(1000); }
262  if (++loop_guard > 10010)
263  {
264  TLOG(TLVL_WARNING) << GetTraceName() << ": receiveFragmentHeader: loop guard triggered, returning RECV_TIMEOUT";
265  usleep(receive_err_wait_us_);
266  active_receive_fd_ = -1;
267  return RECV_TIMEOUT;
268  }
269 
270  if (state == SocketState::Metadata)
271  {
272  //TLOG(TLVL_DEBUG) << GetTraceName() << ": receiveFragmentHeader: Reading Message Header" ;
273  buff = &(mha[offset]);
274  byte_cnt = sizeof(MessHead) - offset;
275  }
276  else
277  {
278  //TLOG(TLVL_DEBUG) << GetTraceName() << ": receiveFragmentHeader: Reading data" ;
279  buff = reinterpret_cast<uint8_t*>(&header) + offset;
280  byte_cnt = target_bytes - offset;
281  }
282  //if (byte_cnt > sizeof(MessHead))
283  // {
284  // TLOG(TLVL_ERROR) << "Invalid byte count for read (count=" << byte_cnt
285  // << ",offset=" << offset << ",mh.byte_count=" << mh.byte_count
286  // << "), skipping read and returning RECV_TIMEOUT";
287  // return RECV_TIMEOUT;
288  //}
289 
290  if (byte_cnt > 0)
291  {
292  TLOG(6) << GetTraceName() << ": receiveFragmentHeader: Reading " << byte_cnt << " bytes from socket";
293  sts = read(active_receive_fd_, buff, byte_cnt);
294  TLOG(6) << GetTraceName() << ": receiveFragmentHeader: Done with read";
295  }
296  if (sts > 0) {
297  loop_guard = 0;
298  last_recv_time_ = std::chrono::steady_clock::now();
299  }
300 
301  TLOG(7) << GetTraceName() << ": receiveFragmentHeader state=" << static_cast<int>(state) << " read=" << sts;
302  if (sts < 0)
303  {
304  TLOG(TLVL_WARNING) << GetTraceName() << ": receiveFragmentHeader: Error on receive, closing socket " << " (errno=" << errno << ": " << strerror(errno) << ")";
305  active_receive_fd_ = disconnect_receive_socket_(active_receive_fd_);
306  }
307  else if (sts == 0)
308  {
309  if (!noDataWarningSent) {
310  TLOG(TLVL_WARNING) << GetTraceName() << ": receiveFragmentHeader: No data received, is the sender still sending?!?";
311  noDataWarningSent = true;
312  }
313  if (TimeUtils::GetElapsedTime(last_recv_time_) > receive_disconnected_wait_s_)
314  {
315  TLOG(TLVL_ERROR) << GetTraceName() << ": receiveFragmentHeader: No data received within timeout, aborting!";
316  return RECV_TIMEOUT;
317  }
318  }
319  else
320  {
321  // see if we're done (with this state)
322  sts = offset += sts;
323  if (sts >= target_bytes)
324  {
325  TLOG(7) << GetTraceName() << ": receiveFragmentHeader: Target read bytes reached. Changing state";
326  offset = 0;
327  if (state == SocketState::Metadata)
328  {
329  state = SocketState::Data;
330  mh.byte_count = ntohl(mh.byte_count);
331  mh.source_id = ntohs(mh.source_id);
332  target_bytes = mh.byte_count;
333  TLOG(7) << GetTraceName() << ": receiveFragmentHeader: Expected header size = " << target_bytes << ", sizeof(RawFragmentHeader) = " << sizeof(artdaq::detail::RawFragmentHeader);
334  //assert(target_bytes == sizeof(artdaq::detail::RawFragmentHeader) || target_bytes == 0);
335 
336  if (mh.message_type == MessHead::stop_v0)
337  {
338  active_receive_fd_ = disconnect_receive_socket_(active_receive_fd_, "Stop Message received.");
339  }
340  else if (mh.message_type == MessHead::data_v0 || mh.message_type == MessHead::data_more_v0)
341  {
342  TLOG(TLVL_WARNING) << GetTraceName() << ": receiveFragmentHeader: Message header indicates that Fragment data follows when I was expecting a Fragment header!";
343  active_receive_fd_ = disconnect_receive_socket_(active_receive_fd_, "Desync detected");
344  }
345 
346  if (target_bytes == 0)
347  {
348  //Probably a stop_v0, return timeout so we can try again.
349  return RECV_TIMEOUT;
350  }
351  }
352  else
353  {
354  ret_rank = source_rank();
355  TLOG(8) << GetTraceName() << ": receiveFragmentHeader done sts=" << sts << " src=" << ret_rank;
356  TLOG(7) << GetTraceName() << ": receiveFragmentHeader: Done receiving fragment header. Moving into output.";
357 
358  done = true; // no more polls
359  //break; // no more read of ready fds
360  }
361  }
362  }
363 
364  } // while(!done)...poll
365 
366  TLOG(5) << GetTraceName() << ": receiveFragmentHeader: Returning " << ret_rank;
367  return ret_rank;
368 }
369 
370 int artdaq::TCPSocketTransfer::disconnect_receive_socket_(int fd, std::string msg)
371 {
372  TLOG(TLVL_WARNING) << GetTraceName() << ": disconnect_receive_socket_: " << msg << " Closing socket " << fd;
373  close(fd);
374  std::unique_lock<std::mutex> lk(connected_fd_mutex_);
375  if (connected_fds_.count(source_rank()))
376  connected_fds_[source_rank()].erase(fd);
377  fd = -1;
378  TLOG(TLVL_DEBUG) << GetTraceName() << ": disconnect_receive_socket_: There are now " << connected_fds_[source_rank()].size() << " active senders.";
379  return fd;
380 }
381 
382 int artdaq::TCPSocketTransfer::receiveFragmentData(RawDataType* destination, size_t)
383 {
384  TLOG(19) << GetTraceName() << ": receiveFragmentData: BEGIN";
385  int ret_rank = RECV_TIMEOUT;
386  if (active_receive_fd_ == -1)
387  { // what if just listen_fd???
388  TLOG(TLVL_ERROR) << GetTraceName() << ": receiveFragmentData: Receive socket not connected, returning RECV_TIMEOUT (Will result in \"Unexpected return code error\")";
389  return RECV_TIMEOUT;
390  }
391 
392  //void* buff=alloca(max_fragment_size_words_*8);
393  uint8_t* buff;
394  size_t byte_cnt = 0;
395  int sts;
396  int offset = 0;
397  SocketState state = SocketState::Metadata;
398  int target_bytes = sizeof(MessHead);
399 
400  pollfd pollfd_s;
401  pollfd_s.events = POLLIN | POLLPRI | POLLERR;
402  pollfd_s.fd = active_receive_fd_;
403 
404  int loop_guard = 0;
405  bool done = false;
406  bool noDataWarningSent = false;
407  last_recv_time_ = std::chrono::steady_clock::now();
408  while (!done)
409  {
410  TLOG(9) << GetTraceName() << ": receiveFragmentData: Polling fd to see if there's data";
411  int num_fds_ready = poll(&pollfd_s, 1, 1000);
412  TLOG(TLVL_TRACE) << GetTraceName() << ": receiveFragmentData: Polled fd to see if there's data"
413  << ", num_fds_ready = " << num_fds_ready;
414  if (num_fds_ready <= 0)
415  {
416  if (num_fds_ready == 0)
417  {
418  TLOG(TLVL_WARNING) << GetTraceName() << ": receiveFragmentData: No data from " << source_rank() << " in " << TimeUtils::GetElapsedTimeMilliseconds(last_recv_time_) << " ms!"
419  << " State = " << (state == SocketState::Metadata ? "Metadata" : "Data") << ", recvd/total=" << offset << "/" << target_bytes << " (delta=" << target_bytes - offset << ")";
420 
421  if (TimeUtils::GetElapsedTime(last_recv_time_) > receive_disconnected_wait_s_)
422  {
423  TLOG(TLVL_WARNING) << GetTraceName() << ": receiveFragmentData: No data received within timeout (" << TimeUtils::GetElapsedTime(last_recv_time_) << " / " << receive_disconnected_wait_s_ << " ), returning RECV_TIMEOUT";
424  disconnect_receive_socket_(active_receive_fd_, "No data on this socket within timeout");
425  active_receive_fd_ = -1;
426  return RECV_TIMEOUT;
427  }
428  continue;
429  }
430 
431  TLOG(TLVL_ERROR) << "Error in poll: errno=" << errno;
432  active_receive_fd_ = -1;
433  break;
434  }
435  else { last_recv_time_ = std::chrono::steady_clock::now(); }
436 
437  if (pollfd_s.revents & (POLLIN | POLLPRI))
438  {
439  // Expected, don't have to check revents any further
440  }
441  else if (pollfd_s.revents & (POLLNVAL))
442  {
443  disconnect_receive_socket_(pollfd_s.fd, "FD is closed, most likely because the peer went away.");
444  break;
445  }
446  else if (pollfd_s.revents & (POLLHUP | POLLERR))
447  {
448  disconnect_receive_socket_(pollfd_s.fd, "Poll returned POLLHUP or POLLERR, indicating problems with the sender.");
449  break;
450  }
451  else
452  {
453  TLOG(TLVL_WARNING) << GetTraceName() << ": receiveFragmentData: Wrong event received from pollfd: " << pollfd_s.revents;
454  disconnect_receive_socket_(pollfd_s.fd);
455  break;
456  }
457 
458  if (state == SocketState::Metadata)
459  {
460  //TLOG(TLVL_DEBUG) << GetTraceName() << ": receiveFragmentData: Reading Message Header" ;
461  buff = &(mha[offset]);
462  byte_cnt = sizeof(MessHead) - offset;
463  }
464  else
465  {
466  //TLOG(TLVL_DEBUG) << GetTraceName() << ": receiveFragmentData: Reading data" ;
467  buff = reinterpret_cast<uint8_t*>(destination) + offset;
468  byte_cnt = mh.byte_count - offset;
469  }
470 
471  TLOG(10) << GetTraceName() << ": receiveFragmentData: Reading " << byte_cnt << " bytes from socket into " << (void*)buff;
472  sts = read(active_receive_fd_, buff, byte_cnt);
473  //TLOG(TLVL_DEBUG) << GetTraceName() << ": receiveFragmentData: Done with read" ;
474 
475  TLOG(10) << GetTraceName() << ": recvFragment state=" << static_cast<int>(state) << " read=" << sts;
476 
477  if (sts == 0)
478  {
479  if (loop_guard > 10) { usleep(1000); }
480  if (++loop_guard > 10010)
481  {
482  TLOG(TLVL_WARNING) << GetTraceName() << ": receiveFragmentData: loop guard triggered, returning RECV_TIMEOUT";
483  active_receive_fd_ = -1;
484  return RECV_TIMEOUT;
485  }
486  }
487  else if(sts > 0)
488  {
489  loop_guard = 0;
490  last_recv_time_ = std::chrono::steady_clock::now();
491  }
492 
493  if (sts < 0)
494  {
495  TLOG(TLVL_WARNING) << GetTraceName() << ": receiveFragmentData: Error on receive, closing socket"
496  << " (errno=" << errno << ": " << strerror(errno) << ")";
497  disconnect_receive_socket_(pollfd_s.fd);
498  }
499  else if (sts == 0)
500  {
501  if (!noDataWarningSent) {
502  TLOG(TLVL_WARNING) << GetTraceName() << ": receiveFragmentData: No data received, is the sender still sending?!?";
503  noDataWarningSent = true;
504  }
505  if (TimeUtils::GetElapsedTime(last_recv_time_) > receive_disconnected_wait_s_)
506  {
507  TLOG(TLVL_ERROR) << GetTraceName() << ": receiveFragmentData: No data received within timeout, aborting!";
508  return RECV_TIMEOUT;
509  }
510  }
511  else
512  {
513  // see if we're done (with this state)
514  sts = offset += sts;
515  if (sts >= target_bytes)
516  {
517  TLOG(9) << GetTraceName() << ": receiveFragmentData: Target read bytes reached. Changing state";
518  offset = 0;
519  if (state == SocketState::Metadata)
520  {
521  state = SocketState::Data;
522  mh.byte_count = ntohl(mh.byte_count);
523  mh.source_id = ntohs(mh.source_id);
524  target_bytes = mh.byte_count;
525 
526 
527  if (mh.message_type == MessHead::header_v0)
528  {
529  TLOG(TLVL_WARNING) << GetTraceName() << ": receiveFragmentData: Message header indicates that a Fragment header follows when I was expecting Fragment data!";
530  active_receive_fd_ = disconnect_receive_socket_(active_receive_fd_, "Desync detected");
531  }
532  }
533  else
534  {
535  ret_rank = source_rank();
536  TLOG(11) << GetTraceName() << ": receiveFragmentData done sts=" << sts << " src=" << ret_rank;
537  TLOG(9) << GetTraceName() << ": receiveFragmentData: Done receiving fragment. Moving into output.";
538 
539 #if USE_ACKS
540  send_ack_(active_receive_fd_);
541 #endif
542 
543  done = true; // no more polls
544  //break; // no more read of ready fds
545  }
546  }
547  }
548 
549  // Check if we were asked to do a 0-size receive
550  if (target_bytes == 0 && state == SocketState::Data)
551  {
552  ret_rank = source_rank();
553  TLOG(11) << GetTraceName() << ": receiveFragmentData done sts=" << sts << " src=" << ret_rank;
554  TLOG(9) << GetTraceName() << ": receiveFragmentData: Done receiving fragment. Moving into output.";
555 
556 #if USE_ACKS
557  send_ack_(active_receive_fd_);
558 #endif
559 
560  done = true; // no more polls
561  }
562 
563  } // while(!done)...poll
564 
565  last_active_receive_fd_ = active_receive_fd_;
566  active_receive_fd_ = -1;
567 
568  TLOG(9) << GetTraceName() << ": receiveFragmentData: Returning rank " << ret_rank;
569  return ret_rank;
570 }
571 
573 {
574  switch (role())
575  {
577  return send_fd_ != -1;
579  TLOG(TLVL_DEBUG) << GetTraceName() << ": isRunning: There are " << getConnectedFDCount(source_rank()) << " fds connected.";
580  return getConnectedFDCount(source_rank()) > 0;
581  }
582  return false;
583 }
584 
586 {
587  while(connected_fds_[source_rank()].size()) {
588  disconnect_receive_socket_(*connected_fds_[source_rank()].begin(), "Flushing connections");
589  }
590 }
591 
592 // Send the given Fragment. Return the rank of the destination to which
593 // the Fragment was sent OR -1 if to none.
594 artdaq::TransferInterface::CopyStatus artdaq::TCPSocketTransfer::sendFragment_(Fragment&& frag, size_t send_timeout_usec)
595 {
596  TLOG(12) << GetTraceName() << ": sendFragment begin send of fragment with sequenceID="<<frag.sequenceID();
597  artdaq::Fragment grab_ownership_frag = std::move(frag);
598 
599  reconnect_();
600  if (send_fd_ == -1 && connection_was_lost_)
601  {
602  TLOG(TLVL_INFO) << GetTraceName() << ": reconnection attempt failed, returning quickly.";
604  }
605 
606  // Send Fragment Header
607 
608 #if USE_ACKS
609  // Wait for fragments to be received
610  while (static_cast<size_t>(send_ack_diff_) > buffer_count_) usleep(10000);
611 #endif
612 
613  iovec iov = { reinterpret_cast<void*>(grab_ownership_frag.headerAddress()),
614  detail::RawFragmentHeader::num_words() * sizeof(RawDataType) };
615 
616  auto sts = sendData_(&iov, 1, send_retry_timeout_us_, true);
617  auto start_time = std::chrono::steady_clock::now();
618  //If it takes more than 10 seconds to write a Fragment header, give up
619  while (sts == CopyStatus::kTimeout && (send_timeout_usec == 0 || TimeUtils::GetElapsedTimeMicroseconds(start_time) < send_timeout_usec) && TimeUtils::GetElapsedTimeMicroseconds(start_time) < 10000000)
620  {
621  TLOG(13) << GetTraceName() << ": sendFragment: Timeout sending fragment";
622  sts = sendData_(&iov, 1, send_retry_timeout_us_, true);
623  usleep(1000);
624  }
625  if (sts != CopyStatus::kSuccess) return sts;
626 
627  // Send Fragment Data
628 
629  iov = { reinterpret_cast<void*>(grab_ownership_frag.headerAddress() + detail::RawFragmentHeader::num_words()),
630  grab_ownership_frag.sizeBytes() - detail::RawFragmentHeader::num_words() * sizeof(RawDataType)
631 };
632  sts = sendData_(&iov, 1, send_retry_timeout_us_);
633  start_time = std::chrono::steady_clock::now();
634  while (sts == CopyStatus::kTimeout && (send_timeout_usec == 0 || TimeUtils::GetElapsedTimeMicroseconds(start_time) < send_timeout_usec) && TimeUtils::GetElapsedTimeMicroseconds(start_time) < 10000000)
635  {
636  TLOG(13) << GetTraceName() << ": sendFragment: Timeout sending fragment";
637  sts = sendData_(&iov, 1, send_retry_timeout_us_);
638  usleep(1000);
639  }
640 
641 #if USE_ACKS
642  send_ack_diff_++;
643 #endif
644 
645  TLOG(12) << GetTraceName() << ": sendFragment returning " << CopyStatusToString(sts);
646  return sts;
647 }
648 
649 artdaq::TransferInterface::CopyStatus artdaq::TCPSocketTransfer::sendData_(const void* buf, size_t bytes, size_t send_timeout_usec, bool isHeader)
650 {
651  TLOG(TLVL_DEBUG) << GetTraceName() << ": sendData_ Converting buf to iovec";
652  iovec iov = { (void*)buf, bytes };
653  return sendData_(&iov, 1, send_timeout_usec, isHeader);
654 }
655 
656 artdaq::TransferInterface::CopyStatus artdaq::TCPSocketTransfer::sendData_(const struct iovec* iov, int iovcnt, size_t send_timeout_usec, bool isHeader)
657 {
658  // check all connected??? -- currently just check fd!=-1
659  if (send_fd_ == -1)
660  {
661  if (timeoutMessageArmed_)
662  {
663  TLOG(TLVL_DEBUG) << GetTraceName() << ": sendData_: Send fd is not open. Returning kTimeout";
664  timeoutMessageArmed_ = false;
665  }
666  return CopyStatus::kTimeout;
667  }
668  timeoutMessageArmed_ = true;
669  TLOG(14) << GetTraceName() << ": send_timeout_usec is " << send_timeout_usec << ", currently unused.";
670 
671  //TLOG(TLVL_DEBUG) << GetTraceName() << ": sendData_: Determining write size" ;
672  uint32_t total_to_write_bytes = 0;
673  std::vector<iovec> iov_in(iovcnt + 1); // need contiguous (for the unlike case that only partial MH
674  std::vector<iovec> iovv(iovcnt + 2); // 1 more for mh and another one for any partial
675  int ii;
676  for (ii = 0; ii < iovcnt; ++ii)
677  {
678  iov_in[ii + 1] = iov[ii];
679  total_to_write_bytes += iov[ii].iov_len;
680  }
681  //TLOG(TLVL_DEBUG) << GetTraceName() << ": sendData_: Constructing Message Header" ;
682  MessHead mh = { 0,isHeader ? MessHead::header_v0 : MessHead::data_v0,htons(source_rank()),{htonl(total_to_write_bytes)} };
683  iov_in[0].iov_base = &mh;
684  iov_in[0].iov_len = sizeof(mh);
685  total_to_write_bytes += sizeof(mh);
686 
687  ssize_t sts = 0;
688  ssize_t total_written_bytes = 0;
689  ssize_t per_write_max_bytes = (32 * 1024);
690 
691  size_t in_iov_idx = 0; // only increment this when we know the associated data has been xferred
692  size_t out_iov_idx = 0;
693  ssize_t this_write_bytes = 0;
694 
695  do
696  {
697  // The first out_iov may be set at the end of the previous loop.
698  // iov looping from below (b/c of the latter, we need to check this_write_bytes)
699  for (;
700  (in_iov_idx + out_iov_idx) < iov_in.size() && this_write_bytes < per_write_max_bytes;
701  ++out_iov_idx)
702  {
703  this_write_bytes += iov_in[in_iov_idx + out_iov_idx].iov_len;
704  iovv[out_iov_idx] = iov_in[in_iov_idx + out_iov_idx];
705  }
706  if (this_write_bytes > per_write_max_bytes)
707  {
708  iovv[out_iov_idx - 1].iov_len -= this_write_bytes - per_write_max_bytes;
709  this_write_bytes = per_write_max_bytes;
710  }
711 
712  // need to do blocking algorithm -- including throttled block notifications
713  do_again:
714 #ifndef __OPTIMIZE__ // This can be an expensive TRACE call (even if disabled) due to multiplicity of calls
715  TLOG(14) << GetTraceName() << ": sendFragment b4 writev " << std::setw(7) << total_written_bytes << " total_written_bytes send_fd_=" << send_fd_ << " in_idx=" << in_iov_idx
716  << " iovcnt=" << out_iov_idx << " 1st.len=" << iovv[0].iov_len;
717 #endif
718  //TLOG(TLVL_DEBUG) << GetTraceName() << " calling writev" ;
719  sts = writev(send_fd_, &(iovv[0]), out_iov_idx);
720  //TLOG(TLVL_DEBUG) << GetTraceName() << " done with writev" ;
721 
722  if (sts == -1)
723  {
724  if (errno == EAGAIN /* same as EWOULDBLOCK */)
725  {
726  TLOG(TLVL_DEBUG) << GetTraceName() << ": sendFragment EWOULDBLOCK";
727  fcntl(send_fd_, F_SETFL, 0); // clear O_NONBLOCK
728  blocking = true;
729  // NOTE: YES -- could drop here
730  goto do_again;
731  }
732  TLOG(TLVL_WARNING) << GetTraceName() << ": sendFragment_: WRITE ERROR: " << strerror(errno);
733  connect_state = 0; // any write error closes
734  close(send_fd_);
735  send_fd_ = -1;
736  connection_was_lost_ = true;
738  }
739  else if (sts != this_write_bytes)
740  {
741  // we'll loop around -- with
742  TLOG(TLVL_DEBUG) << GetTraceName() << ": sendFragment writev sts(" << sts << ")!=requested_send_bytes(" << this_write_bytes << ")";
743  total_written_bytes += sts; // add sts to total_written_bytes now as sts is adjusted next
744  // find which iovs are done
745  for (ii = 0; (size_t)sts >= iovv[ii].iov_len; ++ii)
746  sts -= iovv[ii].iov_len;
747  in_iov_idx += ii; // done with these in_iovs
748  iovv[ii].iov_len -= sts; // adjust partial iov
749  iovv[ii].iov_base = (uint8_t*)(iovv[ii].iov_base) + sts; // adjust partial iov
750 
751  // add more to get up to per_write_max_bytes
752  out_iov_idx = 0;
753  if (ii != 0)
754  iovv[out_iov_idx] = iovv[ii];
755  // starting over
756  this_write_bytes = iovv[out_iov_idx].iov_len;
757  // add any left over from appropriate in_iov_idx --
758  // i.e. match this out_iov with the in_iov that was used to
759  // initialize it; see how close the out base+len is to in base+len
760  // check !>per_write_max_bytes
761  unsigned long additional = ((unsigned long)iov_in[in_iov_idx].iov_base + iov_in[in_iov_idx].iov_len)
762  - ((unsigned long)iovv[out_iov_idx].iov_base + iovv[out_iov_idx].iov_len);
763  if (additional)
764  {
765  iovv[out_iov_idx].iov_len += additional;
766  this_write_bytes += additional;
767  if (this_write_bytes > per_write_max_bytes)
768  {
769  iovv[out_iov_idx].iov_len -= this_write_bytes - per_write_max_bytes;
770  this_write_bytes = per_write_max_bytes;
771  }
772  }
773  ++out_iov_idx; // done with
774  TLOG(TLVL_TRACE) << GetTraceName() << ": sendFragment writev sts!=: this_write_bytes=" << this_write_bytes
775  << " out_iov_idx=" << out_iov_idx
776  << " additional=" << additional
777  << " ii=" << ii;
778  }
779  else
780  {
781 #ifndef __OPTIMIZE__ // This can be an expensive TRACE call (even if disabled) due to multiplicity of calls
782  TLOG(TLVL_TRACE) << GetTraceName() << ": sendFragment writev sts(" << sts << ")==requested_send_bytes(" << this_write_bytes << ")";
783 #endif
784  total_written_bytes += sts;
785  --out_iov_idx; // make it the index of the last iovv
786  iovv[out_iov_idx].iov_base = (uint8_t*)(iovv[out_iov_idx].iov_base) + iovv[out_iov_idx].iov_len;
787  iovv[out_iov_idx].iov_len = 0;
788  in_iov_idx += out_iov_idx; // at least this many complete (one more if "last iovv" is complete
789  this_write_bytes = 0;
790  // need to check last iovv against appropriate iov_in
791  unsigned long additional = ((unsigned long)iov_in[in_iov_idx].iov_base + iov_in[in_iov_idx].iov_len)
792  - ((unsigned long)iovv[out_iov_idx].iov_base + iovv[out_iov_idx].iov_len);
793  if (additional)
794  {
795  iovv[out_iov_idx].iov_len += additional;
796  this_write_bytes += additional;
797  if (this_write_bytes > per_write_max_bytes)
798  {
799  iovv[out_iov_idx].iov_len -= this_write_bytes - per_write_max_bytes;
800  this_write_bytes = per_write_max_bytes;
801  }
802  if (out_iov_idx != 0)
803  iovv[0] = iovv[out_iov_idx];
804  out_iov_idx = 1;
805  }
806  else
807  {
808  ++in_iov_idx;
809  out_iov_idx = 0;
810  }
811  }
812  } while (total_written_bytes < total_to_write_bytes);
813  if (total_written_bytes > total_to_write_bytes)
814  TLOG(TLVL_ERROR) << GetTraceName() << ": sendFragment program error: too many bytes transferred";
815 
816  if (blocking)
817  {
818  blocking = false;
819  fcntl(send_fd_, F_SETFL, O_NONBLOCK); // set O_NONBLOCK
820  }
821  sts = total_written_bytes - sizeof(MessHead);
822 
823  TLOG(14) << GetTraceName() << ": sendFragment sts=" << sts;
825 }
826 
827 void artdaq::TCPSocketTransfer::connect_()
828 {
829  auto start_time = std::chrono::steady_clock::now();
830 
831  // Retry a few times if we can't connect
832  while (send_fd_ == -1 && TimeUtils::GetElapsedTimeMicroseconds(start_time) < send_retry_timeout_us_ * 10)
833  {
834  TLOG(TLVL_DEBUG) << GetTraceName() << ": Connecting sender socket";
835  int sndbuf_bytes = static_cast<int>(sndbuf_);
836  if (sndbuf_ > INT_MAX)
837  {
838  sndbuf_bytes = INT_MAX;
839  TLOG(TLVL_WARNING) << "Requested SNDBUF " << sndbuf_ << " too large, setting to INT_MAX: " << INT_MAX;
840  }
841  TLOG(TLVL_DEBUG) << "Requested SNDBUF is " << sndbuf_bytes;
842 
843  send_fd_ = TCPConnect(hostMap_[destination_rank()].c_str()
844  , portMan->GetTCPSocketTransferPort(destination_rank())
845  , O_NONBLOCK
846  , sndbuf_bytes);
847  if (send_fd_ == -1)
848  {
849  if (connection_was_lost_) {break;}
850  else {usleep(send_retry_timeout_us_);}
851  }
852  }
853  connect_state = 0;
854  blocking = 0;
855  TLOG(TLVL_DEBUG) << GetTraceName() << ": connect_ " + hostMap_[destination_rank()] + ":" << portMan->GetTCPSocketTransferPort(destination_rank()) << " send_fd_=" << send_fd_;
856  if (send_fd_ != -1)
857  {
858  // write connect msg
859  TLOG(TLVL_DEBUG) << GetTraceName() << ": connect_: Writing connect message";
860  MessHead mh = { 0,MessHead::connect_v0,htons(source_rank()),{htonl(CONN_MAGIC)} };
861  ssize_t sts = write(send_fd_, &mh, sizeof(mh));
862  if (sts == -1)
863  {
864  TLOG(TLVL_ERROR) << GetTraceName() << ": connect_: Error writing connect message!";
865  // a write error here is completely unexpected!
866  connect_state = 0;
867  close(send_fd_);
868  send_fd_ = -1;
869  }
870  else
871  {
872  TLOG(TLVL_INFO) << GetTraceName() << ": connect_: Successfully connected";
873  // consider it all connected/established
874  connect_state = 1;
875  connection_was_lost_ = false;
876  }
877 
878 #if USE_ACKS
879  if (ack_listen_thread_ && ack_listen_thread_->joinable()) ack_listen_thread_->join();
880  TLOG(TLVL_INFO) << GetTraceName() << ": Starting Ack Listener Thread";
881 
882  try {
883  ack_listen_thread_ = std::make_unique<boost::thread>(&TCPSocketTransfer::receive_acks_, this);
884  }
885  catch (const boost::exception& e)
886  {
887  TLOG(TLVL_ERROR) << "Caught boost::exception starting TCP Socket Ack Listen thread: " << boost::diagnostic_information(e) << ", errno=" << errno;
888  std::cerr << "Caught boost::exception starting TCP Socket Ack Listen thread: " << boost::diagnostic_information(e) << ", errno=" << errno << std::endl;
889  exit(5);
890  }
891 #endif
892  }
893  }
894 
895 void artdaq::TCPSocketTransfer::reconnect_()
896 {
897  if (send_fd_ == -1 && role() == TransferInterface::Role::kSend)
898  {
899  TLOG(TLVL_TRACE) << GetTraceName() << ": check/reconnect";
900  return connect_();
901  }
902 }
903 
904 void artdaq::TCPSocketTransfer::start_listen_thread_()
905 {
906  std::unique_lock<std::mutex> start_lock(listen_thread_mutex_);
907  if (listen_thread_refcount_ == 0)
908  {
909  if (listen_thread_ && listen_thread_->joinable()) listen_thread_->join();
910  listen_thread_refcount_ = 1;
911  TLOG(TLVL_INFO) << GetTraceName() << ": Starting Listener Thread";
912 
913  try {
914  listen_thread_ = std::make_unique<boost::thread>(&TCPSocketTransfer::listen_, portMan->GetTCPSocketTransferPort(destination_rank()), rcvbuf_);
915  }
916  catch (const boost::exception& e)
917  {
918  TLOG(TLVL_ERROR) << "Caught boost::exception starting TCP Socket Listen thread: " << boost::diagnostic_information(e) << ", errno=" << errno;
919  std::cerr << "Caught boost::exception starting TCP Socket Listen thread: " << boost::diagnostic_information(e) << ", errno=" << errno << std::endl;
920  exit(5);
921  }
922  }
923  else
924  {
925  listen_thread_refcount_++;
926  }
927 }
928 
929 #if USE_ACKS
930 void artdaq::TCPSocketTransfer::receive_acks_()
931 {
932  while (send_fd_ >= 0)
933  {
934  pollfd pollfd_s;
935  pollfd_s.events = POLLIN | POLLPRI;
936  pollfd_s.fd = send_fd_;
937 
938  TLOG(18) << GetTraceName() << ": receive_acks_: Polling fd to see if there's data";
939  int num_fds_ready = poll(&pollfd_s, 1, 1000);
940  if (num_fds_ready <= 0)
941  {
942  if (num_fds_ready == 0)
943  {
944  TLOG(18) << GetTraceName() << ": receive_acks_: No data on receive socket";
945  continue;
946  }
947 
948  TLOG(TLVL_ERROR) << "Error in poll: errno=" << errno;
949  break;
950  }
951 
952  if (pollfd_s.revents & (POLLIN | POLLPRI))
953  {
954  // Expected, don't have to check revents any further
955  }
956  else
957  {
958  TLOG(TLVL_DEBUG) << GetTraceName() << ": receive_acks_: Wrong event received from pollfd: " << pollfd_s.revents;
959  break;
960  }
961 
962  MessHead mh;
963  auto sts = read(send_fd_, &mh, sizeof(mh));
964 
965  if (sts != sizeof(mh))
966  {
967  TLOG(TLVL_ERROR) << GetTraceName() << ": receive_ack_: Wrong message header length received! (actual " << sts << " != " << sizeof(mh) << " expected)";
968  continue;
969  }
970 
971  // check for "magic" and valid source_id(aka rank)
972  mh.source_id = ntohs(mh.source_id); // convert here as it is reference several times
973  if (mh.source_id != my_rank)
974  {
975  TLOG(TLVL_ERROR) << GetTraceName() << ": receive_ack_: Received ack for different sender! Rank=" << my_rank << ", hdr=" << mh.source_id;
976  continue;
977  }
978  if (ntohl(mh.conn_magic) != ACK_MAGIC || !(mh.message_type == MessHead::ack_v0)) // Allow for future connect message versions
979  {
980  TLOG(TLVL_ERROR) << GetTraceName() << ": receive_ack_: Wrong magic bytes in header!";
981  continue;
982  }
983 
984  TLOG(17) << GetTraceName() << ": receive_acks_: Received ack message, diff is now " << (send_ack_diff_.load() - 1);
985  send_ack_diff_--;
986 }
987 }
988 
989 void artdaq::TCPSocketTransfer::send_ack_(int fd)
990 {
991  MessHead mh = { 0,MessHead::ack_v0,htons(source_rank()),{ htonl(ACK_MAGIC) } };
992  write(fd, &mh, sizeof(mh));
993 }
994 #endif
995 
996 void artdaq::TCPSocketTransfer::listen_(int port, size_t rcvbuf)
997 {
998  int listen_fd = -1;
999  while (listen_thread_refcount_ > 0)
1000  {
1001  TLOG(TLVL_TRACE) << "listen_: Listening/accepting new connections on port " << port;
1002  if (listen_fd == -1)
1003  {
1004  TLOG(TLVL_DEBUG) << "listen_: Opening listener";
1005  listen_fd = TCP_listen_fd(port, rcvbuf);
1006  }
1007  if (listen_fd == -1)
1008  {
1009  TLOG(TLVL_DEBUG) << "listen_: Error creating listen_fd!";
1010  break;
1011  }
1012 
1013  int res;
1014  timeval tv = { 2,0 }; // maybe increase of some global "debugging" flag set???
1015  fd_set rfds;
1016  FD_ZERO(&rfds);
1017  FD_SET(listen_fd, &rfds);
1018 
1019  res = select(listen_fd + 1, &rfds, (fd_set *)0, (fd_set *)0, &tv);
1020  if (res > 0)
1021  {
1022  int sts;
1023  sockaddr_un un;
1024  socklen_t arglen = sizeof(un);
1025  int fd;
1026  TLOG(TLVL_DEBUG) << "listen_: Calling accept";
1027  fd = accept(listen_fd, (sockaddr *)&un, &arglen);
1028  TLOG(TLVL_DEBUG) << "listen_: Done with accept";
1029 
1030  TLOG(TLVL_DEBUG) << "listen_: Reading connect message";
1031  socklen_t lenlen = sizeof(tv);
1032  /*sts=*/
1033  setsockopt(fd, SOL_SOCKET, SO_RCVTIMEO, &tv, lenlen); // see man 7 socket.
1034  MessHead mh;
1035  uint64_t mark_us = TimeUtils::gettimeofday_us();
1036  sts = read(fd, &mh, sizeof(mh));
1037  uint64_t delta_us = TimeUtils::gettimeofday_us() - mark_us;
1038  TLOG(TLVL_DEBUG) << "listen_: Read of connect message took " << delta_us << " microseconds.";
1039  if (sts != sizeof(mh))
1040  {
1041  TLOG(TLVL_DEBUG) << "listen_: Wrong message header length received!";
1042  close(fd);
1043  continue;
1044  }
1045 
1046  // check for "magic" and valid source_id(aka rank)
1047  mh.source_id = ntohs(mh.source_id); // convert here as it is reference several times
1048  if (ntohl(mh.conn_magic) != CONN_MAGIC || !(mh.message_type == MessHead::connect_v0)) // Allow for future connect message versions
1049  {
1050  TLOG(TLVL_DEBUG) << "listen_: Wrong magic bytes in header!";
1051  close(fd);
1052  continue;
1053  }
1054 
1055  // now add (new) connection
1056  std::unique_lock<std::mutex> lk(connected_fd_mutex_);
1057  connected_fds_[mh.source_id].insert(fd);
1058 
1059  TLOG(TLVL_INFO) << "listen_: New fd is " << fd << " for source rank " << mh.source_id;
1060  }
1061  else
1062  {
1063  TLOG(16) << "listen_: No connections in timeout interval!";
1064  }
1065  }
1066 
1067  TLOG(TLVL_INFO) << "listen_: Shutting down connection listener";
1068  if (listen_fd != -1) close(listen_fd);
1069  std::unique_lock<std::mutex> lk(connected_fd_mutex_);
1070  auto it = connected_fds_.begin();
1071  while (it != connected_fds_.end())
1072  {
1073  auto& fd_set = it->second;
1074  auto rank_it = fd_set.begin();
1075  while (rank_it != fd_set.end())
1076  {
1077  close(*rank_it);
1078  rank_it = fd_set.erase(rank_it);
1079  }
1080  it = connected_fds_.erase(it);
1081  }
1082 
1083 } // do_connect_
1084 
1085 DEFINE_ARTDAQ_TRANSFER(artdaq::TCPSocketTransfer)
bool isRunning() override
Determine whether the TransferInterface plugin is able to send/receive data.
virtual int source_rank() const
Get the source rank for this TransferInterface instance.
int TCPConnect(char const *host_in, int dflt_port, long flags=0, int sndbufsiz=0)
Connect to a host on a given port.
Definition: TCPConnect.cc:358
uint32_t conn_magic
unsigned first is better for MessHead initializer: {0,0,my_node_idx_,CONN_MAGIC}
Definition: SRSockets.hh:40
hostMap_t MakeHostMap(fhicl::ParameterSet pset, hostMap_t map=hostMap_t())
Make a hostMap_t from a HostMap::Config ParameterSet
Definition: HostMap.hh:66
This TransferInterface is a Receiver.
int receiveFragmentData(RawDataType *destination, size_t wordCount) override
Receive the body of a Fragment to the given destination pointer.
int TCP_listen_fd(int port, int rcvbuf)
Create a TCP listening socket on the given port and INADDR_ANY, with the given receive buffer...
TCPSocketTransfer(fhicl::ParameterSet const &ps, Role role)
TCPSocketTransfer Constructor.
int receiveFragmentHeader(detail::RawFragmentHeader &header, size_t receiveTimeout) override
Receive a Fragment Header from the transport mechanism.
This TransferInterface is a Sender.
void flush_buffers() override
Flush any in-flight data. This should be used by the receiver after the receive loop has ended...
int32_t byte_count
use CONN_MAGIC for connect_v0, data that follow for data_v0 (and 0 lenght data)
Definition: SRSockets.hh:41
Some error occurred, but no exception was thrown.
Role
Used to determine if a TransferInterface is a Sender or Receiver.
int64_t source_id
Rank of the source.
Definition: SRSockets.hh:36
MessType message_type
Message Type.
Definition: SRSockets.hh:35
The send operation completed successfully.
This interface defines the functions used to transfer data between artdaq applications.
TransferInterface implementation plugin that sends data using TCP sockets.
This header is sent by the TCPSocket_transfer to allow for more efficient writev calls.
Definition: SRSockets.hh:15
CopyStatus
Returned from the send functions, this enumeration describes the possible return codes. If an exception occurs, it will be thrown and should be handled normally.