artdaq  v2_02_03
 All Classes Namespaces Files Functions Variables Typedefs Enumerations Enumerator Pages
TCPSocket_transfer.cc
1 // This file (TCPSocketTransfer.cc) was created by Ron Rechenmacher <ron@fnal.gov> on
2 // Sep 14, 2016. "TERMS AND CONDITIONS" governing this file are in the README
3 // or COPYING file. If you do not have such a file, one can be obtained by
4 // contacting Ron or Fermi Lab in Batavia IL, 60510, phone: 630-840-3000.
5 // $RCSfile: .emacs.gnu,v $
6 // rev="$Revision: 1.30 $$Date: 2016/03/01 14:27:27 $";
7 
8 // C Includes
9 #include <stdlib.h> // atoi, strtoul
10 #include <sys/socket.h> // socket, socklen_t
11 #include <sys/un.h> // sockaddr_un
12 #include <arpa/inet.h> // ntohl, ntohs
13 #include <assert.h>
14 
15 // C++ Includes
16 #include <string>
17 #include <fstream>
18 #include <stdexcept>
19 
20 // product Includes
21 #define TRACE_NAME "TCPSocketTransfer"
22 #include "artdaq/DAQdata/Globals.hh"
23 
24 // artdaq Includes
25 #include "artdaq/TransferPlugins/TCPSocketTransfer.hh"
28 #include "artdaq/TransferPlugins/detail/Timeout.hh"
29 #include "artdaq/TransferPlugins/detail/SRSockets.hh"
30 #include "artdaq-core/Data/Fragment.hh"
31 
32 
34 TCPSocketTransfer(fhicl::ParameterSet const& pset, TransferInterface::Role role)
35  : TransferInterface(pset, role)
36  , fd_(-1)
37  , listen_fd_(-1)
38  , state_(SocketState::Metadata)
39  , frag(max_fragment_size_words_)
40  , buffer(frag.headerBeginBytes())
41  , offset(0)
42  , target_bytes(sizeof(MessHead))
43  , rcvbuf_(pset.get<size_t>("tcp_receive_buffer_size", 0))
44  , sndbuf_(max_fragment_size_words_ * sizeof(artdaq::RawDataType) * buffer_count_)
45  , stats_connect_stop_(false)
46  , stats_connect_thread_(std::bind(&TCPSocketTransfer::stats_connect_, this))
47  , timeoutMessageArmed_(true)
48 {
49  TLOG_DEBUG(uniqueLabel()) << "TCPSocketTransfer Constructor: pset=" << pset.to_string() << ", role=" << (role == TransferInterface::Role::kReceive ? "kReceive" : "kSend") << TLOG_ENDL;
50  auto hosts = pset.get<std::vector<fhicl::ParameterSet>>("host_map");
51  for (auto& ps : hosts)
52  {
53  auto rank = ps.get<size_t>("rank", RECV_TIMEOUT);
54  DestinationInfo info;
55  info.hostname = ps.get<std::string>("host", "localhost");
56  info.portOffset = ps.get<int>("portOffset", 5500);
57 
58  hostMap_[rank] = info;
59  }
60 
61  std::function<void()> function = std::bind(&TCPSocketTransfer::reconnect_, this);
62  tmo_.add_periodic("reconnect", NULL, function, 200/*millisec*/);
63 
65  {
66  // Wait for sender to connect...
67  TLOG_DEBUG(uniqueLabel()) << "Listening for connections" << TLOG_ENDL;
68  listen_();
69  TLOG_DEBUG(uniqueLabel()) << "Done Listening" << TLOG_ENDL;
70  }
71  else
72  {
73  TLOG_DEBUG(uniqueLabel()) << "Connecting to destination" << TLOG_ENDL;
74  connect_();
75  TLOG_DEBUG(uniqueLabel()) << "Done Connecting" << TLOG_ENDL;
76  }
77  TLOG_DEBUG(uniqueLabel()) << "End of TCPSocketTransfer Constructor" << TLOG_ENDL;
78 }
79 
80 artdaq::TCPSocketTransfer::~TCPSocketTransfer()
81 {
82  TLOG_DEBUG(uniqueLabel()) << "Shutting down TCPSocketTransfer" << TLOG_ENDL;
83  stats_connect_stop_ = true;
84  stopstatscv_.notify_all();
85  stats_connect_thread_.join();
86 
87  if (role() == TransferInterface::Role::kSend)
88  {
89  // close all open connections (send stop_v0) first
90  MessHead mh = {0,MessHead::stop_v0,htons(source_rank()),0};
91  if (fd_ != -1)
92  {
93  // should be blocking with modest timeo
94  timeval tv = {0,100000};
95  socklen_t len = sizeof(tv);
96  setsockopt(fd_, SOL_SOCKET, SO_SNDTIMEO, &tv, len);
97  write(fd_, &mh, sizeof(mh));
98  }
99  }
100  close(fd_);
101  TLOG_DEBUG(uniqueLabel()) << "End of TCPSocketTransfer Destructor" << TLOG_ENDL;
102  TRACE(4, "TCPSocketTransfer dtor");
103 }
104 
105 
106 // Send the given Fragment. Return the rank of the destination to which
107 // the Fragment was sent OR -1 if to none.
108 artdaq::TransferInterface::CopyStatus artdaq::TCPSocketTransfer::sendFragment_(Fragment&& frag, size_t send_timeout_usec)
109 {
110  TRACE(7, "TCPSocketTransfer::sendFragment begin");
111  artdaq::Fragment grab_ownership_frag = std::move(frag);
112  iovec iov = {(void*)grab_ownership_frag.headerBeginBytes(), grab_ownership_frag.sizeBytes()};
113  auto sts = sendFragment_(&iov, 1, send_timeout_usec);
114  while (sts != CopyStatus::kSuccess)
115  {
116  TRACE(7, "TCPSocketTransfer::sendFragment: Timeout or Error sending fragment");
117  sts = sendFragment_(&iov, 1, send_timeout_usec);
118  usleep(1000);
119  }
120 
121  std::string result = (sts == TransferInterface::CopyStatus::kSuccess ? "kSuccess" : (sts == TransferInterface::CopyStatus::kTimeout ? "kTimeout" : "kErrorNotRequiringException"));
122 
123  TRACE_(7, "TCPSocketTransfer::sendFragment returning " + result);
124  return sts;
125 }
126 
127 artdaq::TransferInterface::CopyStatus artdaq::TCPSocketTransfer::sendFragment_(const void* buf, size_t bytes, size_t send_timeout_usec)
128 {
129  TLOG_DEBUG(uniqueLabel()) << "TCPSocketTransfer::sendFragment_ Converting buf to iovec" << TLOG_ENDL;
130  iovec iov = {(void*)buf, bytes};
131  return sendFragment_(&iov, 1, send_timeout_usec);
132 }
133 
134 artdaq::TransferInterface::CopyStatus artdaq::TCPSocketTransfer::sendFragment_(const struct iovec* iov, int iovcnt, size_t send_timeout_usec)
135 {
136  // check all connected??? -- currently just check fd!=-1
137  if (fd_ == -1)
138  {
139  if (timeoutMessageArmed_)
140  {
141  TLOG_DEBUG(uniqueLabel()) << "TCPSocketTransfer::sendFragment_: Send fd is not open. Returning kTimeout" << TLOG_ENDL;
142  timeoutMessageArmed_ = false;
143  }
145  }
146  timeoutMessageArmed_ = true;
147  TRACE(12, "send_timeout_usec is %zu, currently unused.", send_timeout_usec);
148 
149  //TLOG_DEBUG(uniqueLabel()) << "TCPSocketTransfer::sendFragment_: Determining write size" << TLOG_ENDL;
150  uint32_t total_to_write_bytes = 0;
151  std::vector<iovec> iov_in(iovcnt + 1); // need contiguous (for the unlike case that only partial MH
152  std::vector<iovec> iovv(iovcnt + 2); // 1 more for mh and another one for any partial
153  int ii;
154  for (ii = 0; ii < iovcnt; ++ii)
155  {
156  iov_in[ii + 1] = iov[ii];
157  total_to_write_bytes += iov[ii].iov_len;
158  }
159  //TLOG_DEBUG(uniqueLabel()) << "TCPSocketTransfer::sendFragment_: Constructing Message Header" << TLOG_ENDL;
160  MessHead mh = {0,MessHead::data_v0,htons(source_rank()),htonl(total_to_write_bytes)};
161  iov_in[0].iov_base = &mh;
162  iov_in[0].iov_len = sizeof(mh);
163  total_to_write_bytes += sizeof(mh);
164 
165  ssize_t sts = 0;
166  ssize_t total_written_bytes = 0;
167  ssize_t per_write_max_bytes = (32 * 1024);
168 
169  size_t in_iov_idx = 0; // only increment this when we know the associated data has been xferred
170  size_t out_iov_idx = 0;
171  ssize_t this_write_bytes = 0;
172 
173  do
174  {
175  // The first out_iov may be set at the end of the previous loop.
176  // iov looping from below (b/c of the latter, we need to check this_write_bytes)
177  for (;
178  (in_iov_idx + out_iov_idx) < iov_in.size() && this_write_bytes < per_write_max_bytes;
179  ++out_iov_idx)
180  {
181  this_write_bytes += iov_in[in_iov_idx + out_iov_idx].iov_len;
182  iovv[out_iov_idx] = iov_in[in_iov_idx + out_iov_idx];
183  }
184  if (this_write_bytes > per_write_max_bytes)
185  {
186  iovv[out_iov_idx - 1].iov_len -= this_write_bytes - per_write_max_bytes;
187  this_write_bytes = per_write_max_bytes;
188  }
189 
190  // need to do blocking algorithm -- including throttled block notifications
191  do_again:
192  TRACE(7, "sendFragment b4 writev %7zu total_written_bytes fd=%d in_idx=%zu iovcnt=%zu 1st.len=%zu"
193  , total_written_bytes, fd_, in_iov_idx, out_iov_idx, iovv[0].iov_len);
194  //TLOG_DEBUG(uniqueLabel()) << "TCPSocketTransfer calling writev" << TLOG_ENDL;
195  sts = writev(fd_, &(iovv[0]), out_iov_idx);
196  //TLOG_DEBUG(uniqueLabel()) << "TCPSocketTransfer done with writev" << TLOG_ENDL;
197 
198  if (sts == -1)
199  {
200  if (errno == EAGAIN /* same as EWOULDBLOCK */)
201  {
202  TRACE(2, "sendFragment EWOULDBLOCK");
203  fcntl(fd_, F_SETFL, 0); // clear O_NONBLOCK
204  blocking = true;
205  // NOTE: YES -- could drop here
206  goto do_again;
207  }
208  TLOG_DEBUG(uniqueLabel()) << "TCPSocketTransfer::sendFragment_: WRITE ERROR!" << TLOG_ENDL;
209  connect_state = 0; // any write error closes
210  close(fd_);
211  fd_ = -1;
213  }
214  else if (sts != this_write_bytes)
215  {
216  // we'll loop around -- with
217  TRACE(4, "sendFragment writev sts(%ld)!=requested_send_bytes(%ld)"
218  , sts, this_write_bytes);
219  total_written_bytes += sts; // add sts to total_written_bytes now as sts is adjusted next
220  // find which iovs are done
221  for (ii = 0; (size_t)sts >= iovv[ii].iov_len; ++ii)
222  sts -= iovv[ii].iov_len;
223  in_iov_idx += ii; // done with these in_iovs
224  iovv[ii].iov_len -= sts; // adjust partial iov
225  iovv[ii].iov_base = (uint8_t*)(iovv[ii].iov_base) + sts; // adjust partial iov
226 
227  // add more to get up to per_write_max_bytes
228  out_iov_idx = 0;
229  if (ii != 0)
230  iovv[out_iov_idx] = iovv[ii];
231  // starting over
232  this_write_bytes = iovv[out_iov_idx].iov_len;
233  // add any left over from appropriate in_iov_idx --
234  // i.e. match this out_iov with the in_iov that was used to
235  // initialize it; see how close the out base+len is to in base+len
236  // check !>per_write_max_bytes
237  unsigned long additional = ((unsigned long)iov_in[in_iov_idx].iov_base + iov_in[in_iov_idx].iov_len)
238  - ((unsigned long)iovv[out_iov_idx].iov_base + iovv[out_iov_idx].iov_len);
239  if (additional)
240  {
241  iovv[out_iov_idx].iov_len += additional;
242  this_write_bytes += additional;
243  if (this_write_bytes > per_write_max_bytes)
244  {
245  iovv[out_iov_idx].iov_len -= this_write_bytes - per_write_max_bytes;
246  this_write_bytes = per_write_max_bytes;
247  }
248  }
249  ++out_iov_idx; // done with
250  TRACE(4, "sendFragment writev sts!=: this_write_bytes=%zd out_iov_idx=%zu additional=%lu ii=%d"
251  , this_write_bytes, out_iov_idx, additional, ii);
252  }
253  else
254  {
255  TRACE(4, "sendFragment writev sts(%ld)==requested_send_bytes(%ld)"
256  , sts, this_write_bytes);
257  total_written_bytes += sts;
258  --out_iov_idx; // make it the index of the last iovv
259  iovv[out_iov_idx].iov_base = (uint8_t*)(iovv[out_iov_idx].iov_base) + iovv[out_iov_idx].iov_len;
260  iovv[out_iov_idx].iov_len = 0;
261  in_iov_idx += out_iov_idx; // at least this many complete (one more if "last iovv" is complete
262  this_write_bytes = 0;
263  // need to check last iovv against appropriate iov_in
264  unsigned long additional = ((unsigned long)iov_in[in_iov_idx].iov_base + iov_in[in_iov_idx].iov_len)
265  - ((unsigned long)iovv[out_iov_idx].iov_base + iovv[out_iov_idx].iov_len);
266  if (additional)
267  {
268  iovv[out_iov_idx].iov_len += additional;
269  this_write_bytes += additional;
270  if (this_write_bytes > per_write_max_bytes)
271  {
272  iovv[out_iov_idx].iov_len -= this_write_bytes - per_write_max_bytes;
273  this_write_bytes = per_write_max_bytes;
274  }
275  if (out_iov_idx != 0)
276  iovv[0] = iovv[out_iov_idx];
277  out_iov_idx = 1;
278  }
279  else
280  {
281  ++in_iov_idx;
282  out_iov_idx = 0;
283  }
284  }
285  }
286  while (total_written_bytes < total_to_write_bytes);
287  if (total_written_bytes > total_to_write_bytes)
288  TRACE(0, "sendFragment program error: too many bytes transferred");
289 
290  if (blocking)
291  {
292  blocking = false;
293  fcntl(fd_, F_SETFL, 0); // clear O_NONBLOCK
294  }
295  sts = total_written_bytes - sizeof(MessHead);
296 
297  TRACE(10, "sendFragment sts=%ld", sts);
299 }
300 
301 //=============================================
302 
303 void artdaq::TCPSocketTransfer::connect_()
304 {
305  TLOG_DEBUG(uniqueLabel()) << "Connecting sender socket" << TLOG_ENDL;
306  int sndbuf_bytes = static_cast<int>(sndbuf_);
307  fd_ = TCPConnect(hostMap_[destination_rank()].hostname.c_str()
308  , calculate_port_()
309  , O_NONBLOCK
310  , sndbuf_bytes);
311  connect_state = 0;
312  blocking = 0;
313  TLOG_DEBUG(uniqueLabel()) << "TCPSocketTransfer::connect_ " + hostMap_[destination_rank()].hostname + ":" << calculate_port_() << " fd_=" << fd_ << TLOG_ENDL;
314  if (fd_ != -1)
315  {
316  // write connect msg
317  TLOG_DEBUG(uniqueLabel()) << "TCPSocketTransfer::connect_: Writing connect message" << TLOG_ENDL;
318  MessHead mh = {0,MessHead::connect_v0,htons(source_rank()),htonl(CONN_MAGIC)};
319  ssize_t sts = write(fd_, &mh, sizeof(mh));
320  if (sts == -1)
321  {
322  TLOG_ERROR(uniqueLabel()) << "TCPSocketTransfer::connect_: Error writing connect message!" << TLOG_ENDL;
323  // a write error here is completely unexpected!
324  connect_state = 0;
325  close(fd_);
326  fd_ = -1;
327  }
328  else
329  {
330  TLOG_INFO(uniqueLabel()) << "TCPSocketTransfer::connect_: Successfully connected" << TLOG_ENDL;
331  // consider it all connected/established
332  connect_state = 1;
333  }
334  }
335 }
336 
337 void artdaq::TCPSocketTransfer::reconnect_()
338 {
339  TRACE(5, "check/reconnect");
340  if (fd_ == -1 && role() == TransferInterface::Role::kSend) return connect_();
341  if ((fd_ == -1 || listen_fd_ == -1) && role() == TransferInterface::Role::kReceive) return listen_();
342 }
343 
344 
345 static uint64_t gettimeofday_us()
346 {
347  struct timeval tv;
348  gettimeofday(&tv, NULL);
349  return (uint64_t)tv.tv_sec * 1000000 + tv.tv_usec;
350 }
351 
352 void artdaq::TCPSocketTransfer::listen_()
353 {
354  TLOG_DEBUG(uniqueLabel()) << "TCPSocketTransfer::listen_: Listening/accepting new connections" << TLOG_ENDL;
355  if (listen_fd_ == -1)
356  {
357  TLOG_DEBUG(uniqueLabel()) << "TCPSocketTransfer::listen_: Opening listener" << TLOG_ENDL;
358  listen_fd_ = TCP_listen_fd(calculate_port_(), rcvbuf_);
359  }
360  if (listen_fd_ == -1)
361  {
362  TLOG_DEBUG(uniqueLabel()) << "TCPSocketTransfer::listen_: Error creating listen_fd_!" << TLOG_ENDL;
363  return;
364  }
365 
366  int res;
367  timeval tv = {2,0}; // maybe increase of some global "debugging" flag set???
368  fd_set rfds;
369  FD_ZERO(&rfds);
370  FD_SET(listen_fd_, &rfds);
371 
372  res = select(listen_fd_ + 1, &rfds, (fd_set *)0, (fd_set *)0, &tv);
373  if (res > 0)
374  {
375  int sts;
376  sockaddr_un un;
377  socklen_t arglen = sizeof(un);
378  int fd;
379  TLOG_DEBUG(uniqueLabel()) << "Calling accept" << TLOG_ENDL;
380  fd = accept(listen_fd_, (sockaddr *)&un, &arglen);
381  TLOG_DEBUG(uniqueLabel()) << "Done with accept" << TLOG_ENDL;
382 
383  TLOG_DEBUG(uniqueLabel()) << "TCPSocketTransfer::listen_: Reading connect message" << TLOG_ENDL;
384  socklen_t lenlen = sizeof(tv);
385  /*sts=*/
386  setsockopt(fd, SOL_SOCKET, SO_RCVTIMEO, &tv, lenlen); // see man 7 socket.
387  MessHead mh;
388  uint64_t mark_us = gettimeofday_us();
389  sts = read(fd, &mh, sizeof(mh));
390  uint64_t delta_us = gettimeofday_us() - mark_us;
391  TLOG_DEBUG(uniqueLabel()) << "TCPSocketTransfer::listen_: Read of connect message took " << delta_us << " microseconds." << TLOG_ENDL;
392  TRACE(10, "do_connect read of connect msg (after accept) took %lu microseconds", delta_us); // emperically, read take a couple hundred usecs.
393  if (sts != sizeof(mh))
394  {
395  TLOG_DEBUG(uniqueLabel()) << "TCPSocketTransfer::listen_: Wrong message header length received!" << TLOG_ENDL;
396  TRACE(0, "do_connect_ problem with connect msg sts(%d)!=sizeof(mh)(%ld)", sts, sizeof(mh));
397  close(fd);
398  return;
399  }
400 
401  // check for "magic" and valid source_id(aka rank)
402  mh.source_id = ntohs(mh.source_id); // convert here as it is reference several times
403  if (ntohl(mh.conn_magic) != CONN_MAGIC || mh.source_id != source_rank())
404  {
405  TLOG_DEBUG(uniqueLabel()) << "TCPSocketTransfer::listen_: Wrong magic bytes in header!" << TLOG_ENDL;
406  close(fd);
407  return;
408  }
409  if (fd_ != -1)
410  {
411  // close previous dec connect_count_
412  close(fd_);
413  }
414 
415  // now add (new) connection
416  fd_ = fd;
417  TLOG_INFO(uniqueLabel()) << "TCPSocketTransfer::listen_: New fd is " << fd_ << TLOG_ENDL;
418 
419  TRACE(3, "do_connect_ connection from sender_rank=%zu", mh.source_id);
420  }
421  else
422  {
423  TRACE(10, "TCPSocketTransfer::do_connect_: No connections in timeout interval!");
424  }
425 } // do_connect_
426 
427 // recvFragment() puts the next received fragment in frag, with the
428 // source of that fragment as its return value.
429 //
430 // It is a precondition that a sources_sending() != 0.
431 int artdaq::TCPSocketTransfer::receiveFragment(Fragment& outfrag, size_t timeout_usec)
432 {
433  TRACE(7, "TCPSocketTransfer::receiveFragment: BEGIN");
434  int ret_rank = RECV_TIMEOUT;
435  if (fd_ == -1)
436  { // what if just listen_fd???
437  TLOG_DEBUG(uniqueLabel()) << "TCPSocketTransfer::receiveFragment: Receive socket not connected, returning RECV_TIMEOUT" << TLOG_ENDL;
438  return RECV_TIMEOUT;
439  }
440 
441  TRACE(7, "TCPSocketTransfer::recvFragment timeout_usec=%ld", timeout_usec);
442  //void* buff=alloca(max_fragment_size_words_*8);
443  uint8_t* buff;
444  size_t byte_cnt = 0;
445  int sts;
446  uint64_t start_time_us = gettimeofday_us();
447 
448  pollfd pollfd_s;
449  pollfd_s.events = POLLIN | POLLERR;
450  pollfd_s.fd = fd_;
451 
452  int timeout_ms;
453  if (timeout_usec == 0)
454  timeout_ms = 0;
455  else
456  timeout_ms = (timeout_usec + 999) / 1000; // want at least 1 ms
457 
458  bool done = false;
459  while (!done)
460  {
461  //TLOG_DEBUG(uniqueLabel()) << "TCPSocketTransfer::receiveFragment: Polling fd to see if there's data" << TLOG_ENDL;
462  int num_fds_ready = poll(&pollfd_s, 1, timeout_ms);
463  if (num_fds_ready <= 0)
464  {
465  if (num_fds_ready == 0 && timeout_ms > 0)
466  {
467  TRACE(7, "TCPSocketTransfer::receiveFragment: No data on receive socket, returning RECV_TIMEOUT");
468  return RECV_TIMEOUT;
469  }
470  break;
471  }
472 
473  if (!(pollfd_s.revents & (POLLIN | POLLERR)))
474  {
475  TLOG_DEBUG(uniqueLabel()) << "TCPSocketTransfer::receiveFragment: Wrong event received from pollfd" << TLOG_ENDL;
476  continue;
477  }
478 
479  if (state_ == SocketState::Metadata)
480  {
481  //TLOG_DEBUG(uniqueLabel()) << "TCPSocketTransfer::receiveFragment: Reading Message Header" << TLOG_ENDL;
482  buff = &(mha[offset]);
483  byte_cnt = sizeof(MessHead) - offset;
484  }
485  else
486  {
487  //TLOG_DEBUG(uniqueLabel()) << "TCPSocketTransfer::receiveFragment: Reading data" << TLOG_ENDL;
488  buff = buffer + offset;
489  byte_cnt = mh.byte_count - offset;
490  }
491 
492  //TLOG_DEBUG(uniqueLabel()) << "TCPSocketTransfer::receiveFragment: Reading " << byte_cnt << " bytes from socket" << TLOG_ENDL;
493  sts = read(fd_, buff, byte_cnt);
494  //TLOG_DEBUG(uniqueLabel()) << "TCPSocketTransfer::receiveFragment: Done with read" << TLOG_ENDL;
495 
496  TRACE(9, "recvFragment state=%d read=%d (errno=%d)", static_cast<int>(state_), sts, errno);
497  if (sts <= 0)
498  {
499  TLOG_DEBUG(uniqueLabel()) << "TCPSocketTransfer::receiveFragment: Error on receive, closing socket" << TLOG_ENDL;
500  close(fd_);
501  }
502  else
503  {
504  // see if we're done (with this state)
505  sts = offset += sts;
506  if (sts == target_bytes)
507  {
508  TRACE(7, "TCPSocketTransfer::receiveFragment: Target read bytes reached. Changing state");
509  offset = 0;
510  if (state_ == SocketState::Metadata)
511  {
512  state_ = SocketState::Data;
513  mh.byte_count = ntohl(mh.byte_count);
514  mh.source_id = ntohs(mh.source_id);
515  target_bytes = mh.byte_count;
516  }
517  else
518  {
519  state_ = SocketState::Metadata;
520  target_bytes = sizeof(MessHead);
521  ret_rank = source_rank();
522  TRACE(9, "recvFragment done sts=%d src=%d", sts, ret_rank);
523  TRACE(7, "TCPSocketTransfer::receiveFragment: Done receiving fragment. Moving into output.");
524  frag.autoResize();
525  if (frag.type() == artdaq::Fragment::EndOfDataFragmentType)
526  {
527  stats_connect_stop_ = true; // Don't reconnect if we're done receiving data...
528  stopstatscv_.notify_all();
529  }
530  outfrag.swap(frag);
531  frag.reserve(max_fragment_size_words_);
532  buffer = frag.headerBeginBytes();
533  done = true; // no more polls
534  break; // no more read of ready fds
535  }
536  }
537  }
538 
539  if (!done && timeout_usec > 0)
540  {
541  // calc next timeout_ms (unless timed out)
542  size_t delta_us = gettimeofday_us() - start_time_us;
543  if (delta_us > timeout_usec)
544  return RECV_TIMEOUT;
545  timeout_ms = ((timeout_usec - delta_us) + 999) / 1000; // want at least 1 ms
546  }
547  } // while(!done)...poll
548 
549  TRACE(7, "TCPSocketTransfer::receiveFragment: Returning %d", ret_rank);
550  return ret_rank;
551 } // recvFragment
552 
553 void artdaq::TCPSocketTransfer::stats_connect_() // thread
554 {
555  std::cv_status sts;
556  while (!stats_connect_stop_)
557  {
558  std::string desc;
559  void* tag;
560  std::function<void()> function;
561  uint64_t ts_us;
562 
563  int msdly = tmo_.get_next_timeout_msdly();
564 
565  if (msdly <= 0)
566  msdly = 2000;
567 
568  std::unique_lock<std::mutex> lck(stopstatscvm_);
569  sts = stopstatscv_.wait_until(lck
570  , std::chrono::system_clock::now()
571  + std::chrono::milliseconds(msdly));
572  TRACE(5, "thread1 after wait_until(msdly=%d) - sts=%d", msdly, static_cast<int>(sts));
573 
574  if (sts == std::cv_status::no_timeout)
575  break;
576 
577  auto sts = tmo_.get_next_expired_timeout(desc, &tag, function, &ts_us);
578 
579  while (sts != -1 && desc != "")
580  {
581  if (function != NULL)
582  function();
583 
584  sts = tmo_.get_next_expired_timeout(desc, &tag, function, &ts_us);
585  }
586  }
587 }
588 
589 DEFINE_ARTDAQ_TRANSFER(artdaq::TCPSocketTransfer)
int TCPConnect(char const *host_in, int dflt_port, long flags=0, int sndbufsiz=0)
Connect to a host on a given port.
Definition: TCPConnect.cc:122
uint32_t conn_magic
unsigned first is better for MessHead initializer: {0,0,my_node_idx_,CONN_MAGIC}
Definition: SRSockets.hh:38
static const int RECV_TIMEOUT
Value to be returned upon receive timeout. Because receivers otherwise return rank, this is also the limit on the number of ranks that artdaq currently supports.
This TransferInterface is a Receiver.
int TCP_listen_fd(int port, int rcvbuf)
Create a TCP listening socket on the given port and INADDR_ANY, with the given receive buffer...
TCPSocketTransfer(fhicl::ParameterSet const &ps, Role role)
TCPSocketTransfer Constructor.
This TransferInterface is a Sender.
int receiveFragment(Fragment &frag, size_t timeout_usec=0) override
Receive a Fragment using TCP.
int32_t byte_count
use CONN_MAGIC for connect_v0, data that follow for data_v0 (and 0 lenght data)
Definition: SRSockets.hh:39
Some error occurred, but no exception was thrown.
Role
Used to determine if a TransferInterface is a Sender or Receiver.
int64_t source_id
Rank of the source.
Definition: SRSockets.hh:34
The send operation completed successfully.
std::string uniqueLabel() const
Get the unique label of this TransferInterface instance.
This interface defines the functions used to transfer data between artdaq applications.
TransferInterface implementation plugin that sends data using TCP sockets.
This header is sent by the TCPSocket_transfer to allow for more efficient writev calls.
Definition: SRSockets.hh:15
void add_periodic(const char *desc, void *tag, std::function< void()> &function, uint64_t period_us, uint64_t start_us=0)
Add a periodic timeout to the Timeout container.
Definition: Timeout.cc:60
CopyStatus
Returned from the send functions, this enumeration describes the possible return codes. If an exception occurs, it will be thrown and should be handled normally.