artdaq  v3_00_03
TCPSocket_transfer.cc
1 // Sep 14, 2016. "TERMS AND CONDITIONS" governing this file are in the README
2 // or COPYING file. If you do not have such a file, one can be obtained by
3 // contacting Ron or Fermi Lab in Batavia IL, 60510, phone: 630-840-3000.
4 // $RCSfile: .emacs.gnu,v $
5 // rev="$Revision: 1.30 $$Date: 2016/03/01 14:27:27 $";
6 
7 // C Includes
8 #include <stdlib.h> // atoi, strtoul
9 #include <sys/socket.h> // socket, socklen_t
10 #include <sys/un.h> // sockaddr_un
11 #include <arpa/inet.h> // ntohl, ntohs
12 #include <sys/types.h> // size_t
13 #include <poll.h> // struct pollfd
14 
15 // C++ Includes
16 #include <string>
17 #include <fstream>
18 #include <stdexcept>
19 
20 // product Includes
21 #define TRACE_NAME "TCPSocketTransfer"
22 #include "artdaq/DAQdata/Globals.hh"
23 
24 // artdaq Includes
25 #include "artdaq/TransferPlugins/TCPSocketTransfer.hh"
28 #include "artdaq/TransferPlugins/detail/Timeout.hh"
29 #include "artdaq/TransferPlugins/detail/SRSockets.hh"
30 #include "artdaq-core/Data/Fragment.hh"
31 #include "artdaq-core/Utilities/TimeUtils.hh"
32 #include <iomanip>
33 
35 TCPSocketTransfer(fhicl::ParameterSet const& pset, TransferInterface::Role role)
36  : TransferInterface(pset, role)
37  , fd_(-1)
38  , listen_fd_(-1)
39  , state_(SocketState::Metadata)
40  , offset(0)
41  , target_bytes(sizeof(MessHead))
42  , rcvbuf_(pset.get<size_t>("tcp_receive_buffer_size", 0))
43  , sndbuf_(max_fragment_size_words_ * sizeof(artdaq::RawDataType) * buffer_count_)
44  , stats_connect_stop_(false)
45  , stats_connect_thread_(std::bind(&TCPSocketTransfer::stats_connect_, this))
46  , timeoutMessageArmed_(true)
47 {
48  TLOG_DEBUG("TCPSocketTransfer") << uniqueLabel() << " TCPSocketTransfer Constructor: pset=" << pset.to_string() << ", role=" << (role == TransferInterface::Role::kReceive ? "kReceive" : "kSend") << TLOG_ENDL;
49  auto masterPortOffset = pset.get<int>("offset_all_ports",0);
50  auto hosts = pset.get<std::vector<fhicl::ParameterSet>>("host_map");
51  for (auto& ps : hosts)
52  {
53  auto rank = ps.get<size_t>("rank", RECV_TIMEOUT);
54  DestinationInfo info;
55  info.hostname = ps.get<std::string>("host", "localhost");
56  info.portOffset = ps.get<int>("portOffset", 5500) + masterPortOffset;
57 
58  hostMap_[rank] = info;
59  }
60 
61  std::function<void()> function = std::bind(&TCPSocketTransfer::reconnect_, this);
62  tmo_.add_periodic("reconnect", NULL, function, 200/*millisec*/);
63 
65  {
66  // Wait for sender to connect...
67  TLOG_DEBUG("TCPSocketTransfer") << uniqueLabel() << " Listening for connections" << TLOG_ENDL;
68  listen_();
69  TLOG_DEBUG("TCPSocketTransfer") << uniqueLabel() << " Done Listening" << TLOG_ENDL;
70  }
71  else
72  {
73  TLOG_DEBUG("TCPSocketTransfer") << uniqueLabel() << " Connecting to destination" << TLOG_ENDL;
74  connect_();
75  TLOG_DEBUG("TCPSocketTransfer") << uniqueLabel() << " Done Connecting" << TLOG_ENDL;
76  }
77  TLOG_DEBUG("TCPSocketTransfer") << uniqueLabel() << " End of TCPSocketTransfer Constructor" << TLOG_ENDL;
78 }
79 
80 artdaq::TCPSocketTransfer::~TCPSocketTransfer()
81 {
82  TLOG_DEBUG("TCPSocketTransfer") << uniqueLabel() << " Shutting down TCPSocketTransfer" << TLOG_ENDL;
83  stats_connect_stop_ = true;
84  stopstatscv_.notify_all();
85  stats_connect_thread_.join();
86 
87  if (role() == TransferInterface::Role::kSend)
88  {
89  // close all open connections (send stop_v0) first
90  MessHead mh = { 0,MessHead::stop_v0,htons(TransferInterface::source_rank()),0 };
91  if (fd_ != -1)
92  {
93  // should be blocking with modest timeo
94  timeval tv = { 0,100000 };
95  socklen_t len = sizeof(tv);
96  setsockopt(fd_, SOL_SOCKET, SO_SNDTIMEO, &tv, len);
97  write(fd_, &mh, sizeof(mh));
98  }
99  }
100  close(fd_);
101  TLOG_DEBUG("TCPSocketTransfer") << uniqueLabel() << " End of TCPSocketTransfer Destructor" << TLOG_ENDL;
102 }
103 
104 int artdaq::TCPSocketTransfer::receiveFragmentHeader(detail::RawFragmentHeader& header, size_t timeout_usec)
105 {
106  TLOG_ARB(7, "TCPSocketTransfer") << uniqueLabel() << " TCPSocketTransfer::receiveFragment: BEGIN" << TLOG_ENDL;
107  int ret_rank = RECV_TIMEOUT;
108  if (fd_ == -1)
109  { // what if just listen_fd???
110  TLOG_DEBUG("TCPSocketTransfer") << uniqueLabel() << " TCPSocketTransfer::receiveFragment: Receive socket not connected, returning RECV_TIMEOUT" << TLOG_ENDL;
111  return RECV_TIMEOUT;
112  }
113 
114  TLOG_ARB(7, "TCPSocketTransfer") << uniqueLabel() << " TCPSocketTransfer::recvFragment timeout_usec=" << std::to_string(timeout_usec) << TLOG_ENDL;
115  //void* buff=alloca(max_fragment_size_words_*8);
116  size_t byte_cnt = 0;
117  int sts;
118  uint64_t start_time_us = TimeUtils::gettimeofday_us();
119 
120  pollfd pollfd_s;
121  pollfd_s.events = POLLIN | POLLERR;
122  pollfd_s.fd = fd_;
123  uint8_t* buff;
124 
125  int timeout_ms;
126  if (timeout_usec == 0)
127  timeout_ms = 0;
128  else
129  timeout_ms = (timeout_usec + 999) / 1000; // want at least 1 ms
130 
131  bool done = false;
132  while (!done)
133  {
134  //TLOG_DEBUG("TCPSocketTransfer") << uniqueLabel() << " TCPSocketTransfer::receiveFragment: Polling fd to see if there's data" << TLOG_ENDL;
135  int num_fds_ready = poll(&pollfd_s, 1, timeout_ms);
136  if (num_fds_ready <= 0)
137  {
138  if (num_fds_ready == 0 && timeout_ms > 0)
139  {
140  TLOG_ARB(7, "TCPSocketTransfer") << uniqueLabel() << " TCPSocketTransfer::receiveFragment: No data on receive socket, returning RECV_TIMEOUT" << TLOG_ENDL;
141  return RECV_TIMEOUT;
142  }
143  break;
144  }
145 
146  if (!(pollfd_s.revents & (POLLIN | POLLHUP | POLLERR)))
147  {
148  TLOG_DEBUG("TCPSocketTransfer") << uniqueLabel() << " TCPSocketTransfer::receiveFragment: Wrong event received from pollfd: " << static_cast<int>(pollfd_s.revents) << TLOG_ENDL;
149  continue;
150  }
151 
152  if (state_ == SocketState::Metadata)
153  {
154  //TLOG_DEBUG("TCPSocketTransfer") << uniqueLabel() << " TCPSocketTransfer::receiveFragment: Reading Message Header" << TLOG_ENDL;
155  buff = &(mha[offset]);
156  byte_cnt = sizeof(MessHead) - offset;
157  }
158  else
159  {
160  //TLOG_DEBUG("TCPSocketTransfer") << uniqueLabel() << " TCPSocketTransfer::receiveFragment: Reading data" << TLOG_ENDL;
161  buff = reinterpret_cast<uint8_t*>(&header) + offset;
162  byte_cnt = mh.byte_count - offset;
163  }
164 
165  //TLOG_DEBUG("TCPSocketTransfer") << uniqueLabel() << " TCPSocketTransfer::receiveFragment: Reading " << byte_cnt << " bytes from socket" << TLOG_ENDL;
166  sts = read(fd_, buff, byte_cnt);
167  //TLOG_DEBUG("TCPSocketTransfer") << uniqueLabel() << " TCPSocketTransfer::receiveFragment: Done with read" << TLOG_ENDL;
168 
169  TLOG_ARB(9, "TCPSocketTransfer") << uniqueLabel() << " TCPSocketTransfer::recvFragment state=" << static_cast<int>(state_) << " read=" << sts << " (errno=" << strerror(errno) << ")" << TLOG_ENDL;
170  if (sts <= 0)
171  {
172  TLOG_DEBUG("TCPSocketTransfer") << uniqueLabel() << " TCPSocketTransfer::receiveFragment: Error on receive, closing socket" << TLOG_ENDL;
173  close(fd_);
174  }
175  else
176  {
177  // see if we're done (with this state)
178  sts = offset += sts;
179  if (sts == target_bytes)
180  {
181  TLOG_ARB(7, "TCPSocketTransfer") << uniqueLabel() << " TCPSocketTransfer::receiveFragment: Target read bytes reached. Changing state" << TLOG_ENDL;
182  offset = 0;
183  if (state_ == SocketState::Metadata)
184  {
185  state_ = SocketState::Data;
186  mh.byte_count = ntohl(mh.byte_count);
187  mh.source_id = ntohs(mh.source_id);
188  target_bytes = mh.byte_count;
189  }
190  else
191  {
192  state_ = SocketState::Metadata;
193  target_bytes = sizeof(MessHead);
194  ret_rank = source_rank();
195  TLOG_ARB(9, "TCPSocketTransfer") << uniqueLabel() << " TCPSocketTransfer::recvFragment done sts=" << sts << " src=" << ret_rank << TLOG_ENDL;
196  TLOG_ARB(7, "TCPSocketTransfer") << uniqueLabel() << " TCPSocketTransfer::receiveFragment: Done receiving fragment. Moving into output." << TLOG_ENDL;
197 
198  done = true; // no more polls
199  break; // no more read of ready fds
200  }
201  }
202  }
203 
204  if (!done && timeout_usec > 0)
205  {
206  // calc next timeout_ms (unless timed out)
207  size_t delta_us = TimeUtils::gettimeofday_us() - start_time_us;
208  if (delta_us > timeout_usec)
209  return RECV_TIMEOUT;
210  timeout_ms = ((timeout_usec - delta_us) + 999) / 1000; // want at least 1 ms
211  }
212  } // while(!done)...poll
213 
214  TLOG_ARB(7, "TCPSocketTransfer") << uniqueLabel() << " TCPSocketTransfer::receiveFragment: Returning " << ret_rank << TLOG_ENDL;
215  return ret_rank;
216 }
217 
218 int artdaq::TCPSocketTransfer::receiveFragmentData(RawDataType* destination, size_t)
219 {
220  TLOG_ARB(7, "TCPSocketTransfer") << uniqueLabel() << " TCPSocketTransfer::receiveFragment: BEGIN" << TLOG_ENDL;
221  int ret_rank = RECV_TIMEOUT;
222  if (fd_ == -1)
223  { // what if just listen_fd???
224  TLOG_DEBUG("TCPSocketTransfer") << uniqueLabel() << " TCPSocketTransfer::receiveFragment: Receive socket not connected, returning RECV_TIMEOUT" << TLOG_ENDL;
225  return RECV_TIMEOUT;
226  }
227 
228  //void* buff=alloca(max_fragment_size_words_*8);
229  uint8_t* buff;
230  size_t byte_cnt = 0;
231  int sts;
232 
233  pollfd pollfd_s;
234  pollfd_s.events = POLLIN | POLLERR;
235  pollfd_s.fd = fd_;
236 
237  bool done = false;
238  while (!done)
239  {
240  //TLOG_DEBUG("TCPSocketTransfer") << uniqueLabel() << " TCPSocketTransfer::receiveFragment: Polling fd to see if there's data" << TLOG_ENDL;
241  int num_fds_ready = poll(&pollfd_s, 1, -1);
242  if (num_fds_ready <= 0)
243  {
244  if (num_fds_ready == 0)
245  {
246  TLOG_ARB(7, "TCPSocketTransfer") << uniqueLabel() << " TCPSocketTransfer::receiveFragment: No data on receive socket, returning RECV_TIMEOUT" << TLOG_ENDL;
247  return RECV_TIMEOUT;
248  }
249  break;
250  }
251 
252  if (!(pollfd_s.revents & (POLLIN | POLLERR)))
253  {
254  TLOG_DEBUG("TCPSocketTransfer") << uniqueLabel() << " TCPSocketTransfer::receiveFragment: Wrong event received from pollfd" << TLOG_ENDL;
255  continue;
256  }
257 
258  if (state_ == SocketState::Metadata)
259  {
260  //TLOG_DEBUG("TCPSocketTransfer") << uniqueLabel() << " TCPSocketTransfer::receiveFragment: Reading Message Header" << TLOG_ENDL;
261  buff = &(mha[offset]);
262  byte_cnt = sizeof(MessHead) - offset;
263  }
264  else
265  {
266  //TLOG_DEBUG("TCPSocketTransfer") << uniqueLabel() << " TCPSocketTransfer::receiveFragment: Reading data" << TLOG_ENDL;
267  buff = reinterpret_cast<uint8_t*>(destination) + offset;
268  byte_cnt = mh.byte_count - offset;
269  }
270 
271  //TLOG_DEBUG("TCPSocketTransfer") << uniqueLabel() << " TCPSocketTransfer::receiveFragment: Reading " << byte_cnt << " bytes from socket" << TLOG_ENDL;
272  sts = read(fd_, buff, byte_cnt);
273  //TLOG_DEBUG("TCPSocketTransfer") << uniqueLabel() << " TCPSocketTransfer::receiveFragment: Done with read" << TLOG_ENDL;
274 
275  TLOG_ARB(9, "TCPSocketTransfer") << uniqueLabel() << " recvFragment state=" << static_cast<int>(state_) << " read=" << sts << " (errno=" << strerror(errno) << ")" << TLOG_ENDL;
276  if (sts <= 0)
277  {
278  TLOG_DEBUG("TCPSocketTransfer") << uniqueLabel() << " TCPSocketTransfer::receiveFragment: Error on receive, closing socket" << TLOG_ENDL;
279  close(fd_);
280  }
281  else
282  {
283  // see if we're done (with this state)
284  sts = offset += sts;
285  if (sts == target_bytes)
286  {
287  TLOG_ARB(7, "TCPSocketTransfer") << uniqueLabel() << " TCPSocketTransfer::receiveFragment: Target read bytes reached. Changing state" << TLOG_ENDL;
288  offset = 0;
289  if (state_ == SocketState::Metadata)
290  {
291  state_ = SocketState::Data;
292  mh.byte_count = ntohl(mh.byte_count);
293  mh.source_id = ntohs(mh.source_id);
294  target_bytes = mh.byte_count;
295  }
296  else
297  {
298  state_ = SocketState::Metadata;
299  target_bytes = sizeof(MessHead);
300  ret_rank = source_rank();
301  TLOG_ARB(9, "TCPSocketTransfer") << uniqueLabel() << " recvFragment done sts=" << sts << " src=" << ret_rank << TLOG_ENDL;
302  TLOG_ARB(7, "TCPSocketTransfer") << uniqueLabel() << " TCPSocketTransfer::receiveFragment: Done receiving fragment. Moving into output." << TLOG_ENDL;
303 
304  done = true; // no more polls
305  break; // no more read of ready fds
306  }
307  }
308  }
309  } // while(!done)...poll
310 
311  TLOG_ARB(7, "TCPSocketTransfer") << uniqueLabel() << " TCPSocketTransfer::receiveFragment: Returning " << ret_rank << TLOG_ENDL;
312  return ret_rank;
313 }
314 
315 // Send the given Fragment. Return the rank of the destination to which
316 // the Fragment was sent OR -1 if to none.
317 artdaq::TransferInterface::CopyStatus artdaq::TCPSocketTransfer::sendFragment_(Fragment&& frag, size_t send_timeout_usec)
318 {
319  TLOG_ARB(7, "TCPSocketTransfer") << uniqueLabel() << " TCPSocketTransfer::sendFragment begin" << TLOG_ENDL;
320  artdaq::Fragment grab_ownership_frag = std::move(frag);
321 
322  // Send Fragment Header
323 
324  iovec iov = { reinterpret_cast<void*>(grab_ownership_frag.headerAddress()),
325  detail::RawFragmentHeader::num_words() * sizeof(RawDataType) };
326  auto sts = sendData_(&iov, 1, send_timeout_usec);
327  while (sts != CopyStatus::kSuccess)
328  {
329  TLOG_ARB(7, "TCPSocketTransfer") << uniqueLabel() << " TCPSocketTransfer::sendFragment: Timeout or Error sending fragment" << TLOG_ENDL;
330  sts = sendData_(&iov, 1, send_timeout_usec);
331  usleep(1000);
332  }
333 
334  // Send Fragment Data
335 
336  iov = { reinterpret_cast<void*>(grab_ownership_frag.headerAddress() + detail::RawFragmentHeader::num_words()),
337  grab_ownership_frag.sizeBytes() - detail::RawFragmentHeader::num_words() * sizeof(RawDataType) };
338  sts = sendData_(&iov, 1, send_timeout_usec);
339  while (sts != CopyStatus::kSuccess)
340  {
341  TLOG_ARB(7, "TCPSocketTransfer") << uniqueLabel() << " TCPSocketTransfer::sendFragment: Timeout or Error sending fragment" << TLOG_ENDL;
342  sts = sendData_(&iov, 1, send_timeout_usec);
343  usleep(1000);
344  }
345 
346  TRACE(7, "TCPSocketTransfer::sendFragment returning kSuccess");
347  return sts;
348 }
349 
350 artdaq::TransferInterface::CopyStatus artdaq::TCPSocketTransfer::sendData_(const void* buf, size_t bytes, size_t send_timeout_usec)
351 {
352  TLOG_DEBUG("TCPSocketTransfer") << uniqueLabel() << "TCPSocketTransfer::sendData_ Converting buf to iovec" << TLOG_ENDL;
353  iovec iov = { (void*)buf, bytes };
354  return sendData_(&iov, 1, send_timeout_usec);
355 }
356 
357 artdaq::TransferInterface::CopyStatus artdaq::TCPSocketTransfer::sendData_(const struct iovec* iov, int iovcnt, size_t send_timeout_usec)
358 {
359  // check all connected??? -- currently just check fd!=-1
360  if (fd_ == -1)
361  {
362  if (timeoutMessageArmed_)
363  {
364  TLOG_DEBUG("TCPSocketTransfer") << uniqueLabel() << "TCPSocketTransfer::sendData_: Send fd is not open. Returning kTimeout" << TLOG_ENDL;
365  timeoutMessageArmed_ = false;
366  }
367  return CopyStatus::kTimeout;
368  }
369  timeoutMessageArmed_ = true;
370  TLOG_ARB(12, "TCPSocketTransfer") << uniqueLabel() << "send_timeout_usec is " << std::to_string(send_timeout_usec) << ", currently unused." << TLOG_ENDL;
371 
372  //TLOG_DEBUG("TCPSocketTransfer") << uniqueLabel() << "TCPSocketTransfer::sendData_: Determining write size" << TLOG_ENDL;
373  uint32_t total_to_write_bytes = 0;
374  std::vector<iovec> iov_in(iovcnt + 1); // need contiguous (for the unlike case that only partial MH
375  std::vector<iovec> iovv(iovcnt + 2); // 1 more for mh and another one for any partial
376  int ii;
377  for (ii = 0; ii < iovcnt; ++ii)
378  {
379  iov_in[ii + 1] = iov[ii];
380  total_to_write_bytes += iov[ii].iov_len;
381  }
382  //TLOG_DEBUG("TCPSocketTransfer") << uniqueLabel() << "TCPSocketTransfer::sendData_: Constructing Message Header" << TLOG_ENDL;
383  MessHead mh = { 0,MessHead::data_v0,htons(source_rank()),htonl(total_to_write_bytes) };
384  iov_in[0].iov_base = &mh;
385  iov_in[0].iov_len = sizeof(mh);
386  total_to_write_bytes += sizeof(mh);
387 
388  ssize_t sts = 0;
389  ssize_t total_written_bytes = 0;
390  ssize_t per_write_max_bytes = (32 * 1024);
391 
392  size_t in_iov_idx = 0; // only increment this when we know the associated data has been xferred
393  size_t out_iov_idx = 0;
394  ssize_t this_write_bytes = 0;
395 
396  do
397  {
398  // The first out_iov may be set at the end of the previous loop.
399  // iov looping from below (b/c of the latter, we need to check this_write_bytes)
400  for (;
401  (in_iov_idx + out_iov_idx) < iov_in.size() && this_write_bytes < per_write_max_bytes;
402  ++out_iov_idx)
403  {
404  this_write_bytes += iov_in[in_iov_idx + out_iov_idx].iov_len;
405  iovv[out_iov_idx] = iov_in[in_iov_idx + out_iov_idx];
406  }
407  if (this_write_bytes > per_write_max_bytes)
408  {
409  iovv[out_iov_idx - 1].iov_len -= this_write_bytes - per_write_max_bytes;
410  this_write_bytes = per_write_max_bytes;
411  }
412 
413  // need to do blocking algorithm -- including throttled block notifications
414  do_again:
415  TLOG_ARB(7, "TCPSocketTransfer") << uniqueLabel() << "sendFragment b4 writev " << std::setw(7) << std::to_string(total_written_bytes) << " total_written_bytes fd=" << fd_ << " in_idx=" << std::to_string(in_iov_idx)
416  << " iovcnt=" << std::to_string(out_iov_idx) << " 1st.len=" << std::to_string(iovv[0].iov_len) << TLOG_ENDL;
417  //TLOG_DEBUG("TCPSocketTransfer") << uniqueLabel() << "TCPSocketTransfer calling writev" << TLOG_ENDL;
418  sts = writev(fd_, &(iovv[0]), out_iov_idx);
419  //TLOG_DEBUG("TCPSocketTransfer") << uniqueLabel() << "TCPSocketTransfer done with writev" << TLOG_ENDL;
420 
421  if (sts == -1)
422  {
423  if (errno == EAGAIN /* same as EWOULDBLOCK */)
424  {
425  TLOG_ARB(2, "TCPSocketTransfer") << uniqueLabel() << "sendFragment EWOULDBLOCK" << TLOG_ENDL;
426  fcntl(fd_, F_SETFL, 0); // clear O_NONBLOCK
427  blocking = true;
428  // NOTE: YES -- could drop here
429  goto do_again;
430  }
431  TLOG_DEBUG("TCPSocketTransfer") << uniqueLabel() << "TCPSocketTransfer::sendFragment_: WRITE ERROR: " << strerror(errno) << TLOG_ENDL;
432  connect_state = 0; // any write error closes
433  close(fd_);
434  fd_ = -1;
436  }
437  else if (sts != this_write_bytes)
438  {
439  // we'll loop around -- with
440  TLOG_ARB(4, "TCPSocketTransfer") << uniqueLabel() << "sendFragment writev sts(" << std::to_string(sts) << ")!=requested_send_bytes(" << std::to_string(this_write_bytes) << ")" << TLOG_ENDL;
441  total_written_bytes += sts; // add sts to total_written_bytes now as sts is adjusted next
442  // find which iovs are done
443  for (ii = 0; (size_t)sts >= iovv[ii].iov_len; ++ii)
444  sts -= iovv[ii].iov_len;
445  in_iov_idx += ii; // done with these in_iovs
446  iovv[ii].iov_len -= sts; // adjust partial iov
447  iovv[ii].iov_base = (uint8_t*)(iovv[ii].iov_base) + sts; // adjust partial iov
448 
449  // add more to get up to per_write_max_bytes
450  out_iov_idx = 0;
451  if (ii != 0)
452  iovv[out_iov_idx] = iovv[ii];
453  // starting over
454  this_write_bytes = iovv[out_iov_idx].iov_len;
455  // add any left over from appropriate in_iov_idx --
456  // i.e. match this out_iov with the in_iov that was used to
457  // initialize it; see how close the out base+len is to in base+len
458  // check !>per_write_max_bytes
459  unsigned long additional = ((unsigned long)iov_in[in_iov_idx].iov_base + iov_in[in_iov_idx].iov_len)
460  - ((unsigned long)iovv[out_iov_idx].iov_base + iovv[out_iov_idx].iov_len);
461  if (additional)
462  {
463  iovv[out_iov_idx].iov_len += additional;
464  this_write_bytes += additional;
465  if (this_write_bytes > per_write_max_bytes)
466  {
467  iovv[out_iov_idx].iov_len -= this_write_bytes - per_write_max_bytes;
468  this_write_bytes = per_write_max_bytes;
469  }
470  }
471  ++out_iov_idx; // done with
472  TLOG_ARB(4, "TCPSocketTransfer") << uniqueLabel() << "sendFragment writev sts!=: this_write_bytes=" << std::to_string(this_write_bytes)
473  << " out_iov_idx=" << std::to_string(out_iov_idx)
474  << " additional=" << std::to_string(additional)
475  << " ii=" << ii << TLOG_ENDL;
476  }
477  else
478  {
479  TLOG_ARB(4, "TCPSocketTransfer") << uniqueLabel() << "sendFragment writev sts(" << std::to_string(sts) << ")==requested_send_bytes(" << std::to_string(this_write_bytes) << ")" << TLOG_ENDL;
480  total_written_bytes += sts;
481  --out_iov_idx; // make it the index of the last iovv
482  iovv[out_iov_idx].iov_base = (uint8_t*)(iovv[out_iov_idx].iov_base) + iovv[out_iov_idx].iov_len;
483  iovv[out_iov_idx].iov_len = 0;
484  in_iov_idx += out_iov_idx; // at least this many complete (one more if "last iovv" is complete
485  this_write_bytes = 0;
486  // need to check last iovv against appropriate iov_in
487  unsigned long additional = ((unsigned long)iov_in[in_iov_idx].iov_base + iov_in[in_iov_idx].iov_len)
488  - ((unsigned long)iovv[out_iov_idx].iov_base + iovv[out_iov_idx].iov_len);
489  if (additional)
490  {
491  iovv[out_iov_idx].iov_len += additional;
492  this_write_bytes += additional;
493  if (this_write_bytes > per_write_max_bytes)
494  {
495  iovv[out_iov_idx].iov_len -= this_write_bytes - per_write_max_bytes;
496  this_write_bytes = per_write_max_bytes;
497  }
498  if (out_iov_idx != 0)
499  iovv[0] = iovv[out_iov_idx];
500  out_iov_idx = 1;
501  }
502  else
503  {
504  ++in_iov_idx;
505  out_iov_idx = 0;
506  }
507  }
508  } while (total_written_bytes < total_to_write_bytes);
509  if (total_written_bytes > total_to_write_bytes)
510  TLOG_ARB(0, "TCPSocketTransfer") << uniqueLabel() << "sendFragment program error: too many bytes transferred" << TLOG_ENDL;
511 
512  if (blocking)
513  {
514  blocking = false;
515  fcntl(fd_, F_SETFL, 0); // clear O_NONBLOCK
516  }
517  sts = total_written_bytes - sizeof(MessHead);
518 
519  TLOG_ARB(10, "TCPSocketTransfer") << uniqueLabel() << "sendFragment sts=" << std::to_string(sts) << TLOG_ENDL;
521 }
522 
523 //=============================================
524 
525 void artdaq::TCPSocketTransfer::stats_connect_() // thread
526 {
527  std::cv_status sts;
528  while (!stats_connect_stop_)
529  {
530  std::string desc;
531  void* tag;
532  std::function<void()> function;
533  uint64_t ts_us;
534 
535  int msdly = tmo_.get_next_timeout_msdly();
536 
537  if (msdly <= 0)
538  msdly = 2000;
539 
540  std::unique_lock<std::mutex> lck(stopstatscvm_);
541  sts = stopstatscv_.wait_until(lck
542  , std::chrono::system_clock::now()
543  + std::chrono::milliseconds(msdly));
544  TLOG_ARB(5, "TCPSocketTransfer") << uniqueLabel() << "thread1 after wait_until(msdly=" << msdly << ") - sts=" << static_cast<int>(sts) << TLOG_ENDL;
545 
546  if (sts == std::cv_status::no_timeout)
547  break;
548 
549  auto sts = tmo_.get_next_expired_timeout(desc, &tag, function, &ts_us);
550 
551  while (sts != -1 && desc != "")
552  {
553  if (function != NULL)
554  function();
555 
556  sts = tmo_.get_next_expired_timeout(desc, &tag, function, &ts_us);
557  }
558  }
559 }
560 
561 void artdaq::TCPSocketTransfer::connect_()
562 {
563  TLOG_DEBUG("TCPSocketTransfer") << uniqueLabel() << "Connecting sender socket" << TLOG_ENDL;
564  int sndbuf_bytes = static_cast<int>(sndbuf_);
565  fd_ = TCPConnect(hostMap_[destination_rank()].hostname.c_str()
566  , calculate_port_()
567  , O_NONBLOCK
568  , sndbuf_bytes);
569  connect_state = 0;
570  blocking = 0;
571  TLOG_DEBUG("TCPSocketTransfer") << uniqueLabel() << "TCPSocketTransfer::connect_ " + hostMap_[destination_rank()].hostname + ":" << calculate_port_() << " fd_=" << fd_ << TLOG_ENDL;
572  if (fd_ != -1)
573  {
574  // write connect msg
575  TLOG_DEBUG("TCPSocketTransfer") << uniqueLabel() << "TCPSocketTransfer::connect_: Writing connect message" << TLOG_ENDL;
576  MessHead mh = { 0,MessHead::connect_v0,htons(source_rank()),htonl(CONN_MAGIC) };
577  ssize_t sts = write(fd_, &mh, sizeof(mh));
578  if (sts == -1)
579  {
580  TLOG_ERROR("TCPSocketTransfer") << uniqueLabel() << "TCPSocketTransfer::connect_: Error writing connect message!" << TLOG_ENDL;
581  // a write error here is completely unexpected!
582  connect_state = 0;
583  close(fd_);
584  fd_ = -1;
585  }
586  else
587  {
588  TLOG_INFO("TCPSocketTransfer") << uniqueLabel() << "TCPSocketTransfer::connect_: Successfully connected" << TLOG_ENDL;
589  // consider it all connected/established
590  connect_state = 1;
591  }
592  }
593 }
594 
595 void artdaq::TCPSocketTransfer::reconnect_()
596 {
597  TLOG_ARB(5, "TCPSocketTransfer") << uniqueLabel() << "check/reconnect" << TLOG_ENDL;
598  if (fd_ == -1 && role() == TransferInterface::Role::kSend) return connect_();
599  if ((fd_ == -1 || listen_fd_ == -1) && role() == TransferInterface::Role::kReceive) return listen_();
600 }
601 
602 void artdaq::TCPSocketTransfer::listen_()
603 {
604  TLOG_DEBUG("TCPSocketTransfer") << uniqueLabel() << "TCPSocketTransfer::listen_: Listening/accepting new connections" << TLOG_ENDL;
605  if (listen_fd_ == -1)
606  {
607  TLOG_DEBUG("TCPSocketTransfer") << uniqueLabel() << "TCPSocketTransfer::listen_: Opening listener" << TLOG_ENDL;
608  listen_fd_ = TCP_listen_fd(calculate_port_(), rcvbuf_);
609  }
610  if (listen_fd_ == -1)
611  {
612  TLOG_DEBUG("TCPSocketTransfer") << uniqueLabel() << "TCPSocketTransfer::listen_: Error creating listen_fd_!" << TLOG_ENDL;
613  return;
614  }
615 
616  int res;
617  timeval tv = { 2,0 }; // maybe increase of some global "debugging" flag set???
618  fd_set rfds;
619  FD_ZERO(&rfds);
620  FD_SET(listen_fd_, &rfds);
621 
622  res = select(listen_fd_ + 1, &rfds, (fd_set *)0, (fd_set *)0, &tv);
623  if (res > 0)
624  {
625  int sts;
626  sockaddr_un un;
627  socklen_t arglen = sizeof(un);
628  int fd;
629  TLOG_DEBUG("TCPSocketTransfer") << uniqueLabel() << "Calling accept" << TLOG_ENDL;
630  fd = accept(listen_fd_, (sockaddr *)&un, &arglen);
631  TLOG_DEBUG("TCPSocketTransfer") << uniqueLabel() << "Done with accept" << TLOG_ENDL;
632 
633  TLOG_DEBUG("TCPSocketTransfer") << uniqueLabel() << "TCPSocketTransfer::listen_: Reading connect message" << TLOG_ENDL;
634  socklen_t lenlen = sizeof(tv);
635  /*sts=*/
636  setsockopt(fd, SOL_SOCKET, SO_RCVTIMEO, &tv, lenlen); // see man 7 socket.
637  MessHead mh;
638  uint64_t mark_us = TimeUtils::gettimeofday_us();
639  sts = read(fd, &mh, sizeof(mh));
640  uint64_t delta_us = TimeUtils::gettimeofday_us() - mark_us;
641  TLOG_DEBUG("TCPSocketTransfer") << uniqueLabel() << "TCPSocketTransfer::listen_: Read of connect message took " << delta_us << " microseconds." << TLOG_ENDL;
642  TLOG_ARB(10, "TCPSocketTransfer") << uniqueLabel() << "do_connect read of connect msg (after accept) took " << std::to_string(delta_us) << " microseconds" << TLOG_ENDL; // emperically, read take a couple hundred usecs.
643  if (sts != sizeof(mh))
644  {
645  TLOG_DEBUG("TCPSocketTransfer") << uniqueLabel() << "TCPSocketTransfer::listen_: Wrong message header length received!" << TLOG_ENDL;
646  TLOG_ARB(0, "TCPSocketTransfer") << uniqueLabel() << "do_connect_ problem with connect msg sts(" << sts << ")!=sizeof(mh)("<<std::to_string(sizeof(mh)) << ")" << TLOG_ENDL;
647  close(fd);
648  return;
649  }
650 
651  // check for "magic" and valid source_id(aka rank)
652  mh.source_id = ntohs(mh.source_id); // convert here as it is reference several times
653  if (ntohl(mh.conn_magic) != CONN_MAGIC || mh.source_id != source_rank())
654  {
655  TLOG_DEBUG("TCPSocketTransfer") << uniqueLabel() << "TCPSocketTransfer::listen_: Wrong magic bytes in header!" << TLOG_ENDL;
656  close(fd);
657  return;
658  }
659  if (fd_ != -1)
660  {
661  // close previous dec connect_count_
662  close(fd_);
663  }
664 
665  // now add (new) connection
666  fd_ = fd;
667  TLOG_INFO("TCPSocketTransfer") << uniqueLabel() << "TCPSocketTransfer::listen_: New fd is " << fd_ << TLOG_ENDL;
668 
669  TLOG_ARB(3, "TCPSocketTransfer") << uniqueLabel() << "do_connect_ connection from sender_rank=" << std::to_string(mh.source_id) << TLOG_ENDL;
670  }
671  else
672  {
673  TLOG_ARB(10, "TCPSocketTransfer") << uniqueLabel() << "TCPSocketTransfer::do_connect_: No connections in timeout interval!" << TLOG_ENDL;
674  }
675 } // do_connect_
676 
677 DEFINE_ARTDAQ_TRANSFER(artdaq::TCPSocketTransfer)
virtual int source_rank() const
Get the source rank for this TransferInterface instance.
int TCPConnect(char const *host_in, int dflt_port, long flags=0, int sndbufsiz=0)
Connect to a host on a given port.
Definition: TCPConnect.cc:122
uint32_t conn_magic
unsigned first is better for MessHead initializer: {0,0,my_node_idx_,CONN_MAGIC}
Definition: SRSockets.hh:38
static const int RECV_TIMEOUT
Value to be returned upon receive timeout. Because receivers otherwise return rank, this is also the limit on the number of ranks that artdaq currently supports.
This TransferInterface is a Receiver.
int receiveFragmentData(RawDataType *destination, size_t wordCount) override
Receive the body of a Fragment to the given destination pointer.
int TCP_listen_fd(int port, int rcvbuf)
Create a TCP listening socket on the given port and INADDR_ANY, with the given receive buffer...
TCPSocketTransfer(fhicl::ParameterSet const &ps, Role role)
TCPSocketTransfer Constructor.
int receiveFragmentHeader(detail::RawFragmentHeader &header, size_t receiveTimeout) override
Receive a Fragment Header from the transport mechanism.
This TransferInterface is a Sender.
int32_t byte_count
use CONN_MAGIC for connect_v0, data that follow for data_v0 (and 0 lenght data)
Definition: SRSockets.hh:39
Some error occurred, but no exception was thrown.
Role
Used to determine if a TransferInterface is a Sender or Receiver.
int64_t source_id
Rank of the source.
Definition: SRSockets.hh:34
The send operation completed successfully.
std::string uniqueLabel() const
Get the unique label of this TransferInterface instance.
This interface defines the functions used to transfer data between artdaq applications.
TransferInterface implementation plugin that sends data using TCP sockets.
This header is sent by the TCPSocket_transfer to allow for more efficient writev calls.
Definition: SRSockets.hh:15
void add_periodic(const char *desc, void *tag, std::function< void()> &function, uint64_t period_us, uint64_t start_us=0)
Add a periodic timeout to the Timeout container.
Definition: Timeout.cc:61
CopyStatus
Returned from the send functions, this enumeration describes the possible return codes. If an exception occurs, it will be thrown and should be handled normally.