artdaq  v3_01_00
TCPSocket_transfer.cc
1 // Sep 14, 2016. "TERMS AND CONDITIONS" governing this file are in the README
2 // or COPYING file. If you do not have such a file, one can be obtained by
3 // contacting Ron or Fermi Lab in Batavia IL, 60510, phone: 630-840-3000.
4 // $RCSfile: .emacs.gnu,v $
5 // rev="$Revision: 1.30 $$Date: 2016/03/01 14:27:27 $";
6 
7 // C Includes
8 #include <stdlib.h> // atoi, strtoul
9 #include <sys/socket.h> // socket, socklen_t
10 #include <sys/un.h> // sockaddr_un
11 #include <arpa/inet.h> // ntohl, ntohs
12 #include <sys/types.h> // size_t
13 #include <poll.h> // struct pollfd
14 
15 // C++ Includes
16 #include <string>
17 #include <fstream>
18 #include <stdexcept>
19 
20 // product Includes
21 #define TRACE_NAME "TCPSocketTransfer"
22 #include "artdaq/DAQdata/Globals.hh"
23 
24 // artdaq Includes
25 #include "artdaq/TransferPlugins/TCPSocketTransfer.hh"
28 #include "artdaq/TransferPlugins/detail/Timeout.hh"
29 #include "artdaq/TransferPlugins/detail/SRSockets.hh"
30 #include "artdaq-core/Data/Fragment.hh"
31 #include "artdaq-core/Utilities/TimeUtils.hh"
32 #include <iomanip>
33 
34 std::atomic<int> artdaq::TCPSocketTransfer::listen_thread_refcount_(0);
35 std::unique_ptr<boost::thread> artdaq::TCPSocketTransfer::listen_thread_ = nullptr;
36 std::map<int, std::set<int>> artdaq::TCPSocketTransfer::connected_fds_ = std::map<int, std::set<int>>();
37 std::mutex artdaq::TCPSocketTransfer::listen_thread_mutex_;
38 
40 TCPSocketTransfer(fhicl::ParameterSet const& pset, TransferInterface::Role role)
41  : TransferInterface(pset, role)
42  , send_fd_(-1)
43  , active_receive_fd_(-1)
44  , last_active_receive_fd_(-1)
45  , rcvbuf_(pset.get<size_t>("tcp_receive_buffer_size", 0))
46  , sndbuf_(max_fragment_size_words_ * sizeof(artdaq::RawDataType) * buffer_count_)
47  , send_retry_timeout_us_(pset.get<size_t>("send_retry_timeout_us", 1000000))
48  , stats_connect_stop_(false)
49  , stats_connect_thread_(std::bind(&TCPSocketTransfer::stats_connect_, this))
50  , timeoutMessageArmed_(true)
51  , not_connected_count_(0)
52  , receive_err_threshold_(pset.get<size_t>("receive_socket_disconnected_max_count", 1000))
53  , receive_err_wait_us_(pset.get<size_t>("receive_socket_disconnected_wait_us", 10000))
54 {
55  TLOG(TLVL_DEBUG) << GetTraceName() << " Constructor: pset=" << pset.to_string() << ", role=" << (role == TransferInterface::Role::kReceive ? "kReceive" : "kSend");
56  auto masterPortOffset = pset.get<int>("offset_all_ports", 0);
57  hostMap_ = MakeHostMap(pset, masterPortOffset);
58 
59  std::function<void()> function = std::bind(&TCPSocketTransfer::reconnect_, this);
60  tmo_.add_periodic("reconnect", NULL, function, 200/*millisec*/);
61 
63  {
64  // Wait for sender to connect...
65  TLOG(TLVL_DEBUG) << GetTraceName() << ": Listening for connections";
66  start_listen_thread_();
67  TLOG(TLVL_DEBUG) << GetTraceName() << ": Done Listening";
68  }
69  else
70  {
71  TLOG(TLVL_DEBUG) << GetTraceName() << ": Connecting to destination";
72  connect_();
73  TLOG(TLVL_DEBUG) << GetTraceName() << ": Done Connecting";
74  }
75  TLOG(TLVL_DEBUG) << GetTraceName() << ": End of Constructor";
76 }
77 
78 artdaq::TCPSocketTransfer::~TCPSocketTransfer() noexcept
79 {
80  TLOG(TLVL_DEBUG) << GetTraceName() << ": Shutting down TCPSocketTransfer";
81  stats_connect_stop_ = true;
82  stopstatscv_.notify_all();
83  stats_connect_thread_.join();
84 
85  if (role() == TransferInterface::Role::kSend)
86  {
87  // close all open connections (send stop_v0) first
88  MessHead mh = { 0,MessHead::stop_v0,htons(TransferInterface::source_rank()),{0} };
89  if (send_fd_ != -1)
90  {
91  // should be blocking with modest timeo
92  timeval tv = { 0,100000 };
93  socklen_t len = sizeof(tv);
94  setsockopt(send_fd_, SOL_SOCKET, SO_SNDTIMEO, &tv, len);
95  write(send_fd_, &mh, sizeof(mh));
96  }
97  close(send_fd_);
98  }
99  else
100  {
101  auto it = connected_fds_[source_rank()].begin();
102  while (it != connected_fds_[source_rank()].end())
103  {
104  close(*it);
105  it = connected_fds_[source_rank()].erase(it);
106  }
107 
108  std::unique_lock<std::mutex> lk(listen_thread_mutex_);
109  listen_thread_refcount_--;
110  if (listen_thread_refcount_ == 0 && listen_thread_ && listen_thread_->joinable())
111  {
112  listen_thread_->join();
113  }
114  }
115  TLOG(TLVL_DEBUG) << GetTraceName() << ": End of Destructor";
116 }
117 
118 int artdaq::TCPSocketTransfer::receiveFragmentHeader(detail::RawFragmentHeader& header, size_t timeout_usec)
119 {
120  TLOG(5) << GetTraceName() << ": receiveFragmentHeader: BEGIN";
121  int ret_rank = RECV_TIMEOUT;
122 
123  if (connected_fds_[source_rank()].size() == 0)
124  { // what if just listen_fd???
125  if (++not_connected_count_ > receive_err_threshold_) { return DATA_END; }
126  TLOG(7) << GetTraceName() << ": receiveFragmentHeader: Receive socket not connected, returning RECV_TIMEOUT";
127  usleep(receive_err_wait_us_);
128  return RECV_TIMEOUT;
129  }
130  not_connected_count_ = 0;
131 
132  TLOG(5) << GetTraceName() << ": receiveFragmentHeader timeout_usec=" << std::to_string(timeout_usec);
133  //void* buff=alloca(max_fragment_size_words_*8);
134  size_t byte_cnt = 0;
135  int sts;
136  int offset = 0;
137  SocketState state = SocketState::Metadata;
138  int target_bytes = sizeof(MessHead);
139  uint64_t start_time_us = TimeUtils::gettimeofday_us();
140 
141  //while (active_receive_fd_ != -1)
142  //{
143  // TLOG(TLVL_TRACE) << GetTraceName() << ": Currently receiving from fd " << active_receive_fd_ << ", waiting!";
144  // usleep(1000);
145  //}
146 
147 
148  uint8_t* buff;
149 
150  int timeout_ms;
151  if (timeout_usec == 0)
152  timeout_ms = 0;
153  else
154  timeout_ms = (timeout_usec + 999) / 1000; // want at least 1 ms
155 
156  bool done = false;
157  while (!done && connected_fds_[source_rank()].size() > 0)
158  {
159  if (active_receive_fd_ == -1)
160  {
161  size_t fd_count = connected_fds_[source_rank()].size();
162  auto iter = connected_fds_[source_rank()].begin();
163  std::vector<pollfd> pollfds(fd_count);
164  for (size_t ii = 0; ii < fd_count; ++ii)
165  {
166  pollfds[ii].events = POLLIN | POLLERR;
167  pollfds[ii].fd = *iter;
168  ++iter;
169  }
170 
171  //TLOG(TLVL_DEBUG) << GetTraceName() << ": receiveFragment: Polling fd to see if there's data" ;
172  int num_fds_ready = poll(&pollfds[0], fd_count, timeout_ms);
173  if (num_fds_ready <= 0)
174  {
175  if (num_fds_ready == 0 && timeout_ms > 0)
176  {
177  TLOG(5) << GetTraceName() << ": receiveFragmentHeader: No data on receive socket, returning RECV_TIMEOUT";
178  return RECV_TIMEOUT;
179  }
180  break;
181  }
182 
183  size_t index = 0;
184  if (last_active_receive_fd_ != -1)
185  {
186  for (auto& pollfd : pollfds)
187  {
188  index++;
189  if (pollfd.fd == last_active_receive_fd_)
190  {
191  break;
192  }
193  }
194  }
195 
196  int active_index = -1;
197  short anomolous_events = 0;
198  for (size_t ii = index; ii < index + pollfds.size(); ++ii)
199  {
200  if (pollfds[index % pollfds.size()].revents & (POLLIN | POLLPRI | POLLHUP | POLLERR))
201  {
202  active_index = index % pollfds.size();
203  active_receive_fd_ = pollfds[active_index].fd;
204  break;
205  }
206  else if (pollfds[index % pollfds.size()].revents & (POLLNVAL))
207  {
208  TLOG(TLVL_DEBUG) << GetTraceName() << ": receiveFragmentHeader: FD is closed, most likely because the peer went away. Removing from fd list.";
209  close(pollfds[active_index].fd);
210  connected_fds_[source_rank()].erase(pollfds[active_index].fd);
211  continue;
212  }
213  else if (pollfds[index % pollfds.size()].revents)
214  {
215  anomolous_events |= pollfds[index % pollfds.size()].revents;
216  }
217  }
218 
219  if (active_index == -1)
220  {
221  TLOG(TLVL_DEBUG) << GetTraceName() << ": receiveFragmentHeader: Wrong event received from a pollfd. Mask: " << static_cast<int>(anomolous_events);
222  active_receive_fd_ = -1;
223  continue;
224  }
225 
226  if (!done && timeout_usec > 0)
227  {
228  // calc next timeout_ms (unless timed out)
229  size_t delta_us = TimeUtils::gettimeofday_us() - start_time_us;
230  if (delta_us > timeout_usec)
231  {
232  return RECV_TIMEOUT;
233  }
234  timeout_ms = ((timeout_usec - delta_us) + 999) / 1000; // want at least 1 ms
235  }
236  }
237 
238  if (state == SocketState::Metadata)
239  {
240  //TLOG(TLVL_DEBUG) << GetTraceName() << ": receiveFragmentHeader: Reading Message Header" ;
241  buff = &(mha[offset]);
242  byte_cnt = sizeof(MessHead) - offset;
243  }
244  else
245  {
246  //TLOG(TLVL_DEBUG) << GetTraceName() << ": receiveFragmentHeader: Reading data" ;
247  buff = reinterpret_cast<uint8_t*>(&header) + offset;
248  byte_cnt = mh.byte_count - offset;
249  }
250 
251  if (byte_cnt > 0)
252  {
253  TLOG(6) << GetTraceName() << ": receiveFragmentHeader: Reading " << byte_cnt << " bytes from socket";
254  sts = read(active_receive_fd_, buff, byte_cnt);
255  TLOG(6) << GetTraceName() << ": receiveFragmentHeader: Done with read";
256  }
257 
258  TLOG(7) << GetTraceName() << ": receiveFragmentHeader state=" << static_cast<int>(state) << " read=" << sts;
259  if (sts < 0)
260  {
261  TLOG(TLVL_WARNING) << GetTraceName() << ": receiveFragmentHeader: Error on receive, closing socket " << " (errno=" << errno << ": " << strerror(errno) << ")";
262  close(active_receive_fd_);
263  connected_fds_[source_rank()].erase(active_receive_fd_);
264  active_receive_fd_ = -1;
265  }
266  else
267  {
268  // see if we're done (with this state)
269  sts = offset += sts;
270  if (sts >= target_bytes)
271  {
272  TLOG(7) << GetTraceName() << ": receiveFragmentHeader: Target read bytes reached. Changing state";
273  offset = 0;
274  if (state == SocketState::Metadata)
275  {
276  state = SocketState::Data;
277  mh.byte_count = ntohl(mh.byte_count);
278  mh.source_id = ntohs(mh.source_id);
279  target_bytes = mh.byte_count;
280 
281  if (mh.message_type == MessHead::stop_v0)
282  {
283  TLOG(TLVL_DEBUG) << GetTraceName() << ": receiveFragmentHeader: Stop Message received. Closing socket";
284  close(active_receive_fd_);
285  connected_fds_[source_rank()].erase(active_receive_fd_);
286  active_receive_fd_ = -1;
287  }
288  }
289  else
290  {
291  ret_rank = source_rank();
292  TLOG(8) << GetTraceName() << ": receiveFragmentHeader done sts=" << sts << " src=" << ret_rank;
293  TLOG(7) << GetTraceName() << ": receiveFragmentHeader: Done receiving fragment header. Moving into output.";
294 
295  done = true; // no more polls
296  break; // no more read of ready fds
297  }
298  }
299  }
300 
301  } // while(!done)...poll
302 
303  TLOG(5) << GetTraceName() << ": receiveFragmentHeader: Returning " << ret_rank;
304  return ret_rank;
305 }
306 
307 int artdaq::TCPSocketTransfer::receiveFragmentData(RawDataType* destination, size_t)
308 {
309  TLOG(9) << GetTraceName() << ": receiveFragmentData: BEGIN";
310  int ret_rank = RECV_TIMEOUT;
311  if (active_receive_fd_ == -1)
312  { // what if just listen_fd???
313  TLOG(TLVL_DEBUG) << GetTraceName() << ": receiveFragmentData: Receive socket not connected, returning RECV_TIMEOUT";
314  return RECV_TIMEOUT;
315  }
316 
317  //void* buff=alloca(max_fragment_size_words_*8);
318  uint8_t* buff;
319  size_t byte_cnt = 0;
320  int sts;
321  int offset = 0;
322  SocketState state = SocketState::Metadata;
323  int target_bytes = sizeof(MessHead);
324 
325  pollfd pollfd_s;
326  pollfd_s.events = POLLIN | POLLPRI | POLLERR;
327  pollfd_s.fd = active_receive_fd_;
328 
329  bool done = false;
330  while (!done)
331  {
332  TLOG(TLVL_DEBUG) << GetTraceName() << ": receiveFragmentData: Polling fd to see if there's data";
333  int num_fds_ready = poll(&pollfd_s, 1, 1000);
334  if (num_fds_ready <= 0)
335  {
336  if (num_fds_ready == 0)
337  {
338  TLOG(9) << GetTraceName() << ": receiveFragmentData: No data on receive socket, returning RECV_TIMEOUT";
339  active_receive_fd_ = -1;
340  return RECV_TIMEOUT;
341  }
342 
343  TLOG(TLVL_ERROR) << "Error in poll: errno=" << errno;
344  active_receive_fd_ = -1;
345  break;
346  }
347 
348  if (pollfd_s.revents & (POLLNVAL))
349  {
350  TLOG(TLVL_DEBUG) << GetTraceName() << ": receiveFragmentData: FD is closed, most likely because the peer went away. Removing from fd list.";
351  close(active_receive_fd_);
352  connected_fds_[source_rank()].erase(active_receive_fd_);
353  active_receive_fd_ = -1;
354  break;
355  }
356  else if (!(pollfd_s.revents & (POLLIN | POLLPRI | POLLERR)))
357  {
358  TLOG(TLVL_DEBUG) << GetTraceName() << ": receiveFragmentData: Wrong event received from pollfd: " << pollfd_s.revents;
359  close(active_receive_fd_);
360  connected_fds_[source_rank()].erase(active_receive_fd_);
361  continue;
362  }
363 
364  if (state == SocketState::Metadata)
365  {
366  //TLOG(TLVL_DEBUG) << GetTraceName() << ": receiveFragmentData: Reading Message Header" ;
367  buff = &(mha[offset]);
368  byte_cnt = sizeof(MessHead) - offset;
369  }
370  else
371  {
372  //TLOG(TLVL_DEBUG) << GetTraceName() << ": receiveFragmentData: Reading data" ;
373  buff = reinterpret_cast<uint8_t*>(destination) + offset;
374  byte_cnt = mh.byte_count - offset;
375  }
376 
377  //TLOG(TLVL_DEBUG) << GetTraceName() << ": receiveFragmentData: Reading " << byte_cnt << " bytes from socket" ;
378  sts = read(active_receive_fd_, buff, byte_cnt);
379  //TLOG(TLVL_DEBUG) << GetTraceName() << ": receiveFragmentData: Done with read" ;
380 
381  TLOG(10) << GetTraceName() << ": recvFragment state=" << static_cast<int>(state) << " read=" << sts;
382  if (sts < 0)
383  {
384  TLOG(TLVL_DEBUG) << GetTraceName() << ": receiveFragmentData: Error on receive, closing socket"
385  << " (errno=" << errno << ": " << strerror(errno) << ")";
386  close(active_receive_fd_);
387  connected_fds_[source_rank()].erase(active_receive_fd_);
388  active_receive_fd_ = -1;
389  }
390  else
391  {
392  // see if we're done (with this state)
393  sts = offset += sts;
394  if (sts >= target_bytes)
395  {
396  TLOG(9) << GetTraceName() << ": receiveFragmentData: Target read bytes reached. Changing state";
397  offset = 0;
398  if (state == SocketState::Metadata)
399  {
400  state = SocketState::Data;
401  mh.byte_count = ntohl(mh.byte_count);
402  mh.source_id = ntohs(mh.source_id);
403  target_bytes = mh.byte_count;
404  }
405  else
406  {
407  ret_rank = source_rank();
408  TLOG(11) << GetTraceName() << ": receiveFragmentData done sts=" << sts << " src=" << ret_rank;
409  TLOG(9) << GetTraceName() << ": receiveFragmentData: Done receiving fragment. Moving into output.";
410 
411  done = true; // no more polls
412  break; // no more read of ready fds
413  }
414  }
415  }
416 
417  // Check if we were asked to do a 0-size receive
418  if (target_bytes == 0 && state == SocketState::Data)
419  {
420  ret_rank = source_rank();
421  TLOG(11) << GetTraceName() << ": receiveFragmentData done sts=" << sts << " src=" << ret_rank;
422  TLOG(9) << GetTraceName() << ": receiveFragmentData: Done receiving fragment. Moving into output.";
423 
424  done = true; // no more polls
425  }
426 
427  } // while(!done)...poll
428 
429  last_active_receive_fd_ = active_receive_fd_;
430  active_receive_fd_ = -1;
431 
432  TLOG(9) << GetTraceName() << ": receiveFragmentData: Returning " << ret_rank;
433  return ret_rank;
434 }
435 
436 // Send the given Fragment. Return the rank of the destination to which
437 // the Fragment was sent OR -1 if to none.
438 artdaq::TransferInterface::CopyStatus artdaq::TCPSocketTransfer::sendFragment_(Fragment&& frag, size_t send_timeout_usec)
439 {
440  TLOG(12) << GetTraceName() << ": sendFragment begin";
441  artdaq::Fragment grab_ownership_frag = std::move(frag);
442 
443  // Send Fragment Header
444 
445  iovec iov = { reinterpret_cast<void*>(grab_ownership_frag.headerAddress()),
446  detail::RawFragmentHeader::num_words() * sizeof(RawDataType) };
447 
448  auto sts = sendData_(&iov, 1, send_retry_timeout_us_);
449  auto start_time = std::chrono::steady_clock::now();
450  //If it takes more than 10 seconds to write a Fragment header, give up
451  while (sts != CopyStatus::kSuccess && (send_timeout_usec == 0 || TimeUtils::GetElapsedTimeMicroseconds(start_time) < send_timeout_usec) && TimeUtils::GetElapsedTimeMicroseconds(start_time) < 10000000)
452  {
453  TLOG(13) << GetTraceName() << ": sendFragment: Timeout or Error sending fragment";
454  sts = sendData_(&iov, 1, send_retry_timeout_us_);
455  usleep(1000);
456  }
457  if (sts != CopyStatus::kSuccess) return sts;
458 
459  // Send Fragment Data
460 
461  iov = { reinterpret_cast<void*>(grab_ownership_frag.headerAddress() + detail::RawFragmentHeader::num_words()),
462  grab_ownership_frag.sizeBytes() - detail::RawFragmentHeader::num_words() * sizeof(RawDataType) };
463  sts = sendData_(&iov, 1, send_retry_timeout_us_);
464  start_time = std::chrono::steady_clock::now();
465  while (sts != CopyStatus::kSuccess && (send_timeout_usec == 0 || TimeUtils::GetElapsedTimeMicroseconds(start_time) < send_timeout_usec) && TimeUtils::GetElapsedTimeMicroseconds(start_time) < 10000000)
466  {
467  TLOG(13) << GetTraceName() << ": sendFragment: Timeout or Error sending fragment";
468  sts = sendData_(&iov, 1, send_retry_timeout_us_);
469  usleep(1000);
470  }
471 
472  TLOG(12) << GetTraceName() << ": sendFragment returning kSuccess";
473  return sts;
474 }
475 
476 artdaq::TransferInterface::CopyStatus artdaq::TCPSocketTransfer::sendData_(const void* buf, size_t bytes, size_t send_timeout_usec)
477 {
478  TLOG(TLVL_DEBUG) << GetTraceName() << ": sendData_ Converting buf to iovec";
479  iovec iov = { (void*)buf, bytes };
480  return sendData_(&iov, 1, send_timeout_usec);
481 }
482 
483 artdaq::TransferInterface::CopyStatus artdaq::TCPSocketTransfer::sendData_(const struct iovec* iov, int iovcnt, size_t send_timeout_usec)
484 {
485  // check all connected??? -- currently just check fd!=-1
486  if (send_fd_ == -1)
487  {
488  if (timeoutMessageArmed_)
489  {
490  TLOG(TLVL_DEBUG) << GetTraceName() << ": sendData_: Send fd is not open. Returning kTimeout";
491  timeoutMessageArmed_ = false;
492  }
493  return CopyStatus::kTimeout;
494  }
495  timeoutMessageArmed_ = true;
496  TLOG(14) << GetTraceName() << ": send_timeout_usec is " << std::to_string(send_timeout_usec) << ", currently unused.";
497 
498  //TLOG(TLVL_DEBUG) << GetTraceName() << ": sendData_: Determining write size" ;
499  uint32_t total_to_write_bytes = 0;
500  std::vector<iovec> iov_in(iovcnt + 1); // need contiguous (for the unlike case that only partial MH
501  std::vector<iovec> iovv(iovcnt + 2); // 1 more for mh and another one for any partial
502  int ii;
503  for (ii = 0; ii < iovcnt; ++ii)
504  {
505  iov_in[ii + 1] = iov[ii];
506  total_to_write_bytes += iov[ii].iov_len;
507  }
508  //TLOG(TLVL_DEBUG) << GetTraceName() << ": sendData_: Constructing Message Header" ;
509  MessHead mh = { 0,MessHead::data_v0,htons(source_rank()),{htonl(total_to_write_bytes)} };
510  iov_in[0].iov_base = &mh;
511  iov_in[0].iov_len = sizeof(mh);
512  total_to_write_bytes += sizeof(mh);
513 
514  ssize_t sts = 0;
515  ssize_t total_written_bytes = 0;
516  ssize_t per_write_max_bytes = (32 * 1024);
517 
518  size_t in_iov_idx = 0; // only increment this when we know the associated data has been xferred
519  size_t out_iov_idx = 0;
520  ssize_t this_write_bytes = 0;
521 
522  do
523  {
524  // The first out_iov may be set at the end of the previous loop.
525  // iov looping from below (b/c of the latter, we need to check this_write_bytes)
526  for (;
527  (in_iov_idx + out_iov_idx) < iov_in.size() && this_write_bytes < per_write_max_bytes;
528  ++out_iov_idx)
529  {
530  this_write_bytes += iov_in[in_iov_idx + out_iov_idx].iov_len;
531  iovv[out_iov_idx] = iov_in[in_iov_idx + out_iov_idx];
532  }
533  if (this_write_bytes > per_write_max_bytes)
534  {
535  iovv[out_iov_idx - 1].iov_len -= this_write_bytes - per_write_max_bytes;
536  this_write_bytes = per_write_max_bytes;
537  }
538 
539  // need to do blocking algorithm -- including throttled block notifications
540  do_again:
541  TLOG(14) << GetTraceName() << ": sendFragment b4 writev " << std::setw(7) << std::to_string(total_written_bytes) << " total_written_bytes send_fd_=" << send_fd_ << " in_idx=" << std::to_string(in_iov_idx)
542  << " iovcnt=" << std::to_string(out_iov_idx) << " 1st.len=" << std::to_string(iovv[0].iov_len);
543  //TLOG(TLVL_DEBUG) << GetTraceName() << " calling writev" ;
544  sts = writev(send_fd_, &(iovv[0]), out_iov_idx);
545  //TLOG(TLVL_DEBUG) << GetTraceName() << " done with writev" ;
546 
547  if (sts == -1)
548  {
549  if (errno == EAGAIN /* same as EWOULDBLOCK */)
550  {
551  TLOG(TLVL_DEBUG) << GetTraceName() << ": sendFragment EWOULDBLOCK";
552  fcntl(send_fd_, F_SETFL, 0); // clear O_NONBLOCK
553  blocking = true;
554  // NOTE: YES -- could drop here
555  goto do_again;
556  }
557  TLOG(TLVL_WARNING) << GetTraceName() << ": sendFragment_: WRITE ERROR: " << strerror(errno);
558  connect_state = 0; // any write error closes
559  close(send_fd_);
560  send_fd_ = -1;
562  }
563  else if (sts != this_write_bytes)
564  {
565  // we'll loop around -- with
566  TLOG(TLVL_DEBUG) << GetTraceName() << ": sendFragment writev sts(" << std::to_string(sts) << ")!=requested_send_bytes(" << std::to_string(this_write_bytes) << ")";
567  total_written_bytes += sts; // add sts to total_written_bytes now as sts is adjusted next
568  // find which iovs are done
569  for (ii = 0; (size_t)sts >= iovv[ii].iov_len; ++ii)
570  sts -= iovv[ii].iov_len;
571  in_iov_idx += ii; // done with these in_iovs
572  iovv[ii].iov_len -= sts; // adjust partial iov
573  iovv[ii].iov_base = (uint8_t*)(iovv[ii].iov_base) + sts; // adjust partial iov
574 
575  // add more to get up to per_write_max_bytes
576  out_iov_idx = 0;
577  if (ii != 0)
578  iovv[out_iov_idx] = iovv[ii];
579  // starting over
580  this_write_bytes = iovv[out_iov_idx].iov_len;
581  // add any left over from appropriate in_iov_idx --
582  // i.e. match this out_iov with the in_iov that was used to
583  // initialize it; see how close the out base+len is to in base+len
584  // check !>per_write_max_bytes
585  unsigned long additional = ((unsigned long)iov_in[in_iov_idx].iov_base + iov_in[in_iov_idx].iov_len)
586  - ((unsigned long)iovv[out_iov_idx].iov_base + iovv[out_iov_idx].iov_len);
587  if (additional)
588  {
589  iovv[out_iov_idx].iov_len += additional;
590  this_write_bytes += additional;
591  if (this_write_bytes > per_write_max_bytes)
592  {
593  iovv[out_iov_idx].iov_len -= this_write_bytes - per_write_max_bytes;
594  this_write_bytes = per_write_max_bytes;
595  }
596  }
597  ++out_iov_idx; // done with
598  TLOG(TLVL_TRACE) << GetTraceName() << ": sendFragment writev sts!=: this_write_bytes=" << std::to_string(this_write_bytes)
599  << " out_iov_idx=" << std::to_string(out_iov_idx)
600  << " additional=" << std::to_string(additional)
601  << " ii=" << ii;
602  }
603  else
604  {
605  TLOG(TLVL_TRACE) << GetTraceName() << ": sendFragment writev sts(" << std::to_string(sts) << ")==requested_send_bytes(" << std::to_string(this_write_bytes) << ")";
606  total_written_bytes += sts;
607  --out_iov_idx; // make it the index of the last iovv
608  iovv[out_iov_idx].iov_base = (uint8_t*)(iovv[out_iov_idx].iov_base) + iovv[out_iov_idx].iov_len;
609  iovv[out_iov_idx].iov_len = 0;
610  in_iov_idx += out_iov_idx; // at least this many complete (one more if "last iovv" is complete
611  this_write_bytes = 0;
612  // need to check last iovv against appropriate iov_in
613  unsigned long additional = ((unsigned long)iov_in[in_iov_idx].iov_base + iov_in[in_iov_idx].iov_len)
614  - ((unsigned long)iovv[out_iov_idx].iov_base + iovv[out_iov_idx].iov_len);
615  if (additional)
616  {
617  iovv[out_iov_idx].iov_len += additional;
618  this_write_bytes += additional;
619  if (this_write_bytes > per_write_max_bytes)
620  {
621  iovv[out_iov_idx].iov_len -= this_write_bytes - per_write_max_bytes;
622  this_write_bytes = per_write_max_bytes;
623  }
624  if (out_iov_idx != 0)
625  iovv[0] = iovv[out_iov_idx];
626  out_iov_idx = 1;
627  }
628  else
629  {
630  ++in_iov_idx;
631  out_iov_idx = 0;
632  }
633  }
634  } while (total_written_bytes < total_to_write_bytes);
635  if (total_written_bytes > total_to_write_bytes)
636  TLOG(TLVL_ERROR) << GetTraceName() << ": sendFragment program error: too many bytes transferred";
637 
638  if (blocking)
639  {
640  blocking = false;
641  fcntl(send_fd_, F_SETFL, 0); // clear O_NONBLOCK
642  }
643  sts = total_written_bytes - sizeof(MessHead);
644 
645  TLOG(14) << GetTraceName() << ": sendFragment sts=" << std::to_string(sts);
647 }
648 
649 //=============================================
650 
651 void artdaq::TCPSocketTransfer::stats_connect_() // thread
652 {
653  std::cv_status sts;
654  while (!stats_connect_stop_)
655  {
656  std::string desc;
657  void* tag;
658  std::function<void()> function;
659  uint64_t ts_us;
660 
661  int msdly = tmo_.get_next_timeout_msdly();
662 
663  if (msdly <= 0)
664  msdly = 2000;
665 
666  std::unique_lock<std::mutex> lck(stopstatscvm_);
667  sts = stopstatscv_.wait_until(lck
668  , std::chrono::system_clock::now()
669  + std::chrono::milliseconds(msdly));
670  TLOG(15) << GetTraceName() << ": thread1 after wait_until(msdly=" << msdly << ") - sts=" << static_cast<int>(sts);
671 
672  if (sts == std::cv_status::no_timeout)
673  break;
674 
675  auto sts = tmo_.get_next_expired_timeout(desc, &tag, function, &ts_us);
676 
677  while (sts != -1 && desc != "")
678  {
679  if (function != NULL)
680  function();
681 
682  sts = tmo_.get_next_expired_timeout(desc, &tag, function, &ts_us);
683  }
684  }
685 }
686 
687 void artdaq::TCPSocketTransfer::connect_()
688 {
689  TLOG(TLVL_DEBUG) << GetTraceName() << ": Connecting sender socket";
690  int sndbuf_bytes = static_cast<int>(sndbuf_);
691  send_fd_ = TCPConnect(hostMap_[destination_rank()].hostname.c_str()
692  , calculate_port_()
693  , O_NONBLOCK
694  , sndbuf_bytes);
695  connect_state = 0;
696  blocking = 0;
697  TLOG(TLVL_DEBUG) << GetTraceName() << ": connect_ " + hostMap_[destination_rank()].hostname + ":" << calculate_port_() << " send_fd_=" << send_fd_;
698  if (send_fd_ != -1)
699  {
700  // write connect msg
701  TLOG(TLVL_DEBUG) << GetTraceName() << ": connect_: Writing connect message";
702  MessHead mh = { 0,MessHead::connect_v0,htons(source_rank()),{htonl(CONN_MAGIC)} };
703  ssize_t sts = write(send_fd_, &mh, sizeof(mh));
704  if (sts == -1)
705  {
706  TLOG(TLVL_ERROR) << GetTraceName() << ": connect_: Error writing connect message!";
707  // a write error here is completely unexpected!
708  connect_state = 0;
709  close(send_fd_);
710  send_fd_ = -1;
711  }
712  else
713  {
714  TLOG(TLVL_INFO) << GetTraceName() << ": connect_: Successfully connected";
715  // consider it all connected/established
716  connect_state = 1;
717  }
718  }
719 }
720 
721 void artdaq::TCPSocketTransfer::reconnect_()
722 {
723  TLOG(TLVL_TRACE) << GetTraceName() << ": check/reconnect";
724  if (send_fd_ == -1 && role() == TransferInterface::Role::kSend) return connect_();
725 }
726 
727 void artdaq::TCPSocketTransfer::start_listen_thread_()
728 {
729  std::unique_lock<std::mutex> start_lock(listen_thread_mutex_);
730  if (listen_thread_refcount_ == 0)
731  {
732  if (listen_thread_ && listen_thread_->joinable()) listen_thread_->join();
733  listen_thread_refcount_ = 1;
734  TLOG(TLVL_INFO) << GetTraceName() << ": Starting Listener Thread";
735  listen_thread_ = std::make_unique<boost::thread>(&TCPSocketTransfer::listen_, this);
736  }
737  else
738  {
739  listen_thread_refcount_++;
740  }
741 }
742 
743 void artdaq::TCPSocketTransfer::listen_()
744 {
745  int listen_fd = -1;
746  while (listen_thread_refcount_ > 0)
747  {
748  TLOG(TLVL_TRACE) << "listen_: Listening/accepting new connections";
749  if (listen_fd == -1)
750  {
751  TLOG(TLVL_DEBUG) << "listen_: Opening listener";
752  listen_fd = TCP_listen_fd(calculate_port_(), rcvbuf_);
753  }
754  if (listen_fd == -1)
755  {
756  TLOG(TLVL_DEBUG) << "listen_: Error creating listen_fd!";
757  break;
758  }
759 
760  int res;
761  timeval tv = { 2,0 }; // maybe increase of some global "debugging" flag set???
762  fd_set rfds;
763  FD_ZERO(&rfds);
764  FD_SET(listen_fd, &rfds);
765 
766  res = select(listen_fd + 1, &rfds, (fd_set *)0, (fd_set *)0, &tv);
767  if (res > 0)
768  {
769  int sts;
770  sockaddr_un un;
771  socklen_t arglen = sizeof(un);
772  int fd;
773  TLOG(TLVL_DEBUG) << "listen_: Calling accept";
774  fd = accept(listen_fd, (sockaddr *)&un, &arglen);
775  TLOG(TLVL_DEBUG) << GetTraceName() << ": Done with accept";
776 
777  TLOG(TLVL_DEBUG) << "listen_: Reading connect message";
778  socklen_t lenlen = sizeof(tv);
779  /*sts=*/
780  setsockopt(fd, SOL_SOCKET, SO_RCVTIMEO, &tv, lenlen); // see man 7 socket.
781  MessHead mh;
782  uint64_t mark_us = TimeUtils::gettimeofday_us();
783  sts = read(fd, &mh, sizeof(mh));
784  uint64_t delta_us = TimeUtils::gettimeofday_us() - mark_us;
785  TLOG(TLVL_DEBUG) << "listen_: Read of connect message took " << delta_us << " microseconds.";
786  if (sts != sizeof(mh))
787  {
788  TLOG(TLVL_DEBUG) << "listen_: Wrong message header length received!";
789  close(fd);
790  continue;
791  }
792 
793  // check for "magic" and valid source_id(aka rank)
794  mh.source_id = ntohs(mh.source_id); // convert here as it is reference several times
795  if (ntohl(mh.conn_magic) != CONN_MAGIC || !(mh.message_type == MessHead::connect_v0)) // Allow for future connect message versions
796  {
797  TLOG(TLVL_DEBUG) << "listen_: Wrong magic bytes in header!";
798  close(fd);
799  continue;
800  }
801 
802  // now add (new) connection
803  connected_fds_[mh.source_id].insert(fd);
804 
805  TLOG(TLVL_INFO) << "listen_: New fd is " << fd << " for source rank " << mh.source_id;
806  }
807  else
808  {
809  TLOG(16) << "listen_: No connections in timeout interval!";
810  }
811  }
812 
813  TLOG(TLVL_INFO) << "listen_: Shutting down connection listener";
814  if (listen_fd != -1) close(listen_fd);
815  for (auto& rank : connected_fds_)
816  {
817  for (auto& fd : rank.second)
818  {
819  close(fd);
820  }
821  }
822  connected_fds_.clear();
823 
824 } // do_connect_
825 
826 DEFINE_ARTDAQ_TRANSFER(artdaq::TCPSocketTransfer)
virtual int source_rank() const
Get the source rank for this TransferInterface instance.
int TCPConnect(char const *host_in, int dflt_port, long flags=0, int sndbufsiz=0)
Connect to a host on a given port.
Definition: TCPConnect.cc:124
uint32_t conn_magic
unsigned first is better for MessHead initializer: {0,0,my_node_idx_,CONN_MAGIC}
Definition: SRSockets.hh:38
std::string GetTraceName() const
Constructs a name suitable for TRACE messages.
This TransferInterface is a Receiver.
int receiveFragmentData(RawDataType *destination, size_t wordCount) override
Receive the body of a Fragment to the given destination pointer.
int TCP_listen_fd(int port, int rcvbuf)
Create a TCP listening socket on the given port and INADDR_ANY, with the given receive buffer...
TCPSocketTransfer(fhicl::ParameterSet const &ps, Role role)
TCPSocketTransfer Constructor.
int receiveFragmentHeader(detail::RawFragmentHeader &header, size_t receiveTimeout) override
Receive a Fragment Header from the transport mechanism.
This TransferInterface is a Sender.
int32_t byte_count
use CONN_MAGIC for connect_v0, data that follow for data_v0 (and 0 lenght data)
Definition: SRSockets.hh:39
Some error occurred, but no exception was thrown.
Role
Used to determine if a TransferInterface is a Sender or Receiver.
int64_t source_id
Rank of the source.
Definition: SRSockets.hh:34
MessType message_type
Message Type.
Definition: SRSockets.hh:33
The send operation completed successfully.
This interface defines the functions used to transfer data between artdaq applications.
TransferInterface implementation plugin that sends data using TCP sockets.
This header is sent by the TCPSocket_transfer to allow for more efficient writev calls.
Definition: SRSockets.hh:15
void add_periodic(const char *desc, void *tag, std::function< void()> &function, uint64_t period_us, uint64_t start_us=0)
Add a periodic timeout to the Timeout container.
Definition: Timeout.cc:67
CopyStatus
Returned from the send functions, this enumeration describes the possible return codes. If an exception occurs, it will be thrown and should be handled normally.