10 #include <sys/socket.h>
12 #include <arpa/inet.h>
21 #define TRACE_NAME "TCPSocketTransfer"
22 #include "artdaq/DAQdata/Globals.hh"
25 #include "artdaq/TransferPlugins/TCPSocketTransfer.hh"
28 #include "artdaq/TransferPlugins/detail/Timeout.hh"
29 #include "artdaq/TransferPlugins/detail/SRSockets.hh"
30 #include "artdaq-core/Data/Fragment.hh"
38 , state_(SocketState::Metadata)
39 , frag(max_fragment_size_words_)
40 , buffer(frag.headerBeginBytes())
43 , rcvbuf_(pset.get<size_t>(
"tcp_receive_buffer_size", 0))
44 , sndbuf_(max_fragment_size_words_ * sizeof(artdaq::RawDataType) * buffer_count_)
45 , stats_connect_stop_(false)
47 , timeoutMessageArmed_(true)
50 auto hosts = pset.get<std::vector<fhicl::ParameterSet>>(
"host_map");
51 for (
auto& ps : hosts)
55 info.hostname = ps.get<std::string>(
"host",
"localhost");
56 info.portOffset = ps.get<
int>(
"portOffset", 5500);
58 hostMap_[rank] = info;
61 std::function<void()>
function = std::bind(&TCPSocketTransfer::reconnect_,
this);
67 TLOG_DEBUG(
uniqueLabel()) <<
"Listening for connections" << TLOG_ENDL;
69 TLOG_DEBUG(
uniqueLabel()) <<
"Done Listening" << TLOG_ENDL;
73 TLOG_DEBUG(
uniqueLabel()) <<
"Connecting to destination" << TLOG_ENDL;
75 TLOG_DEBUG(
uniqueLabel()) <<
"Done Connecting" << TLOG_ENDL;
77 TLOG_DEBUG(
uniqueLabel()) <<
"End of TCPSocketTransfer Constructor" << TLOG_ENDL;
80 artdaq::TCPSocketTransfer::~TCPSocketTransfer()
82 TLOG_DEBUG(uniqueLabel()) <<
"Shutting down TCPSocketTransfer" << TLOG_ENDL;
83 stats_connect_stop_ =
true;
84 stopstatscv_.notify_all();
85 stats_connect_thread_.join();
90 MessHead mh = {0,MessHead::stop_v0,htons(source_rank()),0};
94 timeval tv = {0,100000};
95 socklen_t len =
sizeof(tv);
96 setsockopt(fd_, SOL_SOCKET, SO_SNDTIMEO, &tv, len);
97 write(fd_, &mh,
sizeof(mh));
101 TLOG_DEBUG(uniqueLabel()) <<
"End of TCPSocketTransfer Destructor" << TLOG_ENDL;
102 TRACE(4,
"TCPSocketTransfer dtor");
110 TRACE(7,
"TCPSocketTransfer::sendFragment begin");
111 artdaq::Fragment grab_ownership_frag = std::move(frag);
112 iovec iov = {(
void*)grab_ownership_frag.headerBeginBytes(), grab_ownership_frag.sizeBytes()};
113 auto sts = sendFragment_(&iov, 1, send_timeout_usec);
114 while (sts != CopyStatus::kSuccess)
116 TRACE(7,
"TCPSocketTransfer::sendFragment: Timeout or Error sending fragment");
117 sts = sendFragment_(&iov, 1, send_timeout_usec);
123 TRACE_(7,
"TCPSocketTransfer::sendFragment returning " + result);
129 TLOG_DEBUG(uniqueLabel()) <<
"TCPSocketTransfer::sendFragment_ Converting buf to iovec" << TLOG_ENDL;
130 iovec iov = {(
void*)buf, bytes};
131 return sendFragment_(&iov, 1, send_timeout_usec);
139 if (timeoutMessageArmed_)
141 TLOG_DEBUG(uniqueLabel()) <<
"TCPSocketTransfer::sendFragment_: Send fd is not open. Returning kTimeout" << TLOG_ENDL;
142 timeoutMessageArmed_ =
false;
146 timeoutMessageArmed_ =
true;
147 TRACE(12,
"send_timeout_usec is %zu, currently unused.", send_timeout_usec);
150 uint32_t total_to_write_bytes = 0;
151 std::vector<iovec> iov_in(iovcnt + 1);
152 std::vector<iovec> iovv(iovcnt + 2);
154 for (ii = 0; ii < iovcnt; ++ii)
156 iov_in[ii + 1] = iov[ii];
157 total_to_write_bytes += iov[ii].iov_len;
160 MessHead mh = {0,MessHead::data_v0,htons(source_rank()),htonl(total_to_write_bytes)};
161 iov_in[0].iov_base = &mh;
162 iov_in[0].iov_len =
sizeof(mh);
163 total_to_write_bytes +=
sizeof(mh);
166 ssize_t total_written_bytes = 0;
167 ssize_t per_write_max_bytes = (32 * 1024);
169 size_t in_iov_idx = 0;
170 size_t out_iov_idx = 0;
171 ssize_t this_write_bytes = 0;
178 (in_iov_idx + out_iov_idx) < iov_in.size() && this_write_bytes < per_write_max_bytes;
181 this_write_bytes += iov_in[in_iov_idx + out_iov_idx].iov_len;
182 iovv[out_iov_idx] = iov_in[in_iov_idx + out_iov_idx];
184 if (this_write_bytes > per_write_max_bytes)
186 iovv[out_iov_idx - 1].iov_len -= this_write_bytes - per_write_max_bytes;
187 this_write_bytes = per_write_max_bytes;
192 TRACE(7,
"sendFragment b4 writev %7zu total_written_bytes fd=%d in_idx=%zu iovcnt=%zu 1st.len=%zu"
193 , total_written_bytes, fd_, in_iov_idx, out_iov_idx, iovv[0].iov_len);
195 sts = writev(fd_, &(iovv[0]), out_iov_idx);
200 if (errno == EAGAIN )
202 TRACE(2,
"sendFragment EWOULDBLOCK");
203 fcntl(fd_, F_SETFL, 0);
208 TLOG_DEBUG(uniqueLabel()) <<
"TCPSocketTransfer::sendFragment_: WRITE ERROR!" << TLOG_ENDL;
214 else if (sts != this_write_bytes)
217 TRACE(4,
"sendFragment writev sts(%ld)!=requested_send_bytes(%ld)"
218 , sts, this_write_bytes);
219 total_written_bytes += sts;
221 for (ii = 0; (size_t)sts >= iovv[ii].iov_len; ++ii)
222 sts -= iovv[ii].iov_len;
224 iovv[ii].iov_len -= sts;
225 iovv[ii].iov_base = (uint8_t*)(iovv[ii].iov_base) + sts;
230 iovv[out_iov_idx] = iovv[ii];
232 this_write_bytes = iovv[out_iov_idx].iov_len;
237 unsigned long additional = ((
unsigned long)iov_in[in_iov_idx].iov_base + iov_in[in_iov_idx].iov_len)
238 - ((
unsigned long)iovv[out_iov_idx].iov_base + iovv[out_iov_idx].iov_len);
241 iovv[out_iov_idx].iov_len += additional;
242 this_write_bytes += additional;
243 if (this_write_bytes > per_write_max_bytes)
245 iovv[out_iov_idx].iov_len -= this_write_bytes - per_write_max_bytes;
246 this_write_bytes = per_write_max_bytes;
250 TRACE(4,
"sendFragment writev sts!=: this_write_bytes=%zd out_iov_idx=%zu additional=%lu ii=%d"
251 , this_write_bytes, out_iov_idx, additional, ii);
255 TRACE(4,
"sendFragment writev sts(%ld)==requested_send_bytes(%ld)"
256 , sts, this_write_bytes);
257 total_written_bytes += sts;
259 iovv[out_iov_idx].iov_base = (uint8_t*)(iovv[out_iov_idx].iov_base) + iovv[out_iov_idx].iov_len;
260 iovv[out_iov_idx].iov_len = 0;
261 in_iov_idx += out_iov_idx;
262 this_write_bytes = 0;
264 unsigned long additional = ((
unsigned long)iov_in[in_iov_idx].iov_base + iov_in[in_iov_idx].iov_len)
265 - ((
unsigned long)iovv[out_iov_idx].iov_base + iovv[out_iov_idx].iov_len);
268 iovv[out_iov_idx].iov_len += additional;
269 this_write_bytes += additional;
270 if (this_write_bytes > per_write_max_bytes)
272 iovv[out_iov_idx].iov_len -= this_write_bytes - per_write_max_bytes;
273 this_write_bytes = per_write_max_bytes;
275 if (out_iov_idx != 0)
276 iovv[0] = iovv[out_iov_idx];
286 while (total_written_bytes < total_to_write_bytes);
287 if (total_written_bytes > total_to_write_bytes)
288 TRACE(0,
"sendFragment program error: too many bytes transferred");
293 fcntl(fd_, F_SETFL, 0);
295 sts = total_written_bytes -
sizeof(
MessHead);
297 TRACE(10,
"sendFragment sts=%ld", sts);
303 void artdaq::TCPSocketTransfer::connect_()
305 TLOG_DEBUG(uniqueLabel()) <<
"Connecting sender socket" << TLOG_ENDL;
306 int sndbuf_bytes =
static_cast<int>(sndbuf_);
307 fd_ =
TCPConnect(hostMap_[destination_rank()].hostname.c_str()
313 TLOG_DEBUG(uniqueLabel()) <<
"TCPSocketTransfer::connect_ " + hostMap_[destination_rank()].hostname +
":" << calculate_port_() <<
" fd_=" << fd_ << TLOG_ENDL;
317 TLOG_DEBUG(uniqueLabel()) <<
"TCPSocketTransfer::connect_: Writing connect message" << TLOG_ENDL;
318 MessHead mh = {0,MessHead::connect_v0,htons(source_rank()),htonl(CONN_MAGIC)};
319 ssize_t sts = write(fd_, &mh,
sizeof(mh));
322 TLOG_ERROR(uniqueLabel()) <<
"TCPSocketTransfer::connect_: Error writing connect message!" << TLOG_ENDL;
330 TLOG_INFO(uniqueLabel()) <<
"TCPSocketTransfer::connect_: Successfully connected" << TLOG_ENDL;
337 void artdaq::TCPSocketTransfer::reconnect_()
339 TRACE(5,
"check/reconnect");
345 static uint64_t gettimeofday_us()
348 gettimeofday(&tv, NULL);
349 return (uint64_t)tv.tv_sec * 1000000 + tv.tv_usec;
352 void artdaq::TCPSocketTransfer::listen_()
354 TLOG_DEBUG(uniqueLabel()) <<
"TCPSocketTransfer::listen_: Listening/accepting new connections" << TLOG_ENDL;
355 if (listen_fd_ == -1)
357 TLOG_DEBUG(uniqueLabel()) <<
"TCPSocketTransfer::listen_: Opening listener" << TLOG_ENDL;
360 if (listen_fd_ == -1)
362 TLOG_DEBUG(uniqueLabel()) <<
"TCPSocketTransfer::listen_: Error creating listen_fd_!" << TLOG_ENDL;
370 FD_SET(listen_fd_, &rfds);
372 res = select(listen_fd_ + 1, &rfds, (fd_set *)0, (fd_set *)0, &tv);
377 socklen_t arglen =
sizeof(un);
379 TLOG_DEBUG(uniqueLabel()) <<
"Calling accept" << TLOG_ENDL;
380 fd = accept(listen_fd_, (sockaddr *)&un, &arglen);
381 TLOG_DEBUG(uniqueLabel()) <<
"Done with accept" << TLOG_ENDL;
383 TLOG_DEBUG(uniqueLabel()) <<
"TCPSocketTransfer::listen_: Reading connect message" << TLOG_ENDL;
384 socklen_t lenlen =
sizeof(tv);
386 setsockopt(fd, SOL_SOCKET, SO_RCVTIMEO, &tv, lenlen);
388 uint64_t mark_us = gettimeofday_us();
389 sts = read(fd, &mh,
sizeof(mh));
390 uint64_t delta_us = gettimeofday_us() - mark_us;
391 TLOG_DEBUG(uniqueLabel()) <<
"TCPSocketTransfer::listen_: Read of connect message took " << delta_us <<
" microseconds." << TLOG_ENDL;
392 TRACE(10,
"do_connect read of connect msg (after accept) took %lu microseconds", delta_us);
393 if (sts !=
sizeof(mh))
395 TLOG_DEBUG(uniqueLabel()) <<
"TCPSocketTransfer::listen_: Wrong message header length received!" << TLOG_ENDL;
396 TRACE(0,
"do_connect_ problem with connect msg sts(%d)!=sizeof(mh)(%ld)", sts,
sizeof(mh));
405 TLOG_DEBUG(uniqueLabel()) <<
"TCPSocketTransfer::listen_: Wrong magic bytes in header!" << TLOG_ENDL;
417 TLOG_INFO(uniqueLabel()) <<
"TCPSocketTransfer::listen_: New fd is " << fd_ << TLOG_ENDL;
419 TRACE(3,
"do_connect_ connection from sender_rank=%zu", mh.
source_id);
423 TRACE(10,
"TCPSocketTransfer::do_connect_: No connections in timeout interval!");
433 TRACE(7,
"TCPSocketTransfer::receiveFragment: BEGIN");
434 int ret_rank = RECV_TIMEOUT;
437 TLOG_DEBUG(uniqueLabel()) <<
"TCPSocketTransfer::receiveFragment: Receive socket not connected, returning RECV_TIMEOUT" << TLOG_ENDL;
441 TRACE(7,
"TCPSocketTransfer::recvFragment timeout_usec=%ld", timeout_usec);
446 uint64_t start_time_us = gettimeofday_us();
449 pollfd_s.events = POLLIN | POLLERR;
453 if (timeout_usec == 0)
456 timeout_ms = (timeout_usec + 999) / 1000;
462 int num_fds_ready = poll(&pollfd_s, 1, timeout_ms);
463 if (num_fds_ready <= 0)
465 if (num_fds_ready == 0 && timeout_ms > 0)
467 TRACE(7,
"TCPSocketTransfer::receiveFragment: No data on receive socket, returning RECV_TIMEOUT");
473 if (!(pollfd_s.revents & (POLLIN | POLLERR)))
475 TLOG_DEBUG(uniqueLabel()) <<
"TCPSocketTransfer::receiveFragment: Wrong event received from pollfd" << TLOG_ENDL;
479 if (state_ == SocketState::Metadata)
482 buff = &(mha[offset]);
483 byte_cnt =
sizeof(
MessHead) - offset;
488 buff = buffer + offset;
493 sts = read(fd_, buff, byte_cnt);
496 TRACE(9,
"recvFragment state=%d read=%d (errno=%d)", static_cast<int>(state_), sts, errno);
499 TLOG_DEBUG(uniqueLabel()) <<
"TCPSocketTransfer::receiveFragment: Error on receive, closing socket" << TLOG_ENDL;
506 if (sts == target_bytes)
508 TRACE(7,
"TCPSocketTransfer::receiveFragment: Target read bytes reached. Changing state");
510 if (state_ == SocketState::Metadata)
512 state_ = SocketState::Data;
519 state_ = SocketState::Metadata;
521 ret_rank = source_rank();
522 TRACE(9,
"recvFragment done sts=%d src=%d", sts, ret_rank);
523 TRACE(7,
"TCPSocketTransfer::receiveFragment: Done receiving fragment. Moving into output.");
525 if (frag.type() == artdaq::Fragment::EndOfDataFragmentType)
527 stats_connect_stop_ =
true;
528 stopstatscv_.notify_all();
531 frag.reserve(max_fragment_size_words_);
532 buffer = frag.headerBeginBytes();
539 if (!done && timeout_usec > 0)
542 size_t delta_us = gettimeofday_us() - start_time_us;
543 if (delta_us > timeout_usec)
545 timeout_ms = ((timeout_usec - delta_us) + 999) / 1000;
549 TRACE(7,
"TCPSocketTransfer::receiveFragment: Returning %d", ret_rank);
553 void artdaq::TCPSocketTransfer::stats_connect_()
556 while (!stats_connect_stop_)
560 std::function<void()>
function;
563 int msdly = tmo_.get_next_timeout_msdly();
568 std::unique_lock<std::mutex> lck(stopstatscvm_);
569 sts = stopstatscv_.wait_until(lck
570 , std::chrono::system_clock::now()
571 + std::chrono::milliseconds(msdly));
572 TRACE(5,
"thread1 after wait_until(msdly=%d) - sts=%d", msdly, static_cast<int>(sts));
574 if (sts == std::cv_status::no_timeout)
577 auto sts = tmo_.get_next_expired_timeout(desc, &tag,
function, &ts_us);
579 while (sts != -1 && desc !=
"")
581 if (
function != NULL)
584 sts = tmo_.get_next_expired_timeout(desc, &tag,
function, &ts_us);
int TCPConnect(char const *host_in, int dflt_port, long flags=0, int sndbufsiz=0)
Connect to a host on a given port.
uint32_t conn_magic
unsigned first is better for MessHead initializer: {0,0,my_node_idx_,CONN_MAGIC}
The send operation timed out.
static const int RECV_TIMEOUT
Value to be returned upon receive timeout. Because receivers otherwise return rank, this is also the limit on the number of ranks that artdaq currently supports.
This TransferInterface is a Receiver.
int TCP_listen_fd(int port, int rcvbuf)
Create a TCP listening socket on the given port and INADDR_ANY, with the given receive buffer...
TCPSocketTransfer(fhicl::ParameterSet const &ps, Role role)
TCPSocketTransfer Constructor.
This TransferInterface is a Sender.
int receiveFragment(Fragment &frag, size_t timeout_usec=0) override
Receive a Fragment using TCP.
int32_t byte_count
use CONN_MAGIC for connect_v0, data that follow for data_v0 (and 0 lenght data)
Some error occurred, but no exception was thrown.
Role
Used to determine if a TransferInterface is a Sender or Receiver.
int64_t source_id
Rank of the source.
The send operation completed successfully.
std::string uniqueLabel() const
Get the unique label of this TransferInterface instance.
This interface defines the functions used to transfer data between artdaq applications.
TransferInterface implementation plugin that sends data using TCP sockets.
This header is sent by the TCPSocket_transfer to allow for more efficient writev calls.
void add_periodic(const char *desc, void *tag, std::function< void()> &function, uint64_t period_us, uint64_t start_us=0)
Add a periodic timeout to the Timeout container.
CopyStatus
Returned from the send functions, this enumeration describes the possible return codes. If an exception occurs, it will be thrown and should be handled normally.