10 #include <sys/socket.h>
11 #include <sys/types.h>
21 #include "artdaq/DAQdata/Globals.hh"
22 #define TRACE_NAME (app_name + "_TCPSocketTransfer").c_str()
26 #include "artdaq-core/Data/Fragment.hh"
27 #include "artdaq-core/Utilities/TimeUtils.hh"
30 #include "artdaq/TransferPlugins/TCPSocketTransfer.hh"
31 #include "artdaq/TransferPlugins/detail/SRSockets.hh"
32 #include "artdaq/TransferPlugins/detail/Timeout.hh"
36 std::atomic<int> artdaq::TCPSocketTransfer::listen_thread_refcount_(0);
37 std::unique_ptr<boost::thread> artdaq::TCPSocketTransfer::listen_thread_ =
nullptr;
38 std::map<int, std::set<int>> artdaq::TCPSocketTransfer::connected_fds_ = std::map<int, std::set<int>>();
39 std::mutex artdaq::TCPSocketTransfer::listen_thread_mutex_;
40 std::mutex artdaq::TCPSocketTransfer::fd_mutex_;
46 , rcvbuf_(pset.get<size_t>(
"tcp_receive_buffer_size", 0))
47 , sndbuf_(pset.get<size_t>(
"tcp_send_buffer_size", max_fragment_size_words_ * sizeof(artdaq::RawDataType) * buffer_count_))
48 , send_retry_timeout_us_(pset.get<size_t>(
"send_retry_timeout_us", 1000000))
49 , timeoutMessageArmed_(true)
50 , receive_disconnected_wait_s_(pset.get<double>(
"receive_socket_disconnected_wait_s", 10.0))
51 , receive_err_wait_us_(pset.get<size_t>(
"receive_socket_disconnected_wait_us", 10000))
52 , receive_socket_has_been_connected_(false)
55 TLOG(TLVL_DEBUG) << GetTraceName() <<
" Constructor: pset=" << pset.to_string() <<
", role=" << (role ==
TransferInterface::Role::kReceive ?
"kReceive" :
"kSend");
56 connection_was_lost_ =
false;
61 TLOG(TLVL_DEBUG) << GetTraceName() <<
"Listening for connections";
62 start_listen_thread_();
63 TLOG(TLVL_DEBUG) << GetTraceName() <<
"Done Listening";
68 TLOG(TLVL_DEBUG) << GetTraceName() <<
"Connecting to destination";
70 TLOG(TLVL_DEBUG) << GetTraceName() <<
"Done Connecting";
72 TLOG(TLVL_DEBUG) << GetTraceName() <<
"End of Constructor";
75 artdaq::TCPSocketTransfer::~TCPSocketTransfer() noexcept
77 TLOG(TLVL_DEBUG) << GetTraceName() <<
"Shutting down TCPSocketTransfer";
86 timeval tv = {0, 100000};
87 socklen_t len =
sizeof(tv);
88 setsockopt(send_fd_, SOL_SOCKET, SO_SNDTIMEO, &tv, len);
89 write(send_fd_, &mh,
sizeof(mh));
99 if (ack_listen_thread_ && ack_listen_thread_->joinable())
101 ack_listen_thread_->join();
109 std::lock_guard<std::mutex> lk(listen_thread_mutex_);
110 listen_thread_refcount_--;
113 if (listen_thread_refcount_ <= 0 && listen_thread_ && listen_thread_->joinable())
115 listen_thread_->join();
124 TLOG(TLVL_DEBUG) << GetTraceName() <<
"End of Destructor";
129 TLOG(TLVL_DEBUG + 2) << GetTraceName() <<
"receiveFragmentHeader: BEGIN";
130 int ret_rank = RECV_TIMEOUT;
133 if (getConnectedFDCount_(source_rank()) == 0)
142 TLOG(TLVL_DEBUG + 4) << GetTraceName() <<
"receiveFragmentHeader: Receive socket not connected, returning RECV_TIMEOUT";
143 usleep(receive_err_wait_us_);
146 receive_socket_has_been_connected_ =
true;
147 last_recv_time_ = std::chrono::steady_clock::now();
149 TLOG(TLVL_DEBUG + 2) << GetTraceName() <<
"receiveFragmentHeader timeout_usec=" << timeout_usec;
154 SocketState state = SocketState::Metadata;
155 int target_bytes =
sizeof(
MessHead);
156 uint64_t start_time_us = TimeUtils::gettimeofday_us();
167 if (timeout_usec == 0)
173 timeout_ms = (timeout_usec + 999) / 1000;
177 bool noDataWarningSent =
false;
180 while (!done && getConnectedFDCount_(source_rank()) > 0)
182 if (getActiveFD_(source_rank()) == -1)
186 std::vector<pollfd> pollfds;
188 std::lock_guard<std::mutex> lk(fd_mutex_);
189 fd_count = connected_fds_[source_rank()].size();
190 pollfds.resize(fd_count);
191 auto iter = connected_fds_[source_rank()].begin();
192 for (
size_t ii = 0; ii < fd_count; ++ii)
194 pollfds[ii].events = POLLIN | POLLPRI | POLLERR;
195 pollfds[ii].fd = *iter;
200 int num_fds_ready = poll(&pollfds[0], fd_count, timeout_ms);
201 if (num_fds_ready <= 0)
203 TLOG(TLVL_DEBUG + 2) << GetTraceName() <<
"receiveFragmentHeader: No data on receive socket, returning RECV_TIMEOUT";
208 if (getLastActiveFD_(source_rank()) != -1)
210 for (
auto& pollfd : pollfds)
213 if (pollfd.fd == getLastActiveFD_(source_rank()))
220 int active_index = -1;
221 int16_t anomolous_events = 0;
222 for (
size_t ii = index; ii < index + pollfds.size(); ++ii)
224 auto pollfd_index = (ii + index) % pollfds.size();
225 setActiveFD_(source_rank(), pollfds[pollfd_index].fd);
226 if ((pollfds[pollfd_index].revents & (POLLIN | POLLPRI)) != 0)
228 active_index = pollfd_index;
231 if ((pollfds[pollfd_index].revents & (POLLHUP | POLLERR)) != 0)
233 disconnect_receive_socket_(
"Poll returned POLLHUP or POLLERR, indicating problems with the sender.");
236 else if ((pollfds[pollfd_index].revents & (POLLNVAL)) != 0)
238 TLOG(TLVL_DEBUG) << GetTraceName() <<
"receiveFragmentHeader: FD is closed, most likely because the peer went away. Removing from fd list.";
239 disconnect_receive_socket_(
"FD is closed, most likely because the peer went away.");
242 else if (pollfds[pollfd_index].revents != 0)
244 anomolous_events |= pollfds[pollfd_index].revents;
248 if (active_index == -1)
250 if (anomolous_events != 0)
252 TLOG(TLVL_DEBUG) << GetTraceName() <<
"receiveFragmentHeader: Wrong event received from a pollfd. Mask: " <<
static_cast<int>(anomolous_events);
254 setActiveFD_(source_rank(), -1);
258 if (!done && timeout_usec > 0)
261 size_t delta_us = TimeUtils::gettimeofday_us() - start_time_us;
262 if (delta_us > timeout_usec)
266 timeout_ms = ((timeout_usec - delta_us) + 999) / 1000;
269 if (loop_guard > 10) { usleep(1000); }
270 if (++loop_guard > 10010)
272 TLOG(TLVL_WARNING) << GetTraceName() <<
"receiveFragmentHeader: loop guard triggered, returning RECV_TIMEOUT";
273 usleep(receive_err_wait_us_);
274 setActiveFD_(source_rank(), -1);
278 if (state == SocketState::Metadata)
281 buff = &(mha[offset]);
282 byte_cnt =
sizeof(
MessHead) - offset;
287 buff =
reinterpret_cast<uint8_t*
>(&header) + offset;
288 byte_cnt = target_bytes - offset;
298 auto fd = getActiveFD_(source_rank());
301 TLOG(TLVL_DEBUG + 3) << GetTraceName() <<
"receiveFragmentHeader: Reading " << byte_cnt <<
" bytes from socket " << fd;
302 sts = read(fd, buff, byte_cnt);
303 TLOG(TLVL_DEBUG + 3) << GetTraceName() <<
"receiveFragmentHeader: Done with read";
308 last_recv_time_ = std::chrono::steady_clock::now();
311 TLOG(TLVL_DEBUG + 4) << GetTraceName() <<
"receiveFragmentHeader state=" <<
static_cast<int>(state) <<
" read=" << sts;
312 if (sts < 0 && errno != EAGAIN)
314 TLOG(TLVL_WARNING) << GetTraceName() <<
"receiveFragmentHeader: Error on receive, closing socket " << fd
315 <<
" (errno=" << errno <<
": " << strerror(errno) <<
")";
316 disconnect_receive_socket_(
"Error on receive");
318 else if (sts == 0 || errno == EAGAIN)
320 if (!noDataWarningSent)
322 TLOG(TLVL_WARNING) << GetTraceName() <<
"receiveFragmentHeader: No data received, is the sender still sending?!?";
323 noDataWarningSent =
true;
325 if (TimeUtils::GetElapsedTime(last_recv_time_) > receive_disconnected_wait_s_)
327 TLOG(TLVL_ERROR) << GetTraceName() <<
"receiveFragmentHeader: No data received within timeout, aborting!";
335 if (sts >= target_bytes)
337 TLOG(TLVL_DEBUG + 4) << GetTraceName() <<
"receiveFragmentHeader: Target read bytes reached. Changing state";
339 if (state == SocketState::Metadata)
341 state = SocketState::Data;
345 TLOG(TLVL_DEBUG + 4) << GetTraceName() <<
"receiveFragmentHeader: Expected header size = " << target_bytes <<
", sizeof(RawFragmentHeader) = " <<
sizeof(artdaq::detail::RawFragmentHeader);
350 disconnect_receive_socket_(
"Stop Message received.");
354 TLOG(TLVL_WARNING) << GetTraceName() <<
"receiveFragmentHeader: Message header indicates that Fragment data follows when I was expecting a Fragment header!";
355 disconnect_receive_socket_(
"Desync detected");
358 if (target_bytes == 0)
366 ret_rank = source_rank();
367 TLOG(8) << GetTraceName() <<
"receiveFragmentHeader done sts=" << sts <<
" src=" << ret_rank;
368 TLOG(TLVL_DEBUG + 4) << GetTraceName() <<
"receiveFragmentHeader: Done receiving fragment header. Moving into output.";
378 TLOG(TLVL_DEBUG + 2) << GetTraceName() <<
"receiveFragmentHeader: Returning " << ret_rank;
382 void artdaq::TCPSocketTransfer::disconnect_receive_socket_(
const std::string& msg)
384 std::lock_guard<std::mutex> lk(fd_mutex_);
385 auto fd = active_receive_fds_[source_rank()];
386 TLOG(TLVL_WARNING) << GetTraceName() <<
"disconnect_receive_socket_: " << msg <<
" Closing socket " << fd <<
" for rank " << source_rank();
388 if (connected_fds_.count(source_rank()) != 0u)
390 connected_fds_[source_rank()].erase(fd);
392 active_receive_fds_[source_rank()] = -1;
393 TLOG(TLVL_DEBUG) << GetTraceName() <<
"disconnect_receive_socket_: There are now " << connected_fds_[source_rank()].size() <<
" active senders.";
398 TLOG(19) << GetTraceName() <<
"receiveFragmentData: BEGIN";
399 int ret_rank = RECV_TIMEOUT;
400 if (getActiveFD_(source_rank()) == -1)
402 TLOG(TLVL_ERROR) << GetTraceName() <<
"receiveFragmentData: Receive socket not connected, returning RECV_TIMEOUT (Will result in \"Unexpected return code error\")";
411 SocketState state = SocketState::Metadata;
412 int target_bytes =
sizeof(
MessHead);
415 pollfd_s.events = POLLIN | POLLPRI | POLLERR;
416 pollfd_s.fd = getActiveFD_(source_rank());
420 bool noDataWarningSent =
false;
421 last_recv_time_ = std::chrono::steady_clock::now();
424 TLOG(9) << GetTraceName() <<
"receiveFragmentData: Polling fd to see if there's data";
425 int num_fds_ready = poll(&pollfd_s, 1, 1000);
426 TLOG(TLVL_TRACE) << GetTraceName() <<
"receiveFragmentData: Polled fd to see if there's data"
427 <<
", num_fds_ready = " << num_fds_ready;
428 if (num_fds_ready <= 0)
430 if (num_fds_ready == 0)
432 TLOG(TLVL_WARNING) << GetTraceName() <<
"receiveFragmentData: No data from " << source_rank() <<
" in " << TimeUtils::GetElapsedTimeMilliseconds(last_recv_time_) <<
" ms!"
433 <<
" State = " << (state == SocketState::Metadata ?
"Metadata" :
"Data") <<
", recvd/total=" << offset <<
"/" << target_bytes <<
" (delta=" << target_bytes - offset <<
")";
435 if (TimeUtils::GetElapsedTime(last_recv_time_) > receive_disconnected_wait_s_)
437 TLOG(TLVL_WARNING) << GetTraceName() <<
"receiveFragmentData: No data received within timeout (" << TimeUtils::GetElapsedTime(last_recv_time_) <<
" / " << receive_disconnected_wait_s_ <<
" ), returning RECV_TIMEOUT";
438 disconnect_receive_socket_(
"No data on this socket within timeout");
444 TLOG(TLVL_ERROR) <<
"Error in poll: errno=" << errno;
448 last_recv_time_ = std::chrono::steady_clock::now();
450 if ((pollfd_s.revents & (POLLIN | POLLPRI)) != 0)
454 else if ((pollfd_s.revents & (POLLNVAL)) != 0)
456 disconnect_receive_socket_(
"FD is closed, most likely because the peer went away.");
459 else if ((pollfd_s.revents & (POLLHUP | POLLERR)) != 0)
461 disconnect_receive_socket_(
"Poll returned POLLHUP or POLLERR, indicating problems with the sender.");
466 TLOG(TLVL_WARNING) << GetTraceName() <<
"receiveFragmentData: Wrong event received from pollfd: " << pollfd_s.revents;
467 disconnect_receive_socket_(
"Wrong event received from pollfd.");
471 if (state == SocketState::Metadata)
474 buff = &(mha[offset]);
475 byte_cnt =
sizeof(
MessHead) - offset;
480 buff =
reinterpret_cast<uint8_t*
>(destination) + offset;
484 TLOG(10) << GetTraceName() <<
"receiveFragmentData: Reading " << byte_cnt <<
" bytes from socket into " <<
static_cast<void*
>(buff);
485 sts = read(getActiveFD_(source_rank()), buff, byte_cnt);
488 TLOG(10) << GetTraceName() <<
"recvFragment state=" <<
static_cast<int>(state) <<
" read=" << sts;
490 if (sts == 0 || (sts < 0 && errno == EAGAIN))
493 if (loop_guard > 10) { usleep(1000); }
494 if (++loop_guard > 10010)
496 TLOG(TLVL_WARNING) << GetTraceName() <<
"receiveFragmentData: loop guard triggered, returning RECV_TIMEOUT";
497 setActiveFD_(source_rank(), -1);
504 last_recv_time_ = std::chrono::steady_clock::now();
509 TLOG(TLVL_WARNING) << GetTraceName() <<
"receiveFragmentData: Error on receive, closing socket"
510 <<
" (errno=" << errno <<
": " << strerror(errno) <<
")";
511 disconnect_receive_socket_(
"Error on receive");
515 if (!noDataWarningSent)
517 TLOG(TLVL_WARNING) << GetTraceName() <<
"receiveFragmentData: No data received, is the sender still sending?!?";
518 noDataWarningSent =
true;
520 if (TimeUtils::GetElapsedTime(last_recv_time_) > receive_disconnected_wait_s_)
522 TLOG(TLVL_ERROR) << GetTraceName() <<
"receiveFragmentData: No data received within timeout, aborting!";
530 if (sts >= target_bytes)
532 TLOG(9) << GetTraceName() <<
"receiveFragmentData: Target read bytes reached. Changing state";
534 if (state == SocketState::Metadata)
536 state = SocketState::Data;
543 TLOG(TLVL_WARNING) << GetTraceName() <<
"receiveFragmentData: Message header indicates that a Fragment header follows when I was expecting Fragment data!";
544 disconnect_receive_socket_(
"Desync detected");
549 ret_rank = source_rank();
550 TLOG(11) << GetTraceName() <<
"receiveFragmentData done sts=" << sts <<
" src=" << ret_rank;
551 TLOG(9) << GetTraceName() <<
"receiveFragmentData: Done receiving fragment. Moving into output.";
554 send_ack_(active_receive_fd_);
564 if (target_bytes == 0 && state == SocketState::Data)
566 ret_rank = source_rank();
567 TLOG(11) << GetTraceName() <<
"receiveFragmentData done sts=" << sts <<
" src=" << ret_rank;
568 TLOG(9) << GetTraceName() <<
"receiveFragmentData: Done receiving fragment. Moving into output.";
571 send_ack_(active_receive_fd_);
579 setLastActiveFD_(source_rank(), getActiveFD_(source_rank()));
580 setActiveFD_(source_rank(), -1);
582 TLOG(9) << GetTraceName() <<
"receiveFragmentData: Returning rank " << ret_rank;
591 return send_fd_ != -1;
593 auto count = getConnectedFDCount_(source_rank());
594 TLOG(TLVL_DEBUG) << GetTraceName() <<
"isRunning: There are " << count <<
" fds connected.";
602 std::lock_guard<std::mutex> lk(fd_mutex_);
603 if (connected_fds_.count(source_rank()) != 0u)
605 auto it = connected_fds_[source_rank()].begin();
606 char discard_buf[0x1000];
607 while (it != connected_fds_[source_rank()].end())
609 TLOG(TLVL_INFO) << GetTraceName() <<
"flush_buffers: Checking for data in socket " << *it <<
" for rank " << source_rank();
610 size_t bytes_read = 0;
611 while (
int sts = static_cast<int>(read(*it, discard_buf,
sizeof(discard_buf)) > 0))
617 TLOG(TLVL_WARNING) << GetTraceName() <<
"flush_buffers: Flushed " << bytes_read <<
" bytes from socket " << *it <<
" for rank " << source_rank();
619 TLOG(TLVL_INFO) << GetTraceName() <<
"flush_buffers: Closing socket " << *it <<
" for rank " << source_rank();
621 it = connected_fds_[source_rank()].erase(it);
623 connected_fds_.erase(source_rank());
625 active_receive_fds_[source_rank()] = -1;
626 last_active_receive_fds_[source_rank()] = -1;
633 TLOG(12) << GetTraceName() <<
"sendFragment begin send of fragment with sequenceID=" << frag.sequenceID();
634 artdaq::Fragment grab_ownership_frag = std::move(frag);
637 if (send_fd_ == -1 && connection_was_lost_)
639 TLOG(TLVL_INFO) << GetTraceName() <<
"reconnection attempt failed, returning quickly.";
647 while (static_cast<size_t>(send_ack_diff_) > buffer_count_) usleep(10000);
650 iovec iov = {
static_cast<void*
>(grab_ownership_frag.headerAddress()),
651 detail::RawFragmentHeader::num_words() *
sizeof(RawDataType)};
653 auto sts = sendData_(&iov, 1, send_retry_timeout_us_,
true);
654 auto start_time = std::chrono::steady_clock::now();
656 while (sts == CopyStatus::kTimeout && (send_timeout_usec == 0 || TimeUtils::GetElapsedTimeMicroseconds(start_time) < send_timeout_usec) && TimeUtils::GetElapsedTimeMicroseconds(start_time) < 10000000)
658 TLOG(13) << GetTraceName() <<
"sendFragment: Timeout sending fragment";
659 sts = sendData_(&iov, 1, send_retry_timeout_us_,
true);
662 if (sts != CopyStatus::kSuccess)
669 iov = {
static_cast<void*
>(grab_ownership_frag.headerAddress() + detail::RawFragmentHeader::num_words()),
670 grab_ownership_frag.sizeBytes() - detail::RawFragmentHeader::num_words() *
sizeof(RawDataType)};
671 sts = sendData_(&iov, 1, send_retry_timeout_us_);
672 start_time = std::chrono::steady_clock::now();
673 while (sts == CopyStatus::kTimeout && (send_timeout_usec == 0 || TimeUtils::GetElapsedTimeMicroseconds(start_time) < send_timeout_usec) && TimeUtils::GetElapsedTimeMicroseconds(start_time) < 10000000)
675 TLOG(13) << GetTraceName() <<
"sendFragment: Timeout sending fragment";
676 sts = sendData_(&iov, 1, send_retry_timeout_us_);
684 TLOG(12) << GetTraceName() <<
"sendFragment returning " << CopyStatusToString(sts);
690 TLOG(TLVL_DEBUG) << GetTraceName() <<
"sendData_ Converting buf to iovec";
691 iovec iov = {
const_cast<void*
>(buf), bytes};
692 return sendData_(&iov, 1, send_timeout_usec, isHeader);
700 if (timeoutMessageArmed_)
702 TLOG(TLVL_DEBUG) << GetTraceName() <<
"sendData_: Send fd is not open. Returning kTimeout";
703 timeoutMessageArmed_ =
false;
705 return CopyStatus::kTimeout;
707 timeoutMessageArmed_ =
true;
708 TLOG(14) << GetTraceName() <<
"send_timeout_usec is " << send_timeout_usec <<
", currently unused.";
711 uint32_t total_to_write_bytes = 0;
712 std::vector<iovec> iov_in(iovcnt + 1);
713 std::vector<iovec> iovv(iovcnt + 2);
715 for (ii = 0; ii < iovcnt; ++ii)
717 iov_in[ii + 1] = iov[ii];
718 total_to_write_bytes += iov[ii].iov_len;
721 MessHead mh = {0, isHeader ? MessHead::header_v0 : MessHead::data_v0, htons(source_rank()), {htonl(total_to_write_bytes)}};
722 iov_in[0].iov_base = &mh;
723 iov_in[0].iov_len =
sizeof(mh);
724 total_to_write_bytes +=
sizeof(mh);
727 ssize_t total_written_bytes = 0;
728 ssize_t per_write_max_bytes = (32 * 1024);
730 size_t in_iov_idx = 0;
731 size_t out_iov_idx = 0;
732 ssize_t this_write_bytes = 0;
739 (in_iov_idx + out_iov_idx) < iov_in.size() && this_write_bytes < per_write_max_bytes;
742 this_write_bytes += iov_in[in_iov_idx + out_iov_idx].iov_len;
743 iovv[out_iov_idx] = iov_in[in_iov_idx + out_iov_idx];
745 if (this_write_bytes > per_write_max_bytes)
747 iovv[out_iov_idx - 1].iov_len -= this_write_bytes - per_write_max_bytes;
748 this_write_bytes = per_write_max_bytes;
753 #ifndef __OPTIMIZE__ // This can be an expensive TRACE call (even if disabled) due to multiplicity of calls
754 TLOG(14) << GetTraceName() <<
"sendFragment b4 writev " << std::setw(7) << total_written_bytes <<
" total_written_bytes send_fd_=" << send_fd_ <<
" in_idx=" << in_iov_idx
755 <<
" iovcnt=" << out_iov_idx <<
" 1st.len=" << iovv[0].iov_len;
760 memset(&msg, 0,
sizeof(msghdr));
761 msg.msg_iov = &(iovv[0]);
762 msg.msg_iovlen = out_iov_idx;
763 sts = sendmsg(send_fd_, &msg, MSG_NOSIGNAL | (blocking != 0u ? 0 : MSG_DONTWAIT));
765 sts = writev(send_fd_, &(iovv[0]), out_iov_idx);
771 if (errno == EAGAIN )
773 TLOG(TLVL_DEBUG) << GetTraceName() <<
"sendFragment EWOULDBLOCK";
776 fcntl(send_fd_, F_SETFL, 0);
781 TLOG(TLVL_WARNING) << GetTraceName() <<
"sendFragment_: WRITE ERROR " << errno <<
": " << strerror(errno);
785 connection_was_lost_ =
true;
788 if (sts != this_write_bytes)
791 TLOG(TLVL_DEBUG) << GetTraceName() <<
"sendFragment writev sts(" << sts <<
")!=requested_send_bytes(" << this_write_bytes <<
")";
792 total_written_bytes += sts;
794 for (ii = 0;
static_cast<size_t>(sts) >= iovv[ii].iov_len; ++ii)
796 sts -= iovv[ii].iov_len;
799 iovv[ii].iov_len -= sts;
800 iovv[ii].iov_base =
static_cast<uint8_t*
>(iovv[ii].iov_base) + sts;
806 iovv[out_iov_idx] = iovv[ii];
809 this_write_bytes = iovv[out_iov_idx].iov_len;
814 auto additional = (
reinterpret_cast<uintptr_t
>(iov_in[in_iov_idx].iov_base) + iov_in[in_iov_idx].iov_len) - (
reinterpret_cast<uintptr_t
>(iovv[out_iov_idx].iov_base) + iovv[out_iov_idx].iov_len);
815 if (additional != 0u)
817 iovv[out_iov_idx].iov_len += additional;
818 this_write_bytes += additional;
819 if (this_write_bytes > per_write_max_bytes)
821 iovv[out_iov_idx].iov_len -= this_write_bytes - per_write_max_bytes;
822 this_write_bytes = per_write_max_bytes;
826 TLOG(TLVL_TRACE) << GetTraceName() <<
"sendFragment writev sts!=: this_write_bytes=" << this_write_bytes
827 <<
" out_iov_idx=" << out_iov_idx
828 <<
" additional=" << additional
833 #ifndef __OPTIMIZE__ // This can be an expensive TRACE call (even if disabled) due to multiplicity of calls
834 TLOG(TLVL_TRACE) << GetTraceName() <<
"sendFragment writev sts(" << sts <<
")==requested_send_bytes(" << this_write_bytes <<
")";
836 total_written_bytes += sts;
838 iovv[out_iov_idx].iov_base =
static_cast<uint8_t*
>(iovv[out_iov_idx].iov_base) + iovv[out_iov_idx].iov_len;
839 iovv[out_iov_idx].iov_len = 0;
840 in_iov_idx += out_iov_idx;
841 this_write_bytes = 0;
843 auto additional = (
reinterpret_cast<uintptr_t
>(iov_in[in_iov_idx].iov_base) + iov_in[in_iov_idx].iov_len) - (
reinterpret_cast<uintptr_t
>(iovv[out_iov_idx].iov_base) + iovv[out_iov_idx].iov_len);
844 if (additional != 0u)
846 iovv[out_iov_idx].iov_len += additional;
847 this_write_bytes += additional;
848 if (this_write_bytes > per_write_max_bytes)
850 iovv[out_iov_idx].iov_len -= this_write_bytes - per_write_max_bytes;
851 this_write_bytes = per_write_max_bytes;
853 if (out_iov_idx != 0)
855 iovv[0] = iovv[out_iov_idx];
865 }
while (total_written_bytes < total_to_write_bytes);
866 if (total_written_bytes > total_to_write_bytes)
868 TLOG(TLVL_ERROR) << GetTraceName() <<
"sendFragment program error: too many bytes transferred";
874 fcntl(send_fd_, F_SETFL, O_NONBLOCK);
876 sts = total_written_bytes -
sizeof(
MessHead);
878 TLOG(14) << GetTraceName() <<
"sendFragment sts=" << sts;
882 void artdaq::TCPSocketTransfer::connect_()
884 auto start_time = std::chrono::steady_clock::now();
887 while (send_fd_ == -1 && TimeUtils::GetElapsedTimeMicroseconds(start_time) < send_retry_timeout_us_ * 10)
889 TLOG(TLVL_DEBUG) << GetTraceName() <<
"Connecting sender socket";
890 int sndbuf_bytes =
static_cast<int>(sndbuf_);
891 if (sndbuf_ > INT_MAX)
893 sndbuf_bytes = INT_MAX;
894 TLOG(TLVL_WARNING) <<
"Requested SNDBUF " << sndbuf_ <<
" too large, setting to INT_MAX: " << INT_MAX;
896 TLOG(TLVL_DEBUG) <<
"Requested SNDBUF is " << sndbuf_bytes;
898 send_fd_ =
TCPConnect(hostMap_[destination_rank()].c_str(), portMan->GetTCPSocketTransferPort(destination_rank()), O_NONBLOCK, sndbuf_bytes);
901 if (connection_was_lost_) {
break; }
903 usleep(send_retry_timeout_us_);
908 TLOG(TLVL_DEBUG) << GetTraceName() <<
"connect_ " + hostMap_[destination_rank()] +
":" << portMan->GetTCPSocketTransferPort(destination_rank()) <<
" send_fd_=" << send_fd_;
912 TLOG(TLVL_DEBUG) << GetTraceName() <<
"connect_: Writing connect message";
913 MessHead mh = {0, MessHead::connect_v0, htons(source_rank()), {htonl(CONN_MAGIC)}};
914 ssize_t sts = write(send_fd_, &mh,
sizeof(mh));
917 TLOG(TLVL_ERROR) << GetTraceName() <<
"connect_: Error writing connect message!";
925 TLOG(TLVL_INFO) << GetTraceName() <<
"connect_: Successfully connected";
928 connection_was_lost_ =
false;
932 if (ack_listen_thread_ && ack_listen_thread_->joinable()) ack_listen_thread_->join();
933 TLOG(TLVL_INFO) << GetTraceName() <<
"Starting Ack Listener Thread";
937 ack_listen_thread_ = std::make_unique<boost::thread>(&TCPSocketTransfer::receive_acks_,
this);
939 snprintf(tname,
sizeof(tname)-1,
"%d-AckListen", my_rank);
940 tname[
sizeof(tname)-1] =
'\0';
941 auto handle = ack_listen_thread_.native_handle();
942 pthread_setname_np(handle, tname);
944 catch (
const boost::exception& e)
946 TLOG(TLVL_ERROR) <<
"Caught boost::exception starting TCP Socket Ack Listen thread: " << boost::diagnostic_information(e) <<
", errno=" << errno;
947 std::cerr <<
"Caught boost::exception starting TCP Socket Ack Listen thread: " << boost::diagnostic_information(e) <<
", errno=" << errno << std::endl;
954 void artdaq::TCPSocketTransfer::reconnect_()
958 TLOG(TLVL_TRACE) << GetTraceName() <<
"check/reconnect";
963 void artdaq::TCPSocketTransfer::start_listen_thread_()
965 std::lock_guard<std::mutex> start_lock(listen_thread_mutex_);
966 if (listen_thread_refcount_ == 0)
968 if (listen_thread_ && listen_thread_->joinable())
970 listen_thread_->join();
972 listen_thread_refcount_ = 1;
973 TLOG(TLVL_INFO) << GetTraceName() <<
"Starting Listener Thread";
977 listen_thread_ = std::make_unique<boost::thread>(&TCPSocketTransfer::listen_, portMan->GetTCPSocketTransferPort(destination_rank()), rcvbuf_);
979 snprintf(tname,
sizeof(tname)-1,
"%d-Listen", my_rank);
980 tname[
sizeof(tname)-1] =
'\0';
981 auto handle = listen_thread_->native_handle();
982 pthread_setname_np(handle, tname);
984 catch (
const boost::exception& e)
986 TLOG(TLVL_ERROR) <<
"Caught boost::exception starting TCP Socket Listen thread: " << boost::diagnostic_information(e) <<
", errno=" << errno;
987 std::cerr <<
"Caught boost::exception starting TCP Socket Listen thread: " << boost::diagnostic_information(e) <<
", errno=" << errno << std::endl;
993 listen_thread_refcount_++;
998 void artdaq::TCPSocketTransfer::receive_acks_()
1000 while (send_fd_ >= 0)
1003 pollfd_s.events = POLLIN | POLLPRI;
1004 pollfd_s.fd = send_fd_;
1006 TLOG(18) << GetTraceName() <<
"receive_acks_: Polling fd to see if there's data";
1007 int num_fds_ready = poll(&pollfd_s, 1, 1000);
1008 if (num_fds_ready <= 0)
1010 if (num_fds_ready == 0)
1012 TLOG(18) << GetTraceName() <<
"receive_acks_: No data on receive socket";
1016 TLOG(TLVL_ERROR) <<
"Error in poll: errno=" << errno;
1020 if (pollfd_s.revents & (POLLIN | POLLPRI))
1026 TLOG(TLVL_DEBUG) << GetTraceName() <<
"receive_acks_: Wrong event received from pollfd: " << pollfd_s.revents;
1031 auto sts = read(send_fd_, &mh,
sizeof(mh));
1033 if (sts !=
sizeof(mh))
1035 TLOG(TLVL_ERROR) << GetTraceName() <<
"receive_ack_: Wrong message header length received! (actual " << sts <<
" != " <<
sizeof(mh) <<
" expected)";
1043 TLOG(TLVL_ERROR) << GetTraceName() <<
"receive_ack_: Received ack for different sender! Rank=" << my_rank <<
", hdr=" << mh.
source_id;
1048 TLOG(TLVL_ERROR) << GetTraceName() <<
"receive_ack_: Wrong magic bytes in header!";
1052 TLOG(17) << GetTraceName() <<
"receive_acks_: Received ack message, diff is now " << (send_ack_diff_.load() - 1);
1057 void artdaq::TCPSocketTransfer::send_ack_(
int fd)
1059 MessHead mh = {0, MessHead::ack_v0, htons(source_rank()), {htonl(ACK_MAGIC)}};
1060 write(fd, &mh,
sizeof(mh));
1064 void artdaq::TCPSocketTransfer::listen_(
int port,
size_t rcvbuf)
1067 while (listen_thread_refcount_ > 0)
1069 TLOG(TLVL_TRACE) <<
"listen_: Listening/accepting new connections on port " << port;
1070 if (listen_fd == -1)
1072 TLOG(TLVL_DEBUG) <<
"listen_: Opening listener";
1075 if (listen_fd == -1)
1077 TLOG(TLVL_DEBUG) <<
"listen_: Error creating listen_fd!";
1082 timeval tv = {2, 0};
1085 FD_SET(listen_fd, &rfds);
1087 res = select(listen_fd + 1, &rfds, static_cast<fd_set*>(
nullptr), static_cast<fd_set*>(
nullptr), &tv);
1092 socklen_t arglen =
sizeof(un);
1094 TLOG(TLVL_DEBUG) <<
"listen_: Calling accept";
1095 fd = accept(listen_fd, reinterpret_cast<sockaddr*>(&un), &arglen);
1096 TLOG(TLVL_DEBUG) <<
"listen_: Done with accept";
1098 TLOG(TLVL_DEBUG) <<
"listen_: Reading connect message";
1099 socklen_t lenlen =
sizeof(tv);
1101 setsockopt(fd, SOL_SOCKET, SO_RCVTIMEO, &tv, lenlen);
1103 uint64_t mark_us = TimeUtils::gettimeofday_us();
1104 sts = read(fd, &mh,
sizeof(mh));
1105 uint64_t delta_us = TimeUtils::gettimeofday_us() - mark_us;
1106 TLOG(TLVL_DEBUG) <<
"listen_: Read of connect message took " << delta_us <<
" microseconds.";
1107 if (sts !=
sizeof(mh))
1109 TLOG(TLVL_DEBUG) <<
"listen_: Wrong message header length received!";
1118 TLOG(TLVL_DEBUG) <<
"listen_: Wrong magic bytes in header!";
1124 std::lock_guard<std::mutex> lk(fd_mutex_);
1125 connected_fds_[mh.
source_id].insert(fd);
1127 TLOG(TLVL_INFO) <<
"listen_: New fd is " << fd <<
" for source rank " << mh.
source_id;
1131 TLOG(16) <<
"listen_: No connections in timeout interval!";
1135 TLOG(TLVL_INFO) <<
"listen_: Shutting down connection listener";
1136 if (listen_fd != -1)
1140 std::lock_guard<std::mutex> lk(fd_mutex_);
1141 auto it = connected_fds_.begin();
1142 while (it != connected_fds_.end())
1144 auto& fd_set = it->second;
1145 auto rank_it = fd_set.begin();
1146 while (rank_it != fd_set.end())
1149 rank_it = fd_set.erase(rank_it);
1151 it = connected_fds_.erase(it);
1156 size_t artdaq::TCPSocketTransfer::getConnectedFDCount_(
int source_rank)
1158 std::lock_guard<std::mutex> lk(fd_mutex_);
1159 #ifndef __OPTIMIZE__
1160 TLOG(15) << GetTraceName() <<
"getConnectedFDCount_: count is " << (connected_fds_.count(source_rank) != 0u ? connected_fds_[source_rank].size() : 0);
1162 return connected_fds_.count(source_rank) != 0u ? connected_fds_[source_rank].size() : 0;
1165 int artdaq::TCPSocketTransfer::getActiveFD_(
int source_rank)
1167 std::lock_guard<std::mutex> lk(fd_mutex_);
1168 #ifndef __OPTIMIZE__
1169 TLOG(15) << GetTraceName() <<
"getActiveFD_: fd is " << (active_receive_fds_.count(source_rank) != 0u ? active_receive_fds_[source_rank] : -1);
1171 return active_receive_fds_.count(source_rank) != 0u ? active_receive_fds_[source_rank] : -1;
1173 void artdaq::TCPSocketTransfer::setActiveFD_(
int source_rank,
int fd)
1175 std::lock_guard<std::mutex> lk(fd_mutex_);
1176 #ifndef __OPTIMIZE__
1177 TLOG(15) << GetTraceName() <<
"setActiveFD_: setting active fd for rank " << source_rank <<
" to " << fd;
1179 active_receive_fds_[source_rank] = fd;
1181 int artdaq::TCPSocketTransfer::getLastActiveFD_(
int source_rank)
1183 std::lock_guard<std::mutex> lk(fd_mutex_);
1184 #ifndef __OPTIMIZE__
1185 TLOG(15) << GetTraceName() <<
"getLastActiveFD_: fd is " << (last_active_receive_fds_.count(source_rank) != 0u ? last_active_receive_fds_[source_rank] : -1);
1187 return last_active_receive_fds_.count(source_rank) != 0u ? last_active_receive_fds_[source_rank] : -1;
1189 void artdaq::TCPSocketTransfer::setLastActiveFD_(
int source_rank,
int fd)
1191 std::lock_guard<std::mutex> lk(fd_mutex_);
1192 #ifndef __OPTIMIZE__
1193 TLOG(15) << GetTraceName() <<
"setLastActiveFD_: setting last active fd for rank " << source_rank <<
" to " << fd;
1195 last_active_receive_fds_[source_rank] = fd;
bool isRunning() override
Determine whether the TransferInterface plugin is able to send/receive data.
virtual int source_rank() const
Get the source rank for this TransferInterface instance.
uint32_t conn_magic
unsigned first is better for MessHead initializer: {0,0,my_node_idx_,CONN_MAGIC}
int receiveFragmentHeader(detail::RawFragmentHeader &header, size_t timeout_usec) override
Receive a Fragment Header from the transport mechanism.
int TCPConnect(char const *host_in, int dflt_port, int64_t flags=0, int sndbufsiz=0)
Connect to a host on a given port.
This TransferInterface is a Receiver.
int receiveFragmentData(RawDataType *destination, size_t wordCount) override
Receive the body of a Fragment to the given destination pointer.
int TCP_listen_fd(int port, int rcvbuf)
Create a TCP listening socket on the given port and INADDR_ANY, with the given receive buffer...
TCPSocketTransfer(fhicl::ParameterSet const &ps, Role role)
TCPSocketTransfer Constructor.
This TransferInterface is a Sender.
void flush_buffers() override
Flush any in-flight data. This should be used by the receiver after the receive loop has ended...
int32_t byte_count
use CONN_MAGIC for connect_v0, data that follow for data_v0 (and 0 lenght data)
Some error occurred, but no exception was thrown.
Role
Used to determine if a TransferInterface is a Sender or Receiver.
int64_t source_id
Rank of the source.
MessType message_type
Message Type.
The send operation completed successfully.
This interface defines the functions used to transfer data between artdaq applications.
TransferInterface implementation plugin that sends data using TCP sockets.
This header is sent by the TCPSocket_transfer to allow for more efficient writev calls.
hostMap_t MakeHostMap(fhicl::ParameterSet const &pset, hostMap_t map=hostMap_t())
Make a hostMap_t from a HostMap::Config ParameterSet
CopyStatus
Returned from the send functions, this enumeration describes the possible return codes. If an exception occurs, it will be thrown and should be handled normally.