10 #include <sys/socket.h>
11 #include <sys/types.h>
21 #include "artdaq/DAQdata/Globals.hh"
22 #define TRACE_NAME (app_name + "_TCPSocketTransfer").c_str()
26 #include "artdaq-core/Data/Fragment.hh"
27 #include "artdaq-core/Utilities/TimeUtils.hh"
30 #include "artdaq/TransferPlugins/TCPSocketTransfer.hh"
31 #include "artdaq/TransferPlugins/detail/SRSockets.hh"
32 #include "artdaq/TransferPlugins/detail/Timeout.hh"
36 std::atomic<int> artdaq::TCPSocketTransfer::listen_thread_refcount_(0);
37 std::unique_ptr<boost::thread> artdaq::TCPSocketTransfer::listen_thread_ =
nullptr;
38 std::map<int, std::set<int>> artdaq::TCPSocketTransfer::connected_fds_ = std::map<int, std::set<int>>();
39 std::mutex artdaq::TCPSocketTransfer::listen_thread_mutex_;
40 std::mutex artdaq::TCPSocketTransfer::fd_mutex_;
46 , rcvbuf_(pset.get<size_t>(
"tcp_receive_buffer_size", 0))
47 , sndbuf_(pset.get<size_t>(
"tcp_send_buffer_size", max_fragment_size_words_ * sizeof(artdaq::RawDataType) * buffer_count_))
48 , send_retry_timeout_us_(pset.get<size_t>(
"send_retry_timeout_us", 1000000))
49 , timeoutMessageArmed_(true)
50 , receive_disconnected_wait_s_(pset.get<double>(
"receive_socket_disconnected_wait_s", 10.0))
51 , receive_err_wait_us_(pset.get<size_t>(
"receive_socket_disconnected_wait_us", 10000))
52 , receive_socket_has_been_connected_(false)
55 TLOG(TLVL_DEBUG) << GetTraceName() <<
" Constructor: pset=" << pset.to_string() <<
", role=" << (role ==
TransferInterface::Role::kReceive ?
"kReceive" :
"kSend");
56 connection_was_lost_ =
false;
61 TLOG(TLVL_DEBUG) << GetTraceName() <<
"Listening for connections";
62 start_listen_thread_();
63 TLOG(TLVL_DEBUG) << GetTraceName() <<
"Done Listening";
68 TLOG(TLVL_DEBUG) << GetTraceName() <<
"Connecting to destination";
70 TLOG(TLVL_DEBUG) << GetTraceName() <<
"Done Connecting";
72 TLOG(TLVL_DEBUG) << GetTraceName() <<
"End of Constructor";
75 artdaq::TCPSocketTransfer::~TCPSocketTransfer() noexcept
77 TLOG(TLVL_DEBUG) << GetTraceName() <<
"Shutting down TCPSocketTransfer";
86 timeval tv = {0, 100000};
87 socklen_t len =
sizeof(tv);
88 setsockopt(send_fd_, SOL_SOCKET, SO_SNDTIMEO, &tv, len);
89 write(send_fd_, &mh,
sizeof(mh));
99 if (ack_listen_thread_ && ack_listen_thread_->joinable())
101 ack_listen_thread_->join();
109 std::lock_guard<std::mutex> lk(listen_thread_mutex_);
110 listen_thread_refcount_--;
113 if (listen_thread_refcount_ <= 0 && listen_thread_ && listen_thread_->joinable())
115 listen_thread_->join();
124 TLOG(TLVL_DEBUG) << GetTraceName() <<
"End of Destructor";
129 TLOG(5) << GetTraceName() <<
"receiveFragmentHeader: BEGIN";
130 int ret_rank = RECV_TIMEOUT;
133 if (getConnectedFDCount_(source_rank()) == 0)
142 TLOG(7) << GetTraceName() <<
"receiveFragmentHeader: Receive socket not connected, returning RECV_TIMEOUT";
143 usleep(receive_err_wait_us_);
146 receive_socket_has_been_connected_ =
true;
147 last_recv_time_ = std::chrono::steady_clock::now();
149 TLOG(5) << GetTraceName() <<
"receiveFragmentHeader timeout_usec=" << timeout_usec;
154 SocketState state = SocketState::Metadata;
155 int target_bytes =
sizeof(
MessHead);
156 uint64_t start_time_us = TimeUtils::gettimeofday_us();
167 if (timeout_usec == 0)
173 timeout_ms = (timeout_usec + 999) / 1000;
177 bool noDataWarningSent =
false;
180 while (!done && getConnectedFDCount_(source_rank()) > 0)
182 if (getActiveFD_(source_rank()) == -1)
186 std::vector<pollfd> pollfds;
188 std::lock_guard<std::mutex> lk(fd_mutex_);
189 fd_count = connected_fds_[source_rank()].size();
190 pollfds.resize(fd_count);
191 auto iter = connected_fds_[source_rank()].begin();
192 for (
size_t ii = 0; ii < fd_count; ++ii)
194 pollfds[ii].events = POLLIN | POLLPRI | POLLERR;
195 pollfds[ii].fd = *iter;
200 int num_fds_ready = poll(&pollfds[0], fd_count, timeout_ms);
201 if (num_fds_ready <= 0)
203 TLOG(5) << GetTraceName() <<
"receiveFragmentHeader: No data on receive socket, returning RECV_TIMEOUT";
208 if (getLastActiveFD_(source_rank()) != -1)
210 for (
auto& pollfd : pollfds)
213 if (pollfd.fd == getLastActiveFD_(source_rank()))
220 int active_index = -1;
221 int16_t anomolous_events = 0;
222 for (
size_t ii = index; ii < index + pollfds.size(); ++ii)
224 auto pollfd_index = (ii + index) % pollfds.size();
225 setActiveFD_(source_rank(), pollfds[pollfd_index].fd);
226 if ((pollfds[pollfd_index].revents & (POLLIN | POLLPRI)) != 0)
228 active_index = pollfd_index;
231 if ((pollfds[pollfd_index].revents & (POLLHUP | POLLERR)) != 0)
233 disconnect_receive_socket_(
"Poll returned POLLHUP or POLLERR, indicating problems with the sender.");
236 else if ((pollfds[pollfd_index].revents & (POLLNVAL)) != 0)
238 TLOG(TLVL_DEBUG) << GetTraceName() <<
"receiveFragmentHeader: FD is closed, most likely because the peer went away. Removing from fd list.";
239 disconnect_receive_socket_(
"FD is closed, most likely because the peer went away.");
242 else if (pollfds[pollfd_index].revents != 0)
244 anomolous_events |= pollfds[pollfd_index].revents;
248 if (active_index == -1)
250 if (anomolous_events != 0)
252 TLOG(TLVL_DEBUG) << GetTraceName() <<
"receiveFragmentHeader: Wrong event received from a pollfd. Mask: " <<
static_cast<int>(anomolous_events);
254 setActiveFD_(source_rank(), -1);
258 if (!done && timeout_usec > 0)
261 size_t delta_us = TimeUtils::gettimeofday_us() - start_time_us;
262 if (delta_us > timeout_usec)
266 timeout_ms = ((timeout_usec - delta_us) + 999) / 1000;
269 if (loop_guard > 10) { usleep(1000); }
270 if (++loop_guard > 10010)
272 TLOG(TLVL_WARNING) << GetTraceName() <<
"receiveFragmentHeader: loop guard triggered, returning RECV_TIMEOUT";
273 usleep(receive_err_wait_us_);
274 setActiveFD_(source_rank(), -1);
278 if (state == SocketState::Metadata)
281 buff = &(mha[offset]);
282 byte_cnt =
sizeof(
MessHead) - offset;
287 buff =
reinterpret_cast<uint8_t*
>(&header) + offset;
288 byte_cnt = target_bytes - offset;
298 auto fd = getActiveFD_(source_rank());
301 TLOG(6) << GetTraceName() <<
"receiveFragmentHeader: Reading " << byte_cnt <<
" bytes from socket " << fd;
302 sts = read(fd, buff, byte_cnt);
303 TLOG(6) << GetTraceName() <<
"receiveFragmentHeader: Done with read";
308 last_recv_time_ = std::chrono::steady_clock::now();
311 TLOG(7) << GetTraceName() <<
"receiveFragmentHeader state=" <<
static_cast<int>(state) <<
" read=" << sts;
312 if (sts < 0 && errno != EAGAIN)
314 TLOG(TLVL_WARNING) << GetTraceName() <<
"receiveFragmentHeader: Error on receive, closing socket " << fd
315 <<
" (errno=" << errno <<
": " << strerror(errno) <<
")";
316 disconnect_receive_socket_(
"Error on receive");
318 else if (sts == 0 || errno == EAGAIN)
320 if (!noDataWarningSent)
322 TLOG(TLVL_WARNING) << GetTraceName() <<
"receiveFragmentHeader: No data received, is the sender still sending?!?";
323 noDataWarningSent =
true;
325 if (TimeUtils::GetElapsedTime(last_recv_time_) > receive_disconnected_wait_s_)
327 TLOG(TLVL_ERROR) << GetTraceName() <<
"receiveFragmentHeader: No data received within timeout, aborting!";
335 if (sts >= target_bytes)
337 TLOG(7) << GetTraceName() <<
"receiveFragmentHeader: Target read bytes reached. Changing state";
339 if (state == SocketState::Metadata)
341 state = SocketState::Data;
345 TLOG(7) << GetTraceName() <<
"receiveFragmentHeader: Expected header size = " << target_bytes <<
", sizeof(RawFragmentHeader) = " <<
sizeof(artdaq::detail::RawFragmentHeader);
350 disconnect_receive_socket_(
"Stop Message received.");
354 TLOG(TLVL_WARNING) << GetTraceName() <<
"receiveFragmentHeader: Message header indicates that Fragment data follows when I was expecting a Fragment header!";
355 disconnect_receive_socket_(
"Desync detected");
358 if (target_bytes == 0)
366 ret_rank = source_rank();
367 TLOG(8) << GetTraceName() <<
"receiveFragmentHeader done sts=" << sts <<
" src=" << ret_rank;
368 TLOG(7) << GetTraceName() <<
"receiveFragmentHeader: Done receiving fragment header. Moving into output.";
378 TLOG(5) << GetTraceName() <<
"receiveFragmentHeader: Returning " << ret_rank;
382 void artdaq::TCPSocketTransfer::disconnect_receive_socket_(
const std::string& msg)
384 std::lock_guard<std::mutex> lk(fd_mutex_);
385 auto fd = active_receive_fds_[source_rank()];
386 TLOG(TLVL_WARNING) << GetTraceName() <<
"disconnect_receive_socket_: " << msg <<
" Closing socket " << fd <<
" for rank " << source_rank();
388 if (connected_fds_.count(source_rank()) != 0u)
390 connected_fds_[source_rank()].erase(fd);
392 active_receive_fds_[source_rank()] = -1;
393 TLOG(TLVL_DEBUG) << GetTraceName() <<
"disconnect_receive_socket_: There are now " << connected_fds_[source_rank()].size() <<
" active senders.";
398 TLOG(19) << GetTraceName() <<
"receiveFragmentData: BEGIN";
399 int ret_rank = RECV_TIMEOUT;
400 if (getActiveFD_(source_rank()) == -1)
402 TLOG(TLVL_ERROR) << GetTraceName() <<
"receiveFragmentData: Receive socket not connected, returning RECV_TIMEOUT (Will result in \"Unexpected return code error\")";
411 SocketState state = SocketState::Metadata;
412 int target_bytes =
sizeof(
MessHead);
415 pollfd_s.events = POLLIN | POLLPRI | POLLERR;
416 pollfd_s.fd = getActiveFD_(source_rank());
420 bool noDataWarningSent =
false;
421 last_recv_time_ = std::chrono::steady_clock::now();
424 TLOG(9) << GetTraceName() <<
"receiveFragmentData: Polling fd to see if there's data";
425 int num_fds_ready = poll(&pollfd_s, 1, 1000);
426 TLOG(TLVL_TRACE) << GetTraceName() <<
"receiveFragmentData: Polled fd to see if there's data"
427 <<
", num_fds_ready = " << num_fds_ready;
428 if (num_fds_ready <= 0)
430 if (num_fds_ready == 0)
432 TLOG(TLVL_WARNING) << GetTraceName() <<
"receiveFragmentData: No data from " << source_rank() <<
" in " << TimeUtils::GetElapsedTimeMilliseconds(last_recv_time_) <<
" ms!"
433 <<
" State = " << (state == SocketState::Metadata ?
"Metadata" :
"Data") <<
", recvd/total=" << offset <<
"/" << target_bytes <<
" (delta=" << target_bytes - offset <<
")";
435 if (TimeUtils::GetElapsedTime(last_recv_time_) > receive_disconnected_wait_s_)
437 TLOG(TLVL_WARNING) << GetTraceName() <<
"receiveFragmentData: No data received within timeout (" << TimeUtils::GetElapsedTime(last_recv_time_) <<
" / " << receive_disconnected_wait_s_ <<
" ), returning RECV_TIMEOUT";
438 disconnect_receive_socket_(
"No data on this socket within timeout");
444 TLOG(TLVL_ERROR) <<
"Error in poll: errno=" << errno;
448 last_recv_time_ = std::chrono::steady_clock::now();
450 if ((pollfd_s.revents & (POLLIN | POLLPRI)) != 0)
454 else if ((pollfd_s.revents & (POLLNVAL)) != 0)
456 disconnect_receive_socket_(
"FD is closed, most likely because the peer went away.");
459 else if ((pollfd_s.revents & (POLLHUP | POLLERR)) != 0)
461 disconnect_receive_socket_(
"Poll returned POLLHUP or POLLERR, indicating problems with the sender.");
466 TLOG(TLVL_WARNING) << GetTraceName() <<
"receiveFragmentData: Wrong event received from pollfd: " << pollfd_s.revents;
467 disconnect_receive_socket_(
"Wrong event received from pollfd.");
471 if (state == SocketState::Metadata)
474 buff = &(mha[offset]);
475 byte_cnt =
sizeof(
MessHead) - offset;
480 buff =
reinterpret_cast<uint8_t*
>(destination) + offset;
484 TLOG(10) << GetTraceName() <<
"receiveFragmentData: Reading " << byte_cnt <<
" bytes from socket into " <<
static_cast<void*
>(buff);
485 sts = read(getActiveFD_(source_rank()), buff, byte_cnt);
488 TLOG(10) << GetTraceName() <<
"recvFragment state=" <<
static_cast<int>(state) <<
" read=" << sts;
490 if (sts == 0 || (sts < 0 && errno == EAGAIN))
493 if (loop_guard > 10) { usleep(1000); }
494 if (++loop_guard > 10010)
496 TLOG(TLVL_WARNING) << GetTraceName() <<
"receiveFragmentData: loop guard triggered, returning RECV_TIMEOUT";
497 setActiveFD_(source_rank(), -1);
504 last_recv_time_ = std::chrono::steady_clock::now();
509 TLOG(TLVL_WARNING) << GetTraceName() <<
"receiveFragmentData: Error on receive, closing socket"
510 <<
" (errno=" << errno <<
": " << strerror(errno) <<
")";
511 disconnect_receive_socket_(
"Error on receive");
515 if (!noDataWarningSent)
517 TLOG(TLVL_WARNING) << GetTraceName() <<
"receiveFragmentData: No data received, is the sender still sending?!?";
518 noDataWarningSent =
true;
520 if (TimeUtils::GetElapsedTime(last_recv_time_) > receive_disconnected_wait_s_)
522 TLOG(TLVL_ERROR) << GetTraceName() <<
"receiveFragmentData: No data received within timeout, aborting!";
530 if (sts >= target_bytes)
532 TLOG(9) << GetTraceName() <<
"receiveFragmentData: Target read bytes reached. Changing state";
534 if (state == SocketState::Metadata)
536 state = SocketState::Data;
543 TLOG(TLVL_WARNING) << GetTraceName() <<
"receiveFragmentData: Message header indicates that a Fragment header follows when I was expecting Fragment data!";
544 disconnect_receive_socket_(
"Desync detected");
549 ret_rank = source_rank();
550 TLOG(11) << GetTraceName() <<
"receiveFragmentData done sts=" << sts <<
" src=" << ret_rank;
551 TLOG(9) << GetTraceName() <<
"receiveFragmentData: Done receiving fragment. Moving into output.";
554 send_ack_(active_receive_fd_);
564 if (target_bytes == 0 && state == SocketState::Data)
566 ret_rank = source_rank();
567 TLOG(11) << GetTraceName() <<
"receiveFragmentData done sts=" << sts <<
" src=" << ret_rank;
568 TLOG(9) << GetTraceName() <<
"receiveFragmentData: Done receiving fragment. Moving into output.";
571 send_ack_(active_receive_fd_);
579 setLastActiveFD_(source_rank(), getActiveFD_(source_rank()));
580 setActiveFD_(source_rank(), -1);
582 TLOG(9) << GetTraceName() <<
"receiveFragmentData: Returning rank " << ret_rank;
591 return send_fd_ != -1;
593 auto count = getConnectedFDCount_(source_rank());
594 TLOG(TLVL_DEBUG) << GetTraceName() <<
"isRunning: There are " << count <<
" fds connected.";
602 std::lock_guard<std::mutex> lk(fd_mutex_);
603 if (connected_fds_.count(source_rank()) != 0u)
605 auto it = connected_fds_[source_rank()].begin();
606 char discard_buf[0x1000];
607 while (it != connected_fds_[source_rank()].end())
609 TLOG(TLVL_INFO) << GetTraceName() <<
"flush_buffers: Checking for data in socket " << *it <<
" for rank " << source_rank();
610 size_t bytes_read = 0;
611 while (
int sts = static_cast<int>(read(*it, discard_buf,
sizeof(discard_buf)) > 0))
617 TLOG(TLVL_WARNING) << GetTraceName() <<
"flush_buffers: Flushed " << bytes_read <<
" bytes from socket " << *it <<
" for rank " << source_rank();
619 TLOG(TLVL_INFO) << GetTraceName() <<
"flush_buffers: Closing socket " << *it <<
" for rank " << source_rank();
621 it = connected_fds_[source_rank()].erase(it);
623 connected_fds_.erase(source_rank());
625 active_receive_fds_[source_rank()] = -1;
626 last_active_receive_fds_[source_rank()] = -1;
633 TLOG(12) << GetTraceName() <<
"sendFragment begin send of fragment with sequenceID=" << frag.sequenceID();
634 artdaq::Fragment grab_ownership_frag = std::move(frag);
637 if (send_fd_ == -1 && connection_was_lost_)
639 TLOG(TLVL_INFO) << GetTraceName() <<
"reconnection attempt failed, returning quickly.";
647 while (static_cast<size_t>(send_ack_diff_) > buffer_count_) usleep(10000);
650 iovec iov = {
static_cast<void*
>(grab_ownership_frag.headerAddress()),
651 detail::RawFragmentHeader::num_words() *
sizeof(RawDataType)};
653 auto sts = sendData_(&iov, 1, send_retry_timeout_us_,
true);
654 auto start_time = std::chrono::steady_clock::now();
656 while (sts == CopyStatus::kTimeout && (send_timeout_usec == 0 || TimeUtils::GetElapsedTimeMicroseconds(start_time) < send_timeout_usec) && TimeUtils::GetElapsedTimeMicroseconds(start_time) < 10000000)
658 TLOG(13) << GetTraceName() <<
"sendFragment: Timeout sending fragment";
659 sts = sendData_(&iov, 1, send_retry_timeout_us_,
true);
662 if (sts != CopyStatus::kSuccess)
669 iov = {
static_cast<void*
>(grab_ownership_frag.headerAddress() + detail::RawFragmentHeader::num_words()),
670 grab_ownership_frag.sizeBytes() - detail::RawFragmentHeader::num_words() *
sizeof(RawDataType)};
671 sts = sendData_(&iov, 1, send_retry_timeout_us_);
672 start_time = std::chrono::steady_clock::now();
673 while (sts == CopyStatus::kTimeout && (send_timeout_usec == 0 || TimeUtils::GetElapsedTimeMicroseconds(start_time) < send_timeout_usec) && TimeUtils::GetElapsedTimeMicroseconds(start_time) < 10000000)
675 TLOG(13) << GetTraceName() <<
"sendFragment: Timeout sending fragment";
676 sts = sendData_(&iov, 1, send_retry_timeout_us_);
684 TLOG(12) << GetTraceName() <<
"sendFragment returning " << CopyStatusToString(sts);
690 TLOG(TLVL_DEBUG) << GetTraceName() <<
"sendData_ Converting buf to iovec";
691 iovec iov = {
const_cast<void*
>(buf), bytes};
692 return sendData_(&iov, 1, send_timeout_usec, isHeader);
700 if (timeoutMessageArmed_)
702 TLOG(TLVL_DEBUG) << GetTraceName() <<
"sendData_: Send fd is not open. Returning kTimeout";
703 timeoutMessageArmed_ =
false;
705 return CopyStatus::kTimeout;
707 timeoutMessageArmed_ =
true;
708 TLOG(14) << GetTraceName() <<
"send_timeout_usec is " << send_timeout_usec <<
", currently unused.";
711 uint32_t total_to_write_bytes = 0;
712 std::vector<iovec> iov_in(iovcnt + 1);
713 std::vector<iovec> iovv(iovcnt + 2);
715 for (ii = 0; ii < iovcnt; ++ii)
717 iov_in[ii + 1] = iov[ii];
718 total_to_write_bytes += iov[ii].iov_len;
721 MessHead mh = {0, isHeader ? MessHead::header_v0 : MessHead::data_v0, htons(source_rank()), {htonl(total_to_write_bytes)}};
722 iov_in[0].iov_base = &mh;
723 iov_in[0].iov_len =
sizeof(mh);
724 total_to_write_bytes +=
sizeof(mh);
727 ssize_t total_written_bytes = 0;
728 ssize_t per_write_max_bytes = (32 * 1024);
730 size_t in_iov_idx = 0;
731 size_t out_iov_idx = 0;
732 ssize_t this_write_bytes = 0;
739 (in_iov_idx + out_iov_idx) < iov_in.size() && this_write_bytes < per_write_max_bytes;
742 this_write_bytes += iov_in[in_iov_idx + out_iov_idx].iov_len;
743 iovv[out_iov_idx] = iov_in[in_iov_idx + out_iov_idx];
745 if (this_write_bytes > per_write_max_bytes)
747 iovv[out_iov_idx - 1].iov_len -= this_write_bytes - per_write_max_bytes;
748 this_write_bytes = per_write_max_bytes;
753 #ifndef __OPTIMIZE__ // This can be an expensive TRACE call (even if disabled) due to multiplicity of calls
754 TLOG(14) << GetTraceName() <<
"sendFragment b4 writev " << std::setw(7) << total_written_bytes <<
" total_written_bytes send_fd_=" << send_fd_ <<
" in_idx=" << in_iov_idx
755 <<
" iovcnt=" << out_iov_idx <<
" 1st.len=" << iovv[0].iov_len;
760 memset(&msg, 0,
sizeof(msghdr));
761 msg.msg_iov = &(iovv[0]);
762 msg.msg_iovlen = out_iov_idx;
763 sts = sendmsg(send_fd_, &msg, MSG_NOSIGNAL | (blocking != 0u ? 0 : MSG_DONTWAIT));
765 sts = writev(send_fd_, &(iovv[0]), out_iov_idx);
771 if (errno == EAGAIN )
773 TLOG(TLVL_DEBUG) << GetTraceName() <<
"sendFragment EWOULDBLOCK";
776 fcntl(send_fd_, F_SETFL, 0);
781 TLOG(TLVL_WARNING) << GetTraceName() <<
"sendFragment_: WRITE ERROR " << errno <<
": " << strerror(errno);
785 connection_was_lost_ =
true;
788 if (sts != this_write_bytes)
791 TLOG(TLVL_DEBUG) << GetTraceName() <<
"sendFragment writev sts(" << sts <<
")!=requested_send_bytes(" << this_write_bytes <<
")";
792 total_written_bytes += sts;
794 for (ii = 0;
static_cast<size_t>(sts) >= iovv[ii].iov_len; ++ii)
796 sts -= iovv[ii].iov_len;
799 iovv[ii].iov_len -= sts;
800 iovv[ii].iov_base =
static_cast<uint8_t*
>(iovv[ii].iov_base) + sts;
806 iovv[out_iov_idx] = iovv[ii];
809 this_write_bytes = iovv[out_iov_idx].iov_len;
814 auto additional = (
reinterpret_cast<uintptr_t
>(iov_in[in_iov_idx].iov_base) + iov_in[in_iov_idx].iov_len) - (
reinterpret_cast<uintptr_t
>(iovv[out_iov_idx].iov_base) + iovv[out_iov_idx].iov_len);
815 if (additional != 0u)
817 iovv[out_iov_idx].iov_len += additional;
818 this_write_bytes += additional;
819 if (this_write_bytes > per_write_max_bytes)
821 iovv[out_iov_idx].iov_len -= this_write_bytes - per_write_max_bytes;
822 this_write_bytes = per_write_max_bytes;
826 TLOG(TLVL_TRACE) << GetTraceName() <<
"sendFragment writev sts!=: this_write_bytes=" << this_write_bytes
827 <<
" out_iov_idx=" << out_iov_idx
828 <<
" additional=" << additional
833 #ifndef __OPTIMIZE__ // This can be an expensive TRACE call (even if disabled) due to multiplicity of calls
834 TLOG(TLVL_TRACE) << GetTraceName() <<
"sendFragment writev sts(" << sts <<
")==requested_send_bytes(" << this_write_bytes <<
")";
836 total_written_bytes += sts;
838 iovv[out_iov_idx].iov_base =
static_cast<uint8_t*
>(iovv[out_iov_idx].iov_base) + iovv[out_iov_idx].iov_len;
839 iovv[out_iov_idx].iov_len = 0;
840 in_iov_idx += out_iov_idx;
841 this_write_bytes = 0;
843 auto additional = (
reinterpret_cast<uintptr_t
>(iov_in[in_iov_idx].iov_base) + iov_in[in_iov_idx].iov_len) - (
reinterpret_cast<uintptr_t
>(iovv[out_iov_idx].iov_base) + iovv[out_iov_idx].iov_len);
844 if (additional != 0u)
846 iovv[out_iov_idx].iov_len += additional;
847 this_write_bytes += additional;
848 if (this_write_bytes > per_write_max_bytes)
850 iovv[out_iov_idx].iov_len -= this_write_bytes - per_write_max_bytes;
851 this_write_bytes = per_write_max_bytes;
853 if (out_iov_idx != 0)
855 iovv[0] = iovv[out_iov_idx];
865 }
while (total_written_bytes < total_to_write_bytes);
866 if (total_written_bytes > total_to_write_bytes)
868 TLOG(TLVL_ERROR) << GetTraceName() <<
"sendFragment program error: too many bytes transferred";
874 fcntl(send_fd_, F_SETFL, O_NONBLOCK);
876 sts = total_written_bytes -
sizeof(
MessHead);
878 TLOG(14) << GetTraceName() <<
"sendFragment sts=" << sts;
882 void artdaq::TCPSocketTransfer::connect_()
884 auto start_time = std::chrono::steady_clock::now();
887 while (send_fd_ == -1 && TimeUtils::GetElapsedTimeMicroseconds(start_time) < send_retry_timeout_us_ * 10)
889 TLOG(TLVL_DEBUG) << GetTraceName() <<
"Connecting sender socket";
890 int sndbuf_bytes =
static_cast<int>(sndbuf_);
891 if (sndbuf_ > INT_MAX)
893 sndbuf_bytes = INT_MAX;
894 TLOG(TLVL_WARNING) <<
"Requested SNDBUF " << sndbuf_ <<
" too large, setting to INT_MAX: " << INT_MAX;
896 TLOG(TLVL_DEBUG) <<
"Requested SNDBUF is " << sndbuf_bytes;
898 send_fd_ =
TCPConnect(hostMap_[destination_rank()].c_str(), portMan->GetTCPSocketTransferPort(destination_rank()), O_NONBLOCK, sndbuf_bytes);
901 if (connection_was_lost_) {
break; }
903 usleep(send_retry_timeout_us_);
908 TLOG(TLVL_DEBUG) << GetTraceName() <<
"connect_ " + hostMap_[destination_rank()] +
":" << portMan->GetTCPSocketTransferPort(destination_rank()) <<
" send_fd_=" << send_fd_;
912 TLOG(TLVL_DEBUG) << GetTraceName() <<
"connect_: Writing connect message";
913 MessHead mh = {0, MessHead::connect_v0, htons(source_rank()), {htonl(CONN_MAGIC)}};
914 ssize_t sts = write(send_fd_, &mh,
sizeof(mh));
917 TLOG(TLVL_ERROR) << GetTraceName() <<
"connect_: Error writing connect message!";
925 TLOG(TLVL_INFO) << GetTraceName() <<
"connect_: Successfully connected";
928 connection_was_lost_ =
false;
932 if (ack_listen_thread_ && ack_listen_thread_->joinable()) ack_listen_thread_->join();
933 TLOG(TLVL_INFO) << GetTraceName() <<
"Starting Ack Listener Thread";
937 ack_listen_thread_ = std::make_unique<boost::thread>(&TCPSocketTransfer::receive_acks_,
this);
939 catch (
const boost::exception& e)
941 TLOG(TLVL_ERROR) <<
"Caught boost::exception starting TCP Socket Ack Listen thread: " << boost::diagnostic_information(e) <<
", errno=" << errno;
942 std::cerr <<
"Caught boost::exception starting TCP Socket Ack Listen thread: " << boost::diagnostic_information(e) <<
", errno=" << errno << std::endl;
949 void artdaq::TCPSocketTransfer::reconnect_()
953 TLOG(TLVL_TRACE) << GetTraceName() <<
"check/reconnect";
958 void artdaq::TCPSocketTransfer::start_listen_thread_()
960 std::lock_guard<std::mutex> start_lock(listen_thread_mutex_);
961 if (listen_thread_refcount_ == 0)
963 if (listen_thread_ && listen_thread_->joinable())
965 listen_thread_->join();
967 listen_thread_refcount_ = 1;
968 TLOG(TLVL_INFO) << GetTraceName() <<
"Starting Listener Thread";
972 listen_thread_ = std::make_unique<boost::thread>(&TCPSocketTransfer::listen_, portMan->GetTCPSocketTransferPort(destination_rank()), rcvbuf_);
974 catch (
const boost::exception& e)
976 TLOG(TLVL_ERROR) <<
"Caught boost::exception starting TCP Socket Listen thread: " << boost::diagnostic_information(e) <<
", errno=" << errno;
977 std::cerr <<
"Caught boost::exception starting TCP Socket Listen thread: " << boost::diagnostic_information(e) <<
", errno=" << errno << std::endl;
983 listen_thread_refcount_++;
988 void artdaq::TCPSocketTransfer::receive_acks_()
990 while (send_fd_ >= 0)
993 pollfd_s.events = POLLIN | POLLPRI;
994 pollfd_s.fd = send_fd_;
996 TLOG(18) << GetTraceName() <<
"receive_acks_: Polling fd to see if there's data";
997 int num_fds_ready = poll(&pollfd_s, 1, 1000);
998 if (num_fds_ready <= 0)
1000 if (num_fds_ready == 0)
1002 TLOG(18) << GetTraceName() <<
"receive_acks_: No data on receive socket";
1006 TLOG(TLVL_ERROR) <<
"Error in poll: errno=" << errno;
1010 if (pollfd_s.revents & (POLLIN | POLLPRI))
1016 TLOG(TLVL_DEBUG) << GetTraceName() <<
"receive_acks_: Wrong event received from pollfd: " << pollfd_s.revents;
1021 auto sts = read(send_fd_, &mh,
sizeof(mh));
1023 if (sts !=
sizeof(mh))
1025 TLOG(TLVL_ERROR) << GetTraceName() <<
"receive_ack_: Wrong message header length received! (actual " << sts <<
" != " <<
sizeof(mh) <<
" expected)";
1033 TLOG(TLVL_ERROR) << GetTraceName() <<
"receive_ack_: Received ack for different sender! Rank=" << my_rank <<
", hdr=" << mh.
source_id;
1038 TLOG(TLVL_ERROR) << GetTraceName() <<
"receive_ack_: Wrong magic bytes in header!";
1042 TLOG(17) << GetTraceName() <<
"receive_acks_: Received ack message, diff is now " << (send_ack_diff_.load() - 1);
1047 void artdaq::TCPSocketTransfer::send_ack_(
int fd)
1049 MessHead mh = {0, MessHead::ack_v0, htons(source_rank()), {htonl(ACK_MAGIC)}};
1050 write(fd, &mh,
sizeof(mh));
1054 void artdaq::TCPSocketTransfer::listen_(
int port,
size_t rcvbuf)
1057 while (listen_thread_refcount_ > 0)
1059 TLOG(TLVL_TRACE) <<
"listen_: Listening/accepting new connections on port " << port;
1060 if (listen_fd == -1)
1062 TLOG(TLVL_DEBUG) <<
"listen_: Opening listener";
1065 if (listen_fd == -1)
1067 TLOG(TLVL_DEBUG) <<
"listen_: Error creating listen_fd!";
1072 timeval tv = {2, 0};
1075 FD_SET(listen_fd, &rfds);
1077 res = select(listen_fd + 1, &rfds, static_cast<fd_set*>(
nullptr), static_cast<fd_set*>(
nullptr), &tv);
1082 socklen_t arglen =
sizeof(un);
1084 TLOG(TLVL_DEBUG) <<
"listen_: Calling accept";
1085 fd = accept(listen_fd, reinterpret_cast<sockaddr*>(&un), &arglen);
1086 TLOG(TLVL_DEBUG) <<
"listen_: Done with accept";
1088 TLOG(TLVL_DEBUG) <<
"listen_: Reading connect message";
1089 socklen_t lenlen =
sizeof(tv);
1091 setsockopt(fd, SOL_SOCKET, SO_RCVTIMEO, &tv, lenlen);
1093 uint64_t mark_us = TimeUtils::gettimeofday_us();
1094 sts = read(fd, &mh,
sizeof(mh));
1095 uint64_t delta_us = TimeUtils::gettimeofday_us() - mark_us;
1096 TLOG(TLVL_DEBUG) <<
"listen_: Read of connect message took " << delta_us <<
" microseconds.";
1097 if (sts !=
sizeof(mh))
1099 TLOG(TLVL_DEBUG) <<
"listen_: Wrong message header length received!";
1108 TLOG(TLVL_DEBUG) <<
"listen_: Wrong magic bytes in header!";
1114 std::lock_guard<std::mutex> lk(fd_mutex_);
1115 connected_fds_[mh.
source_id].insert(fd);
1117 TLOG(TLVL_INFO) <<
"listen_: New fd is " << fd <<
" for source rank " << mh.
source_id;
1121 TLOG(16) <<
"listen_: No connections in timeout interval!";
1125 TLOG(TLVL_INFO) <<
"listen_: Shutting down connection listener";
1126 if (listen_fd != -1)
1130 std::lock_guard<std::mutex> lk(fd_mutex_);
1131 auto it = connected_fds_.begin();
1132 while (it != connected_fds_.end())
1134 auto& fd_set = it->second;
1135 auto rank_it = fd_set.begin();
1136 while (rank_it != fd_set.end())
1139 rank_it = fd_set.erase(rank_it);
1141 it = connected_fds_.erase(it);
1146 size_t artdaq::TCPSocketTransfer::getConnectedFDCount_(
int source_rank)
1148 std::lock_guard<std::mutex> lk(fd_mutex_);
1149 #ifndef __OPTIMIZE__
1150 TLOG(15) << GetTraceName() <<
"getConnectedFDCount_: count is " << (connected_fds_.count(source_rank) != 0u ? connected_fds_[source_rank].size() : 0);
1152 return connected_fds_.count(source_rank) != 0u ? connected_fds_[source_rank].size() : 0;
1155 int artdaq::TCPSocketTransfer::getActiveFD_(
int source_rank)
1157 std::lock_guard<std::mutex> lk(fd_mutex_);
1158 #ifndef __OPTIMIZE__
1159 TLOG(15) << GetTraceName() <<
"getActiveFD_: fd is " << (active_receive_fds_.count(source_rank) != 0u ? active_receive_fds_[source_rank] : -1);
1161 return active_receive_fds_.count(source_rank) != 0u ? active_receive_fds_[source_rank] : -1;
1163 void artdaq::TCPSocketTransfer::setActiveFD_(
int source_rank,
int fd)
1165 std::lock_guard<std::mutex> lk(fd_mutex_);
1166 #ifndef __OPTIMIZE__
1167 TLOG(15) << GetTraceName() <<
"setActiveFD_: setting active fd for rank " << source_rank <<
" to " << fd;
1169 active_receive_fds_[source_rank] = fd;
1171 int artdaq::TCPSocketTransfer::getLastActiveFD_(
int source_rank)
1173 std::lock_guard<std::mutex> lk(fd_mutex_);
1174 #ifndef __OPTIMIZE__
1175 TLOG(15) << GetTraceName() <<
"getLastActiveFD_: fd is " << (last_active_receive_fds_.count(source_rank) != 0u ? last_active_receive_fds_[source_rank] : -1);
1177 return last_active_receive_fds_.count(source_rank) != 0u ? last_active_receive_fds_[source_rank] : -1;
1179 void artdaq::TCPSocketTransfer::setLastActiveFD_(
int source_rank,
int fd)
1181 std::lock_guard<std::mutex> lk(fd_mutex_);
1182 #ifndef __OPTIMIZE__
1183 TLOG(15) << GetTraceName() <<
"setLastActiveFD_: setting last active fd for rank " << source_rank <<
" to " << fd;
1185 last_active_receive_fds_[source_rank] = fd;
bool isRunning() override
Determine whether the TransferInterface plugin is able to send/receive data.
virtual int source_rank() const
Get the source rank for this TransferInterface instance.
uint32_t conn_magic
unsigned first is better for MessHead initializer: {0,0,my_node_idx_,CONN_MAGIC}
int receiveFragmentHeader(detail::RawFragmentHeader &header, size_t timeout_usec) override
Receive a Fragment Header from the transport mechanism.
int TCPConnect(char const *host_in, int dflt_port, int64_t flags=0, int sndbufsiz=0)
Connect to a host on a given port.
This TransferInterface is a Receiver.
int receiveFragmentData(RawDataType *destination, size_t wordCount) override
Receive the body of a Fragment to the given destination pointer.
int TCP_listen_fd(int port, int rcvbuf)
Create a TCP listening socket on the given port and INADDR_ANY, with the given receive buffer...
TCPSocketTransfer(fhicl::ParameterSet const &ps, Role role)
TCPSocketTransfer Constructor.
This TransferInterface is a Sender.
void flush_buffers() override
Flush any in-flight data. This should be used by the receiver after the receive loop has ended...
int32_t byte_count
use CONN_MAGIC for connect_v0, data that follow for data_v0 (and 0 lenght data)
Some error occurred, but no exception was thrown.
Role
Used to determine if a TransferInterface is a Sender or Receiver.
int64_t source_id
Rank of the source.
MessType message_type
Message Type.
The send operation completed successfully.
This interface defines the functions used to transfer data between artdaq applications.
TransferInterface implementation plugin that sends data using TCP sockets.
This header is sent by the TCPSocket_transfer to allow for more efficient writev calls.
hostMap_t MakeHostMap(fhicl::ParameterSet const &pset, hostMap_t map=hostMap_t())
Make a hostMap_t from a HostMap::Config ParameterSet
CopyStatus
Returned from the send functions, this enumeration describes the possible return codes. If an exception occurs, it will be thrown and should be handled normally.