10 #include <sys/socket.h>
11 #include <sys/types.h>
24 #include "artdaq/DAQdata/Globals.hh"
25 #define TRACE_NAME (app_name + "_TCPSocketTransfer").c_str()
29 #include "artdaq-core/Data/Fragment.hh"
30 #include "artdaq-core/Utilities/TimeUtils.hh"
33 #include "artdaq/TransferPlugins/TCPSocketTransfer.hh"
34 #include "artdaq/TransferPlugins/detail/SRSockets.hh"
35 #include "artdaq/TransferPlugins/detail/Timeout.hh"
39 std::atomic<int> artdaq::TCPSocketTransfer::listen_thread_refcount_(0);
40 std::unique_ptr<boost::thread> artdaq::TCPSocketTransfer::listen_thread_ =
nullptr;
41 std::map<int, std::set<int>> artdaq::TCPSocketTransfer::connected_fds_ = std::map<int, std::set<int>>();
42 std::mutex artdaq::TCPSocketTransfer::listen_thread_mutex_;
43 std::mutex artdaq::TCPSocketTransfer::fd_mutex_;
49 , rcvbuf_(pset.get<size_t>(
"tcp_receive_buffer_size", 0))
50 , sndbuf_(pset.get<size_t>(
"tcp_send_buffer_size", max_fragment_size_words_ * sizeof(artdaq::RawDataType) * buffer_count_))
51 , send_retry_timeout_us_(pset.get<size_t>(
"send_retry_timeout_us", 1000000))
52 , timeoutMessageArmed_(true)
53 , receive_disconnected_wait_s_(pset.get<double>(
"receive_socket_disconnected_wait_s", 10.0))
54 , receive_err_wait_us_(pset.get<size_t>(
"receive_socket_disconnected_wait_us", 10000))
55 , receive_socket_has_been_connected_(false)
58 TLOG(TLVL_DEBUG + 32) << GetTraceName() <<
" Constructor: pset=" << pset.to_string() <<
", role=" << (role ==
TransferInterface::Role::kReceive ?
"kReceive" :
"kSend");
59 connection_was_lost_ =
false;
64 TLOG(TLVL_DEBUG + 32) << GetTraceName() <<
"Listening for connections";
65 start_listen_thread_();
66 TLOG(TLVL_DEBUG + 32) << GetTraceName() <<
"Done Listening";
71 TLOG(TLVL_DEBUG + 32) << GetTraceName() <<
"Connecting to destination";
73 TLOG(TLVL_DEBUG + 32) << GetTraceName() <<
"Done Connecting";
75 TLOG(TLVL_DEBUG + 32) << GetTraceName() <<
"End of Constructor";
78 artdaq::TCPSocketTransfer::~TCPSocketTransfer() noexcept
80 TLOG(TLVL_DEBUG + 32) << GetTraceName() <<
"Shutting down TCPSocketTransfer";
89 timeval tv = {0, 100000};
90 socklen_t len =
sizeof(tv);
91 setsockopt(send_fd_, SOL_SOCKET, SO_SNDTIMEO, &tv, len);
92 write(send_fd_, &mh,
sizeof(mh));
102 if (ack_listen_thread_ && ack_listen_thread_->joinable())
104 ack_listen_thread_->join();
112 std::lock_guard<std::mutex> lk(listen_thread_mutex_);
113 listen_thread_refcount_--;
116 if (listen_thread_refcount_ <= 0 && listen_thread_ && listen_thread_->joinable())
118 listen_thread_->join();
127 TLOG(TLVL_DEBUG + 32) << GetTraceName() <<
"End of Destructor";
132 TLOG(TLVL_DEBUG + 34) << GetTraceName() <<
"receiveFragmentHeader: BEGIN";
133 int ret_rank = RECV_TIMEOUT;
136 if (getConnectedFDCount_(source_rank()) == 0)
145 TLOG(TLVL_DEBUG + 36) << GetTraceName() <<
"receiveFragmentHeader: Receive socket not connected, returning RECV_TIMEOUT";
146 usleep(receive_err_wait_us_);
149 receive_socket_has_been_connected_ =
true;
150 last_recv_time_ = std::chrono::steady_clock::now();
152 TLOG(TLVL_DEBUG + 34) << GetTraceName() <<
"receiveFragmentHeader timeout_usec=" << timeout_usec;
157 SocketState state = SocketState::Metadata;
158 int target_bytes =
sizeof(
MessHead);
159 uint64_t start_time_us = TimeUtils::gettimeofday_us();
170 if (timeout_usec == 0)
176 timeout_ms = (timeout_usec + 999) / 1000;
180 bool noDataWarningSent =
false;
183 while (!done && getConnectedFDCount_(source_rank()) > 0)
185 if (getActiveFD_(source_rank()) == -1)
189 std::vector<pollfd> pollfds;
191 std::lock_guard<std::mutex> lk(fd_mutex_);
192 fd_count = connected_fds_[source_rank()].size();
193 pollfds.resize(fd_count);
194 auto iter = connected_fds_[source_rank()].begin();
195 for (
size_t ii = 0; ii < fd_count; ++ii)
197 pollfds[ii].events = POLLIN | POLLPRI | POLLERR;
198 pollfds[ii].fd = *iter;
203 int num_fds_ready = poll(&pollfds[0], fd_count, timeout_ms);
204 if (num_fds_ready <= 0)
206 TLOG(TLVL_DEBUG + 34) << GetTraceName() <<
"receiveFragmentHeader: No data on receive socket, returning RECV_TIMEOUT";
211 if (getLastActiveFD_(source_rank()) != -1)
213 for (
auto& pollfd : pollfds)
216 if (pollfd.fd == getLastActiveFD_(source_rank()))
223 int active_index = -1;
224 int16_t anomolous_events = 0;
225 for (
size_t ii = index; ii < index + pollfds.size(); ++ii)
227 auto pollfd_index = (ii + index) % pollfds.size();
228 setActiveFD_(source_rank(), pollfds[pollfd_index].fd);
229 if ((pollfds[pollfd_index].revents & (POLLIN | POLLPRI)) != 0)
231 active_index = pollfd_index;
234 if ((pollfds[pollfd_index].revents & (POLLHUP | POLLERR)) != 0)
236 disconnect_receive_socket_(
"Poll returned POLLHUP or POLLERR, indicating problems with the sender.");
239 else if ((pollfds[pollfd_index].revents & (POLLNVAL)) != 0)
241 TLOG(TLVL_DEBUG + 32) << GetTraceName() <<
"receiveFragmentHeader: FD is closed, most likely because the peer went away. Removing from fd list.";
242 disconnect_receive_socket_(
"FD is closed, most likely because the peer went away.");
245 else if (pollfds[pollfd_index].revents != 0)
247 anomolous_events |= pollfds[pollfd_index].revents;
251 if (active_index == -1)
253 if (anomolous_events != 0)
255 TLOG(TLVL_DEBUG + 32) << GetTraceName() <<
"receiveFragmentHeader: Wrong event received from a pollfd. Mask: " <<
static_cast<int>(anomolous_events);
257 setActiveFD_(source_rank(), -1);
261 if (!done && timeout_usec > 0)
264 size_t delta_us = TimeUtils::gettimeofday_us() - start_time_us;
265 if (delta_us > timeout_usec)
269 timeout_ms = ((timeout_usec - delta_us) + 999) / 1000;
272 if (loop_guard > 10) { usleep(1000); }
273 if (++loop_guard > 10010)
275 TLOG(TLVL_WARNING) << GetTraceName() <<
"receiveFragmentHeader: loop guard triggered, returning RECV_TIMEOUT";
276 usleep(receive_err_wait_us_);
277 setActiveFD_(source_rank(), -1);
281 if (state == SocketState::Metadata)
284 buff = &(mha[offset]);
285 byte_cnt =
sizeof(
MessHead) - offset;
290 buff =
reinterpret_cast<uint8_t*
>(&header) + offset;
291 byte_cnt = target_bytes - offset;
301 auto fd = getActiveFD_(source_rank());
304 TLOG(TLVL_DEBUG + 35) << GetTraceName() <<
"receiveFragmentHeader: Reading " << byte_cnt <<
" bytes from socket " << fd;
305 sts = read(fd, buff, byte_cnt);
306 TLOG(TLVL_DEBUG + 35) << GetTraceName() <<
"receiveFragmentHeader: Done with read";
311 last_recv_time_ = std::chrono::steady_clock::now();
314 TLOG(TLVL_DEBUG + 36) << GetTraceName() <<
"receiveFragmentHeader state=" <<
static_cast<int>(state) <<
" read=" << sts;
315 if (sts < 0 && errno != EAGAIN)
317 TLOG(TLVL_WARNING) << GetTraceName() <<
"receiveFragmentHeader: Error on receive, closing socket " << fd
318 <<
" (errno=" << errno <<
": " << strerror(errno) <<
")";
319 disconnect_receive_socket_(
"Error on receive");
321 else if (sts == 0 || errno == EAGAIN)
323 if (!noDataWarningSent)
325 TLOG(TLVL_WARNING) << GetTraceName() <<
"receiveFragmentHeader: No data received, is the sender still sending?!?";
326 noDataWarningSent =
true;
328 if (TimeUtils::GetElapsedTime(last_recv_time_) > receive_disconnected_wait_s_)
330 TLOG(TLVL_ERROR) << GetTraceName() <<
"receiveFragmentHeader: No data received within timeout, aborting!";
338 if (sts >= target_bytes)
340 TLOG(TLVL_DEBUG + 36) << GetTraceName() <<
"receiveFragmentHeader: Target read bytes reached. Changing state";
342 if (state == SocketState::Metadata)
344 state = SocketState::Data;
348 TLOG(TLVL_DEBUG + 36) << GetTraceName() <<
"receiveFragmentHeader: Expected header size = " << target_bytes <<
", sizeof(RawFragmentHeader) = " <<
sizeof(artdaq::detail::RawFragmentHeader);
353 disconnect_receive_socket_(
"Stop Message received.");
357 TLOG(TLVL_WARNING) << GetTraceName() <<
"receiveFragmentHeader: Message header indicates that Fragment data follows when I was expecting a Fragment header!";
358 disconnect_receive_socket_(
"Desync detected");
361 if (target_bytes == 0)
369 ret_rank = source_rank();
370 TLOG(TLVL_DEBUG + 35) << GetTraceName() <<
"receiveFragmentHeader done sts=" << sts <<
" src=" << ret_rank;
371 TLOG(TLVL_DEBUG + 36) << GetTraceName() <<
"receiveFragmentHeader: Done receiving fragment header. Moving into output.";
381 TLOG(TLVL_DEBUG + 34) << GetTraceName() <<
"receiveFragmentHeader: Returning " << ret_rank;
385 void artdaq::TCPSocketTransfer::disconnect_receive_socket_(
const std::string& msg)
387 std::lock_guard<std::mutex> lk(fd_mutex_);
388 auto fd = active_receive_fds_[source_rank()];
389 TLOG(TLVL_WARNING) << GetTraceName() <<
"disconnect_receive_socket_: " << msg <<
" Closing socket " << fd <<
" for rank " << source_rank();
391 if (connected_fds_.count(source_rank()) != 0u)
393 connected_fds_[source_rank()].erase(fd);
395 active_receive_fds_[source_rank()] = -1;
396 TLOG(TLVL_DEBUG + 32) << GetTraceName() <<
"disconnect_receive_socket_: There are now " << connected_fds_[source_rank()].size() <<
" active senders.";
401 TLOG(TLVL_DEBUG + 39) << GetTraceName() <<
"receiveFragmentData: BEGIN";
402 int ret_rank = RECV_TIMEOUT;
403 if (getActiveFD_(source_rank()) == -1)
405 TLOG(TLVL_ERROR) << GetTraceName() <<
"receiveFragmentData: Receive socket not connected, returning RECV_TIMEOUT (Will result in \"Unexpected return code error\")";
414 SocketState state = SocketState::Metadata;
415 int target_bytes =
sizeof(
MessHead);
418 pollfd_s.events = POLLIN | POLLPRI | POLLERR;
419 pollfd_s.fd = getActiveFD_(source_rank());
423 bool noDataWarningSent =
false;
424 last_recv_time_ = std::chrono::steady_clock::now();
427 TLOG(TLVL_DEBUG + 33) << GetTraceName() <<
"receiveFragmentData: Polling fd to see if there's data";
428 int num_fds_ready = poll(&pollfd_s, 1, 1000);
429 TLOG(TLVL_DEBUG + 33) << GetTraceName() <<
"receiveFragmentData: Polled fd to see if there's data"
430 <<
", num_fds_ready = " << num_fds_ready;
431 if (num_fds_ready <= 0)
433 if (num_fds_ready == 0)
435 TLOG(TLVL_WARNING) << GetTraceName() <<
"receiveFragmentData: No data from " << source_rank() <<
" in " << TimeUtils::GetElapsedTimeMilliseconds(last_recv_time_) <<
" ms!"
436 <<
" State = " << (state == SocketState::Metadata ?
"Metadata" :
"Data") <<
", recvd/total=" << offset <<
"/" << target_bytes <<
" (delta=" << target_bytes - offset <<
")";
438 if (TimeUtils::GetElapsedTime(last_recv_time_) > receive_disconnected_wait_s_)
440 TLOG(TLVL_WARNING) << GetTraceName() <<
"receiveFragmentData: No data received within timeout (" << TimeUtils::GetElapsedTime(last_recv_time_) <<
" / " << receive_disconnected_wait_s_ <<
" ), returning RECV_TIMEOUT";
441 disconnect_receive_socket_(
"No data on this socket within timeout");
447 TLOG(TLVL_ERROR) <<
"Error in poll: errno=" << errno;
451 last_recv_time_ = std::chrono::steady_clock::now();
453 if ((pollfd_s.revents & (POLLIN | POLLPRI)) != 0)
457 else if ((pollfd_s.revents & (POLLNVAL)) != 0)
459 disconnect_receive_socket_(
"FD is closed, most likely because the peer went away.");
462 else if ((pollfd_s.revents & (POLLHUP | POLLERR)) != 0)
464 disconnect_receive_socket_(
"Poll returned POLLHUP or POLLERR, indicating problems with the sender.");
469 TLOG(TLVL_WARNING) << GetTraceName() <<
"receiveFragmentData: Wrong event received from pollfd: " << pollfd_s.revents;
470 disconnect_receive_socket_(
"Wrong event received from pollfd.");
474 if (state == SocketState::Metadata)
477 buff = &(mha[offset]);
478 byte_cnt =
sizeof(
MessHead) - offset;
483 buff =
reinterpret_cast<uint8_t*
>(destination) + offset;
487 TLOG(TLVL_DEBUG + 38) << GetTraceName() <<
"receiveFragmentData: Reading " << byte_cnt <<
" bytes from socket into " <<
static_cast<void*
>(buff);
488 sts = read(getActiveFD_(source_rank()), buff, byte_cnt);
491 TLOG(TLVL_DEBUG + 38) << GetTraceName() <<
"recvFragment state=" <<
static_cast<int>(state) <<
" read=" << sts;
493 if (sts == 0 || (sts < 0 && errno == EAGAIN))
496 if (loop_guard > 10) { usleep(1000); }
497 if (++loop_guard > 10010)
499 TLOG(TLVL_WARNING) << GetTraceName() <<
"receiveFragmentData: loop guard triggered, returning RECV_TIMEOUT";
500 setActiveFD_(source_rank(), -1);
507 last_recv_time_ = std::chrono::steady_clock::now();
512 TLOG(TLVL_WARNING) << GetTraceName() <<
"receiveFragmentData: Error on receive, closing socket"
513 <<
" (errno=" << errno <<
": " << strerror(errno) <<
")";
514 disconnect_receive_socket_(
"Error on receive");
518 if (!noDataWarningSent)
520 TLOG(TLVL_WARNING) << GetTraceName() <<
"receiveFragmentData: No data received, is the sender still sending?!?";
521 noDataWarningSent =
true;
523 if (TimeUtils::GetElapsedTime(last_recv_time_) > receive_disconnected_wait_s_)
525 TLOG(TLVL_ERROR) << GetTraceName() <<
"receiveFragmentData: No data received within timeout, aborting!";
533 if (sts >= target_bytes)
535 TLOG(TLVL_DEBUG + 42) << GetTraceName() <<
"receiveFragmentData: Target read bytes reached. Changing state";
537 if (state == SocketState::Metadata)
539 state = SocketState::Data;
546 TLOG(TLVL_WARNING) << GetTraceName() <<
"receiveFragmentData: Message header indicates that a Fragment header follows when I was expecting Fragment data!";
547 disconnect_receive_socket_(
"Desync detected");
552 ret_rank = source_rank();
553 TLOG(TLVL_DEBUG + 41) << GetTraceName() <<
"receiveFragmentData done sts=" << sts <<
" src=" << ret_rank;
554 TLOG(TLVL_DEBUG + 39) << GetTraceName() <<
"receiveFragmentData: Done receiving fragment. Moving into output.";
557 send_ack_(active_receive_fd_);
567 if (target_bytes == 0 && state == SocketState::Data)
569 ret_rank = source_rank();
570 TLOG(TLVL_DEBUG + 41) << GetTraceName() <<
"receiveFragmentData done sts=" << sts <<
" src=" << ret_rank;
571 TLOG(TLVL_DEBUG + 39) << GetTraceName() <<
"receiveFragmentData: Done receiving fragment. Moving into output.";
574 send_ack_(active_receive_fd_);
582 setLastActiveFD_(source_rank(), getActiveFD_(source_rank()));
583 setActiveFD_(source_rank(), -1);
585 TLOG(TLVL_DEBUG + 39) << GetTraceName() <<
"receiveFragmentData: Returning rank " << ret_rank;
594 return send_fd_ != -1;
596 auto count = getConnectedFDCount_(source_rank());
597 TLOG(TLVL_DEBUG + 32) << GetTraceName() <<
"isRunning: There are " << count <<
" fds connected.";
608 std::lock_guard<std::mutex> lk(fd_mutex_);
609 if (connected_fds_.count(rank) != 0)
611 fds = connected_fds_[rank];
612 connected_fds_.erase(rank);
616 char discard_buf[0x1000];
619 TLOG(TLVL_INFO) << GetTraceName() <<
"flush_buffers: Checking for data in socket " << fd <<
" for rank " << rank;
620 size_t bytes_read = 0;
621 while (
int sts = static_cast<int>(read(fd, discard_buf,
sizeof(discard_buf)) > 0))
627 TLOG(TLVL_WARNING) << GetTraceName() <<
"flush_buffers: Flushed " << bytes_read <<
" bytes from socket " << fd <<
" for rank " << rank;
629 TLOG(TLVL_INFO) << GetTraceName() <<
"flush_buffers: Closing socket " << fd <<
" for rank " << rank;
632 active_receive_fds_[rank] = -1;
633 last_active_receive_fds_[rank] = -1;
640 TLOG(TLVL_DEBUG + 42) << GetTraceName() <<
"sendFragment begin send of fragment with sequenceID=" << frag.sequenceID();
641 artdaq::Fragment grab_ownership_frag = std::move(frag);
644 if (send_fd_ == -1 && connection_was_lost_)
646 TLOG(TLVL_INFO) << GetTraceName() <<
"reconnection attempt failed, returning quickly.";
654 while (static_cast<size_t>(send_ack_diff_) > buffer_count_) usleep(10000);
657 iovec iov = {
static_cast<void*
>(grab_ownership_frag.headerAddress()),
658 detail::RawFragmentHeader::num_words() *
sizeof(RawDataType)};
660 auto sts = sendData_(&iov, 1, send_retry_timeout_us_,
true);
661 auto start_time = std::chrono::steady_clock::now();
663 while (sts == CopyStatus::kTimeout && (send_timeout_usec == 0 || TimeUtils::GetElapsedTimeMicroseconds(start_time) < send_timeout_usec) && TimeUtils::GetElapsedTimeMicroseconds(start_time) < 10000000)
665 TLOG(TLVL_DEBUG + 43) << GetTraceName() <<
"sendFragment: Timeout sending fragment";
666 sts = sendData_(&iov, 1, send_retry_timeout_us_,
true);
669 if (sts != CopyStatus::kSuccess)
676 iov = {
static_cast<void*
>(grab_ownership_frag.headerAddress() + detail::RawFragmentHeader::num_words()),
677 grab_ownership_frag.sizeBytes() - detail::RawFragmentHeader::num_words() *
sizeof(RawDataType)};
678 sts = sendData_(&iov, 1, send_retry_timeout_us_);
679 start_time = std::chrono::steady_clock::now();
680 while (sts == CopyStatus::kTimeout && (send_timeout_usec == 0 || TimeUtils::GetElapsedTimeMicroseconds(start_time) < send_timeout_usec) && TimeUtils::GetElapsedTimeMicroseconds(start_time) < 10000000)
682 TLOG(TLVL_DEBUG + 43) << GetTraceName() <<
"sendFragment: Timeout sending fragment";
683 sts = sendData_(&iov, 1, send_retry_timeout_us_);
691 TLOG(TLVL_DEBUG + 42) << GetTraceName() <<
"sendFragment returning " << CopyStatusToString(sts);
697 TLOG(TLVL_DEBUG + 32) << GetTraceName() <<
"sendData_ Converting buf to iovec";
698 iovec iov = {
const_cast<void*
>(buf), bytes};
699 return sendData_(&iov, 1, send_timeout_usec, isHeader);
707 if (timeoutMessageArmed_)
709 TLOG(TLVL_DEBUG + 32) << GetTraceName() <<
"sendData_: Send fd is not open. Returning kTimeout";
710 timeoutMessageArmed_ =
false;
712 return CopyStatus::kTimeout;
714 timeoutMessageArmed_ =
true;
715 TLOG(TLVL_DEBUG + 44) << GetTraceName() <<
"send_timeout_usec is " << send_timeout_usec <<
", currently unused.";
718 uint32_t total_to_write_bytes = 0;
719 std::vector<iovec> iov_in(iovcnt + 1);
720 std::vector<iovec> iovv(iovcnt + 2);
722 for (ii = 0; ii < iovcnt; ++ii)
724 iov_in[ii + 1] = iov[ii];
725 total_to_write_bytes += iov[ii].iov_len;
728 MessHead mh = {0, isHeader ? MessHead::header_v0 : MessHead::data_v0, htons(source_rank()), {htonl(total_to_write_bytes)}};
729 iov_in[0].iov_base = &mh;
730 iov_in[0].iov_len =
sizeof(mh);
731 total_to_write_bytes +=
sizeof(mh);
734 ssize_t total_written_bytes = 0;
735 ssize_t per_write_max_bytes = (32 * 1024);
737 size_t in_iov_idx = 0;
738 size_t out_iov_idx = 0;
739 ssize_t this_write_bytes = 0;
746 (in_iov_idx + out_iov_idx) < iov_in.size() && this_write_bytes < per_write_max_bytes;
749 this_write_bytes += iov_in[in_iov_idx + out_iov_idx].iov_len;
750 iovv[out_iov_idx] = iov_in[in_iov_idx + out_iov_idx];
752 if (this_write_bytes > per_write_max_bytes)
754 iovv[out_iov_idx - 1].iov_len -= this_write_bytes - per_write_max_bytes;
755 this_write_bytes = per_write_max_bytes;
760 #ifndef __OPTIMIZE__ // This can be an expensive TRACE call (even if disabled) due to multiplicity of calls
761 TLOG(TLVL_DEBUG + 44) << GetTraceName() <<
"sendFragment b4 writev " << std::setw(7) << total_written_bytes <<
" total_written_bytes send_fd_=" << send_fd_ <<
" in_idx=" << in_iov_idx
762 <<
" iovcnt=" << out_iov_idx <<
" 1st.len=" << iovv[0].iov_len;
767 memset(&msg, 0,
sizeof(msghdr));
768 msg.msg_iov = &(iovv[0]);
769 msg.msg_iovlen = out_iov_idx;
770 sts = sendmsg(send_fd_, &msg, MSG_NOSIGNAL | (blocking != 0u ? 0 : MSG_DONTWAIT));
772 sts = writev(send_fd_, &(iovv[0]), out_iov_idx);
778 if (errno == EAGAIN )
780 TLOG(TLVL_DEBUG + 32) << GetTraceName() <<
"sendFragment EWOULDBLOCK";
783 fcntl(send_fd_, F_SETFL, 0);
788 TLOG(TLVL_WARNING) << GetTraceName() <<
"sendFragment_: WRITE ERROR " << errno <<
": " << strerror(errno);
792 connection_was_lost_ =
true;
795 if (sts != this_write_bytes)
798 TLOG(TLVL_DEBUG + 32) << GetTraceName() <<
"sendFragment writev sts(" << sts <<
")!=requested_send_bytes(" << this_write_bytes <<
")";
799 total_written_bytes += sts;
801 for (ii = 0;
static_cast<size_t>(sts) >= iovv[ii].iov_len; ++ii)
803 sts -= iovv[ii].iov_len;
806 iovv[ii].iov_len -= sts;
807 iovv[ii].iov_base =
static_cast<uint8_t*
>(iovv[ii].iov_base) + sts;
813 iovv[out_iov_idx] = iovv[ii];
816 this_write_bytes = iovv[out_iov_idx].iov_len;
821 auto additional = (
reinterpret_cast<uintptr_t
>(iov_in[in_iov_idx].iov_base) + iov_in[in_iov_idx].iov_len) - (
reinterpret_cast<uintptr_t
>(iovv[out_iov_idx].iov_base) + iovv[out_iov_idx].iov_len);
822 if (additional != 0u)
824 iovv[out_iov_idx].iov_len += additional;
825 this_write_bytes += additional;
826 if (this_write_bytes > per_write_max_bytes)
828 iovv[out_iov_idx].iov_len -= this_write_bytes - per_write_max_bytes;
829 this_write_bytes = per_write_max_bytes;
833 TLOG(TLVL_DEBUG + 33) << GetTraceName() <<
"sendFragment writev sts!=: this_write_bytes=" << this_write_bytes
834 <<
" out_iov_idx=" << out_iov_idx
835 <<
" additional=" << additional
840 #ifndef __OPTIMIZE__ // This can be an expensive TRACE call (even if disabled) due to multiplicity of calls
841 TLOG(TLVL_DEBUG + 33) << GetTraceName() <<
"sendFragment writev sts(" << sts <<
")==requested_send_bytes(" << this_write_bytes <<
")";
843 total_written_bytes += sts;
845 iovv[out_iov_idx].iov_base =
static_cast<uint8_t*
>(iovv[out_iov_idx].iov_base) + iovv[out_iov_idx].iov_len;
846 iovv[out_iov_idx].iov_len = 0;
847 in_iov_idx += out_iov_idx;
848 this_write_bytes = 0;
850 auto additional = (
reinterpret_cast<uintptr_t
>(iov_in[in_iov_idx].iov_base) + iov_in[in_iov_idx].iov_len) - (
reinterpret_cast<uintptr_t
>(iovv[out_iov_idx].iov_base) + iovv[out_iov_idx].iov_len);
851 if (additional != 0u)
853 iovv[out_iov_idx].iov_len += additional;
854 this_write_bytes += additional;
855 if (this_write_bytes > per_write_max_bytes)
857 iovv[out_iov_idx].iov_len -= this_write_bytes - per_write_max_bytes;
858 this_write_bytes = per_write_max_bytes;
860 if (out_iov_idx != 0)
862 iovv[0] = iovv[out_iov_idx];
872 }
while (total_written_bytes < total_to_write_bytes);
873 if (total_written_bytes > total_to_write_bytes)
875 TLOG(TLVL_ERROR) << GetTraceName() <<
"sendFragment program error: too many bytes transferred";
881 fcntl(send_fd_, F_SETFL, O_NONBLOCK);
883 sts = total_written_bytes -
sizeof(
MessHead);
885 TLOG(TLVL_DEBUG + 44) << GetTraceName() <<
"sendFragment sts=" << sts;
889 void artdaq::TCPSocketTransfer::connect_()
891 auto start_time = std::chrono::steady_clock::now();
894 while (send_fd_ == -1 && TimeUtils::GetElapsedTimeMicroseconds(start_time) < send_retry_timeout_us_ * 10)
896 TLOG(TLVL_DEBUG + 32) << GetTraceName() <<
"Connecting sender socket";
897 int sndbuf_bytes =
static_cast<int>(sndbuf_);
898 if (sndbuf_ > INT_MAX)
900 sndbuf_bytes = INT_MAX;
901 TLOG(TLVL_WARNING) <<
"Requested SNDBUF " << sndbuf_ <<
" too large, setting to INT_MAX: " << INT_MAX;
903 TLOG(TLVL_DEBUG + 32) <<
"Requested SNDBUF is " << sndbuf_bytes;
905 send_fd_ =
TCPConnect(hostMap_[destination_rank()].c_str(), portMan->GetTCPSocketTransferPort(destination_rank()), O_NONBLOCK, sndbuf_bytes);
908 if (connection_was_lost_) {
break; }
910 usleep(send_retry_timeout_us_);
915 TLOG(TLVL_DEBUG + 32) << GetTraceName() <<
"connect_ " + hostMap_[destination_rank()] +
":" << portMan->GetTCPSocketTransferPort(destination_rank()) <<
" send_fd_=" << send_fd_;
919 TLOG(TLVL_DEBUG + 32) << GetTraceName() <<
"connect_: Writing connect message";
920 MessHead mh = {0, MessHead::connect_v0, htons(source_rank()), {htonl(CONN_MAGIC)}};
921 ssize_t sts = write(send_fd_, &mh,
sizeof(mh));
924 TLOG(TLVL_ERROR) << GetTraceName() <<
"connect_: Error writing connect message!";
932 TLOG(TLVL_INFO) << GetTraceName() <<
"connect_: Successfully connected";
935 connection_was_lost_ =
false;
939 if (ack_listen_thread_ && ack_listen_thread_->joinable()) ack_listen_thread_->join();
940 TLOG(TLVL_INFO) << GetTraceName() <<
"Starting Ack Listener Thread";
944 ack_listen_thread_ = std::make_unique<boost::thread>(&TCPSocketTransfer::receive_acks_,
this);
946 snprintf(tname,
sizeof(tname) - 1,
"%d-AckListen", my_rank);
947 tname[
sizeof(tname) - 1] =
'\0';
948 auto handle = ack_listen_thread_.native_handle();
949 pthread_setname_np(handle, tname);
951 catch (
const boost::exception& e)
953 TLOG(TLVL_ERROR) <<
"Caught boost::exception starting TCP Socket Ack Listen thread: " << boost::diagnostic_information(e) <<
", errno=" << errno;
954 std::cerr <<
"Caught boost::exception starting TCP Socket Ack Listen thread: " << boost::diagnostic_information(e) <<
", errno=" << errno << std::endl;
961 void artdaq::TCPSocketTransfer::reconnect_()
965 TLOG(TLVL_DEBUG + 33) << GetTraceName() <<
"check/reconnect";
970 void artdaq::TCPSocketTransfer::start_listen_thread_()
972 std::lock_guard<std::mutex> start_lock(listen_thread_mutex_);
973 if (listen_thread_refcount_ == 0)
975 if (listen_thread_ && listen_thread_->joinable())
977 listen_thread_->join();
979 listen_thread_refcount_ = 1;
980 TLOG(TLVL_INFO) << GetTraceName() <<
"Starting Listener Thread";
984 listen_thread_ = std::make_unique<boost::thread>(&TCPSocketTransfer::listen_, portMan->GetTCPSocketTransferPort(destination_rank()), rcvbuf_);
986 snprintf(tname,
sizeof(tname) - 1,
"%d-Listen", my_rank);
987 tname[
sizeof(tname) - 1] =
'\0';
988 auto handle = listen_thread_->native_handle();
989 pthread_setname_np(handle, tname);
991 catch (
const boost::exception& e)
993 TLOG(TLVL_ERROR) <<
"Caught boost::exception starting TCP Socket Listen thread: " << boost::diagnostic_information(e) <<
", errno=" << errno;
994 std::cerr <<
"Caught boost::exception starting TCP Socket Listen thread: " << boost::diagnostic_information(e) <<
", errno=" << errno << std::endl;
1000 listen_thread_refcount_++;
1005 void artdaq::TCPSocketTransfer::receive_acks_()
1007 while (send_fd_ >= 0)
1010 pollfd_s.events = POLLIN | POLLPRI;
1011 pollfd_s.fd = send_fd_;
1013 TLOG(TLVL_DEBUG + 48) << GetTraceName() <<
"receive_acks_: Polling fd to see if there's data";
1014 int num_fds_ready = poll(&pollfd_s, 1, 1000);
1015 if (num_fds_ready <= 0)
1017 if (num_fds_ready == 0)
1019 TLOG(TLVL_DEBUG + 48) << GetTraceName() <<
"receive_acks_: No data on receive socket";
1023 TLOG(TLVL_ERROR) <<
"Error in poll: errno=" << errno;
1027 if (pollfd_s.revents & (POLLIN | POLLPRI))
1033 TLOG(TLVL_DEBUG + 32) << GetTraceName() <<
"receive_acks_: Wrong event received from pollfd: " << pollfd_s.revents;
1038 auto sts = read(send_fd_, &mh,
sizeof(mh));
1040 if (sts !=
sizeof(mh))
1042 TLOG(TLVL_ERROR) << GetTraceName() <<
"receive_ack_: Wrong message header length received! (actual " << sts <<
" != " <<
sizeof(mh) <<
" expected)";
1050 TLOG(TLVL_ERROR) << GetTraceName() <<
"receive_ack_: Received ack for different sender! Rank=" << my_rank <<
", hdr=" << mh.
source_id;
1055 TLOG(TLVL_ERROR) << GetTraceName() <<
"receive_ack_: Wrong magic bytes in header!";
1059 TLOG(TLVL_DEBUG + 47) << GetTraceName() <<
"receive_acks_: Received ack message, diff is now " << (send_ack_diff_.load() - 1);
1064 void artdaq::TCPSocketTransfer::send_ack_(
int fd)
1066 MessHead mh = {0, MessHead::ack_v0, htons(source_rank()), {htonl(ACK_MAGIC)}};
1067 write(fd, &mh,
sizeof(mh));
1071 void artdaq::TCPSocketTransfer::listen_(
int port,
size_t rcvbuf)
1074 while (listen_thread_refcount_ > 0)
1076 TLOG(TLVL_DEBUG + 33) <<
"listen_: Listening/accepting new connections on port " << port;
1077 if (listen_fd == -1)
1079 TLOG(TLVL_DEBUG + 32) <<
"listen_: Opening listener";
1082 if (listen_fd == -1)
1084 TLOG(TLVL_DEBUG + 32) <<
"listen_: Error creating listen_fd!";
1089 timeval tv = {2, 0};
1092 FD_SET(listen_fd, &rfds);
1094 res = select(listen_fd + 1, &rfds, static_cast<fd_set*>(
nullptr), static_cast<fd_set*>(
nullptr), &tv);
1099 socklen_t arglen =
sizeof(un);
1101 TLOG(TLVL_DEBUG + 32) <<
"listen_: Calling accept";
1102 fd = accept(listen_fd, reinterpret_cast<sockaddr*>(&un), &arglen);
1103 TLOG(TLVL_DEBUG + 32) <<
"listen_: Done with accept";
1105 TLOG(TLVL_DEBUG + 32) <<
"listen_: Reading connect message";
1106 socklen_t lenlen =
sizeof(tv);
1108 setsockopt(fd, SOL_SOCKET, SO_RCVTIMEO, &tv, lenlen);
1110 uint64_t mark_us = TimeUtils::gettimeofday_us();
1111 sts = read(fd, &mh,
sizeof(mh));
1112 uint64_t delta_us = TimeUtils::gettimeofday_us() - mark_us;
1113 TLOG(TLVL_DEBUG + 32) <<
"listen_: Read of connect message took " << delta_us <<
" microseconds.";
1114 if (sts !=
sizeof(mh))
1116 TLOG(TLVL_DEBUG + 32) <<
"listen_: Wrong message header length received!";
1125 TLOG(TLVL_DEBUG + 32) <<
"listen_: Wrong magic bytes in header!";
1131 std::lock_guard<std::mutex> lk(fd_mutex_);
1132 connected_fds_[mh.
source_id].insert(fd);
1134 TLOG(TLVL_INFO) <<
"listen_: New fd is " << fd <<
" for source rank " << mh.
source_id;
1138 TLOG(TLVL_DEBUG + 46) <<
"listen_: No connections in timeout interval!";
1142 TLOG(TLVL_INFO) <<
"listen_: Shutting down connection listener";
1143 if (listen_fd != -1)
1147 std::lock_guard<std::mutex> lk(fd_mutex_);
1148 auto it = connected_fds_.begin();
1149 while (it != connected_fds_.end())
1151 auto& fd_set = it->second;
1152 auto rank_it = fd_set.begin();
1153 while (rank_it != fd_set.end())
1156 rank_it = fd_set.erase(rank_it);
1158 it = connected_fds_.erase(it);
1163 size_t artdaq::TCPSocketTransfer::getConnectedFDCount_(
int source_rank)
1165 std::lock_guard<std::mutex> lk(fd_mutex_);
1166 #ifndef __OPTIMIZE__
1167 TLOG(TLVL_DEBUG + 45) << GetTraceName() <<
"getConnectedFDCount_: count is " << (connected_fds_.count(source_rank) != 0u ? connected_fds_[source_rank].size() : 0);
1169 return connected_fds_.count(source_rank) != 0u ? connected_fds_[source_rank].size() : 0;
1172 int artdaq::TCPSocketTransfer::getActiveFD_(
int source_rank)
1174 std::lock_guard<std::mutex> lk(fd_mutex_);
1175 #ifndef __OPTIMIZE__
1176 TLOG(TLVL_DEBUG + 45) << GetTraceName() <<
"getActiveFD_: fd is " << (active_receive_fds_.count(source_rank) != 0u ? active_receive_fds_[source_rank] : -1);
1178 return active_receive_fds_.count(source_rank) != 0u ? active_receive_fds_[source_rank] : -1;
1180 void artdaq::TCPSocketTransfer::setActiveFD_(
int source_rank,
int fd)
1182 std::lock_guard<std::mutex> lk(fd_mutex_);
1183 #ifndef __OPTIMIZE__
1184 TLOG(TLVL_DEBUG + 45) << GetTraceName() <<
"setActiveFD_: setting active fd for rank " << source_rank <<
" to " << fd;
1186 active_receive_fds_[source_rank] = fd;
1188 int artdaq::TCPSocketTransfer::getLastActiveFD_(
int source_rank)
1190 std::lock_guard<std::mutex> lk(fd_mutex_);
1191 #ifndef __OPTIMIZE__
1192 TLOG(TLVL_DEBUG + 45) << GetTraceName() <<
"getLastActiveFD_: fd is " << (last_active_receive_fds_.count(source_rank) != 0u ? last_active_receive_fds_[source_rank] : -1);
1194 return last_active_receive_fds_.count(source_rank) != 0u ? last_active_receive_fds_[source_rank] : -1;
1196 void artdaq::TCPSocketTransfer::setLastActiveFD_(
int source_rank,
int fd)
1198 std::lock_guard<std::mutex> lk(fd_mutex_);
1199 #ifndef __OPTIMIZE__
1200 TLOG(TLVL_DEBUG + 45) << GetTraceName() <<
"setLastActiveFD_: setting last active fd for rank " << source_rank <<
" to " << fd;
1202 last_active_receive_fds_[source_rank] = fd;
bool isRunning() override
Determine whether the TransferInterface plugin is able to send/receive data.
virtual int source_rank() const
Get the source rank for this TransferInterface instance.
uint32_t conn_magic
unsigned first is better for MessHead initializer: {0,0,my_node_idx_,CONN_MAGIC}
int receiveFragmentHeader(detail::RawFragmentHeader &header, size_t timeout_usec) override
Receive a Fragment Header from the transport mechanism.
int TCPConnect(char const *host_in, int dflt_port, int64_t flags=0, int sndbufsiz=0)
Connect to a host on a given port.
This TransferInterface is a Receiver.
int receiveFragmentData(RawDataType *destination, size_t wordCount) override
Receive the body of a Fragment to the given destination pointer.
int TCP_listen_fd(int port, int rcvbuf)
Create a TCP listening socket on the given port and INADDR_ANY, with the given receive buffer...
TCPSocketTransfer(fhicl::ParameterSet const &ps, Role role)
TCPSocketTransfer Constructor.
This TransferInterface is a Sender.
void flush_buffers() override
Flush any in-flight data. This should be used by the receiver after the receive loop has ended...
int32_t byte_count
use CONN_MAGIC for connect_v0, data that follow for data_v0 (and 0 lenght data)
Some error occurred, but no exception was thrown.
Role
Used to determine if a TransferInterface is a Sender or Receiver.
int64_t source_id
Rank of the source.
MessType message_type
Message Type.
The send operation completed successfully.
This interface defines the functions used to transfer data between artdaq applications.
This header is sent by the TCPSocket_transfer to allow for more efficient writev calls.
hostMap_t MakeHostMap(fhicl::ParameterSet const &pset, hostMap_t map=hostMap_t())
Make a hostMap_t from a HostMap::Config ParameterSet
CopyStatus
Returned from the send functions, this enumeration describes the possible return codes. If an exception occurs, it will be thrown and should be handled normally.