9 #include <sys/socket.h>
11 #include <arpa/inet.h>
12 #include <sys/types.h>
21 #define TRACE_NAME (app_name + "_TCPSocketTransfer").c_str()
22 #include "artdaq/DAQdata/Globals.hh"
25 #include "artdaq/TransferPlugins/TCPSocketTransfer.hh"
28 #include "artdaq/TransferPlugins/detail/Timeout.hh"
29 #include "artdaq/TransferPlugins/detail/SRSockets.hh"
30 #include "artdaq-core/Data/Fragment.hh"
31 #include "artdaq-core/Utilities/TimeUtils.hh"
34 std::atomic<int> artdaq::TCPSocketTransfer::listen_thread_refcount_(0);
35 std::unique_ptr<boost::thread> artdaq::TCPSocketTransfer::listen_thread_ =
nullptr;
36 std::map<int, std::set<int>> artdaq::TCPSocketTransfer::connected_fds_ = std::map<int, std::set<int>>();
37 std::mutex artdaq::TCPSocketTransfer::listen_thread_mutex_;
38 std::mutex artdaq::TCPSocketTransfer::connected_fd_mutex_;
44 , active_receive_fd_(-1)
45 , last_active_receive_fd_(-1)
46 , rcvbuf_(pset.get<size_t>(
"tcp_receive_buffer_size", 0))
47 , sndbuf_(pset.get<size_t>(
"tcp_send_buffer_size", max_fragment_size_words_ * sizeof(artdaq::RawDataType) * buffer_count_))
48 , send_retry_timeout_us_(pset.get<size_t>(
"send_retry_timeout_us", 1000000))
49 , timeoutMessageArmed_(true)
51 , receive_disconnected_wait_s_(pset.get<double>(
"receive_socket_disconnected_wait_s", 10.0))
52 , receive_err_wait_us_(pset.get<size_t>(
"receive_socket_disconnected_wait_us", 10000))
53 , receive_socket_has_been_connected_(false)
56 TLOG(TLVL_DEBUG) << GetTraceName() <<
" Constructor: pset=" << pset.to_string() <<
", role=" << (role ==
TransferInterface::Role::kReceive ?
"kReceive" :
"kSend");
57 connection_was_lost_ =
false;
62 TLOG(TLVL_DEBUG) << GetTraceName() <<
": Listening for connections";
63 start_listen_thread_();
64 TLOG(TLVL_DEBUG) << GetTraceName() <<
": Done Listening";
69 TLOG(TLVL_DEBUG) << GetTraceName() <<
": Connecting to destination";
71 TLOG(TLVL_DEBUG) << GetTraceName() <<
": Done Connecting";
73 TLOG(TLVL_DEBUG) << GetTraceName() <<
": End of Constructor";
76 artdaq::TCPSocketTransfer::~TCPSocketTransfer() noexcept
78 TLOG(TLVL_DEBUG) << GetTraceName() <<
": Shutting down TCPSocketTransfer";
87 timeval tv = { 0,100000 };
88 socklen_t len =
sizeof(tv);
89 setsockopt(send_fd_, SOL_SOCKET, SO_SNDTIMEO, &tv, len);
90 write(send_fd_, &mh,
sizeof(mh));
98 std::unique_lock<std::mutex> fd_lock(connected_fd_mutex_);
99 if (connected_fds_.count(source_rank()))
101 auto it = connected_fds_[source_rank()].begin();
102 while (it != connected_fds_[source_rank()].end())
105 it = connected_fds_[source_rank()].erase(it);
107 connected_fds_.erase(source_rank());
109 if (ack_listen_thread_ && ack_listen_thread_->joinable()) ack_listen_thread_->join();
112 std::unique_lock<std::mutex> lk(listen_thread_mutex_);
113 listen_thread_refcount_--;
114 if (listen_thread_refcount_ <= 0 && listen_thread_ && listen_thread_->joinable())
116 listen_thread_->join();
120 TLOG(TLVL_DEBUG) << GetTraceName() <<
": End of Destructor";
125 TLOG(5) << GetTraceName() <<
": receiveFragmentHeader: BEGIN";
126 int ret_rank = RECV_TIMEOUT;
129 if (getConnectedFDCount(source_rank()) == 0)
138 TLOG(7) << GetTraceName() <<
": receiveFragmentHeader: Receive socket not connected, returning RECV_TIMEOUT";
139 usleep(receive_err_wait_us_);
142 receive_socket_has_been_connected_ =
true;
143 last_recv_time_ = std::chrono::steady_clock::now();
145 TLOG(5) << GetTraceName() <<
": receiveFragmentHeader timeout_usec=" << timeout_usec;
150 SocketState state = SocketState::Metadata;
151 int target_bytes =
sizeof(
MessHead);
152 uint64_t start_time_us = TimeUtils::gettimeofday_us();
164 if (timeout_usec == 0)
167 timeout_ms = (timeout_usec + 999) / 1000;
170 bool noDataWarningSent =
false;
173 while (!done && getConnectedFDCount(source_rank()) > 0)
175 if (active_receive_fd_ == -1)
179 std::vector<pollfd> pollfds;
181 std::unique_lock<std::mutex> lk(connected_fd_mutex_);
182 fd_count = connected_fds_[source_rank()].size();
183 pollfds.resize(fd_count);
184 auto iter = connected_fds_[source_rank()].begin();
185 for (
size_t ii = 0; ii < fd_count; ++ii)
187 pollfds[ii].events = POLLIN | POLLPRI | POLLERR;
188 pollfds[ii].fd = *iter;
193 int num_fds_ready = poll(&pollfds[0], fd_count, timeout_ms);
194 if (num_fds_ready <= 0)
196 TLOG(5) << GetTraceName() <<
": receiveFragmentHeader: No data on receive socket, returning RECV_TIMEOUT";
201 if (last_active_receive_fd_ != -1)
203 for (
auto& pollfd : pollfds)
206 if (pollfd.fd == last_active_receive_fd_)
213 int active_index = -1;
214 short anomolous_events = 0;
215 for (
size_t ii = index; ii < index + pollfds.size(); ++ii)
217 auto pollfd_index = (ii + index) % pollfds.size();
218 if (pollfds[pollfd_index].revents & (POLLIN | POLLPRI))
220 active_index = pollfd_index;
221 active_receive_fd_ = pollfds[active_index].fd;
222 active_revents_ = pollfds[active_index].revents;;
225 else if (pollfds[pollfd_index].revents & (POLLHUP | POLLERR))
227 disconnect_receive_socket_(pollfds[pollfd_index].fd,
"Poll returned POLLHUP or POLLERR, indicating problems with the sender.");
230 else if (pollfds[pollfd_index].revents & (POLLNVAL))
232 TLOG(TLVL_DEBUG) << GetTraceName() <<
": receiveFragmentHeader: FD is closed, most likely because the peer went away. Removing from fd list.";
233 disconnect_receive_socket_(pollfds[pollfd_index].fd,
"FD is closed, most likely because the peer went away.");
236 else if (pollfds[pollfd_index].revents)
238 anomolous_events |= pollfds[pollfd_index].revents;
242 if (active_index == -1)
244 if (anomolous_events)
245 TLOG(TLVL_DEBUG) << GetTraceName() <<
": receiveFragmentHeader: Wrong event received from a pollfd. Mask: " <<
static_cast<int>(anomolous_events);
246 active_receive_fd_ = -1;
250 if (!done && timeout_usec > 0)
253 size_t delta_us = TimeUtils::gettimeofday_us() - start_time_us;
254 if (delta_us > timeout_usec)
258 timeout_ms = ((timeout_usec - delta_us) + 999) / 1000;
261 if (loop_guard > 10) { usleep(1000); }
262 if (++loop_guard > 10010)
264 TLOG(TLVL_WARNING) << GetTraceName() <<
": receiveFragmentHeader: loop guard triggered, returning RECV_TIMEOUT";
265 usleep(receive_err_wait_us_);
266 active_receive_fd_ = -1;
270 if (state == SocketState::Metadata)
273 buff = &(mha[offset]);
274 byte_cnt =
sizeof(
MessHead) - offset;
279 buff =
reinterpret_cast<uint8_t*
>(&header) + offset;
280 byte_cnt = target_bytes - offset;
292 TLOG(6) << GetTraceName() <<
": receiveFragmentHeader: Reading " << byte_cnt <<
" bytes from socket";
293 sts = read(active_receive_fd_, buff, byte_cnt);
294 TLOG(6) << GetTraceName() <<
": receiveFragmentHeader: Done with read";
298 last_recv_time_ = std::chrono::steady_clock::now();
301 TLOG(7) << GetTraceName() <<
": receiveFragmentHeader state=" <<
static_cast<int>(state) <<
" read=" << sts;
304 TLOG(TLVL_WARNING) << GetTraceName() <<
": receiveFragmentHeader: Error on receive, closing socket " <<
" (errno=" << errno <<
": " << strerror(errno) <<
")";
305 active_receive_fd_ = disconnect_receive_socket_(active_receive_fd_);
309 if (!noDataWarningSent) {
310 TLOG(TLVL_WARNING) << GetTraceName() <<
": receiveFragmentHeader: No data received, is the sender still sending?!?";
311 noDataWarningSent =
true;
313 if (TimeUtils::GetElapsedTime(last_recv_time_) > receive_disconnected_wait_s_)
315 TLOG(TLVL_ERROR) << GetTraceName() <<
": receiveFragmentHeader: No data received within timeout, aborting!";
323 if (sts >= target_bytes)
325 TLOG(7) << GetTraceName() <<
": receiveFragmentHeader: Target read bytes reached. Changing state";
327 if (state == SocketState::Metadata)
329 state = SocketState::Data;
333 TLOG(7) << GetTraceName() <<
": receiveFragmentHeader: Expected header size = " << target_bytes <<
", sizeof(RawFragmentHeader) = " <<
sizeof(artdaq::detail::RawFragmentHeader);
338 active_receive_fd_ = disconnect_receive_socket_(active_receive_fd_,
"Stop Message received.");
342 TLOG(TLVL_WARNING) << GetTraceName() <<
": receiveFragmentHeader: Message header indicates that Fragment data follows when I was expecting a Fragment header!";
343 active_receive_fd_ = disconnect_receive_socket_(active_receive_fd_,
"Desync detected");
346 if (target_bytes == 0)
354 ret_rank = source_rank();
355 TLOG(8) << GetTraceName() <<
": receiveFragmentHeader done sts=" << sts <<
" src=" << ret_rank;
356 TLOG(7) << GetTraceName() <<
": receiveFragmentHeader: Done receiving fragment header. Moving into output.";
366 TLOG(5) << GetTraceName() <<
": receiveFragmentHeader: Returning " << ret_rank;
370 int artdaq::TCPSocketTransfer::disconnect_receive_socket_(
int fd, std::string msg)
372 TLOG(TLVL_WARNING) << GetTraceName() <<
": disconnect_receive_socket_: " << msg <<
" Closing socket " << fd;
374 std::unique_lock<std::mutex> lk(connected_fd_mutex_);
375 if (connected_fds_.count(source_rank()))
376 connected_fds_[source_rank()].erase(fd);
378 TLOG(TLVL_DEBUG) << GetTraceName() <<
": disconnect_receive_socket_: There are now " << connected_fds_[source_rank()].size() <<
" active senders.";
384 TLOG(19) << GetTraceName() <<
": receiveFragmentData: BEGIN";
385 int ret_rank = RECV_TIMEOUT;
386 if (active_receive_fd_ == -1)
388 TLOG(TLVL_ERROR) << GetTraceName() <<
": receiveFragmentData: Receive socket not connected, returning RECV_TIMEOUT (Will result in \"Unexpected return code error\")";
397 SocketState state = SocketState::Metadata;
398 int target_bytes =
sizeof(
MessHead);
401 pollfd_s.events = POLLIN | POLLPRI | POLLERR;
402 pollfd_s.fd = active_receive_fd_;
406 bool noDataWarningSent =
false;
407 last_recv_time_ = std::chrono::steady_clock::now();
410 TLOG(9) << GetTraceName() <<
": receiveFragmentData: Polling fd to see if there's data";
411 int num_fds_ready = poll(&pollfd_s, 1, 1000);
412 TLOG(TLVL_TRACE) << GetTraceName() <<
": receiveFragmentData: Polled fd to see if there's data"
413 <<
", num_fds_ready = " << num_fds_ready;
414 if (num_fds_ready <= 0)
416 if (num_fds_ready == 0)
418 TLOG(TLVL_WARNING) << GetTraceName() <<
": receiveFragmentData: No data from " << source_rank() <<
" in " << TimeUtils::GetElapsedTimeMilliseconds(last_recv_time_) <<
" ms!"
419 <<
" State = " << (state == SocketState::Metadata ?
"Metadata" :
"Data") <<
", recvd/total=" << offset <<
"/" << target_bytes <<
" (delta=" << target_bytes - offset <<
")";
421 if (TimeUtils::GetElapsedTime(last_recv_time_) > receive_disconnected_wait_s_)
423 TLOG(TLVL_WARNING) << GetTraceName() <<
": receiveFragmentData: No data received within timeout (" << TimeUtils::GetElapsedTime(last_recv_time_) <<
" / " << receive_disconnected_wait_s_ <<
" ), returning RECV_TIMEOUT";
424 disconnect_receive_socket_(active_receive_fd_,
"No data on this socket within timeout");
425 active_receive_fd_ = -1;
431 TLOG(TLVL_ERROR) <<
"Error in poll: errno=" << errno;
432 active_receive_fd_ = -1;
435 else { last_recv_time_ = std::chrono::steady_clock::now(); }
437 if (pollfd_s.revents & (POLLIN | POLLPRI))
441 else if (pollfd_s.revents & (POLLNVAL))
443 disconnect_receive_socket_(pollfd_s.fd,
"FD is closed, most likely because the peer went away.");
446 else if (pollfd_s.revents & (POLLHUP | POLLERR))
448 disconnect_receive_socket_(pollfd_s.fd,
"Poll returned POLLHUP or POLLERR, indicating problems with the sender.");
453 TLOG(TLVL_WARNING) << GetTraceName() <<
": receiveFragmentData: Wrong event received from pollfd: " << pollfd_s.revents;
454 disconnect_receive_socket_(pollfd_s.fd);
458 if (state == SocketState::Metadata)
461 buff = &(mha[offset]);
462 byte_cnt =
sizeof(
MessHead) - offset;
467 buff =
reinterpret_cast<uint8_t*
>(destination) + offset;
471 TLOG(10) << GetTraceName() <<
": receiveFragmentData: Reading " << byte_cnt <<
" bytes from socket into " << (
void*)buff;
472 sts = read(active_receive_fd_, buff, byte_cnt);
475 TLOG(10) << GetTraceName() <<
": recvFragment state=" <<
static_cast<int>(state) <<
" read=" << sts;
479 if (loop_guard > 10) { usleep(1000); }
480 if (++loop_guard > 10010)
482 TLOG(TLVL_WARNING) << GetTraceName() <<
": receiveFragmentData: loop guard triggered, returning RECV_TIMEOUT";
483 active_receive_fd_ = -1;
490 last_recv_time_ = std::chrono::steady_clock::now();
495 TLOG(TLVL_WARNING) << GetTraceName() <<
": receiveFragmentData: Error on receive, closing socket"
496 <<
" (errno=" << errno <<
": " << strerror(errno) <<
")";
497 disconnect_receive_socket_(pollfd_s.fd);
501 if (!noDataWarningSent) {
502 TLOG(TLVL_WARNING) << GetTraceName() <<
": receiveFragmentData: No data received, is the sender still sending?!?";
503 noDataWarningSent =
true;
505 if (TimeUtils::GetElapsedTime(last_recv_time_) > receive_disconnected_wait_s_)
507 TLOG(TLVL_ERROR) << GetTraceName() <<
": receiveFragmentData: No data received within timeout, aborting!";
515 if (sts >= target_bytes)
517 TLOG(9) << GetTraceName() <<
": receiveFragmentData: Target read bytes reached. Changing state";
519 if (state == SocketState::Metadata)
521 state = SocketState::Data;
529 TLOG(TLVL_WARNING) << GetTraceName() <<
": receiveFragmentData: Message header indicates that a Fragment header follows when I was expecting Fragment data!";
530 active_receive_fd_ = disconnect_receive_socket_(active_receive_fd_,
"Desync detected");
535 ret_rank = source_rank();
536 TLOG(11) << GetTraceName() <<
": receiveFragmentData done sts=" << sts <<
" src=" << ret_rank;
537 TLOG(9) << GetTraceName() <<
": receiveFragmentData: Done receiving fragment. Moving into output.";
540 send_ack_(active_receive_fd_);
550 if (target_bytes == 0 && state == SocketState::Data)
552 ret_rank = source_rank();
553 TLOG(11) << GetTraceName() <<
": receiveFragmentData done sts=" << sts <<
" src=" << ret_rank;
554 TLOG(9) << GetTraceName() <<
": receiveFragmentData: Done receiving fragment. Moving into output.";
557 send_ack_(active_receive_fd_);
565 last_active_receive_fd_ = active_receive_fd_;
566 active_receive_fd_ = -1;
568 TLOG(9) << GetTraceName() <<
": receiveFragmentData: Returning rank " << ret_rank;
577 return send_fd_ != -1;
579 TLOG(TLVL_DEBUG) << GetTraceName() <<
": isRunning: There are " << getConnectedFDCount(source_rank()) <<
" fds connected.";
580 return getConnectedFDCount(source_rank()) > 0;
587 while(connected_fds_[source_rank()].size()) {
588 disconnect_receive_socket_(*connected_fds_[source_rank()].begin(),
"Flushing connections");
596 TLOG(12) << GetTraceName() <<
": sendFragment begin send of fragment with sequenceID="<<frag.sequenceID();
597 artdaq::Fragment grab_ownership_frag = std::move(frag);
600 if (send_fd_ == -1 && connection_was_lost_)
602 TLOG(TLVL_INFO) << GetTraceName() <<
": reconnection attempt failed, returning quickly.";
610 while (static_cast<size_t>(send_ack_diff_) > buffer_count_) usleep(10000);
613 iovec iov = {
reinterpret_cast<void*
>(grab_ownership_frag.headerAddress()),
614 detail::RawFragmentHeader::num_words() *
sizeof(RawDataType) };
616 auto sts = sendData_(&iov, 1, send_retry_timeout_us_,
true);
617 auto start_time = std::chrono::steady_clock::now();
619 while (sts == CopyStatus::kTimeout && (send_timeout_usec == 0 || TimeUtils::GetElapsedTimeMicroseconds(start_time) < send_timeout_usec) && TimeUtils::GetElapsedTimeMicroseconds(start_time) < 10000000)
621 TLOG(13) << GetTraceName() <<
": sendFragment: Timeout sending fragment";
622 sts = sendData_(&iov, 1, send_retry_timeout_us_,
true);
625 if (sts != CopyStatus::kSuccess)
return sts;
629 iov = {
reinterpret_cast<void*
>(grab_ownership_frag.headerAddress() + detail::RawFragmentHeader::num_words()),
630 grab_ownership_frag.sizeBytes() - detail::RawFragmentHeader::num_words() *
sizeof(RawDataType)
632 sts = sendData_(&iov, 1, send_retry_timeout_us_);
633 start_time = std::chrono::steady_clock::now();
634 while (sts == CopyStatus::kTimeout && (send_timeout_usec == 0 || TimeUtils::GetElapsedTimeMicroseconds(start_time) < send_timeout_usec) && TimeUtils::GetElapsedTimeMicroseconds(start_time) < 10000000)
636 TLOG(13) << GetTraceName() <<
": sendFragment: Timeout sending fragment";
637 sts = sendData_(&iov, 1, send_retry_timeout_us_);
645 TLOG(12) << GetTraceName() <<
": sendFragment returning " << CopyStatusToString(sts);
651 TLOG(TLVL_DEBUG) << GetTraceName() <<
": sendData_ Converting buf to iovec";
652 iovec iov = { (
void*)buf, bytes };
653 return sendData_(&iov, 1, send_timeout_usec, isHeader);
661 if (timeoutMessageArmed_)
663 TLOG(TLVL_DEBUG) << GetTraceName() <<
": sendData_: Send fd is not open. Returning kTimeout";
664 timeoutMessageArmed_ =
false;
666 return CopyStatus::kTimeout;
668 timeoutMessageArmed_ =
true;
669 TLOG(14) << GetTraceName() <<
": send_timeout_usec is " << send_timeout_usec <<
", currently unused.";
672 uint32_t total_to_write_bytes = 0;
673 std::vector<iovec> iov_in(iovcnt + 1);
674 std::vector<iovec> iovv(iovcnt + 2);
676 for (ii = 0; ii < iovcnt; ++ii)
678 iov_in[ii + 1] = iov[ii];
679 total_to_write_bytes += iov[ii].iov_len;
682 MessHead mh = { 0,isHeader ? MessHead::header_v0 : MessHead::data_v0,htons(source_rank()),{htonl(total_to_write_bytes)} };
683 iov_in[0].iov_base = &mh;
684 iov_in[0].iov_len =
sizeof(mh);
685 total_to_write_bytes +=
sizeof(mh);
688 ssize_t total_written_bytes = 0;
689 ssize_t per_write_max_bytes = (32 * 1024);
691 size_t in_iov_idx = 0;
692 size_t out_iov_idx = 0;
693 ssize_t this_write_bytes = 0;
700 (in_iov_idx + out_iov_idx) < iov_in.size() && this_write_bytes < per_write_max_bytes;
703 this_write_bytes += iov_in[in_iov_idx + out_iov_idx].iov_len;
704 iovv[out_iov_idx] = iov_in[in_iov_idx + out_iov_idx];
706 if (this_write_bytes > per_write_max_bytes)
708 iovv[out_iov_idx - 1].iov_len -= this_write_bytes - per_write_max_bytes;
709 this_write_bytes = per_write_max_bytes;
714 #ifndef __OPTIMIZE__ // This can be an expensive TRACE call (even if disabled) due to multiplicity of calls
715 TLOG(14) << GetTraceName() <<
": sendFragment b4 writev " << std::setw(7) << total_written_bytes <<
" total_written_bytes send_fd_=" << send_fd_ <<
" in_idx=" << in_iov_idx
716 <<
" iovcnt=" << out_iov_idx <<
" 1st.len=" << iovv[0].iov_len;
719 sts = writev(send_fd_, &(iovv[0]), out_iov_idx);
724 if (errno == EAGAIN )
726 TLOG(TLVL_DEBUG) << GetTraceName() <<
": sendFragment EWOULDBLOCK";
727 fcntl(send_fd_, F_SETFL, 0);
732 TLOG(TLVL_WARNING) << GetTraceName() <<
": sendFragment_: WRITE ERROR: " << strerror(errno);
736 connection_was_lost_ =
true;
739 else if (sts != this_write_bytes)
742 TLOG(TLVL_DEBUG) << GetTraceName() <<
": sendFragment writev sts(" << sts <<
")!=requested_send_bytes(" << this_write_bytes <<
")";
743 total_written_bytes += sts;
745 for (ii = 0; (size_t)sts >= iovv[ii].iov_len; ++ii)
746 sts -= iovv[ii].iov_len;
748 iovv[ii].iov_len -= sts;
749 iovv[ii].iov_base = (uint8_t*)(iovv[ii].iov_base) + sts;
754 iovv[out_iov_idx] = iovv[ii];
756 this_write_bytes = iovv[out_iov_idx].iov_len;
761 unsigned long additional = ((
unsigned long)iov_in[in_iov_idx].iov_base + iov_in[in_iov_idx].iov_len)
762 - ((
unsigned long)iovv[out_iov_idx].iov_base + iovv[out_iov_idx].iov_len);
765 iovv[out_iov_idx].iov_len += additional;
766 this_write_bytes += additional;
767 if (this_write_bytes > per_write_max_bytes)
769 iovv[out_iov_idx].iov_len -= this_write_bytes - per_write_max_bytes;
770 this_write_bytes = per_write_max_bytes;
774 TLOG(TLVL_TRACE) << GetTraceName() <<
": sendFragment writev sts!=: this_write_bytes=" << this_write_bytes
775 <<
" out_iov_idx=" << out_iov_idx
776 <<
" additional=" << additional
781 #ifndef __OPTIMIZE__ // This can be an expensive TRACE call (even if disabled) due to multiplicity of calls
782 TLOG(TLVL_TRACE) << GetTraceName() <<
": sendFragment writev sts(" << sts <<
")==requested_send_bytes(" << this_write_bytes <<
")";
784 total_written_bytes += sts;
786 iovv[out_iov_idx].iov_base = (uint8_t*)(iovv[out_iov_idx].iov_base) + iovv[out_iov_idx].iov_len;
787 iovv[out_iov_idx].iov_len = 0;
788 in_iov_idx += out_iov_idx;
789 this_write_bytes = 0;
791 unsigned long additional = ((
unsigned long)iov_in[in_iov_idx].iov_base + iov_in[in_iov_idx].iov_len)
792 - ((
unsigned long)iovv[out_iov_idx].iov_base + iovv[out_iov_idx].iov_len);
795 iovv[out_iov_idx].iov_len += additional;
796 this_write_bytes += additional;
797 if (this_write_bytes > per_write_max_bytes)
799 iovv[out_iov_idx].iov_len -= this_write_bytes - per_write_max_bytes;
800 this_write_bytes = per_write_max_bytes;
802 if (out_iov_idx != 0)
803 iovv[0] = iovv[out_iov_idx];
812 }
while (total_written_bytes < total_to_write_bytes);
813 if (total_written_bytes > total_to_write_bytes)
814 TLOG(TLVL_ERROR) << GetTraceName() <<
": sendFragment program error: too many bytes transferred";
819 fcntl(send_fd_, F_SETFL, O_NONBLOCK);
821 sts = total_written_bytes -
sizeof(
MessHead);
823 TLOG(14) << GetTraceName() <<
": sendFragment sts=" << sts;
827 void artdaq::TCPSocketTransfer::connect_()
829 auto start_time = std::chrono::steady_clock::now();
832 while (send_fd_ == -1 && TimeUtils::GetElapsedTimeMicroseconds(start_time) < send_retry_timeout_us_ * 10)
834 TLOG(TLVL_DEBUG) << GetTraceName() <<
": Connecting sender socket";
835 int sndbuf_bytes =
static_cast<int>(sndbuf_);
836 if (sndbuf_ > INT_MAX)
838 sndbuf_bytes = INT_MAX;
839 TLOG(TLVL_WARNING) <<
"Requested SNDBUF " << sndbuf_ <<
" too large, setting to INT_MAX: " << INT_MAX;
841 TLOG(TLVL_DEBUG) <<
"Requested SNDBUF is " << sndbuf_bytes;
843 send_fd_ =
TCPConnect(hostMap_[destination_rank()].c_str()
844 , portMan->GetTCPSocketTransferPort(destination_rank())
849 if (connection_was_lost_) {
break;}
850 else {usleep(send_retry_timeout_us_);}
855 TLOG(TLVL_DEBUG) << GetTraceName() <<
": connect_ " + hostMap_[destination_rank()] +
":" << portMan->GetTCPSocketTransferPort(destination_rank()) <<
" send_fd_=" << send_fd_;
859 TLOG(TLVL_DEBUG) << GetTraceName() <<
": connect_: Writing connect message";
860 MessHead mh = { 0,MessHead::connect_v0,htons(source_rank()),{htonl(CONN_MAGIC)} };
861 ssize_t sts = write(send_fd_, &mh,
sizeof(mh));
864 TLOG(TLVL_ERROR) << GetTraceName() <<
": connect_: Error writing connect message!";
872 TLOG(TLVL_INFO) << GetTraceName() <<
": connect_: Successfully connected";
875 connection_was_lost_ =
false;
879 if (ack_listen_thread_ && ack_listen_thread_->joinable()) ack_listen_thread_->join();
880 TLOG(TLVL_INFO) << GetTraceName() <<
": Starting Ack Listener Thread";
883 ack_listen_thread_ = std::make_unique<boost::thread>(&TCPSocketTransfer::receive_acks_,
this);
885 catch (
const boost::exception& e)
887 TLOG(TLVL_ERROR) <<
"Caught boost::exception starting TCP Socket Ack Listen thread: " << boost::diagnostic_information(e) <<
", errno=" << errno;
888 std::cerr <<
"Caught boost::exception starting TCP Socket Ack Listen thread: " << boost::diagnostic_information(e) <<
", errno=" << errno << std::endl;
895 void artdaq::TCPSocketTransfer::reconnect_()
899 TLOG(TLVL_TRACE) << GetTraceName() <<
": check/reconnect";
904 void artdaq::TCPSocketTransfer::start_listen_thread_()
906 std::unique_lock<std::mutex> start_lock(listen_thread_mutex_);
907 if (listen_thread_refcount_ == 0)
909 if (listen_thread_ && listen_thread_->joinable()) listen_thread_->join();
910 listen_thread_refcount_ = 1;
911 TLOG(TLVL_INFO) << GetTraceName() <<
": Starting Listener Thread";
914 listen_thread_ = std::make_unique<boost::thread>(&TCPSocketTransfer::listen_, portMan->GetTCPSocketTransferPort(destination_rank()), rcvbuf_);
916 catch (
const boost::exception& e)
918 TLOG(TLVL_ERROR) <<
"Caught boost::exception starting TCP Socket Listen thread: " << boost::diagnostic_information(e) <<
", errno=" << errno;
919 std::cerr <<
"Caught boost::exception starting TCP Socket Listen thread: " << boost::diagnostic_information(e) <<
", errno=" << errno << std::endl;
925 listen_thread_refcount_++;
930 void artdaq::TCPSocketTransfer::receive_acks_()
932 while (send_fd_ >= 0)
935 pollfd_s.events = POLLIN | POLLPRI;
936 pollfd_s.fd = send_fd_;
938 TLOG(18) << GetTraceName() <<
": receive_acks_: Polling fd to see if there's data";
939 int num_fds_ready = poll(&pollfd_s, 1, 1000);
940 if (num_fds_ready <= 0)
942 if (num_fds_ready == 0)
944 TLOG(18) << GetTraceName() <<
": receive_acks_: No data on receive socket";
948 TLOG(TLVL_ERROR) <<
"Error in poll: errno=" << errno;
952 if (pollfd_s.revents & (POLLIN | POLLPRI))
958 TLOG(TLVL_DEBUG) << GetTraceName() <<
": receive_acks_: Wrong event received from pollfd: " << pollfd_s.revents;
963 auto sts = read(send_fd_, &mh,
sizeof(mh));
965 if (sts !=
sizeof(mh))
967 TLOG(TLVL_ERROR) << GetTraceName() <<
": receive_ack_: Wrong message header length received! (actual " << sts <<
" != " <<
sizeof(mh) <<
" expected)";
975 TLOG(TLVL_ERROR) << GetTraceName() <<
": receive_ack_: Received ack for different sender! Rank=" << my_rank <<
", hdr=" << mh.
source_id;
980 TLOG(TLVL_ERROR) << GetTraceName() <<
": receive_ack_: Wrong magic bytes in header!";
984 TLOG(17) << GetTraceName() <<
": receive_acks_: Received ack message, diff is now " << (send_ack_diff_.load() - 1);
989 void artdaq::TCPSocketTransfer::send_ack_(
int fd)
991 MessHead mh = { 0,MessHead::ack_v0,htons(source_rank()),{ htonl(ACK_MAGIC) } };
992 write(fd, &mh,
sizeof(mh));
996 void artdaq::TCPSocketTransfer::listen_(
int port,
size_t rcvbuf)
999 while (listen_thread_refcount_ > 0)
1001 TLOG(TLVL_TRACE) <<
"listen_: Listening/accepting new connections on port " << port;
1002 if (listen_fd == -1)
1004 TLOG(TLVL_DEBUG) <<
"listen_: Opening listener";
1007 if (listen_fd == -1)
1009 TLOG(TLVL_DEBUG) <<
"listen_: Error creating listen_fd!";
1014 timeval tv = { 2,0 };
1017 FD_SET(listen_fd, &rfds);
1019 res = select(listen_fd + 1, &rfds, (fd_set *)0, (fd_set *)0, &tv);
1024 socklen_t arglen =
sizeof(un);
1026 TLOG(TLVL_DEBUG) <<
"listen_: Calling accept";
1027 fd = accept(listen_fd, (sockaddr *)&un, &arglen);
1028 TLOG(TLVL_DEBUG) <<
"listen_: Done with accept";
1030 TLOG(TLVL_DEBUG) <<
"listen_: Reading connect message";
1031 socklen_t lenlen =
sizeof(tv);
1033 setsockopt(fd, SOL_SOCKET, SO_RCVTIMEO, &tv, lenlen);
1035 uint64_t mark_us = TimeUtils::gettimeofday_us();
1036 sts = read(fd, &mh,
sizeof(mh));
1037 uint64_t delta_us = TimeUtils::gettimeofday_us() - mark_us;
1038 TLOG(TLVL_DEBUG) <<
"listen_: Read of connect message took " << delta_us <<
" microseconds.";
1039 if (sts !=
sizeof(mh))
1041 TLOG(TLVL_DEBUG) <<
"listen_: Wrong message header length received!";
1050 TLOG(TLVL_DEBUG) <<
"listen_: Wrong magic bytes in header!";
1056 std::unique_lock<std::mutex> lk(connected_fd_mutex_);
1057 connected_fds_[mh.
source_id].insert(fd);
1059 TLOG(TLVL_INFO) <<
"listen_: New fd is " << fd <<
" for source rank " << mh.
source_id;
1063 TLOG(16) <<
"listen_: No connections in timeout interval!";
1067 TLOG(TLVL_INFO) <<
"listen_: Shutting down connection listener";
1068 if (listen_fd != -1) close(listen_fd);
1069 std::unique_lock<std::mutex> lk(connected_fd_mutex_);
1070 auto it = connected_fds_.begin();
1071 while (it != connected_fds_.end())
1073 auto& fd_set = it->second;
1074 auto rank_it = fd_set.begin();
1075 while (rank_it != fd_set.end())
1078 rank_it = fd_set.erase(rank_it);
1080 it = connected_fds_.erase(it);
bool isRunning() override
Determine whether the TransferInterface plugin is able to send/receive data.
virtual int source_rank() const
Get the source rank for this TransferInterface instance.
int TCPConnect(char const *host_in, int dflt_port, long flags=0, int sndbufsiz=0)
Connect to a host on a given port.
uint32_t conn_magic
unsigned first is better for MessHead initializer: {0,0,my_node_idx_,CONN_MAGIC}
hostMap_t MakeHostMap(fhicl::ParameterSet pset, hostMap_t map=hostMap_t())
Make a hostMap_t from a HostMap::Config ParameterSet
This TransferInterface is a Receiver.
int receiveFragmentData(RawDataType *destination, size_t wordCount) override
Receive the body of a Fragment to the given destination pointer.
int TCP_listen_fd(int port, int rcvbuf)
Create a TCP listening socket on the given port and INADDR_ANY, with the given receive buffer...
TCPSocketTransfer(fhicl::ParameterSet const &ps, Role role)
TCPSocketTransfer Constructor.
int receiveFragmentHeader(detail::RawFragmentHeader &header, size_t receiveTimeout) override
Receive a Fragment Header from the transport mechanism.
This TransferInterface is a Sender.
void flush_buffers() override
Flush any in-flight data. This should be used by the receiver after the receive loop has ended...
int32_t byte_count
use CONN_MAGIC for connect_v0, data that follow for data_v0 (and 0 lenght data)
Some error occurred, but no exception was thrown.
Role
Used to determine if a TransferInterface is a Sender or Receiver.
int64_t source_id
Rank of the source.
MessType message_type
Message Type.
The send operation completed successfully.
This interface defines the functions used to transfer data between artdaq applications.
TransferInterface implementation plugin that sends data using TCP sockets.
This header is sent by the TCPSocket_transfer to allow for more efficient writev calls.
CopyStatus
Returned from the send functions, this enumeration describes the possible return codes. If an exception occurs, it will be thrown and should be handled normally.