11 #include <sys/socket.h>
12 #include <sys/types.h>
21 #include "artdaq/DAQdata/Globals.hh"
22 #define TRACE_NAME (app_name + "_TCPSocketTransfer").c_str()
26 #include "artdaq-core/Data/Fragment.hh"
27 #include "artdaq-core/Utilities/TimeUtils.hh"
30 #include "artdaq/TransferPlugins/TCPSocketTransfer.hh"
31 #include "artdaq/TransferPlugins/detail/SRSockets.hh"
32 #include "artdaq/TransferPlugins/detail/Timeout.hh"
34 std::atomic<int> artdaq::TCPSocketTransfer::listen_thread_refcount_(0);
35 std::unique_ptr<boost::thread> artdaq::TCPSocketTransfer::listen_thread_ =
nullptr;
36 std::map<int, std::set<int>> artdaq::TCPSocketTransfer::connected_fds_ = std::map<int, std::set<int>>();
37 std::mutex artdaq::TCPSocketTransfer::listen_thread_mutex_;
38 std::mutex artdaq::TCPSocketTransfer::connected_fd_mutex_;
44 , active_receive_fd_(-1)
45 , last_active_receive_fd_(-1)
46 , rcvbuf_(pset.get<size_t>(
"tcp_receive_buffer_size", 0))
47 , sndbuf_(pset.get<size_t>(
"tcp_send_buffer_size", max_fragment_size_words_ * sizeof(artdaq::RawDataType) * buffer_count_))
48 , send_retry_timeout_us_(pset.get<size_t>(
"send_retry_timeout_us", 1000000))
49 , timeoutMessageArmed_(true)
51 , receive_disconnected_wait_s_(pset.get<double>(
"receive_socket_disconnected_wait_s", 10.0))
52 , receive_err_wait_us_(pset.get<size_t>(
"receive_socket_disconnected_wait_us", 10000))
53 , receive_socket_has_been_connected_(false)
56 TLOG(TLVL_DEBUG) << GetTraceName() <<
" Constructor: pset=" << pset.to_string() <<
", role=" << (role ==
TransferInterface::Role::kReceive ?
"kReceive" :
"kSend");
57 connection_was_lost_ =
false;
62 TLOG(TLVL_DEBUG) << GetTraceName() <<
": Listening for connections";
63 start_listen_thread_();
64 TLOG(TLVL_DEBUG) << GetTraceName() <<
": Done Listening";
69 TLOG(TLVL_DEBUG) << GetTraceName() <<
": Connecting to destination";
71 TLOG(TLVL_DEBUG) << GetTraceName() <<
": Done Connecting";
73 TLOG(TLVL_DEBUG) << GetTraceName() <<
": End of Constructor";
76 artdaq::TCPSocketTransfer::~TCPSocketTransfer() noexcept
78 TLOG(TLVL_DEBUG) << GetTraceName() <<
": Shutting down TCPSocketTransfer";
87 timeval tv = {0, 100000};
88 socklen_t len =
sizeof(tv);
89 setsockopt(send_fd_, SOL_SOCKET, SO_SNDTIMEO, &tv, len);
90 write(send_fd_, &mh,
sizeof(mh));
98 std::unique_lock<std::mutex> fd_lock(connected_fd_mutex_);
99 if (connected_fds_.count(source_rank()))
101 auto it = connected_fds_[source_rank()].begin();
102 while (it != connected_fds_[source_rank()].end())
105 it = connected_fds_[source_rank()].erase(it);
107 connected_fds_.erase(source_rank());
109 if (ack_listen_thread_ && ack_listen_thread_->joinable()) ack_listen_thread_->join();
112 std::unique_lock<std::mutex> lk(listen_thread_mutex_);
113 listen_thread_refcount_--;
114 if (listen_thread_refcount_ <= 0 && listen_thread_ && listen_thread_->joinable())
116 listen_thread_->join();
120 TLOG(TLVL_DEBUG) << GetTraceName() <<
": End of Destructor";
125 TLOG(5) << GetTraceName() <<
": receiveFragmentHeader: BEGIN";
126 int ret_rank = RECV_TIMEOUT;
129 if (getConnectedFDCount(source_rank()) == 0)
138 TLOG(7) << GetTraceName() <<
": receiveFragmentHeader: Receive socket not connected, returning RECV_TIMEOUT";
139 usleep(receive_err_wait_us_);
142 receive_socket_has_been_connected_ =
true;
143 last_recv_time_ = std::chrono::steady_clock::now();
145 TLOG(5) << GetTraceName() <<
": receiveFragmentHeader timeout_usec=" << timeout_usec;
150 SocketState state = SocketState::Metadata;
151 int target_bytes =
sizeof(
MessHead);
152 uint64_t start_time_us = TimeUtils::gettimeofday_us();
163 if (timeout_usec == 0)
166 timeout_ms = (timeout_usec + 999) / 1000;
169 bool noDataWarningSent =
false;
172 while (!done && getConnectedFDCount(source_rank()) > 0)
174 if (active_receive_fd_ == -1)
178 std::vector<pollfd> pollfds;
180 std::unique_lock<std::mutex> lk(connected_fd_mutex_);
181 fd_count = connected_fds_[source_rank()].size();
182 pollfds.resize(fd_count);
183 auto iter = connected_fds_[source_rank()].begin();
184 for (
size_t ii = 0; ii < fd_count; ++ii)
186 pollfds[ii].events = POLLIN | POLLPRI | POLLERR;
187 pollfds[ii].fd = *iter;
192 int num_fds_ready = poll(&pollfds[0], fd_count, timeout_ms);
193 if (num_fds_ready <= 0)
195 TLOG(5) << GetTraceName() <<
": receiveFragmentHeader: No data on receive socket, returning RECV_TIMEOUT";
200 if (last_active_receive_fd_ != -1)
202 for (
auto& pollfd : pollfds)
205 if (pollfd.fd == last_active_receive_fd_)
212 int active_index = -1;
213 short anomolous_events = 0;
214 for (
size_t ii = index; ii < index + pollfds.size(); ++ii)
216 auto pollfd_index = (ii + index) % pollfds.size();
217 if (pollfds[pollfd_index].revents & (POLLIN | POLLPRI))
219 active_index = pollfd_index;
220 active_receive_fd_ = pollfds[active_index].fd;
221 active_revents_ = pollfds[active_index].revents;
225 else if (pollfds[pollfd_index].revents & (POLLHUP | POLLERR))
227 disconnect_receive_socket_(pollfds[pollfd_index].fd,
"Poll returned POLLHUP or POLLERR, indicating problems with the sender.");
230 else if (pollfds[pollfd_index].revents & (POLLNVAL))
232 TLOG(TLVL_DEBUG) << GetTraceName() <<
": receiveFragmentHeader: FD is closed, most likely because the peer went away. Removing from fd list.";
233 disconnect_receive_socket_(pollfds[pollfd_index].fd,
"FD is closed, most likely because the peer went away.");
236 else if (pollfds[pollfd_index].revents)
238 anomolous_events |= pollfds[pollfd_index].revents;
242 if (active_index == -1)
244 if (anomolous_events)
245 TLOG(TLVL_DEBUG) << GetTraceName() <<
": receiveFragmentHeader: Wrong event received from a pollfd. Mask: " <<
static_cast<int>(anomolous_events);
246 active_receive_fd_ = -1;
250 if (!done && timeout_usec > 0)
253 size_t delta_us = TimeUtils::gettimeofday_us() - start_time_us;
254 if (delta_us > timeout_usec)
258 timeout_ms = ((timeout_usec - delta_us) + 999) / 1000;
261 if (loop_guard > 10) { usleep(1000); }
262 if (++loop_guard > 10010)
264 TLOG(TLVL_WARNING) << GetTraceName() <<
": receiveFragmentHeader: loop guard triggered, returning RECV_TIMEOUT";
265 usleep(receive_err_wait_us_);
266 active_receive_fd_ = -1;
270 if (state == SocketState::Metadata)
273 buff = &(mha[offset]);
274 byte_cnt =
sizeof(
MessHead) - offset;
279 buff =
reinterpret_cast<uint8_t*
>(&header) + offset;
280 byte_cnt = target_bytes - offset;
292 TLOG(6) << GetTraceName() <<
": receiveFragmentHeader: Reading " << byte_cnt <<
" bytes from socket";
293 sts = read(active_receive_fd_, buff, byte_cnt);
294 TLOG(6) << GetTraceName() <<
": receiveFragmentHeader: Done with read";
299 last_recv_time_ = std::chrono::steady_clock::now();
302 TLOG(7) << GetTraceName() <<
": receiveFragmentHeader state=" <<
static_cast<int>(state) <<
" read=" << sts;
305 TLOG(TLVL_WARNING) << GetTraceName() <<
": receiveFragmentHeader: Error on receive, closing socket "
306 <<
" (errno=" << errno <<
": " << strerror(errno) <<
")";
307 active_receive_fd_ = disconnect_receive_socket_(active_receive_fd_);
311 if (!noDataWarningSent)
313 TLOG(TLVL_WARNING) << GetTraceName() <<
": receiveFragmentHeader: No data received, is the sender still sending?!?";
314 noDataWarningSent =
true;
316 if (TimeUtils::GetElapsedTime(last_recv_time_) > receive_disconnected_wait_s_)
318 TLOG(TLVL_ERROR) << GetTraceName() <<
": receiveFragmentHeader: No data received within timeout, aborting!";
326 if (sts >= target_bytes)
328 TLOG(7) << GetTraceName() <<
": receiveFragmentHeader: Target read bytes reached. Changing state";
330 if (state == SocketState::Metadata)
332 state = SocketState::Data;
336 TLOG(7) << GetTraceName() <<
": receiveFragmentHeader: Expected header size = " << target_bytes <<
", sizeof(RawFragmentHeader) = " <<
sizeof(artdaq::detail::RawFragmentHeader);
341 active_receive_fd_ = disconnect_receive_socket_(active_receive_fd_,
"Stop Message received.");
345 TLOG(TLVL_WARNING) << GetTraceName() <<
": receiveFragmentHeader: Message header indicates that Fragment data follows when I was expecting a Fragment header!";
346 active_receive_fd_ = disconnect_receive_socket_(active_receive_fd_,
"Desync detected");
349 if (target_bytes == 0)
357 ret_rank = source_rank();
358 TLOG(8) << GetTraceName() <<
": receiveFragmentHeader done sts=" << sts <<
" src=" << ret_rank;
359 TLOG(7) << GetTraceName() <<
": receiveFragmentHeader: Done receiving fragment header. Moving into output.";
369 TLOG(5) << GetTraceName() <<
": receiveFragmentHeader: Returning " << ret_rank;
373 int artdaq::TCPSocketTransfer::disconnect_receive_socket_(
int fd, std::string msg)
375 TLOG(TLVL_WARNING) << GetTraceName() <<
": disconnect_receive_socket_: " << msg <<
" Closing socket " << fd;
377 std::unique_lock<std::mutex> lk(connected_fd_mutex_);
378 if (connected_fds_.count(source_rank()))
379 connected_fds_[source_rank()].erase(fd);
381 TLOG(TLVL_DEBUG) << GetTraceName() <<
": disconnect_receive_socket_: There are now " << connected_fds_[source_rank()].size() <<
" active senders.";
387 TLOG(19) << GetTraceName() <<
": receiveFragmentData: BEGIN";
388 int ret_rank = RECV_TIMEOUT;
389 if (active_receive_fd_ == -1)
391 TLOG(TLVL_ERROR) << GetTraceName() <<
": receiveFragmentData: Receive socket not connected, returning RECV_TIMEOUT (Will result in \"Unexpected return code error\")";
400 SocketState state = SocketState::Metadata;
401 int target_bytes =
sizeof(
MessHead);
404 pollfd_s.events = POLLIN | POLLPRI | POLLERR;
405 pollfd_s.fd = active_receive_fd_;
409 bool noDataWarningSent =
false;
410 last_recv_time_ = std::chrono::steady_clock::now();
413 TLOG(9) << GetTraceName() <<
": receiveFragmentData: Polling fd to see if there's data";
414 int num_fds_ready = poll(&pollfd_s, 1, 1000);
415 TLOG(TLVL_TRACE) << GetTraceName() <<
": receiveFragmentData: Polled fd to see if there's data"
416 <<
", num_fds_ready = " << num_fds_ready;
417 if (num_fds_ready <= 0)
419 if (num_fds_ready == 0)
421 TLOG(TLVL_WARNING) << GetTraceName() <<
": receiveFragmentData: No data from " << source_rank() <<
" in " << TimeUtils::GetElapsedTimeMilliseconds(last_recv_time_) <<
" ms!"
422 <<
" State = " << (state == SocketState::Metadata ?
"Metadata" :
"Data") <<
", recvd/total=" << offset <<
"/" << target_bytes <<
" (delta=" << target_bytes - offset <<
")";
424 if (TimeUtils::GetElapsedTime(last_recv_time_) > receive_disconnected_wait_s_)
426 TLOG(TLVL_WARNING) << GetTraceName() <<
": receiveFragmentData: No data received within timeout (" << TimeUtils::GetElapsedTime(last_recv_time_) <<
" / " << receive_disconnected_wait_s_ <<
" ), returning RECV_TIMEOUT";
427 disconnect_receive_socket_(active_receive_fd_,
"No data on this socket within timeout");
428 active_receive_fd_ = -1;
434 TLOG(TLVL_ERROR) <<
"Error in poll: errno=" << errno;
435 active_receive_fd_ = -1;
440 last_recv_time_ = std::chrono::steady_clock::now();
443 if (pollfd_s.revents & (POLLIN | POLLPRI))
447 else if (pollfd_s.revents & (POLLNVAL))
449 disconnect_receive_socket_(pollfd_s.fd,
"FD is closed, most likely because the peer went away.");
452 else if (pollfd_s.revents & (POLLHUP | POLLERR))
454 disconnect_receive_socket_(pollfd_s.fd,
"Poll returned POLLHUP or POLLERR, indicating problems with the sender.");
459 TLOG(TLVL_WARNING) << GetTraceName() <<
": receiveFragmentData: Wrong event received from pollfd: " << pollfd_s.revents;
460 disconnect_receive_socket_(pollfd_s.fd);
464 if (state == SocketState::Metadata)
467 buff = &(mha[offset]);
468 byte_cnt =
sizeof(
MessHead) - offset;
473 buff =
reinterpret_cast<uint8_t*
>(destination) + offset;
477 TLOG(10) << GetTraceName() <<
": receiveFragmentData: Reading " << byte_cnt <<
" bytes from socket into " << (
void*)buff;
478 sts = read(active_receive_fd_, buff, byte_cnt);
481 TLOG(10) << GetTraceName() <<
": recvFragment state=" <<
static_cast<int>(state) <<
" read=" << sts;
485 if (loop_guard > 10) { usleep(1000); }
486 if (++loop_guard > 10010)
488 TLOG(TLVL_WARNING) << GetTraceName() <<
": receiveFragmentData: loop guard triggered, returning RECV_TIMEOUT";
489 active_receive_fd_ = -1;
496 last_recv_time_ = std::chrono::steady_clock::now();
501 TLOG(TLVL_WARNING) << GetTraceName() <<
": receiveFragmentData: Error on receive, closing socket"
502 <<
" (errno=" << errno <<
": " << strerror(errno) <<
")";
503 disconnect_receive_socket_(pollfd_s.fd);
507 if (!noDataWarningSent)
509 TLOG(TLVL_WARNING) << GetTraceName() <<
": receiveFragmentData: No data received, is the sender still sending?!?";
510 noDataWarningSent =
true;
512 if (TimeUtils::GetElapsedTime(last_recv_time_) > receive_disconnected_wait_s_)
514 TLOG(TLVL_ERROR) << GetTraceName() <<
": receiveFragmentData: No data received within timeout, aborting!";
522 if (sts >= target_bytes)
524 TLOG(9) << GetTraceName() <<
": receiveFragmentData: Target read bytes reached. Changing state";
526 if (state == SocketState::Metadata)
528 state = SocketState::Data;
535 TLOG(TLVL_WARNING) << GetTraceName() <<
": receiveFragmentData: Message header indicates that a Fragment header follows when I was expecting Fragment data!";
536 active_receive_fd_ = disconnect_receive_socket_(active_receive_fd_,
"Desync detected");
541 ret_rank = source_rank();
542 TLOG(11) << GetTraceName() <<
": receiveFragmentData done sts=" << sts <<
" src=" << ret_rank;
543 TLOG(9) << GetTraceName() <<
": receiveFragmentData: Done receiving fragment. Moving into output.";
546 send_ack_(active_receive_fd_);
556 if (target_bytes == 0 && state == SocketState::Data)
558 ret_rank = source_rank();
559 TLOG(11) << GetTraceName() <<
": receiveFragmentData done sts=" << sts <<
" src=" << ret_rank;
560 TLOG(9) << GetTraceName() <<
": receiveFragmentData: Done receiving fragment. Moving into output.";
563 send_ack_(active_receive_fd_);
571 last_active_receive_fd_ = active_receive_fd_;
572 active_receive_fd_ = -1;
574 TLOG(9) << GetTraceName() <<
": receiveFragmentData: Returning rank " << ret_rank;
583 return send_fd_ != -1;
585 TLOG(TLVL_DEBUG) << GetTraceName() <<
": isRunning: There are " << getConnectedFDCount(source_rank()) <<
" fds connected.";
586 return getConnectedFDCount(source_rank()) > 0;
593 while (connected_fds_[source_rank()].size())
595 disconnect_receive_socket_(*connected_fds_[source_rank()].begin(),
"Flushing connections");
603 TLOG(12) << GetTraceName() <<
": sendFragment begin send of fragment with sequenceID=" << frag.sequenceID();
604 artdaq::Fragment grab_ownership_frag = std::move(frag);
607 if (send_fd_ == -1 && connection_was_lost_)
609 TLOG(TLVL_INFO) << GetTraceName() <<
": reconnection attempt failed, returning quickly.";
617 while (static_cast<size_t>(send_ack_diff_) > buffer_count_) usleep(10000);
620 iovec iov = {
reinterpret_cast<void*
>(grab_ownership_frag.headerAddress()),
621 detail::RawFragmentHeader::num_words() *
sizeof(RawDataType)};
623 auto sts = sendData_(&iov, 1, send_retry_timeout_us_,
true);
624 auto start_time = std::chrono::steady_clock::now();
626 while (sts == CopyStatus::kTimeout && (send_timeout_usec == 0 || TimeUtils::GetElapsedTimeMicroseconds(start_time) < send_timeout_usec) && TimeUtils::GetElapsedTimeMicroseconds(start_time) < 10000000)
628 TLOG(13) << GetTraceName() <<
": sendFragment: Timeout sending fragment";
629 sts = sendData_(&iov, 1, send_retry_timeout_us_,
true);
632 if (sts != CopyStatus::kSuccess)
return sts;
636 iov = {
reinterpret_cast<void*
>(grab_ownership_frag.headerAddress() + detail::RawFragmentHeader::num_words()),
637 grab_ownership_frag.sizeBytes() - detail::RawFragmentHeader::num_words() *
sizeof(RawDataType)};
638 sts = sendData_(&iov, 1, send_retry_timeout_us_);
639 start_time = std::chrono::steady_clock::now();
640 while (sts == CopyStatus::kTimeout && (send_timeout_usec == 0 || TimeUtils::GetElapsedTimeMicroseconds(start_time) < send_timeout_usec) && TimeUtils::GetElapsedTimeMicroseconds(start_time) < 10000000)
642 TLOG(13) << GetTraceName() <<
": sendFragment: Timeout sending fragment";
643 sts = sendData_(&iov, 1, send_retry_timeout_us_);
651 TLOG(12) << GetTraceName() <<
": sendFragment returning " << CopyStatusToString(sts);
657 TLOG(TLVL_DEBUG) << GetTraceName() <<
": sendData_ Converting buf to iovec";
658 iovec iov = {(
void*)buf, bytes};
659 return sendData_(&iov, 1, send_timeout_usec, isHeader);
667 if (timeoutMessageArmed_)
669 TLOG(TLVL_DEBUG) << GetTraceName() <<
": sendData_: Send fd is not open. Returning kTimeout";
670 timeoutMessageArmed_ =
false;
672 return CopyStatus::kTimeout;
674 timeoutMessageArmed_ =
true;
675 TLOG(14) << GetTraceName() <<
": send_timeout_usec is " << send_timeout_usec <<
", currently unused.";
678 uint32_t total_to_write_bytes = 0;
679 std::vector<iovec> iov_in(iovcnt + 1);
680 std::vector<iovec> iovv(iovcnt + 2);
682 for (ii = 0; ii < iovcnt; ++ii)
684 iov_in[ii + 1] = iov[ii];
685 total_to_write_bytes += iov[ii].iov_len;
688 MessHead mh = {0, isHeader ? MessHead::header_v0 : MessHead::data_v0, htons(source_rank()), {htonl(total_to_write_bytes)}};
689 iov_in[0].iov_base = &mh;
690 iov_in[0].iov_len =
sizeof(mh);
691 total_to_write_bytes +=
sizeof(mh);
694 ssize_t total_written_bytes = 0;
695 ssize_t per_write_max_bytes = (32 * 1024);
697 size_t in_iov_idx = 0;
698 size_t out_iov_idx = 0;
699 ssize_t this_write_bytes = 0;
706 (in_iov_idx + out_iov_idx) < iov_in.size() && this_write_bytes < per_write_max_bytes;
709 this_write_bytes += iov_in[in_iov_idx + out_iov_idx].iov_len;
710 iovv[out_iov_idx] = iov_in[in_iov_idx + out_iov_idx];
712 if (this_write_bytes > per_write_max_bytes)
714 iovv[out_iov_idx - 1].iov_len -= this_write_bytes - per_write_max_bytes;
715 this_write_bytes = per_write_max_bytes;
720 #ifndef __OPTIMIZE__ // This can be an expensive TRACE call (even if disabled) due to multiplicity of calls
721 TLOG(14) << GetTraceName() <<
": sendFragment b4 writev " << std::setw(7) << total_written_bytes <<
" total_written_bytes send_fd_=" << send_fd_ <<
" in_idx=" << in_iov_idx
722 <<
" iovcnt=" << out_iov_idx <<
" 1st.len=" << iovv[0].iov_len;
725 sts = writev(send_fd_, &(iovv[0]), out_iov_idx);
730 if (errno == EAGAIN )
732 TLOG(TLVL_DEBUG) << GetTraceName() <<
": sendFragment EWOULDBLOCK";
733 fcntl(send_fd_, F_SETFL, 0);
738 TLOG(TLVL_WARNING) << GetTraceName() <<
": sendFragment_: WRITE ERROR: " << strerror(errno);
742 connection_was_lost_ =
true;
745 else if (sts != this_write_bytes)
748 TLOG(TLVL_DEBUG) << GetTraceName() <<
": sendFragment writev sts(" << sts <<
")!=requested_send_bytes(" << this_write_bytes <<
")";
749 total_written_bytes += sts;
751 for (ii = 0; (size_t)sts >= iovv[ii].iov_len; ++ii)
752 sts -= iovv[ii].iov_len;
754 iovv[ii].iov_len -= sts;
755 iovv[ii].iov_base = (uint8_t*)(iovv[ii].iov_base) + sts;
760 iovv[out_iov_idx] = iovv[ii];
762 this_write_bytes = iovv[out_iov_idx].iov_len;
767 unsigned long additional = ((
unsigned long)iov_in[in_iov_idx].iov_base + iov_in[in_iov_idx].iov_len) - ((
unsigned long)iovv[out_iov_idx].iov_base + iovv[out_iov_idx].iov_len);
770 iovv[out_iov_idx].iov_len += additional;
771 this_write_bytes += additional;
772 if (this_write_bytes > per_write_max_bytes)
774 iovv[out_iov_idx].iov_len -= this_write_bytes - per_write_max_bytes;
775 this_write_bytes = per_write_max_bytes;
779 TLOG(TLVL_TRACE) << GetTraceName() <<
": sendFragment writev sts!=: this_write_bytes=" << this_write_bytes
780 <<
" out_iov_idx=" << out_iov_idx
781 <<
" additional=" << additional
786 #ifndef __OPTIMIZE__ // This can be an expensive TRACE call (even if disabled) due to multiplicity of calls
787 TLOG(TLVL_TRACE) << GetTraceName() <<
": sendFragment writev sts(" << sts <<
")==requested_send_bytes(" << this_write_bytes <<
")";
789 total_written_bytes += sts;
791 iovv[out_iov_idx].iov_base = (uint8_t*)(iovv[out_iov_idx].iov_base) + iovv[out_iov_idx].iov_len;
792 iovv[out_iov_idx].iov_len = 0;
793 in_iov_idx += out_iov_idx;
794 this_write_bytes = 0;
796 unsigned long additional = ((
unsigned long)iov_in[in_iov_idx].iov_base + iov_in[in_iov_idx].iov_len) - ((
unsigned long)iovv[out_iov_idx].iov_base + iovv[out_iov_idx].iov_len);
799 iovv[out_iov_idx].iov_len += additional;
800 this_write_bytes += additional;
801 if (this_write_bytes > per_write_max_bytes)
803 iovv[out_iov_idx].iov_len -= this_write_bytes - per_write_max_bytes;
804 this_write_bytes = per_write_max_bytes;
806 if (out_iov_idx != 0)
807 iovv[0] = iovv[out_iov_idx];
816 }
while (total_written_bytes < total_to_write_bytes);
817 if (total_written_bytes > total_to_write_bytes)
818 TLOG(TLVL_ERROR) << GetTraceName() <<
": sendFragment program error: too many bytes transferred";
823 fcntl(send_fd_, F_SETFL, O_NONBLOCK);
825 sts = total_written_bytes -
sizeof(
MessHead);
827 TLOG(14) << GetTraceName() <<
": sendFragment sts=" << sts;
831 void artdaq::TCPSocketTransfer::connect_()
833 auto start_time = std::chrono::steady_clock::now();
836 while (send_fd_ == -1 && TimeUtils::GetElapsedTimeMicroseconds(start_time) < send_retry_timeout_us_ * 10)
838 TLOG(TLVL_DEBUG) << GetTraceName() <<
": Connecting sender socket";
839 int sndbuf_bytes =
static_cast<int>(sndbuf_);
840 if (sndbuf_ > INT_MAX)
842 sndbuf_bytes = INT_MAX;
843 TLOG(TLVL_WARNING) <<
"Requested SNDBUF " << sndbuf_ <<
" too large, setting to INT_MAX: " << INT_MAX;
845 TLOG(TLVL_DEBUG) <<
"Requested SNDBUF is " << sndbuf_bytes;
847 send_fd_ =
TCPConnect(hostMap_[destination_rank()].c_str(), portMan->GetTCPSocketTransferPort(destination_rank()), O_NONBLOCK, sndbuf_bytes);
850 if (connection_was_lost_) {
break; }
853 usleep(send_retry_timeout_us_);
859 TLOG(TLVL_DEBUG) << GetTraceName() <<
": connect_ " + hostMap_[destination_rank()] +
":" << portMan->GetTCPSocketTransferPort(destination_rank()) <<
" send_fd_=" << send_fd_;
863 TLOG(TLVL_DEBUG) << GetTraceName() <<
": connect_: Writing connect message";
864 MessHead mh = {0, MessHead::connect_v0, htons(source_rank()), {htonl(CONN_MAGIC)}};
865 ssize_t sts = write(send_fd_, &mh,
sizeof(mh));
868 TLOG(TLVL_ERROR) << GetTraceName() <<
": connect_: Error writing connect message!";
876 TLOG(TLVL_INFO) << GetTraceName() <<
": connect_: Successfully connected";
879 connection_was_lost_ =
false;
883 if (ack_listen_thread_ && ack_listen_thread_->joinable()) ack_listen_thread_->join();
884 TLOG(TLVL_INFO) << GetTraceName() <<
": Starting Ack Listener Thread";
888 ack_listen_thread_ = std::make_unique<boost::thread>(&TCPSocketTransfer::receive_acks_,
this);
890 catch (
const boost::exception& e)
892 TLOG(TLVL_ERROR) <<
"Caught boost::exception starting TCP Socket Ack Listen thread: " << boost::diagnostic_information(e) <<
", errno=" << errno;
893 std::cerr <<
"Caught boost::exception starting TCP Socket Ack Listen thread: " << boost::diagnostic_information(e) <<
", errno=" << errno << std::endl;
900 void artdaq::TCPSocketTransfer::reconnect_()
904 TLOG(TLVL_TRACE) << GetTraceName() <<
": check/reconnect";
909 void artdaq::TCPSocketTransfer::start_listen_thread_()
911 std::unique_lock<std::mutex> start_lock(listen_thread_mutex_);
912 if (listen_thread_refcount_ == 0)
914 if (listen_thread_ && listen_thread_->joinable()) listen_thread_->join();
915 listen_thread_refcount_ = 1;
916 TLOG(TLVL_INFO) << GetTraceName() <<
": Starting Listener Thread";
920 listen_thread_ = std::make_unique<boost::thread>(&TCPSocketTransfer::listen_, portMan->GetTCPSocketTransferPort(destination_rank()), rcvbuf_);
922 catch (
const boost::exception& e)
924 TLOG(TLVL_ERROR) <<
"Caught boost::exception starting TCP Socket Listen thread: " << boost::diagnostic_information(e) <<
", errno=" << errno;
925 std::cerr <<
"Caught boost::exception starting TCP Socket Listen thread: " << boost::diagnostic_information(e) <<
", errno=" << errno << std::endl;
931 listen_thread_refcount_++;
936 void artdaq::TCPSocketTransfer::receive_acks_()
938 while (send_fd_ >= 0)
941 pollfd_s.events = POLLIN | POLLPRI;
942 pollfd_s.fd = send_fd_;
944 TLOG(18) << GetTraceName() <<
": receive_acks_: Polling fd to see if there's data";
945 int num_fds_ready = poll(&pollfd_s, 1, 1000);
946 if (num_fds_ready <= 0)
948 if (num_fds_ready == 0)
950 TLOG(18) << GetTraceName() <<
": receive_acks_: No data on receive socket";
954 TLOG(TLVL_ERROR) <<
"Error in poll: errno=" << errno;
958 if (pollfd_s.revents & (POLLIN | POLLPRI))
964 TLOG(TLVL_DEBUG) << GetTraceName() <<
": receive_acks_: Wrong event received from pollfd: " << pollfd_s.revents;
969 auto sts = read(send_fd_, &mh,
sizeof(mh));
971 if (sts !=
sizeof(mh))
973 TLOG(TLVL_ERROR) << GetTraceName() <<
": receive_ack_: Wrong message header length received! (actual " << sts <<
" != " <<
sizeof(mh) <<
" expected)";
981 TLOG(TLVL_ERROR) << GetTraceName() <<
": receive_ack_: Received ack for different sender! Rank=" << my_rank <<
", hdr=" << mh.
source_id;
986 TLOG(TLVL_ERROR) << GetTraceName() <<
": receive_ack_: Wrong magic bytes in header!";
990 TLOG(17) << GetTraceName() <<
": receive_acks_: Received ack message, diff is now " << (send_ack_diff_.load() - 1);
995 void artdaq::TCPSocketTransfer::send_ack_(
int fd)
997 MessHead mh = {0, MessHead::ack_v0, htons(source_rank()), {htonl(ACK_MAGIC)}};
998 write(fd, &mh,
sizeof(mh));
1002 void artdaq::TCPSocketTransfer::listen_(
int port,
size_t rcvbuf)
1005 while (listen_thread_refcount_ > 0)
1007 TLOG(TLVL_TRACE) <<
"listen_: Listening/accepting new connections on port " << port;
1008 if (listen_fd == -1)
1010 TLOG(TLVL_DEBUG) <<
"listen_: Opening listener";
1013 if (listen_fd == -1)
1015 TLOG(TLVL_DEBUG) <<
"listen_: Error creating listen_fd!";
1020 timeval tv = {2, 0};
1023 FD_SET(listen_fd, &rfds);
1025 res = select(listen_fd + 1, &rfds, (fd_set*)0, (fd_set*)0, &tv);
1030 socklen_t arglen =
sizeof(un);
1032 TLOG(TLVL_DEBUG) <<
"listen_: Calling accept";
1033 fd = accept(listen_fd, (sockaddr*)&un, &arglen);
1034 TLOG(TLVL_DEBUG) <<
"listen_: Done with accept";
1036 TLOG(TLVL_DEBUG) <<
"listen_: Reading connect message";
1037 socklen_t lenlen =
sizeof(tv);
1039 setsockopt(fd, SOL_SOCKET, SO_RCVTIMEO, &tv, lenlen);
1041 uint64_t mark_us = TimeUtils::gettimeofday_us();
1042 sts = read(fd, &mh,
sizeof(mh));
1043 uint64_t delta_us = TimeUtils::gettimeofday_us() - mark_us;
1044 TLOG(TLVL_DEBUG) <<
"listen_: Read of connect message took " << delta_us <<
" microseconds.";
1045 if (sts !=
sizeof(mh))
1047 TLOG(TLVL_DEBUG) <<
"listen_: Wrong message header length received!";
1056 TLOG(TLVL_DEBUG) <<
"listen_: Wrong magic bytes in header!";
1062 std::unique_lock<std::mutex> lk(connected_fd_mutex_);
1063 connected_fds_[mh.
source_id].insert(fd);
1065 TLOG(TLVL_INFO) <<
"listen_: New fd is " << fd <<
" for source rank " << mh.
source_id;
1069 TLOG(16) <<
"listen_: No connections in timeout interval!";
1073 TLOG(TLVL_INFO) <<
"listen_: Shutting down connection listener";
1074 if (listen_fd != -1) close(listen_fd);
1075 std::unique_lock<std::mutex> lk(connected_fd_mutex_);
1076 auto it = connected_fds_.begin();
1077 while (it != connected_fds_.end())
1079 auto& fd_set = it->second;
1080 auto rank_it = fd_set.begin();
1081 while (rank_it != fd_set.end())
1084 rank_it = fd_set.erase(rank_it);
1086 it = connected_fds_.erase(it);
bool isRunning() override
Determine whether the TransferInterface plugin is able to send/receive data.
virtual int source_rank() const
Get the source rank for this TransferInterface instance.
int TCPConnect(char const *host_in, int dflt_port, long flags=0, int sndbufsiz=0)
Connect to a host on a given port.
uint32_t conn_magic
unsigned first is better for MessHead initializer: {0,0,my_node_idx_,CONN_MAGIC}
hostMap_t MakeHostMap(fhicl::ParameterSet pset, hostMap_t map=hostMap_t())
Make a hostMap_t from a HostMap::Config ParameterSet
This TransferInterface is a Receiver.
int receiveFragmentData(RawDataType *destination, size_t wordCount) override
Receive the body of a Fragment to the given destination pointer.
int TCP_listen_fd(int port, int rcvbuf)
Create a TCP listening socket on the given port and INADDR_ANY, with the given receive buffer...
TCPSocketTransfer(fhicl::ParameterSet const &ps, Role role)
TCPSocketTransfer Constructor.
int receiveFragmentHeader(detail::RawFragmentHeader &header, size_t receiveTimeout) override
Receive a Fragment Header from the transport mechanism.
This TransferInterface is a Sender.
void flush_buffers() override
Flush any in-flight data. This should be used by the receiver after the receive loop has ended...
int32_t byte_count
use CONN_MAGIC for connect_v0, data that follow for data_v0 (and 0 lenght data)
Some error occurred, but no exception was thrown.
Role
Used to determine if a TransferInterface is a Sender or Receiver.
int64_t source_id
Rank of the source.
MessType message_type
Message Type.
The send operation completed successfully.
This interface defines the functions used to transfer data between artdaq applications.
TransferInterface implementation plugin that sends data using TCP sockets.
This header is sent by the TCPSocket_transfer to allow for more efficient writev calls.
CopyStatus
Returned from the send functions, this enumeration describes the possible return codes. If an exception occurs, it will be thrown and should be handled normally.