11 #include <sys/socket.h>
12 #include <sys/types.h>
21 #include "artdaq/DAQdata/Globals.hh"
22 #define TRACE_NAME (app_name + "_TCPSocketTransfer").c_str()
26 #include "artdaq-core/Data/Fragment.hh"
27 #include "artdaq-core/Utilities/TimeUtils.hh"
30 #include "artdaq/TransferPlugins/TCPSocketTransfer.hh"
31 #include "artdaq/TransferPlugins/detail/SRSockets.hh"
32 #include "artdaq/TransferPlugins/detail/Timeout.hh"
36 std::atomic<int> artdaq::TCPSocketTransfer::listen_thread_refcount_(0);
37 std::unique_ptr<boost::thread> artdaq::TCPSocketTransfer::listen_thread_ =
nullptr;
38 std::map<int, std::set<int>> artdaq::TCPSocketTransfer::connected_fds_ = std::map<int, std::set<int>>();
39 std::mutex artdaq::TCPSocketTransfer::listen_thread_mutex_;
40 std::mutex artdaq::TCPSocketTransfer::connected_fd_mutex_;
46 , active_receive_fd_(-1)
47 , last_active_receive_fd_(-1)
48 , rcvbuf_(pset.get<size_t>(
"tcp_receive_buffer_size", 0))
49 , sndbuf_(pset.get<size_t>(
"tcp_send_buffer_size", max_fragment_size_words_ * sizeof(artdaq::RawDataType) * buffer_count_))
50 , send_retry_timeout_us_(pset.get<size_t>(
"send_retry_timeout_us", 1000000))
51 , timeoutMessageArmed_(true)
53 , receive_disconnected_wait_s_(pset.get<double>(
"receive_socket_disconnected_wait_s", 10.0))
54 , receive_err_wait_us_(pset.get<size_t>(
"receive_socket_disconnected_wait_us", 10000))
55 , receive_socket_has_been_connected_(false)
58 TLOG(TLVL_DEBUG) << GetTraceName() <<
" Constructor: pset=" << pset.to_string() <<
", role=" << (role ==
TransferInterface::Role::kReceive ?
"kReceive" :
"kSend");
59 connection_was_lost_ =
false;
64 TLOG(TLVL_DEBUG) << GetTraceName() <<
": Listening for connections";
65 start_listen_thread_();
66 TLOG(TLVL_DEBUG) << GetTraceName() <<
": Done Listening";
71 TLOG(TLVL_DEBUG) << GetTraceName() <<
": Connecting to destination";
73 TLOG(TLVL_DEBUG) << GetTraceName() <<
": Done Connecting";
75 TLOG(TLVL_DEBUG) << GetTraceName() <<
": End of Constructor";
78 artdaq::TCPSocketTransfer::~TCPSocketTransfer() noexcept
80 TLOG(TLVL_DEBUG) << GetTraceName() <<
": Shutting down TCPSocketTransfer";
89 timeval tv = {0, 100000};
90 socklen_t len =
sizeof(tv);
91 setsockopt(send_fd_, SOL_SOCKET, SO_SNDTIMEO, &tv, len);
92 write(send_fd_, &mh,
sizeof(mh));
100 std::unique_lock<std::mutex> fd_lock(connected_fd_mutex_);
101 if (connected_fds_.count(source_rank()))
103 auto it = connected_fds_[source_rank()].begin();
104 while (it != connected_fds_[source_rank()].end())
107 it = connected_fds_[source_rank()].erase(it);
109 connected_fds_.erase(source_rank());
111 if (ack_listen_thread_ && ack_listen_thread_->joinable()) ack_listen_thread_->join();
114 std::unique_lock<std::mutex> lk(listen_thread_mutex_);
115 listen_thread_refcount_--;
116 if (listen_thread_refcount_ <= 0 && listen_thread_ && listen_thread_->joinable())
118 listen_thread_->join();
122 TLOG(TLVL_DEBUG) << GetTraceName() <<
": End of Destructor";
127 TLOG(5) << GetTraceName() <<
": receiveFragmentHeader: BEGIN";
128 int ret_rank = RECV_TIMEOUT;
131 if (getConnectedFDCount(source_rank()) == 0)
140 TLOG(7) << GetTraceName() <<
": receiveFragmentHeader: Receive socket not connected, returning RECV_TIMEOUT";
141 usleep(receive_err_wait_us_);
144 receive_socket_has_been_connected_ =
true;
145 last_recv_time_ = std::chrono::steady_clock::now();
147 TLOG(5) << GetTraceName() <<
": receiveFragmentHeader timeout_usec=" << timeout_usec;
152 SocketState state = SocketState::Metadata;
153 int target_bytes =
sizeof(
MessHead);
154 uint64_t start_time_us = TimeUtils::gettimeofday_us();
165 if (timeout_usec == 0)
168 timeout_ms = (timeout_usec + 999) / 1000;
171 bool noDataWarningSent =
false;
174 while (!done && getConnectedFDCount(source_rank()) > 0)
176 if (active_receive_fd_ == -1)
180 std::vector<pollfd> pollfds;
182 std::unique_lock<std::mutex> lk(connected_fd_mutex_);
183 fd_count = connected_fds_[source_rank()].size();
184 pollfds.resize(fd_count);
185 auto iter = connected_fds_[source_rank()].begin();
186 for (
size_t ii = 0; ii < fd_count; ++ii)
188 pollfds[ii].events = POLLIN | POLLPRI | POLLERR;
189 pollfds[ii].fd = *iter;
194 int num_fds_ready = poll(&pollfds[0], fd_count, timeout_ms);
195 if (num_fds_ready <= 0)
197 TLOG(5) << GetTraceName() <<
": receiveFragmentHeader: No data on receive socket, returning RECV_TIMEOUT";
202 if (last_active_receive_fd_ != -1)
204 for (
auto& pollfd : pollfds)
207 if (pollfd.fd == last_active_receive_fd_)
214 int active_index = -1;
215 short anomolous_events = 0;
216 for (
size_t ii = index; ii < index + pollfds.size(); ++ii)
218 auto pollfd_index = (ii + index) % pollfds.size();
219 if (pollfds[pollfd_index].revents & (POLLIN | POLLPRI))
221 active_index = pollfd_index;
222 active_receive_fd_ = pollfds[active_index].fd;
223 active_revents_ = pollfds[active_index].revents;
227 else if (pollfds[pollfd_index].revents & (POLLHUP | POLLERR))
229 disconnect_receive_socket_(pollfds[pollfd_index].fd,
"Poll returned POLLHUP or POLLERR, indicating problems with the sender.");
232 else if (pollfds[pollfd_index].revents & (POLLNVAL))
234 TLOG(TLVL_DEBUG) << GetTraceName() <<
": receiveFragmentHeader: FD is closed, most likely because the peer went away. Removing from fd list.";
235 disconnect_receive_socket_(pollfds[pollfd_index].fd,
"FD is closed, most likely because the peer went away.");
238 else if (pollfds[pollfd_index].revents)
240 anomolous_events |= pollfds[pollfd_index].revents;
244 if (active_index == -1)
246 if (anomolous_events)
247 TLOG(TLVL_DEBUG) << GetTraceName() <<
": receiveFragmentHeader: Wrong event received from a pollfd. Mask: " <<
static_cast<int>(anomolous_events);
248 active_receive_fd_ = -1;
252 if (!done && timeout_usec > 0)
255 size_t delta_us = TimeUtils::gettimeofday_us() - start_time_us;
256 if (delta_us > timeout_usec)
260 timeout_ms = ((timeout_usec - delta_us) + 999) / 1000;
263 if (loop_guard > 10) { usleep(1000); }
264 if (++loop_guard > 10010)
266 TLOG(TLVL_WARNING) << GetTraceName() <<
": receiveFragmentHeader: loop guard triggered, returning RECV_TIMEOUT";
267 usleep(receive_err_wait_us_);
268 active_receive_fd_ = -1;
272 if (state == SocketState::Metadata)
275 buff = &(mha[offset]);
276 byte_cnt =
sizeof(
MessHead) - offset;
281 buff =
reinterpret_cast<uint8_t*
>(&header) + offset;
282 byte_cnt = target_bytes - offset;
294 TLOG(6) << GetTraceName() <<
": receiveFragmentHeader: Reading " << byte_cnt <<
" bytes from socket";
295 sts = read(active_receive_fd_, buff, byte_cnt);
296 TLOG(6) << GetTraceName() <<
": receiveFragmentHeader: Done with read";
301 last_recv_time_ = std::chrono::steady_clock::now();
304 TLOG(7) << GetTraceName() <<
": receiveFragmentHeader state=" <<
static_cast<int>(state) <<
" read=" << sts;
307 TLOG(TLVL_WARNING) << GetTraceName() <<
": receiveFragmentHeader: Error on receive, closing socket "
308 <<
" (errno=" << errno <<
": " << strerror(errno) <<
")";
309 active_receive_fd_ = disconnect_receive_socket_(active_receive_fd_);
313 if (!noDataWarningSent)
315 TLOG(TLVL_WARNING) << GetTraceName() <<
": receiveFragmentHeader: No data received, is the sender still sending?!?";
316 noDataWarningSent =
true;
318 if (TimeUtils::GetElapsedTime(last_recv_time_) > receive_disconnected_wait_s_)
320 TLOG(TLVL_ERROR) << GetTraceName() <<
": receiveFragmentHeader: No data received within timeout, aborting!";
328 if (sts >= target_bytes)
330 TLOG(7) << GetTraceName() <<
": receiveFragmentHeader: Target read bytes reached. Changing state";
332 if (state == SocketState::Metadata)
334 state = SocketState::Data;
338 TLOG(7) << GetTraceName() <<
": receiveFragmentHeader: Expected header size = " << target_bytes <<
", sizeof(RawFragmentHeader) = " <<
sizeof(artdaq::detail::RawFragmentHeader);
343 active_receive_fd_ = disconnect_receive_socket_(active_receive_fd_,
"Stop Message received.");
347 TLOG(TLVL_WARNING) << GetTraceName() <<
": receiveFragmentHeader: Message header indicates that Fragment data follows when I was expecting a Fragment header!";
348 active_receive_fd_ = disconnect_receive_socket_(active_receive_fd_,
"Desync detected");
351 if (target_bytes == 0)
359 ret_rank = source_rank();
360 TLOG(8) << GetTraceName() <<
": receiveFragmentHeader done sts=" << sts <<
" src=" << ret_rank;
361 TLOG(7) << GetTraceName() <<
": receiveFragmentHeader: Done receiving fragment header. Moving into output.";
371 TLOG(5) << GetTraceName() <<
": receiveFragmentHeader: Returning " << ret_rank;
375 int artdaq::TCPSocketTransfer::disconnect_receive_socket_(
int fd, std::string msg)
377 TLOG(TLVL_WARNING) << GetTraceName() <<
": disconnect_receive_socket_: " << msg <<
" Closing socket " << fd;
379 std::unique_lock<std::mutex> lk(connected_fd_mutex_);
380 if (connected_fds_.count(source_rank()))
381 connected_fds_[source_rank()].erase(fd);
383 TLOG(TLVL_DEBUG) << GetTraceName() <<
": disconnect_receive_socket_: There are now " << connected_fds_[source_rank()].size() <<
" active senders.";
389 TLOG(19) << GetTraceName() <<
": receiveFragmentData: BEGIN";
390 int ret_rank = RECV_TIMEOUT;
391 if (active_receive_fd_ == -1)
393 TLOG(TLVL_ERROR) << GetTraceName() <<
": receiveFragmentData: Receive socket not connected, returning RECV_TIMEOUT (Will result in \"Unexpected return code error\")";
402 SocketState state = SocketState::Metadata;
403 int target_bytes =
sizeof(
MessHead);
406 pollfd_s.events = POLLIN | POLLPRI | POLLERR;
407 pollfd_s.fd = active_receive_fd_;
411 bool noDataWarningSent =
false;
412 last_recv_time_ = std::chrono::steady_clock::now();
415 TLOG(9) << GetTraceName() <<
": receiveFragmentData: Polling fd to see if there's data";
416 int num_fds_ready = poll(&pollfd_s, 1, 1000);
417 TLOG(TLVL_TRACE) << GetTraceName() <<
": receiveFragmentData: Polled fd to see if there's data"
418 <<
", num_fds_ready = " << num_fds_ready;
419 if (num_fds_ready <= 0)
421 if (num_fds_ready == 0)
423 TLOG(TLVL_WARNING) << GetTraceName() <<
": receiveFragmentData: No data from " << source_rank() <<
" in " << TimeUtils::GetElapsedTimeMilliseconds(last_recv_time_) <<
" ms!"
424 <<
" State = " << (state == SocketState::Metadata ?
"Metadata" :
"Data") <<
", recvd/total=" << offset <<
"/" << target_bytes <<
" (delta=" << target_bytes - offset <<
")";
426 if (TimeUtils::GetElapsedTime(last_recv_time_) > receive_disconnected_wait_s_)
428 TLOG(TLVL_WARNING) << GetTraceName() <<
": receiveFragmentData: No data received within timeout (" << TimeUtils::GetElapsedTime(last_recv_time_) <<
" / " << receive_disconnected_wait_s_ <<
" ), returning RECV_TIMEOUT";
429 disconnect_receive_socket_(active_receive_fd_,
"No data on this socket within timeout");
430 active_receive_fd_ = -1;
436 TLOG(TLVL_ERROR) <<
"Error in poll: errno=" << errno;
437 active_receive_fd_ = -1;
442 last_recv_time_ = std::chrono::steady_clock::now();
445 if (pollfd_s.revents & (POLLIN | POLLPRI))
449 else if (pollfd_s.revents & (POLLNVAL))
451 disconnect_receive_socket_(pollfd_s.fd,
"FD is closed, most likely because the peer went away.");
454 else if (pollfd_s.revents & (POLLHUP | POLLERR))
456 disconnect_receive_socket_(pollfd_s.fd,
"Poll returned POLLHUP or POLLERR, indicating problems with the sender.");
461 TLOG(TLVL_WARNING) << GetTraceName() <<
": receiveFragmentData: Wrong event received from pollfd: " << pollfd_s.revents;
462 disconnect_receive_socket_(pollfd_s.fd);
466 if (state == SocketState::Metadata)
469 buff = &(mha[offset]);
470 byte_cnt =
sizeof(
MessHead) - offset;
475 buff =
reinterpret_cast<uint8_t*
>(destination) + offset;
479 TLOG(10) << GetTraceName() <<
": receiveFragmentData: Reading " << byte_cnt <<
" bytes from socket into " << (
void*)buff;
480 sts = read(active_receive_fd_, buff, byte_cnt);
483 TLOG(10) << GetTraceName() <<
": recvFragment state=" <<
static_cast<int>(state) <<
" read=" << sts;
487 if (loop_guard > 10) { usleep(1000); }
488 if (++loop_guard > 10010)
490 TLOG(TLVL_WARNING) << GetTraceName() <<
": receiveFragmentData: loop guard triggered, returning RECV_TIMEOUT";
491 active_receive_fd_ = -1;
498 last_recv_time_ = std::chrono::steady_clock::now();
503 TLOG(TLVL_WARNING) << GetTraceName() <<
": receiveFragmentData: Error on receive, closing socket"
504 <<
" (errno=" << errno <<
": " << strerror(errno) <<
")";
505 disconnect_receive_socket_(pollfd_s.fd);
509 if (!noDataWarningSent)
511 TLOG(TLVL_WARNING) << GetTraceName() <<
": receiveFragmentData: No data received, is the sender still sending?!?";
512 noDataWarningSent =
true;
514 if (TimeUtils::GetElapsedTime(last_recv_time_) > receive_disconnected_wait_s_)
516 TLOG(TLVL_ERROR) << GetTraceName() <<
": receiveFragmentData: No data received within timeout, aborting!";
524 if (sts >= target_bytes)
526 TLOG(9) << GetTraceName() <<
": receiveFragmentData: Target read bytes reached. Changing state";
528 if (state == SocketState::Metadata)
530 state = SocketState::Data;
537 TLOG(TLVL_WARNING) << GetTraceName() <<
": receiveFragmentData: Message header indicates that a Fragment header follows when I was expecting Fragment data!";
538 active_receive_fd_ = disconnect_receive_socket_(active_receive_fd_,
"Desync detected");
543 ret_rank = source_rank();
544 TLOG(11) << GetTraceName() <<
": receiveFragmentData done sts=" << sts <<
" src=" << ret_rank;
545 TLOG(9) << GetTraceName() <<
": receiveFragmentData: Done receiving fragment. Moving into output.";
548 send_ack_(active_receive_fd_);
558 if (target_bytes == 0 && state == SocketState::Data)
560 ret_rank = source_rank();
561 TLOG(11) << GetTraceName() <<
": receiveFragmentData done sts=" << sts <<
" src=" << ret_rank;
562 TLOG(9) << GetTraceName() <<
": receiveFragmentData: Done receiving fragment. Moving into output.";
565 send_ack_(active_receive_fd_);
573 last_active_receive_fd_ = active_receive_fd_;
574 active_receive_fd_ = -1;
576 TLOG(9) << GetTraceName() <<
": receiveFragmentData: Returning rank " << ret_rank;
585 return send_fd_ != -1;
587 TLOG(TLVL_DEBUG) << GetTraceName() <<
": isRunning: There are " << getConnectedFDCount(source_rank()) <<
" fds connected.";
588 return getConnectedFDCount(source_rank()) > 0;
595 while (connected_fds_[source_rank()].size())
597 disconnect_receive_socket_(*connected_fds_[source_rank()].begin(),
"Flushing connections");
605 TLOG(12) << GetTraceName() <<
": sendFragment begin send of fragment with sequenceID=" << frag.sequenceID();
606 artdaq::Fragment grab_ownership_frag = std::move(frag);
609 if (send_fd_ == -1 && connection_was_lost_)
611 TLOG(TLVL_INFO) << GetTraceName() <<
": reconnection attempt failed, returning quickly.";
619 while (static_cast<size_t>(send_ack_diff_) > buffer_count_) usleep(10000);
622 iovec iov = {
reinterpret_cast<void*
>(grab_ownership_frag.headerAddress()),
623 detail::RawFragmentHeader::num_words() *
sizeof(RawDataType)};
625 auto sts = sendData_(&iov, 1, send_retry_timeout_us_,
true);
626 auto start_time = std::chrono::steady_clock::now();
628 while (sts == CopyStatus::kTimeout && (send_timeout_usec == 0 || TimeUtils::GetElapsedTimeMicroseconds(start_time) < send_timeout_usec) && TimeUtils::GetElapsedTimeMicroseconds(start_time) < 10000000)
630 TLOG(13) << GetTraceName() <<
": sendFragment: Timeout sending fragment";
631 sts = sendData_(&iov, 1, send_retry_timeout_us_,
true);
634 if (sts != CopyStatus::kSuccess)
return sts;
638 iov = {
reinterpret_cast<void*
>(grab_ownership_frag.headerAddress() + detail::RawFragmentHeader::num_words()),
639 grab_ownership_frag.sizeBytes() - detail::RawFragmentHeader::num_words() *
sizeof(RawDataType)};
640 sts = sendData_(&iov, 1, send_retry_timeout_us_);
641 start_time = std::chrono::steady_clock::now();
642 while (sts == CopyStatus::kTimeout && (send_timeout_usec == 0 || TimeUtils::GetElapsedTimeMicroseconds(start_time) < send_timeout_usec) && TimeUtils::GetElapsedTimeMicroseconds(start_time) < 10000000)
644 TLOG(13) << GetTraceName() <<
": sendFragment: Timeout sending fragment";
645 sts = sendData_(&iov, 1, send_retry_timeout_us_);
653 TLOG(12) << GetTraceName() <<
": sendFragment returning " << CopyStatusToString(sts);
659 TLOG(TLVL_DEBUG) << GetTraceName() <<
": sendData_ Converting buf to iovec";
660 iovec iov = {(
void*)buf, bytes};
661 return sendData_(&iov, 1, send_timeout_usec, isHeader);
669 if (timeoutMessageArmed_)
671 TLOG(TLVL_DEBUG) << GetTraceName() <<
": sendData_: Send fd is not open. Returning kTimeout";
672 timeoutMessageArmed_ =
false;
674 return CopyStatus::kTimeout;
676 timeoutMessageArmed_ =
true;
677 TLOG(14) << GetTraceName() <<
": send_timeout_usec is " << send_timeout_usec <<
", currently unused.";
680 uint32_t total_to_write_bytes = 0;
681 std::vector<iovec> iov_in(iovcnt + 1);
682 std::vector<iovec> iovv(iovcnt + 2);
684 for (ii = 0; ii < iovcnt; ++ii)
686 iov_in[ii + 1] = iov[ii];
687 total_to_write_bytes += iov[ii].iov_len;
690 MessHead mh = {0, isHeader ? MessHead::header_v0 : MessHead::data_v0, htons(source_rank()), {htonl(total_to_write_bytes)}};
691 iov_in[0].iov_base = &mh;
692 iov_in[0].iov_len =
sizeof(mh);
693 total_to_write_bytes +=
sizeof(mh);
696 ssize_t total_written_bytes = 0;
697 ssize_t per_write_max_bytes = (32 * 1024);
699 size_t in_iov_idx = 0;
700 size_t out_iov_idx = 0;
701 ssize_t this_write_bytes = 0;
708 (in_iov_idx + out_iov_idx) < iov_in.size() && this_write_bytes < per_write_max_bytes;
711 this_write_bytes += iov_in[in_iov_idx + out_iov_idx].iov_len;
712 iovv[out_iov_idx] = iov_in[in_iov_idx + out_iov_idx];
714 if (this_write_bytes > per_write_max_bytes)
716 iovv[out_iov_idx - 1].iov_len -= this_write_bytes - per_write_max_bytes;
717 this_write_bytes = per_write_max_bytes;
722 #ifndef __OPTIMIZE__ // This can be an expensive TRACE call (even if disabled) due to multiplicity of calls
723 TLOG(14) << GetTraceName() <<
": sendFragment b4 writev " << std::setw(7) << total_written_bytes <<
" total_written_bytes send_fd_=" << send_fd_ <<
" in_idx=" << in_iov_idx
724 <<
" iovcnt=" << out_iov_idx <<
" 1st.len=" << iovv[0].iov_len;
729 memset(&msg, 0,
sizeof(msghdr));
730 msg.msg_iov = &(iovv[0]);
731 msg.msg_iovlen = out_iov_idx;
732 sts = sendmsg(send_fd_, &msg, MSG_NOSIGNAL | (blocking ? 0 : MSG_DONTWAIT));
734 sts = writev(send_fd_, &(iovv[0]), out_iov_idx);
740 if (errno == EAGAIN )
742 TLOG(TLVL_DEBUG) << GetTraceName() <<
": sendFragment EWOULDBLOCK";
746 fcntl(send_fd_, F_SETFL, 0);
751 TLOG(TLVL_WARNING) << GetTraceName() <<
": sendFragment_: WRITE ERROR " << errno <<
": " << strerror(errno);
755 connection_was_lost_ =
true;
758 else if (sts != this_write_bytes)
761 TLOG(TLVL_DEBUG) << GetTraceName() <<
": sendFragment writev sts(" << sts <<
")!=requested_send_bytes(" << this_write_bytes <<
")";
762 total_written_bytes += sts;
764 for (ii = 0; (size_t)sts >= iovv[ii].iov_len; ++ii)
765 sts -= iovv[ii].iov_len;
767 iovv[ii].iov_len -= sts;
768 iovv[ii].iov_base = (uint8_t*)(iovv[ii].iov_base) + sts;
773 iovv[out_iov_idx] = iovv[ii];
775 this_write_bytes = iovv[out_iov_idx].iov_len;
780 unsigned long additional = ((
unsigned long)iov_in[in_iov_idx].iov_base + iov_in[in_iov_idx].iov_len) - ((
unsigned long)iovv[out_iov_idx].iov_base + iovv[out_iov_idx].iov_len);
783 iovv[out_iov_idx].iov_len += additional;
784 this_write_bytes += additional;
785 if (this_write_bytes > per_write_max_bytes)
787 iovv[out_iov_idx].iov_len -= this_write_bytes - per_write_max_bytes;
788 this_write_bytes = per_write_max_bytes;
792 TLOG(TLVL_TRACE) << GetTraceName() <<
": sendFragment writev sts!=: this_write_bytes=" << this_write_bytes
793 <<
" out_iov_idx=" << out_iov_idx
794 <<
" additional=" << additional
799 #ifndef __OPTIMIZE__ // This can be an expensive TRACE call (even if disabled) due to multiplicity of calls
800 TLOG(TLVL_TRACE) << GetTraceName() <<
": sendFragment writev sts(" << sts <<
")==requested_send_bytes(" << this_write_bytes <<
")";
802 total_written_bytes += sts;
804 iovv[out_iov_idx].iov_base = (uint8_t*)(iovv[out_iov_idx].iov_base) + iovv[out_iov_idx].iov_len;
805 iovv[out_iov_idx].iov_len = 0;
806 in_iov_idx += out_iov_idx;
807 this_write_bytes = 0;
809 unsigned long additional = ((
unsigned long)iov_in[in_iov_idx].iov_base + iov_in[in_iov_idx].iov_len) - ((
unsigned long)iovv[out_iov_idx].iov_base + iovv[out_iov_idx].iov_len);
812 iovv[out_iov_idx].iov_len += additional;
813 this_write_bytes += additional;
814 if (this_write_bytes > per_write_max_bytes)
816 iovv[out_iov_idx].iov_len -= this_write_bytes - per_write_max_bytes;
817 this_write_bytes = per_write_max_bytes;
819 if (out_iov_idx != 0)
820 iovv[0] = iovv[out_iov_idx];
829 }
while (total_written_bytes < total_to_write_bytes);
830 if (total_written_bytes > total_to_write_bytes)
831 TLOG(TLVL_ERROR) << GetTraceName() <<
": sendFragment program error: too many bytes transferred";
837 fcntl(send_fd_, F_SETFL, O_NONBLOCK);
840 sts = total_written_bytes -
sizeof(
MessHead);
842 TLOG(14) << GetTraceName() <<
": sendFragment sts=" << sts;
846 void artdaq::TCPSocketTransfer::connect_()
848 auto start_time = std::chrono::steady_clock::now();
851 while (send_fd_ == -1 && TimeUtils::GetElapsedTimeMicroseconds(start_time) < send_retry_timeout_us_ * 10)
853 TLOG(TLVL_DEBUG) << GetTraceName() <<
": Connecting sender socket";
854 int sndbuf_bytes =
static_cast<int>(sndbuf_);
855 if (sndbuf_ > INT_MAX)
857 sndbuf_bytes = INT_MAX;
858 TLOG(TLVL_WARNING) <<
"Requested SNDBUF " << sndbuf_ <<
" too large, setting to INT_MAX: " << INT_MAX;
860 TLOG(TLVL_DEBUG) <<
"Requested SNDBUF is " << sndbuf_bytes;
862 send_fd_ =
TCPConnect(hostMap_[destination_rank()].c_str(), portMan->GetTCPSocketTransferPort(destination_rank()), O_NONBLOCK, sndbuf_bytes);
865 if (connection_was_lost_) {
break; }
868 usleep(send_retry_timeout_us_);
874 TLOG(TLVL_DEBUG) << GetTraceName() <<
": connect_ " + hostMap_[destination_rank()] +
":" << portMan->GetTCPSocketTransferPort(destination_rank()) <<
" send_fd_=" << send_fd_;
878 TLOG(TLVL_DEBUG) << GetTraceName() <<
": connect_: Writing connect message";
879 MessHead mh = {0, MessHead::connect_v0, htons(source_rank()), {htonl(CONN_MAGIC)}};
880 ssize_t sts = write(send_fd_, &mh,
sizeof(mh));
883 TLOG(TLVL_ERROR) << GetTraceName() <<
": connect_: Error writing connect message!";
891 TLOG(TLVL_INFO) << GetTraceName() <<
": connect_: Successfully connected";
894 connection_was_lost_ =
false;
898 if (ack_listen_thread_ && ack_listen_thread_->joinable()) ack_listen_thread_->join();
899 TLOG(TLVL_INFO) << GetTraceName() <<
": Starting Ack Listener Thread";
903 ack_listen_thread_ = std::make_unique<boost::thread>(&TCPSocketTransfer::receive_acks_,
this);
905 catch (
const boost::exception& e)
907 TLOG(TLVL_ERROR) <<
"Caught boost::exception starting TCP Socket Ack Listen thread: " << boost::diagnostic_information(e) <<
", errno=" << errno;
908 std::cerr <<
"Caught boost::exception starting TCP Socket Ack Listen thread: " << boost::diagnostic_information(e) <<
", errno=" << errno << std::endl;
915 void artdaq::TCPSocketTransfer::reconnect_()
919 TLOG(TLVL_TRACE) << GetTraceName() <<
": check/reconnect";
924 void artdaq::TCPSocketTransfer::start_listen_thread_()
926 std::unique_lock<std::mutex> start_lock(listen_thread_mutex_);
927 if (listen_thread_refcount_ == 0)
929 if (listen_thread_ && listen_thread_->joinable()) listen_thread_->join();
930 listen_thread_refcount_ = 1;
931 TLOG(TLVL_INFO) << GetTraceName() <<
": Starting Listener Thread";
935 listen_thread_ = std::make_unique<boost::thread>(&TCPSocketTransfer::listen_, portMan->GetTCPSocketTransferPort(destination_rank()), rcvbuf_);
937 catch (
const boost::exception& e)
939 TLOG(TLVL_ERROR) <<
"Caught boost::exception starting TCP Socket Listen thread: " << boost::diagnostic_information(e) <<
", errno=" << errno;
940 std::cerr <<
"Caught boost::exception starting TCP Socket Listen thread: " << boost::diagnostic_information(e) <<
", errno=" << errno << std::endl;
946 listen_thread_refcount_++;
951 void artdaq::TCPSocketTransfer::receive_acks_()
953 while (send_fd_ >= 0)
956 pollfd_s.events = POLLIN | POLLPRI;
957 pollfd_s.fd = send_fd_;
959 TLOG(18) << GetTraceName() <<
": receive_acks_: Polling fd to see if there's data";
960 int num_fds_ready = poll(&pollfd_s, 1, 1000);
961 if (num_fds_ready <= 0)
963 if (num_fds_ready == 0)
965 TLOG(18) << GetTraceName() <<
": receive_acks_: No data on receive socket";
969 TLOG(TLVL_ERROR) <<
"Error in poll: errno=" << errno;
973 if (pollfd_s.revents & (POLLIN | POLLPRI))
979 TLOG(TLVL_DEBUG) << GetTraceName() <<
": receive_acks_: Wrong event received from pollfd: " << pollfd_s.revents;
984 auto sts = read(send_fd_, &mh,
sizeof(mh));
986 if (sts !=
sizeof(mh))
988 TLOG(TLVL_ERROR) << GetTraceName() <<
": receive_ack_: Wrong message header length received! (actual " << sts <<
" != " <<
sizeof(mh) <<
" expected)";
996 TLOG(TLVL_ERROR) << GetTraceName() <<
": receive_ack_: Received ack for different sender! Rank=" << my_rank <<
", hdr=" << mh.
source_id;
1001 TLOG(TLVL_ERROR) << GetTraceName() <<
": receive_ack_: Wrong magic bytes in header!";
1005 TLOG(17) << GetTraceName() <<
": receive_acks_: Received ack message, diff is now " << (send_ack_diff_.load() - 1);
1010 void artdaq::TCPSocketTransfer::send_ack_(
int fd)
1012 MessHead mh = {0, MessHead::ack_v0, htons(source_rank()), {htonl(ACK_MAGIC)}};
1013 write(fd, &mh,
sizeof(mh));
1017 void artdaq::TCPSocketTransfer::listen_(
int port,
size_t rcvbuf)
1020 while (listen_thread_refcount_ > 0)
1022 TLOG(TLVL_TRACE) <<
"listen_: Listening/accepting new connections on port " << port;
1023 if (listen_fd == -1)
1025 TLOG(TLVL_DEBUG) <<
"listen_: Opening listener";
1028 if (listen_fd == -1)
1030 TLOG(TLVL_DEBUG) <<
"listen_: Error creating listen_fd!";
1035 timeval tv = {2, 0};
1038 FD_SET(listen_fd, &rfds);
1040 res = select(listen_fd + 1, &rfds, (fd_set*)0, (fd_set*)0, &tv);
1045 socklen_t arglen =
sizeof(un);
1047 TLOG(TLVL_DEBUG) <<
"listen_: Calling accept";
1048 fd = accept(listen_fd, (sockaddr*)&un, &arglen);
1049 TLOG(TLVL_DEBUG) <<
"listen_: Done with accept";
1051 TLOG(TLVL_DEBUG) <<
"listen_: Reading connect message";
1052 socklen_t lenlen =
sizeof(tv);
1054 setsockopt(fd, SOL_SOCKET, SO_RCVTIMEO, &tv, lenlen);
1056 uint64_t mark_us = TimeUtils::gettimeofday_us();
1057 sts = read(fd, &mh,
sizeof(mh));
1058 uint64_t delta_us = TimeUtils::gettimeofday_us() - mark_us;
1059 TLOG(TLVL_DEBUG) <<
"listen_: Read of connect message took " << delta_us <<
" microseconds.";
1060 if (sts !=
sizeof(mh))
1062 TLOG(TLVL_DEBUG) <<
"listen_: Wrong message header length received!";
1071 TLOG(TLVL_DEBUG) <<
"listen_: Wrong magic bytes in header!";
1077 std::unique_lock<std::mutex> lk(connected_fd_mutex_);
1078 connected_fds_[mh.
source_id].insert(fd);
1080 TLOG(TLVL_INFO) <<
"listen_: New fd is " << fd <<
" for source rank " << mh.
source_id;
1084 TLOG(16) <<
"listen_: No connections in timeout interval!";
1088 TLOG(TLVL_INFO) <<
"listen_: Shutting down connection listener";
1089 if (listen_fd != -1) close(listen_fd);
1090 std::unique_lock<std::mutex> lk(connected_fd_mutex_);
1091 auto it = connected_fds_.begin();
1092 while (it != connected_fds_.end())
1094 auto& fd_set = it->second;
1095 auto rank_it = fd_set.begin();
1096 while (rank_it != fd_set.end())
1099 rank_it = fd_set.erase(rank_it);
1101 it = connected_fds_.erase(it);
bool isRunning() override
Determine whether the TransferInterface plugin is able to send/receive data.
virtual int source_rank() const
Get the source rank for this TransferInterface instance.
int TCPConnect(char const *host_in, int dflt_port, long flags=0, int sndbufsiz=0)
Connect to a host on a given port.
uint32_t conn_magic
unsigned first is better for MessHead initializer: {0,0,my_node_idx_,CONN_MAGIC}
hostMap_t MakeHostMap(fhicl::ParameterSet pset, hostMap_t map=hostMap_t())
Make a hostMap_t from a HostMap::Config ParameterSet
This TransferInterface is a Receiver.
int receiveFragmentData(RawDataType *destination, size_t wordCount) override
Receive the body of a Fragment to the given destination pointer.
int TCP_listen_fd(int port, int rcvbuf)
Create a TCP listening socket on the given port and INADDR_ANY, with the given receive buffer...
TCPSocketTransfer(fhicl::ParameterSet const &ps, Role role)
TCPSocketTransfer Constructor.
int receiveFragmentHeader(detail::RawFragmentHeader &header, size_t receiveTimeout) override
Receive a Fragment Header from the transport mechanism.
This TransferInterface is a Sender.
void flush_buffers() override
Flush any in-flight data. This should be used by the receiver after the receive loop has ended...
int32_t byte_count
use CONN_MAGIC for connect_v0, data that follow for data_v0 (and 0 lenght data)
Some error occurred, but no exception was thrown.
Role
Used to determine if a TransferInterface is a Sender or Receiver.
int64_t source_id
Rank of the source.
MessType message_type
Message Type.
The send operation completed successfully.
This interface defines the functions used to transfer data between artdaq applications.
TransferInterface implementation plugin that sends data using TCP sockets.
This header is sent by the TCPSocket_transfer to allow for more efficient writev calls.
CopyStatus
Returned from the send functions, this enumeration describes the possible return codes. If an exception occurs, it will be thrown and should be handled normally.