9 #include <sys/socket.h>
11 #include <arpa/inet.h>
12 #include <sys/types.h>
21 #define TRACE_NAME (app_name + "_TCPSocketTransfer").c_str()
22 #include "artdaq/DAQdata/Globals.hh"
25 #include "artdaq/TransferPlugins/TCPSocketTransfer.hh"
28 #include "artdaq/TransferPlugins/detail/Timeout.hh"
29 #include "artdaq/TransferPlugins/detail/SRSockets.hh"
30 #include "artdaq-core/Data/Fragment.hh"
31 #include "artdaq-core/Utilities/TimeUtils.hh"
34 std::atomic<int> artdaq::TCPSocketTransfer::listen_thread_refcount_(0);
35 std::unique_ptr<boost::thread> artdaq::TCPSocketTransfer::listen_thread_ =
nullptr;
36 std::map<int, std::set<int>> artdaq::TCPSocketTransfer::connected_fds_ = std::map<int, std::set<int>>();
37 std::mutex artdaq::TCPSocketTransfer::listen_thread_mutex_;
38 std::mutex artdaq::TCPSocketTransfer::connected_fd_mutex_;
44 , active_receive_fd_(-1)
45 , last_active_receive_fd_(-1)
46 , rcvbuf_(pset.get<size_t>(
"tcp_receive_buffer_size", 0))
47 , sndbuf_(pset.get<size_t>(
"tcp_send_buffer_size", max_fragment_size_words_ * sizeof(artdaq::RawDataType) * buffer_count_))
48 , send_retry_timeout_us_(pset.get<size_t>(
"send_retry_timeout_us", 1000000))
49 , timeoutMessageArmed_(true)
51 , receive_disconnected_wait_s_(pset.get<double>(
"receive_socket_disconnected_wait_s", 10.0))
52 , receive_err_wait_us_(pset.get<size_t>(
"receive_socket_disconnected_wait_us", 10000))
53 , receive_socket_has_been_connected_(false)
56 TLOG(TLVL_DEBUG) << GetTraceName() <<
" Constructor: pset=" << pset.to_string() <<
", role=" << (role ==
TransferInterface::Role::kReceive ?
"kReceive" :
"kSend");
61 TLOG(TLVL_DEBUG) << GetTraceName() <<
": Listening for connections";
62 start_listen_thread_();
63 TLOG(TLVL_DEBUG) << GetTraceName() <<
": Done Listening";
68 TLOG(TLVL_DEBUG) << GetTraceName() <<
": Connecting to destination";
70 TLOG(TLVL_DEBUG) << GetTraceName() <<
": Done Connecting";
72 TLOG(TLVL_DEBUG) << GetTraceName() <<
": End of Constructor";
75 artdaq::TCPSocketTransfer::~TCPSocketTransfer() noexcept
77 TLOG(TLVL_DEBUG) << GetTraceName() <<
": Shutting down TCPSocketTransfer";
86 timeval tv = { 0,100000 };
87 socklen_t len =
sizeof(tv);
88 setsockopt(send_fd_, SOL_SOCKET, SO_SNDTIMEO, &tv, len);
89 write(send_fd_, &mh,
sizeof(mh));
97 std::unique_lock<std::mutex> fd_lock(connected_fd_mutex_);
98 if (connected_fds_.count(source_rank()))
100 auto it = connected_fds_[source_rank()].begin();
101 while (it != connected_fds_[source_rank()].end())
104 it = connected_fds_[source_rank()].erase(it);
106 connected_fds_.erase(source_rank());
108 if (ack_listen_thread_ && ack_listen_thread_->joinable()) ack_listen_thread_->join();
111 std::unique_lock<std::mutex> lk(listen_thread_mutex_);
112 listen_thread_refcount_--;
113 if (listen_thread_refcount_ <= 0 && listen_thread_ && listen_thread_->joinable())
115 listen_thread_->join();
119 TLOG(TLVL_DEBUG) << GetTraceName() <<
": End of Destructor";
124 TLOG(5) << GetTraceName() <<
": receiveFragmentHeader: BEGIN";
125 int ret_rank = RECV_TIMEOUT;
128 if (getConnectedFDCount(source_rank()) == 0)
137 TLOG(7) << GetTraceName() <<
": receiveFragmentHeader: Receive socket not connected, returning RECV_TIMEOUT";
138 usleep(receive_err_wait_us_);
141 receive_socket_has_been_connected_ =
true;
142 last_recv_time_ = std::chrono::steady_clock::now();
144 TLOG(5) << GetTraceName() <<
": receiveFragmentHeader timeout_usec=" << timeout_usec;
149 SocketState state = SocketState::Metadata;
150 int target_bytes =
sizeof(
MessHead);
151 uint64_t start_time_us = TimeUtils::gettimeofday_us();
163 if (timeout_usec == 0)
166 timeout_ms = (timeout_usec + 999) / 1000;
169 bool noDataWarningSent =
false;
172 while (!done && getConnectedFDCount(source_rank()) > 0)
174 if (active_receive_fd_ == -1)
178 std::vector<pollfd> pollfds;
180 std::unique_lock<std::mutex> lk(connected_fd_mutex_);
181 fd_count = connected_fds_[source_rank()].size();
182 pollfds.resize(fd_count);
183 auto iter = connected_fds_[source_rank()].begin();
184 for (
size_t ii = 0; ii < fd_count; ++ii)
186 pollfds[ii].events = POLLIN | POLLPRI | POLLERR;
187 pollfds[ii].fd = *iter;
192 int num_fds_ready = poll(&pollfds[0], fd_count, timeout_ms);
193 if (num_fds_ready <= 0)
195 TLOG(5) << GetTraceName() <<
": receiveFragmentHeader: No data on receive socket, returning RECV_TIMEOUT";
200 if (last_active_receive_fd_ != -1)
202 for (
auto& pollfd : pollfds)
205 if (pollfd.fd == last_active_receive_fd_)
212 int active_index = -1;
213 short anomolous_events = 0;
214 for (
size_t ii = index; ii < index + pollfds.size(); ++ii)
216 auto pollfd_index = (ii + index) % pollfds.size();
217 if (pollfds[pollfd_index].revents & (POLLIN | POLLPRI))
219 active_index = pollfd_index;
220 active_receive_fd_ = pollfds[active_index].fd;
221 active_revents_ = pollfds[active_index].revents;;
224 else if (pollfds[pollfd_index].revents & (POLLHUP | POLLERR))
226 disconnect_receive_socket_(pollfds[pollfd_index].fd,
"Poll returned POLLHUP or POLLERR, indicating problems with the sender.");
229 else if (pollfds[pollfd_index].revents & (POLLNVAL))
231 TLOG(TLVL_DEBUG) << GetTraceName() <<
": receiveFragmentHeader: FD is closed, most likely because the peer went away. Removing from fd list.";
232 disconnect_receive_socket_(pollfds[pollfd_index].fd,
"FD is closed, most likely because the peer went away.");
235 else if (pollfds[pollfd_index].revents)
237 anomolous_events |= pollfds[pollfd_index].revents;
241 if (active_index == -1)
243 if (anomolous_events)
244 TLOG(TLVL_DEBUG) << GetTraceName() <<
": receiveFragmentHeader: Wrong event received from a pollfd. Mask: " <<
static_cast<int>(anomolous_events);
245 active_receive_fd_ = -1;
249 if (!done && timeout_usec > 0)
252 size_t delta_us = TimeUtils::gettimeofday_us() - start_time_us;
253 if (delta_us > timeout_usec)
257 timeout_ms = ((timeout_usec - delta_us) + 999) / 1000;
260 if (loop_guard > 10) { usleep(1000); }
261 if (++loop_guard > 10010)
263 TLOG(TLVL_WARNING) << GetTraceName() <<
": receiveFragmentHeader: loop guard triggered, returning RECV_TIMEOUT";
264 usleep(receive_err_wait_us_);
265 active_receive_fd_ = -1;
269 if (state == SocketState::Metadata)
272 buff = &(mha[offset]);
273 byte_cnt =
sizeof(
MessHead) - offset;
278 buff =
reinterpret_cast<uint8_t*
>(&header) + offset;
279 byte_cnt = target_bytes - offset;
291 TLOG(6) << GetTraceName() <<
": receiveFragmentHeader: Reading " << byte_cnt <<
" bytes from socket";
292 sts = read(active_receive_fd_, buff, byte_cnt);
293 TLOG(6) << GetTraceName() <<
": receiveFragmentHeader: Done with read";
297 last_recv_time_ = std::chrono::steady_clock::now();
300 TLOG(7) << GetTraceName() <<
": receiveFragmentHeader state=" <<
static_cast<int>(state) <<
" read=" << sts;
303 TLOG(TLVL_WARNING) << GetTraceName() <<
": receiveFragmentHeader: Error on receive, closing socket " <<
" (errno=" << errno <<
": " << strerror(errno) <<
")";
304 active_receive_fd_ = disconnect_receive_socket_(active_receive_fd_);
308 if (!noDataWarningSent) {
309 TLOG(TLVL_WARNING) << GetTraceName() <<
": receiveFragmentHeader: No data received, is the sender still sending?!?";
310 noDataWarningSent =
true;
312 if (TimeUtils::GetElapsedTime(last_recv_time_) > receive_disconnected_wait_s_)
314 TLOG(TLVL_ERROR) << GetTraceName() <<
": receiveFragmentHeader: No data received within timeout, aborting!";
322 if (sts >= target_bytes)
324 TLOG(7) << GetTraceName() <<
": receiveFragmentHeader: Target read bytes reached. Changing state";
326 if (state == SocketState::Metadata)
328 state = SocketState::Data;
332 TLOG(7) << GetTraceName() <<
": receiveFragmentHeader: Expected header size = " << target_bytes <<
", sizeof(RawFragmentHeader) = " <<
sizeof(artdaq::detail::RawFragmentHeader);
337 active_receive_fd_ = disconnect_receive_socket_(active_receive_fd_,
"Stop Message received.");
341 TLOG(TLVL_WARNING) << GetTraceName() <<
": receiveFragmentHeader: Message header indicates that Fragment data follows when I was expecting a Fragment header!";
342 active_receive_fd_ = disconnect_receive_socket_(active_receive_fd_,
"Desync detected");
345 if (target_bytes == 0)
353 ret_rank = source_rank();
354 TLOG(8) << GetTraceName() <<
": receiveFragmentHeader done sts=" << sts <<
" src=" << ret_rank;
355 TLOG(7) << GetTraceName() <<
": receiveFragmentHeader: Done receiving fragment header. Moving into output.";
365 TLOG(5) << GetTraceName() <<
": receiveFragmentHeader: Returning " << ret_rank;
369 int artdaq::TCPSocketTransfer::disconnect_receive_socket_(
int fd, std::string msg)
371 TLOG(TLVL_WARNING) << GetTraceName() <<
": disconnect_receive_socket_: " << msg <<
" Closing socket " << fd;
373 std::unique_lock<std::mutex> lk(connected_fd_mutex_);
374 if (connected_fds_.count(source_rank()))
375 connected_fds_[source_rank()].erase(fd);
377 TLOG(TLVL_DEBUG) << GetTraceName() <<
": disconnect_receive_socket_: There are now " << connected_fds_[source_rank()].size() <<
" active senders.";
383 TLOG(19) << GetTraceName() <<
": receiveFragmentData: BEGIN";
384 int ret_rank = RECV_TIMEOUT;
385 if (active_receive_fd_ == -1)
387 TLOG(TLVL_ERROR) << GetTraceName() <<
": receiveFragmentData: Receive socket not connected, returning RECV_TIMEOUT (Will result in \"Unexpected return code error\")";
396 SocketState state = SocketState::Metadata;
397 int target_bytes =
sizeof(
MessHead);
400 pollfd_s.events = POLLIN | POLLPRI | POLLERR;
401 pollfd_s.fd = active_receive_fd_;
405 bool noDataWarningSent =
false;
406 last_recv_time_ = std::chrono::steady_clock::now();
409 TLOG(9) << GetTraceName() <<
": receiveFragmentData: Polling fd to see if there's data";
410 int num_fds_ready = poll(&pollfd_s, 1, 1000);
411 TLOG(TLVL_TRACE) << GetTraceName() <<
": receiveFragmentData: Polled fd to see if there's data"
412 <<
", num_fds_ready = " << num_fds_ready;
413 if (num_fds_ready <= 0)
415 if (num_fds_ready == 0)
417 TLOG(TLVL_WARNING) << GetTraceName() <<
": receiveFragmentData: No data from " << source_rank() <<
" in " << TimeUtils::GetElapsedTimeMilliseconds(last_recv_time_) <<
" ms!"
418 <<
" State = " << (state == SocketState::Metadata ?
"Metadata" :
"Data") <<
", recvd/total=" << offset <<
"/" << target_bytes <<
" (delta=" << target_bytes - offset <<
")";
420 if (TimeUtils::GetElapsedTime(last_recv_time_) > receive_disconnected_wait_s_)
422 TLOG(TLVL_WARNING) << GetTraceName() <<
": receiveFragmentData: No data received within timeout (" << TimeUtils::GetElapsedTime(last_recv_time_) <<
" / " << receive_disconnected_wait_s_ <<
" ), returning RECV_TIMEOUT";
423 disconnect_receive_socket_(active_receive_fd_,
"No data on this socket within timeout");
424 active_receive_fd_ = -1;
430 TLOG(TLVL_ERROR) <<
"Error in poll: errno=" << errno;
431 active_receive_fd_ = -1;
434 else { last_recv_time_ = std::chrono::steady_clock::now(); }
436 if (pollfd_s.revents & (POLLIN | POLLPRI))
440 else if (pollfd_s.revents & (POLLNVAL))
442 disconnect_receive_socket_(pollfd_s.fd,
"FD is closed, most likely because the peer went away.");
445 else if (pollfd_s.revents & (POLLHUP | POLLERR))
447 disconnect_receive_socket_(pollfd_s.fd,
"Poll returned POLLHUP or POLLERR, indicating problems with the sender.");
452 TLOG(TLVL_WARNING) << GetTraceName() <<
": receiveFragmentData: Wrong event received from pollfd: " << pollfd_s.revents;
453 disconnect_receive_socket_(pollfd_s.fd);
457 if (state == SocketState::Metadata)
460 buff = &(mha[offset]);
461 byte_cnt =
sizeof(
MessHead) - offset;
466 buff =
reinterpret_cast<uint8_t*
>(destination) + offset;
470 TLOG(10) << GetTraceName() <<
": receiveFragmentData: Reading " << byte_cnt <<
" bytes from socket into " << (
void*)buff;
471 sts = read(active_receive_fd_, buff, byte_cnt);
474 TLOG(10) << GetTraceName() <<
": recvFragment state=" <<
static_cast<int>(state) <<
" read=" << sts;
478 if (loop_guard > 10) { usleep(1000); }
479 if (++loop_guard > 10010)
481 TLOG(TLVL_WARNING) << GetTraceName() <<
": receiveFragmentData: loop guard triggered, returning RECV_TIMEOUT";
482 active_receive_fd_ = -1;
489 last_recv_time_ = std::chrono::steady_clock::now();
494 TLOG(TLVL_WARNING) << GetTraceName() <<
": receiveFragmentData: Error on receive, closing socket"
495 <<
" (errno=" << errno <<
": " << strerror(errno) <<
")";
496 disconnect_receive_socket_(pollfd_s.fd);
500 if (!noDataWarningSent) {
501 TLOG(TLVL_WARNING) << GetTraceName() <<
": receiveFragmentData: No data received, is the sender still sending?!?";
502 noDataWarningSent =
true;
504 if (TimeUtils::GetElapsedTime(last_recv_time_) > receive_disconnected_wait_s_)
506 TLOG(TLVL_ERROR) << GetTraceName() <<
": receiveFragmentData: No data received within timeout, aborting!";
514 if (sts >= target_bytes)
516 TLOG(9) << GetTraceName() <<
": receiveFragmentData: Target read bytes reached. Changing state";
518 if (state == SocketState::Metadata)
520 state = SocketState::Data;
528 TLOG(TLVL_WARNING) << GetTraceName() <<
": receiveFragmentData: Message header indicates that a Fragment header follows when I was expecting Fragment data!";
529 active_receive_fd_ = disconnect_receive_socket_(active_receive_fd_,
"Desync detected");
534 ret_rank = source_rank();
535 TLOG(11) << GetTraceName() <<
": receiveFragmentData done sts=" << sts <<
" src=" << ret_rank;
536 TLOG(9) << GetTraceName() <<
": receiveFragmentData: Done receiving fragment. Moving into output.";
539 send_ack_(active_receive_fd_);
549 if (target_bytes == 0 && state == SocketState::Data)
551 ret_rank = source_rank();
552 TLOG(11) << GetTraceName() <<
": receiveFragmentData done sts=" << sts <<
" src=" << ret_rank;
553 TLOG(9) << GetTraceName() <<
": receiveFragmentData: Done receiving fragment. Moving into output.";
556 send_ack_(active_receive_fd_);
564 last_active_receive_fd_ = active_receive_fd_;
565 active_receive_fd_ = -1;
567 TLOG(9) << GetTraceName() <<
": receiveFragmentData: Returning rank " << ret_rank;
576 return send_fd_ != -1;
578 TLOG(TLVL_DEBUG) << GetTraceName() <<
": isRunning: There are " << getConnectedFDCount(source_rank()) <<
" fds connected.";
579 return getConnectedFDCount(source_rank()) > 0;
586 while(connected_fds_[source_rank()].size()) {
587 disconnect_receive_socket_(*connected_fds_[source_rank()].begin(),
"Flushing connections");
595 TLOG(12) << GetTraceName() <<
": sendFragment begin send of fragment with sequenceID="<<frag.sequenceID();
596 artdaq::Fragment grab_ownership_frag = std::move(frag);
603 while (static_cast<size_t>(send_ack_diff_) > buffer_count_) usleep(10000);
606 iovec iov = {
reinterpret_cast<void*
>(grab_ownership_frag.headerAddress()),
607 detail::RawFragmentHeader::num_words() *
sizeof(RawDataType) };
609 auto sts = sendData_(&iov, 1, send_retry_timeout_us_,
true);
610 auto start_time = std::chrono::steady_clock::now();
612 while (sts == CopyStatus::kTimeout && (send_timeout_usec == 0 || TimeUtils::GetElapsedTimeMicroseconds(start_time) < send_timeout_usec) && TimeUtils::GetElapsedTimeMicroseconds(start_time) < 10000000)
614 TLOG(13) << GetTraceName() <<
": sendFragment: Timeout sending fragment";
615 sts = sendData_(&iov, 1, send_retry_timeout_us_,
true);
618 if (sts != CopyStatus::kSuccess)
return sts;
622 iov = {
reinterpret_cast<void*
>(grab_ownership_frag.headerAddress() + detail::RawFragmentHeader::num_words()),
623 grab_ownership_frag.sizeBytes() - detail::RawFragmentHeader::num_words() *
sizeof(RawDataType)
625 sts = sendData_(&iov, 1, send_retry_timeout_us_);
626 start_time = std::chrono::steady_clock::now();
627 while (sts == CopyStatus::kTimeout && (send_timeout_usec == 0 || TimeUtils::GetElapsedTimeMicroseconds(start_time) < send_timeout_usec) && TimeUtils::GetElapsedTimeMicroseconds(start_time) < 10000000)
629 TLOG(13) << GetTraceName() <<
": sendFragment: Timeout sending fragment";
630 sts = sendData_(&iov, 1, send_retry_timeout_us_);
638 TLOG(12) << GetTraceName() <<
": sendFragment returning " << CopyStatusToString(sts);
644 TLOG(TLVL_DEBUG) << GetTraceName() <<
": sendData_ Converting buf to iovec";
645 iovec iov = { (
void*)buf, bytes };
646 return sendData_(&iov, 1, send_timeout_usec, isHeader);
654 if (timeoutMessageArmed_)
656 TLOG(TLVL_DEBUG) << GetTraceName() <<
": sendData_: Send fd is not open. Returning kTimeout";
657 timeoutMessageArmed_ =
false;
659 return CopyStatus::kTimeout;
661 timeoutMessageArmed_ =
true;
662 TLOG(14) << GetTraceName() <<
": send_timeout_usec is " << send_timeout_usec <<
", currently unused.";
665 uint32_t total_to_write_bytes = 0;
666 std::vector<iovec> iov_in(iovcnt + 1);
667 std::vector<iovec> iovv(iovcnt + 2);
669 for (ii = 0; ii < iovcnt; ++ii)
671 iov_in[ii + 1] = iov[ii];
672 total_to_write_bytes += iov[ii].iov_len;
675 MessHead mh = { 0,isHeader ? MessHead::header_v0 : MessHead::data_v0,htons(source_rank()),{htonl(total_to_write_bytes)} };
676 iov_in[0].iov_base = &mh;
677 iov_in[0].iov_len =
sizeof(mh);
678 total_to_write_bytes +=
sizeof(mh);
681 ssize_t total_written_bytes = 0;
682 ssize_t per_write_max_bytes = (32 * 1024);
684 size_t in_iov_idx = 0;
685 size_t out_iov_idx = 0;
686 ssize_t this_write_bytes = 0;
693 (in_iov_idx + out_iov_idx) < iov_in.size() && this_write_bytes < per_write_max_bytes;
696 this_write_bytes += iov_in[in_iov_idx + out_iov_idx].iov_len;
697 iovv[out_iov_idx] = iov_in[in_iov_idx + out_iov_idx];
699 if (this_write_bytes > per_write_max_bytes)
701 iovv[out_iov_idx - 1].iov_len -= this_write_bytes - per_write_max_bytes;
702 this_write_bytes = per_write_max_bytes;
707 #ifndef __OPTIMIZE__ // This can be an expensive TRACE call (even if disabled) due to multiplicity of calls
708 TLOG(14) << GetTraceName() <<
": sendFragment b4 writev " << std::setw(7) << total_written_bytes <<
" total_written_bytes send_fd_=" << send_fd_ <<
" in_idx=" << in_iov_idx
709 <<
" iovcnt=" << out_iov_idx <<
" 1st.len=" << iovv[0].iov_len;
712 sts = writev(send_fd_, &(iovv[0]), out_iov_idx);
717 if (errno == EAGAIN )
719 TLOG(TLVL_DEBUG) << GetTraceName() <<
": sendFragment EWOULDBLOCK";
720 fcntl(send_fd_, F_SETFL, 0);
725 TLOG(TLVL_WARNING) << GetTraceName() <<
": sendFragment_: WRITE ERROR: " << strerror(errno);
731 else if (sts != this_write_bytes)
734 TLOG(TLVL_DEBUG) << GetTraceName() <<
": sendFragment writev sts(" << sts <<
")!=requested_send_bytes(" << this_write_bytes <<
")";
735 total_written_bytes += sts;
737 for (ii = 0; (size_t)sts >= iovv[ii].iov_len; ++ii)
738 sts -= iovv[ii].iov_len;
740 iovv[ii].iov_len -= sts;
741 iovv[ii].iov_base = (uint8_t*)(iovv[ii].iov_base) + sts;
746 iovv[out_iov_idx] = iovv[ii];
748 this_write_bytes = iovv[out_iov_idx].iov_len;
753 unsigned long additional = ((
unsigned long)iov_in[in_iov_idx].iov_base + iov_in[in_iov_idx].iov_len)
754 - ((
unsigned long)iovv[out_iov_idx].iov_base + iovv[out_iov_idx].iov_len);
757 iovv[out_iov_idx].iov_len += additional;
758 this_write_bytes += additional;
759 if (this_write_bytes > per_write_max_bytes)
761 iovv[out_iov_idx].iov_len -= this_write_bytes - per_write_max_bytes;
762 this_write_bytes = per_write_max_bytes;
766 TLOG(TLVL_TRACE) << GetTraceName() <<
": sendFragment writev sts!=: this_write_bytes=" << this_write_bytes
767 <<
" out_iov_idx=" << out_iov_idx
768 <<
" additional=" << additional
773 #ifndef __OPTIMIZE__ // This can be an expensive TRACE call (even if disabled) due to multiplicity of calls
774 TLOG(TLVL_TRACE) << GetTraceName() <<
": sendFragment writev sts(" << sts <<
")==requested_send_bytes(" << this_write_bytes <<
")";
776 total_written_bytes += sts;
778 iovv[out_iov_idx].iov_base = (uint8_t*)(iovv[out_iov_idx].iov_base) + iovv[out_iov_idx].iov_len;
779 iovv[out_iov_idx].iov_len = 0;
780 in_iov_idx += out_iov_idx;
781 this_write_bytes = 0;
783 unsigned long additional = ((
unsigned long)iov_in[in_iov_idx].iov_base + iov_in[in_iov_idx].iov_len)
784 - ((
unsigned long)iovv[out_iov_idx].iov_base + iovv[out_iov_idx].iov_len);
787 iovv[out_iov_idx].iov_len += additional;
788 this_write_bytes += additional;
789 if (this_write_bytes > per_write_max_bytes)
791 iovv[out_iov_idx].iov_len -= this_write_bytes - per_write_max_bytes;
792 this_write_bytes = per_write_max_bytes;
794 if (out_iov_idx != 0)
795 iovv[0] = iovv[out_iov_idx];
804 }
while (total_written_bytes < total_to_write_bytes);
805 if (total_written_bytes > total_to_write_bytes)
806 TLOG(TLVL_ERROR) << GetTraceName() <<
": sendFragment program error: too many bytes transferred";
811 fcntl(send_fd_, F_SETFL, O_NONBLOCK);
813 sts = total_written_bytes -
sizeof(
MessHead);
815 TLOG(14) << GetTraceName() <<
": sendFragment sts=" << sts;
819 void artdaq::TCPSocketTransfer::connect_()
821 auto start_time = std::chrono::steady_clock::now();
824 while (send_fd_ == -1 && TimeUtils::GetElapsedTimeMicroseconds(start_time) < send_retry_timeout_us_ * 10)
826 TLOG(TLVL_DEBUG) << GetTraceName() <<
": Connecting sender socket";
827 int sndbuf_bytes =
static_cast<int>(sndbuf_);
828 if (sndbuf_ > INT_MAX)
830 sndbuf_bytes = INT_MAX;
831 TLOG(TLVL_WARNING) <<
"Requested SNDBUF " << sndbuf_ <<
" too large, setting to INT_MAX: " << INT_MAX;
833 TLOG(TLVL_DEBUG) <<
"Requested SNDBUF is " << sndbuf_bytes;
835 send_fd_ =
TCPConnect(hostMap_[destination_rank()].c_str()
836 , portMan->GetTCPSocketTransferPort(destination_rank())
840 usleep(send_retry_timeout_us_);
844 TLOG(TLVL_DEBUG) << GetTraceName() <<
": connect_ " + hostMap_[destination_rank()] +
":" << portMan->GetTCPSocketTransferPort(destination_rank()) <<
" send_fd_=" << send_fd_;
848 TLOG(TLVL_DEBUG) << GetTraceName() <<
": connect_: Writing connect message";
849 MessHead mh = { 0,MessHead::connect_v0,htons(source_rank()),{htonl(CONN_MAGIC)} };
850 ssize_t sts = write(send_fd_, &mh,
sizeof(mh));
853 TLOG(TLVL_ERROR) << GetTraceName() <<
": connect_: Error writing connect message!";
861 TLOG(TLVL_INFO) << GetTraceName() <<
": connect_: Successfully connected";
867 if (ack_listen_thread_ && ack_listen_thread_->joinable()) ack_listen_thread_->join();
868 TLOG(TLVL_INFO) << GetTraceName() <<
": Starting Ack Listener Thread";
871 ack_listen_thread_ = std::make_unique<boost::thread>(&TCPSocketTransfer::receive_acks_,
this);
873 catch (
const boost::exception& e)
875 TLOG(TLVL_ERROR) <<
"Caught boost::exception starting TCP Socket Ack Listen thread: " << boost::diagnostic_information(e) <<
", errno=" << errno;
876 std::cerr <<
"Caught boost::exception starting TCP Socket Ack Listen thread: " << boost::diagnostic_information(e) <<
", errno=" << errno << std::endl;
883 void artdaq::TCPSocketTransfer::reconnect_()
887 TLOG(TLVL_TRACE) << GetTraceName() <<
": check/reconnect";
892 void artdaq::TCPSocketTransfer::start_listen_thread_()
894 std::unique_lock<std::mutex> start_lock(listen_thread_mutex_);
895 if (listen_thread_refcount_ == 0)
897 if (listen_thread_ && listen_thread_->joinable()) listen_thread_->join();
898 listen_thread_refcount_ = 1;
899 TLOG(TLVL_INFO) << GetTraceName() <<
": Starting Listener Thread";
902 listen_thread_ = std::make_unique<boost::thread>(&TCPSocketTransfer::listen_, portMan->GetTCPSocketTransferPort(destination_rank()), rcvbuf_);
904 catch (
const boost::exception& e)
906 TLOG(TLVL_ERROR) <<
"Caught boost::exception starting TCP Socket Listen thread: " << boost::diagnostic_information(e) <<
", errno=" << errno;
907 std::cerr <<
"Caught boost::exception starting TCP Socket Listen thread: " << boost::diagnostic_information(e) <<
", errno=" << errno << std::endl;
913 listen_thread_refcount_++;
918 void artdaq::TCPSocketTransfer::receive_acks_()
920 while (send_fd_ >= 0)
923 pollfd_s.events = POLLIN | POLLPRI;
924 pollfd_s.fd = send_fd_;
926 TLOG(18) << GetTraceName() <<
": receive_acks_: Polling fd to see if there's data";
927 int num_fds_ready = poll(&pollfd_s, 1, 1000);
928 if (num_fds_ready <= 0)
930 if (num_fds_ready == 0)
932 TLOG(18) << GetTraceName() <<
": receive_acks_: No data on receive socket";
936 TLOG(TLVL_ERROR) <<
"Error in poll: errno=" << errno;
940 if (pollfd_s.revents & (POLLIN | POLLPRI))
946 TLOG(TLVL_DEBUG) << GetTraceName() <<
": receive_acks_: Wrong event received from pollfd: " << pollfd_s.revents;
951 auto sts = read(send_fd_, &mh,
sizeof(mh));
953 if (sts !=
sizeof(mh))
955 TLOG(TLVL_ERROR) << GetTraceName() <<
": receive_ack_: Wrong message header length received! (actual " << sts <<
" != " <<
sizeof(mh) <<
" expected)";
963 TLOG(TLVL_ERROR) << GetTraceName() <<
": receive_ack_: Received ack for different sender! Rank=" << my_rank <<
", hdr=" << mh.
source_id;
968 TLOG(TLVL_ERROR) << GetTraceName() <<
": receive_ack_: Wrong magic bytes in header!";
972 TLOG(17) << GetTraceName() <<
": receive_acks_: Received ack message, diff is now " << (send_ack_diff_.load() - 1);
977 void artdaq::TCPSocketTransfer::send_ack_(
int fd)
979 MessHead mh = { 0,MessHead::ack_v0,htons(source_rank()),{ htonl(ACK_MAGIC) } };
980 write(fd, &mh,
sizeof(mh));
984 void artdaq::TCPSocketTransfer::listen_(
int port,
size_t rcvbuf)
987 while (listen_thread_refcount_ > 0)
989 TLOG(TLVL_TRACE) <<
"listen_: Listening/accepting new connections on port " << port;
992 TLOG(TLVL_DEBUG) <<
"listen_: Opening listener";
997 TLOG(TLVL_DEBUG) <<
"listen_: Error creating listen_fd!";
1002 timeval tv = { 2,0 };
1005 FD_SET(listen_fd, &rfds);
1007 res = select(listen_fd + 1, &rfds, (fd_set *)0, (fd_set *)0, &tv);
1012 socklen_t arglen =
sizeof(un);
1014 TLOG(TLVL_DEBUG) <<
"listen_: Calling accept";
1015 fd = accept(listen_fd, (sockaddr *)&un, &arglen);
1016 TLOG(TLVL_DEBUG) <<
"listen_: Done with accept";
1018 TLOG(TLVL_DEBUG) <<
"listen_: Reading connect message";
1019 socklen_t lenlen =
sizeof(tv);
1021 setsockopt(fd, SOL_SOCKET, SO_RCVTIMEO, &tv, lenlen);
1023 uint64_t mark_us = TimeUtils::gettimeofday_us();
1024 sts = read(fd, &mh,
sizeof(mh));
1025 uint64_t delta_us = TimeUtils::gettimeofday_us() - mark_us;
1026 TLOG(TLVL_DEBUG) <<
"listen_: Read of connect message took " << delta_us <<
" microseconds.";
1027 if (sts !=
sizeof(mh))
1029 TLOG(TLVL_DEBUG) <<
"listen_: Wrong message header length received!";
1038 TLOG(TLVL_DEBUG) <<
"listen_: Wrong magic bytes in header!";
1044 std::unique_lock<std::mutex> lk(connected_fd_mutex_);
1045 connected_fds_[mh.
source_id].insert(fd);
1047 TLOG(TLVL_INFO) <<
"listen_: New fd is " << fd <<
" for source rank " << mh.
source_id;
1051 TLOG(16) <<
"listen_: No connections in timeout interval!";
1055 TLOG(TLVL_INFO) <<
"listen_: Shutting down connection listener";
1056 if (listen_fd != -1) close(listen_fd);
1057 std::unique_lock<std::mutex> lk(connected_fd_mutex_);
1058 auto it = connected_fds_.begin();
1059 while (it != connected_fds_.end())
1061 auto& fd_set = it->second;
1062 auto rank_it = fd_set.begin();
1063 while (rank_it != fd_set.end())
1066 rank_it = fd_set.erase(rank_it);
1068 it = connected_fds_.erase(it);
bool isRunning() override
Determine whether the TransferInterface plugin is able to send/receive data.
virtual int source_rank() const
Get the source rank for this TransferInterface instance.
int TCPConnect(char const *host_in, int dflt_port, long flags=0, int sndbufsiz=0)
Connect to a host on a given port.
uint32_t conn_magic
unsigned first is better for MessHead initializer: {0,0,my_node_idx_,CONN_MAGIC}
hostMap_t MakeHostMap(fhicl::ParameterSet pset, hostMap_t map=hostMap_t())
Make a hostMap_t from a HostMap::Config ParameterSet
This TransferInterface is a Receiver.
int receiveFragmentData(RawDataType *destination, size_t wordCount) override
Receive the body of a Fragment to the given destination pointer.
int TCP_listen_fd(int port, int rcvbuf)
Create a TCP listening socket on the given port and INADDR_ANY, with the given receive buffer...
TCPSocketTransfer(fhicl::ParameterSet const &ps, Role role)
TCPSocketTransfer Constructor.
int receiveFragmentHeader(detail::RawFragmentHeader &header, size_t receiveTimeout) override
Receive a Fragment Header from the transport mechanism.
This TransferInterface is a Sender.
void flush_buffers() override
Flush any in-flight data. This should be used by the receiver after the receive loop has ended...
int32_t byte_count
use CONN_MAGIC for connect_v0, data that follow for data_v0 (and 0 lenght data)
Some error occurred, but no exception was thrown.
Role
Used to determine if a TransferInterface is a Sender or Receiver.
int64_t source_id
Rank of the source.
MessType message_type
Message Type.
The send operation completed successfully.
This interface defines the functions used to transfer data between artdaq applications.
TransferInterface implementation plugin that sends data using TCP sockets.
This header is sent by the TCPSocket_transfer to allow for more efficient writev calls.
CopyStatus
Returned from the send functions, this enumeration describes the possible return codes. If an exception occurs, it will be thrown and should be handled normally.