9 #include <sys/socket.h>
11 #include <arpa/inet.h>
12 #include <sys/types.h>
21 #define TRACE_NAME (app_name + "_TCPSocketTransfer").c_str()
22 #include "artdaq/DAQdata/Globals.hh"
25 #include "artdaq/TransferPlugins/TCPSocketTransfer.hh"
28 #include "artdaq/TransferPlugins/detail/Timeout.hh"
29 #include "artdaq/TransferPlugins/detail/SRSockets.hh"
30 #include "artdaq-core/Data/Fragment.hh"
31 #include "artdaq-core/Utilities/TimeUtils.hh"
34 std::atomic<int> artdaq::TCPSocketTransfer::listen_thread_refcount_(0);
35 std::unique_ptr<boost::thread> artdaq::TCPSocketTransfer::listen_thread_ =
nullptr;
36 std::map<int, std::set<int>> artdaq::TCPSocketTransfer::connected_fds_ = std::map<int, std::set<int>>();
37 std::mutex artdaq::TCPSocketTransfer::listen_thread_mutex_;
38 std::mutex artdaq::TCPSocketTransfer::connected_fd_mutex_;
44 , active_receive_fd_(-1)
45 , last_active_receive_fd_(-1)
46 , rcvbuf_(pset.get<size_t>(
"tcp_receive_buffer_size", 0))
47 , sndbuf_(pset.get<size_t>(
"tcp_send_buffer_size", max_fragment_size_words_ * sizeof(artdaq::RawDataType) * buffer_count_))
48 , send_retry_timeout_us_(pset.get<size_t>(
"send_retry_timeout_us", 1000000))
49 , timeoutMessageArmed_(true)
51 , receive_disconnected_wait_s_(pset.get<double>(
"receive_socket_disconnected_wait_s", 10.0))
52 , receive_err_wait_us_(pset.get<size_t>(
"receive_socket_disconnected_wait_us", 10000))
53 , receive_socket_has_been_connected_(false)
56 TLOG(TLVL_DEBUG) << GetTraceName() <<
" Constructor: pset=" << pset.to_string() <<
", role=" << (role ==
TransferInterface::Role::kReceive ?
"kReceive" :
"kSend");
61 TLOG(TLVL_DEBUG) << GetTraceName() <<
": Listening for connections";
62 start_listen_thread_();
63 TLOG(TLVL_DEBUG) << GetTraceName() <<
": Done Listening";
68 TLOG(TLVL_DEBUG) << GetTraceName() <<
": Connecting to destination";
70 TLOG(TLVL_DEBUG) << GetTraceName() <<
": Done Connecting";
72 TLOG(TLVL_DEBUG) << GetTraceName() <<
": End of Constructor";
75 artdaq::TCPSocketTransfer::~TCPSocketTransfer() noexcept
77 TLOG(TLVL_DEBUG) << GetTraceName() <<
": Shutting down TCPSocketTransfer";
86 timeval tv = { 0,100000 };
87 socklen_t len =
sizeof(tv);
88 setsockopt(send_fd_, SOL_SOCKET, SO_SNDTIMEO, &tv, len);
89 write(send_fd_, &mh,
sizeof(mh));
97 std::unique_lock<std::mutex> fd_lock(connected_fd_mutex_);
98 if (connected_fds_.count(source_rank()))
100 auto it = connected_fds_[source_rank()].begin();
101 while (it != connected_fds_[source_rank()].end())
104 it = connected_fds_[source_rank()].erase(it);
106 connected_fds_.erase(source_rank());
108 if (ack_listen_thread_ && ack_listen_thread_->joinable()) ack_listen_thread_->join();
111 std::unique_lock<std::mutex> lk(listen_thread_mutex_);
112 listen_thread_refcount_--;
113 if (listen_thread_refcount_ <= 0 && listen_thread_ && listen_thread_->joinable())
115 listen_thread_->join();
119 TLOG(TLVL_DEBUG) << GetTraceName() <<
": End of Destructor";
124 TLOG(5) << GetTraceName() <<
": receiveFragmentHeader: BEGIN";
125 int ret_rank = RECV_TIMEOUT;
128 if (getConnectedFDCount(source_rank()) == 0)
137 TLOG(7) << GetTraceName() <<
": receiveFragmentHeader: Receive socket not connected, returning RECV_TIMEOUT";
138 usleep(receive_err_wait_us_);
141 receive_socket_has_been_connected_ =
true;
142 last_recv_time_ = std::chrono::steady_clock::now();
144 TLOG(5) << GetTraceName() <<
": receiveFragmentHeader timeout_usec=" << timeout_usec;
149 SocketState state = SocketState::Metadata;
150 int target_bytes =
sizeof(
MessHead);
151 uint64_t start_time_us = TimeUtils::gettimeofday_us();
163 if (timeout_usec == 0)
166 timeout_ms = (timeout_usec + 999) / 1000;
169 bool noDataWarningSent =
false;
172 while (!done && getConnectedFDCount(source_rank()) > 0)
174 if (active_receive_fd_ == -1)
178 std::vector<pollfd> pollfds;
180 std::unique_lock<std::mutex> lk(connected_fd_mutex_);
181 fd_count = connected_fds_[source_rank()].size();
182 pollfds.resize(fd_count);
183 auto iter = connected_fds_[source_rank()].begin();
184 for (
size_t ii = 0; ii < fd_count; ++ii)
186 pollfds[ii].events = POLLIN | POLLPRI | POLLERR;
187 pollfds[ii].fd = *iter;
192 int num_fds_ready = poll(&pollfds[0], fd_count, timeout_ms);
193 if (num_fds_ready <= 0)
195 TLOG(5) << GetTraceName() <<
": receiveFragmentHeader: No data on receive socket, returning RECV_TIMEOUT";
200 if (last_active_receive_fd_ != -1)
202 for (
auto& pollfd : pollfds)
205 if (pollfd.fd == last_active_receive_fd_)
212 int active_index = -1;
213 short anomolous_events = 0;
214 for (
size_t ii = index; ii < index + pollfds.size(); ++ii)
216 auto pollfd_index = (ii + index) % pollfds.size();
217 if (pollfds[pollfd_index].revents & (POLLIN | POLLPRI))
219 active_index = pollfd_index;
220 active_receive_fd_ = pollfds[active_index].fd;
221 active_revents_ = pollfds[active_index].revents;;
224 else if (pollfds[pollfd_index].revents & (POLLHUP | POLLERR))
226 disconnect_receive_socket_(pollfds[pollfd_index].fd,
"Poll returned POLLHUP or POLLERR, indicating problems with the sender.");
229 else if (pollfds[pollfd_index].revents & (POLLNVAL))
231 TLOG(TLVL_DEBUG) << GetTraceName() <<
": receiveFragmentHeader: FD is closed, most likely because the peer went away. Removing from fd list.";
232 disconnect_receive_socket_(pollfds[pollfd_index].fd,
"FD is closed, most likely because the peer went away.");
235 else if (pollfds[pollfd_index].revents)
237 anomolous_events |= pollfds[pollfd_index].revents;
241 if (active_index == -1)
243 if (anomolous_events)
244 TLOG(TLVL_DEBUG) << GetTraceName() <<
": receiveFragmentHeader: Wrong event received from a pollfd. Mask: " <<
static_cast<int>(anomolous_events);
245 active_receive_fd_ = -1;
249 if (!done && timeout_usec > 0)
252 size_t delta_us = TimeUtils::gettimeofday_us() - start_time_us;
253 if (delta_us > timeout_usec)
257 timeout_ms = ((timeout_usec - delta_us) + 999) / 1000;
260 if (loop_guard > 10) { usleep(1000); }
261 if (++loop_guard > 10010)
263 TLOG(TLVL_WARNING) << GetTraceName() <<
": receiveFragmentHeader: loop guard triggered, returning RECV_TIMEOUT";
264 usleep(receive_err_wait_us_);
265 active_receive_fd_ = -1;
269 if (state == SocketState::Metadata)
272 buff = &(mha[offset]);
273 byte_cnt =
sizeof(
MessHead) - offset;
278 buff =
reinterpret_cast<uint8_t*
>(&header) + offset;
279 byte_cnt = target_bytes - offset;
291 TLOG(6) << GetTraceName() <<
": receiveFragmentHeader: Reading " << byte_cnt <<
" bytes from socket";
292 sts = read(active_receive_fd_, buff, byte_cnt);
293 TLOG(6) << GetTraceName() <<
": receiveFragmentHeader: Done with read";
297 last_recv_time_ = std::chrono::steady_clock::now();
300 TLOG(7) << GetTraceName() <<
": receiveFragmentHeader state=" <<
static_cast<int>(state) <<
" read=" << sts;
303 TLOG(TLVL_WARNING) << GetTraceName() <<
": receiveFragmentHeader: Error on receive, closing socket " <<
" (errno=" << errno <<
": " << strerror(errno) <<
")";
304 active_receive_fd_ = disconnect_receive_socket_(active_receive_fd_);
308 if (!noDataWarningSent) {
309 TLOG(TLVL_WARNING) << GetTraceName() <<
": receiveFragmentHeader: No data received, is the sender still sending?!?";
310 noDataWarningSent =
true;
312 if (TimeUtils::GetElapsedTime(last_recv_time_) > receive_disconnected_wait_s_)
314 TLOG(TLVL_ERROR) << GetTraceName() <<
": receiveFragmentHeader: No data received within timeout, aborting!";
322 if (sts >= target_bytes)
324 TLOG(7) << GetTraceName() <<
": receiveFragmentHeader: Target read bytes reached. Changing state";
326 if (state == SocketState::Metadata)
328 state = SocketState::Data;
332 TLOG(7) << GetTraceName() <<
": receiveFragmentHeader: Expected header size = " << target_bytes <<
", sizeof(RawFragmentHeader) = " <<
sizeof(artdaq::detail::RawFragmentHeader);
337 active_receive_fd_ = disconnect_receive_socket_(active_receive_fd_,
"Stop Message received.");
341 TLOG(TLVL_WARNING) << GetTraceName() <<
": receiveFragmentHeader: Message header indicates that Fragment data follows when I was expecting a Fragment header!";
342 active_receive_fd_ = disconnect_receive_socket_(active_receive_fd_,
"Desync detected");
345 if (target_bytes == 0)
353 ret_rank = source_rank();
354 TLOG(8) << GetTraceName() <<
": receiveFragmentHeader done sts=" << sts <<
" src=" << ret_rank;
355 TLOG(7) << GetTraceName() <<
": receiveFragmentHeader: Done receiving fragment header. Moving into output.";
365 TLOG(5) << GetTraceName() <<
": receiveFragmentHeader: Returning " << ret_rank;
369 int artdaq::TCPSocketTransfer::disconnect_receive_socket_(
int fd, std::string msg)
371 TLOG(TLVL_WARNING) << GetTraceName() <<
": disconnect_receive_socket_: " << msg <<
" Closing socket " << fd;
373 std::unique_lock<std::mutex> lk(connected_fd_mutex_);
374 if (connected_fds_.count(source_rank()))
375 connected_fds_[source_rank()].erase(fd);
377 TLOG(TLVL_DEBUG) << GetTraceName() <<
": disconnect_receive_socket_: There are now " << connected_fds_[source_rank()].size() <<
" active senders.";
383 TLOG(9) << GetTraceName() <<
": receiveFragmentData: BEGIN";
384 int ret_rank = RECV_TIMEOUT;
385 if (active_receive_fd_ == -1)
387 TLOG(TLVL_ERROR) << GetTraceName() <<
": receiveFragmentData: Receive socket not connected, returning RECV_TIMEOUT (Will result in \"Unexpected return code error\")";
396 SocketState state = SocketState::Metadata;
397 int target_bytes =
sizeof(
MessHead);
400 pollfd_s.events = POLLIN | POLLPRI | POLLERR;
401 pollfd_s.fd = active_receive_fd_;
405 bool noDataWarningSent =
false;
406 last_recv_time_ = std::chrono::steady_clock::now();
409 TLOG(9) << GetTraceName() <<
": receiveFragmentData: Polling fd to see if there's data";
410 int num_fds_ready = poll(&pollfd_s, 1, 1000);
411 TLOG(TLVL_TRACE) << GetTraceName() <<
": receiveFragmentData: Polled fd to see if there's data"
412 <<
", num_fds_ready = " << num_fds_ready;
413 if (num_fds_ready <= 0)
415 if (num_fds_ready == 0)
417 TLOG(TLVL_WARNING) << GetTraceName() <<
": receiveFragmentData: No data from " << source_rank() <<
" in " << TimeUtils::GetElapsedTimeMilliseconds(last_recv_time_) <<
" ms!"
418 <<
" State = " << (state == SocketState::Metadata ?
"Metadata" :
"Data") <<
", recvd/total=" << offset <<
"/" << target_bytes <<
" (delta=" << target_bytes - offset <<
")";
420 if (TimeUtils::GetElapsedTime(last_recv_time_) > receive_disconnected_wait_s_)
422 TLOG(TLVL_WARNING) << GetTraceName() <<
": receiveFragmentData: No data received within timeout (" << TimeUtils::GetElapsedTime(last_recv_time_) <<
" / " << receive_disconnected_wait_s_ <<
" ), returning RECV_TIMEOUT";
423 disconnect_receive_socket_(active_receive_fd_,
"No data on this socket within timeout");
424 active_receive_fd_ = -1;
430 TLOG(TLVL_ERROR) <<
"Error in poll: errno=" << errno;
431 active_receive_fd_ = -1;
434 else { last_recv_time_ = std::chrono::steady_clock::now(); }
436 if (pollfd_s.revents & (POLLIN | POLLPRI))
440 else if (pollfd_s.revents & (POLLNVAL))
442 disconnect_receive_socket_(pollfd_s.fd,
"FD is closed, most likely because the peer went away.");
445 else if (pollfd_s.revents & (POLLHUP | POLLERR))
447 disconnect_receive_socket_(pollfd_s.fd,
"Poll returned POLLHUP or POLLERR, indicating problems with the sender.");
452 TLOG(TLVL_WARNING) << GetTraceName() <<
": receiveFragmentData: Wrong event received from pollfd: " << pollfd_s.revents;
453 disconnect_receive_socket_(pollfd_s.fd);
457 if (state == SocketState::Metadata)
460 buff = &(mha[offset]);
461 byte_cnt =
sizeof(
MessHead) - offset;
466 buff =
reinterpret_cast<uint8_t*
>(destination) + offset;
470 TLOG(10) << GetTraceName() <<
": receiveFragmentData: Reading " << byte_cnt <<
" bytes from socket into " << (
void*)buff;
471 sts = read(active_receive_fd_, buff, byte_cnt);
474 TLOG(10) << GetTraceName() <<
": recvFragment state=" <<
static_cast<int>(state) <<
" read=" << sts;
478 if (loop_guard > 10) { usleep(1000); }
479 if (++loop_guard > 10010)
481 TLOG(TLVL_WARNING) << GetTraceName() <<
": receiveFragmentData: loop guard triggered, returning RECV_TIMEOUT";
482 active_receive_fd_ = -1;
489 last_recv_time_ = std::chrono::steady_clock::now();
494 TLOG(TLVL_WARNING) << GetTraceName() <<
": receiveFragmentData: Error on receive, closing socket"
495 <<
" (errno=" << errno <<
": " << strerror(errno) <<
")";
496 disconnect_receive_socket_(pollfd_s.fd);
500 if (!noDataWarningSent) {
501 TLOG(TLVL_WARNING) << GetTraceName() <<
": receiveFragmentData: No data received, is the sender still sending?!?";
502 noDataWarningSent =
true;
504 if (TimeUtils::GetElapsedTime(last_recv_time_) > receive_disconnected_wait_s_)
506 TLOG(TLVL_ERROR) << GetTraceName() <<
": receiveFragmentData: No data received within timeout, aborting!";
514 if (sts >= target_bytes)
516 TLOG(9) << GetTraceName() <<
": receiveFragmentData: Target read bytes reached. Changing state";
518 if (state == SocketState::Metadata)
520 state = SocketState::Data;
528 TLOG(TLVL_WARNING) << GetTraceName() <<
": receiveFragmentData: Message header indicates that a Fragment header follows when I was expecting Fragment data!";
529 active_receive_fd_ = disconnect_receive_socket_(active_receive_fd_,
"Desync detected");
534 ret_rank = source_rank();
535 TLOG(11) << GetTraceName() <<
": receiveFragmentData done sts=" << sts <<
" src=" << ret_rank;
536 TLOG(9) << GetTraceName() <<
": receiveFragmentData: Done receiving fragment. Moving into output.";
539 send_ack_(active_receive_fd_);
549 if (target_bytes == 0 && state == SocketState::Data)
551 ret_rank = source_rank();
552 TLOG(11) << GetTraceName() <<
": receiveFragmentData done sts=" << sts <<
" src=" << ret_rank;
553 TLOG(9) << GetTraceName() <<
": receiveFragmentData: Done receiving fragment. Moving into output.";
556 send_ack_(active_receive_fd_);
564 last_active_receive_fd_ = active_receive_fd_;
565 active_receive_fd_ = -1;
567 TLOG(9) << GetTraceName() <<
": receiveFragmentData: Returning " << ret_rank;
576 return send_fd_ != -1;
578 TLOG(TLVL_DEBUG) << GetTraceName() <<
": isRunning: There are " << getConnectedFDCount(source_rank()) <<
" fds connected.";
579 return getConnectedFDCount(source_rank()) > 0;
588 TLOG(12) << GetTraceName() <<
": sendFragment begin send of fragment with sequenceID="<<frag.sequenceID();
589 artdaq::Fragment grab_ownership_frag = std::move(frag);
596 while (static_cast<size_t>(send_ack_diff_) > buffer_count_) usleep(10000);
599 iovec iov = {
reinterpret_cast<void*
>(grab_ownership_frag.headerAddress()),
600 detail::RawFragmentHeader::num_words() *
sizeof(RawDataType) };
602 auto sts = sendData_(&iov, 1, send_retry_timeout_us_,
true);
603 auto start_time = std::chrono::steady_clock::now();
605 while (sts == CopyStatus::kTimeout && (send_timeout_usec == 0 || TimeUtils::GetElapsedTimeMicroseconds(start_time) < send_timeout_usec) && TimeUtils::GetElapsedTimeMicroseconds(start_time) < 10000000)
607 TLOG(13) << GetTraceName() <<
": sendFragment: Timeout sending fragment";
608 sts = sendData_(&iov, 1, send_retry_timeout_us_,
true);
611 if (sts != CopyStatus::kSuccess)
return sts;
615 iov = {
reinterpret_cast<void*
>(grab_ownership_frag.headerAddress() + detail::RawFragmentHeader::num_words()),
616 grab_ownership_frag.sizeBytes() - detail::RawFragmentHeader::num_words() *
sizeof(RawDataType)
618 sts = sendData_(&iov, 1, send_retry_timeout_us_);
619 start_time = std::chrono::steady_clock::now();
620 while (sts == CopyStatus::kTimeout && (send_timeout_usec == 0 || TimeUtils::GetElapsedTimeMicroseconds(start_time) < send_timeout_usec) && TimeUtils::GetElapsedTimeMicroseconds(start_time) < 10000000)
622 TLOG(13) << GetTraceName() <<
": sendFragment: Timeout sending fragment";
623 sts = sendData_(&iov, 1, send_retry_timeout_us_);
631 TLOG(12) << GetTraceName() <<
": sendFragment returning " << CopyStatusToString(sts);
637 TLOG(TLVL_DEBUG) << GetTraceName() <<
": sendData_ Converting buf to iovec";
638 iovec iov = { (
void*)buf, bytes };
639 return sendData_(&iov, 1, send_timeout_usec, isHeader);
647 if (timeoutMessageArmed_)
649 TLOG(TLVL_DEBUG) << GetTraceName() <<
": sendData_: Send fd is not open. Returning kTimeout";
650 timeoutMessageArmed_ =
false;
652 return CopyStatus::kTimeout;
654 timeoutMessageArmed_ =
true;
655 TLOG(14) << GetTraceName() <<
": send_timeout_usec is " << send_timeout_usec <<
", currently unused.";
658 uint32_t total_to_write_bytes = 0;
659 std::vector<iovec> iov_in(iovcnt + 1);
660 std::vector<iovec> iovv(iovcnt + 2);
662 for (ii = 0; ii < iovcnt; ++ii)
664 iov_in[ii + 1] = iov[ii];
665 total_to_write_bytes += iov[ii].iov_len;
668 MessHead mh = { 0,isHeader ? MessHead::header_v0 : MessHead::data_v0,htons(source_rank()),{htonl(total_to_write_bytes)} };
669 iov_in[0].iov_base = &mh;
670 iov_in[0].iov_len =
sizeof(mh);
671 total_to_write_bytes +=
sizeof(mh);
674 ssize_t total_written_bytes = 0;
675 ssize_t per_write_max_bytes = (32 * 1024);
677 size_t in_iov_idx = 0;
678 size_t out_iov_idx = 0;
679 ssize_t this_write_bytes = 0;
686 (in_iov_idx + out_iov_idx) < iov_in.size() && this_write_bytes < per_write_max_bytes;
689 this_write_bytes += iov_in[in_iov_idx + out_iov_idx].iov_len;
690 iovv[out_iov_idx] = iov_in[in_iov_idx + out_iov_idx];
692 if (this_write_bytes > per_write_max_bytes)
694 iovv[out_iov_idx - 1].iov_len -= this_write_bytes - per_write_max_bytes;
695 this_write_bytes = per_write_max_bytes;
700 #ifndef __OPTIMIZE__ // This can be an expensive TRACE call (even if disabled) due to multiplicity of calls
701 TLOG(14) << GetTraceName() <<
": sendFragment b4 writev " << std::setw(7) << total_written_bytes <<
" total_written_bytes send_fd_=" << send_fd_ <<
" in_idx=" << in_iov_idx
702 <<
" iovcnt=" << out_iov_idx <<
" 1st.len=" << iovv[0].iov_len;
705 sts = writev(send_fd_, &(iovv[0]), out_iov_idx);
710 if (errno == EAGAIN )
712 TLOG(TLVL_DEBUG) << GetTraceName() <<
": sendFragment EWOULDBLOCK";
713 fcntl(send_fd_, F_SETFL, 0);
718 TLOG(TLVL_WARNING) << GetTraceName() <<
": sendFragment_: WRITE ERROR: " << strerror(errno);
724 else if (sts != this_write_bytes)
727 TLOG(TLVL_DEBUG) << GetTraceName() <<
": sendFragment writev sts(" << sts <<
")!=requested_send_bytes(" << this_write_bytes <<
")";
728 total_written_bytes += sts;
730 for (ii = 0; (size_t)sts >= iovv[ii].iov_len; ++ii)
731 sts -= iovv[ii].iov_len;
733 iovv[ii].iov_len -= sts;
734 iovv[ii].iov_base = (uint8_t*)(iovv[ii].iov_base) + sts;
739 iovv[out_iov_idx] = iovv[ii];
741 this_write_bytes = iovv[out_iov_idx].iov_len;
746 unsigned long additional = ((
unsigned long)iov_in[in_iov_idx].iov_base + iov_in[in_iov_idx].iov_len)
747 - ((
unsigned long)iovv[out_iov_idx].iov_base + iovv[out_iov_idx].iov_len);
750 iovv[out_iov_idx].iov_len += additional;
751 this_write_bytes += additional;
752 if (this_write_bytes > per_write_max_bytes)
754 iovv[out_iov_idx].iov_len -= this_write_bytes - per_write_max_bytes;
755 this_write_bytes = per_write_max_bytes;
759 TLOG(TLVL_TRACE) << GetTraceName() <<
": sendFragment writev sts!=: this_write_bytes=" << this_write_bytes
760 <<
" out_iov_idx=" << out_iov_idx
761 <<
" additional=" << additional
766 #ifndef __OPTIMIZE__ // This can be an expensive TRACE call (even if disabled) due to multiplicity of calls
767 TLOG(TLVL_TRACE) << GetTraceName() <<
": sendFragment writev sts(" << sts <<
")==requested_send_bytes(" << this_write_bytes <<
")";
769 total_written_bytes += sts;
771 iovv[out_iov_idx].iov_base = (uint8_t*)(iovv[out_iov_idx].iov_base) + iovv[out_iov_idx].iov_len;
772 iovv[out_iov_idx].iov_len = 0;
773 in_iov_idx += out_iov_idx;
774 this_write_bytes = 0;
776 unsigned long additional = ((
unsigned long)iov_in[in_iov_idx].iov_base + iov_in[in_iov_idx].iov_len)
777 - ((
unsigned long)iovv[out_iov_idx].iov_base + iovv[out_iov_idx].iov_len);
780 iovv[out_iov_idx].iov_len += additional;
781 this_write_bytes += additional;
782 if (this_write_bytes > per_write_max_bytes)
784 iovv[out_iov_idx].iov_len -= this_write_bytes - per_write_max_bytes;
785 this_write_bytes = per_write_max_bytes;
787 if (out_iov_idx != 0)
788 iovv[0] = iovv[out_iov_idx];
797 }
while (total_written_bytes < total_to_write_bytes);
798 if (total_written_bytes > total_to_write_bytes)
799 TLOG(TLVL_ERROR) << GetTraceName() <<
": sendFragment program error: too many bytes transferred";
804 fcntl(send_fd_, F_SETFL, O_NONBLOCK);
806 sts = total_written_bytes -
sizeof(
MessHead);
808 TLOG(14) << GetTraceName() <<
": sendFragment sts=" << sts;
812 void artdaq::TCPSocketTransfer::connect_()
814 auto start_time = std::chrono::steady_clock::now();
817 while (send_fd_ == -1 && TimeUtils::GetElapsedTimeMicroseconds(start_time) < send_retry_timeout_us_ * 10)
819 TLOG(TLVL_DEBUG) << GetTraceName() <<
": Connecting sender socket";
820 int sndbuf_bytes =
static_cast<int>(sndbuf_);
821 if (sndbuf_ > INT_MAX)
823 sndbuf_bytes = INT_MAX;
824 TLOG(TLVL_WARNING) <<
"Requested SNDBUF " << sndbuf_ <<
" too large, setting to INT_MAX: " << INT_MAX;
826 TLOG(TLVL_DEBUG) <<
"Requested SNDBUF is " << sndbuf_bytes;
828 send_fd_ =
TCPConnect(hostMap_[destination_rank()].c_str()
829 , portMan->GetTCPSocketTransferPort(destination_rank())
833 usleep(send_retry_timeout_us_);
837 TLOG(TLVL_DEBUG) << GetTraceName() <<
": connect_ " + hostMap_[destination_rank()] +
":" << portMan->GetTCPSocketTransferPort(destination_rank()) <<
" send_fd_=" << send_fd_;
841 TLOG(TLVL_DEBUG) << GetTraceName() <<
": connect_: Writing connect message";
842 MessHead mh = { 0,MessHead::connect_v0,htons(source_rank()),{htonl(CONN_MAGIC)} };
843 ssize_t sts = write(send_fd_, &mh,
sizeof(mh));
846 TLOG(TLVL_ERROR) << GetTraceName() <<
": connect_: Error writing connect message!";
854 TLOG(TLVL_INFO) << GetTraceName() <<
": connect_: Successfully connected";
860 if (ack_listen_thread_ && ack_listen_thread_->joinable()) ack_listen_thread_->join();
861 TLOG(TLVL_INFO) << GetTraceName() <<
": Starting Ack Listener Thread";
864 ack_listen_thread_ = std::make_unique<boost::thread>(&TCPSocketTransfer::receive_acks_,
this);
866 catch (
const boost::exception& e)
868 TLOG(TLVL_ERROR) <<
"Caught boost::exception starting TCP Socket Ack Listen thread: " << boost::diagnostic_information(e) <<
", errno=" << errno;
869 std::cerr <<
"Caught boost::exception starting TCP Socket Ack Listen thread: " << boost::diagnostic_information(e) <<
", errno=" << errno << std::endl;
876 void artdaq::TCPSocketTransfer::reconnect_()
880 TLOG(TLVL_TRACE) << GetTraceName() <<
": check/reconnect";
885 void artdaq::TCPSocketTransfer::start_listen_thread_()
887 std::unique_lock<std::mutex> start_lock(listen_thread_mutex_);
888 if (listen_thread_refcount_ == 0)
890 if (listen_thread_ && listen_thread_->joinable()) listen_thread_->join();
891 listen_thread_refcount_ = 1;
892 TLOG(TLVL_INFO) << GetTraceName() <<
": Starting Listener Thread";
895 listen_thread_ = std::make_unique<boost::thread>(&TCPSocketTransfer::listen_, portMan->GetTCPSocketTransferPort(destination_rank()), rcvbuf_);
897 catch (
const boost::exception& e)
899 TLOG(TLVL_ERROR) <<
"Caught boost::exception starting TCP Socket Listen thread: " << boost::diagnostic_information(e) <<
", errno=" << errno;
900 std::cerr <<
"Caught boost::exception starting TCP Socket Listen thread: " << boost::diagnostic_information(e) <<
", errno=" << errno << std::endl;
906 listen_thread_refcount_++;
911 void artdaq::TCPSocketTransfer::receive_acks_()
913 while (send_fd_ >= 0)
916 pollfd_s.events = POLLIN | POLLPRI;
917 pollfd_s.fd = send_fd_;
919 TLOG(18) << GetTraceName() <<
": receive_acks_: Polling fd to see if there's data";
920 int num_fds_ready = poll(&pollfd_s, 1, 1000);
921 if (num_fds_ready <= 0)
923 if (num_fds_ready == 0)
925 TLOG(18) << GetTraceName() <<
": receive_acks_: No data on receive socket";
929 TLOG(TLVL_ERROR) <<
"Error in poll: errno=" << errno;
933 if (pollfd_s.revents & (POLLIN | POLLPRI))
939 TLOG(TLVL_DEBUG) << GetTraceName() <<
": receive_acks_: Wrong event received from pollfd: " << pollfd_s.revents;
944 auto sts = read(send_fd_, &mh,
sizeof(mh));
946 if (sts !=
sizeof(mh))
948 TLOG(TLVL_ERROR) << GetTraceName() <<
": receive_ack_: Wrong message header length received! (actual " << sts <<
" != " <<
sizeof(mh) <<
" expected)";
956 TLOG(TLVL_ERROR) << GetTraceName() <<
": receive_ack_: Received ack for different sender! Rank=" << my_rank <<
", hdr=" << mh.
source_id;
961 TLOG(TLVL_ERROR) << GetTraceName() <<
": receive_ack_: Wrong magic bytes in header!";
965 TLOG(17) << GetTraceName() <<
": receive_acks_: Received ack message, diff is now " << (send_ack_diff_.load() - 1);
970 void artdaq::TCPSocketTransfer::send_ack_(
int fd)
972 MessHead mh = { 0,MessHead::ack_v0,htons(source_rank()),{ htonl(ACK_MAGIC) } };
973 write(fd, &mh,
sizeof(mh));
977 void artdaq::TCPSocketTransfer::listen_(
int port,
size_t rcvbuf)
980 while (listen_thread_refcount_ > 0)
982 TLOG(TLVL_TRACE) <<
"listen_: Listening/accepting new connections on port " << port;
985 TLOG(TLVL_DEBUG) <<
"listen_: Opening listener";
990 TLOG(TLVL_DEBUG) <<
"listen_: Error creating listen_fd!";
995 timeval tv = { 2,0 };
998 FD_SET(listen_fd, &rfds);
1000 res = select(listen_fd + 1, &rfds, (fd_set *)0, (fd_set *)0, &tv);
1005 socklen_t arglen =
sizeof(un);
1007 TLOG(TLVL_DEBUG) <<
"listen_: Calling accept";
1008 fd = accept(listen_fd, (sockaddr *)&un, &arglen);
1009 TLOG(TLVL_DEBUG) <<
"listen_: Done with accept";
1011 TLOG(TLVL_DEBUG) <<
"listen_: Reading connect message";
1012 socklen_t lenlen =
sizeof(tv);
1014 setsockopt(fd, SOL_SOCKET, SO_RCVTIMEO, &tv, lenlen);
1016 uint64_t mark_us = TimeUtils::gettimeofday_us();
1017 sts = read(fd, &mh,
sizeof(mh));
1018 uint64_t delta_us = TimeUtils::gettimeofday_us() - mark_us;
1019 TLOG(TLVL_DEBUG) <<
"listen_: Read of connect message took " << delta_us <<
" microseconds.";
1020 if (sts !=
sizeof(mh))
1022 TLOG(TLVL_DEBUG) <<
"listen_: Wrong message header length received!";
1031 TLOG(TLVL_DEBUG) <<
"listen_: Wrong magic bytes in header!";
1037 std::unique_lock<std::mutex> lk(connected_fd_mutex_);
1038 connected_fds_[mh.
source_id].insert(fd);
1040 TLOG(TLVL_INFO) <<
"listen_: New fd is " << fd <<
" for source rank " << mh.
source_id;
1044 TLOG(16) <<
"listen_: No connections in timeout interval!";
1048 TLOG(TLVL_INFO) <<
"listen_: Shutting down connection listener";
1049 if (listen_fd != -1) close(listen_fd);
1050 std::unique_lock<std::mutex> lk(connected_fd_mutex_);
1051 auto it = connected_fds_.begin();
1052 while (it != connected_fds_.end())
1054 auto& fd_set = it->second;
1055 auto rank_it = fd_set.begin();
1056 while (rank_it != fd_set.end())
1059 rank_it = fd_set.erase(rank_it);
1061 it = connected_fds_.erase(it);
bool isRunning() override
Determine whether the TransferInterface plugin is able to send/receive data.
virtual int source_rank() const
Get the source rank for this TransferInterface instance.
int TCPConnect(char const *host_in, int dflt_port, long flags=0, int sndbufsiz=0)
Connect to a host on a given port.
uint32_t conn_magic
unsigned first is better for MessHead initializer: {0,0,my_node_idx_,CONN_MAGIC}
hostMap_t MakeHostMap(fhicl::ParameterSet pset, hostMap_t map=hostMap_t())
Make a hostMap_t from a HostMap::Config ParameterSet
This TransferInterface is a Receiver.
int receiveFragmentData(RawDataType *destination, size_t wordCount) override
Receive the body of a Fragment to the given destination pointer.
int TCP_listen_fd(int port, int rcvbuf)
Create a TCP listening socket on the given port and INADDR_ANY, with the given receive buffer...
TCPSocketTransfer(fhicl::ParameterSet const &ps, Role role)
TCPSocketTransfer Constructor.
int receiveFragmentHeader(detail::RawFragmentHeader &header, size_t receiveTimeout) override
Receive a Fragment Header from the transport mechanism.
This TransferInterface is a Sender.
int32_t byte_count
use CONN_MAGIC for connect_v0, data that follow for data_v0 (and 0 lenght data)
Some error occurred, but no exception was thrown.
Role
Used to determine if a TransferInterface is a Sender or Receiver.
int64_t source_id
Rank of the source.
MessType message_type
Message Type.
The send operation completed successfully.
This interface defines the functions used to transfer data between artdaq applications.
TransferInterface implementation plugin that sends data using TCP sockets.
This header is sent by the TCPSocket_transfer to allow for more efficient writev calls.
CopyStatus
Returned from the send functions, this enumeration describes the possible return codes. If an exception occurs, it will be thrown and should be handled normally.