9 #include <sys/socket.h>
11 #include <arpa/inet.h>
12 #include <sys/types.h>
21 #define TRACE_NAME "TCPSocketTransfer"
22 #include "artdaq/DAQdata/Globals.hh"
25 #include "artdaq/TransferPlugins/TCPSocketTransfer.hh"
28 #include "artdaq/TransferPlugins/detail/Timeout.hh"
29 #include "artdaq/TransferPlugins/detail/SRSockets.hh"
30 #include "artdaq-core/Data/Fragment.hh"
31 #include "artdaq-core/Utilities/TimeUtils.hh"
34 std::atomic<int> artdaq::TCPSocketTransfer::listen_thread_refcount_(0);
35 std::unique_ptr<boost::thread> artdaq::TCPSocketTransfer::listen_thread_ =
nullptr;
36 std::map<int, std::set<int>> artdaq::TCPSocketTransfer::connected_fds_ = std::map<int, std::set<int>>();
37 std::mutex artdaq::TCPSocketTransfer::listen_thread_mutex_;
38 std::mutex artdaq::TCPSocketTransfer::connected_fd_mutex_;
44 , active_receive_fd_(-1)
45 , last_active_receive_fd_(-1)
46 , rcvbuf_(pset.get<size_t>(
"tcp_receive_buffer_size", 0))
47 , sndbuf_(max_fragment_size_words_ * sizeof(artdaq::RawDataType) * buffer_count_)
48 , send_retry_timeout_us_(pset.get<size_t>(
"send_retry_timeout_us", 1000000))
49 , stats_connect_stop_(false)
51 , timeoutMessageArmed_(true)
52 , not_connected_count_(0)
53 , receive_err_threshold_(pset.get<size_t>(
"receive_socket_disconnected_max_count", 1000))
54 , receive_err_wait_us_(pset.get<size_t>(
"receive_socket_disconnected_wait_us", 10000))
57 auto masterPortOffset = pset.get<
int>(
"offset_all_ports", 0);
58 hostMap_ = MakeHostMap(pset, masterPortOffset);
60 std::function<void()>
function = std::bind(&TCPSocketTransfer::reconnect_,
this);
66 TLOG(TLVL_DEBUG) <<
GetTraceName() <<
": Listening for connections";
67 start_listen_thread_();
68 TLOG(TLVL_DEBUG) <<
GetTraceName() <<
": Done Listening";
72 TLOG(TLVL_DEBUG) <<
GetTraceName() <<
": Connecting to destination";
74 TLOG(TLVL_DEBUG) <<
GetTraceName() <<
": Done Connecting";
76 TLOG(TLVL_DEBUG) <<
GetTraceName() <<
": End of Constructor";
79 artdaq::TCPSocketTransfer::~TCPSocketTransfer() noexcept
81 TLOG(TLVL_DEBUG) << GetTraceName() <<
": Shutting down TCPSocketTransfer";
82 stats_connect_stop_ =
true;
83 stopstatscv_.notify_all();
84 stats_connect_thread_.join();
93 timeval tv = { 0,100000 };
94 socklen_t len =
sizeof(tv);
95 setsockopt(send_fd_, SOL_SOCKET, SO_SNDTIMEO, &tv, len);
96 write(send_fd_, &mh,
sizeof(mh));
103 std::unique_lock<std::mutex> fd_lock(connected_fd_mutex_);
104 auto it = connected_fds_[source_rank()].begin();
105 while (it != connected_fds_[source_rank()].end())
108 it = connected_fds_[source_rank()].erase(it);
110 connected_fds_.erase(source_rank());
113 std::unique_lock<std::mutex> lk(listen_thread_mutex_);
114 listen_thread_refcount_--;
115 if (listen_thread_refcount_ == 0 && listen_thread_ && listen_thread_->joinable())
117 listen_thread_->join();
120 TLOG(TLVL_DEBUG) << GetTraceName() <<
": End of Destructor";
125 TLOG(5) << GetTraceName() <<
": receiveFragmentHeader: BEGIN";
126 int ret_rank = RECV_TIMEOUT;
128 if (getConnectedFDCount(source_rank()) == 0)
130 if (++not_connected_count_ > receive_err_threshold_) {
return DATA_END; }
131 TLOG(7) << GetTraceName() <<
": receiveFragmentHeader: Receive socket not connected, returning RECV_TIMEOUT";
132 usleep(receive_err_wait_us_);
135 not_connected_count_ = 0;
137 TLOG(5) << GetTraceName() <<
": receiveFragmentHeader timeout_usec=" << std::to_string(timeout_usec);
142 SocketState state = SocketState::Metadata;
143 int target_bytes =
sizeof(
MessHead);
144 uint64_t start_time_us = TimeUtils::gettimeofday_us();
156 if (timeout_usec == 0)
159 timeout_ms = (timeout_usec + 999) / 1000;
162 while (!done && getConnectedFDCount(source_rank()) > 0)
164 if (active_receive_fd_ == -1)
166 std::unique_lock<std::mutex> lk(connected_fd_mutex_);
167 size_t fd_count = connected_fds_[source_rank()].size();
168 auto iter = connected_fds_[source_rank()].begin();
169 std::vector<pollfd> pollfds(fd_count);
170 for (
size_t ii = 0; ii < fd_count; ++ii)
172 pollfds[ii].events = POLLIN | POLLERR;
173 pollfds[ii].fd = *iter;
178 int num_fds_ready = poll(&pollfds[0], fd_count, timeout_ms);
179 if (num_fds_ready <= 0)
181 if (num_fds_ready == 0 && timeout_ms > 0)
183 TLOG(5) << GetTraceName() <<
": receiveFragmentHeader: No data on receive socket, returning RECV_TIMEOUT";
190 if (last_active_receive_fd_ != -1)
192 for (
auto& pollfd : pollfds)
195 if (pollfd.fd == last_active_receive_fd_)
202 int active_index = -1;
203 short anomolous_events = 0;
204 for (
size_t ii = index; ii < index + pollfds.size(); ++ii)
206 if (pollfds[index % pollfds.size()].revents & (POLLIN | POLLPRI | POLLHUP | POLLERR))
208 active_index = index % pollfds.size();
209 active_receive_fd_ = pollfds[active_index].fd;
212 else if (pollfds[index % pollfds.size()].revents & (POLLNVAL))
214 TLOG(TLVL_DEBUG) << GetTraceName() <<
": receiveFragmentHeader: FD is closed, most likely because the peer went away. Removing from fd list.";
215 close(pollfds[index].fd);
216 std::unique_lock<std::mutex> lk(connected_fd_mutex_);
217 connected_fds_[source_rank()].erase(pollfds[index].fd);
220 else if (pollfds[index % pollfds.size()].revents)
222 anomolous_events |= pollfds[index % pollfds.size()].revents;
226 if (active_index == -1)
228 if (anomolous_events)
229 TLOG(TLVL_DEBUG) << GetTraceName() <<
": receiveFragmentHeader: Wrong event received from a pollfd. Mask: " <<
static_cast<int>(anomolous_events);
230 active_receive_fd_ = -1;
234 if (!done && timeout_usec > 0)
237 size_t delta_us = TimeUtils::gettimeofday_us() - start_time_us;
238 if (delta_us > timeout_usec)
242 timeout_ms = ((timeout_usec - delta_us) + 999) / 1000;
246 if (state == SocketState::Metadata)
249 buff = &(mha[offset]);
250 byte_cnt =
sizeof(
MessHead) - offset;
255 buff =
reinterpret_cast<uint8_t*
>(&header) + offset;
261 TLOG(6) << GetTraceName() <<
": receiveFragmentHeader: Reading " << byte_cnt <<
" bytes from socket";
262 sts = read(active_receive_fd_, buff, byte_cnt);
263 TLOG(6) << GetTraceName() <<
": receiveFragmentHeader: Done with read";
266 TLOG(7) << GetTraceName() <<
": receiveFragmentHeader state=" <<
static_cast<int>(state) <<
" read=" << sts;
269 TLOG(TLVL_WARNING) << GetTraceName() <<
": receiveFragmentHeader: Error on receive, closing socket " <<
" (errno=" << errno <<
": " << strerror(errno) <<
")";
270 close(active_receive_fd_);
271 std::unique_lock<std::mutex> lk(connected_fd_mutex_);
272 connected_fds_[source_rank()].erase(active_receive_fd_);
273 active_receive_fd_ = -1;
279 if (sts >= target_bytes)
281 TLOG(7) << GetTraceName() <<
": receiveFragmentHeader: Target read bytes reached. Changing state";
283 if (state == SocketState::Metadata)
285 state = SocketState::Data;
292 TLOG(TLVL_DEBUG) << GetTraceName() <<
": receiveFragmentHeader: Stop Message received. Closing socket " << active_receive_fd_;
293 close(active_receive_fd_);
294 std::unique_lock<std::mutex> lk(connected_fd_mutex_);
295 connected_fds_[source_rank()].erase(active_receive_fd_);
296 active_receive_fd_ = -1;
297 TLOG(TLVL_DEBUG) << GetTraceName() <<
": receiveFragmentHeader: There are now " << connected_fds_[source_rank()].size() <<
" active senders.";
300 if (target_bytes == 0)
308 ret_rank = source_rank();
309 TLOG(8) << GetTraceName() <<
": receiveFragmentHeader done sts=" << sts <<
" src=" << ret_rank;
310 TLOG(7) << GetTraceName() <<
": receiveFragmentHeader: Done receiving fragment header. Moving into output.";
320 TLOG(5) << GetTraceName() <<
": receiveFragmentHeader: Returning " << ret_rank;
326 TLOG(9) << GetTraceName() <<
": receiveFragmentData: BEGIN";
327 int ret_rank = RECV_TIMEOUT;
328 if (active_receive_fd_ == -1)
330 TLOG(TLVL_DEBUG) << GetTraceName() <<
": receiveFragmentData: Receive socket not connected, returning RECV_TIMEOUT";
339 SocketState state = SocketState::Metadata;
340 int target_bytes =
sizeof(
MessHead);
343 pollfd_s.events = POLLIN | POLLPRI | POLLERR;
344 pollfd_s.fd = active_receive_fd_;
349 TLOG(TLVL_DEBUG) << GetTraceName() <<
": receiveFragmentData: Polling fd to see if there's data";
350 int num_fds_ready = poll(&pollfd_s, 1, 1000);
351 if (num_fds_ready <= 0)
353 if (num_fds_ready == 0)
355 TLOG(9) << GetTraceName() <<
": receiveFragmentData: No data on receive socket, returning RECV_TIMEOUT";
356 active_receive_fd_ = -1;
360 TLOG(TLVL_ERROR) <<
"Error in poll: errno=" << errno;
361 active_receive_fd_ = -1;
365 if (pollfd_s.revents & (POLLNVAL))
367 TLOG(TLVL_DEBUG) << GetTraceName() <<
": receiveFragmentData: FD is closed, most likely because the peer went away. Removing from fd list.";
368 close(active_receive_fd_);
369 std::unique_lock<std::mutex> lk(connected_fd_mutex_);
370 connected_fds_[source_rank()].erase(active_receive_fd_);
371 active_receive_fd_ = -1;
374 else if (!(pollfd_s.revents & (POLLIN | POLLPRI | POLLERR)))
376 TLOG(TLVL_DEBUG) << GetTraceName() <<
": receiveFragmentData: Wrong event received from pollfd: " << pollfd_s.revents;
377 close(active_receive_fd_);
378 std::unique_lock<std::mutex> lk(connected_fd_mutex_);
379 connected_fds_[source_rank()].erase(active_receive_fd_);
383 if (state == SocketState::Metadata)
386 buff = &(mha[offset]);
387 byte_cnt =
sizeof(
MessHead) - offset;
392 buff =
reinterpret_cast<uint8_t*
>(destination) + offset;
397 sts = read(active_receive_fd_, buff, byte_cnt);
400 TLOG(10) << GetTraceName() <<
": recvFragment state=" <<
static_cast<int>(state) <<
" read=" << sts;
403 TLOG(TLVL_DEBUG) << GetTraceName() <<
": receiveFragmentData: Error on receive, closing socket"
404 <<
" (errno=" << errno <<
": " << strerror(errno) <<
")";
405 close(active_receive_fd_);
406 std::unique_lock<std::mutex> lk(connected_fd_mutex_);
407 connected_fds_[source_rank()].erase(active_receive_fd_);
408 active_receive_fd_ = -1;
414 if (sts >= target_bytes)
416 TLOG(9) << GetTraceName() <<
": receiveFragmentData: Target read bytes reached. Changing state";
418 if (state == SocketState::Metadata)
420 state = SocketState::Data;
427 ret_rank = source_rank();
428 TLOG(11) << GetTraceName() <<
": receiveFragmentData done sts=" << sts <<
" src=" << ret_rank;
429 TLOG(9) << GetTraceName() <<
": receiveFragmentData: Done receiving fragment. Moving into output.";
438 if (target_bytes == 0 && state == SocketState::Data)
440 ret_rank = source_rank();
441 TLOG(11) << GetTraceName() <<
": receiveFragmentData done sts=" << sts <<
" src=" << ret_rank;
442 TLOG(9) << GetTraceName() <<
": receiveFragmentData: Done receiving fragment. Moving into output.";
449 last_active_receive_fd_ = active_receive_fd_;
450 active_receive_fd_ = -1;
452 TLOG(9) << GetTraceName() <<
": receiveFragmentData: Returning " << ret_rank;
461 return send_fd_ != -1;
463 TLOG(TLVL_DEBUG) << GetTraceName() <<
": isRunning: There are " << getConnectedFDCount(source_rank()) <<
" fds connected.";
464 return getConnectedFDCount(source_rank()) > 0;
473 TLOG(12) << GetTraceName() <<
": sendFragment begin";
474 artdaq::Fragment grab_ownership_frag = std::move(frag);
478 iovec iov = {
reinterpret_cast<void*
>(grab_ownership_frag.headerAddress()),
479 detail::RawFragmentHeader::num_words() *
sizeof(RawDataType) };
481 auto sts = sendData_(&iov, 1, send_retry_timeout_us_);
482 auto start_time = std::chrono::steady_clock::now();
484 while (sts != CopyStatus::kSuccess && (send_timeout_usec == 0 || TimeUtils::GetElapsedTimeMicroseconds(start_time) < send_timeout_usec) && TimeUtils::GetElapsedTimeMicroseconds(start_time) < 10000000)
486 TLOG(13) << GetTraceName() <<
": sendFragment: Timeout or Error sending fragment";
487 sts = sendData_(&iov, 1, send_retry_timeout_us_);
490 if (sts != CopyStatus::kSuccess)
return sts;
494 iov = {
reinterpret_cast<void*
>(grab_ownership_frag.headerAddress() + detail::RawFragmentHeader::num_words()),
495 grab_ownership_frag.sizeBytes() - detail::RawFragmentHeader::num_words() *
sizeof(RawDataType) };
496 sts = sendData_(&iov, 1, send_retry_timeout_us_);
497 start_time = std::chrono::steady_clock::now();
498 while (sts != CopyStatus::kSuccess && (send_timeout_usec == 0 || TimeUtils::GetElapsedTimeMicroseconds(start_time) < send_timeout_usec) && TimeUtils::GetElapsedTimeMicroseconds(start_time) < 10000000)
500 TLOG(13) << GetTraceName() <<
": sendFragment: Timeout or Error sending fragment";
501 sts = sendData_(&iov, 1, send_retry_timeout_us_);
505 TLOG(12) << GetTraceName() <<
": sendFragment returning kSuccess";
511 TLOG(TLVL_DEBUG) << GetTraceName() <<
": sendData_ Converting buf to iovec";
512 iovec iov = { (
void*)buf, bytes };
513 return sendData_(&iov, 1, send_timeout_usec);
521 if (timeoutMessageArmed_)
523 TLOG(TLVL_DEBUG) << GetTraceName() <<
": sendData_: Send fd is not open. Returning kTimeout";
524 timeoutMessageArmed_ =
false;
526 return CopyStatus::kTimeout;
528 timeoutMessageArmed_ =
true;
529 TLOG(14) << GetTraceName() <<
": send_timeout_usec is " << std::to_string(send_timeout_usec) <<
", currently unused.";
532 uint32_t total_to_write_bytes = 0;
533 std::vector<iovec> iov_in(iovcnt + 1);
534 std::vector<iovec> iovv(iovcnt + 2);
536 for (ii = 0; ii < iovcnt; ++ii)
538 iov_in[ii + 1] = iov[ii];
539 total_to_write_bytes += iov[ii].iov_len;
542 MessHead mh = { 0,MessHead::data_v0,htons(source_rank()),{htonl(total_to_write_bytes)} };
543 iov_in[0].iov_base = &mh;
544 iov_in[0].iov_len =
sizeof(mh);
545 total_to_write_bytes +=
sizeof(mh);
548 ssize_t total_written_bytes = 0;
549 ssize_t per_write_max_bytes = (32 * 1024);
551 size_t in_iov_idx = 0;
552 size_t out_iov_idx = 0;
553 ssize_t this_write_bytes = 0;
560 (in_iov_idx + out_iov_idx) < iov_in.size() && this_write_bytes < per_write_max_bytes;
563 this_write_bytes += iov_in[in_iov_idx + out_iov_idx].iov_len;
564 iovv[out_iov_idx] = iov_in[in_iov_idx + out_iov_idx];
566 if (this_write_bytes > per_write_max_bytes)
568 iovv[out_iov_idx - 1].iov_len -= this_write_bytes - per_write_max_bytes;
569 this_write_bytes = per_write_max_bytes;
574 TLOG(14) << GetTraceName() <<
": sendFragment b4 writev " << std::setw(7) << std::to_string(total_written_bytes) <<
" total_written_bytes send_fd_=" << send_fd_ <<
" in_idx=" << std::to_string(in_iov_idx)
575 <<
" iovcnt=" << std::to_string(out_iov_idx) <<
" 1st.len=" << std::to_string(iovv[0].iov_len);
577 sts = writev(send_fd_, &(iovv[0]), out_iov_idx);
582 if (errno == EAGAIN )
584 TLOG(TLVL_DEBUG) << GetTraceName() <<
": sendFragment EWOULDBLOCK";
585 fcntl(send_fd_, F_SETFL, 0);
590 TLOG(TLVL_WARNING) << GetTraceName() <<
": sendFragment_: WRITE ERROR: " << strerror(errno);
596 else if (sts != this_write_bytes)
599 TLOG(TLVL_DEBUG) << GetTraceName() <<
": sendFragment writev sts(" << std::to_string(sts) <<
")!=requested_send_bytes(" << std::to_string(this_write_bytes) <<
")";
600 total_written_bytes += sts;
602 for (ii = 0; (size_t)sts >= iovv[ii].iov_len; ++ii)
603 sts -= iovv[ii].iov_len;
605 iovv[ii].iov_len -= sts;
606 iovv[ii].iov_base = (uint8_t*)(iovv[ii].iov_base) + sts;
611 iovv[out_iov_idx] = iovv[ii];
613 this_write_bytes = iovv[out_iov_idx].iov_len;
618 unsigned long additional = ((
unsigned long)iov_in[in_iov_idx].iov_base + iov_in[in_iov_idx].iov_len)
619 - ((
unsigned long)iovv[out_iov_idx].iov_base + iovv[out_iov_idx].iov_len);
622 iovv[out_iov_idx].iov_len += additional;
623 this_write_bytes += additional;
624 if (this_write_bytes > per_write_max_bytes)
626 iovv[out_iov_idx].iov_len -= this_write_bytes - per_write_max_bytes;
627 this_write_bytes = per_write_max_bytes;
631 TLOG(TLVL_TRACE) << GetTraceName() <<
": sendFragment writev sts!=: this_write_bytes=" << std::to_string(this_write_bytes)
632 <<
" out_iov_idx=" << std::to_string(out_iov_idx)
633 <<
" additional=" << std::to_string(additional)
638 TLOG(TLVL_TRACE) << GetTraceName() <<
": sendFragment writev sts(" << std::to_string(sts) <<
")==requested_send_bytes(" << std::to_string(this_write_bytes) <<
")";
639 total_written_bytes += sts;
641 iovv[out_iov_idx].iov_base = (uint8_t*)(iovv[out_iov_idx].iov_base) + iovv[out_iov_idx].iov_len;
642 iovv[out_iov_idx].iov_len = 0;
643 in_iov_idx += out_iov_idx;
644 this_write_bytes = 0;
646 unsigned long additional = ((
unsigned long)iov_in[in_iov_idx].iov_base + iov_in[in_iov_idx].iov_len)
647 - ((
unsigned long)iovv[out_iov_idx].iov_base + iovv[out_iov_idx].iov_len);
650 iovv[out_iov_idx].iov_len += additional;
651 this_write_bytes += additional;
652 if (this_write_bytes > per_write_max_bytes)
654 iovv[out_iov_idx].iov_len -= this_write_bytes - per_write_max_bytes;
655 this_write_bytes = per_write_max_bytes;
657 if (out_iov_idx != 0)
658 iovv[0] = iovv[out_iov_idx];
667 }
while (total_written_bytes < total_to_write_bytes);
668 if (total_written_bytes > total_to_write_bytes)
669 TLOG(TLVL_ERROR) << GetTraceName() <<
": sendFragment program error: too many bytes transferred";
674 fcntl(send_fd_, F_SETFL, 0);
676 sts = total_written_bytes -
sizeof(
MessHead);
678 TLOG(14) << GetTraceName() <<
": sendFragment sts=" << std::to_string(sts);
684 void artdaq::TCPSocketTransfer::stats_connect_()
687 while (!stats_connect_stop_)
691 std::function<void()>
function;
694 int msdly = tmo_.get_next_timeout_msdly();
699 std::unique_lock<std::mutex> lck(stopstatscvm_);
700 sts = stopstatscv_.wait_until(lck
701 , std::chrono::system_clock::now()
702 + std::chrono::milliseconds(msdly));
703 TLOG(15) << GetTraceName() <<
": thread1 after wait_until(msdly=" << msdly <<
") - sts=" <<
static_cast<int>(sts);
705 if (sts == std::cv_status::no_timeout)
708 auto sts = tmo_.get_next_expired_timeout(desc, &tag,
function, &ts_us);
710 while (sts != -1 && desc !=
"")
712 if (
function != NULL)
715 sts = tmo_.get_next_expired_timeout(desc, &tag,
function, &ts_us);
720 void artdaq::TCPSocketTransfer::connect_()
722 TLOG(TLVL_DEBUG) << GetTraceName() <<
": Connecting sender socket";
723 int sndbuf_bytes =
static_cast<int>(sndbuf_);
724 send_fd_ =
TCPConnect(hostMap_[destination_rank()].hostname.c_str()
730 TLOG(TLVL_DEBUG) << GetTraceName() <<
": connect_ " + hostMap_[destination_rank()].hostname +
":" << calculate_port_() <<
" send_fd_=" << send_fd_;
734 TLOG(TLVL_DEBUG) << GetTraceName() <<
": connect_: Writing connect message";
735 MessHead mh = { 0,MessHead::connect_v0,htons(source_rank()),{htonl(CONN_MAGIC)} };
736 ssize_t sts = write(send_fd_, &mh,
sizeof(mh));
739 TLOG(TLVL_ERROR) << GetTraceName() <<
": connect_: Error writing connect message!";
747 TLOG(TLVL_INFO) << GetTraceName() <<
": connect_: Successfully connected";
754 void artdaq::TCPSocketTransfer::reconnect_()
756 TLOG(TLVL_TRACE) << GetTraceName() <<
": check/reconnect";
760 void artdaq::TCPSocketTransfer::start_listen_thread_()
762 std::unique_lock<std::mutex> start_lock(listen_thread_mutex_);
763 if (listen_thread_refcount_ == 0)
765 if (listen_thread_ && listen_thread_->joinable()) listen_thread_->join();
766 listen_thread_refcount_ = 1;
767 TLOG(TLVL_INFO) << GetTraceName() <<
": Starting Listener Thread";
768 listen_thread_ = std::make_unique<boost::thread>(&TCPSocketTransfer::listen_, calculate_port_(), rcvbuf_);
772 listen_thread_refcount_++;
776 void artdaq::TCPSocketTransfer::listen_(
int port,
size_t rcvbuf)
779 while (listen_thread_refcount_ > 0)
781 TLOG(TLVL_TRACE) <<
"listen_: Listening/accepting new connections";
784 TLOG(TLVL_DEBUG) <<
"listen_: Opening listener";
789 TLOG(TLVL_DEBUG) <<
"listen_: Error creating listen_fd!";
794 timeval tv = { 2,0 };
797 FD_SET(listen_fd, &rfds);
799 res = select(listen_fd + 1, &rfds, (fd_set *)0, (fd_set *)0, &tv);
804 socklen_t arglen =
sizeof(un);
806 TLOG(TLVL_DEBUG) <<
"listen_: Calling accept";
807 fd = accept(listen_fd, (sockaddr *)&un, &arglen);
808 TLOG(TLVL_DEBUG) <<
"listen_: Done with accept";
810 TLOG(TLVL_DEBUG) <<
"listen_: Reading connect message";
811 socklen_t lenlen =
sizeof(tv);
813 setsockopt(fd, SOL_SOCKET, SO_RCVTIMEO, &tv, lenlen);
815 uint64_t mark_us = TimeUtils::gettimeofday_us();
816 sts = read(fd, &mh,
sizeof(mh));
817 uint64_t delta_us = TimeUtils::gettimeofday_us() - mark_us;
818 TLOG(TLVL_DEBUG) <<
"listen_: Read of connect message took " << delta_us <<
" microseconds.";
819 if (sts !=
sizeof(mh))
821 TLOG(TLVL_DEBUG) <<
"listen_: Wrong message header length received!";
830 TLOG(TLVL_DEBUG) <<
"listen_: Wrong magic bytes in header!";
836 std::unique_lock<std::mutex> lk(connected_fd_mutex_);
839 TLOG(TLVL_INFO) <<
"listen_: New fd is " << fd <<
" for source rank " << mh.
source_id;
843 TLOG(16) <<
"listen_: No connections in timeout interval!";
847 TLOG(TLVL_INFO) <<
"listen_: Shutting down connection listener";
848 if (listen_fd != -1) close(listen_fd);
849 std::unique_lock<std::mutex> lk(connected_fd_mutex_);
850 auto it = connected_fds_.begin();
851 while (it != connected_fds_.end())
853 auto& fd_set = it->second;
854 auto rank_it = fd_set.begin();
855 while (rank_it != fd_set.end())
858 rank_it = fd_set.erase(rank_it);
860 it = connected_fds_.erase(it);
bool isRunning() override
Determine whether the TransferInterface plugin is able to send/receive data.
virtual int source_rank() const
Get the source rank for this TransferInterface instance.
int TCPConnect(char const *host_in, int dflt_port, long flags=0, int sndbufsiz=0)
Connect to a host on a given port.
uint32_t conn_magic
unsigned first is better for MessHead initializer: {0,0,my_node_idx_,CONN_MAGIC}
std::string GetTraceName() const
Constructs a name suitable for TRACE messages.
This TransferInterface is a Receiver.
int receiveFragmentData(RawDataType *destination, size_t wordCount) override
Receive the body of a Fragment to the given destination pointer.
int TCP_listen_fd(int port, int rcvbuf)
Create a TCP listening socket on the given port and INADDR_ANY, with the given receive buffer...
TCPSocketTransfer(fhicl::ParameterSet const &ps, Role role)
TCPSocketTransfer Constructor.
int receiveFragmentHeader(detail::RawFragmentHeader &header, size_t receiveTimeout) override
Receive a Fragment Header from the transport mechanism.
This TransferInterface is a Sender.
int32_t byte_count
use CONN_MAGIC for connect_v0, data that follow for data_v0 (and 0 lenght data)
Some error occurred, but no exception was thrown.
Role
Used to determine if a TransferInterface is a Sender or Receiver.
int64_t source_id
Rank of the source.
MessType message_type
Message Type.
The send operation completed successfully.
This interface defines the functions used to transfer data between artdaq applications.
TransferInterface implementation plugin that sends data using TCP sockets.
This header is sent by the TCPSocket_transfer to allow for more efficient writev calls.
void add_periodic(const char *desc, void *tag, std::function< void()> &function, uint64_t period_us, uint64_t start_us=0)
Add a periodic timeout to the Timeout container.
CopyStatus
Returned from the send functions, this enumeration describes the possible return codes. If an exception occurs, it will be thrown and should be handled normally.