9 #include <sys/socket.h>
11 #include <arpa/inet.h>
12 #include <sys/types.h>
21 #define TRACE_NAME "TCPSocketTransfer"
22 #include "artdaq/DAQdata/Globals.hh"
25 #include "artdaq/TransferPlugins/TCPSocketTransfer.hh"
28 #include "artdaq/TransferPlugins/detail/Timeout.hh"
29 #include "artdaq/TransferPlugins/detail/SRSockets.hh"
30 #include "artdaq-core/Data/Fragment.hh"
31 #include "artdaq-core/Utilities/TimeUtils.hh"
34 std::atomic<int> artdaq::TCPSocketTransfer::listen_thread_refcount_(0);
35 std::unique_ptr<boost::thread> artdaq::TCPSocketTransfer::listen_thread_ =
nullptr;
36 std::map<int, std::set<int>> artdaq::TCPSocketTransfer::connected_fds_ = std::map<int, std::set<int>>();
37 std::mutex artdaq::TCPSocketTransfer::listen_thread_mutex_;
38 std::mutex artdaq::TCPSocketTransfer::connected_fd_mutex_;
44 , active_receive_fd_(-1)
45 , last_active_receive_fd_(-1)
46 , rcvbuf_(pset.get<size_t>(
"tcp_receive_buffer_size", 0))
47 , sndbuf_(max_fragment_size_words_ * sizeof(artdaq::RawDataType) * buffer_count_)
48 , send_retry_timeout_us_(pset.get<size_t>(
"send_retry_timeout_us", 1000000))
49 , timeoutMessageArmed_(true)
50 , not_connected_count_(0)
51 , receive_err_threshold_(pset.get<size_t>(
"receive_socket_disconnected_max_count", 1000))
52 , receive_err_wait_us_(pset.get<size_t>(
"receive_socket_disconnected_wait_us", 10000))
54 TLOG(TLVL_DEBUG) << GetTraceName() <<
" Constructor: pset=" << pset.to_string() <<
", role=" << (role ==
TransferInterface::Role::kReceive ?
"kReceive" :
"kSend");
55 auto masterPortOffset = pset.get<
int>(
"offset_all_ports", 0);
61 TLOG(TLVL_DEBUG) << GetTraceName() <<
": Listening for connections";
62 start_listen_thread_();
63 TLOG(TLVL_DEBUG) << GetTraceName() <<
": Done Listening";
67 TLOG(TLVL_DEBUG) << GetTraceName() <<
": Connecting to destination";
69 TLOG(TLVL_DEBUG) << GetTraceName() <<
": Done Connecting";
71 TLOG(TLVL_DEBUG) << GetTraceName() <<
": End of Constructor";
74 artdaq::TCPSocketTransfer::~TCPSocketTransfer() noexcept
76 TLOG(TLVL_DEBUG) << GetTraceName() <<
": Shutting down TCPSocketTransfer";
85 timeval tv = { 0,100000 };
86 socklen_t len =
sizeof(tv);
87 setsockopt(send_fd_, SOL_SOCKET, SO_SNDTIMEO, &tv, len);
88 write(send_fd_, &mh,
sizeof(mh));
95 std::unique_lock<std::mutex> fd_lock(connected_fd_mutex_);
96 if (connected_fds_.count(source_rank()))
98 auto it = connected_fds_[source_rank()].begin();
99 while (it != connected_fds_[source_rank()].end())
102 it = connected_fds_[source_rank()].erase(it);
104 connected_fds_.erase(source_rank());
108 std::unique_lock<std::mutex> lk(listen_thread_mutex_);
109 listen_thread_refcount_--;
110 if (listen_thread_refcount_ == 0 && listen_thread_ && listen_thread_->joinable())
112 listen_thread_->join();
115 TLOG(TLVL_DEBUG) << GetTraceName() <<
": End of Destructor";
120 TLOG(5) << GetTraceName() <<
": receiveFragmentHeader: BEGIN";
121 int ret_rank = RECV_TIMEOUT;
123 if (getConnectedFDCount(source_rank()) == 0)
125 if (++not_connected_count_ > receive_err_threshold_) {
return DATA_END; }
126 TLOG(7) << GetTraceName() <<
": receiveFragmentHeader: Receive socket not connected, returning RECV_TIMEOUT";
127 usleep(receive_err_wait_us_);
130 not_connected_count_ = 0;
132 TLOG(5) << GetTraceName() <<
": receiveFragmentHeader timeout_usec=" << timeout_usec;
137 SocketState state = SocketState::Metadata;
138 int target_bytes =
sizeof(
MessHead);
139 uint64_t start_time_us = TimeUtils::gettimeofday_us();
151 if (timeout_usec == 0)
154 timeout_ms = (timeout_usec + 999) / 1000;
157 while (!done && getConnectedFDCount(source_rank()) > 0)
159 if (active_receive_fd_ == -1)
162 std::vector<pollfd> pollfds;
164 std::unique_lock<std::mutex> lk(connected_fd_mutex_);
165 fd_count = connected_fds_[source_rank()].size();
166 pollfds.resize(fd_count);
167 auto iter = connected_fds_[source_rank()].begin();
168 for (
size_t ii = 0; ii < fd_count; ++ii)
170 pollfds[ii].events = POLLIN | POLLPRI | POLLERR;
171 pollfds[ii].fd = *iter;
176 int num_fds_ready = poll(&pollfds[0], fd_count, timeout_ms);
177 if (num_fds_ready <= 0)
179 TLOG(5) << GetTraceName() <<
": receiveFragmentHeader: No data on receive socket, returning RECV_TIMEOUT";
184 if (last_active_receive_fd_ != -1)
186 for (
auto& pollfd : pollfds)
189 if (pollfd.fd == last_active_receive_fd_)
196 int active_index = -1;
197 short anomolous_events = 0;
198 for (
size_t ii = index; ii < index + pollfds.size(); ++ii)
200 auto pollfd_index = (ii + index) % pollfds.size();
201 if (pollfds[pollfd_index].revents & (POLLIN | POLLPRI))
203 active_index = pollfd_index;
204 active_receive_fd_ = pollfds[active_index].fd;
207 else if (pollfds[pollfd_index].revents & (POLLHUP | POLLERR))
209 disconnect_receive_socket_(pollfds[pollfd_index].fd,
"Poll returned POLLHUP or POLLERR, indicating problems with the sender.");
212 else if (pollfds[pollfd_index].revents & (POLLNVAL))
214 TLOG(TLVL_DEBUG) << GetTraceName() <<
": receiveFragmentHeader: FD is closed, most likely because the peer went away. Removing from fd list.";
215 disconnect_receive_socket_(pollfds[pollfd_index].fd,
"FD is closed, most likely because the peer went away.");
218 else if (pollfds[pollfd_index].revents)
220 anomolous_events |= pollfds[pollfd_index].revents;
224 if (active_index == -1)
226 if (anomolous_events)
227 TLOG(TLVL_DEBUG) << GetTraceName() <<
": receiveFragmentHeader: Wrong event received from a pollfd. Mask: " <<
static_cast<int>(anomolous_events);
228 active_receive_fd_ = -1;
232 if (!done && timeout_usec > 0)
235 size_t delta_us = TimeUtils::gettimeofday_us() - start_time_us;
236 if (delta_us > timeout_usec)
240 timeout_ms = ((timeout_usec - delta_us) + 999) / 1000;
244 if (state == SocketState::Metadata)
247 buff = &(mha[offset]);
248 byte_cnt =
sizeof(
MessHead) - offset;
253 buff =
reinterpret_cast<uint8_t*
>(&header) + offset;
259 TLOG(6) << GetTraceName() <<
": receiveFragmentHeader: Reading " << byte_cnt <<
" bytes from socket";
260 sts = read(active_receive_fd_, buff, byte_cnt);
261 TLOG(6) << GetTraceName() <<
": receiveFragmentHeader: Done with read";
264 TLOG(7) << GetTraceName() <<
": receiveFragmentHeader state=" <<
static_cast<int>(state) <<
" read=" << sts;
267 TLOG(TLVL_WARNING) << GetTraceName() <<
": receiveFragmentHeader: Error on receive, closing socket " <<
" (errno=" << errno <<
": " << strerror(errno) <<
")";
268 active_receive_fd_ = disconnect_receive_socket_(active_receive_fd_);
274 if (sts >= target_bytes)
276 TLOG(7) << GetTraceName() <<
": receiveFragmentHeader: Target read bytes reached. Changing state";
278 if (state == SocketState::Metadata)
280 state = SocketState::Data;
287 active_receive_fd_ = disconnect_receive_socket_(active_receive_fd_,
"Stop Message received.");
290 if (target_bytes == 0)
298 ret_rank = source_rank();
299 TLOG(8) << GetTraceName() <<
": receiveFragmentHeader done sts=" << sts <<
" src=" << ret_rank;
300 TLOG(7) << GetTraceName() <<
": receiveFragmentHeader: Done receiving fragment header. Moving into output.";
310 TLOG(5) << GetTraceName() <<
": receiveFragmentHeader: Returning " << ret_rank;
314 int artdaq::TCPSocketTransfer::disconnect_receive_socket_(
int fd, std::string msg)
316 TLOG(TLVL_DEBUG) << GetTraceName() <<
": disconnect_receive_socket_: " << msg <<
" Closing socket " << fd;
318 std::unique_lock<std::mutex> lk(connected_fd_mutex_);
319 if (connected_fds_.count(source_rank()))
320 connected_fds_[source_rank()].erase(fd);
322 TLOG(TLVL_DEBUG) << GetTraceName() <<
": disconnect_receive_socket_: There are now " << connected_fds_[source_rank()].size() <<
" active senders.";
328 TLOG(9) << GetTraceName() <<
": receiveFragmentData: BEGIN";
329 int ret_rank = RECV_TIMEOUT;
330 if (active_receive_fd_ == -1)
332 TLOG(TLVL_DEBUG) << GetTraceName() <<
": receiveFragmentData: Receive socket not connected, returning RECV_TIMEOUT";
341 SocketState state = SocketState::Metadata;
342 int target_bytes =
sizeof(
MessHead);
345 pollfd_s.events = POLLIN | POLLPRI | POLLERR;
346 pollfd_s.fd = active_receive_fd_;
351 TLOG(9) << GetTraceName() <<
": receiveFragmentData: Polling fd to see if there's data";
352 int num_fds_ready = poll(&pollfd_s, 1, 1000);
353 if (num_fds_ready <= 0)
355 if (num_fds_ready == 0)
357 TLOG(9) << GetTraceName() <<
": receiveFragmentData: No data on receive socket, returning RECV_TIMEOUT";
358 active_receive_fd_ = -1;
362 TLOG(TLVL_ERROR) <<
"Error in poll: errno=" << errno;
363 active_receive_fd_ = -1;
367 if (pollfd_s.revents & (POLLIN | POLLPRI))
371 else if (pollfd_s.revents & (POLLNVAL))
373 disconnect_receive_socket_(pollfd_s.fd,
"FD is closed, most likely because the peer went away.");
376 else if (pollfd_s.revents & (POLLHUP | POLLERR))
378 disconnect_receive_socket_(pollfd_s.fd,
"Poll returned POLLHUP or POLLERR, indicating problems with the sender.");
383 TLOG(TLVL_DEBUG) << GetTraceName() <<
": receiveFragmentData: Wrong event received from pollfd: " << pollfd_s.revents;
384 disconnect_receive_socket_(pollfd_s.fd);
388 if (state == SocketState::Metadata)
391 buff = &(mha[offset]);
392 byte_cnt =
sizeof(
MessHead) - offset;
397 buff =
reinterpret_cast<uint8_t*
>(destination) + offset;
401 TLOG(10) << GetTraceName() <<
": receiveFragmentData: Reading " << byte_cnt <<
" bytes from socket into " << (
void*)buff;
402 sts = read(active_receive_fd_, buff, byte_cnt);
405 TLOG(10) << GetTraceName() <<
": recvFragment state=" <<
static_cast<int>(state) <<
" read=" << sts;
408 TLOG(TLVL_DEBUG) << GetTraceName() <<
": receiveFragmentData: Error on receive, closing socket"
409 <<
" (errno=" << errno <<
": " << strerror(errno) <<
")";
410 disconnect_receive_socket_(pollfd_s.fd);
416 if (sts >= target_bytes)
418 TLOG(9) << GetTraceName() <<
": receiveFragmentData: Target read bytes reached. Changing state";
420 if (state == SocketState::Metadata)
422 state = SocketState::Data;
429 ret_rank = source_rank();
430 TLOG(11) << GetTraceName() <<
": receiveFragmentData done sts=" << sts <<
" src=" << ret_rank;
431 TLOG(9) << GetTraceName() <<
": receiveFragmentData: Done receiving fragment. Moving into output.";
434 send_ack_(active_receive_fd_);
444 if (target_bytes == 0 && state == SocketState::Data)
446 ret_rank = source_rank();
447 TLOG(11) << GetTraceName() <<
": receiveFragmentData done sts=" << sts <<
" src=" << ret_rank;
448 TLOG(9) << GetTraceName() <<
": receiveFragmentData: Done receiving fragment. Moving into output.";
451 send_ack_(active_receive_fd_);
459 last_active_receive_fd_ = active_receive_fd_;
460 active_receive_fd_ = -1;
462 TLOG(9) << GetTraceName() <<
": receiveFragmentData: Returning " << ret_rank;
471 return send_fd_ != -1;
473 TLOG(TLVL_DEBUG) << GetTraceName() <<
": isRunning: There are " << getConnectedFDCount(source_rank()) <<
" fds connected.";
474 return getConnectedFDCount(source_rank()) > 0;
483 TLOG(12) << GetTraceName() <<
": sendFragment begin";
484 artdaq::Fragment grab_ownership_frag = std::move(frag);
489 iovec iov = {
reinterpret_cast<void*
>(grab_ownership_frag.headerAddress()),
490 detail::RawFragmentHeader::num_words() *
sizeof(RawDataType) };
492 auto sts = sendData_(&iov, 1, send_retry_timeout_us_);
493 auto start_time = std::chrono::steady_clock::now();
495 while (sts != CopyStatus::kSuccess && (send_timeout_usec == 0 || TimeUtils::GetElapsedTimeMicroseconds(start_time) < send_timeout_usec) && TimeUtils::GetElapsedTimeMicroseconds(start_time) < 10000000)
497 TLOG(13) << GetTraceName() <<
": sendFragment: Timeout or Error sending fragment";
498 sts = sendData_(&iov, 1, send_retry_timeout_us_);
501 if (sts != CopyStatus::kSuccess)
return sts;
505 iov = {
reinterpret_cast<void*
>(grab_ownership_frag.headerAddress() + detail::RawFragmentHeader::num_words()),
506 grab_ownership_frag.sizeBytes() - detail::RawFragmentHeader::num_words() *
sizeof(RawDataType) };
507 sts = sendData_(&iov, 1, send_retry_timeout_us_);
508 start_time = std::chrono::steady_clock::now();
509 while (sts != CopyStatus::kSuccess && (send_timeout_usec == 0 || TimeUtils::GetElapsedTimeMicroseconds(start_time) < send_timeout_usec) && TimeUtils::GetElapsedTimeMicroseconds(start_time) < 10000000)
511 TLOG(13) << GetTraceName() <<
": sendFragment: Timeout or Error sending fragment";
512 sts = sendData_(&iov, 1, send_retry_timeout_us_);
517 receive_ack_(send_fd_);
520 TLOG(12) << GetTraceName() <<
": sendFragment returning kSuccess";
526 TLOG(TLVL_DEBUG) << GetTraceName() <<
": sendData_ Converting buf to iovec";
527 iovec iov = { (
void*)buf, bytes };
528 return sendData_(&iov, 1, send_timeout_usec);
536 if (timeoutMessageArmed_)
538 TLOG(TLVL_DEBUG) << GetTraceName() <<
": sendData_: Send fd is not open. Returning kTimeout";
539 timeoutMessageArmed_ =
false;
541 return CopyStatus::kTimeout;
543 timeoutMessageArmed_ =
true;
544 TLOG(14) << GetTraceName() <<
": send_timeout_usec is " << send_timeout_usec <<
", currently unused.";
547 uint32_t total_to_write_bytes = 0;
548 std::vector<iovec> iov_in(iovcnt + 1);
549 std::vector<iovec> iovv(iovcnt + 2);
551 for (ii = 0; ii < iovcnt; ++ii)
553 iov_in[ii + 1] = iov[ii];
554 total_to_write_bytes += iov[ii].iov_len;
557 MessHead mh = { 0,MessHead::data_v0,htons(source_rank()),{htonl(total_to_write_bytes)} };
558 iov_in[0].iov_base = &mh;
559 iov_in[0].iov_len =
sizeof(mh);
560 total_to_write_bytes +=
sizeof(mh);
563 ssize_t total_written_bytes = 0;
564 ssize_t per_write_max_bytes = (32 * 1024);
566 size_t in_iov_idx = 0;
567 size_t out_iov_idx = 0;
568 ssize_t this_write_bytes = 0;
575 (in_iov_idx + out_iov_idx) < iov_in.size() && this_write_bytes < per_write_max_bytes;
578 this_write_bytes += iov_in[in_iov_idx + out_iov_idx].iov_len;
579 iovv[out_iov_idx] = iov_in[in_iov_idx + out_iov_idx];
581 if (this_write_bytes > per_write_max_bytes)
583 iovv[out_iov_idx - 1].iov_len -= this_write_bytes - per_write_max_bytes;
584 this_write_bytes = per_write_max_bytes;
589 #ifndef __OPTIMIZE__ // This can be an expensive TRACE call (even if disabled) due to multiplicity of calls
590 TLOG(14) << GetTraceName() <<
": sendFragment b4 writev " << std::setw(7) << total_written_bytes <<
" total_written_bytes send_fd_=" << send_fd_ <<
" in_idx=" << in_iov_idx
591 <<
" iovcnt=" << out_iov_idx <<
" 1st.len=" << iovv[0].iov_len;
594 sts = writev(send_fd_, &(iovv[0]), out_iov_idx);
599 if (errno == EAGAIN )
601 TLOG(TLVL_DEBUG) << GetTraceName() <<
": sendFragment EWOULDBLOCK";
602 fcntl(send_fd_, F_SETFL, 0);
607 TLOG(TLVL_WARNING) << GetTraceName() <<
": sendFragment_: WRITE ERROR: " << strerror(errno);
613 else if (sts != this_write_bytes)
616 TLOG(TLVL_DEBUG) << GetTraceName() <<
": sendFragment writev sts(" << sts <<
")!=requested_send_bytes(" << this_write_bytes <<
")";
617 total_written_bytes += sts;
619 for (ii = 0; (size_t)sts >= iovv[ii].iov_len; ++ii)
620 sts -= iovv[ii].iov_len;
622 iovv[ii].iov_len -= sts;
623 iovv[ii].iov_base = (uint8_t*)(iovv[ii].iov_base) + sts;
628 iovv[out_iov_idx] = iovv[ii];
630 this_write_bytes = iovv[out_iov_idx].iov_len;
635 unsigned long additional = ((
unsigned long)iov_in[in_iov_idx].iov_base + iov_in[in_iov_idx].iov_len)
636 - ((
unsigned long)iovv[out_iov_idx].iov_base + iovv[out_iov_idx].iov_len);
639 iovv[out_iov_idx].iov_len += additional;
640 this_write_bytes += additional;
641 if (this_write_bytes > per_write_max_bytes)
643 iovv[out_iov_idx].iov_len -= this_write_bytes - per_write_max_bytes;
644 this_write_bytes = per_write_max_bytes;
648 TLOG(TLVL_TRACE) << GetTraceName() <<
": sendFragment writev sts!=: this_write_bytes=" << this_write_bytes
649 <<
" out_iov_idx=" << out_iov_idx
650 <<
" additional=" << additional
655 #ifndef __OPTIMIZE__ // This can be an expensive TRACE call (even if disabled) due to multiplicity of calls
656 TLOG(TLVL_TRACE) << GetTraceName() <<
": sendFragment writev sts(" << sts <<
")==requested_send_bytes(" << this_write_bytes <<
")";
658 total_written_bytes += sts;
660 iovv[out_iov_idx].iov_base = (uint8_t*)(iovv[out_iov_idx].iov_base) + iovv[out_iov_idx].iov_len;
661 iovv[out_iov_idx].iov_len = 0;
662 in_iov_idx += out_iov_idx;
663 this_write_bytes = 0;
665 unsigned long additional = ((
unsigned long)iov_in[in_iov_idx].iov_base + iov_in[in_iov_idx].iov_len)
666 - ((
unsigned long)iovv[out_iov_idx].iov_base + iovv[out_iov_idx].iov_len);
669 iovv[out_iov_idx].iov_len += additional;
670 this_write_bytes += additional;
671 if (this_write_bytes > per_write_max_bytes)
673 iovv[out_iov_idx].iov_len -= this_write_bytes - per_write_max_bytes;
674 this_write_bytes = per_write_max_bytes;
676 if (out_iov_idx != 0)
677 iovv[0] = iovv[out_iov_idx];
686 }
while (total_written_bytes < total_to_write_bytes);
687 if (total_written_bytes > total_to_write_bytes)
688 TLOG(TLVL_ERROR) << GetTraceName() <<
": sendFragment program error: too many bytes transferred";
693 fcntl(send_fd_, F_SETFL, O_NONBLOCK);
695 sts = total_written_bytes -
sizeof(
MessHead);
697 TLOG(14) << GetTraceName() <<
": sendFragment sts=" << sts;
701 void artdaq::TCPSocketTransfer::connect_()
703 auto start_time = std::chrono::steady_clock::now();
706 while (send_fd_ == -1 && TimeUtils::GetElapsedTimeMicroseconds(start_time) < send_retry_timeout_us_ * 10)
708 TLOG(TLVL_DEBUG) << GetTraceName() <<
": Connecting sender socket";
709 int sndbuf_bytes =
static_cast<int>(sndbuf_);
710 send_fd_ =
TCPConnect(hostMap_[destination_rank()].hostname.c_str()
715 usleep(send_retry_timeout_us_);
719 TLOG(TLVL_DEBUG) << GetTraceName() <<
": connect_ " + hostMap_[destination_rank()].hostname +
":" << calculate_port_() <<
" send_fd_=" << send_fd_;
723 TLOG(TLVL_DEBUG) << GetTraceName() <<
": connect_: Writing connect message";
724 MessHead mh = { 0,MessHead::connect_v0,htons(source_rank()),{htonl(CONN_MAGIC)} };
725 ssize_t sts = write(send_fd_, &mh,
sizeof(mh));
728 TLOG(TLVL_ERROR) << GetTraceName() <<
": connect_: Error writing connect message!";
736 TLOG(TLVL_INFO) << GetTraceName() <<
": connect_: Successfully connected";
743 void artdaq::TCPSocketTransfer::reconnect_()
747 TLOG(TLVL_TRACE) << GetTraceName() <<
": check/reconnect";
752 void artdaq::TCPSocketTransfer::start_listen_thread_()
754 std::unique_lock<std::mutex> start_lock(listen_thread_mutex_);
755 if (listen_thread_refcount_ == 0)
757 if (listen_thread_ && listen_thread_->joinable()) listen_thread_->join();
758 listen_thread_refcount_ = 1;
759 TLOG(TLVL_INFO) << GetTraceName() <<
": Starting Listener Thread";
760 listen_thread_ = std::make_unique<boost::thread>(&TCPSocketTransfer::listen_, calculate_port_(), rcvbuf_);
764 listen_thread_refcount_++;
769 void artdaq::TCPSocketTransfer::receive_ack_(
int fd)
772 uint64_t mark_us = TimeUtils::gettimeofday_us();
773 fcntl(send_fd_, F_SETFL, 0);
774 auto sts = read(fd, &mh,
sizeof(mh));
775 fcntl(send_fd_, F_SETFL, O_NONBLOCK);
776 uint64_t delta_us = TimeUtils::gettimeofday_us() - mark_us;
777 TLOG(17) << GetTraceName() <<
": receive_ack_: Read of ack message took " << delta_us <<
" microseconds.";
778 if (sts !=
sizeof(mh))
780 TLOG(TLVL_ERROR) << GetTraceName() <<
": receive_ack_: Wrong message header length received! (actual " << sts <<
" != " <<
sizeof(mh) <<
" expected)";
790 TLOG(TLVL_ERROR) << GetTraceName() <<
": receive_ack_: Received ack for different sender! Rank=" << my_rank <<
", hdr=" << mh.
source_id;
797 TLOG(TLVL_ERROR) << GetTraceName() <<
": receive_ack_: Wrong magic bytes in header!";
804 void artdaq::TCPSocketTransfer::send_ack_(
int fd)
806 MessHead mh = { 0,MessHead::ack_v0,htons(source_rank()),{ htonl(ACK_MAGIC) } };
807 write(fd, &mh,
sizeof(mh));
811 void artdaq::TCPSocketTransfer::listen_(
int port,
size_t rcvbuf)
814 while (listen_thread_refcount_ > 0)
816 TLOG(TLVL_TRACE) <<
"listen_: Listening/accepting new connections";
819 TLOG(TLVL_DEBUG) <<
"listen_: Opening listener";
824 TLOG(TLVL_DEBUG) <<
"listen_: Error creating listen_fd!";
829 timeval tv = { 2,0 };
832 FD_SET(listen_fd, &rfds);
834 res = select(listen_fd + 1, &rfds, (fd_set *)0, (fd_set *)0, &tv);
839 socklen_t arglen =
sizeof(un);
841 TLOG(TLVL_DEBUG) <<
"listen_: Calling accept";
842 fd = accept(listen_fd, (sockaddr *)&un, &arglen);
843 TLOG(TLVL_DEBUG) <<
"listen_: Done with accept";
845 TLOG(TLVL_DEBUG) <<
"listen_: Reading connect message";
846 socklen_t lenlen =
sizeof(tv);
848 setsockopt(fd, SOL_SOCKET, SO_RCVTIMEO, &tv, lenlen);
850 uint64_t mark_us = TimeUtils::gettimeofday_us();
851 sts = read(fd, &mh,
sizeof(mh));
852 uint64_t delta_us = TimeUtils::gettimeofday_us() - mark_us;
853 TLOG(TLVL_DEBUG) <<
"listen_: Read of connect message took " << delta_us <<
" microseconds.";
854 if (sts !=
sizeof(mh))
856 TLOG(TLVL_DEBUG) <<
"listen_: Wrong message header length received!";
865 TLOG(TLVL_DEBUG) <<
"listen_: Wrong magic bytes in header!";
871 std::unique_lock<std::mutex> lk(connected_fd_mutex_);
874 TLOG(TLVL_INFO) <<
"listen_: New fd is " << fd <<
" for source rank " << mh.
source_id;
878 TLOG(16) <<
"listen_: No connections in timeout interval!";
882 TLOG(TLVL_INFO) <<
"listen_: Shutting down connection listener";
883 if (listen_fd != -1) close(listen_fd);
884 std::unique_lock<std::mutex> lk(connected_fd_mutex_);
885 auto it = connected_fds_.begin();
886 while (it != connected_fds_.end())
888 auto& fd_set = it->second;
889 auto rank_it = fd_set.begin();
890 while (rank_it != fd_set.end())
893 rank_it = fd_set.erase(rank_it);
895 it = connected_fds_.erase(it);
bool isRunning() override
Determine whether the TransferInterface plugin is able to send/receive data.
virtual int source_rank() const
Get the source rank for this TransferInterface instance.
int TCPConnect(char const *host_in, int dflt_port, long flags=0, int sndbufsiz=0)
Connect to a host on a given port.
uint32_t conn_magic
unsigned first is better for MessHead initializer: {0,0,my_node_idx_,CONN_MAGIC}
This TransferInterface is a Receiver.
int receiveFragmentData(RawDataType *destination, size_t wordCount) override
Receive the body of a Fragment to the given destination pointer.
int TCP_listen_fd(int port, int rcvbuf)
Create a TCP listening socket on the given port and INADDR_ANY, with the given receive buffer...
TCPSocketTransfer(fhicl::ParameterSet const &ps, Role role)
TCPSocketTransfer Constructor.
int receiveFragmentHeader(detail::RawFragmentHeader &header, size_t receiveTimeout) override
Receive a Fragment Header from the transport mechanism.
This TransferInterface is a Sender.
int32_t byte_count
use CONN_MAGIC for connect_v0, data that follow for data_v0 (and 0 lenght data)
Some error occurred, but no exception was thrown.
Role
Used to determine if a TransferInterface is a Sender or Receiver.
int64_t source_id
Rank of the source.
MessType message_type
Message Type.
The send operation completed successfully.
hostMap_t MakeHostMap(fhicl::ParameterSet pset, int masterPortOffset=0, hostMap_t map=hostMap_t())
Make a hostMap_t from a HostMap::Config ParameterSet
This interface defines the functions used to transfer data between artdaq applications.
TransferInterface implementation plugin that sends data using TCP sockets.
This header is sent by the TCPSocket_transfer to allow for more efficient writev calls.
CopyStatus
Returned from the send functions, this enumeration describes the possible return codes. If an exception occurs, it will be thrown and should be handled normally.