00001
00002
00003
00004
00005
00006
00007
00008 #include <stdlib.h>
00009 #include <sys/socket.h>
00010 #include <sys/un.h>
00011 #include <arpa/inet.h>
00012 #include <sys/types.h>
00013 #include <poll.h>
00014
00015
00016 #include <string>
00017 #include <fstream>
00018 #include <stdexcept>
00019
00020
00021 #define TRACE_NAME "TCPSocketTransfer"
00022 #include "artdaq/DAQdata/Globals.hh"
00023
00024
00025 #include "artdaq/TransferPlugins/TCPSocketTransfer.hh"
00026 #include "artdaq/DAQdata/TCP_listen_fd.hh"
00027 #include "artdaq/DAQdata/TCPConnect.hh"
00028 #include "artdaq/TransferPlugins/detail/Timeout.hh"
00029 #include "artdaq/TransferPlugins/detail/SRSockets.hh"
00030 #include "artdaq-core/Data/Fragment.hh"
00031 #include "artdaq-core/Utilities/TimeUtils.hh"
00032 #include <iomanip>
00033
00034 std::atomic<int> artdaq::TCPSocketTransfer::listen_thread_refcount_(0);
00035 std::unique_ptr<boost::thread> artdaq::TCPSocketTransfer::listen_thread_ = nullptr;
00036 std::map<int, std::set<int>> artdaq::TCPSocketTransfer::connected_fds_ = std::map<int, std::set<int>>();
00037 std::mutex artdaq::TCPSocketTransfer::listen_thread_mutex_;
00038 std::mutex artdaq::TCPSocketTransfer::connected_fd_mutex_;
00039
00040 artdaq::TCPSocketTransfer::
00041 TCPSocketTransfer(fhicl::ParameterSet const& pset, TransferInterface::Role role)
00042 : TransferInterface(pset, role)
00043 , send_fd_(-1)
00044 , active_receive_fd_(-1)
00045 , last_active_receive_fd_(-1)
00046 , rcvbuf_(pset.get<size_t>("tcp_receive_buffer_size", 0))
00047 , sndbuf_(max_fragment_size_words_ * sizeof(artdaq::RawDataType) * buffer_count_)
00048 , send_retry_timeout_us_(pset.get<size_t>("send_retry_timeout_us", 1000000))
00049 , timeoutMessageArmed_(true)
00050 , not_connected_count_(0)
00051 , receive_err_threshold_(pset.get<size_t>("receive_socket_disconnected_max_count", 1000))
00052 , receive_err_wait_us_(pset.get<size_t>("receive_socket_disconnected_wait_us", 10000))
00053 {
00054 TLOG(TLVL_DEBUG) << GetTraceName() << " Constructor: pset=" << pset.to_string() << ", role=" << (role == TransferInterface::Role::kReceive ? "kReceive" : "kSend");
00055 auto masterPortOffset = pset.get<int>("offset_all_ports", 0);
00056 hostMap_ = MakeHostMap(pset, masterPortOffset);
00057
00058 if (role == TransferInterface::Role::kReceive)
00059 {
00060
00061 TLOG(TLVL_DEBUG) << GetTraceName() << ": Listening for connections";
00062 start_listen_thread_();
00063 TLOG(TLVL_DEBUG) << GetTraceName() << ": Done Listening";
00064 }
00065 else
00066 {
00067 TLOG(TLVL_DEBUG) << GetTraceName() << ": Connecting to destination";
00068 connect_();
00069 TLOG(TLVL_DEBUG) << GetTraceName() << ": Done Connecting";
00070 }
00071 TLOG(TLVL_DEBUG) << GetTraceName() << ": End of Constructor";
00072 }
00073
00074 artdaq::TCPSocketTransfer::~TCPSocketTransfer() noexcept
00075 {
00076 TLOG(TLVL_DEBUG) << GetTraceName() << ": Shutting down TCPSocketTransfer";
00077
00078 if (role() == TransferInterface::Role::kSend)
00079 {
00080
00081 MessHead mh = { 0,MessHead::stop_v0,htons(TransferInterface::source_rank()),{0} };
00082 if (send_fd_ != -1)
00083 {
00084
00085 timeval tv = { 0,100000 };
00086 socklen_t len = sizeof(tv);
00087 setsockopt(send_fd_, SOL_SOCKET, SO_SNDTIMEO, &tv, len);
00088 write(send_fd_, &mh, sizeof(mh));
00089 }
00090 close(send_fd_);
00091 }
00092 else
00093 {
00094 {
00095 std::unique_lock<std::mutex> fd_lock(connected_fd_mutex_);
00096 if (connected_fds_.count(source_rank()))
00097 {
00098 auto it = connected_fds_[source_rank()].begin();
00099 while (it != connected_fds_[source_rank()].end())
00100 {
00101 close(*it);
00102 it = connected_fds_[source_rank()].erase(it);
00103 }
00104 connected_fds_.erase(source_rank());
00105 }
00106 }
00107
00108 std::unique_lock<std::mutex> lk(listen_thread_mutex_);
00109 listen_thread_refcount_--;
00110 if (listen_thread_refcount_ == 0 && listen_thread_ && listen_thread_->joinable())
00111 {
00112 listen_thread_->join();
00113 }
00114 }
00115 TLOG(TLVL_DEBUG) << GetTraceName() << ": End of Destructor";
00116 }
00117
00118 int artdaq::TCPSocketTransfer::receiveFragmentHeader(detail::RawFragmentHeader& header, size_t timeout_usec)
00119 {
00120 TLOG(5) << GetTraceName() << ": receiveFragmentHeader: BEGIN";
00121 int ret_rank = RECV_TIMEOUT;
00122
00123 if (getConnectedFDCount(source_rank()) == 0)
00124 {
00125 if (++not_connected_count_ > receive_err_threshold_) { return DATA_END; }
00126 TLOG(7) << GetTraceName() << ": receiveFragmentHeader: Receive socket not connected, returning RECV_TIMEOUT";
00127 usleep(receive_err_wait_us_);
00128 return RECV_TIMEOUT;
00129 }
00130 not_connected_count_ = 0;
00131
00132 TLOG(5) << GetTraceName() << ": receiveFragmentHeader timeout_usec=" << timeout_usec;
00133
00134 size_t byte_cnt = 0;
00135 int sts;
00136 int offset = 0;
00137 SocketState state = SocketState::Metadata;
00138 int target_bytes = sizeof(MessHead);
00139 uint64_t start_time_us = TimeUtils::gettimeofday_us();
00140
00141
00142
00143
00144
00145
00146
00147
00148 uint8_t* buff;
00149
00150 int timeout_ms;
00151 if (timeout_usec == 0)
00152 timeout_ms = 0;
00153 else
00154 timeout_ms = (timeout_usec + 999) / 1000;
00155
00156 bool done = false;
00157 while (!done && getConnectedFDCount(source_rank()) > 0)
00158 {
00159 if (active_receive_fd_ == -1)
00160 {
00161 size_t fd_count = 0;
00162 std::vector<pollfd> pollfds;
00163 {
00164 std::unique_lock<std::mutex> lk(connected_fd_mutex_);
00165 fd_count = connected_fds_[source_rank()].size();
00166 pollfds.resize(fd_count);
00167 auto iter = connected_fds_[source_rank()].begin();
00168 for (size_t ii = 0; ii < fd_count; ++ii)
00169 {
00170 pollfds[ii].events = POLLIN | POLLPRI | POLLERR;
00171 pollfds[ii].fd = *iter;
00172 ++iter;
00173 }
00174 }
00175
00176 int num_fds_ready = poll(&pollfds[0], fd_count, timeout_ms);
00177 if (num_fds_ready <= 0)
00178 {
00179 TLOG(5) << GetTraceName() << ": receiveFragmentHeader: No data on receive socket, returning RECV_TIMEOUT";
00180 return RECV_TIMEOUT;
00181 }
00182
00183 size_t index = 0;
00184 if (last_active_receive_fd_ != -1)
00185 {
00186 for (auto& pollfd : pollfds)
00187 {
00188 index++;
00189 if (pollfd.fd == last_active_receive_fd_)
00190 {
00191 break;
00192 }
00193 }
00194 }
00195
00196 int active_index = -1;
00197 short anomolous_events = 0;
00198 for (size_t ii = index; ii < index + pollfds.size(); ++ii)
00199 {
00200 auto pollfd_index = (ii + index) % pollfds.size();
00201 if (pollfds[pollfd_index].revents & (POLLIN | POLLPRI))
00202 {
00203 active_index = pollfd_index;
00204 active_receive_fd_ = pollfds[active_index].fd;
00205 break;
00206 }
00207 else if (pollfds[pollfd_index].revents & (POLLHUP | POLLERR))
00208 {
00209 disconnect_receive_socket_(pollfds[pollfd_index].fd, "Poll returned POLLHUP or POLLERR, indicating problems with the sender.");
00210 continue;
00211 }
00212 else if (pollfds[pollfd_index].revents & (POLLNVAL))
00213 {
00214 TLOG(TLVL_DEBUG) << GetTraceName() << ": receiveFragmentHeader: FD is closed, most likely because the peer went away. Removing from fd list.";
00215 disconnect_receive_socket_(pollfds[pollfd_index].fd, "FD is closed, most likely because the peer went away.");
00216 continue;
00217 }
00218 else if (pollfds[pollfd_index].revents)
00219 {
00220 anomolous_events |= pollfds[pollfd_index].revents;
00221 }
00222 }
00223
00224 if (active_index == -1)
00225 {
00226 if (anomolous_events)
00227 TLOG(TLVL_DEBUG) << GetTraceName() << ": receiveFragmentHeader: Wrong event received from a pollfd. Mask: " << static_cast<int>(anomolous_events);
00228 active_receive_fd_ = -1;
00229 continue;
00230 }
00231
00232 if (!done && timeout_usec > 0)
00233 {
00234
00235 size_t delta_us = TimeUtils::gettimeofday_us() - start_time_us;
00236 if (delta_us > timeout_usec)
00237 {
00238 return RECV_TIMEOUT;
00239 }
00240 timeout_ms = ((timeout_usec - delta_us) + 999) / 1000;
00241 }
00242 }
00243
00244 if (state == SocketState::Metadata)
00245 {
00246
00247 buff = &(mha[offset]);
00248 byte_cnt = sizeof(MessHead) - offset;
00249 }
00250 else
00251 {
00252
00253 buff = reinterpret_cast<uint8_t*>(&header) + offset;
00254 byte_cnt = mh.byte_count - offset;
00255 }
00256
00257 if (byte_cnt > 0)
00258 {
00259 TLOG(6) << GetTraceName() << ": receiveFragmentHeader: Reading " << byte_cnt << " bytes from socket";
00260 sts = read(active_receive_fd_, buff, byte_cnt);
00261 TLOG(6) << GetTraceName() << ": receiveFragmentHeader: Done with read";
00262 }
00263
00264 TLOG(7) << GetTraceName() << ": receiveFragmentHeader state=" << static_cast<int>(state) << " read=" << sts;
00265 if (sts < 0)
00266 {
00267 TLOG(TLVL_WARNING) << GetTraceName() << ": receiveFragmentHeader: Error on receive, closing socket " << " (errno=" << errno << ": " << strerror(errno) << ")";
00268 active_receive_fd_ = disconnect_receive_socket_(active_receive_fd_);
00269 }
00270 else
00271 {
00272
00273 sts = offset += sts;
00274 if (sts >= target_bytes)
00275 {
00276 TLOG(7) << GetTraceName() << ": receiveFragmentHeader: Target read bytes reached. Changing state";
00277 offset = 0;
00278 if (state == SocketState::Metadata)
00279 {
00280 state = SocketState::Data;
00281 mh.byte_count = ntohl(mh.byte_count);
00282 mh.source_id = ntohs(mh.source_id);
00283 target_bytes = mh.byte_count;
00284
00285 if (mh.message_type == MessHead::stop_v0)
00286 {
00287 active_receive_fd_ = disconnect_receive_socket_(active_receive_fd_, "Stop Message received.");
00288 }
00289
00290 if (target_bytes == 0)
00291 {
00292
00293 return RECV_TIMEOUT;
00294 }
00295 }
00296 else
00297 {
00298 ret_rank = source_rank();
00299 TLOG(8) << GetTraceName() << ": receiveFragmentHeader done sts=" << sts << " src=" << ret_rank;
00300 TLOG(7) << GetTraceName() << ": receiveFragmentHeader: Done receiving fragment header. Moving into output.";
00301
00302 done = true;
00303 break;
00304 }
00305 }
00306 }
00307
00308 }
00309
00310 TLOG(5) << GetTraceName() << ": receiveFragmentHeader: Returning " << ret_rank;
00311 return ret_rank;
00312 }
00313
00314 int artdaq::TCPSocketTransfer::disconnect_receive_socket_(int fd, std::string msg)
00315 {
00316 TLOG(TLVL_DEBUG) << GetTraceName() << ": disconnect_receive_socket_: " << msg << " Closing socket " << fd;
00317 close(fd);
00318 std::unique_lock<std::mutex> lk(connected_fd_mutex_);
00319 if (connected_fds_.count(source_rank()))
00320 connected_fds_[source_rank()].erase(fd);
00321 fd = -1;
00322 TLOG(TLVL_DEBUG) << GetTraceName() << ": disconnect_receive_socket_: There are now " << connected_fds_[source_rank()].size() << " active senders.";
00323 return fd;
00324 }
00325
00326 int artdaq::TCPSocketTransfer::receiveFragmentData(RawDataType* destination, size_t)
00327 {
00328 TLOG(9) << GetTraceName() << ": receiveFragmentData: BEGIN";
00329 int ret_rank = RECV_TIMEOUT;
00330 if (active_receive_fd_ == -1)
00331 {
00332 TLOG(TLVL_DEBUG) << GetTraceName() << ": receiveFragmentData: Receive socket not connected, returning RECV_TIMEOUT";
00333 return RECV_TIMEOUT;
00334 }
00335
00336
00337 uint8_t* buff;
00338 size_t byte_cnt = 0;
00339 int sts;
00340 int offset = 0;
00341 SocketState state = SocketState::Metadata;
00342 int target_bytes = sizeof(MessHead);
00343
00344 pollfd pollfd_s;
00345 pollfd_s.events = POLLIN | POLLPRI | POLLERR;
00346 pollfd_s.fd = active_receive_fd_;
00347
00348 bool done = false;
00349 while (!done)
00350 {
00351 TLOG(9) << GetTraceName() << ": receiveFragmentData: Polling fd to see if there's data";
00352 int num_fds_ready = poll(&pollfd_s, 1, 1000);
00353 if (num_fds_ready <= 0)
00354 {
00355 if (num_fds_ready == 0)
00356 {
00357 TLOG(9) << GetTraceName() << ": receiveFragmentData: No data on receive socket, returning RECV_TIMEOUT";
00358 active_receive_fd_ = -1;
00359 return RECV_TIMEOUT;
00360 }
00361
00362 TLOG(TLVL_ERROR) << "Error in poll: errno=" << errno;
00363 active_receive_fd_ = -1;
00364 break;
00365 }
00366
00367 if (pollfd_s.revents & (POLLIN | POLLPRI))
00368 {
00369
00370 }
00371 else if (pollfd_s.revents & (POLLNVAL))
00372 {
00373 disconnect_receive_socket_(pollfd_s.fd, "FD is closed, most likely because the peer went away.");
00374 break;
00375 }
00376 else if (pollfd_s.revents & (POLLHUP | POLLERR))
00377 {
00378 disconnect_receive_socket_(pollfd_s.fd, "Poll returned POLLHUP or POLLERR, indicating problems with the sender.");
00379 break;
00380 }
00381 else
00382 {
00383 TLOG(TLVL_DEBUG) << GetTraceName() << ": receiveFragmentData: Wrong event received from pollfd: " << pollfd_s.revents;
00384 disconnect_receive_socket_(pollfd_s.fd);
00385 break;
00386 }
00387
00388 if (state == SocketState::Metadata)
00389 {
00390
00391 buff = &(mha[offset]);
00392 byte_cnt = sizeof(MessHead) - offset;
00393 }
00394 else
00395 {
00396
00397 buff = reinterpret_cast<uint8_t*>(destination) + offset;
00398 byte_cnt = mh.byte_count - offset;
00399 }
00400
00401 TLOG(10) << GetTraceName() << ": receiveFragmentData: Reading " << byte_cnt << " bytes from socket into " << (void*)buff;
00402 sts = read(active_receive_fd_, buff, byte_cnt);
00403
00404
00405 TLOG(10) << GetTraceName() << ": recvFragment state=" << static_cast<int>(state) << " read=" << sts;
00406 if (sts < 0)
00407 {
00408 TLOG(TLVL_DEBUG) << GetTraceName() << ": receiveFragmentData: Error on receive, closing socket"
00409 << " (errno=" << errno << ": " << strerror(errno) << ")";
00410 disconnect_receive_socket_(pollfd_s.fd);
00411 }
00412 else
00413 {
00414
00415 sts = offset += sts;
00416 if (sts >= target_bytes)
00417 {
00418 TLOG(9) << GetTraceName() << ": receiveFragmentData: Target read bytes reached. Changing state";
00419 offset = 0;
00420 if (state == SocketState::Metadata)
00421 {
00422 state = SocketState::Data;
00423 mh.byte_count = ntohl(mh.byte_count);
00424 mh.source_id = ntohs(mh.source_id);
00425 target_bytes = mh.byte_count;
00426 }
00427 else
00428 {
00429 ret_rank = source_rank();
00430 TLOG(11) << GetTraceName() << ": receiveFragmentData done sts=" << sts << " src=" << ret_rank;
00431 TLOG(9) << GetTraceName() << ": receiveFragmentData: Done receiving fragment. Moving into output.";
00432
00433 #if USE_ACKS
00434 send_ack_(active_receive_fd_);
00435 #endif
00436
00437 done = true;
00438 break;
00439 }
00440 }
00441 }
00442
00443
00444 if (target_bytes == 0 && state == SocketState::Data)
00445 {
00446 ret_rank = source_rank();
00447 TLOG(11) << GetTraceName() << ": receiveFragmentData done sts=" << sts << " src=" << ret_rank;
00448 TLOG(9) << GetTraceName() << ": receiveFragmentData: Done receiving fragment. Moving into output.";
00449
00450 #if USE_ACKS
00451 send_ack_(active_receive_fd_);
00452 #endif
00453
00454 done = true;
00455 }
00456
00457 }
00458
00459 last_active_receive_fd_ = active_receive_fd_;
00460 active_receive_fd_ = -1;
00461
00462 TLOG(9) << GetTraceName() << ": receiveFragmentData: Returning " << ret_rank;
00463 return ret_rank;
00464 }
00465
00466 bool artdaq::TCPSocketTransfer::isRunning()
00467 {
00468 switch (role())
00469 {
00470 case TransferInterface::Role::kSend:
00471 return send_fd_ != -1;
00472 case TransferInterface::Role::kReceive:
00473 TLOG(TLVL_DEBUG) << GetTraceName() << ": isRunning: There are " << getConnectedFDCount(source_rank()) << " fds connected.";
00474 return getConnectedFDCount(source_rank()) > 0;
00475 }
00476 return false;
00477 }
00478
00479
00480
00481 artdaq::TransferInterface::CopyStatus artdaq::TCPSocketTransfer::sendFragment_(Fragment&& frag, size_t send_timeout_usec)
00482 {
00483 TLOG(12) << GetTraceName() << ": sendFragment begin";
00484 artdaq::Fragment grab_ownership_frag = std::move(frag);
00485
00486 reconnect_();
00487
00488
00489 iovec iov = { reinterpret_cast<void*>(grab_ownership_frag.headerAddress()),
00490 detail::RawFragmentHeader::num_words() * sizeof(RawDataType) };
00491
00492 auto sts = sendData_(&iov, 1, send_retry_timeout_us_);
00493 auto start_time = std::chrono::steady_clock::now();
00494
00495 while (sts != CopyStatus::kSuccess && (send_timeout_usec == 0 || TimeUtils::GetElapsedTimeMicroseconds(start_time) < send_timeout_usec) && TimeUtils::GetElapsedTimeMicroseconds(start_time) < 10000000)
00496 {
00497 TLOG(13) << GetTraceName() << ": sendFragment: Timeout or Error sending fragment";
00498 sts = sendData_(&iov, 1, send_retry_timeout_us_);
00499 usleep(1000);
00500 }
00501 if (sts != CopyStatus::kSuccess) return sts;
00502
00503
00504
00505 iov = { reinterpret_cast<void*>(grab_ownership_frag.headerAddress() + detail::RawFragmentHeader::num_words()),
00506 grab_ownership_frag.sizeBytes() - detail::RawFragmentHeader::num_words() * sizeof(RawDataType) };
00507 sts = sendData_(&iov, 1, send_retry_timeout_us_);
00508 start_time = std::chrono::steady_clock::now();
00509 while (sts != CopyStatus::kSuccess && (send_timeout_usec == 0 || TimeUtils::GetElapsedTimeMicroseconds(start_time) < send_timeout_usec) && TimeUtils::GetElapsedTimeMicroseconds(start_time) < 10000000)
00510 {
00511 TLOG(13) << GetTraceName() << ": sendFragment: Timeout or Error sending fragment";
00512 sts = sendData_(&iov, 1, send_retry_timeout_us_);
00513 usleep(1000);
00514 }
00515
00516 #if USE_ACKS
00517 receive_ack_(send_fd_);
00518 #endif
00519
00520 TLOG(12) << GetTraceName() << ": sendFragment returning kSuccess";
00521 return sts;
00522 }
00523
00524 artdaq::TransferInterface::CopyStatus artdaq::TCPSocketTransfer::sendData_(const void* buf, size_t bytes, size_t send_timeout_usec)
00525 {
00526 TLOG(TLVL_DEBUG) << GetTraceName() << ": sendData_ Converting buf to iovec";
00527 iovec iov = { (void*)buf, bytes };
00528 return sendData_(&iov, 1, send_timeout_usec);
00529 }
00530
00531 artdaq::TransferInterface::CopyStatus artdaq::TCPSocketTransfer::sendData_(const struct iovec* iov, int iovcnt, size_t send_timeout_usec)
00532 {
00533
00534 if (send_fd_ == -1)
00535 {
00536 if (timeoutMessageArmed_)
00537 {
00538 TLOG(TLVL_DEBUG) << GetTraceName() << ": sendData_: Send fd is not open. Returning kTimeout";
00539 timeoutMessageArmed_ = false;
00540 }
00541 return CopyStatus::kTimeout;
00542 }
00543 timeoutMessageArmed_ = true;
00544 TLOG(14) << GetTraceName() << ": send_timeout_usec is " << send_timeout_usec << ", currently unused.";
00545
00546
00547 uint32_t total_to_write_bytes = 0;
00548 std::vector<iovec> iov_in(iovcnt + 1);
00549 std::vector<iovec> iovv(iovcnt + 2);
00550 int ii;
00551 for (ii = 0; ii < iovcnt; ++ii)
00552 {
00553 iov_in[ii + 1] = iov[ii];
00554 total_to_write_bytes += iov[ii].iov_len;
00555 }
00556
00557 MessHead mh = { 0,MessHead::data_v0,htons(source_rank()),{htonl(total_to_write_bytes)} };
00558 iov_in[0].iov_base = &mh;
00559 iov_in[0].iov_len = sizeof(mh);
00560 total_to_write_bytes += sizeof(mh);
00561
00562 ssize_t sts = 0;
00563 ssize_t total_written_bytes = 0;
00564 ssize_t per_write_max_bytes = (32 * 1024);
00565
00566 size_t in_iov_idx = 0;
00567 size_t out_iov_idx = 0;
00568 ssize_t this_write_bytes = 0;
00569
00570 do
00571 {
00572
00573
00574 for (;
00575 (in_iov_idx + out_iov_idx) < iov_in.size() && this_write_bytes < per_write_max_bytes;
00576 ++out_iov_idx)
00577 {
00578 this_write_bytes += iov_in[in_iov_idx + out_iov_idx].iov_len;
00579 iovv[out_iov_idx] = iov_in[in_iov_idx + out_iov_idx];
00580 }
00581 if (this_write_bytes > per_write_max_bytes)
00582 {
00583 iovv[out_iov_idx - 1].iov_len -= this_write_bytes - per_write_max_bytes;
00584 this_write_bytes = per_write_max_bytes;
00585 }
00586
00587
00588 do_again:
00589 #ifndef __OPTIMIZE__ // This can be an expensive TRACE call (even if disabled) due to multiplicity of calls
00590 TLOG(14) << GetTraceName() << ": sendFragment b4 writev " << std::setw(7) << total_written_bytes << " total_written_bytes send_fd_=" << send_fd_ << " in_idx=" << in_iov_idx
00591 << " iovcnt=" << out_iov_idx << " 1st.len=" << iovv[0].iov_len;
00592 #endif
00593
00594 sts = writev(send_fd_, &(iovv[0]), out_iov_idx);
00595
00596
00597 if (sts == -1)
00598 {
00599 if (errno == EAGAIN )
00600 {
00601 TLOG(TLVL_DEBUG) << GetTraceName() << ": sendFragment EWOULDBLOCK";
00602 fcntl(send_fd_, F_SETFL, 0);
00603 blocking = true;
00604
00605 goto do_again;
00606 }
00607 TLOG(TLVL_WARNING) << GetTraceName() << ": sendFragment_: WRITE ERROR: " << strerror(errno);
00608 connect_state = 0;
00609 close(send_fd_);
00610 send_fd_ = -1;
00611 return TransferInterface::CopyStatus::kErrorNotRequiringException;
00612 }
00613 else if (sts != this_write_bytes)
00614 {
00615
00616 TLOG(TLVL_DEBUG) << GetTraceName() << ": sendFragment writev sts(" << sts << ")!=requested_send_bytes(" << this_write_bytes << ")";
00617 total_written_bytes += sts;
00618
00619 for (ii = 0; (size_t)sts >= iovv[ii].iov_len; ++ii)
00620 sts -= iovv[ii].iov_len;
00621 in_iov_idx += ii;
00622 iovv[ii].iov_len -= sts;
00623 iovv[ii].iov_base = (uint8_t*)(iovv[ii].iov_base) + sts;
00624
00625
00626 out_iov_idx = 0;
00627 if (ii != 0)
00628 iovv[out_iov_idx] = iovv[ii];
00629
00630 this_write_bytes = iovv[out_iov_idx].iov_len;
00631
00632
00633
00634
00635 unsigned long additional = ((unsigned long)iov_in[in_iov_idx].iov_base + iov_in[in_iov_idx].iov_len)
00636 - ((unsigned long)iovv[out_iov_idx].iov_base + iovv[out_iov_idx].iov_len);
00637 if (additional)
00638 {
00639 iovv[out_iov_idx].iov_len += additional;
00640 this_write_bytes += additional;
00641 if (this_write_bytes > per_write_max_bytes)
00642 {
00643 iovv[out_iov_idx].iov_len -= this_write_bytes - per_write_max_bytes;
00644 this_write_bytes = per_write_max_bytes;
00645 }
00646 }
00647 ++out_iov_idx;
00648 TLOG(TLVL_TRACE) << GetTraceName() << ": sendFragment writev sts!=: this_write_bytes=" << this_write_bytes
00649 << " out_iov_idx=" << out_iov_idx
00650 << " additional=" << additional
00651 << " ii=" << ii;
00652 }
00653 else
00654 {
00655 #ifndef __OPTIMIZE__ // This can be an expensive TRACE call (even if disabled) due to multiplicity of calls
00656 TLOG(TLVL_TRACE) << GetTraceName() << ": sendFragment writev sts(" << sts << ")==requested_send_bytes(" << this_write_bytes << ")";
00657 #endif
00658 total_written_bytes += sts;
00659 --out_iov_idx;
00660 iovv[out_iov_idx].iov_base = (uint8_t*)(iovv[out_iov_idx].iov_base) + iovv[out_iov_idx].iov_len;
00661 iovv[out_iov_idx].iov_len = 0;
00662 in_iov_idx += out_iov_idx;
00663 this_write_bytes = 0;
00664
00665 unsigned long additional = ((unsigned long)iov_in[in_iov_idx].iov_base + iov_in[in_iov_idx].iov_len)
00666 - ((unsigned long)iovv[out_iov_idx].iov_base + iovv[out_iov_idx].iov_len);
00667 if (additional)
00668 {
00669 iovv[out_iov_idx].iov_len += additional;
00670 this_write_bytes += additional;
00671 if (this_write_bytes > per_write_max_bytes)
00672 {
00673 iovv[out_iov_idx].iov_len -= this_write_bytes - per_write_max_bytes;
00674 this_write_bytes = per_write_max_bytes;
00675 }
00676 if (out_iov_idx != 0)
00677 iovv[0] = iovv[out_iov_idx];
00678 out_iov_idx = 1;
00679 }
00680 else
00681 {
00682 ++in_iov_idx;
00683 out_iov_idx = 0;
00684 }
00685 }
00686 } while (total_written_bytes < total_to_write_bytes);
00687 if (total_written_bytes > total_to_write_bytes)
00688 TLOG(TLVL_ERROR) << GetTraceName() << ": sendFragment program error: too many bytes transferred";
00689
00690 if (blocking)
00691 {
00692 blocking = false;
00693 fcntl(send_fd_, F_SETFL, O_NONBLOCK);
00694 }
00695 sts = total_written_bytes - sizeof(MessHead);
00696
00697 TLOG(14) << GetTraceName() << ": sendFragment sts=" << sts;
00698 return TransferInterface::CopyStatus::kSuccess;
00699 }
00700
00701 void artdaq::TCPSocketTransfer::connect_()
00702 {
00703 auto start_time = std::chrono::steady_clock::now();
00704
00705
00706 while (send_fd_ == -1 && TimeUtils::GetElapsedTimeMicroseconds(start_time) < send_retry_timeout_us_ * 10)
00707 {
00708 TLOG(TLVL_DEBUG) << GetTraceName() << ": Connecting sender socket";
00709 int sndbuf_bytes = static_cast<int>(sndbuf_);
00710 send_fd_ = TCPConnect(hostMap_[destination_rank()].hostname.c_str()
00711 , calculate_port_()
00712 , O_NONBLOCK
00713 , sndbuf_bytes);
00714 if (send_fd_ == -1)
00715 usleep(send_retry_timeout_us_);
00716 }
00717 connect_state = 0;
00718 blocking = 0;
00719 TLOG(TLVL_DEBUG) << GetTraceName() << ": connect_ " + hostMap_[destination_rank()].hostname + ":" << calculate_port_() << " send_fd_=" << send_fd_;
00720 if (send_fd_ != -1)
00721 {
00722
00723 TLOG(TLVL_DEBUG) << GetTraceName() << ": connect_: Writing connect message";
00724 MessHead mh = { 0,MessHead::connect_v0,htons(source_rank()),{htonl(CONN_MAGIC)} };
00725 ssize_t sts = write(send_fd_, &mh, sizeof(mh));
00726 if (sts == -1)
00727 {
00728 TLOG(TLVL_ERROR) << GetTraceName() << ": connect_: Error writing connect message!";
00729
00730 connect_state = 0;
00731 close(send_fd_);
00732 send_fd_ = -1;
00733 }
00734 else
00735 {
00736 TLOG(TLVL_INFO) << GetTraceName() << ": connect_: Successfully connected";
00737
00738 connect_state = 1;
00739 }
00740 }
00741 }
00742
00743 void artdaq::TCPSocketTransfer::reconnect_()
00744 {
00745 if (send_fd_ == -1 && role() == TransferInterface::Role::kSend)
00746 {
00747 TLOG(TLVL_TRACE) << GetTraceName() << ": check/reconnect";
00748 return connect_();
00749 }
00750 }
00751
00752 void artdaq::TCPSocketTransfer::start_listen_thread_()
00753 {
00754 std::unique_lock<std::mutex> start_lock(listen_thread_mutex_);
00755 if (listen_thread_refcount_ == 0)
00756 {
00757 if (listen_thread_ && listen_thread_->joinable()) listen_thread_->join();
00758 listen_thread_refcount_ = 1;
00759 TLOG(TLVL_INFO) << GetTraceName() << ": Starting Listener Thread";
00760 listen_thread_ = std::make_unique<boost::thread>(&TCPSocketTransfer::listen_, calculate_port_(), rcvbuf_);
00761 }
00762 else
00763 {
00764 listen_thread_refcount_++;
00765 }
00766 }
00767
00768 #if USE_ACKS
00769 void artdaq::TCPSocketTransfer::receive_ack_(int fd)
00770 {
00771 MessHead mh;
00772 uint64_t mark_us = TimeUtils::gettimeofday_us();
00773 fcntl(send_fd_, F_SETFL, 0);
00774 auto sts = read(fd, &mh, sizeof(mh));
00775 fcntl(send_fd_, F_SETFL, O_NONBLOCK);
00776 uint64_t delta_us = TimeUtils::gettimeofday_us() - mark_us;
00777 TLOG(17) << GetTraceName() << ": receive_ack_: Read of ack message took " << delta_us << " microseconds.";
00778 if (sts != sizeof(mh))
00779 {
00780 TLOG(TLVL_ERROR) << GetTraceName() << ": receive_ack_: Wrong message header length received! (actual " << sts << " != " << sizeof(mh) << " expected)";
00781 close(fd);
00782 send_fd_ = -1;
00783 return;
00784 }
00785
00786
00787 mh.source_id = ntohs(mh.source_id);
00788 if (mh.source_id != my_rank)
00789 {
00790 TLOG(TLVL_ERROR) << GetTraceName() << ": receive_ack_: Received ack for different sender! Rank=" << my_rank << ", hdr=" << mh.source_id;
00791 close(fd);
00792 send_fd_ = -1;
00793 return;
00794 }
00795 if (ntohl(mh.conn_magic) != ACK_MAGIC || !(mh.message_type == MessHead::ack_v0))
00796 {
00797 TLOG(TLVL_ERROR) << GetTraceName() << ": receive_ack_: Wrong magic bytes in header!";
00798 close(fd);
00799 send_fd_ = -1;
00800 return;
00801 }
00802 }
00803
00804 void artdaq::TCPSocketTransfer::send_ack_(int fd)
00805 {
00806 MessHead mh = { 0,MessHead::ack_v0,htons(source_rank()),{ htonl(ACK_MAGIC) } };
00807 write(fd, &mh, sizeof(mh));
00808 }
00809 #endif
00810
00811 void artdaq::TCPSocketTransfer::listen_(int port, size_t rcvbuf)
00812 {
00813 int listen_fd = -1;
00814 while (listen_thread_refcount_ > 0)
00815 {
00816 TLOG(TLVL_TRACE) << "listen_: Listening/accepting new connections";
00817 if (listen_fd == -1)
00818 {
00819 TLOG(TLVL_DEBUG) << "listen_: Opening listener";
00820 listen_fd = TCP_listen_fd(port, rcvbuf);
00821 }
00822 if (listen_fd == -1)
00823 {
00824 TLOG(TLVL_DEBUG) << "listen_: Error creating listen_fd!";
00825 break;
00826 }
00827
00828 int res;
00829 timeval tv = { 2,0 };
00830 fd_set rfds;
00831 FD_ZERO(&rfds);
00832 FD_SET(listen_fd, &rfds);
00833
00834 res = select(listen_fd + 1, &rfds, (fd_set *)0, (fd_set *)0, &tv);
00835 if (res > 0)
00836 {
00837 int sts;
00838 sockaddr_un un;
00839 socklen_t arglen = sizeof(un);
00840 int fd;
00841 TLOG(TLVL_DEBUG) << "listen_: Calling accept";
00842 fd = accept(listen_fd, (sockaddr *)&un, &arglen);
00843 TLOG(TLVL_DEBUG) << "listen_: Done with accept";
00844
00845 TLOG(TLVL_DEBUG) << "listen_: Reading connect message";
00846 socklen_t lenlen = sizeof(tv);
00847
00848 setsockopt(fd, SOL_SOCKET, SO_RCVTIMEO, &tv, lenlen);
00849 MessHead mh;
00850 uint64_t mark_us = TimeUtils::gettimeofday_us();
00851 sts = read(fd, &mh, sizeof(mh));
00852 uint64_t delta_us = TimeUtils::gettimeofday_us() - mark_us;
00853 TLOG(TLVL_DEBUG) << "listen_: Read of connect message took " << delta_us << " microseconds.";
00854 if (sts != sizeof(mh))
00855 {
00856 TLOG(TLVL_DEBUG) << "listen_: Wrong message header length received!";
00857 close(fd);
00858 continue;
00859 }
00860
00861
00862 mh.source_id = ntohs(mh.source_id);
00863 if (ntohl(mh.conn_magic) != CONN_MAGIC || !(mh.message_type == MessHead::connect_v0))
00864 {
00865 TLOG(TLVL_DEBUG) << "listen_: Wrong magic bytes in header!";
00866 close(fd);
00867 continue;
00868 }
00869
00870
00871 std::unique_lock<std::mutex> lk(connected_fd_mutex_);
00872 connected_fds_[mh.source_id].insert(fd);
00873
00874 TLOG(TLVL_INFO) << "listen_: New fd is " << fd << " for source rank " << mh.source_id;
00875 }
00876 else
00877 {
00878 TLOG(16) << "listen_: No connections in timeout interval!";
00879 }
00880 }
00881
00882 TLOG(TLVL_INFO) << "listen_: Shutting down connection listener";
00883 if (listen_fd != -1) close(listen_fd);
00884 std::unique_lock<std::mutex> lk(connected_fd_mutex_);
00885 auto it = connected_fds_.begin();
00886 while (it != connected_fds_.end())
00887 {
00888 auto& fd_set = it->second;
00889 auto rank_it = fd_set.begin();
00890 while (rank_it != fd_set.end())
00891 {
00892 close(*rank_it);
00893 rank_it = fd_set.erase(rank_it);
00894 }
00895 it = connected_fds_.erase(it);
00896 }
00897
00898 }
00899
00900 DEFINE_ARTDAQ_TRANSFER(artdaq::TCPSocketTransfer)