00001
00002
00003
00004
00005
00006
00007
00008 #include <stdlib.h>
00009 #include <sys/socket.h>
00010 #include <sys/un.h>
00011 #include <arpa/inet.h>
00012 #include <sys/types.h>
00013 #include <poll.h>
00014
00015
00016 #include <string>
00017 #include <fstream>
00018 #include <stdexcept>
00019
00020
00021 #define TRACE_NAME "TCPSocketTransfer"
00022 #include "artdaq/DAQdata/Globals.hh"
00023
00024
00025 #include "artdaq/TransferPlugins/TCPSocketTransfer.hh"
00026 #include "artdaq/DAQdata/TCP_listen_fd.hh"
00027 #include "artdaq/DAQdata/TCPConnect.hh"
00028 #include "artdaq/TransferPlugins/detail/Timeout.hh"
00029 #include "artdaq/TransferPlugins/detail/SRSockets.hh"
00030 #include "artdaq-core/Data/Fragment.hh"
00031 #include "artdaq-core/Utilities/TimeUtils.hh"
00032 #include <iomanip>
00033
00034 artdaq::TCPSocketTransfer::
00035 TCPSocketTransfer(fhicl::ParameterSet const& pset, TransferInterface::Role role)
00036 : TransferInterface(pset, role)
00037 , fd_(-1)
00038 , listen_fd_(-1)
00039 , state_(SocketState::Metadata)
00040 , offset(0)
00041 , target_bytes(sizeof(MessHead))
00042 , rcvbuf_(pset.get<size_t>("tcp_receive_buffer_size", 0))
00043 , sndbuf_(max_fragment_size_words_ * sizeof(artdaq::RawDataType) * buffer_count_)
00044 , stats_connect_stop_(false)
00045 , stats_connect_thread_(std::bind(&TCPSocketTransfer::stats_connect_, this))
00046 , timeoutMessageArmed_(true)
00047 {
00048 TLOG_DEBUG("TCPSocketTransfer") << uniqueLabel() << " TCPSocketTransfer Constructor: pset=" << pset.to_string() << ", role=" << (role == TransferInterface::Role::kReceive ? "kReceive" : "kSend") << TLOG_ENDL;
00049 auto masterPortOffset = pset.get<int>("offset_all_ports",0);
00050 auto hosts = pset.get<std::vector<fhicl::ParameterSet>>("host_map");
00051 for (auto& ps : hosts)
00052 {
00053 auto rank = ps.get<size_t>("rank", RECV_TIMEOUT);
00054 DestinationInfo info;
00055 info.hostname = ps.get<std::string>("host", "localhost");
00056 info.portOffset = ps.get<int>("portOffset", 5500) + masterPortOffset;
00057
00058 hostMap_[rank] = info;
00059 }
00060
00061 std::function<void()> function = std::bind(&TCPSocketTransfer::reconnect_, this);
00062 tmo_.add_periodic("reconnect", NULL, function, 200);
00063
00064 if (role == TransferInterface::Role::kReceive)
00065 {
00066
00067 TLOG_DEBUG("TCPSocketTransfer") << uniqueLabel() << " Listening for connections" << TLOG_ENDL;
00068 listen_();
00069 TLOG_DEBUG("TCPSocketTransfer") << uniqueLabel() << " Done Listening" << TLOG_ENDL;
00070 }
00071 else
00072 {
00073 TLOG_DEBUG("TCPSocketTransfer") << uniqueLabel() << " Connecting to destination" << TLOG_ENDL;
00074 connect_();
00075 TLOG_DEBUG("TCPSocketTransfer") << uniqueLabel() << " Done Connecting" << TLOG_ENDL;
00076 }
00077 TLOG_DEBUG("TCPSocketTransfer") << uniqueLabel() << " End of TCPSocketTransfer Constructor" << TLOG_ENDL;
00078 }
00079
00080 artdaq::TCPSocketTransfer::~TCPSocketTransfer()
00081 {
00082 TLOG_DEBUG("TCPSocketTransfer") << uniqueLabel() << " Shutting down TCPSocketTransfer" << TLOG_ENDL;
00083 stats_connect_stop_ = true;
00084 stopstatscv_.notify_all();
00085 stats_connect_thread_.join();
00086
00087 if (role() == TransferInterface::Role::kSend)
00088 {
00089
00090 MessHead mh = { 0,MessHead::stop_v0,htons(TransferInterface::source_rank()),0 };
00091 if (fd_ != -1)
00092 {
00093
00094 timeval tv = { 0,100000 };
00095 socklen_t len = sizeof(tv);
00096 setsockopt(fd_, SOL_SOCKET, SO_SNDTIMEO, &tv, len);
00097 write(fd_, &mh, sizeof(mh));
00098 }
00099 }
00100 close(fd_);
00101 TLOG_DEBUG("TCPSocketTransfer") << uniqueLabel() << " End of TCPSocketTransfer Destructor" << TLOG_ENDL;
00102 }
00103
00104 int artdaq::TCPSocketTransfer::receiveFragmentHeader(detail::RawFragmentHeader& header, size_t timeout_usec)
00105 {
00106 TLOG_ARB(7, "TCPSocketTransfer") << uniqueLabel() << " TCPSocketTransfer::receiveFragment: BEGIN" << TLOG_ENDL;
00107 int ret_rank = RECV_TIMEOUT;
00108 if (fd_ == -1)
00109 {
00110 TLOG_DEBUG("TCPSocketTransfer") << uniqueLabel() << " TCPSocketTransfer::receiveFragment: Receive socket not connected, returning RECV_TIMEOUT" << TLOG_ENDL;
00111 return RECV_TIMEOUT;
00112 }
00113
00114 TLOG_ARB(7, "TCPSocketTransfer") << uniqueLabel() << " TCPSocketTransfer::recvFragment timeout_usec=" << std::to_string(timeout_usec) << TLOG_ENDL;
00115
00116 size_t byte_cnt = 0;
00117 int sts;
00118 uint64_t start_time_us = TimeUtils::gettimeofday_us();
00119
00120 pollfd pollfd_s;
00121 pollfd_s.events = POLLIN | POLLERR;
00122 pollfd_s.fd = fd_;
00123 uint8_t* buff;
00124
00125 int timeout_ms;
00126 if (timeout_usec == 0)
00127 timeout_ms = 0;
00128 else
00129 timeout_ms = (timeout_usec + 999) / 1000;
00130
00131 bool done = false;
00132 while (!done)
00133 {
00134
00135 int num_fds_ready = poll(&pollfd_s, 1, timeout_ms);
00136 if (num_fds_ready <= 0)
00137 {
00138 if (num_fds_ready == 0 && timeout_ms > 0)
00139 {
00140 TLOG_ARB(7, "TCPSocketTransfer") << uniqueLabel() << " TCPSocketTransfer::receiveFragment: No data on receive socket, returning RECV_TIMEOUT" << TLOG_ENDL;
00141 return RECV_TIMEOUT;
00142 }
00143 break;
00144 }
00145
00146 if (!(pollfd_s.revents & (POLLIN | POLLHUP | POLLERR)))
00147 {
00148 TLOG_DEBUG("TCPSocketTransfer") << uniqueLabel() << " TCPSocketTransfer::receiveFragment: Wrong event received from pollfd: " << static_cast<int>(pollfd_s.revents) << TLOG_ENDL;
00149 continue;
00150 }
00151
00152 if (state_ == SocketState::Metadata)
00153 {
00154
00155 buff = &(mha[offset]);
00156 byte_cnt = sizeof(MessHead) - offset;
00157 }
00158 else
00159 {
00160
00161 buff = reinterpret_cast<uint8_t*>(&header) + offset;
00162 byte_cnt = mh.byte_count - offset;
00163 }
00164
00165
00166 sts = read(fd_, buff, byte_cnt);
00167
00168
00169 TLOG_ARB(9, "TCPSocketTransfer") << uniqueLabel() << " TCPSocketTransfer::recvFragment state=" << static_cast<int>(state_) << " read=" << sts << " (errno=" << strerror(errno) << ")" << TLOG_ENDL;
00170 if (sts <= 0)
00171 {
00172 TLOG_DEBUG("TCPSocketTransfer") << uniqueLabel() << " TCPSocketTransfer::receiveFragment: Error on receive, closing socket" << TLOG_ENDL;
00173 close(fd_);
00174 }
00175 else
00176 {
00177
00178 sts = offset += sts;
00179 if (sts == target_bytes)
00180 {
00181 TLOG_ARB(7, "TCPSocketTransfer") << uniqueLabel() << " TCPSocketTransfer::receiveFragment: Target read bytes reached. Changing state" << TLOG_ENDL;
00182 offset = 0;
00183 if (state_ == SocketState::Metadata)
00184 {
00185 state_ = SocketState::Data;
00186 mh.byte_count = ntohl(mh.byte_count);
00187 mh.source_id = ntohs(mh.source_id);
00188 target_bytes = mh.byte_count;
00189 }
00190 else
00191 {
00192 state_ = SocketState::Metadata;
00193 target_bytes = sizeof(MessHead);
00194 ret_rank = source_rank();
00195 TLOG_ARB(9, "TCPSocketTransfer") << uniqueLabel() << " TCPSocketTransfer::recvFragment done sts=" << sts << " src=" << ret_rank << TLOG_ENDL;
00196 TLOG_ARB(7, "TCPSocketTransfer") << uniqueLabel() << " TCPSocketTransfer::receiveFragment: Done receiving fragment. Moving into output." << TLOG_ENDL;
00197
00198 done = true;
00199 break;
00200 }
00201 }
00202 }
00203
00204 if (!done && timeout_usec > 0)
00205 {
00206
00207 size_t delta_us = TimeUtils::gettimeofday_us() - start_time_us;
00208 if (delta_us > timeout_usec)
00209 return RECV_TIMEOUT;
00210 timeout_ms = ((timeout_usec - delta_us) + 999) / 1000;
00211 }
00212 }
00213
00214 TLOG_ARB(7, "TCPSocketTransfer") << uniqueLabel() << " TCPSocketTransfer::receiveFragment: Returning " << ret_rank << TLOG_ENDL;
00215 return ret_rank;
00216 }
00217
00218 int artdaq::TCPSocketTransfer::receiveFragmentData(RawDataType* destination, size_t)
00219 {
00220 TLOG_ARB(7, "TCPSocketTransfer") << uniqueLabel() << " TCPSocketTransfer::receiveFragment: BEGIN" << TLOG_ENDL;
00221 int ret_rank = RECV_TIMEOUT;
00222 if (fd_ == -1)
00223 {
00224 TLOG_DEBUG("TCPSocketTransfer") << uniqueLabel() << " TCPSocketTransfer::receiveFragment: Receive socket not connected, returning RECV_TIMEOUT" << TLOG_ENDL;
00225 return RECV_TIMEOUT;
00226 }
00227
00228
00229 uint8_t* buff;
00230 size_t byte_cnt = 0;
00231 int sts;
00232
00233 pollfd pollfd_s;
00234 pollfd_s.events = POLLIN | POLLERR;
00235 pollfd_s.fd = fd_;
00236
00237 bool done = false;
00238 while (!done)
00239 {
00240
00241 int num_fds_ready = poll(&pollfd_s, 1, -1);
00242 if (num_fds_ready <= 0)
00243 {
00244 if (num_fds_ready == 0)
00245 {
00246 TLOG_ARB(7, "TCPSocketTransfer") << uniqueLabel() << " TCPSocketTransfer::receiveFragment: No data on receive socket, returning RECV_TIMEOUT" << TLOG_ENDL;
00247 return RECV_TIMEOUT;
00248 }
00249 break;
00250 }
00251
00252 if (!(pollfd_s.revents & (POLLIN | POLLERR)))
00253 {
00254 TLOG_DEBUG("TCPSocketTransfer") << uniqueLabel() << " TCPSocketTransfer::receiveFragment: Wrong event received from pollfd" << TLOG_ENDL;
00255 continue;
00256 }
00257
00258 if (state_ == SocketState::Metadata)
00259 {
00260
00261 buff = &(mha[offset]);
00262 byte_cnt = sizeof(MessHead) - offset;
00263 }
00264 else
00265 {
00266
00267 buff = reinterpret_cast<uint8_t*>(destination) + offset;
00268 byte_cnt = mh.byte_count - offset;
00269 }
00270
00271
00272 sts = read(fd_, buff, byte_cnt);
00273
00274
00275 TLOG_ARB(9, "TCPSocketTransfer") << uniqueLabel() << " recvFragment state=" << static_cast<int>(state_) << " read=" << sts << " (errno=" << strerror(errno) << ")" << TLOG_ENDL;
00276 if (sts <= 0)
00277 {
00278 TLOG_DEBUG("TCPSocketTransfer") << uniqueLabel() << " TCPSocketTransfer::receiveFragment: Error on receive, closing socket" << TLOG_ENDL;
00279 close(fd_);
00280 }
00281 else
00282 {
00283
00284 sts = offset += sts;
00285 if (sts == target_bytes)
00286 {
00287 TLOG_ARB(7, "TCPSocketTransfer") << uniqueLabel() << " TCPSocketTransfer::receiveFragment: Target read bytes reached. Changing state" << TLOG_ENDL;
00288 offset = 0;
00289 if (state_ == SocketState::Metadata)
00290 {
00291 state_ = SocketState::Data;
00292 mh.byte_count = ntohl(mh.byte_count);
00293 mh.source_id = ntohs(mh.source_id);
00294 target_bytes = mh.byte_count;
00295 }
00296 else
00297 {
00298 state_ = SocketState::Metadata;
00299 target_bytes = sizeof(MessHead);
00300 ret_rank = source_rank();
00301 TLOG_ARB(9, "TCPSocketTransfer") << uniqueLabel() << " recvFragment done sts=" << sts << " src=" << ret_rank << TLOG_ENDL;
00302 TLOG_ARB(7, "TCPSocketTransfer") << uniqueLabel() << " TCPSocketTransfer::receiveFragment: Done receiving fragment. Moving into output." << TLOG_ENDL;
00303
00304 done = true;
00305 break;
00306 }
00307 }
00308 }
00309 }
00310
00311 TLOG_ARB(7, "TCPSocketTransfer") << uniqueLabel() << " TCPSocketTransfer::receiveFragment: Returning " << ret_rank << TLOG_ENDL;
00312 return ret_rank;
00313 }
00314
00315
00316
00317 artdaq::TransferInterface::CopyStatus artdaq::TCPSocketTransfer::sendFragment_(Fragment&& frag, size_t send_timeout_usec)
00318 {
00319 TLOG_ARB(7, "TCPSocketTransfer") << uniqueLabel() << " TCPSocketTransfer::sendFragment begin" << TLOG_ENDL;
00320 artdaq::Fragment grab_ownership_frag = std::move(frag);
00321
00322
00323
00324 iovec iov = { reinterpret_cast<void*>(grab_ownership_frag.headerAddress()),
00325 detail::RawFragmentHeader::num_words() * sizeof(RawDataType) };
00326 auto sts = sendData_(&iov, 1, send_timeout_usec);
00327 while (sts != CopyStatus::kSuccess)
00328 {
00329 TLOG_ARB(7, "TCPSocketTransfer") << uniqueLabel() << " TCPSocketTransfer::sendFragment: Timeout or Error sending fragment" << TLOG_ENDL;
00330 sts = sendData_(&iov, 1, send_timeout_usec);
00331 usleep(1000);
00332 }
00333
00334
00335
00336 iov = { reinterpret_cast<void*>(grab_ownership_frag.headerAddress() + detail::RawFragmentHeader::num_words()),
00337 grab_ownership_frag.sizeBytes() - detail::RawFragmentHeader::num_words() * sizeof(RawDataType) };
00338 sts = sendData_(&iov, 1, send_timeout_usec);
00339 while (sts != CopyStatus::kSuccess)
00340 {
00341 TLOG_ARB(7, "TCPSocketTransfer") << uniqueLabel() << " TCPSocketTransfer::sendFragment: Timeout or Error sending fragment" << TLOG_ENDL;
00342 sts = sendData_(&iov, 1, send_timeout_usec);
00343 usleep(1000);
00344 }
00345
00346 TRACE(7, "TCPSocketTransfer::sendFragment returning kSuccess");
00347 return sts;
00348 }
00349
00350 artdaq::TransferInterface::CopyStatus artdaq::TCPSocketTransfer::sendData_(const void* buf, size_t bytes, size_t send_timeout_usec)
00351 {
00352 TLOG_DEBUG("TCPSocketTransfer") << uniqueLabel() << "TCPSocketTransfer::sendData_ Converting buf to iovec" << TLOG_ENDL;
00353 iovec iov = { (void*)buf, bytes };
00354 return sendData_(&iov, 1, send_timeout_usec);
00355 }
00356
00357 artdaq::TransferInterface::CopyStatus artdaq::TCPSocketTransfer::sendData_(const struct iovec* iov, int iovcnt, size_t send_timeout_usec)
00358 {
00359
00360 if (fd_ == -1)
00361 {
00362 if (timeoutMessageArmed_)
00363 {
00364 TLOG_DEBUG("TCPSocketTransfer") << uniqueLabel() << "TCPSocketTransfer::sendData_: Send fd is not open. Returning kTimeout" << TLOG_ENDL;
00365 timeoutMessageArmed_ = false;
00366 }
00367 return CopyStatus::kTimeout;
00368 }
00369 timeoutMessageArmed_ = true;
00370 TLOG_ARB(12, "TCPSocketTransfer") << uniqueLabel() << "send_timeout_usec is " << std::to_string(send_timeout_usec) << ", currently unused." << TLOG_ENDL;
00371
00372
00373 uint32_t total_to_write_bytes = 0;
00374 std::vector<iovec> iov_in(iovcnt + 1);
00375 std::vector<iovec> iovv(iovcnt + 2);
00376 int ii;
00377 for (ii = 0; ii < iovcnt; ++ii)
00378 {
00379 iov_in[ii + 1] = iov[ii];
00380 total_to_write_bytes += iov[ii].iov_len;
00381 }
00382
00383 MessHead mh = { 0,MessHead::data_v0,htons(source_rank()),htonl(total_to_write_bytes) };
00384 iov_in[0].iov_base = &mh;
00385 iov_in[0].iov_len = sizeof(mh);
00386 total_to_write_bytes += sizeof(mh);
00387
00388 ssize_t sts = 0;
00389 ssize_t total_written_bytes = 0;
00390 ssize_t per_write_max_bytes = (32 * 1024);
00391
00392 size_t in_iov_idx = 0;
00393 size_t out_iov_idx = 0;
00394 ssize_t this_write_bytes = 0;
00395
00396 do
00397 {
00398
00399
00400 for (;
00401 (in_iov_idx + out_iov_idx) < iov_in.size() && this_write_bytes < per_write_max_bytes;
00402 ++out_iov_idx)
00403 {
00404 this_write_bytes += iov_in[in_iov_idx + out_iov_idx].iov_len;
00405 iovv[out_iov_idx] = iov_in[in_iov_idx + out_iov_idx];
00406 }
00407 if (this_write_bytes > per_write_max_bytes)
00408 {
00409 iovv[out_iov_idx - 1].iov_len -= this_write_bytes - per_write_max_bytes;
00410 this_write_bytes = per_write_max_bytes;
00411 }
00412
00413
00414 do_again:
00415 TLOG_ARB(7, "TCPSocketTransfer") << uniqueLabel() << "sendFragment b4 writev " << std::setw(7) << std::to_string(total_written_bytes) << " total_written_bytes fd=" << fd_ << " in_idx=" << std::to_string(in_iov_idx)
00416 << " iovcnt=" << std::to_string(out_iov_idx) << " 1st.len=" << std::to_string(iovv[0].iov_len) << TLOG_ENDL;
00417
00418 sts = writev(fd_, &(iovv[0]), out_iov_idx);
00419
00420
00421 if (sts == -1)
00422 {
00423 if (errno == EAGAIN )
00424 {
00425 TLOG_ARB(2, "TCPSocketTransfer") << uniqueLabel() << "sendFragment EWOULDBLOCK" << TLOG_ENDL;
00426 fcntl(fd_, F_SETFL, 0);
00427 blocking = true;
00428
00429 goto do_again;
00430 }
00431 TLOG_DEBUG("TCPSocketTransfer") << uniqueLabel() << "TCPSocketTransfer::sendFragment_: WRITE ERROR: " << strerror(errno) << TLOG_ENDL;
00432 connect_state = 0;
00433 close(fd_);
00434 fd_ = -1;
00435 return TransferInterface::CopyStatus::kErrorNotRequiringException;
00436 }
00437 else if (sts != this_write_bytes)
00438 {
00439
00440 TLOG_ARB(4, "TCPSocketTransfer") << uniqueLabel() << "sendFragment writev sts(" << std::to_string(sts) << ")!=requested_send_bytes(" << std::to_string(this_write_bytes) << ")" << TLOG_ENDL;
00441 total_written_bytes += sts;
00442
00443 for (ii = 0; (size_t)sts >= iovv[ii].iov_len; ++ii)
00444 sts -= iovv[ii].iov_len;
00445 in_iov_idx += ii;
00446 iovv[ii].iov_len -= sts;
00447 iovv[ii].iov_base = (uint8_t*)(iovv[ii].iov_base) + sts;
00448
00449
00450 out_iov_idx = 0;
00451 if (ii != 0)
00452 iovv[out_iov_idx] = iovv[ii];
00453
00454 this_write_bytes = iovv[out_iov_idx].iov_len;
00455
00456
00457
00458
00459 unsigned long additional = ((unsigned long)iov_in[in_iov_idx].iov_base + iov_in[in_iov_idx].iov_len)
00460 - ((unsigned long)iovv[out_iov_idx].iov_base + iovv[out_iov_idx].iov_len);
00461 if (additional)
00462 {
00463 iovv[out_iov_idx].iov_len += additional;
00464 this_write_bytes += additional;
00465 if (this_write_bytes > per_write_max_bytes)
00466 {
00467 iovv[out_iov_idx].iov_len -= this_write_bytes - per_write_max_bytes;
00468 this_write_bytes = per_write_max_bytes;
00469 }
00470 }
00471 ++out_iov_idx;
00472 TLOG_ARB(4, "TCPSocketTransfer") << uniqueLabel() << "sendFragment writev sts!=: this_write_bytes=" << std::to_string(this_write_bytes)
00473 << " out_iov_idx=" << std::to_string(out_iov_idx)
00474 << " additional=" << std::to_string(additional)
00475 << " ii=" << ii << TLOG_ENDL;
00476 }
00477 else
00478 {
00479 TLOG_ARB(4, "TCPSocketTransfer") << uniqueLabel() << "sendFragment writev sts(" << std::to_string(sts) << ")==requested_send_bytes(" << std::to_string(this_write_bytes) << ")" << TLOG_ENDL;
00480 total_written_bytes += sts;
00481 --out_iov_idx;
00482 iovv[out_iov_idx].iov_base = (uint8_t*)(iovv[out_iov_idx].iov_base) + iovv[out_iov_idx].iov_len;
00483 iovv[out_iov_idx].iov_len = 0;
00484 in_iov_idx += out_iov_idx;
00485 this_write_bytes = 0;
00486
00487 unsigned long additional = ((unsigned long)iov_in[in_iov_idx].iov_base + iov_in[in_iov_idx].iov_len)
00488 - ((unsigned long)iovv[out_iov_idx].iov_base + iovv[out_iov_idx].iov_len);
00489 if (additional)
00490 {
00491 iovv[out_iov_idx].iov_len += additional;
00492 this_write_bytes += additional;
00493 if (this_write_bytes > per_write_max_bytes)
00494 {
00495 iovv[out_iov_idx].iov_len -= this_write_bytes - per_write_max_bytes;
00496 this_write_bytes = per_write_max_bytes;
00497 }
00498 if (out_iov_idx != 0)
00499 iovv[0] = iovv[out_iov_idx];
00500 out_iov_idx = 1;
00501 }
00502 else
00503 {
00504 ++in_iov_idx;
00505 out_iov_idx = 0;
00506 }
00507 }
00508 } while (total_written_bytes < total_to_write_bytes);
00509 if (total_written_bytes > total_to_write_bytes)
00510 TLOG_ARB(0, "TCPSocketTransfer") << uniqueLabel() << "sendFragment program error: too many bytes transferred" << TLOG_ENDL;
00511
00512 if (blocking)
00513 {
00514 blocking = false;
00515 fcntl(fd_, F_SETFL, 0);
00516 }
00517 sts = total_written_bytes - sizeof(MessHead);
00518
00519 TLOG_ARB(10, "TCPSocketTransfer") << uniqueLabel() << "sendFragment sts=" << std::to_string(sts) << TLOG_ENDL;
00520 return TransferInterface::CopyStatus::kSuccess;
00521 }
00522
00523
00524
00525 void artdaq::TCPSocketTransfer::stats_connect_()
00526 {
00527 std::cv_status sts;
00528 while (!stats_connect_stop_)
00529 {
00530 std::string desc;
00531 void* tag;
00532 std::function<void()> function;
00533 uint64_t ts_us;
00534
00535 int msdly = tmo_.get_next_timeout_msdly();
00536
00537 if (msdly <= 0)
00538 msdly = 2000;
00539
00540 std::unique_lock<std::mutex> lck(stopstatscvm_);
00541 sts = stopstatscv_.wait_until(lck
00542 , std::chrono::system_clock::now()
00543 + std::chrono::milliseconds(msdly));
00544 TLOG_ARB(5, "TCPSocketTransfer") << uniqueLabel() << "thread1 after wait_until(msdly=" << msdly << ") - sts=" << static_cast<int>(sts) << TLOG_ENDL;
00545
00546 if (sts == std::cv_status::no_timeout)
00547 break;
00548
00549 auto sts = tmo_.get_next_expired_timeout(desc, &tag, function, &ts_us);
00550
00551 while (sts != -1 && desc != "")
00552 {
00553 if (function != NULL)
00554 function();
00555
00556 sts = tmo_.get_next_expired_timeout(desc, &tag, function, &ts_us);
00557 }
00558 }
00559 }
00560
00561 void artdaq::TCPSocketTransfer::connect_()
00562 {
00563 TLOG_DEBUG("TCPSocketTransfer") << uniqueLabel() << "Connecting sender socket" << TLOG_ENDL;
00564 int sndbuf_bytes = static_cast<int>(sndbuf_);
00565 fd_ = TCPConnect(hostMap_[destination_rank()].hostname.c_str()
00566 , calculate_port_()
00567 , O_NONBLOCK
00568 , sndbuf_bytes);
00569 connect_state = 0;
00570 blocking = 0;
00571 TLOG_DEBUG("TCPSocketTransfer") << uniqueLabel() << "TCPSocketTransfer::connect_ " + hostMap_[destination_rank()].hostname + ":" << calculate_port_() << " fd_=" << fd_ << TLOG_ENDL;
00572 if (fd_ != -1)
00573 {
00574
00575 TLOG_DEBUG("TCPSocketTransfer") << uniqueLabel() << "TCPSocketTransfer::connect_: Writing connect message" << TLOG_ENDL;
00576 MessHead mh = { 0,MessHead::connect_v0,htons(source_rank()),htonl(CONN_MAGIC) };
00577 ssize_t sts = write(fd_, &mh, sizeof(mh));
00578 if (sts == -1)
00579 {
00580 TLOG_ERROR("TCPSocketTransfer") << uniqueLabel() << "TCPSocketTransfer::connect_: Error writing connect message!" << TLOG_ENDL;
00581
00582 connect_state = 0;
00583 close(fd_);
00584 fd_ = -1;
00585 }
00586 else
00587 {
00588 TLOG_INFO("TCPSocketTransfer") << uniqueLabel() << "TCPSocketTransfer::connect_: Successfully connected" << TLOG_ENDL;
00589
00590 connect_state = 1;
00591 }
00592 }
00593 }
00594
00595 void artdaq::TCPSocketTransfer::reconnect_()
00596 {
00597 TLOG_ARB(5, "TCPSocketTransfer") << uniqueLabel() << "check/reconnect" << TLOG_ENDL;
00598 if (fd_ == -1 && role() == TransferInterface::Role::kSend) return connect_();
00599 if ((fd_ == -1 || listen_fd_ == -1) && role() == TransferInterface::Role::kReceive) return listen_();
00600 }
00601
00602 void artdaq::TCPSocketTransfer::listen_()
00603 {
00604 TLOG_DEBUG("TCPSocketTransfer") << uniqueLabel() << "TCPSocketTransfer::listen_: Listening/accepting new connections" << TLOG_ENDL;
00605 if (listen_fd_ == -1)
00606 {
00607 TLOG_DEBUG("TCPSocketTransfer") << uniqueLabel() << "TCPSocketTransfer::listen_: Opening listener" << TLOG_ENDL;
00608 listen_fd_ = TCP_listen_fd(calculate_port_(), rcvbuf_);
00609 }
00610 if (listen_fd_ == -1)
00611 {
00612 TLOG_DEBUG("TCPSocketTransfer") << uniqueLabel() << "TCPSocketTransfer::listen_: Error creating listen_fd_!" << TLOG_ENDL;
00613 return;
00614 }
00615
00616 int res;
00617 timeval tv = { 2,0 };
00618 fd_set rfds;
00619 FD_ZERO(&rfds);
00620 FD_SET(listen_fd_, &rfds);
00621
00622 res = select(listen_fd_ + 1, &rfds, (fd_set *)0, (fd_set *)0, &tv);
00623 if (res > 0)
00624 {
00625 int sts;
00626 sockaddr_un un;
00627 socklen_t arglen = sizeof(un);
00628 int fd;
00629 TLOG_DEBUG("TCPSocketTransfer") << uniqueLabel() << "Calling accept" << TLOG_ENDL;
00630 fd = accept(listen_fd_, (sockaddr *)&un, &arglen);
00631 TLOG_DEBUG("TCPSocketTransfer") << uniqueLabel() << "Done with accept" << TLOG_ENDL;
00632
00633 TLOG_DEBUG("TCPSocketTransfer") << uniqueLabel() << "TCPSocketTransfer::listen_: Reading connect message" << TLOG_ENDL;
00634 socklen_t lenlen = sizeof(tv);
00635
00636 setsockopt(fd, SOL_SOCKET, SO_RCVTIMEO, &tv, lenlen);
00637 MessHead mh;
00638 uint64_t mark_us = TimeUtils::gettimeofday_us();
00639 sts = read(fd, &mh, sizeof(mh));
00640 uint64_t delta_us = TimeUtils::gettimeofday_us() - mark_us;
00641 TLOG_DEBUG("TCPSocketTransfer") << uniqueLabel() << "TCPSocketTransfer::listen_: Read of connect message took " << delta_us << " microseconds." << TLOG_ENDL;
00642 TLOG_ARB(10, "TCPSocketTransfer") << uniqueLabel() << "do_connect read of connect msg (after accept) took " << std::to_string(delta_us) << " microseconds" << TLOG_ENDL;
00643 if (sts != sizeof(mh))
00644 {
00645 TLOG_DEBUG("TCPSocketTransfer") << uniqueLabel() << "TCPSocketTransfer::listen_: Wrong message header length received!" << TLOG_ENDL;
00646 TLOG_ARB(0, "TCPSocketTransfer") << uniqueLabel() << "do_connect_ problem with connect msg sts(" << sts << ")!=sizeof(mh)("<<std::to_string(sizeof(mh)) << ")" << TLOG_ENDL;
00647 close(fd);
00648 return;
00649 }
00650
00651
00652 mh.source_id = ntohs(mh.source_id);
00653 if (ntohl(mh.conn_magic) != CONN_MAGIC || mh.source_id != source_rank())
00654 {
00655 TLOG_DEBUG("TCPSocketTransfer") << uniqueLabel() << "TCPSocketTransfer::listen_: Wrong magic bytes in header!" << TLOG_ENDL;
00656 close(fd);
00657 return;
00658 }
00659 if (fd_ != -1)
00660 {
00661
00662 close(fd_);
00663 }
00664
00665
00666 fd_ = fd;
00667 TLOG_INFO("TCPSocketTransfer") << uniqueLabel() << "TCPSocketTransfer::listen_: New fd is " << fd_ << TLOG_ENDL;
00668
00669 TLOG_ARB(3, "TCPSocketTransfer") << uniqueLabel() << "do_connect_ connection from sender_rank=" << std::to_string(mh.source_id) << TLOG_ENDL;
00670 }
00671 else
00672 {
00673 TLOG_ARB(10, "TCPSocketTransfer") << uniqueLabel() << "TCPSocketTransfer::do_connect_: No connections in timeout interval!" << TLOG_ENDL;
00674 }
00675 }
00676
00677 DEFINE_ARTDAQ_TRANSFER(artdaq::TCPSocketTransfer)