00001
00002
00003
00004
00005
00006
00007
00008
00009 #include <stdlib.h>
00010 #include <sys/socket.h>
00011 #include <sys/un.h>
00012 #include <arpa/inet.h>
00013 #include <assert.h>
00014
00015
00016 #include <string>
00017 #include <fstream>
00018 #include <stdexcept>
00019
00020
00021 #define TRACE_NAME "TCPSocketTransfer"
00022 #include "artdaq/DAQdata/Globals.hh"
00023
00024
00025 #include "artdaq/TransferPlugins/TCPSocketTransfer.hh"
00026 #include "artdaq/DAQdata/TCP_listen_fd.hh"
00027 #include "artdaq/DAQdata/TCPConnect.hh"
00028 #include "artdaq/TransferPlugins/detail/Timeout.hh"
00029 #include "artdaq/TransferPlugins/detail/SRSockets.hh"
00030 #include "artdaq-core/Data/Fragment.hh"
00031
00032
00033 artdaq::TCPSocketTransfer::
00034 TCPSocketTransfer(fhicl::ParameterSet const& pset, TransferInterface::Role role)
00035 : TransferInterface(pset, role)
00036 , fd_(-1)
00037 , listen_fd_(-1)
00038 , state_(SocketState::Metadata)
00039 , frag(max_fragment_size_words_)
00040 , buffer(frag.headerBeginBytes())
00041 , offset(0)
00042 , target_bytes(sizeof(MessHead))
00043 , rcvbuf_(pset.get<size_t>("tcp_receive_buffer_size", 0))
00044 , sndbuf_(max_fragment_size_words_ * sizeof(artdaq::RawDataType) * buffer_count_)
00045 , stats_connect_stop_(false)
00046 , stats_connect_thread_(std::bind(&TCPSocketTransfer::stats_connect_, this))
00047 , timeoutMessageArmed_(true)
00048 {
00049 TLOG_DEBUG(uniqueLabel()) << "TCPSocketTransfer Constructor: pset=" << pset.to_string() << ", role=" << (role == TransferInterface::Role::kReceive ? "kReceive" : "kSend") << TLOG_ENDL;
00050 auto hosts = pset.get<std::vector<fhicl::ParameterSet>>("host_map");
00051 for (auto& ps : hosts)
00052 {
00053 auto rank = ps.get<size_t>("rank", RECV_TIMEOUT);
00054 DestinationInfo info;
00055 info.hostname = ps.get<std::string>("host", "localhost");
00056 info.portOffset = ps.get<int>("portOffset", 5500);
00057
00058 hostMap_[rank] = info;
00059 }
00060
00061 std::function<void()> function = std::bind(&TCPSocketTransfer::reconnect_, this);
00062 tmo_.add_periodic("reconnect", NULL, function, 200);
00063
00064 if (role == TransferInterface::Role::kReceive)
00065 {
00066
00067 TLOG_DEBUG(uniqueLabel()) << "Listening for connections" << TLOG_ENDL;
00068 listen_();
00069 TLOG_DEBUG(uniqueLabel()) << "Done Listening" << TLOG_ENDL;
00070 }
00071 else
00072 {
00073 TLOG_DEBUG(uniqueLabel()) << "Connecting to destination" << TLOG_ENDL;
00074 connect_();
00075 TLOG_DEBUG(uniqueLabel()) << "Done Connecting" << TLOG_ENDL;
00076 }
00077 TLOG_DEBUG(uniqueLabel()) << "End of TCPSocketTransfer Constructor" << TLOG_ENDL;
00078 }
00079
00080 artdaq::TCPSocketTransfer::~TCPSocketTransfer()
00081 {
00082 TLOG_DEBUG(uniqueLabel()) << "Shutting down TCPSocketTransfer" << TLOG_ENDL;
00083 stats_connect_stop_ = true;
00084 stopstatscv_.notify_all();
00085 stats_connect_thread_.join();
00086
00087 if (role() == TransferInterface::Role::kSend)
00088 {
00089
00090 MessHead mh = {0,MessHead::stop_v0,htons(source_rank()),0};
00091 if (fd_ != -1)
00092 {
00093
00094 timeval tv = {0,100000};
00095 socklen_t len = sizeof(tv);
00096 setsockopt(fd_, SOL_SOCKET, SO_SNDTIMEO, &tv, len);
00097 write(fd_, &mh, sizeof(mh));
00098 }
00099 }
00100 close(fd_);
00101 TLOG_DEBUG(uniqueLabel()) << "End of TCPSocketTransfer Destructor" << TLOG_ENDL;
00102 TRACE(4, "TCPSocketTransfer dtor");
00103 }
00104
00105
00106
00107
00108 artdaq::TransferInterface::CopyStatus artdaq::TCPSocketTransfer::sendFragment_(Fragment&& frag, size_t send_timeout_usec)
00109 {
00110 TRACE(7, "TCPSocketTransfer::sendFragment begin");
00111 artdaq::Fragment grab_ownership_frag = std::move(frag);
00112 iovec iov = {(void*)grab_ownership_frag.headerBeginBytes(), grab_ownership_frag.sizeBytes()};
00113 auto sts = sendFragment_(&iov, 1, send_timeout_usec);
00114 while (sts != CopyStatus::kSuccess)
00115 {
00116 TRACE(7, "TCPSocketTransfer::sendFragment: Timeout or Error sending fragment");
00117 sts = sendFragment_(&iov, 1, send_timeout_usec);
00118 usleep(1000);
00119 }
00120
00121 std::string result = (sts == TransferInterface::CopyStatus::kSuccess ? "kSuccess" : (sts == TransferInterface::CopyStatus::kTimeout ? "kTimeout" : "kErrorNotRequiringException"));
00122
00123 TRACE_(7, "TCPSocketTransfer::sendFragment returning " + result);
00124 return sts;
00125 }
00126
00127 artdaq::TransferInterface::CopyStatus artdaq::TCPSocketTransfer::sendFragment_(const void* buf, size_t bytes, size_t send_timeout_usec)
00128 {
00129 TLOG_DEBUG(uniqueLabel()) << "TCPSocketTransfer::sendFragment_ Converting buf to iovec" << TLOG_ENDL;
00130 iovec iov = {(void*)buf, bytes};
00131 return sendFragment_(&iov, 1, send_timeout_usec);
00132 }
00133
00134 artdaq::TransferInterface::CopyStatus artdaq::TCPSocketTransfer::sendFragment_(const struct iovec* iov, int iovcnt, size_t send_timeout_usec)
00135 {
00136
00137 if (fd_ == -1)
00138 {
00139 if (timeoutMessageArmed_)
00140 {
00141 TLOG_DEBUG(uniqueLabel()) << "TCPSocketTransfer::sendFragment_: Send fd is not open. Returning kTimeout" << TLOG_ENDL;
00142 timeoutMessageArmed_ = false;
00143 }
00144 return TransferInterface::CopyStatus::kTimeout;
00145 }
00146 timeoutMessageArmed_ = true;
00147 TRACE(12, "send_timeout_usec is %zu, currently unused.", send_timeout_usec);
00148
00149
00150 uint32_t total_to_write_bytes = 0;
00151 std::vector<iovec> iov_in(iovcnt + 1);
00152 std::vector<iovec> iovv(iovcnt + 2);
00153 int ii;
00154 for (ii = 0; ii < iovcnt; ++ii)
00155 {
00156 iov_in[ii + 1] = iov[ii];
00157 total_to_write_bytes += iov[ii].iov_len;
00158 }
00159
00160 MessHead mh = {0,MessHead::data_v0,htons(source_rank()),htonl(total_to_write_bytes)};
00161 iov_in[0].iov_base = &mh;
00162 iov_in[0].iov_len = sizeof(mh);
00163 total_to_write_bytes += sizeof(mh);
00164
00165 ssize_t sts = 0;
00166 ssize_t total_written_bytes = 0;
00167 ssize_t per_write_max_bytes = (32 * 1024);
00168
00169 size_t in_iov_idx = 0;
00170 size_t out_iov_idx = 0;
00171 ssize_t this_write_bytes = 0;
00172
00173 do
00174 {
00175
00176
00177 for (;
00178 (in_iov_idx + out_iov_idx) < iov_in.size() && this_write_bytes < per_write_max_bytes;
00179 ++out_iov_idx)
00180 {
00181 this_write_bytes += iov_in[in_iov_idx + out_iov_idx].iov_len;
00182 iovv[out_iov_idx] = iov_in[in_iov_idx + out_iov_idx];
00183 }
00184 if (this_write_bytes > per_write_max_bytes)
00185 {
00186 iovv[out_iov_idx - 1].iov_len -= this_write_bytes - per_write_max_bytes;
00187 this_write_bytes = per_write_max_bytes;
00188 }
00189
00190
00191 do_again:
00192 TRACE(7, "sendFragment b4 writev %7zu total_written_bytes fd=%d in_idx=%zu iovcnt=%zu 1st.len=%zu"
00193 , total_written_bytes, fd_, in_iov_idx, out_iov_idx, iovv[0].iov_len);
00194
00195 sts = writev(fd_, &(iovv[0]), out_iov_idx);
00196
00197
00198 if (sts == -1)
00199 {
00200 if (errno == EAGAIN )
00201 {
00202 TRACE(2, "sendFragment EWOULDBLOCK");
00203 fcntl(fd_, F_SETFL, 0);
00204 blocking = true;
00205
00206 goto do_again;
00207 }
00208 TLOG_DEBUG(uniqueLabel()) << "TCPSocketTransfer::sendFragment_: WRITE ERROR!" << TLOG_ENDL;
00209 connect_state = 0;
00210 close(fd_);
00211 fd_ = -1;
00212 return TransferInterface::CopyStatus::kErrorNotRequiringException;
00213 }
00214 else if (sts != this_write_bytes)
00215 {
00216
00217 TRACE(4, "sendFragment writev sts(%ld)!=requested_send_bytes(%ld)"
00218 , sts, this_write_bytes);
00219 total_written_bytes += sts;
00220
00221 for (ii = 0; (size_t)sts >= iovv[ii].iov_len; ++ii)
00222 sts -= iovv[ii].iov_len;
00223 in_iov_idx += ii;
00224 iovv[ii].iov_len -= sts;
00225 iovv[ii].iov_base = (uint8_t*)(iovv[ii].iov_base) + sts;
00226
00227
00228 out_iov_idx = 0;
00229 if (ii != 0)
00230 iovv[out_iov_idx] = iovv[ii];
00231
00232 this_write_bytes = iovv[out_iov_idx].iov_len;
00233
00234
00235
00236
00237 unsigned long additional = ((unsigned long)iov_in[in_iov_idx].iov_base + iov_in[in_iov_idx].iov_len)
00238 - ((unsigned long)iovv[out_iov_idx].iov_base + iovv[out_iov_idx].iov_len);
00239 if (additional)
00240 {
00241 iovv[out_iov_idx].iov_len += additional;
00242 this_write_bytes += additional;
00243 if (this_write_bytes > per_write_max_bytes)
00244 {
00245 iovv[out_iov_idx].iov_len -= this_write_bytes - per_write_max_bytes;
00246 this_write_bytes = per_write_max_bytes;
00247 }
00248 }
00249 ++out_iov_idx;
00250 TRACE(4, "sendFragment writev sts!=: this_write_bytes=%zd out_iov_idx=%zu additional=%lu ii=%d"
00251 , this_write_bytes, out_iov_idx, additional, ii);
00252 }
00253 else
00254 {
00255 TRACE(4, "sendFragment writev sts(%ld)==requested_send_bytes(%ld)"
00256 , sts, this_write_bytes);
00257 total_written_bytes += sts;
00258 --out_iov_idx;
00259 iovv[out_iov_idx].iov_base = (uint8_t*)(iovv[out_iov_idx].iov_base) + iovv[out_iov_idx].iov_len;
00260 iovv[out_iov_idx].iov_len = 0;
00261 in_iov_idx += out_iov_idx;
00262 this_write_bytes = 0;
00263
00264 unsigned long additional = ((unsigned long)iov_in[in_iov_idx].iov_base + iov_in[in_iov_idx].iov_len)
00265 - ((unsigned long)iovv[out_iov_idx].iov_base + iovv[out_iov_idx].iov_len);
00266 if (additional)
00267 {
00268 iovv[out_iov_idx].iov_len += additional;
00269 this_write_bytes += additional;
00270 if (this_write_bytes > per_write_max_bytes)
00271 {
00272 iovv[out_iov_idx].iov_len -= this_write_bytes - per_write_max_bytes;
00273 this_write_bytes = per_write_max_bytes;
00274 }
00275 if (out_iov_idx != 0)
00276 iovv[0] = iovv[out_iov_idx];
00277 out_iov_idx = 1;
00278 }
00279 else
00280 {
00281 ++in_iov_idx;
00282 out_iov_idx = 0;
00283 }
00284 }
00285 }
00286 while (total_written_bytes < total_to_write_bytes);
00287 if (total_written_bytes > total_to_write_bytes)
00288 TRACE(0, "sendFragment program error: too many bytes transferred");
00289
00290 if (blocking)
00291 {
00292 blocking = false;
00293 fcntl(fd_, F_SETFL, 0);
00294 }
00295 sts = total_written_bytes - sizeof(MessHead);
00296
00297 TRACE(10, "sendFragment sts=%ld", sts);
00298 return TransferInterface::CopyStatus::kSuccess;
00299 }
00300
00301
00302
00303 void artdaq::TCPSocketTransfer::connect_()
00304 {
00305 TLOG_DEBUG(uniqueLabel()) << "Connecting sender socket" << TLOG_ENDL;
00306 int sndbuf_bytes = static_cast<int>(sndbuf_);
00307 fd_ = TCPConnect(hostMap_[destination_rank()].hostname.c_str()
00308 , calculate_port_()
00309 , O_NONBLOCK
00310 , sndbuf_bytes);
00311 connect_state = 0;
00312 blocking = 0;
00313 TLOG_DEBUG(uniqueLabel()) << "TCPSocketTransfer::connect_ " + hostMap_[destination_rank()].hostname + ":" << calculate_port_() << " fd_=" << fd_ << TLOG_ENDL;
00314 if (fd_ != -1)
00315 {
00316
00317 TLOG_DEBUG(uniqueLabel()) << "TCPSocketTransfer::connect_: Writing connect message" << TLOG_ENDL;
00318 MessHead mh = {0,MessHead::connect_v0,htons(source_rank()),htonl(CONN_MAGIC)};
00319 ssize_t sts = write(fd_, &mh, sizeof(mh));
00320 if (sts == -1)
00321 {
00322 TLOG_ERROR(uniqueLabel()) << "TCPSocketTransfer::connect_: Error writing connect message!" << TLOG_ENDL;
00323
00324 connect_state = 0;
00325 close(fd_);
00326 fd_ = -1;
00327 }
00328 else
00329 {
00330 TLOG_INFO(uniqueLabel()) << "TCPSocketTransfer::connect_: Successfully connected" << TLOG_ENDL;
00331
00332 connect_state = 1;
00333 }
00334 }
00335 }
00336
00337 void artdaq::TCPSocketTransfer::reconnect_()
00338 {
00339 TRACE(5, "check/reconnect");
00340 if (fd_ == -1 && role() == TransferInterface::Role::kSend) return connect_();
00341 if ((fd_ == -1 || listen_fd_ == -1) && role() == TransferInterface::Role::kReceive) return listen_();
00342 }
00343
00344
00345 static uint64_t gettimeofday_us()
00346 {
00347 struct timeval tv;
00348 gettimeofday(&tv, NULL);
00349 return (uint64_t)tv.tv_sec * 1000000 + tv.tv_usec;
00350 }
00351
00352 void artdaq::TCPSocketTransfer::listen_()
00353 {
00354 TLOG_DEBUG(uniqueLabel()) << "TCPSocketTransfer::listen_: Listening/accepting new connections" << TLOG_ENDL;
00355 if (listen_fd_ == -1)
00356 {
00357 TLOG_DEBUG(uniqueLabel()) << "TCPSocketTransfer::listen_: Opening listener" << TLOG_ENDL;
00358 listen_fd_ = TCP_listen_fd(calculate_port_(), rcvbuf_);
00359 }
00360 if (listen_fd_ == -1)
00361 {
00362 TLOG_DEBUG(uniqueLabel()) << "TCPSocketTransfer::listen_: Error creating listen_fd_!" << TLOG_ENDL;
00363 return;
00364 }
00365
00366 int res;
00367 timeval tv = {2,0};
00368 fd_set rfds;
00369 FD_ZERO(&rfds);
00370 FD_SET(listen_fd_, &rfds);
00371
00372 res = select(listen_fd_ + 1, &rfds, (fd_set *)0, (fd_set *)0, &tv);
00373 if (res > 0)
00374 {
00375 int sts;
00376 sockaddr_un un;
00377 socklen_t arglen = sizeof(un);
00378 int fd;
00379 TLOG_DEBUG(uniqueLabel()) << "Calling accept" << TLOG_ENDL;
00380 fd = accept(listen_fd_, (sockaddr *)&un, &arglen);
00381 TLOG_DEBUG(uniqueLabel()) << "Done with accept" << TLOG_ENDL;
00382
00383 TLOG_DEBUG(uniqueLabel()) << "TCPSocketTransfer::listen_: Reading connect message" << TLOG_ENDL;
00384 socklen_t lenlen = sizeof(tv);
00385
00386 setsockopt(fd, SOL_SOCKET, SO_RCVTIMEO, &tv, lenlen);
00387 MessHead mh;
00388 uint64_t mark_us = gettimeofday_us();
00389 sts = read(fd, &mh, sizeof(mh));
00390 uint64_t delta_us = gettimeofday_us() - mark_us;
00391 TLOG_DEBUG(uniqueLabel()) << "TCPSocketTransfer::listen_: Read of connect message took " << delta_us << " microseconds." << TLOG_ENDL;
00392 TRACE(10, "do_connect read of connect msg (after accept) took %lu microseconds", delta_us);
00393 if (sts != sizeof(mh))
00394 {
00395 TLOG_DEBUG(uniqueLabel()) << "TCPSocketTransfer::listen_: Wrong message header length received!" << TLOG_ENDL;
00396 TRACE(0, "do_connect_ problem with connect msg sts(%d)!=sizeof(mh)(%ld)", sts, sizeof(mh));
00397 close(fd);
00398 return;
00399 }
00400
00401
00402 mh.source_id = ntohs(mh.source_id);
00403 if (ntohl(mh.conn_magic) != CONN_MAGIC || mh.source_id != source_rank())
00404 {
00405 TLOG_DEBUG(uniqueLabel()) << "TCPSocketTransfer::listen_: Wrong magic bytes in header!" << TLOG_ENDL;
00406 close(fd);
00407 return;
00408 }
00409 if (fd_ != -1)
00410 {
00411
00412 close(fd_);
00413 }
00414
00415
00416 fd_ = fd;
00417 TLOG_INFO(uniqueLabel()) << "TCPSocketTransfer::listen_: New fd is " << fd_ << TLOG_ENDL;
00418
00419 TRACE(3, "do_connect_ connection from sender_rank=%zu", mh.source_id);
00420 }
00421 else
00422 {
00423 TRACE(10, "TCPSocketTransfer::do_connect_: No connections in timeout interval!");
00424 }
00425 }
00426
00427
00428
00429
00430
00431 int artdaq::TCPSocketTransfer::receiveFragment(Fragment& outfrag, size_t timeout_usec)
00432 {
00433 TRACE(7, "TCPSocketTransfer::receiveFragment: BEGIN");
00434 int ret_rank = RECV_TIMEOUT;
00435 if (fd_ == -1)
00436 {
00437 TLOG_DEBUG(uniqueLabel()) << "TCPSocketTransfer::receiveFragment: Receive socket not connected, returning RECV_TIMEOUT" << TLOG_ENDL;
00438 return RECV_TIMEOUT;
00439 }
00440
00441 TRACE(7, "TCPSocketTransfer::recvFragment timeout_usec=%ld", timeout_usec);
00442
00443 uint8_t* buff;
00444 size_t byte_cnt = 0;
00445 int sts;
00446 uint64_t start_time_us = gettimeofday_us();
00447
00448 pollfd pollfd_s;
00449 pollfd_s.events = POLLIN | POLLERR;
00450 pollfd_s.fd = fd_;
00451
00452 int timeout_ms;
00453 if (timeout_usec == 0)
00454 timeout_ms = 0;
00455 else
00456 timeout_ms = (timeout_usec + 999) / 1000;
00457
00458 bool done = false;
00459 while (!done)
00460 {
00461
00462 int num_fds_ready = poll(&pollfd_s, 1, timeout_ms);
00463 if (num_fds_ready <= 0)
00464 {
00465 if (num_fds_ready == 0 && timeout_ms > 0)
00466 {
00467 TRACE(7, "TCPSocketTransfer::receiveFragment: No data on receive socket, returning RECV_TIMEOUT");
00468 return RECV_TIMEOUT;
00469 }
00470 break;
00471 }
00472
00473 if (!(pollfd_s.revents & (POLLIN | POLLERR)))
00474 {
00475 TLOG_DEBUG(uniqueLabel()) << "TCPSocketTransfer::receiveFragment: Wrong event received from pollfd" << TLOG_ENDL;
00476 continue;
00477 }
00478
00479 if (state_ == SocketState::Metadata)
00480 {
00481
00482 buff = &(mha[offset]);
00483 byte_cnt = sizeof(MessHead) - offset;
00484 }
00485 else
00486 {
00487
00488 buff = buffer + offset;
00489 byte_cnt = mh.byte_count - offset;
00490 }
00491
00492
00493 sts = read(fd_, buff, byte_cnt);
00494
00495
00496 TRACE(9, "recvFragment state=%d read=%d (errno=%d)", static_cast<int>(state_), sts, errno);
00497 if (sts <= 0)
00498 {
00499 TLOG_DEBUG(uniqueLabel()) << "TCPSocketTransfer::receiveFragment: Error on receive, closing socket" << TLOG_ENDL;
00500 close(fd_);
00501 }
00502 else
00503 {
00504
00505 sts = offset += sts;
00506 if (sts == target_bytes)
00507 {
00508 TRACE(7, "TCPSocketTransfer::receiveFragment: Target read bytes reached. Changing state");
00509 offset = 0;
00510 if (state_ == SocketState::Metadata)
00511 {
00512 state_ = SocketState::Data;
00513 mh.byte_count = ntohl(mh.byte_count);
00514 mh.source_id = ntohs(mh.source_id);
00515 target_bytes = mh.byte_count;
00516 }
00517 else
00518 {
00519 state_ = SocketState::Metadata;
00520 target_bytes = sizeof(MessHead);
00521 ret_rank = source_rank();
00522 TRACE(9, "recvFragment done sts=%d src=%d", sts, ret_rank);
00523 TRACE(7, "TCPSocketTransfer::receiveFragment: Done receiving fragment. Moving into output.");
00524 frag.autoResize();
00525 if (frag.type() == artdaq::Fragment::EndOfDataFragmentType)
00526 {
00527 stats_connect_stop_ = true;
00528 stopstatscv_.notify_all();
00529 }
00530 outfrag.swap(frag);
00531 frag.reserve(max_fragment_size_words_);
00532 buffer = frag.headerBeginBytes();
00533 done = true;
00534 break;
00535 }
00536 }
00537 }
00538
00539 if (!done && timeout_usec > 0)
00540 {
00541
00542 size_t delta_us = gettimeofday_us() - start_time_us;
00543 if (delta_us > timeout_usec)
00544 return RECV_TIMEOUT;
00545 timeout_ms = ((timeout_usec - delta_us) + 999) / 1000;
00546 }
00547 }
00548
00549 TRACE(7, "TCPSocketTransfer::receiveFragment: Returning %d", ret_rank);
00550 return ret_rank;
00551 }
00552
00553 void artdaq::TCPSocketTransfer::stats_connect_()
00554 {
00555 std::cv_status sts;
00556 while (!stats_connect_stop_)
00557 {
00558 std::string desc;
00559 void* tag;
00560 std::function<void()> function;
00561 uint64_t ts_us;
00562
00563 int msdly = tmo_.get_next_timeout_msdly();
00564
00565 if (msdly <= 0)
00566 msdly = 2000;
00567
00568 std::unique_lock<std::mutex> lck(stopstatscvm_);
00569 sts = stopstatscv_.wait_until(lck
00570 , std::chrono::system_clock::now()
00571 + std::chrono::milliseconds(msdly));
00572 TRACE(5, "thread1 after wait_until(msdly=%d) - sts=%d", msdly, static_cast<int>(sts));
00573
00574 if (sts == std::cv_status::no_timeout)
00575 break;
00576
00577 auto sts = tmo_.get_next_expired_timeout(desc, &tag, function, &ts_us);
00578
00579 while (sts != -1 && desc != "")
00580 {
00581 if (function != NULL)
00582 function();
00583
00584 sts = tmo_.get_next_expired_timeout(desc, &tag, function, &ts_us);
00585 }
00586 }
00587 }
00588
00589 DEFINE_ARTDAQ_TRANSFER(artdaq::TCPSocketTransfer)