00001 #ifndef TCPSocketTransfer_hh
00002 #define TCPSocketTransfer_hh
00003
00004
00005
00006
00007
00008
00009
00010
00011 #include <stdint.h>
00012 #include <sys/uio.h>
00013
00014
00015 #include <condition_variable>
00016 #include <boost/thread.hpp>
00017
00018
00019 #include "fhiclcpp/fwd.h"
00020
00021
00022 #include "artdaq/TransferPlugins/TransferInterface.hh"
00023 #include "artdaq/TransferPlugins/detail/SRSockets.hh"
00024 #include "artdaq/TransferPlugins/detail/Timeout.hh"
00025 #include "artdaq-core/Data/Fragment.hh"
00026 #include "artdaq/TransferPlugins/detail/HostMap.hh"
00027
00028 #ifndef USE_ACKS
00029 #define USE_ACKS 0
00030 #endif
00031
00032 namespace artdaq
00033 {
00034 class TCPSocketTransfer;
00035 }
00036
00040 class artdaq::TCPSocketTransfer : public TransferInterface
00041 {
00042 public:
00061 TCPSocketTransfer(fhicl::ParameterSet const& ps, Role role);
00062
00063 virtual ~TCPSocketTransfer() noexcept;
00064
00071 int receiveFragmentHeader(detail::RawFragmentHeader& header, size_t receiveTimeout) override;
00072
00079 int receiveFragmentData(RawDataType* destination, size_t wordCount) override;
00080
00087 CopyStatus copyFragment(Fragment& frag, size_t timeout_usec) override { return sendFragment_(std::move(frag), timeout_usec); }
00088
00094 CopyStatus moveFragment(Fragment&& frag) override { return sendFragment_(std::move(frag), 0); }
00095
00100 bool isRunning() override;
00101 private:
00102
00103 static std::atomic<int> listen_thread_refcount_;
00104 static std::mutex listen_thread_mutex_;
00105 static std::unique_ptr<boost::thread> listen_thread_;
00106 static std::map<int, std::set<int>> connected_fds_;
00107 static std::mutex connected_fd_mutex_;
00108 int send_fd_;
00109 int active_receive_fd_;
00110 int last_active_receive_fd_;
00111 short active_revents_;
00112
00113 union
00114 {
00115 MessHead mh;
00116 uint8_t mha[sizeof(MessHead)];
00117 };
00118
00119 enum class SocketState
00120 {
00121 Metadata,
00122 Data
00123 };
00124
00125 size_t rcvbuf_;
00126 size_t sndbuf_;
00127 size_t send_retry_timeout_us_;
00128
00129 hostMap_t hostMap_;
00130
00131 volatile unsigned connect_state : 1;
00132 unsigned blocking : 1;
00133
00134 bool timeoutMessageArmed_;
00135 std::chrono::steady_clock::time_point last_recv_time_;
00136 double receive_disconnected_wait_s_;
00137 size_t receive_err_wait_us_;
00138 std::atomic<bool> receive_socket_has_been_connected_;
00139 std::atomic<int> send_ack_diff_;
00140 std::unique_ptr<boost::thread> ack_listen_thread_;
00141
00142 private:
00143 CopyStatus sendFragment_(Fragment&& frag, size_t timeout_usec);
00144
00145 CopyStatus sendData_(const void* buf, size_t bytes, size_t tmo, bool isHeader = false);
00146
00147 CopyStatus sendData_(const struct iovec* iov, int iovcnt, size_t tmo, bool isHeader = false);
00148
00149 #if USE_ACKS
00150 void receive_acks_();
00151 void send_ack_(int fd);
00152 #endif
00153
00154
00155 void connect_();
00156
00157 void reconnect_();
00158
00159 int disconnect_receive_socket_(int fd, std::string msg = "");
00160
00161
00162 void start_listen_thread_();
00163 static void listen_(int port, size_t rcvbuf);
00164
00165 size_t getConnectedFDCount(int source_rank)
00166 {
00167 std::unique_lock<std::mutex> lk(connected_fd_mutex_);
00168 return connected_fds_.count(source_rank) ? connected_fds_[source_rank].size() : 0;
00169 }
00170 };
00171
00172 #endif // TCPSocketTransfer_hh