00001 #ifndef TCPSocketTransfer_hh
00002 #define TCPSocketTransfer_hh
00003
00004
00005
00006
00007
00008
00009
00010
00011 #include <stdint.h>
00012 #include <sys/uio.h>
00013
00014
00015 #include <condition_variable>
00016 #include <boost/thread.hpp>
00017
00018
00019 #include "fhiclcpp/fwd.h"
00020
00021
00022 #include "artdaq/TransferPlugins/TransferInterface.hh"
00023 #include "artdaq/TransferPlugins/detail/SRSockets.hh"
00024 #include "artdaq/TransferPlugins/detail/Timeout.hh"
00025 #include "artdaq-core/Data/Fragment.hh"
00026 #include "artdaq/TransferPlugins/detail/HostMap.hh"
00027
00028 #define USE_ACKS 0
00029
00030 namespace artdaq
00031 {
00032 class TCPSocketTransfer;
00033 }
00034
00038 class artdaq::TCPSocketTransfer : public TransferInterface
00039 {
00040 public:
00059 TCPSocketTransfer(fhicl::ParameterSet const& ps, Role role);
00060
00061 virtual ~TCPSocketTransfer() noexcept;
00062
00069 int receiveFragmentHeader(detail::RawFragmentHeader& header, size_t receiveTimeout) override;
00070
00077 int receiveFragmentData(RawDataType* destination, size_t wordCount) override;
00078
00085 CopyStatus copyFragment(Fragment& frag, size_t timeout_usec) override { return sendFragment_(std::move(frag), timeout_usec); }
00086
00092 CopyStatus moveFragment(Fragment&& frag) override { return sendFragment_(std::move(frag), 0); }
00093
00098 bool isRunning() override;
00099 private:
00100
00101 static std::atomic<int> listen_thread_refcount_;
00102 static std::mutex listen_thread_mutex_;
00103 static std::unique_ptr<boost::thread> listen_thread_;
00104 static std::map<int, std::set<int>> connected_fds_;
00105 static std::mutex connected_fd_mutex_;
00106 int send_fd_;
00107 int active_receive_fd_;
00108 int last_active_receive_fd_;
00109
00110 union
00111 {
00112 MessHead mh;
00113 uint8_t mha[sizeof(MessHead)];
00114 };
00115
00116 enum class SocketState
00117 {
00118 Metadata,
00119 Data
00120 };
00121
00122 size_t rcvbuf_;
00123 size_t sndbuf_;
00124 size_t send_retry_timeout_us_;
00125
00126 hostMap_t hostMap_;
00127
00128 volatile unsigned connect_state : 1;
00129 unsigned blocking : 1;
00130
00131 bool timeoutMessageArmed_;
00132 size_t not_connected_count_;
00133 size_t receive_err_threshold_;
00134 size_t receive_err_wait_us_;
00135
00136 private:
00137 CopyStatus sendFragment_(Fragment&& frag, size_t timeout_usec);
00138
00139 CopyStatus sendData_(const void* buf, size_t bytes, size_t tmo);
00140
00141 CopyStatus sendData_(const struct iovec* iov, int iovcnt, size_t tmo);
00142
00143 #if USE_ACKS
00144 void receive_ack_(int fd);
00145 void send_ack_(int fd);
00146 #endif
00147
00148
00149 void connect_();
00150
00151 void reconnect_();
00152
00153 int disconnect_receive_socket_(int fd, std::string msg = "");
00154
00155
00156 void start_listen_thread_();
00157 static void listen_(int port, size_t rcvbuf);
00158
00159 int calculate_port_() const {
00160 return destination_rank() + ((partition_number_ % 22) * 1000) + 10000;
00161 }
00162
00163 size_t getConnectedFDCount(int source_rank)
00164 {
00165 std::unique_lock<std::mutex> lk(connected_fd_mutex_);
00166 return connected_fds_.count(source_rank) ? connected_fds_[source_rank].size() : 0;
00167 }
00168 };
00169
00170 #endif // TCPSocketTransfer_hh