00001 #ifndef TCPSocketTransfer_hh
00002 #define TCPSocketTransfer_hh
00003
00004
00005
00006
00007
00008
00009
00010
00011 #include <stdint.h>
00012 #include <sys/uio.h>
00013
00014
00015 #include <condition_variable>
00016 #include <boost/thread.hpp>
00017
00018
00019 #include "fhiclcpp/fwd.h"
00020
00021
00022 #include "artdaq/TransferPlugins/TransferInterface.hh"
00023 #include "artdaq/TransferPlugins/detail/SRSockets.hh"
00024 #include "artdaq/TransferPlugins/detail/Timeout.hh"
00025 #include "artdaq-core/Data/Fragment.hh"
00026 #include "artdaq/TransferPlugins/detail/HostMap.hh"
00027
00028 namespace artdaq
00029 {
00030 class TCPSocketTransfer;
00031 }
00032
00036 class artdaq::TCPSocketTransfer : public TransferInterface
00037 {
00038 public:
00057 TCPSocketTransfer(fhicl::ParameterSet const& ps, Role role);
00058
00059 virtual ~TCPSocketTransfer() noexcept;
00060
00067 int receiveFragmentHeader(detail::RawFragmentHeader& header, size_t receiveTimeout) override;
00068
00075 int receiveFragmentData(RawDataType* destination, size_t wordCount) override;
00076
00083 CopyStatus copyFragment(Fragment& frag, size_t timeout_usec) override { return sendFragment_(std::move(frag), timeout_usec); }
00084
00090 CopyStatus moveFragment(Fragment&& frag) override { return sendFragment_(std::move(frag), 0); }
00091
00096 bool isRunning() override;
00097 private:
00098
00099 static std::atomic<int> listen_thread_refcount_;
00100 static std::mutex listen_thread_mutex_;
00101 static std::unique_ptr<boost::thread> listen_thread_;
00102 static std::map<int, std::set<int>> connected_fds_;
00103 static std::mutex connected_fd_mutex_;
00104 int send_fd_;
00105 int active_receive_fd_;
00106 int last_active_receive_fd_;
00107
00108 union
00109 {
00110 MessHead mh;
00111 uint8_t mha[sizeof(MessHead)];
00112 };
00113
00114 enum class SocketState
00115 {
00116 Metadata,
00117 Data
00118 };
00119
00120 size_t rcvbuf_;
00121 size_t sndbuf_;
00122 size_t send_retry_timeout_us_;
00123
00124 hostMap_t hostMap_;
00125
00126 volatile unsigned connect_state : 1;
00127 unsigned blocking : 1;
00128
00129
00130 Timeout tmo_;
00131 bool stats_connect_stop_;
00132 boost::thread stats_connect_thread_;
00133 std::condition_variable stopstatscv_;
00134 std::mutex stopstatscvm_;
00135
00136 bool timeoutMessageArmed_;
00137 size_t not_connected_count_;
00138 size_t receive_err_threshold_;
00139 size_t receive_err_wait_us_;
00140
00141 private:
00142 CopyStatus sendFragment_(Fragment&& frag, size_t timeout_usec);
00143
00144 CopyStatus sendData_(const void* buf, size_t bytes, size_t tmo);
00145
00146 CopyStatus sendData_(const struct iovec* iov, int iovcnt, size_t tmo);
00147
00148
00149 void stats_connect_();
00150
00151
00152 void connect_();
00153
00154 void reconnect_();
00155
00156
00157 void start_listen_thread_();
00158 static void listen_(int port, size_t rcvbuf);
00159
00160 int calculate_port_() const {
00161 return destination_rank() + ((partition_number_ % 22) * 1000) + 10000;
00162 }
00163
00164 size_t getConnectedFDCount(int source_rank)
00165 {
00166 std::unique_lock<std::mutex> lk(connected_fd_mutex_);
00167 return connected_fds_.count(source_rank) ? connected_fds_[source_rank].size() : 0;
00168 }
00169 };
00170
00171 #endif // TCPSocketTransfer_hh