$treeview $search $mathjax $extrastylesheet
artdaq
v3_04_01
$projectbrief
|
$projectbrief
|
$searchbox |
00001 #ifndef TCPSocketTransfer_hh 00002 #define TCPSocketTransfer_hh 00003 // This file (TCPSocketTransfer.hh) was created by Ron Rechenmacher <ron@fnal.gov> on 00004 // Sep 14, 2016. "TERMS AND CONDITIONS" governing this file are in the README 00005 // or COPYING file. If you do not have such a file, one can be obtained by 00006 // contacting Ron or Fermi Lab in Batavia IL, 60510, phone: 630-840-3000. 00007 // $RCSfile: .emacs.gnu,v $ 00008 // rev="$Revision: 1.30 $$Date: 2016/03/01 14:27:27 $"; 00009 00010 // C Includes 00011 #include <stdint.h> // uint64_t 00012 #include <sys/uio.h> // iovec 00013 00014 // C++ Includes 00015 #include <condition_variable> 00016 #include <boost/thread.hpp> 00017 00018 // Products includes 00019 #include "fhiclcpp/fwd.h" 00020 00021 // artdaq Includes 00022 #include "artdaq/TransferPlugins/TransferInterface.hh" 00023 #include "artdaq/TransferPlugins/detail/SRSockets.hh" 00024 #include "artdaq/TransferPlugins/detail/Timeout.hh" // Timeout 00025 #include "artdaq-core/Data/Fragment.hh" 00026 #include "artdaq/TransferPlugins/detail/HostMap.hh" 00027 00028 #ifndef USE_ACKS 00029 #define USE_ACKS 0 00030 #endif 00031 00032 namespace artdaq 00033 { 00034 class TCPSocketTransfer; 00035 } 00036 00040 class artdaq::TCPSocketTransfer : public TransferInterface 00041 { 00042 public: 00061 TCPSocketTransfer(fhicl::ParameterSet const& ps, Role role); 00062 00063 virtual ~TCPSocketTransfer() noexcept; 00064 00071 int receiveFragmentHeader(detail::RawFragmentHeader& header, size_t receiveTimeout) override; 00072 00079 int receiveFragmentData(RawDataType* destination, size_t wordCount) override; 00080 00087 CopyStatus transfer_fragment_min_blocking_mode(Fragment const& frag, size_t timeout_usec) override { return sendFragment_(Fragment(frag), timeout_usec); } 00088 00094 CopyStatus transfer_fragment_reliable_mode(Fragment&& frag) override { return sendFragment_(std::move(frag), 0); } 00095 00100 bool isRunning() override; 00101 00106 void flush_buffers() override; 00107 00108 private: 00109 00110 static std::atomic<int> listen_thread_refcount_; 00111 static std::mutex listen_thread_mutex_; 00112 static std::unique_ptr<boost::thread> listen_thread_; 00113 static std::map<int, std::set<int>> connected_fds_; 00114 static std::mutex connected_fd_mutex_; 00115 int send_fd_; 00116 int active_receive_fd_; 00117 int last_active_receive_fd_; 00118 short active_revents_; 00119 00120 union 00121 { 00122 MessHead mh; 00123 uint8_t mha[sizeof(MessHead)]; 00124 }; 00125 00126 enum class SocketState 00127 { 00128 Metadata, 00129 Data 00130 }; 00131 00132 size_t rcvbuf_; 00133 size_t sndbuf_; 00134 size_t send_retry_timeout_us_; 00135 00136 hostMap_t hostMap_; 00137 00138 volatile unsigned connect_state : 1; // 0=not "connected" (initial msg not sent) 00139 unsigned blocking : 1; // compatible with bool (true/false) 00140 00141 bool timeoutMessageArmed_; // don't repeatedly print about the send fd not being open... 00142 std::chrono::steady_clock::time_point last_recv_time_; // Time of last successful receive 00143 double receive_disconnected_wait_s_; // How long to wait between messages before returning DATA_END 00144 size_t receive_err_wait_us_; // Amount of time to wait if there are no connected receive sockets 00145 std::atomic<bool> receive_socket_has_been_connected_; // Whether the receiver has ever been connected to a sender 00146 std::atomic<int> send_ack_diff_; // Number of sends - number of acks received. Not allowed to exceed buffer_count. 00147 std::unique_ptr<boost::thread> ack_listen_thread_; // Thread to listen for ack messages on the sender 00148 00149 private: // methods 00150 CopyStatus sendFragment_(Fragment&& frag, size_t timeout_usec); 00151 00152 CopyStatus sendData_(const void* buf, size_t bytes, size_t tmo, bool isHeader = false); 00153 00154 CopyStatus sendData_(const struct iovec* iov, int iovcnt, size_t tmo, bool isHeader = false); 00155 00156 #if USE_ACKS 00157 void receive_acks_(); 00158 void send_ack_(int fd); 00159 #endif 00160 00161 // Sender is responsible for connecting to receiver 00162 void connect_(); 00163 00164 void reconnect_(); 00165 00166 int disconnect_receive_socket_(int fd, std::string msg = ""); 00167 00168 // Receiver should listen for connections 00169 void start_listen_thread_(); 00170 static void listen_(int port, size_t rcvbuf); 00171 00172 size_t getConnectedFDCount(int source_rank) 00173 { 00174 std::unique_lock<std::mutex> lk(connected_fd_mutex_); 00175 return connected_fds_.count(source_rank) ? connected_fds_[source_rank].size() : 0; 00176 } 00177 }; 00178 00179 #endif // TCPSocketTransfer_hh