artdaq  v3_12_02
TCPSocketTransfer.hh
1 #ifndef TCPSocketTransfer_hh
2 #define TCPSocketTransfer_hh
3 // This file (TCPSocketTransfer.hh) was created by Ron Rechenmacher <ron@fnal.gov> on
4 // Sep 14, 2016. "TERMS AND CONDITIONS" governing this file are in the README
5 // or COPYING file. If you do not have such a file, one can be obtained by
6 // contacting Ron or Fermi Lab in Batavia IL, 60510, phone: 630-840-3000.
7 // $RCSfile: .emacs.gnu,v $
8 // rev="$Revision: 1.30 $$Date: 2016/03/01 14:27:27 $";
9 
10 // artdaq Includes
11 #include "artdaq/DAQdata/HostMap.hh"
12 #include "artdaq/TransferPlugins/TransferInterface.hh"
13 #include "artdaq/TransferPlugins/detail/SRSockets.hh"
14 
15 // C Includes
16 #include <sys/uio.h> // iovec
17 #include <cstdint> // uint64_t
18 
19 // C++ Includes
20 #include <boost/thread.hpp>
21 
22 #include <atomic>
23 #include <chrono>
24 #include <condition_variable>
25 #include <map>
26 #include <memory>
27 #include <mutex>
28 #include <string>
29 #include <utility> // std::move()
30 
31 #ifndef USE_ACKS
32 #define USE_ACKS 0
33 #endif
34 
35 namespace artdaq {
36 class TCPSocketTransfer;
37 }
38 
43 {
44 public:
63  TCPSocketTransfer(fhicl::ParameterSet const& ps, Role role);
64 
65  virtual ~TCPSocketTransfer() noexcept;
66 
73  int receiveFragmentHeader(detail::RawFragmentHeader& header, size_t timeout_usec) override;
74 
81  int receiveFragmentData(RawDataType* destination, size_t wordCount) override;
82 
89  CopyStatus transfer_fragment_min_blocking_mode(Fragment const& frag, size_t timeout_usec) override { return sendFragment_(Fragment(frag), timeout_usec); }
90 
96  CopyStatus transfer_fragment_reliable_mode(Fragment&& frag) override { return sendFragment_(std::move(frag), 0); }
97 
102  bool isRunning() override;
103 
108  void flush_buffers() override;
109 
110 private:
111  static std::atomic<int> listen_thread_refcount_;
112  static std::mutex listen_thread_mutex_;
113  static std::unique_ptr<boost::thread> listen_thread_;
114  static std::map<int, std::set<int>> connected_fds_;
115  static std::mutex fd_mutex_;
116  int send_fd_;
117  std::map<int, int> active_receive_fds_;
118  std::map<int, int> last_active_receive_fds_;
119 
120  union
121  {
122  MessHead mh;
123  uint8_t mha[sizeof(MessHead)];
124  };
125 
126  enum class SocketState
127  {
128  Metadata,
129  Data
130  };
131 
132  size_t rcvbuf_;
133  size_t sndbuf_;
134  size_t send_retry_timeout_us_;
135 
136  hostMap_t hostMap_;
137 
138  volatile unsigned connect_state : 1; // 0=not "connected" (initial msg not sent)
139  unsigned blocking : 1; // compatible with bool (true/false)
140 
141  bool connection_was_lost_;
142 
143  bool timeoutMessageArmed_; // don't repeatedly print about the send fd not being open...
144  std::chrono::steady_clock::time_point last_recv_time_; // Time of last successful receive
145  double receive_disconnected_wait_s_; // How long to wait between messages before returning DATA_END
146  size_t receive_err_wait_us_; // Amount of time to wait if there are no connected receive sockets
147  std::atomic<bool> receive_socket_has_been_connected_; // Whether the receiver has ever been connected to a sender
148  std::atomic<int> send_ack_diff_; // Number of sends - number of acks received. Not allowed to exceed buffer_count.
149  std::unique_ptr<boost::thread> ack_listen_thread_; // Thread to listen for ack messages on the sender
150 
151 private: // methods
152  TCPSocketTransfer(TCPSocketTransfer const&) = delete;
154  TCPSocketTransfer& operator=(TCPSocketTransfer const&) = delete;
155  TCPSocketTransfer& operator=(TCPSocketTransfer&&) = delete;
156 
157  CopyStatus sendFragment_(Fragment&& frag, size_t timeout_usec);
158 
159  CopyStatus sendData_(const void* buf, size_t bytes, size_t send_timeout_usec, bool isHeader = false);
160 
161  CopyStatus sendData_(const struct iovec* iov, int iovcnt, size_t send_timeout_usec, bool isHeader = false);
162 
163 #if USE_ACKS
164  void receive_acks_();
165  void send_ack_(int fd);
166 #endif
167 
168  // Sender is responsible for connecting to receiver
169  void connect_();
170 
171  void reconnect_();
172 
173  void disconnect_receive_socket_(const std::string& msg = "");
174 
175  // Receiver should listen for connections
176  void start_listen_thread_();
177  static void listen_(int port, size_t rcvbuf);
178 
179  size_t getConnectedFDCount_(int source_rank);
180  int getActiveFD_(int source_rank);
181  void setActiveFD_(int source_rank, int fd);
182  int getLastActiveFD_(int source_rank);
183  void setLastActiveFD_(int source_rank, int fd);
184 };
185 
186 #endif // TCPSocketTransfer_hh
bool isRunning() override
Determine whether the TransferInterface plugin is able to send/receive data.
virtual int source_rank() const
Get the source rank for this TransferInterface instance.
Role role() const
Get the TransferInterface::Role of this TransferInterface.
int receiveFragmentHeader(detail::RawFragmentHeader &header, size_t timeout_usec) override
Receive a Fragment Header from the transport mechanism.
CopyStatus transfer_fragment_reliable_mode(Fragment &&frag) override
Transfer a Fragment to the destination. This should be reliable, if the underlying transport mechanis...
int receiveFragmentData(RawDataType *destination, size_t wordCount) override
Receive the body of a Fragment to the given destination pointer.
TCPSocketTransfer(fhicl::ParameterSet const &ps, Role role)
TCPSocketTransfer Constructor.
void flush_buffers() override
Flush any in-flight data. This should be used by the receiver after the receive loop has ended...
Role
Used to determine if a TransferInterface is a Sender or Receiver.
This interface defines the functions used to transfer data between artdaq applications.
TransferInterface implementation plugin that sends data using TCP sockets.
This header is sent by the TCPSocket_transfer to allow for more efficient writev calls.
Definition: SRSockets.hh:15
std::map< int, std::string > hostMap_t
The host_map is a map associating ranks with artdaq::DestinationInfo objects.
Definition: HostMap.hh:42
CopyStatus transfer_fragment_min_blocking_mode(Fragment const &frag, size_t timeout_usec) override
Transfer a Fragment to the destination. May not necessarily be reliable, but will not block longer th...
CopyStatus
Returned from the send functions, this enumeration describes the possible return codes. If an exception occurs, it will be thrown and should be handled normally.