artdaq  v3_07_01
TCPSocketTransfer.hh
1 #ifndef TCPSocketTransfer_hh
2 #define TCPSocketTransfer_hh
3 // This file (TCPSocketTransfer.hh) was created by Ron Rechenmacher <ron@fnal.gov> on
4 // Sep 14, 2016. "TERMS AND CONDITIONS" governing this file are in the README
5 // or COPYING file. If you do not have such a file, one can be obtained by
6 // contacting Ron or Fermi Lab in Batavia IL, 60510, phone: 630-840-3000.
7 // $RCSfile: .emacs.gnu,v $
8 // rev="$Revision: 1.30 $$Date: 2016/03/01 14:27:27 $";
9 
10 // C Includes
11 #include <stdint.h> // uint64_t
12 #include <sys/uio.h> // iovec
13 
14 // C++ Includes
15 #include <boost/thread.hpp>
16 #include <condition_variable>
17 
18 // Products includes
19 #include "fhiclcpp/fwd.h"
20 
21 // artdaq Includes
22 #include "artdaq-core/Data/Fragment.hh"
23 #include "artdaq/TransferPlugins/TransferInterface.hh"
24 #include "artdaq/TransferPlugins/detail/HostMap.hh"
25 #include "artdaq/TransferPlugins/detail/SRSockets.hh"
26 #include "artdaq/TransferPlugins/detail/Timeout.hh" // Timeout
27 
28 #ifndef USE_ACKS
29 #define USE_ACKS 0
30 #endif
31 
32 namespace artdaq {
33 class TCPSocketTransfer;
34 }
35 
40 {
41 public:
60  TCPSocketTransfer(fhicl::ParameterSet const& ps, Role role);
61 
62  virtual ~TCPSocketTransfer() noexcept;
63 
70  int receiveFragmentHeader(detail::RawFragmentHeader& header, size_t receiveTimeout) override;
71 
78  int receiveFragmentData(RawDataType* destination, size_t wordCount) override;
79 
86  CopyStatus transfer_fragment_min_blocking_mode(Fragment const& frag, size_t timeout_usec) override { return sendFragment_(Fragment(frag), timeout_usec); }
87 
93  CopyStatus transfer_fragment_reliable_mode(Fragment&& frag) override { return sendFragment_(std::move(frag), 0); }
94 
99  bool isRunning() override;
100 
105  void flush_buffers() override;
106 
107 private:
108  static std::atomic<int> listen_thread_refcount_;
109  static std::mutex listen_thread_mutex_;
110  static std::unique_ptr<boost::thread> listen_thread_;
111  static std::map<int, std::set<int>> connected_fds_;
112  static std::mutex connected_fd_mutex_;
113  int send_fd_;
114  int active_receive_fd_;
115  int last_active_receive_fd_;
116  short active_revents_;
117 
118  union
119  {
120  MessHead mh;
121  uint8_t mha[sizeof(MessHead)];
122  };
123 
124  enum class SocketState
125  {
126  Metadata,
127  Data
128  };
129 
130  size_t rcvbuf_;
131  size_t sndbuf_;
132  size_t send_retry_timeout_us_;
133 
134  hostMap_t hostMap_;
135 
136  volatile unsigned connect_state : 1; // 0=not "connected" (initial msg not sent)
137  unsigned blocking : 1; // compatible with bool (true/false)
138 
139  bool connection_was_lost_;
140 
141  bool timeoutMessageArmed_; // don't repeatedly print about the send fd not being open...
142  std::chrono::steady_clock::time_point last_recv_time_; // Time of last successful receive
143  double receive_disconnected_wait_s_; // How long to wait between messages before returning DATA_END
144  size_t receive_err_wait_us_; // Amount of time to wait if there are no connected receive sockets
145  std::atomic<bool> receive_socket_has_been_connected_; // Whether the receiver has ever been connected to a sender
146  std::atomic<int> send_ack_diff_; // Number of sends - number of acks received. Not allowed to exceed buffer_count.
147  std::unique_ptr<boost::thread> ack_listen_thread_; // Thread to listen for ack messages on the sender
148 
149 private: // methods
150  CopyStatus sendFragment_(Fragment&& frag, size_t timeout_usec);
151 
152  CopyStatus sendData_(const void* buf, size_t bytes, size_t tmo, bool isHeader = false);
153 
154  CopyStatus sendData_(const struct iovec* iov, int iovcnt, size_t tmo, bool isHeader = false);
155 
156 #if USE_ACKS
157  void receive_acks_();
158  void send_ack_(int fd);
159 #endif
160 
161  // Sender is responsible for connecting to receiver
162  void connect_();
163 
164  void reconnect_();
165 
166  int disconnect_receive_socket_(int fd, std::string msg = "");
167 
168  // Receiver should listen for connections
169  void start_listen_thread_();
170  static void listen_(int port, size_t rcvbuf);
171 
172  size_t getConnectedFDCount(int source_rank)
173  {
174  std::unique_lock<std::mutex> lk(connected_fd_mutex_);
175  return connected_fds_.count(source_rank) ? connected_fds_[source_rank].size() : 0;
176  }
177 };
178 
179 #endif // TCPSocketTransfer_hh
bool isRunning() override
Determine whether the TransferInterface plugin is able to send/receive data.
virtual int source_rank() const
Get the source rank for this TransferInterface instance.
Role role() const
Get the TransferInterface::Role of this TransferInterface.
CopyStatus transfer_fragment_reliable_mode(Fragment &&frag) override
Transfer a Fragment to the destination. This should be reliable, if the underlying transport mechanis...
int receiveFragmentData(RawDataType *destination, size_t wordCount) override
Receive the body of a Fragment to the given destination pointer.
TCPSocketTransfer(fhicl::ParameterSet const &ps, Role role)
TCPSocketTransfer Constructor.
int receiveFragmentHeader(detail::RawFragmentHeader &header, size_t receiveTimeout) override
Receive a Fragment Header from the transport mechanism.
void flush_buffers() override
Flush any in-flight data. This should be used by the receiver after the receive loop has ended...
Role
Used to determine if a TransferInterface is a Sender or Receiver.
This interface defines the functions used to transfer data between artdaq applications.
TransferInterface implementation plugin that sends data using TCP sockets.
This header is sent by the TCPSocket_transfer to allow for more efficient writev calls.
Definition: SRSockets.hh:15
std::map< int, std::string > hostMap_t
The host_map is a map associating ranks with artdaq::DestinationInfo objects.
Definition: HostMap.hh:39
CopyStatus transfer_fragment_min_blocking_mode(Fragment const &frag, size_t timeout_usec) override
Transfer a Fragment to the destination. May not necessarily be reliable, but will not block longer th...
CopyStatus
Returned from the send functions, this enumeration describes the possible return codes. If an exception occurs, it will be thrown and should be handled normally.