artdaq  v3_03_00
TCPSocketTransfer.hh
1 #ifndef TCPSocketTransfer_hh
2 #define TCPSocketTransfer_hh
3 // This file (TCPSocketTransfer.hh) was created by Ron Rechenmacher <ron@fnal.gov> on
4 // Sep 14, 2016. "TERMS AND CONDITIONS" governing this file are in the README
5 // or COPYING file. If you do not have such a file, one can be obtained by
6 // contacting Ron or Fermi Lab in Batavia IL, 60510, phone: 630-840-3000.
7 // $RCSfile: .emacs.gnu,v $
8 // rev="$Revision: 1.30 $$Date: 2016/03/01 14:27:27 $";
9 
10 // C Includes
11 #include <stdint.h> // uint64_t
12 #include <sys/uio.h> // iovec
13 
14 // C++ Includes
15 #include <condition_variable>
16 #include <boost/thread.hpp>
17 
18 // Products includes
19 #include "fhiclcpp/fwd.h"
20 
21 // artdaq Includes
22 #include "artdaq/TransferPlugins/TransferInterface.hh"
23 #include "artdaq/TransferPlugins/detail/SRSockets.hh"
24 #include "artdaq/TransferPlugins/detail/Timeout.hh" // Timeout
25 #include "artdaq-core/Data/Fragment.hh"
26 #include "artdaq/TransferPlugins/detail/HostMap.hh"
27 
28 #ifndef USE_ACKS
29 #define USE_ACKS 0
30 #endif
31 
32 namespace artdaq
33 {
34  class TCPSocketTransfer;
35 }
36 
41 {
42 public:
61  TCPSocketTransfer(fhicl::ParameterSet const& ps, Role role);
62 
63  virtual ~TCPSocketTransfer() noexcept;
64 
71  int receiveFragmentHeader(detail::RawFragmentHeader& header, size_t receiveTimeout) override;
72 
79  int receiveFragmentData(RawDataType* destination, size_t wordCount) override;
80 
87  CopyStatus copyFragment(Fragment& frag, size_t timeout_usec) override { return sendFragment_(std::move(frag), timeout_usec); }
88 
94  CopyStatus moveFragment(Fragment&& frag) override { return sendFragment_(std::move(frag), 0); }
95 
100  bool isRunning() override;
101 private:
102 
103  static std::atomic<int> listen_thread_refcount_;
104  static std::mutex listen_thread_mutex_;
105  static std::unique_ptr<boost::thread> listen_thread_;
106  static std::map<int, std::set<int>> connected_fds_;
107  static std::mutex connected_fd_mutex_;
108  int send_fd_;
109  int active_receive_fd_;
110  int last_active_receive_fd_;
111  short active_revents_;
112 
113  union
114  {
115  MessHead mh;
116  uint8_t mha[sizeof(MessHead)];
117  };
118 
119  enum class SocketState
120  {
121  Metadata,
122  Data
123  };
124 
125  size_t rcvbuf_;
126  size_t sndbuf_;
127  size_t send_retry_timeout_us_;
128 
129  hostMap_t hostMap_;
130 
131  volatile unsigned connect_state : 1; // 0=not "connected" (initial msg not sent)
132  unsigned blocking : 1; // compatible with bool (true/false)
133 
134  bool timeoutMessageArmed_; // don't repeatedly print about the send fd not being open...
135  std::chrono::steady_clock::time_point last_recv_time_; // Time of last successful receive
136  double receive_disconnected_wait_s_; // How long to wait between messages before returning DATA_END
137  size_t receive_err_wait_us_; // Amount of time to wait if there are no connected receive sockets
138  std::atomic<bool> receive_socket_has_been_connected_; // Whether the receiver has ever been connected to a sender
139  std::atomic<int> send_ack_diff_; // Number of sends - number of acks received. Not allowed to exceed buffer_count.
140  std::unique_ptr<boost::thread> ack_listen_thread_; // Thread to listen for ack messages on the sender
141 
142 private: // methods
143  CopyStatus sendFragment_(Fragment&& frag, size_t timeout_usec);
144 
145  CopyStatus sendData_(const void* buf, size_t bytes, size_t tmo, bool isHeader = false);
146 
147  CopyStatus sendData_(const struct iovec* iov, int iovcnt, size_t tmo, bool isHeader = false);
148 
149 #if USE_ACKS
150  void receive_acks_();
151  void send_ack_(int fd);
152 #endif
153 
154  // Sender is responsible for connecting to receiver
155  void connect_();
156 
157  void reconnect_();
158 
159  int disconnect_receive_socket_(int fd, std::string msg = "");
160 
161  // Receiver should listen for connections
162  void start_listen_thread_();
163  static void listen_(int port, size_t rcvbuf);
164 
165  size_t getConnectedFDCount(int source_rank)
166  {
167  std::unique_lock<std::mutex> lk(connected_fd_mutex_);
168  return connected_fds_.count(source_rank) ? connected_fds_[source_rank].size() : 0;
169  }
170 };
171 
172 #endif // TCPSocketTransfer_hh
bool isRunning() override
Determine whether the TransferInterface plugin is able to send/receive data.
virtual int source_rank() const
Get the source rank for this TransferInterface instance.
Role role() const
Get the TransferInterface::Role of this TransferInterface.
CopyStatus copyFragment(Fragment &frag, size_t timeout_usec) override
Copy a Fragment to the destination. Same implementation as moveFragment, as TCP is always reliable...
int receiveFragmentData(RawDataType *destination, size_t wordCount) override
Receive the body of a Fragment to the given destination pointer.
TCPSocketTransfer(fhicl::ParameterSet const &ps, Role role)
TCPSocketTransfer Constructor.
int receiveFragmentHeader(detail::RawFragmentHeader &header, size_t receiveTimeout) override
Receive a Fragment Header from the transport mechanism.
Role
Used to determine if a TransferInterface is a Sender or Receiver.
CopyStatus moveFragment(Fragment &&frag) override
Move a Fragment to the destination.
This interface defines the functions used to transfer data between artdaq applications.
TransferInterface implementation plugin that sends data using TCP sockets.
This header is sent by the TCPSocket_transfer to allow for more efficient writev calls.
Definition: SRSockets.hh:15
std::map< int, std::string > hostMap_t
The host_map is a map associating ranks with artdaq::DestinationInfo objects.
Definition: HostMap.hh:39
CopyStatus
Returned from the send functions, this enumeration describes the possible return codes. If an exception occurs, it will be thrown and should be handled normally.