artdaq  v3_02_00
TCPSocketTransfer.hh
1 #ifndef TCPSocketTransfer_hh
2 #define TCPSocketTransfer_hh
3 // This file (TCPSocketTransfer.hh) was created by Ron Rechenmacher <ron@fnal.gov> on
4 // Sep 14, 2016. "TERMS AND CONDITIONS" governing this file are in the README
5 // or COPYING file. If you do not have such a file, one can be obtained by
6 // contacting Ron or Fermi Lab in Batavia IL, 60510, phone: 630-840-3000.
7 // $RCSfile: .emacs.gnu,v $
8 // rev="$Revision: 1.30 $$Date: 2016/03/01 14:27:27 $";
9 
10 // C Includes
11 #include <stdint.h> // uint64_t
12 #include <sys/uio.h> // iovec
13 
14 // C++ Includes
15 #include <condition_variable>
16 #include <boost/thread.hpp>
17 
18 // Products includes
19 #include "fhiclcpp/fwd.h"
20 
21 // artdaq Includes
22 #include "artdaq/TransferPlugins/TransferInterface.hh"
23 #include "artdaq/TransferPlugins/detail/SRSockets.hh"
24 #include "artdaq/TransferPlugins/detail/Timeout.hh" // Timeout
25 #include "artdaq-core/Data/Fragment.hh"
26 #include "artdaq/TransferPlugins/detail/HostMap.hh"
27 
28 namespace artdaq
29 {
30  class TCPSocketTransfer;
31 }
32 
37 {
38 public:
57  TCPSocketTransfer(fhicl::ParameterSet const& ps, Role role);
58 
59  virtual ~TCPSocketTransfer() noexcept;
60 
67  int receiveFragmentHeader(detail::RawFragmentHeader& header, size_t receiveTimeout) override;
68 
75  int receiveFragmentData(RawDataType* destination, size_t wordCount) override;
76 
83  CopyStatus copyFragment(Fragment& frag, size_t timeout_usec) override { return sendFragment_(std::move(frag), timeout_usec); }
84 
90  CopyStatus moveFragment(Fragment&& frag) override { return sendFragment_(std::move(frag), 0); }
91 
96  bool isRunning() override;
97 private:
98 
99  static std::atomic<int> listen_thread_refcount_;
100  static std::mutex listen_thread_mutex_;
101  static std::unique_ptr<boost::thread> listen_thread_;
102  static std::map<int, std::set<int>> connected_fds_;
103  static std::mutex connected_fd_mutex_;
104  int send_fd_;
105  int active_receive_fd_;
106  int last_active_receive_fd_;
107 
108  union
109  {
110  MessHead mh;
111  uint8_t mha[sizeof(MessHead)];
112  };
113 
114  enum class SocketState
115  {
116  Metadata,
117  Data
118  };
119 
120  size_t rcvbuf_;
121  size_t sndbuf_;
122  size_t send_retry_timeout_us_;
123 
124  hostMap_t hostMap_;
125 
126  volatile unsigned connect_state : 1; // 0=not "connected" (initial msg not sent)
127  unsigned blocking : 1; // compatible with bool (true/false)
128 
129 
130  Timeout tmo_;
131  bool stats_connect_stop_;
132  boost::thread stats_connect_thread_;
133  std::condition_variable stopstatscv_;
134  std::mutex stopstatscvm_; // protects 'stopcv'
135 
136  bool timeoutMessageArmed_; // don't repeatedly print about the send fd not being open...
137  size_t not_connected_count_; // Number of times returned RECV_TIMEOUT because no receive sockets open
138  size_t receive_err_threshold_; // Number of times TO print RECV_TIMEOUT before starting to return DATA_END
139  size_t receive_err_wait_us_; // Amount of time to wait if there are no connected receive sockets
140 
141 private: // methods
142  CopyStatus sendFragment_(Fragment&& frag, size_t timeout_usec);
143 
144  CopyStatus sendData_(const void* buf, size_t bytes, size_t tmo);
145 
146  CopyStatus sendData_(const struct iovec* iov, int iovcnt, size_t tmo);
147 
148  // Thread to drive reconnect_ requests
149  void stats_connect_();
150 
151  // Sender is responsible for connecting to receiver
152  void connect_();
153 
154  void reconnect_();
155 
156  // Receiver should listen for connections
157  void start_listen_thread_();
158  static void listen_(int port, size_t rcvbuf);
159 
160  int calculate_port_() const {
161  return destination_rank() + ((partition_number_ % 22) * 1000) + 10000;
162  }
163 
164  size_t getConnectedFDCount(int source_rank)
165  {
166  std::unique_lock<std::mutex> lk(connected_fd_mutex_);
167  return connected_fds_.count(source_rank) ? connected_fds_[source_rank].size() : 0;
168  }
169 };
170 
171 #endif // TCPSocketTransfer_hh
bool isRunning() override
Determine whether the TransferInterface plugin is able to send/receive data.
virtual int source_rank() const
Get the source rank for this TransferInterface instance.
Role role() const
Get the TransferInterface::Role of this TransferInterface.
CopyStatus copyFragment(Fragment &frag, size_t timeout_usec) override
Copy a Fragment to the destination. Same implementation as moveFragment, as TCP is always reliable...
The Timeout class performs registered actions at specified intervals.
Definition: Timeout.hh:22
const short partition_number_
The partition number of the DAQ.
int receiveFragmentData(RawDataType *destination, size_t wordCount) override
Receive the body of a Fragment to the given destination pointer.
TCPSocketTransfer(fhicl::ParameterSet const &ps, Role role)
TCPSocketTransfer Constructor.
int receiveFragmentHeader(detail::RawFragmentHeader &header, size_t receiveTimeout) override
Receive a Fragment Header from the transport mechanism.
Role
Used to determine if a TransferInterface is a Sender or Receiver.
CopyStatus moveFragment(Fragment &&frag) override
Move a Fragment to the destination.
This interface defines the functions used to transfer data between artdaq applications.
virtual int destination_rank() const
Get the destination rank for this TransferInterface instance.
TransferInterface implementation plugin that sends data using TCP sockets.
This header is sent by the TCPSocket_transfer to allow for more efficient writev calls.
Definition: SRSockets.hh:15
CopyStatus
Returned from the send functions, this enumeration describes the possible return codes. If an exception occurs, it will be thrown and should be handled normally.