artdaq  v3_02_01
TCPSocketTransfer.hh
1 #ifndef TCPSocketTransfer_hh
2 #define TCPSocketTransfer_hh
3 // This file (TCPSocketTransfer.hh) was created by Ron Rechenmacher <ron@fnal.gov> on
4 // Sep 14, 2016. "TERMS AND CONDITIONS" governing this file are in the README
5 // or COPYING file. If you do not have such a file, one can be obtained by
6 // contacting Ron or Fermi Lab in Batavia IL, 60510, phone: 630-840-3000.
7 // $RCSfile: .emacs.gnu,v $
8 // rev="$Revision: 1.30 $$Date: 2016/03/01 14:27:27 $";
9 
10 // C Includes
11 #include <stdint.h> // uint64_t
12 #include <sys/uio.h> // iovec
13 
14 // C++ Includes
15 #include <condition_variable>
16 #include <boost/thread.hpp>
17 
18 // Products includes
19 #include "fhiclcpp/fwd.h"
20 
21 // artdaq Includes
22 #include "artdaq/TransferPlugins/TransferInterface.hh"
23 #include "artdaq/TransferPlugins/detail/SRSockets.hh"
24 #include "artdaq/TransferPlugins/detail/Timeout.hh" // Timeout
25 #include "artdaq-core/Data/Fragment.hh"
26 #include "artdaq/TransferPlugins/detail/HostMap.hh"
27 
28 #define USE_ACKS 0
29 
30 namespace artdaq
31 {
32  class TCPSocketTransfer;
33 }
34 
39 {
40 public:
59  TCPSocketTransfer(fhicl::ParameterSet const& ps, Role role);
60 
61  virtual ~TCPSocketTransfer() noexcept;
62 
69  int receiveFragmentHeader(detail::RawFragmentHeader& header, size_t receiveTimeout) override;
70 
77  int receiveFragmentData(RawDataType* destination, size_t wordCount) override;
78 
85  CopyStatus copyFragment(Fragment& frag, size_t timeout_usec) override { return sendFragment_(std::move(frag), timeout_usec); }
86 
92  CopyStatus moveFragment(Fragment&& frag) override { return sendFragment_(std::move(frag), 0); }
93 
98  bool isRunning() override;
99 private:
100 
101  static std::atomic<int> listen_thread_refcount_;
102  static std::mutex listen_thread_mutex_;
103  static std::unique_ptr<boost::thread> listen_thread_;
104  static std::map<int, std::set<int>> connected_fds_;
105  static std::mutex connected_fd_mutex_;
106  int send_fd_;
107  int active_receive_fd_;
108  int last_active_receive_fd_;
109 
110  union
111  {
112  MessHead mh;
113  uint8_t mha[sizeof(MessHead)];
114  };
115 
116  enum class SocketState
117  {
118  Metadata,
119  Data
120  };
121 
122  size_t rcvbuf_;
123  size_t sndbuf_;
124  size_t send_retry_timeout_us_;
125 
126  hostMap_t hostMap_;
127 
128  volatile unsigned connect_state : 1; // 0=not "connected" (initial msg not sent)
129  unsigned blocking : 1; // compatible with bool (true/false)
130 
131  bool timeoutMessageArmed_; // don't repeatedly print about the send fd not being open...
132  size_t not_connected_count_; // Number of times returned RECV_TIMEOUT because no receive sockets open
133  size_t receive_err_threshold_; // Number of times TO print RECV_TIMEOUT before starting to return DATA_END
134  size_t receive_err_wait_us_; // Amount of time to wait if there are no connected receive sockets
135 
136 private: // methods
137  CopyStatus sendFragment_(Fragment&& frag, size_t timeout_usec);
138 
139  CopyStatus sendData_(const void* buf, size_t bytes, size_t tmo);
140 
141  CopyStatus sendData_(const struct iovec* iov, int iovcnt, size_t tmo);
142 
143 #if USE_ACKS
144  void receive_ack_(int fd);
145  void send_ack_(int fd);
146 #endif
147 
148  // Sender is responsible for connecting to receiver
149  void connect_();
150 
151  void reconnect_();
152 
153  int disconnect_receive_socket_(int fd, std::string msg = "");
154 
155  // Receiver should listen for connections
156  void start_listen_thread_();
157  static void listen_(int port, size_t rcvbuf);
158 
159  int calculate_port_() const {
160  return destination_rank() + ((partition_number_ % 22) * 1000) + 10000;
161  }
162 
163  size_t getConnectedFDCount(int source_rank)
164  {
165  std::unique_lock<std::mutex> lk(connected_fd_mutex_);
166  return connected_fds_.count(source_rank) ? connected_fds_[source_rank].size() : 0;
167  }
168 };
169 
170 #endif // TCPSocketTransfer_hh
bool isRunning() override
Determine whether the TransferInterface plugin is able to send/receive data.
virtual int source_rank() const
Get the source rank for this TransferInterface instance.
Role role() const
Get the TransferInterface::Role of this TransferInterface.
CopyStatus copyFragment(Fragment &frag, size_t timeout_usec) override
Copy a Fragment to the destination. Same implementation as moveFragment, as TCP is always reliable...
const short partition_number_
The partition number of the DAQ.
int receiveFragmentData(RawDataType *destination, size_t wordCount) override
Receive the body of a Fragment to the given destination pointer.
TCPSocketTransfer(fhicl::ParameterSet const &ps, Role role)
TCPSocketTransfer Constructor.
int receiveFragmentHeader(detail::RawFragmentHeader &header, size_t receiveTimeout) override
Receive a Fragment Header from the transport mechanism.
std::map< int, DestinationInfo > hostMap_t
The host_map is a map associating ranks with artdaq::DestinationInfo objects.
Definition: HostMap.hh:49
Role
Used to determine if a TransferInterface is a Sender or Receiver.
CopyStatus moveFragment(Fragment &&frag) override
Move a Fragment to the destination.
This interface defines the functions used to transfer data between artdaq applications.
virtual int destination_rank() const
Get the destination rank for this TransferInterface instance.
TransferInterface implementation plugin that sends data using TCP sockets.
This header is sent by the TCPSocket_transfer to allow for more efficient writev calls.
Definition: SRSockets.hh:15
CopyStatus
Returned from the send functions, this enumeration describes the possible return codes. If an exception occurs, it will be thrown and should be handled normally.