artdaq  v3_09_00
DataSenderManager.hh
1 #ifndef ARTDAQ_DAQRATE_DATASENDERMANAGER_HH
2 #define ARTDAQ_DAQRATE_DATASENDERMANAGER_HH
3 
4 #include <netinet/in.h>
5 #include <map>
6 #include <memory>
7 #include <set>
8 
9 #include "fhiclcpp/fwd.h"
10 
11 #include "artdaq-core/Data/Fragment.hh"
12 #include "artdaq-utilities/Plugins/MetricManager.hh"
13 #include "artdaq/DAQrate/detail/FragCounter.hh"
14 #include "artdaq/DAQrate/detail/RoutingPacket.hh"
15 #include "artdaq/TransferPlugins/TransferInterface.hh"
16 #include "artdaq/TransferPlugins/detail/HostMap.hh"
17 #include "fhiclcpp/types/Atom.h"
18 #include "fhiclcpp/types/OptionalTable.h"
19 #include "fhiclcpp/types/TableFragment.h"
20 
21 namespace artdaq {
22 class DataSenderManager;
23 }
24 
30 {
31 public:
38  {
40  fhicl::Atom<bool> use_routing_manager{fhicl::Name{"use_routing_manager"}, fhicl::Comment{"True if using the Routing Manager"}, false};
42  fhicl::Atom<int> table_port{fhicl::Name{"table_update_port"}, fhicl::Comment{"Port that table updates should arrive on"}, 35556};
44  fhicl::Atom<std::string> table_address{fhicl::Name{"table_update_address"}, fhicl::Comment{"Address that table updates should arrive on"}, "227.128.12.28"};
46  fhicl::Atom<std::string> table_multicast_interface{fhicl::Name{"table_update_multicast_interface"}, fhicl::Comment{"Network interface that table updates should arrive on"}, "localhost"};
48  fhicl::Atom<int> ack_port{fhicl::Name{"table_acknowledge_port"}, fhicl::Comment{"Port that acknowledgements should be sent to"}, 35557};
50  fhicl::Atom<std::string> ack_address{fhicl::Name{"routing_manager_hostname"}, fhicl::Comment{"Host that acknowledgements should be sent to"}, "localhost"};
52  fhicl::Atom<int> routing_timeout_ms{fhicl::Name{"routing_timeout_ms"}, fhicl::Comment{"Time to wait (in ms) for a routing table update if the table is exhausted"}, 1000};
54  fhicl::Atom<int> routing_retry_count{fhicl::Name{"routing_retry_count"}, fhicl::Comment{"Number of times to retry getting destination from routing table"}, 5};
56  fhicl::Atom<size_t> routing_table_max_size{fhicl::Name{"routing_table_max_size"}, fhicl::Comment{"Maximum number of entries in the routing table"}, 1000};
57  };
58 
63  {
65  fhicl::OptionalTable<artdaq::TransferInterface::Config> dest{fhicl::Name{"d1"}, fhicl::Comment{"Configuration for transfer to destination"}};
66  };
67 
71  struct Config
72  {
74  fhicl::Atom<bool> broadcast_sends{fhicl::Name{"broadcast_sends"}, fhicl::Comment{"Send all Fragments to all destinations"}, false};
76  fhicl::Atom<bool> nonblocking_sends{fhicl::Name{"nonblocking_sends"}, fhicl::Comment{"Whether sends should block. Used for DL->DISP connection."}, false};
78  fhicl::Atom<size_t> send_timeout_us{fhicl::Name{"send_timeout_usec"}, fhicl::Comment{"Timeout for sends in non-reliable modes (broadcast and nonblocking)"}, 5000000};
80  fhicl::Atom<size_t> send_retry_count{fhicl::Name{"send_retry_count"}, fhicl::Comment{"Number of times to retry a send in non-reliable mode"}, 2};
81  fhicl::OptionalTable<RoutingTableConfig> routing_table_config{fhicl::Name{"routing_table_config"}};
82  fhicl::OptionalTable<DestinationsConfig> destinations{fhicl::Name{"destinations"}};
86  fhicl::TableFragment<artdaq::HostMap::Config> host_map;
88  fhicl::Sequence<size_t> enabled_destinations{fhicl::Name{"enabled_destinations"}, fhicl::Comment{"List of destiantion ranks to activate (must be defined in destinations block)"}, std::vector<size_t>()};
89  };
91  using Parameters = fhicl::WrappedTable<Config>;
92 
97  explicit DataSenderManager(const fhicl::ParameterSet& ps);
98 
102  virtual ~DataSenderManager();
103 
109  std::pair<int, TransferInterface::CopyStatus> sendFragment(Fragment&& frag);
110 
115  size_t count() const;
116 
122  size_t slotCount(size_t rank) const;
123 
128  size_t destinationCount() const { return destinations_.size(); }
129 
134  std::set<int> enabled_destinations() const { return enabled_destinations_; }
135 
140  size_t GetRoutingTableEntryCount() const;
141 
146  size_t GetRemainingRoutingTableEntries() const;
147 
151  void StopSender() { should_stop_ = true; }
152 
157  void RemoveRoutingTableEntry(Fragment::sequence_id_t seq);
163  size_t GetSentSequenceIDCount(Fragment::sequence_id_t seq);
164 
165 private:
166  DataSenderManager(DataSenderManager const&) = delete;
168  DataSenderManager& operator=(DataSenderManager const&) = delete;
169  DataSenderManager& operator=(DataSenderManager&&) = delete;
170 
171  // Calculate where the fragment with this sequenceID should go.
172  int calcDest_(Fragment::sequence_id_t) const;
173 
174  void setupTableListener_();
175 
176  void startTableReceiverThread_();
177 
178  void receiveTableUpdatesLoop_();
179 
180 private:
181  std::map<int, std::unique_ptr<artdaq::TransferInterface>> destinations_;
182  std::set<int> enabled_destinations_;
183 
184  detail::FragCounter sent_frag_count_;
185 
186  bool broadcast_sends_;
187  bool non_blocking_mode_;
188  size_t send_timeout_us_;
189  size_t send_retry_count_;
190 
191  bool use_routing_manager_;
192  detail::RoutingManagerMode routing_manager_mode_;
193  std::atomic<bool> should_stop_;
194  int table_port_;
195  std::string table_address_;
196  std::string table_multicast_interface_;
197  int ack_port_;
198  std::string ack_address_;
199  struct sockaddr_in ack_addr_;
200  int ack_socket_;
201  int table_socket_;
202  std::map<Fragment::sequence_id_t, int> routing_table_;
203  std::map<Fragment::sequence_id_t, size_t> sent_sequence_id_count_;
204  Fragment::sequence_id_t routing_table_last_;
205  size_t routing_table_max_size_;
206  mutable std::mutex routing_mutex_;
207  boost::thread routing_thread_;
208  mutable std::atomic<size_t> routing_wait_time_;
209 
210  int routing_timeout_ms_;
211  int routing_retry_count_;
212 
213  mutable std::atomic<uint64_t> highest_sequence_id_routed_;
214 };
215 
216 inline size_t
218  count() const
219 {
220  return sent_frag_count_.count();
221 }
222 
223 inline size_t
225  slotCount(size_t rank) const
226 {
227  return sent_frag_count_.slotCount(rank);
228 }
229 #endif //ARTDAQ_DAQRATE_DATASENDERMANAGER_HH
fhicl::TableFragment< artdaq::HostMap::Config > host_map
Optional host_map configuration (Can also be specified in each DestinationsConfig entry...
void RemoveRoutingTableEntry(Fragment::sequence_id_t seq)
Remove the given sequence ID from the routing table and sent_count lists.
size_t destinationCount() const
Get the number of configured destinations.
fhicl::WrappedTable< Config > Parameters
Used for ParameterSet validation (if desired)
Sends Fragment objects using TransferInterface plugins. Uses Routing Tables if confgiured, otherwise will Round-Robin Fragments to the destinations.
fhicl::Atom< int > ack_port
&quot;table_acknowledge_port&quot; (Default: 35557): Port that acknowledgements should be sent to ...
virtual ~DataSenderManager()
DataSenderManager Destructor.
fhicl::Atom< size_t > routing_table_max_size
&quot;routing_table_max_size&quot; (Default: 1000): Maximum number of entries in the routing table ...
fhicl::Atom< bool > use_routing_manager
&quot;use_routing_manager&quot; (Default: false): True if using the Routing Manager
size_t slotCount(size_t rank) const
Get the count of Fragment objects sent by this DataSenderManager to a given destination.
fhicl::OptionalTable< artdaq::TransferInterface::Config > dest
Example Configuration for transfer to destination. See artdaq::TransferInterface::Config.
Keep track of the count of Fragments received from a set of sources.
Definition: FragCounter.hh:18
size_t GetSentSequenceIDCount(Fragment::sequence_id_t seq)
Get the number of Fragments sent with a given Sequence ID.
RoutingManagerMode
Mode indicating whether the RoutingManager is routing events by Sequence ID or by Send Count...
fhicl::Atom< size_t > send_timeout_us
&quot;send_timeout_usec&quot; (Default: 5000000 (5 seconds): Timeout for sends in non-reliable modes (broadcast...
fhicl::Atom< std::string > table_multicast_interface
&quot;table_update_multicast_interface&quot; (Default: &quot;localhost&quot;): Network interface that table updates shoul...
fhicl::OptionalTable< DestinationsConfig > destinations
std::pair< int, TransferInterface::CopyStatus > sendFragment(Fragment &&frag)
Send the given Fragment. Return the rank of the destination to which the Fragment was sent...
size_t GetRemainingRoutingTableEntries() const
Gets the number of sends remaining in the routing table, in case other parts of the system want to us...
fhicl::Atom< int > table_port
&quot;table_update_port&quot; (Default: 35556): Port that table updates should arrive on
DataSenderManager(const fhicl::ParameterSet &ps)
DataSenderManager Constructor.
fhicl::Atom< size_t > send_retry_count
&quot;send_retry_count&quot; (Default: 2): Number of times to retry a send in non-reliable mode ...
fhicl::Atom< bool > broadcast_sends
&quot;broadcast_sends&quot; (Default: false): Send all Fragments to all destinations
size_t GetRoutingTableEntryCount() const
Gets the current size of the Routing Table, in case other parts of the system want to use this inform...
size_t count() const
Return the count of Fragment objects sent by this DataSenderManagerq.
fhicl::Atom< std::string > table_address
&quot;table_update_address&quot; (Default: &quot;227.128.12.28&quot;): Address that table updates should arrive on ...
fhicl::Atom< int > routing_retry_count
&quot;routing_retry_count&quot; (Default: 5): Number of times to retry calculating destination before giving up...
Configuration for transfers to destinations
size_t slotCount(size_t slot) const
Get the current count for the requested slot.
Definition: FragCounter.hh:133
Configuration of DataSenderManager. May be used for parameter validation
fhicl::Sequence< size_t > enabled_destinations
enabled_destinations" (OPTIONAL): If specified, only the destination ranks listed will be enabled...
Configuration for Routing table reception
void StopSender()
Stop the DataSenderManager, aborting any sends in progress.
std::set< int > enabled_destinations() const
Get the list of enabled destinations.
size_t count() const
Get the total number of Fragments received.
Definition: FragCounter.hh:120
fhicl::OptionalTable< RoutingTableConfig > routing_table_config
fhicl::Atom< int > routing_timeout_ms
&quot;routing_timeout_ms&quot; (Default: 1000): Time to wait for a routing table update if the table is exhaust...
fhicl::Atom< std::string > ack_address
&quot;routing_manager_hostname&quot; (Default: &quot;localhost&quot;): Host that acknowledgements should be sent to ...
fhicl::Atom< bool > nonblocking_sends
&quot;nonblocking_sends&quot; (Default: false): If true, will use non-reliable mode of TransferInterface plugin...