1 #ifndef ARTDAQ_DAQRATE_DATASENDERMANAGER_HH
2 #define ARTDAQ_DAQRATE_DATASENDERMANAGER_HH
4 #include <netinet/in.h>
9 #include "fhiclcpp/fwd.h"
11 #include "artdaq-core/Data/Fragment.hh"
12 #include "artdaq-utilities/Plugins/MetricManager.hh"
13 #include "artdaq/DAQrate/detail/FragCounter.hh"
14 #include "artdaq/DAQrate/detail/RoutingPacket.hh"
15 #include "artdaq/TransferPlugins/TransferInterface.hh"
16 #include "artdaq/TransferPlugins/detail/HostMap.hh"
17 #include "fhiclcpp/types/Atom.h"
18 #include "fhiclcpp/types/OptionalTable.h"
19 #include "fhiclcpp/types/TableFragment.h"
22 class DataSenderManager;
40 fhicl::Atom<bool>
use_routing_master{fhicl::Name{
"use_routing_master"}, fhicl::Comment{
"True if using the Routing Master"},
false};
42 fhicl::Atom<int>
table_port{fhicl::Name{
"table_update_port"}, fhicl::Comment{
"Port that table updates should arrive on"}, 35556};
44 fhicl::Atom<std::string>
table_address{fhicl::Name{
"table_update_address"}, fhicl::Comment{
"Address that table updates should arrive on"},
"227.128.12.28"};
46 fhicl::Atom<std::string>
table_multicast_interface{fhicl::Name{
"table_update_multicast_interface"}, fhicl::Comment{
"Network interface that table updates should arrive on"},
"localhost"};
48 fhicl::Atom<int>
ack_port{fhicl::Name{
"table_acknowledge_port"}, fhicl::Comment{
"Port that acknowledgements should be sent to"}, 35557};
50 fhicl::Atom<std::string>
ack_address{fhicl::Name{
"routing_master_hostname"}, fhicl::Comment{
"Host that acknowledgements should be sent to"},
"localhost"};
52 fhicl::Atom<int>
routing_timeout_ms{fhicl::Name{
"routing_timeout_ms"}, fhicl::Comment{
"Time to wait (in ms) for a routing table update if the table is exhausted"}, 1000};
54 fhicl::Atom<int>
routing_retry_count{fhicl::Name{
"routing_retry_count"}, fhicl::Comment{
"Number of times to retry getting destination from routing table"}, 5};
56 fhicl::Atom<size_t>
routing_table_max_size{fhicl::Name{
"routing_table_max_size"}, fhicl::Comment{
"Maximum number of entries in the routing table"}, 1000};
65 fhicl::OptionalTable<artdaq::TransferInterface::Config>
dest{fhicl::Name{
"d1"}, fhicl::Comment{
"Configuration for transfer to destination"}};
74 fhicl::Atom<bool>
broadcast_sends{fhicl::Name{
"broadcast_sends"}, fhicl::Comment{
"Send all Fragments to all destinations"},
false};
76 fhicl::Atom<bool>
nonblocking_sends{fhicl::Name{
"nonblocking_sends"}, fhicl::Comment{
"Whether sends should block. Used for DL->DISP connection."},
false};
78 fhicl::Atom<size_t>
send_timeout_us{fhicl::Name{
"send_timeout_usec"}, fhicl::Comment{
"Timeout for sends in non-reliable modes (broadcast and nonblocking)"}, 5000000};
80 fhicl::Atom<size_t>
send_retry_count{fhicl::Name{
"send_retry_count"}, fhicl::Comment{
"Number of times to retry a send in non-reliable mode"}, 2};
82 fhicl::OptionalTable<DestinationsConfig>
destinations{fhicl::Name{
"destinations"}};
85 fhicl::TableFragment<artdaq::HostMap::Config>
host_map;
86 fhicl::Sequence<size_t> enabled_destinations{fhicl::Name{
"enabled_destinations"}, fhicl::Comment{
"List of destiantion ranks to activate (must be defined in destinations block)"}, std::vector<size_t>()};
108 std::pair<int, TransferInterface::CopyStatus>
sendFragment(Fragment&& frag);
114 size_t count()
const;
166 int calcDest_(Fragment::sequence_id_t)
const;
168 void setupTableListener_();
170 void startTableReceiverThread_();
172 void receiveTableUpdatesLoop_();
175 std::map<int, std::unique_ptr<artdaq::TransferInterface>> destinations_;
176 std::set<int> enabled_destinations_;
180 bool broadcast_sends_;
181 bool non_blocking_mode_;
182 size_t send_timeout_us_;
183 size_t send_retry_count_;
185 bool use_routing_master_;
187 std::atomic<bool> should_stop_;
189 std::string table_address_;
190 std::string table_multicast_interface_;
192 std::string ack_address_;
193 struct sockaddr_in ack_addr_;
196 std::map<Fragment::sequence_id_t, int> routing_table_;
197 std::map<Fragment::sequence_id_t, size_t> sent_sequence_id_count_;
198 Fragment::sequence_id_t routing_table_last_;
199 size_t routing_table_max_size_;
200 mutable std::mutex routing_mutex_;
201 boost::thread routing_thread_;
202 mutable std::atomic<size_t> routing_wait_time_;
204 int routing_timeout_ms_;
205 int routing_retry_count_;
207 mutable std::atomic<uint64_t> highest_sequence_id_routed_;
214 return sent_frag_count_.
count();
223 #endif //ARTDAQ_DAQRATE_DATASENDERMANAGER_HH
fhicl::TableFragment< artdaq::HostMap::Config > host_map
void RemoveRoutingTableEntry(Fragment::sequence_id_t seq)
Remove the given sequence ID from the routing table and sent_count lists.
RoutingMasterMode
Mode indicating whether the RoutingMaster is routing events by Sequence ID or by Send Count...
size_t destinationCount() const
Get the number of configured destinations.
fhicl::WrappedTable< Config > Parameters
Used for ParameterSet validation (if desired)
fhicl::Atom< bool > use_routing_master
"use_routing_master" (Default: false): True if using the Routing Master
Sends Fragment objects using TransferInterface plugins. Uses Routing Tables if confgiured, otherwise will Round-Robin Fragments to the destinations.
fhicl::Atom< int > ack_port
"table_acknowledge_port" (Default: 35557): Port that acknowledgements should be sent to ...
virtual ~DataSenderManager()
DataSenderManager Destructor.
fhicl::Atom< size_t > routing_table_max_size
"routing_table_max_size" (Default: 1000): Maximum number of entries in the routing table ...
size_t slotCount(size_t rank) const
Get the count of Fragment objects sent by this DataSenderManager to a given destination.
fhicl::OptionalTable< artdaq::TransferInterface::Config > dest
Example Configuration for transfer to destination. See artdaq::TransferInterface::Config.
Keep track of the count of Fragments received from a set of sources.
size_t GetSentSequenceIDCount(Fragment::sequence_id_t seq)
Get the number of Fragments sent with a given Sequence ID.
fhicl::Atom< size_t > send_timeout_us
"send_timeout_usec" (Default: 5000000 (5 seconds): Timeout for sends in non-reliable modes (broadcast...
fhicl::Atom< std::string > table_multicast_interface
"table_update_multicast_interface" (Default: "localhost"): Network interface that table updates shoul...
fhicl::OptionalTable< DestinationsConfig > destinations
std::pair< int, TransferInterface::CopyStatus > sendFragment(Fragment &&frag)
Send the given Fragment. Return the rank of the destination to which the Fragment was sent...
size_t GetRemainingRoutingTableEntries() const
Gets the number of sends remaining in the routing table, in case other parts of the system want to us...
fhicl::Atom< int > table_port
"table_update_port" (Default: 35556): Port that table updates should arrive on
DataSenderManager(const fhicl::ParameterSet &ps)
DataSenderManager Constructor.
fhicl::Atom< size_t > send_retry_count
"send_retry_count" (Default: 2): Number of times to retry a send in non-reliable mode ...
fhicl::Atom< bool > broadcast_sends
"broadcast_sends" (Default: false): Send all Fragments to all destinations
size_t GetRoutingTableEntryCount() const
Gets the current size of the Routing Table, in case other parts of the system want to use this inform...
size_t count() const
Return the count of Fragment objects sent by this DataSenderManagerq.
fhicl::Atom< std::string > table_address
"table_update_address" (Default: "227.128.12.28"): Address that table updates should arrive on ...
fhicl::Atom< int > routing_retry_count
"routing_retry_count" (Default: 5): Number of times to retry calculating destination before giving up...
Configuration for transfers to destinations
size_t slotCount(size_t slot) const
Get the current count for the requested slot.
Configuration of DataSenderManager. May be used for parameter validation
Configuration for Routing table reception
void StopSender()
Stop the DataSenderManager, aborting any sends in progress.
std::set< int > enabled_destinations() const
Get the list of enabled destinations.
size_t count() const
Get the total number of Fragments received.
fhicl::OptionalTable< RoutingTableConfig > routing_table_config
fhicl::Atom< int > routing_timeout_ms
"routing_timeout_ms" (Default: 1000): Time to wait for a routing table update if the table is exhaust...
fhicl::Atom< std::string > ack_address
"routing_master_hostname" (Default: "localhost"): Host that acknowledgements should be sent to ...
fhicl::Atom< bool > nonblocking_sends
"nonblocking_sends" (Default: false): If true, will use non-reliable mode of TransferInterface plugin...