1 #ifndef ARTDAQ_DAQRATE_DATASENDERMANAGER_HH
2 #define ARTDAQ_DAQRATE_DATASENDERMANAGER_HH
4 #include <netinet/in.h>
9 #include "fhiclcpp/fwd.h"
11 #include "artdaq-core/Data/Fragment.hh"
12 #include "artdaq-utilities/Plugins/MetricManager.hh"
13 #include "artdaq/DAQrate/detail/FragCounter.hh"
14 #include "artdaq/DAQrate/detail/RoutingPacket.hh"
15 #include "artdaq/TransferPlugins/TransferInterface.hh"
16 #include "artdaq/TransferPlugins/detail/HostMap.hh"
17 #include "fhiclcpp/types/Atom.h"
18 #include "fhiclcpp/types/OptionalTable.h"
19 #include "fhiclcpp/types/TableFragment.h"
22 class DataSenderManager;
40 fhicl::Atom<bool>
use_routing_master{fhicl::Name{
"use_routing_master"}, fhicl::Comment{
"True if using the Routing Master"},
false};
42 fhicl::Atom<int>
table_port{fhicl::Name{
"table_update_port"}, fhicl::Comment{
"Port that table updates should arrive on"}, 35556};
44 fhicl::Atom<std::string>
table_address{fhicl::Name{
"table_update_address"}, fhicl::Comment{
"Address that table updates should arrive on"},
"227.128.12.28"};
46 fhicl::Atom<std::string>
table_multicast_interface{fhicl::Name{
"table_update_multicast_interface"}, fhicl::Comment{
"Network interface that table updates should arrive on"},
"localhost"};
48 fhicl::Atom<int>
ack_port{fhicl::Name{
"table_acknowledge_port"}, fhicl::Comment{
"Port that acknowledgements should be sent to"}, 35557};
50 fhicl::Atom<std::string>
ack_address{fhicl::Name{
"routing_master_hostname"}, fhicl::Comment{
"Host that acknowledgements should be sent to"},
"localhost"};
52 fhicl::Atom<int>
routing_timeout_ms{fhicl::Name{
"routing_timeout_ms"}, fhicl::Comment{
"Time to wait (in ms) for a routing table update if the table is exhausted"}, 1000};
54 fhicl::Atom<int>
routing_retry_count{fhicl::Name{
"routing_retry_count"}, fhicl::Comment{
"Number of times to retry getting destination from routing table"}, 5};
56 fhicl::Atom<size_t>
routing_table_max_size{fhicl::Name{
"routing_table_max_size"}, fhicl::Comment{
"Maximum number of entries in the routing table"}, 1000};
65 fhicl::OptionalTable<artdaq::TransferInterface::Config>
dest{fhicl::Name{
"d1"}, fhicl::Comment{
"Configuration for transfer to destination"}};
74 fhicl::Atom<bool>
broadcast_sends{fhicl::Name{
"broadcast_sends"}, fhicl::Comment{
"Send all Fragments to all destinations"},
false};
76 fhicl::Atom<bool>
nonblocking_sends{fhicl::Name{
"nonblocking_sends"}, fhicl::Comment{
"Whether sends should block. Used for DL->DISP connection."},
false};
78 fhicl::Atom<size_t>
send_timeout_us{fhicl::Name{
"send_timeout_usec"}, fhicl::Comment{
"Timeout for sends in non-reliable modes (broadcast and nonblocking)"}, 5000000};
80 fhicl::Atom<size_t>
send_retry_count{fhicl::Name{
"send_retry_count"}, fhicl::Comment{
"Number of times to retry a send in non-reliable mode"}, 2};
82 fhicl::OptionalTable<DestinationsConfig>
destinations{fhicl::Name{
"destinations"}};
86 fhicl::TableFragment<artdaq::HostMap::Config>
host_map;
88 fhicl::Sequence<size_t>
enabled_destinations{fhicl::Name{
"enabled_destinations"}, fhicl::Comment{
"List of destiantion ranks to activate (must be defined in destinations block)"}, std::vector<size_t>()};
109 std::pair<int, TransferInterface::CopyStatus>
sendFragment(Fragment&& frag);
115 size_t count()
const;
167 int calcDest_(Fragment::sequence_id_t)
const;
169 void setupTableListener_();
171 void startTableReceiverThread_();
173 void receiveTableUpdatesLoop_();
176 std::map<int, std::unique_ptr<artdaq::TransferInterface>> destinations_;
177 std::set<int> enabled_destinations_;
181 bool broadcast_sends_;
182 bool non_blocking_mode_;
183 size_t send_timeout_us_;
184 size_t send_retry_count_;
186 bool use_routing_master_;
188 std::atomic<bool> should_stop_;
190 std::string table_address_;
191 std::string table_multicast_interface_;
193 std::string ack_address_;
194 struct sockaddr_in ack_addr_;
197 std::map<Fragment::sequence_id_t, int> routing_table_;
198 std::map<Fragment::sequence_id_t, size_t> sent_sequence_id_count_;
199 Fragment::sequence_id_t routing_table_last_;
200 size_t routing_table_max_size_;
201 mutable std::mutex routing_mutex_;
202 boost::thread routing_thread_;
203 mutable std::atomic<size_t> routing_wait_time_;
205 int routing_timeout_ms_;
206 int routing_retry_count_;
208 mutable std::atomic<uint64_t> highest_sequence_id_routed_;
215 return sent_frag_count_.
count();
224 #endif //ARTDAQ_DAQRATE_DATASENDERMANAGER_HH
fhicl::TableFragment< artdaq::HostMap::Config > host_map
Optional host_map configuration (Can also be specified in each DestinationsConfig entry...
void RemoveRoutingTableEntry(Fragment::sequence_id_t seq)
Remove the given sequence ID from the routing table and sent_count lists.
RoutingMasterMode
Mode indicating whether the RoutingMaster is routing events by Sequence ID or by Send Count...
size_t destinationCount() const
Get the number of configured destinations.
fhicl::WrappedTable< Config > Parameters
Used for ParameterSet validation (if desired)
fhicl::Atom< bool > use_routing_master
"use_routing_master" (Default: false): True if using the Routing Master
Sends Fragment objects using TransferInterface plugins. Uses Routing Tables if confgiured, otherwise will Round-Robin Fragments to the destinations.
fhicl::Atom< int > ack_port
"table_acknowledge_port" (Default: 35557): Port that acknowledgements should be sent to ...
virtual ~DataSenderManager()
DataSenderManager Destructor.
fhicl::Atom< size_t > routing_table_max_size
"routing_table_max_size" (Default: 1000): Maximum number of entries in the routing table ...
size_t slotCount(size_t rank) const
Get the count of Fragment objects sent by this DataSenderManager to a given destination.
fhicl::OptionalTable< artdaq::TransferInterface::Config > dest
Example Configuration for transfer to destination. See artdaq::TransferInterface::Config.
Keep track of the count of Fragments received from a set of sources.
size_t GetSentSequenceIDCount(Fragment::sequence_id_t seq)
Get the number of Fragments sent with a given Sequence ID.
fhicl::Atom< size_t > send_timeout_us
"send_timeout_usec" (Default: 5000000 (5 seconds): Timeout for sends in non-reliable modes (broadcast...
fhicl::Atom< std::string > table_multicast_interface
"table_update_multicast_interface" (Default: "localhost"): Network interface that table updates shoul...
fhicl::OptionalTable< DestinationsConfig > destinations
std::pair< int, TransferInterface::CopyStatus > sendFragment(Fragment &&frag)
Send the given Fragment. Return the rank of the destination to which the Fragment was sent...
size_t GetRemainingRoutingTableEntries() const
Gets the number of sends remaining in the routing table, in case other parts of the system want to us...
fhicl::Atom< int > table_port
"table_update_port" (Default: 35556): Port that table updates should arrive on
DataSenderManager(const fhicl::ParameterSet &ps)
DataSenderManager Constructor.
fhicl::Atom< size_t > send_retry_count
"send_retry_count" (Default: 2): Number of times to retry a send in non-reliable mode ...
fhicl::Atom< bool > broadcast_sends
"broadcast_sends" (Default: false): Send all Fragments to all destinations
size_t GetRoutingTableEntryCount() const
Gets the current size of the Routing Table, in case other parts of the system want to use this inform...
size_t count() const
Return the count of Fragment objects sent by this DataSenderManagerq.
fhicl::Atom< std::string > table_address
"table_update_address" (Default: "227.128.12.28"): Address that table updates should arrive on ...
fhicl::Atom< int > routing_retry_count
"routing_retry_count" (Default: 5): Number of times to retry calculating destination before giving up...
Configuration for transfers to destinations
size_t slotCount(size_t slot) const
Get the current count for the requested slot.
Configuration of DataSenderManager. May be used for parameter validation
fhicl::Sequence< size_t > enabled_destinations
enabled_destinations" (OPTIONAL): If specified, only the destination ranks listed will be enabled...
Configuration for Routing table reception
void StopSender()
Stop the DataSenderManager, aborting any sends in progress.
std::set< int > enabled_destinations() const
Get the list of enabled destinations.
size_t count() const
Get the total number of Fragments received.
fhicl::OptionalTable< RoutingTableConfig > routing_table_config
fhicl::Atom< int > routing_timeout_ms
"routing_timeout_ms" (Default: 1000): Time to wait for a routing table update if the table is exhaust...
fhicl::Atom< std::string > ack_address
"routing_master_hostname" (Default: "localhost"): Host that acknowledgements should be sent to ...
fhicl::Atom< bool > nonblocking_sends
"nonblocking_sends" (Default: false): If true, will use non-reliable mode of TransferInterface plugin...