00001 #ifndef ARTDAQ_DAQRATE_DATASENDERMANAGER_HH
00002 #define ARTDAQ_DAQRATE_DATASENDERMANAGER_HH
00003
00004 #include <map>
00005 #include <set>
00006 #include <memory>
00007 #include <netinet/in.h>
00008
00009 #include "fhiclcpp/fwd.h"
00010
00011 #include "artdaq-core/Data/Fragment.hh"
00012 #include "artdaq/TransferPlugins/TransferInterface.hh"
00013 #include "artdaq/DAQrate/detail/FragCounter.hh"
00014 #include "artdaq-utilities/Plugins/MetricManager.hh"
00015 #include "artdaq/DAQrate/detail/RoutingPacket.hh"
00016 #include "artdaq/TransferPlugins/detail/HostMap.hh"
00017 #include "fhiclcpp/types/Atom.h"
00018 #include "fhiclcpp/types/OptionalTable.h"
00019 #include "fhiclcpp/types/TableFragment.h"
00020
00021 namespace artdaq
00022 {
00023 class DataSenderManager;
00024 }
00025
00030 class artdaq::DataSenderManager
00031 {
00032 public:
00038 struct RoutingTableConfig
00039 {
00041 fhicl::Atom<bool> use_routing_master{ fhicl::Name{ "use_routing_master"}, fhicl::Comment{ "True if using the Routing Master"}, false };
00043 fhicl::Atom<int> table_port{ fhicl::Name{ "table_update_port"}, fhicl::Comment{ "Port that table updates should arrive on" },35556 };
00045 fhicl::Atom<std::string> table_address{ fhicl::Name{ "table_update_address"}, fhicl::Comment{ "Address that table updates should arrive on" }, "227.128.12.28" };
00047 fhicl::Atom<int> ack_port{ fhicl::Name{ "table_acknowledge_port" },fhicl::Comment{ "Port that acknowledgements should be sent to" },35557 };
00049 fhicl::Atom<std::string> ack_address{ fhicl::Name{ "routing_master_hostname"}, fhicl::Comment{ "Host that acknowledgements should be sent to" },"localhost" };
00051 fhicl::Atom<int> routing_timeout_ms{ fhicl::Name{"routing_timeout_ms"}, fhicl::Comment{"Time to wait (in ms) for a routing table update if the table is exhausted"}, 1000 };
00053 fhicl::Atom<int> routing_retry_count{ fhicl::Name{"routing_retry_count"}, fhicl::Comment{"Number of times to retry getting destination from routing table"}, 5 };
00055 fhicl::Atom<size_t> routing_table_max_size{ fhicl::Name{"routing_table_max_size"}, fhicl::Comment{"Maximum number of entries in the routing table"}, 1000 };
00056 };
00057
00061 struct DestinationsConfig
00062 {
00064 fhicl::OptionalTable<artdaq::TransferInterface::Config> dest{ fhicl::Name{"d1"}, fhicl::Comment{"Configuration for transfer to destination"} };
00065 };
00066
00070 struct Config
00071 {
00073 fhicl::Atom<bool> broadcast_sends{ fhicl::Name{"broadcast_sends"}, fhicl::Comment{"Send all Fragments to all destinations"}, false };
00075 fhicl::Atom<bool> nonblocking_sends{ fhicl::Name{"nonblocking_sends"}, fhicl::Comment{"Whether sends should block. Used for DL->DISP connection."}, false };
00077 fhicl::Atom<size_t> send_timeout_us{ fhicl::Name{"send_timeout_usec"}, fhicl::Comment{"Timeout for sends in non-reliable modes (broadcast and nonblocking)"},5000000 };
00079 fhicl::Atom<size_t> send_retry_count{ fhicl::Name{"send_retry_count"}, fhicl::Comment{"Number of times to retry a send in non-reliable mode"}, 2 };
00080 fhicl::OptionalTable<RoutingTableConfig> routing_table_config{ fhicl::Name{"routing_table_config"} };
00081
00082
00083 fhicl::OptionalTable<DestinationsConfig> destinations{ fhicl::Name{"destinations"} };
00084 fhicl::TableFragment<artdaq::HostMap::Config> host_map;
00085
00086 fhicl::Sequence<size_t> enabled_destinations{ fhicl::Name{"enabled_destinations"}, fhicl::Comment{"List of destiantion ranks to activate (must be defined in destinations block)"}, std::vector<size_t>() };
00087 };
00088 using Parameters = fhicl::WrappedTable<Config>;
00089
00094 explicit DataSenderManager(const fhicl::ParameterSet& ps);
00095
00099 virtual ~DataSenderManager();
00100
00106 std::pair<int, TransferInterface::CopyStatus> sendFragment(Fragment&& frag);
00107
00112 size_t count() const;
00113
00119 size_t slotCount(size_t rank) const;
00120
00125 size_t destinationCount() const { return destinations_.size(); }
00126
00131 std::set<int> enabled_destinations() const { return enabled_destinations_; }
00132
00137 size_t GetRoutingTableEntryCount() const;
00138
00143 size_t GetRemainingRoutingTableEntries() const;
00144
00148 void StopSender() { should_stop_ = true; }
00149
00150 private:
00151
00152
00153 int calcDest_(Fragment::sequence_id_t) const;
00154
00155 void setupTableListener_();
00156
00157 void startTableReceiverThread_();
00158
00159 void receiveTableUpdatesLoop_();
00160 private:
00161
00162 std::map<int, std::unique_ptr<artdaq::TransferInterface>> destinations_;
00163 std::unordered_map<int, std::pair<size_t, double>> destination_metric_data_;
00164 std::unordered_map<int, std::chrono::steady_clock::time_point> destination_metric_send_time_;
00165 std::set<int> enabled_destinations_;
00166
00167 detail::FragCounter sent_frag_count_;
00168
00169 bool broadcast_sends_;
00170 bool non_blocking_mode_;
00171 size_t send_timeout_us_;
00172 size_t send_retry_count_;
00173
00174 bool use_routing_master_;
00175 detail::RoutingMasterMode routing_master_mode_;
00176 std::atomic<bool> should_stop_;
00177 int table_port_;
00178 std::string table_address_;
00179 int ack_port_;
00180 std::string ack_address_;
00181 struct sockaddr_in ack_addr_;
00182 int ack_socket_;
00183 int table_socket_;
00184 std::map<Fragment::sequence_id_t, int> routing_table_;
00185 Fragment::sequence_id_t routing_table_last_;
00186 size_t routing_table_max_size_;
00187 mutable std::mutex routing_mutex_;
00188 boost::thread routing_thread_;
00189 mutable std::atomic<size_t> routing_wait_time_;
00190
00191 int routing_timeout_ms_;
00192 int routing_retry_count_;
00193
00194 mutable std::atomic<uint64_t> highest_sequence_id_routed_;
00195
00196 };
00197
00198 inline
00199 size_t
00200 artdaq::DataSenderManager::
00201 count() const
00202 {
00203 return sent_frag_count_.count();
00204 }
00205
00206 inline
00207 size_t
00208 artdaq::DataSenderManager::
00209 slotCount(size_t rank) const
00210 {
00211 return sent_frag_count_.slotCount(rank);
00212 }
00213 #endif //ARTDAQ_DAQRATE_DATASENDERMANAGER_HH