$treeview $search $mathjax $extrastylesheet
artdaq
v3_04_00
$projectbrief
|
$projectbrief
|
$searchbox |
00001 #ifndef ARTDAQ_DAQRATE_DATASENDERMANAGER_HH 00002 #define ARTDAQ_DAQRATE_DATASENDERMANAGER_HH 00003 00004 #include <map> 00005 #include <set> 00006 #include <memory> 00007 #include <netinet/in.h> 00008 00009 #include "fhiclcpp/fwd.h" 00010 00011 #include "artdaq-core/Data/Fragment.hh" 00012 #include "artdaq/TransferPlugins/TransferInterface.hh" 00013 #include "artdaq/DAQrate/detail/FragCounter.hh" 00014 #include "artdaq-utilities/Plugins/MetricManager.hh" 00015 #include "artdaq/DAQrate/detail/RoutingPacket.hh" 00016 #include "artdaq/TransferPlugins/detail/HostMap.hh" 00017 #include "fhiclcpp/types/Atom.h" 00018 #include "fhiclcpp/types/OptionalTable.h" 00019 #include "fhiclcpp/types/TableFragment.h" 00020 00021 namespace artdaq 00022 { 00023 class DataSenderManager; 00024 } 00025 00030 class artdaq::DataSenderManager 00031 { 00032 public: 00038 struct RoutingTableConfig 00039 { 00041 fhicl::Atom<bool> use_routing_master{ fhicl::Name{ "use_routing_master"}, fhicl::Comment{ "True if using the Routing Master"}, false }; 00043 fhicl::Atom<int> table_port{ fhicl::Name{ "table_update_port"}, fhicl::Comment{ "Port that table updates should arrive on" },35556 }; 00045 fhicl::Atom<std::string> table_address{ fhicl::Name{ "table_update_address"}, fhicl::Comment{ "Address that table updates should arrive on" }, "227.128.12.28" }; 00047 fhicl::Atom<int> ack_port{ fhicl::Name{ "table_acknowledge_port" },fhicl::Comment{ "Port that acknowledgements should be sent to" },35557 }; 00049 fhicl::Atom<std::string> ack_address{ fhicl::Name{ "routing_master_hostname"}, fhicl::Comment{ "Host that acknowledgements should be sent to" },"localhost" }; 00051 fhicl::Atom<int> routing_timeout_ms{ fhicl::Name{"routing_timeout_ms"}, fhicl::Comment{"Time to wait (in ms) for a routing table update if the table is exhausted"}, 1000 }; 00053 fhicl::Atom<int> routing_retry_count{ fhicl::Name{"routing_retry_count"}, fhicl::Comment{"Number of times to retry getting destination from routing table"}, 5 }; 00055 fhicl::Atom<size_t> routing_table_max_size{ fhicl::Name{"routing_table_max_size"}, fhicl::Comment{"Maximum number of entries in the routing table"}, 1000 }; 00056 }; 00057 00061 struct DestinationsConfig 00062 { 00064 fhicl::OptionalTable<artdaq::TransferInterface::Config> dest{ fhicl::Name{"d1"}, fhicl::Comment{"Configuration for transfer to destination"} }; 00065 }; 00066 00070 struct Config 00071 { 00073 fhicl::Atom<bool> broadcast_sends{ fhicl::Name{"broadcast_sends"}, fhicl::Comment{"Send all Fragments to all destinations"}, false }; 00075 fhicl::Atom<bool> nonblocking_sends{ fhicl::Name{"nonblocking_sends"}, fhicl::Comment{"Whether sends should block. Used for DL->DISP connection."}, false }; 00077 fhicl::Atom<size_t> send_timeout_us{ fhicl::Name{"send_timeout_usec"}, fhicl::Comment{"Timeout for sends in non-reliable modes (broadcast and nonblocking)"},5000000 }; 00079 fhicl::Atom<size_t> send_retry_count{ fhicl::Name{"send_retry_count"}, fhicl::Comment{"Number of times to retry a send in non-reliable mode"}, 2 }; 00080 fhicl::OptionalTable<RoutingTableConfig> routing_table_config{ fhicl::Name{"routing_table_config"} }; 00081 00082 00083 fhicl::OptionalTable<DestinationsConfig> destinations{ fhicl::Name{"destinations"} }; 00084 fhicl::TableFragment<artdaq::HostMap::Config> host_map; 00085 00086 fhicl::Sequence<size_t> enabled_destinations{ fhicl::Name{"enabled_destinations"}, fhicl::Comment{"List of destiantion ranks to activate (must be defined in destinations block)"}, std::vector<size_t>() }; 00087 }; 00088 using Parameters = fhicl::WrappedTable<Config>; 00089 00094 explicit DataSenderManager(const fhicl::ParameterSet& ps); 00095 00099 virtual ~DataSenderManager(); 00100 00106 std::pair<int, TransferInterface::CopyStatus> sendFragment(Fragment&& frag); 00107 00112 size_t count() const; 00113 00119 size_t slotCount(size_t rank) const; 00120 00125 size_t destinationCount() const { return destinations_.size(); } 00126 00131 std::set<int> enabled_destinations() const { return enabled_destinations_; } 00132 00137 size_t GetRoutingTableEntryCount() const; 00138 00143 size_t GetRemainingRoutingTableEntries() const; 00144 00148 void StopSender() { should_stop_ = true; } 00149 00150 private: 00151 00152 // Calculate where the fragment with this sequenceID should go. 00153 int calcDest_(Fragment::sequence_id_t) const; 00154 00155 void setupTableListener_(); 00156 00157 void startTableReceiverThread_(); 00158 00159 void receiveTableUpdatesLoop_(); 00160 private: 00161 00162 std::map<int, std::unique_ptr<artdaq::TransferInterface>> destinations_; 00163 std::unordered_map<int, std::pair<size_t, double>> destination_metric_data_; 00164 std::unordered_map<int, std::chrono::steady_clock::time_point> destination_metric_send_time_; 00165 std::set<int> enabled_destinations_; 00166 00167 detail::FragCounter sent_frag_count_; 00168 00169 bool broadcast_sends_; 00170 bool non_blocking_mode_; 00171 size_t send_timeout_us_; 00172 size_t send_retry_count_; 00173 00174 bool use_routing_master_; 00175 detail::RoutingMasterMode routing_master_mode_; 00176 std::atomic<bool> should_stop_; 00177 int table_port_; 00178 std::string table_address_; 00179 int ack_port_; 00180 std::string ack_address_; 00181 struct sockaddr_in ack_addr_; 00182 int ack_socket_; 00183 int table_socket_; 00184 std::map<Fragment::sequence_id_t, int> routing_table_; 00185 Fragment::sequence_id_t routing_table_last_; 00186 size_t routing_table_max_size_; 00187 mutable std::mutex routing_mutex_; 00188 boost::thread routing_thread_; 00189 mutable std::atomic<size_t> routing_wait_time_; 00190 00191 int routing_timeout_ms_; 00192 int routing_retry_count_; 00193 00194 mutable std::atomic<uint64_t> highest_sequence_id_routed_; 00195 00196 }; 00197 00198 inline 00199 size_t 00200 artdaq::DataSenderManager:: 00201 count() const 00202 { 00203 return sent_frag_count_.count(); 00204 } 00205 00206 inline 00207 size_t 00208 artdaq::DataSenderManager:: 00209 slotCount(size_t rank) const 00210 { 00211 return sent_frag_count_.slotCount(rank); 00212 } 00213 #endif //ARTDAQ_DAQRATE_DATASENDERMANAGER_HH