00001 #ifndef artdaq_Application_MPI2_RoutingMasterCore_hh
00002 #define artdaq_Application_MPI2_RoutingMasterCore_hh
00003
00004 #include <string>
00005 #include <vector>
00006
00007
00008 #include <netinet/in.h>
00009 #include <sys/epoll.h>
00010
00011 #include "fhiclcpp/ParameterSet.h"
00012 #include "canvas/Persistency/Provenance/RunID.h"
00013
00014 #include "artdaq-utilities/Plugins/MetricManager.hh"
00015
00016 #include "artdaq/Application/StatisticsHelper.hh"
00017 #include "artdaq/Application/Routing/RoutingPacket.hh"
00018 #include "artdaq/Application/Routing/RoutingMasterPolicy.hh"
00019 #include "artdaq/DAQrate/detail/FragCounter.hh"
00020
00021 namespace artdaq
00022 {
00023 class RoutingMasterCore;
00024 }
00025
00031 class artdaq::RoutingMasterCore
00032 {
00033 public:
00034 static const std::string TABLE_UPDATES_STAT_KEY;
00035 static const std::string TOKENS_RECEIVED_STAT_KEY;
00036
00042 RoutingMasterCore(int rank, std::string name);
00043
00047 RoutingMasterCore(RoutingMasterCore const&) = delete;
00048
00052 ~RoutingMasterCore();
00053
00058 RoutingMasterCore& operator=(RoutingMasterCore const&) = delete;
00059
00083 bool initialize(fhicl::ParameterSet const& pset, uint64_t, uint64_t);
00084
00090 bool start(art::RunID id, uint64_t, uint64_t);
00091
00096 bool stop(uint64_t, uint64_t);
00097
00102 bool pause(uint64_t, uint64_t);
00103
00108 bool resume(uint64_t, uint64_t);
00109
00114 bool shutdown(uint64_t);
00115
00123 bool soft_initialize(fhicl::ParameterSet const& pset, uint64_t timeout, uint64_t timestamp);
00124
00132 bool reinitialize(fhicl::ParameterSet const& pset, uint64_t timeout, uint64_t timestamp);
00133
00139 size_t process_event_table();
00140
00151 void send_event_table(detail::RoutingPacket table);
00152
00158 std::string report(std::string const&) const;
00159
00160 private:
00161 void receive_tokens_();
00162 void start_recieve_token_thread_();
00163
00164 art::RunID run_id_;
00165 std::string name_;
00166
00167 fhicl::ParameterSet policy_pset_;
00168 int rt_priority_;
00169
00170 size_t max_table_update_interval_ms_;
00171 size_t max_ack_cycle_count_;
00172 detail::RoutingMasterMode routing_mode_;
00173 size_t current_table_interval_ms_;
00174 std::atomic<size_t> table_update_count_;
00175 std::atomic<size_t> received_token_count_;
00176 std::unordered_map<int,size_t> received_token_counter_;
00177
00178 std::vector<int> sender_ranks_;
00179 size_t num_receivers_;
00180
00181 std::unique_ptr<RoutingMasterPolicy> policy_;
00182
00183 std::atomic<bool> shutdown_requested_;
00184 std::atomic<bool> stop_requested_;
00185 std::atomic<bool> pause_requested_;
00186
00187
00188 artdaq::StatisticsHelper statsHelper_;
00189
00190 std::string buildStatisticsString_() const;
00191
00192 artdaq::MetricManager metricMan_;
00193
00194 void sendMetrics_();
00195
00196
00197
00198
00199 int receive_token_port_;
00200 int send_tables_port_;
00201 int receive_acks_port_;
00202 std::string send_tables_address_;
00203 std::string receive_address_;
00204 struct sockaddr_in receive_addr_;
00205 struct sockaddr_in send_tables_addr_;
00206 std::vector<epoll_event> receive_ack_events_;
00207 std::vector<epoll_event> receive_token_events_;
00208 std::unordered_map<int, std::string> receive_token_addrs_;
00209 int token_epoll_fd_;
00210
00211
00212 int token_socket_;
00213 int table_socket_;
00214 int ack_socket_;
00215 mutable std::mutex request_mutex_;
00216 std::thread ev_token_receive_thread_;
00217
00218 };
00219
00220 #endif