00001 #ifndef artdaq_Application_MPI2_RoutingMasterCore_hh
00002 #define artdaq_Application_MPI2_RoutingMasterCore_hh
00003
00004 #include <string>
00005 #include <vector>
00006
00007
00008 #include <netinet/in.h>
00009 #include <sys/epoll.h>
00010
00011 #include "fhiclcpp/ParameterSet.h"
00012 #include "canvas/Persistency/Provenance/RunID.h"
00013
00014 #include "artdaq-utilities/Plugins/MetricManager.hh"
00015
00016 #include "artdaq/Application/StatisticsHelper.hh"
00017 #include "artdaq/DAQrate/detail/RoutingPacket.hh"
00018 #include "artdaq/Application/Routing/RoutingMasterPolicy.hh"
00019 #include "artdaq/DAQrate/detail/FragCounter.hh"
00020
00021 namespace artdaq
00022 {
00023 class RoutingMasterCore;
00024 }
00025
00031 class artdaq::RoutingMasterCore
00032 {
00033 public:
00034 static const std::string TABLE_UPDATES_STAT_KEY;
00035 static const std::string TOKENS_RECEIVED_STAT_KEY;
00036
00040 RoutingMasterCore();
00041
00045 RoutingMasterCore(RoutingMasterCore const&) = delete;
00046
00050 ~RoutingMasterCore();
00051
00056 RoutingMasterCore& operator=(RoutingMasterCore const&) = delete;
00057
00081 bool initialize(fhicl::ParameterSet const& pset, uint64_t, uint64_t);
00082
00088 bool start(art::RunID id, uint64_t, uint64_t);
00089
00094 bool stop(uint64_t, uint64_t);
00095
00100 bool pause(uint64_t, uint64_t);
00101
00106 bool resume(uint64_t, uint64_t);
00107
00112 bool shutdown(uint64_t);
00113
00121 bool soft_initialize(fhicl::ParameterSet const& pset, uint64_t timeout, uint64_t timestamp);
00122
00130 bool reinitialize(fhicl::ParameterSet const& pset, uint64_t timeout, uint64_t timestamp);
00131
00136 void process_event_table();
00137
00148 void send_event_table(detail::RoutingPacket table);
00149
00155 std::string report(std::string const&) const;
00156
00161 size_t get_update_count() const { return table_update_count_; }
00162
00163 private:
00164 void receive_tokens_();
00165 void start_recieve_token_thread_();
00166
00167 art::RunID run_id_;
00168
00169 fhicl::ParameterSet policy_pset_;
00170 int rt_priority_;
00171
00172 size_t max_table_update_interval_ms_;
00173 size_t max_ack_cycle_count_;
00174 detail::RoutingMasterMode routing_mode_;
00175 std::atomic<size_t> current_table_interval_ms_;
00176 std::atomic<size_t> table_update_count_;
00177 std::atomic<size_t> received_token_count_;
00178 std::unordered_map<int, size_t> received_token_counter_;
00179
00180 std::vector<int> sender_ranks_;
00181 size_t num_receivers_;
00182
00183 std::unique_ptr<RoutingMasterPolicy> policy_;
00184
00185 std::atomic<bool> shutdown_requested_;
00186 std::atomic<bool> stop_requested_;
00187 std::atomic<bool> pause_requested_;
00188
00189
00190 artdaq::StatisticsHelper statsHelper_;
00191
00192 std::string buildStatisticsString_() const;
00193
00194 void sendMetrics_();
00195
00196
00197
00198
00199 int receive_token_port_;
00200 int send_tables_port_;
00201 int receive_acks_port_;
00202 std::string send_tables_address_;
00203 std::string receive_address_;
00204 struct sockaddr_in send_tables_addr_;
00205 std::vector<epoll_event> receive_ack_events_;
00206 std::vector<epoll_event> receive_token_events_;
00207 std::unordered_map<int, std::string> receive_token_addrs_;
00208 int token_epoll_fd_;
00209
00210
00211 int token_socket_;
00212 int table_socket_;
00213 int ack_socket_;
00214 mutable std::mutex request_mutex_;
00215 boost::thread ev_token_receive_thread_;
00216
00217 };
00218
00219 #endif