00001 #ifndef artdaq_Application_MPI2_RoutingMasterCore_hh
00002 #define artdaq_Application_MPI2_RoutingMasterCore_hh
00003
00004 #include <string>
00005 #include <vector>
00006
00007
00008 #include <netinet/in.h>
00009 #include <sys/epoll.h>
00010
00011 #include "fhiclcpp/ParameterSet.h"
00012 #include "canvas/Persistency/Provenance/RunID.h"
00013
00014 #include "artdaq-utilities/Plugins/MetricManager.hh"
00015
00016 #include "artdaq/Application/StatisticsHelper.hh"
00017 #include "artdaq/DAQrate/detail/RoutingPacket.hh"
00018 #include "artdaq/Application/Routing/RoutingMasterPolicy.hh"
00019 #include "artdaq/DAQrate/detail/FragCounter.hh"
00020
00021 namespace artdaq
00022 {
00023 class RoutingMasterCore;
00024 }
00025
00031 class artdaq::RoutingMasterCore
00032 {
00033 public:
00034 static const std::string TABLE_UPDATES_STAT_KEY;
00035 static const std::string TOKENS_RECEIVED_STAT_KEY;
00036
00040 RoutingMasterCore();
00041
00045 RoutingMasterCore(RoutingMasterCore const&) = delete;
00046
00050 ~RoutingMasterCore();
00051
00056 RoutingMasterCore& operator=(RoutingMasterCore const&) = delete;
00057
00081 bool initialize(fhicl::ParameterSet const& pset, uint64_t, uint64_t);
00082
00088 bool start(art::RunID id, uint64_t, uint64_t);
00089
00094 bool stop(uint64_t, uint64_t);
00095
00100 bool pause(uint64_t, uint64_t);
00101
00106 bool resume(uint64_t, uint64_t);
00107
00112 bool shutdown(uint64_t);
00113
00121 bool soft_initialize(fhicl::ParameterSet const& pset, uint64_t timeout, uint64_t timestamp);
00122
00130 bool reinitialize(fhicl::ParameterSet const& pset, uint64_t timeout, uint64_t timestamp);
00131
00137 size_t process_event_table();
00138
00149 void send_event_table(detail::RoutingPacket table);
00150
00156 std::string report(std::string const&) const;
00157
00158 private:
00159 void receive_tokens_();
00160 void start_recieve_token_thread_();
00161
00162 art::RunID run_id_;
00163
00164 fhicl::ParameterSet policy_pset_;
00165 int rt_priority_;
00166
00167 size_t max_table_update_interval_ms_;
00168 size_t max_ack_cycle_count_;
00169 detail::RoutingMasterMode routing_mode_;
00170 size_t current_table_interval_ms_;
00171 std::atomic<size_t> table_update_count_;
00172 std::atomic<size_t> received_token_count_;
00173 std::unordered_map<int,size_t> received_token_counter_;
00174
00175 std::vector<int> sender_ranks_;
00176 size_t num_receivers_;
00177
00178 std::unique_ptr<RoutingMasterPolicy> policy_;
00179
00180 std::atomic<bool> shutdown_requested_;
00181 std::atomic<bool> stop_requested_;
00182 std::atomic<bool> pause_requested_;
00183
00184
00185 artdaq::StatisticsHelper statsHelper_;
00186
00187 std::string buildStatisticsString_() const;
00188
00189 artdaq::MetricManager metricMan_;
00190
00191 void sendMetrics_();
00192
00193
00194
00195
00196 int receive_token_port_;
00197 int send_tables_port_;
00198 int receive_acks_port_;
00199 std::string send_tables_address_;
00200 std::string receive_address_;
00201 struct sockaddr_in receive_addr_;
00202 struct sockaddr_in send_tables_addr_;
00203 std::vector<epoll_event> receive_ack_events_;
00204 std::vector<epoll_event> receive_token_events_;
00205 std::unordered_map<int, std::string> receive_token_addrs_;
00206 int token_epoll_fd_;
00207
00208
00209 int token_socket_;
00210 int table_socket_;
00211 int ack_socket_;
00212 mutable std::mutex request_mutex_;
00213 boost::thread ev_token_receive_thread_;
00214
00215 };
00216
00217 #endif