9 #define TRACE_NAME "MetricManager"
11 #include "artdaq-utilities/Plugins/MetricManager.hh"
12 #include "artdaq-utilities/Plugins/makeMetricPlugin.hh"
13 #include "fhiclcpp/ParameterSet.h"
16 #include <boost/exception/all.hpp>
23 , missed_metric_calls_(0)
24 , metric_queue_max_size_(10000)
39 TLOG_INFO(
"MetricManager") <<
"Configuring metrics with parameter set:\n" << pset.to_string() << TLOG_ENDL;
41 std::vector<std::string> names = pset.get_pset_names();
43 for (
auto name : names)
45 if (name ==
"metric_queue_size")
47 metric_queue_max_size_ = pset.get<
size_t>(
"metric_queue_size");
53 TLOG_DEBUG(
"MetricManager") <<
"Constructing metric plugin with name " << name << TLOG_ENDL;
54 fhicl::ParameterSet plugin_pset = pset.get<fhicl::ParameterSet>(name);
56 plugin_pset.get<std::string>(
"metricPluginType",
""), plugin_pset));
58 catch (
const cet::exception& e)
60 TLOG_ERROR(
"MetricManager") <<
"Exception caught in MetricManager::initialize, error loading plugin with name " << name <<
61 ", cet::exception object caught:" << e.explain_self() << TLOG_ENDL;
63 catch (
const boost::exception& e)
65 TLOG_ERROR(
"MetricManager") <<
"Exception caught in MetricManager::initialize, error loading plugin with name " << name <<
66 ", boost::exception object caught: " << boost::diagnostic_information(e) << TLOG_ENDL;
68 catch (
const std::exception& e)
70 TLOG_ERROR(
"MetricManager") <<
"Exception caught in MetricManager::initialize, error loading plugin with name " << name <<
71 ", std::exception caught: " << e.what() << TLOG_ENDL;
75 TLOG_ERROR(
"MetricManager") <<
"Unknown Exception caught in MetricManager::initialize, error loading plugin with name " << name << TLOG_ENDL;
87 TLOG_DEBUG(
"MetricManager") <<
"Starting MetricManager" << TLOG_ENDL;
88 for (
auto& metric : metric_plugins_)
92 metric->startMetrics();
93 TLOG_INFO(
"MetricManager") <<
"Metric Plugin " << metric->getLibName() <<
" started." << TLOG_ENDL;
98 TLOG_ERROR(
"MetricManager") <<
99 "Exception caught in MetricManager::do_start(), error starting plugin with name " <<
100 metric->getLibName() << TLOG_ENDL;
110 TLOG_DEBUG(
"MetricManager") <<
"Stopping Metrics" << TLOG_ENDL;
112 metric_cv_.notify_all();
113 TLOG_DEBUG(
"MetricManager") <<
"Joining Metric-Sending thread" << TLOG_ENDL;
114 if (metric_sending_thread_.joinable()) metric_sending_thread_.join();
115 TLOG_DEBUG(
"MetricManager") <<
"do_stop Complete" << TLOG_ENDL;
124 initialize(pset, prefix);
129 TLOG_DEBUG(
"MetricManager") <<
"MetricManager is shutting down..." << TLOG_ENDL;
134 for (
auto& i : metric_plugins_)
138 std::string name = i->getLibName();
140 TLOG_DEBUG(
"MetricManager") <<
"Metric Plugin " << name <<
" shutdown." << TLOG_ENDL;
144 TLOG_ERROR(
"MetricManager") <<
145 "Exception caught in MetricManager::shutdown(), error shutting down metric with name " <<
146 i->getLibName() << TLOG_ENDL;
149 initialized_ =
false;
155 if (!initialized_) { TLOG_WARNING(
"MetricManager") <<
"Attempted to send metric when MetricManager has not yet been initialized!" << TLOG_ENDL; }
156 else if (!running_) { TLOG_WARNING(
"MetricManager") <<
"Attempted to send metric when MetricManager stopped!" << TLOG_ENDL; }
159 if (metric_queue_.size() < metric_queue_max_size_)
161 std::unique_ptr<MetricData> metric(
new MetricData(name, value, unit, level, mode, metricPrefix, useNameOverride));
163 std::unique_lock<std::mutex> lk(metric_queue_mutex_);
164 metric_queue_.emplace(std::move(metric));
169 TLOG_ARB(10,
"MetricManager") <<
"Rejecting metric because queue full" << TLOG_ENDL;
170 missed_metric_calls_++;
172 metric_cv_.notify_all();
178 if (!initialized_) { TLOG_WARNING(
"MetricManager") <<
"Attempted to send metric when MetricManager has not yet been initialized!" << TLOG_ENDL; }
179 else if (!running_) { TLOG_WARNING(
"MetricManager") <<
"Attempted to send metric when MetricManager stopped!" << TLOG_ENDL; }
182 if (metric_queue_.size() < metric_queue_max_size_)
184 std::unique_ptr<MetricData> metric(
new MetricData(name, value, unit, level, mode, metricPrefix, useNameOverride));
186 std::unique_lock<std::mutex> lk(metric_queue_mutex_);
187 metric_queue_.emplace(std::move(metric));
192 TLOG_ARB(10,
"MetricManager") <<
"Rejecting metric because queue full" << TLOG_ENDL;
193 missed_metric_calls_++;
195 metric_cv_.notify_all();
201 if (!initialized_) { TLOG_WARNING(
"MetricManager") <<
"Attempted to send metric when MetricManager has not yet been initialized!" << TLOG_ENDL; }
202 else if (!running_) { TLOG_WARNING(
"MetricManager") <<
"Attempted to send metric when MetricManager stopped!" << TLOG_ENDL; }
205 if (metric_queue_.size() < metric_queue_max_size_)
207 std::unique_ptr<MetricData> metric(
new MetricData(name, value, unit, level, mode, metricPrefix, useNameOverride));
209 std::unique_lock<std::mutex> lk(metric_queue_mutex_);
210 metric_queue_.emplace(std::move(metric));
215 TLOG_ARB(10,
"MetricManager") <<
"Rejecting metric because queue full" << TLOG_ENDL;
216 missed_metric_calls_++;
218 metric_cv_.notify_all();
224 if (!initialized_) { TLOG_WARNING(
"MetricManager") <<
"Attempted to send metric when MetricManager has not yet been initialized!" << TLOG_ENDL; }
225 else if (!running_) { TLOG_WARNING(
"MetricManager") <<
"Attempted to send metric when MetricManager stopped!" << TLOG_ENDL; }
228 if (metric_queue_.size() < metric_queue_max_size_)
230 std::unique_ptr<MetricData> metric(
new MetricData(name, value, unit, level, mode, metricPrefix, useNameOverride));
232 std::unique_lock<std::mutex> lk(metric_queue_mutex_);
233 metric_queue_.emplace(std::move(metric));
238 TLOG_ARB(10,
"MetricManager") <<
"Rejecting metric because queue full" << TLOG_ENDL;
239 missed_metric_calls_++;
241 metric_cv_.notify_all();
247 if (!initialized_) { TLOG_WARNING(
"MetricManager") <<
"Attempted to send metric when MetricManager has not yet been initialized!" << TLOG_ENDL; }
248 else if (!running_) { TLOG_WARNING(
"MetricManager") <<
"Attempted to send metric when MetricManager stopped!" << TLOG_ENDL; }
251 if (metric_queue_.size() < metric_queue_max_size_)
253 std::unique_ptr<MetricData> metric(
new MetricData(name, value, unit, level, mode, metricPrefix, useNameOverride));
255 std::unique_lock<std::mutex> lk(metric_queue_mutex_);
256 metric_queue_.emplace(std::move(metric));
261 TLOG_ARB(10,
"MetricManager") <<
"Rejecting metric because queue full" << TLOG_ENDL;
262 missed_metric_calls_++;
264 metric_cv_.notify_all();
268 void artdaq::MetricManager::startMetricLoop_()
270 if (metric_sending_thread_.joinable()) metric_sending_thread_.join();
271 TLOG_INFO(
"MetricManager") <<
"Starting Metric Sending Thread" << TLOG_ENDL;
272 boost::thread::attributes attrs;
273 attrs.set_stack_size(4096 * 200);
274 metric_sending_thread_ = boost::thread(attrs, boost::bind(&MetricManager::sendMetricLoop_,
this));
277 void artdaq::MetricManager::sendMetricLoop_()
279 auto last_send_time = std::chrono::steady_clock::time_point();
282 while (metric_queue_.empty() && running_)
284 std::unique_lock<std::mutex> lk(metric_mutex_);
285 metric_cv_.wait_for(lk, std::chrono::milliseconds(100));
286 auto now = std::chrono::steady_clock::now();
287 if (std::chrono::duration_cast<std::chrono::milliseconds>(now - last_send_time).count() > metric_send_interval_ms_)
289 for (
auto& metric : metric_plugins_) { metric->sendMetrics(); }
290 last_send_time = now;
294 auto temp_list = std::queue<std::unique_ptr<MetricData>>();
296 std::unique_lock<std::mutex> lk(metric_queue_mutex_);
297 temp_list.swap(metric_queue_);
298 temp_list.emplace(
new MetricData(
"Metric Calls", temp_list.size(),
"metrics", 4,
MetricMode::Accumulate,
"",
false));
299 auto missed = missed_metric_calls_.exchange(0);
301 temp_list.emplace(
new MetricData(
"Missed Metric Calls", missed,
"metrics", 4,
MetricMode::Accumulate,
"",
false));
302 TLOG_TRACE(
"MetricManager") <<
"There are " << temp_list.size() <<
" Metric Calls to process (missed " << missed <<
")" << TLOG_ENDL;
305 while (temp_list.size() > 0)
307 auto data_ = std::move(temp_list.front());
310 if (!data_->UseNameOverride)
312 if (data_->MetricPrefix.size() > 0)
314 data_->Name = prefix_ +
"." + data_->MetricPrefix +
"." + data_->Name;
318 data_->Name = prefix_ +
"." + data_->Name;
322 for (
auto& metric : metric_plugins_)
324 if (metric->getRunLevel() >= data_->Level)
328 metric->addMetricData(*data_);
329 last_send_time = std::chrono::steady_clock::now();
333 TLOG_ERROR(
"MetricManager") <<
334 "Error in MetricManager::sendMetric: error sending value to metric plugin with name "
335 << metric->getLibName() << TLOG_ENDL;
342 for (
auto& metric : metric_plugins_)
346 metric->stopMetrics();
347 TLOG_DEBUG(
"MetricManager") <<
"Metric Plugin " << metric->getLibName() <<
" stopped." << TLOG_ENDL;
351 TLOG_ERROR(
"MetricManager") <<
352 "Exception caught in MetricManager::do_stop(), error stopping plugin with name " <<
353 metric->getLibName() << TLOG_ENDL;
356 TLOG_DEBUG(
"MetricManager") <<
"MetricManager has been stopped." << TLOG_ENDL;
void reinitialize(fhicl::ParameterSet const &pset, std::string prefix="")
Reinitialize all MetricPlugin Instances.
void shutdown()
Call the destructors for all configured MetricPlugin instances.
void initialize(fhicl::ParameterSet const &pset, std::string prefix="")
Initialize the MetricPlugin instances.
void sendMetric(std::string const &name, std::string const &value, std::string const &unit, int level, MetricMode mode, std::string const &metricPrefix="", bool useNameOverride=false)
Send a metric with the given parameters to any MetricPlugins with a threshold level >= to level...
std::unique_ptr< MetricPlugin > makeMetricPlugin(std::string const &generator_plugin_spec, fhicl::ParameterSet const &ps)
Load a given MetricPlugin and return a pointer to it.
MetricMode
The Mode of the metric indicates how multiple metric values should be combined within a reporting int...
MetricManager()
Construct an instance of the MetricManager class.
void do_start()
Perform startup actions for each configured MetricPlugin.
void do_stop()
Stop sending metrics to the MetricPlugin instances.
virtual ~MetricManager() noexcept
MetricManager destructor.
Report the sum of all values. Use for counters to report accurate results.
Small structure used to hold a metric data point before sending to the metric plugins ...
void do_resume()
Resume metric sending. Currently a No-Op.
void do_pause()
Pause metric sending. Currently a No-Op.