artdaq_utilities  v1_07_02
MetricManager.cc
1 // MetricManager.cc: MetricManager class implementation file
2 // Author: Eric Flumerfelt
3 // Last Modified: 11/14/2014
4 //
5 // MetricManager loads a user-specified set of plugins, sends them their configuration,
6 // and sends them data as it is recieved. It also maintains the state of the plugins
7 // relative to the application state.
8 
9 #include "tracemf.h"
10 #define TRACE_NAME "MetricManager"
11 
12 #include "artdaq-utilities/Plugins/MetricManager.hh"
13 #include "artdaq-utilities/Plugins/makeMetricPlugin.hh"
14 #include "fhiclcpp/ParameterSet.h"
15 
16 #include <pthread.h>
17 #include <boost/exception/all.hpp>
18 #include <chrono>
19 #include <memory>
20 
22  : metric_plugins_(0)
23  , system_metric_collector_(nullptr)
24  , initialized_(false)
25  , running_(false)
26  , active_(false)
27  , busy_(false)
28  , missed_metric_calls_(0)
29  , metric_calls_(0) {}
30 
31 artdaq::MetricManager::~MetricManager() noexcept { shutdown(); }
32 
33 void artdaq::MetricManager::initialize(fhicl::ParameterSet const& pset, std::string const& prefix)
34 {
35  prefix_ = prefix;
36  if (initialized_)
37  {
38  shutdown();
39  }
40  TLOG(TLVL_INFO) << "Configuring metrics with parameter set: " << pset.to_string();
41 
42  std::vector<std::string> names = pset.get_names();
43 
44  metric_plugins_.clear();
45  bool send_system_metrics = false;
46  bool send_process_metrics = false;
47 
48  for (const auto& name : names)
49  {
50  if (name == "metric_queue_size")
51  {
52  metric_cache_max_size_ = pset.get<size_t>("metric_queue_size");
53  }
54  else if (name == "metric_queue_notify_size")
55  {
56  metric_cache_notify_size_ = pset.get<size_t>("metric_queue_notify_size");
57  }
58  else if (name == "metric_cache_size")
59  {
60  metric_cache_max_size_ = pset.get<size_t>("metric_cache_size");
61  }
62  else if (name == "metric_cache_notify_size")
63  {
64  metric_cache_notify_size_ = pset.get<size_t>("metric_cache_notify_size");
65  }
66  else if (name == "metric_send_maximum_delay_ms")
67  {
68  TLOG(TLVL_INFO) << "Setting metric_send_interval_ms_ to " << pset.get<int>("metric_send_maximum_delay_ms");
69  metric_send_interval_ms_ = pset.get<int>("metric_send_maximum_delay_ms");
70  }
71  else if (name == "metric_holdoff_us")
72  {
73  TLOG(TLVL_INFO) << "Setting metric_holdoff_us_ to " << pset.get<int>("metric_holdoff_us");
74  metric_holdoff_us_ = pset.get<int>("metric_holdoff_us");
75  }
76  else if (name == "send_system_metrics")
77  {
78  send_system_metrics = pset.get<bool>("send_system_metrics");
79  }
80  else if (name == "send_process_metrics")
81  {
82  send_process_metrics = pset.get<bool>("send_process_metrics");
83  }
84  else
85  {
86  try
87  {
88  TLOG(TLVL_DEBUG + 32) << "Constructing metric plugin with name " << name;
89  auto plugin_pset = pset.get<fhicl::ParameterSet>(name);
90  metric_plugins_.push_back(
91  makeMetricPlugin(plugin_pset.get<std::string>("metricPluginType", ""), plugin_pset, prefix_, name));
92  }
93  catch (const cet::exception& e)
94  {
95  TLOG(TLVL_ERROR) << "Exception caught in MetricManager::initialize, error loading plugin with name " << name
96  << ", cet::exception object caught:" << e.explain_self();
97  }
98  catch (const boost::exception& e)
99  {
100  TLOG(TLVL_ERROR) << "Exception caught in MetricManager::initialize, error loading plugin with name " << name
101  << ", boost::exception object caught: " << boost::diagnostic_information(e);
102  }
103  catch (const std::exception& e)
104  {
105  TLOG(TLVL_ERROR) << "Exception caught in MetricManager::initialize, error loading plugin with name " << name
106  << ", std::exception caught: " << e.what();
107  }
108  catch (...)
109  {
110  TLOG(TLVL_ERROR) << "Unknown Exception caught in MetricManager::initialize, error loading plugin with name "
111  << name;
112  }
113  }
114  }
115 
116  if (send_system_metrics || send_process_metrics)
117  {
118  system_metric_collector_ = std::make_unique<SystemMetricCollector>(send_process_metrics, send_system_metrics);
119  }
120 
121  initialized_ = true;
122 }
123 
125 {
126  std::lock_guard<std::mutex> lk(metric_mutex_);
127  if (!running_)
128  {
129  TLOG(TLVL_DEBUG + 32) << "Starting MetricManager";
130  for (auto& metric : metric_plugins_)
131  {
132  if (!metric)
133  {
134  continue;
135  }
136  try
137  {
138  metric->startMetrics();
139  TLOG(TLVL_INFO) << "Metric Plugin " << metric->getLibName() << " started.";
140  active_ = true;
141  }
142  catch (...)
143  {
144  TLOG(TLVL_ERROR) << "Exception caught in MetricManager::do_start(), error starting plugin with name "
145  << metric->getLibName();
146  }
147  }
148  running_ = true;
149  startMetricLoop_();
150  }
151 }
152 
154 {
155  std::unique_lock<std::mutex> lk(metric_mutex_);
156  TLOG(TLVL_DEBUG + 32) << "Stopping Metrics";
157  running_ = false;
158  metric_cv_.notify_all();
159  TLOG(TLVL_DEBUG + 32) << "Joining Metric-Sending thread";
160  lk.unlock();
161  try
162  {
163  if (metric_sending_thread_.joinable())
164  {
165  metric_sending_thread_.join();
166  }
167  }
168  catch (...)
169  {
170  // IGNORED
171  }
172  TLOG(TLVL_DEBUG + 32) << "do_stop Complete";
173 }
174 
176 { /*do_stop();*/
177 }
179 { /*do_start();*/
180 }
181 
182 void artdaq::MetricManager::reinitialize(fhicl::ParameterSet const& pset, std::string const& prefix)
183 {
184  shutdown();
185  initialize(pset, prefix);
186 }
187 
189 {
190  TLOG(TLVL_DEBUG + 32) << "MetricManager is shutting down...";
191  do_stop();
192 
193  std::lock_guard<std::mutex> lk(metric_mutex_);
194  if (initialized_)
195  {
196  initialized_ = false;
197  for (auto& i : metric_plugins_)
198  {
199  try
200  {
201  std::string name = i->getLibName();
202  i.reset(nullptr);
203  TLOG(TLVL_DEBUG + 32) << "Metric Plugin " << name << " shutdown.";
204  }
205  catch (...)
206  {
207  TLOG(TLVL_ERROR) << "Exception caught in MetricManager::shutdown(), error shutting down metric with name "
208  << i->getLibName();
209  }
210  }
211  metric_plugins_.clear();
212  }
213 }
214 
215 void artdaq::MetricManager::sendMetric(std::string const& name, std::string const& value, std::string const& unit,
216  int level, MetricMode mode, std::string const& metricPrefix,
217  bool useNameOverride)
218 {
219  if (!initialized_)
220  {
221  TLOG(TLVL_WARNING) << "Attempted to send metric " << name << " when MetricManager has not yet been initialized!";
222  }
223  else if (!running_)
224  {
225  TLOG(TLVL_INFO) << "Attempted to send metric when MetricManager stopped!";
226  }
227  else if (active_)
228  {
229  {
230  std::lock_guard<std::mutex> lk(metric_cache_mutex_);
231  metric_calls_++;
232  last_metric_received_ = std::chrono::steady_clock::now();
233  auto& cached = metric_cache_[name];
234  if (cached == nullptr)
235  {
236  metric_cache_[name] = std::make_unique<MetricData>(name, value, unit, level, mode, metricPrefix, useNameOverride);
237  }
238  else
239  {
240  auto size = cached->DataPointCount;
241  if (size < metric_cache_max_size_)
242  {
243  if (size >= metric_cache_notify_size_)
244  {
245  TLOG(TLVL_DEBUG + 35) << "Metric cache is at size " << size << " of " << metric_cache_max_size_ << " for metric " << name
246  << ".";
247  }
248  if (mode == MetricMode::LastPoint)
249  {
250  cached->StringValue = value;
251  cached->DataPointCount = 1;
252  }
253  else
254  {
255  cached->StringValue += " " + value;
256  cached->DataPointCount++;
257  }
258  }
259  else
260  {
261  TLOG(TLVL_DEBUG + 36) << "Rejecting metric because queue full";
262  missed_metric_calls_++;
263  }
264  }
265  }
266  metric_cv_.notify_all();
267  }
268 }
269 
270 void artdaq::MetricManager::sendMetric(std::string const& name, int const& value, std::string const& unit, int level,
271  MetricMode mode, std::string const& metricPrefix, bool useNameOverride)
272 {
273  if (!initialized_)
274  {
275  TLOG(TLVL_WARNING) << "Attempted to send metric " << name << " when MetricManager has not yet been initialized!";
276  }
277  else if (!running_)
278  {
279  TLOG(TLVL_INFO) << "Attempted to send metric when MetricManager stopped!";
280  }
281  else if (active_)
282  {
283  {
284  std::lock_guard<std::mutex> lk(metric_cache_mutex_);
285  metric_calls_++;
286  last_metric_received_ = std::chrono::steady_clock::now();
287  auto& cached = metric_cache_[name];
288  if (cached == nullptr)
289  {
290  metric_cache_[name] = std::make_unique<MetricData>(name, value, unit, level, mode, metricPrefix, useNameOverride);
291  }
292  else
293  {
294  auto size = cached->DataPointCount;
295  if (size < metric_cache_max_size_)
296  {
297  if (size >= metric_cache_notify_size_)
298  {
299  TLOG(TLVL_DEBUG + 35) << "Metric cache is at size " << size << " of " << metric_cache_max_size_ << " for metric " << name
300  << ".";
301  }
302  cached->AddPoint(value);
303  }
304  else
305  {
306  TLOG(TLVL_DEBUG + 36) << "Rejecting metric because queue full";
307  missed_metric_calls_++;
308  }
309  }
310  }
311  metric_cv_.notify_all();
312  }
313 }
314 
315 void artdaq::MetricManager::sendMetric(std::string const& name, double const& value, std::string const& unit, int level,
316  MetricMode mode, std::string const& metricPrefix, bool useNameOverride)
317 {
318  if (!initialized_)
319  {
320  TLOG(TLVL_WARNING) << "Attempted to send metric " << name << " when MetricManager has not yet been initialized!";
321  }
322  else if (!running_)
323  {
324  TLOG(TLVL_INFO) << "Attempted to send metric when MetricManager stopped!";
325  }
326  else if (active_)
327  {
328  {
329  std::lock_guard<std::mutex> lk(metric_cache_mutex_);
330  metric_calls_++;
331  last_metric_received_ = std::chrono::steady_clock::now();
332  auto& cached = metric_cache_[name];
333  if (cached == nullptr)
334  {
335  metric_cache_[name] = std::make_unique<MetricData>(name, value, unit, level, mode, metricPrefix, useNameOverride);
336  }
337  else
338  {
339  auto size = cached->DataPointCount;
340  if (size < metric_cache_max_size_)
341  {
342  if (size >= metric_cache_notify_size_)
343  {
344  TLOG(TLVL_DEBUG + 35) << "Metric cache is at size " << size << " of " << metric_cache_max_size_ << " for metric " << name
345  << ".";
346  }
347  cached->AddPoint(value);
348  }
349  else
350  {
351  TLOG(TLVL_DEBUG + 36) << "Rejecting metric because queue full";
352  missed_metric_calls_++;
353  }
354  }
355  }
356  metric_cv_.notify_all();
357  }
358 }
359 
360 void artdaq::MetricManager::sendMetric(std::string const& name, float const& value, std::string const& unit, int level,
361  MetricMode mode, std::string const& metricPrefix, bool useNameOverride)
362 {
363  if (!initialized_)
364  {
365  TLOG(TLVL_WARNING) << "Attempted to send metric " << name << " when MetricManager has not yet been initialized!";
366  }
367  else if (!running_)
368  {
369  TLOG(TLVL_INFO) << "Attempted to send metric when MetricManager stopped!";
370  }
371  else if (active_)
372  {
373  {
374  std::lock_guard<std::mutex> lk(metric_cache_mutex_);
375  metric_calls_++;
376  last_metric_received_ = std::chrono::steady_clock::now();
377  auto& cached = metric_cache_[name];
378  if (cached == nullptr)
379  {
380  metric_cache_[name] = std::make_unique<MetricData>(name, value, unit, level, mode, metricPrefix, useNameOverride);
381  }
382  else
383  {
384  auto size = cached->DataPointCount;
385  if (size < metric_cache_max_size_)
386  {
387  if (size >= metric_cache_notify_size_)
388  {
389  TLOG(TLVL_DEBUG + 35) << "Metric cache is at size " << size << " of " << metric_cache_max_size_ << " for metric " << name
390  << ".";
391  }
392  cached->AddPoint(value);
393  }
394  else
395  {
396  TLOG(TLVL_DEBUG + 36) << "Rejecting metric because queue full";
397  missed_metric_calls_++;
398  }
399  }
400  }
401  metric_cv_.notify_all();
402  }
403 }
404 
405 void artdaq::MetricManager::sendMetric(std::string const& name, uint64_t const& value, std::string const& unit,
406  int level, MetricMode mode, std::string const& metricPrefix,
407  bool useNameOverride)
408 {
409  if (!initialized_)
410  {
411  TLOG(TLVL_WARNING) << "Attempted to send metric " << name << " when MetricManager has not yet been initialized!";
412  }
413  else if (!running_)
414  {
415  TLOG(TLVL_INFO) << "Attempted to send metric when MetricManager stopped!";
416  }
417  else if (active_)
418  {
419  {
420  std::lock_guard<std::mutex> lk(metric_cache_mutex_);
421  metric_calls_++;
422  last_metric_received_ = std::chrono::steady_clock::now();
423  auto& cached = metric_cache_[name];
424  if (cached == nullptr)
425  {
426  metric_cache_[name] = std::make_unique<MetricData>(name, value, unit, level, mode, metricPrefix, useNameOverride);
427  }
428  else
429  {
430  auto size = cached->DataPointCount;
431  if (size < metric_cache_max_size_)
432  {
433  if (size >= metric_cache_notify_size_)
434  {
435  TLOG(TLVL_DEBUG + 35) << "Metric cache is at size " << size << " of " << metric_cache_max_size_ << " for metric " << name
436  << ".";
437  }
438  cached->AddPoint(value);
439  }
440  else
441  {
442  TLOG(TLVL_DEBUG + 36) << "Rejecting metric because queue full";
443  missed_metric_calls_++;
444  }
445  }
446  }
447  metric_cv_.notify_all();
448  }
449 }
450 
451 void artdaq::MetricManager::startMetricLoop_()
452 {
453  if (metric_sending_thread_.joinable())
454  {
455  metric_sending_thread_.join();
456  }
457  boost::thread::attributes attrs;
458  attrs.set_stack_size(4096 * 2000); // 8000 KB
459  TLOG(TLVL_INFO) << "Starting Metric Sending Thread";
460  try
461  {
462  metric_sending_thread_ = boost::thread(attrs, boost::bind(&MetricManager::sendMetricLoop_, this));
463 
464  char tname[16]; // Size 16 - see man page pthread_setname_np(3) and/or prctl(2)
465  snprintf(tname, sizeof(tname) - 1, "%s", "MetricSend"); // NOLINT
466  tname[sizeof(tname) - 1] = '\0'; // assure term. snprintf is not too evil :)
467  auto handle = metric_sending_thread_.native_handle();
468  pthread_setname_np(handle, tname);
469  }
470  catch (const boost::exception& e)
471  {
472  TLOG(TLVL_ERROR) << "Caught boost::exception starting Metric Sending thread: " << boost::diagnostic_information(e)
473  << ", errno=" << errno;
474  std::cerr << "Caught boost::exception starting Metric Sending thread: " << boost::diagnostic_information(e)
475  << ", errno=" << errno << std::endl;
476  exit(5);
477  }
478  TLOG(TLVL_INFO) << "Metric Sending thread started";
479 }
480 
482 {
483  std::lock_guard<std::mutex> lk(metric_cache_mutex_);
484  for (auto& cache_entry : metric_cache_)
485  {
486  if (cache_entry.second->DataPointCount > 0)
487  {
488  return false;
489  }
490  }
491 
492  return true;
493 }
494 
496 {
497  bool pluginsBusy = false;
498 
499  for (auto& p : metric_plugins_)
500  {
501  if (p->metricsPending())
502  {
503  pluginsBusy = true;
504  break;
505  }
506  }
507 
508  TLOG(TLVL_DEBUG + 33) << "Metric queue empty: " << metricQueueEmpty() << ", busy_: " << busy_ << ", Plugins busy: " << pluginsBusy;
509  return !metricQueueEmpty() || busy_ || pluginsBusy;
510 }
511 
512 size_t artdaq::MetricManager::metricQueueSize(std::string const& name)
513 {
514  std::lock_guard<std::mutex> lk(metric_cache_mutex_);
515  size_t size = 0;
516  if (name.empty())
517  {
518  for (auto& q : metric_cache_)
519  {
520  size += q.second->DataPointCount;
521  }
522  }
523  else
524  {
525  if (metric_cache_.count(name) != 0u)
526  {
527  size = metric_cache_[name]->DataPointCount;
528  }
529  }
530 
531  return size;
532 }
533 
534 void artdaq::MetricManager::sendMetricLoop_()
535 {
536  TLOG(TLVL_INFO) << "sendMetricLoop_ START";
537  auto last_send_time = std::chrono::steady_clock::time_point();
538  while (running_)
539  {
540  TLOG(TLVL_DEBUG + 34) << "sendMetricLoop_: Entering Metric input wait loop";
541  while (metricQueueEmpty() && running_)
542  {
543  std::unique_lock<std::mutex> lk(metric_mutex_);
544  metric_cv_.wait_for(lk, std::chrono::milliseconds(100));
545  auto now = std::chrono::steady_clock::now();
546  if (std::chrono::duration_cast<std::chrono::milliseconds>(now - last_send_time).count() >
547  metric_send_interval_ms_)
548  {
549  TLOG(TLVL_DEBUG + 34) << "sendMetricLoop_: Metric send interval exceeded: Sending metrics";
550  {
551  std::unique_lock<std::mutex> lk(metric_cache_mutex_); // last_metric_received_ is protected by metric_cache_mutex_
552  if (std::chrono::duration_cast<std::chrono::microseconds>(now - last_metric_received_).count() < metric_holdoff_us_)
553  {
554  lk.unlock();
555  usleep(metric_holdoff_us_);
556  }
557  }
558  for (auto& metric : metric_plugins_)
559  {
560  if (metric)
561  {
562  metric->sendMetrics();
563  }
564  }
565  last_send_time = now;
566  }
567  }
568  {
569  std::unique_lock<std::mutex> lk(metric_cache_mutex_); // last_metric_received_ is protected by metric_cache_mutex_
570  if (std::chrono::duration_cast<std::chrono::microseconds>(std::chrono::steady_clock::now() - last_metric_received_).count() < metric_holdoff_us_)
571  {
572  lk.unlock();
573  usleep(metric_holdoff_us_);
574  }
575  }
576 
577  TLOG(TLVL_DEBUG + 34) << "sendMetricLoop_: After Metric input wait loop";
578  busy_ = true;
579  auto processing_start = std::chrono::steady_clock::now();
580  auto temp_list = std::list<std::unique_ptr<MetricData>>();
581  {
582  std::lock_guard<std::mutex> lk(metric_cache_mutex_);
583 
584  for (auto& q : metric_cache_)
585  {
586  if (q.second != nullptr && q.second->DataPointCount > 0)
587  {
588  temp_list.emplace_back(new MetricData(*q.second));
589  q.second->Reset();
590  }
591  }
592  }
593 
594  auto calls = metric_calls_.exchange(0);
595  temp_list.emplace_back(
596  new MetricData("Metric Calls", calls, "metrics", 4, MetricMode::Accumulate | MetricMode::Rate, "", false));
597 
598  auto missed = missed_metric_calls_.exchange(0);
599  temp_list.emplace_back(
600  new MetricData("Missed Metric Calls", missed, "metrics", 4, MetricMode::Accumulate | MetricMode::Rate, "", false));
601 
602  TLOG(TLVL_DEBUG + 33) << "There are " << temp_list.size() << " Metrics to process (" << calls << " calls, " << missed
603  << " missed)";
604 
605  if (system_metric_collector_ != nullptr)
606  {
607  TLOG(TLVL_DEBUG + 33) << "Collecting System metrics (CPU, RAM, Network)";
608  auto systemMetrics = system_metric_collector_->SendMetrics();
609  for (auto& m : systemMetrics) { temp_list.emplace_back(std::move(m)); }
610  }
611 
612  TLOG(TLVL_DEBUG + 34) << "sendMetricLoop_: Before processing temp_list";
613  while (!temp_list.empty())
614  {
615  auto data_ = std::move(temp_list.front());
616  temp_list.pop_front();
617  if (data_->Type == MetricType::InvalidMetric)
618  {
619  continue;
620  }
621  if (!data_->UseNameOverride)
622  {
623  if (!data_->MetricPrefix.empty())
624  {
625  data_->Name = prefix_ + "." + data_->MetricPrefix + "." + data_->Name;
626  }
627  else
628  {
629  data_->Name = prefix_ + "." + data_->Name;
630  }
631  }
632 
633  for (auto& metric : metric_plugins_)
634  {
635  if (!metric)
636  {
637  continue;
638  }
639  if (metric->IsLevelEnabled(data_->Level))
640  {
641  try
642  {
643  metric->addMetricData(data_);
644  last_send_time = std::chrono::steady_clock::now();
645  }
646  catch (...)
647  {
648  TLOG(TLVL_ERROR) << "Error in MetricManager::sendMetric: error sending value to metric plugin with name "
649  << metric->getLibName();
650  }
651  }
652  }
653  }
654 
655  TLOG(TLVL_DEBUG + 34) << "sendMetricLoop_: Before sending metrics";
656  for (auto& metric : metric_plugins_)
657  {
658  if (!metric)
659  {
660  continue;
661  }
662  metric->sendMetrics(false, processing_start);
663  }
664 
665  // Limit rate of metrics going to plugins
666  TLOG(TLVL_DEBUG + 34) << "sendMetricLoop_: End of working loop";
667  busy_ = false;
668  usleep(10000);
669  }
670 
671  busy_ = true;
672  auto temp_list = std::list<std::unique_ptr<MetricData>>();
673  {
674  std::lock_guard<std::mutex> lk(metric_cache_mutex_);
675 
676  for (auto& q : metric_cache_)
677  {
678  if (q.second != nullptr && q.second->DataPointCount > 0)
679  {
680  temp_list.emplace_back(new MetricData(*q.second));
681  q.second->Reset();
682  }
683  }
684  //metric_cache_.clear();
685  }
686 
687  auto calls = metric_calls_.exchange(0);
688  temp_list.emplace_back(
689  new MetricData("Metric Calls", calls, "metrics", 4, MetricMode::Accumulate | MetricMode::Rate, "", false));
690 
691  auto missed = missed_metric_calls_.exchange(0);
692  temp_list.emplace_back(
693  new MetricData("Missed Metric Calls", missed, "metrics", 4, MetricMode::Accumulate | MetricMode::Rate, "", false));
694 
695  TLOG(TLVL_DEBUG + 33) << "There are " << temp_list.size() << " Metrics to process (" << calls << " calls, " << missed
696  << " missed)";
697 
698  while (!temp_list.empty())
699  {
700  auto data_ = std::move(temp_list.front());
701  temp_list.pop_front();
702  if (data_->Type == MetricType::InvalidMetric)
703  {
704  continue;
705  }
706  if (!data_->UseNameOverride)
707  {
708  if (!data_->MetricPrefix.empty())
709  {
710  data_->Name = prefix_ + "." + data_->MetricPrefix + "." + data_->Name;
711  }
712  else
713  {
714  data_->Name = prefix_ + "." + data_->Name;
715  }
716  }
717 
718  for (auto& metric : metric_plugins_)
719  {
720  if (!metric)
721  {
722  continue;
723  }
724  if (metric->IsLevelEnabled(data_->Level))
725  {
726  try
727  {
728  metric->addMetricData(data_);
729  last_send_time = std::chrono::steady_clock::now();
730  }
731  catch (...)
732  {
733  TLOG(TLVL_ERROR) << "Error in MetricManager::sendMetric: error sending value to metric plugin with name "
734  << metric->getLibName();
735  }
736  }
737  }
738  }
739 
740  for (auto& metric : metric_plugins_)
741  {
742  if (!metric)
743  {
744  continue;
745  }
746  try
747  {
748  metric->stopMetrics();
749  TLOG(TLVL_DEBUG + 32) << "Metric Plugin " << metric->getLibName() << " stopped.";
750  }
751  catch (...)
752  {
753  TLOG(TLVL_ERROR) << "Exception caught in MetricManager::do_stop(), error stopping plugin with name "
754  << metric->getLibName();
755  }
756  }
757  busy_ = false;
758  TLOG(TLVL_DEBUG + 32) << "MetricManager has been stopped.";
759 }
void shutdown()
Call the destructors for all configured MetricPlugin instances.
void initialize(fhicl::ParameterSet const &pset, std::string const &prefix="")
Initialize the MetricPlugin instances.
void sendMetric(std::string const &name, std::string const &value, std::string const &unit, int level, MetricMode mode, std::string const &metricPrefix="", bool useNameOverride=false)
Send a metric with the given parameters to any MetricPlugins with a threshold level &gt;= to level...
bool metricManagerBusy()
Determine whether the MetricManager or any of its plugins are currently processing metrics...
size_t metricQueueSize(std::string const &name="")
Return the size of the named metric queue
void reinitialize(fhicl::ParameterSet const &pset, std::string const &prefix="")
Reinitialize all MetricPlugin Instances.
Report the sum of all values. Use for counters to report accurate results.
MetricManager()
Construct an instance of the MetricManager class.
void do_start()
Perform startup actions for each configured MetricPlugin.
void do_stop()
Stop sending metrics to the MetricPlugin instances.
virtual ~MetricManager() noexcept
MetricManager destructor.
over. Use to create rates from counters.
Report only the last value recorded. Useful for event counters, run numbers, etc. ...
MetricMode
The Mode of the metric indicates how multiple metric values should be combined within a reporting int...
Definition: MetricData.hh:29
std::unique_ptr< MetricPlugin > makeMetricPlugin(std::string const &generator_plugin_spec, fhicl::ParameterSet const &ps, std::string const &app_name, std::string const &metric_name)
Load a given MetricPlugin and return a pointer to it.
Default, invalid value.
void do_resume()
Resume metric sending. Currently a No-Op.
bool metricQueueEmpty()
Returns whether the metric queue is completely empty
void do_pause()
Pause metric sending. Currently a No-Op.