artdaq_utilities  v1_08_03
MetricManager.cc
1 // MetricManager.cc: MetricManager class implementation file
2 // Author: Eric Flumerfelt
3 // Last Modified: 11/14/2014
4 //
5 // MetricManager loads a user-specified set of plugins, sends them their configuration,
6 // and sends them data as it is recieved. It also maintains the state of the plugins
7 // relative to the application state.
8 
9 #include "TRACE/tracemf.h"
10 #define TRACE_NAME "MetricManager"
11 
12 #include "artdaq-utilities/Plugins/MetricManager.hh"
13 #include "artdaq-utilities/Plugins/makeMetricPlugin.hh"
14 #include "fhiclcpp/ParameterSet.h"
15 
16 #include <pthread.h>
17 #include <boost/exception/all.hpp>
18 #include <chrono>
19 #include <memory>
20 
22  : metric_plugins_(0)
23  , system_metric_collector_(nullptr)
24  , initialized_(false)
25  , running_(false)
26  , active_(false)
27  , busy_(false)
28  , missed_metric_calls_(0)
29  , metric_calls_(0)
30 {
31  TLOG(TLVL_INFO) << "MetricManager CONSTRUCTOR";
32 }
33 
34 artdaq::MetricManager::~MetricManager() noexcept { shutdown(); }
35 
36 void artdaq::MetricManager::initialize(fhicl::ParameterSet const& pset, std::string const& prefix)
37 {
38  prefix_ = prefix;
39  if (initialized_)
40  {
41  shutdown();
42  }
43  TLOG(TLVL_INFO) << "Configuring metrics with parameter set: " << pset.to_string();
44 
45  std::vector<std::string> names = pset.get_names();
46 
47  metric_plugins_.clear();
48  bool send_system_metrics = false;
49  bool send_process_metrics = false;
50 
51  for (const auto& name : names)
52  {
53  if (name == "metric_queue_size")
54  {
55  metric_cache_max_size_ = pset.get<size_t>("metric_queue_size");
56  }
57  else if (name == "metric_queue_notify_size")
58  {
59  metric_cache_notify_size_ = pset.get<size_t>("metric_queue_notify_size");
60  }
61  else if (name == "metric_cache_size")
62  {
63  metric_cache_max_size_ = pset.get<size_t>("metric_cache_size");
64  }
65  else if (name == "metric_cache_notify_size")
66  {
67  metric_cache_notify_size_ = pset.get<size_t>("metric_cache_notify_size");
68  }
69  else if (name == "metric_send_maximum_delay_ms")
70  {
71  TLOG(TLVL_INFO) << "Setting metric_send_interval_ms_ to " << pset.get<int>("metric_send_maximum_delay_ms");
72  metric_send_interval_ms_ = pset.get<int>("metric_send_maximum_delay_ms");
73  }
74  else if (name == "metric_holdoff_us")
75  {
76  TLOG(TLVL_INFO) << "Setting metric_holdoff_us_ to " << pset.get<int>("metric_holdoff_us");
77  metric_holdoff_us_ = pset.get<int>("metric_holdoff_us");
78  }
79  else if (name == "send_system_metrics")
80  {
81  send_system_metrics = pset.get<bool>("send_system_metrics");
82  }
83  else if (name == "send_process_metrics")
84  {
85  send_process_metrics = pset.get<bool>("send_process_metrics");
86  }
87  else
88  {
89  try
90  {
91  TLOG(TLVL_DEBUG + 32) << "Constructing metric plugin with name " << name;
92  auto plugin_pset = pset.get<fhicl::ParameterSet>(name);
93  metric_plugins_.push_back(
94  makeMetricPlugin(plugin_pset.get<std::string>("metricPluginType", ""), plugin_pset, prefix_, name));
95  }
96  catch (const cet::exception& e)
97  {
98  TLOG(TLVL_ERROR) << "Exception caught in MetricManager::initialize, error loading plugin with name " << name
99  << ", cet::exception object caught:" << e.explain_self();
100  }
101  catch (const boost::exception& e)
102  {
103  TLOG(TLVL_ERROR) << "Exception caught in MetricManager::initialize, error loading plugin with name " << name
104  << ", boost::exception object caught: " << boost::diagnostic_information(e);
105  }
106  catch (const std::exception& e)
107  {
108  TLOG(TLVL_ERROR) << "Exception caught in MetricManager::initialize, error loading plugin with name " << name
109  << ", std::exception caught: " << e.what();
110  }
111  catch (...)
112  {
113  TLOG(TLVL_ERROR) << "Unknown Exception caught in MetricManager::initialize, error loading plugin with name "
114  << name;
115  }
116  }
117  }
118 
119  if (send_system_metrics || send_process_metrics)
120  {
121  system_metric_collector_ = std::make_unique<SystemMetricCollector>(send_process_metrics, send_system_metrics);
122  }
123 
124  initialized_ = true;
125 }
126 
128 {
129  std::lock_guard<std::mutex> lk(metric_mutex_);
130  if (!running_)
131  {
132  TLOG(TLVL_DEBUG + 32) << "Starting MetricManager";
133  for (auto& metric : metric_plugins_)
134  {
135  if (!metric)
136  {
137  continue;
138  }
139  try
140  {
141  metric->startMetrics();
142  TLOG(TLVL_INFO) << "Metric Plugin " << metric->getLibName() << " started.";
143  active_ = true;
144  }
145  catch (...)
146  {
147  TLOG(TLVL_ERROR) << "Exception caught in MetricManager::do_start(), error starting plugin with name "
148  << metric->getLibName();
149  }
150  }
151  running_ = true;
152  startMetricLoop_();
153  }
154 }
155 
157 {
158  std::unique_lock<std::mutex> lk(metric_mutex_);
159  TLOG(TLVL_DEBUG + 32) << "Stopping Metrics";
160  running_ = false;
161  metric_cv_.notify_all();
162  TLOG(TLVL_DEBUG + 32) << "Joining Metric-Sending thread";
163  lk.unlock();
164  try
165  {
166  if (metric_sending_thread_.joinable())
167  {
168  metric_sending_thread_.join();
169  }
170  }
171  catch (...)
172  {
173  // IGNORED
174  }
175  TLOG(TLVL_DEBUG + 32) << "do_stop Complete";
176 }
177 
179 { /*do_stop();*/
180 }
182 { /*do_start();*/
183 }
184 
185 void artdaq::MetricManager::reinitialize(fhicl::ParameterSet const& pset, std::string const& prefix)
186 {
187  shutdown();
188  initialize(pset, prefix);
189 }
190 
192 {
193  TRACE_STREAMER(TLVL_DEBUG + 32, TLOG2("MetricManager", 0), 0) << "MetricManager is shutting down..."; // Using TRACE_STREAMER in case MessageFacility is already gone
194  do_stop();
195 
196  std::lock_guard<std::mutex> lk(metric_mutex_);
197  if (initialized_)
198  {
199  TRACE_STREAMER(TLVL_DEBUG + 32, TLOG2("MetricManager", 0), 0) << "MetricManager is initialized shutting down...";
200  initialized_ = false;
201  for (auto& i : metric_plugins_)
202  {
203  try
204  {
205  std::string name = i->getLibName();
206  i.reset(nullptr);
207  TRACE_STREAMER(TLVL_DEBUG + 32, TLOG2("MetricManager", 0), 0) << "Metric Plugin " << name << " shutdown.";
208  }
209  catch (...)
210  {
211  TRACE_STREAMER(TLVL_ERROR, TLOG2("MetricManager", 0), 0) << "Exception caught in MetricManager::shutdown(), error shutting down metric with name "
212  << i->getLibName();
213  }
214  }
215  metric_plugins_.clear();
216  }
217 }
218 
219 void artdaq::MetricManager::sendMetric(std::string const& name, std::string const& value, std::string const& unit,
220  int level, MetricMode mode, std::string const& metricPrefix,
221  bool useNameOverride)
222 {
223  if (!initialized_)
224  {
225  TLOG(TLVL_WARNING) << "Attempted to send metric " << name << " when MetricManager has not yet been initialized!";
226  }
227  else if (!running_)
228  {
229  TLOG(TLVL_INFO) << "Attempted to send metric when MetricManager stopped!";
230  }
231  else if (active_)
232  {
233  {
234  std::lock_guard<std::mutex> lk(metric_cache_mutex_);
235  metric_calls_++;
236  last_metric_received_ = std::chrono::steady_clock::now();
237  auto& cached = metric_cache_[name];
238  if (cached == nullptr)
239  {
240  metric_cache_[name] = std::make_unique<MetricData>(name, value, unit, level, mode, metricPrefix, useNameOverride);
241  }
242  else
243  {
244  auto size = cached->DataPointCount;
245  if (size < metric_cache_max_size_)
246  {
247  if (size >= metric_cache_notify_size_)
248  {
249  TLOG(TLVL_DEBUG + 35) << "Metric cache is at size " << size << " of " << metric_cache_max_size_ << " for metric " << name
250  << ".";
251  }
252  if (mode == MetricMode::LastPoint)
253  {
254  cached->StringValue = value;
255  cached->DataPointCount = 1;
256  }
257  else
258  {
259  cached->StringValue += " " + value;
260  cached->DataPointCount++;
261  }
262  }
263  else
264  {
265  TLOG(TLVL_DEBUG + 36) << "Rejecting metric because queue full";
266  missed_metric_calls_++;
267  }
268  }
269  }
270  metric_cv_.notify_all();
271  }
272 }
273 
274 void artdaq::MetricManager::sendMetric(std::string const& name, int const& value, std::string const& unit, int level,
275  MetricMode mode, std::string const& metricPrefix, bool useNameOverride)
276 {
277  if (!initialized_)
278  {
279  TLOG(TLVL_WARNING) << "Attempted to send metric " << name << " when MetricManager has not yet been initialized!";
280  }
281  else if (!running_)
282  {
283  TLOG(TLVL_INFO) << "Attempted to send metric when MetricManager stopped!";
284  }
285  else if (active_)
286  {
287  {
288  std::lock_guard<std::mutex> lk(metric_cache_mutex_);
289  metric_calls_++;
290  last_metric_received_ = std::chrono::steady_clock::now();
291  auto& cached = metric_cache_[name];
292  if (cached == nullptr)
293  {
294  metric_cache_[name] = std::make_unique<MetricData>(name, value, unit, level, mode, metricPrefix, useNameOverride);
295  }
296  else
297  {
298  auto size = cached->DataPointCount;
299  if (size < metric_cache_max_size_)
300  {
301  if (size >= metric_cache_notify_size_)
302  {
303  TLOG(TLVL_DEBUG + 35) << "Metric cache is at size " << size << " of " << metric_cache_max_size_ << " for metric " << name
304  << ".";
305  }
306  cached->AddPoint(value);
307  }
308  else
309  {
310  TLOG(TLVL_DEBUG + 36) << "Rejecting metric because queue full";
311  missed_metric_calls_++;
312  }
313  }
314  }
315  metric_cv_.notify_all();
316  }
317 }
318 
319 void artdaq::MetricManager::sendMetric(std::string const& name, double const& value, std::string const& unit, int level,
320  MetricMode mode, std::string const& metricPrefix, bool useNameOverride)
321 {
322  if (!initialized_)
323  {
324  TLOG(TLVL_WARNING) << "Attempted to send metric " << name << " when MetricManager has not yet been initialized!";
325  }
326  else if (!running_)
327  {
328  TLOG(TLVL_INFO) << "Attempted to send metric when MetricManager stopped!";
329  }
330  else if (active_)
331  {
332  {
333  std::lock_guard<std::mutex> lk(metric_cache_mutex_);
334  metric_calls_++;
335  last_metric_received_ = std::chrono::steady_clock::now();
336  auto& cached = metric_cache_[name];
337  if (cached == nullptr)
338  {
339  metric_cache_[name] = std::make_unique<MetricData>(name, value, unit, level, mode, metricPrefix, useNameOverride);
340  }
341  else
342  {
343  auto size = cached->DataPointCount;
344  if (size < metric_cache_max_size_)
345  {
346  if (size >= metric_cache_notify_size_)
347  {
348  TLOG(TLVL_DEBUG + 35) << "Metric cache is at size " << size << " of " << metric_cache_max_size_ << " for metric " << name
349  << ".";
350  }
351  cached->AddPoint(value);
352  }
353  else
354  {
355  TLOG(TLVL_DEBUG + 36) << "Rejecting metric because queue full";
356  missed_metric_calls_++;
357  }
358  }
359  }
360  metric_cv_.notify_all();
361  }
362 }
363 
364 void artdaq::MetricManager::sendMetric(std::string const& name, float const& value, std::string const& unit, int level,
365  MetricMode mode, std::string const& metricPrefix, bool useNameOverride)
366 {
367  if (!initialized_)
368  {
369  TLOG(TLVL_WARNING) << "Attempted to send metric " << name << " when MetricManager has not yet been initialized!";
370  }
371  else if (!running_)
372  {
373  TLOG(TLVL_INFO) << "Attempted to send metric when MetricManager stopped!";
374  }
375  else if (active_)
376  {
377  {
378  std::lock_guard<std::mutex> lk(metric_cache_mutex_);
379  metric_calls_++;
380  last_metric_received_ = std::chrono::steady_clock::now();
381  auto& cached = metric_cache_[name];
382  if (cached == nullptr)
383  {
384  metric_cache_[name] = std::make_unique<MetricData>(name, value, unit, level, mode, metricPrefix, useNameOverride);
385  }
386  else
387  {
388  auto size = cached->DataPointCount;
389  if (size < metric_cache_max_size_)
390  {
391  if (size >= metric_cache_notify_size_)
392  {
393  TLOG(TLVL_DEBUG + 35) << "Metric cache is at size " << size << " of " << metric_cache_max_size_ << " for metric " << name
394  << ".";
395  }
396  cached->AddPoint(value);
397  }
398  else
399  {
400  TLOG(TLVL_DEBUG + 36) << "Rejecting metric because queue full";
401  missed_metric_calls_++;
402  }
403  }
404  }
405  metric_cv_.notify_all();
406  }
407 }
408 
409 void artdaq::MetricManager::sendMetric(std::string const& name, uint64_t const& value, std::string const& unit,
410  int level, MetricMode mode, std::string const& metricPrefix,
411  bool useNameOverride)
412 {
413  if (!initialized_)
414  {
415  TLOG(TLVL_WARNING) << "Attempted to send metric " << name << " when MetricManager has not yet been initialized!";
416  }
417  else if (!running_)
418  {
419  TLOG(TLVL_INFO) << "Attempted to send metric when MetricManager stopped!";
420  }
421  else if (active_)
422  {
423  {
424  std::lock_guard<std::mutex> lk(metric_cache_mutex_);
425  metric_calls_++;
426  last_metric_received_ = std::chrono::steady_clock::now();
427  auto& cached = metric_cache_[name];
428  if (cached == nullptr)
429  {
430  metric_cache_[name] = std::make_unique<MetricData>(name, value, unit, level, mode, metricPrefix, useNameOverride);
431  }
432  else
433  {
434  auto size = cached->DataPointCount;
435  if (size < metric_cache_max_size_)
436  {
437  if (size >= metric_cache_notify_size_)
438  {
439  TLOG(TLVL_DEBUG + 35) << "Metric cache is at size " << size << " of " << metric_cache_max_size_ << " for metric " << name
440  << ".";
441  }
442  cached->AddPoint(value);
443  }
444  else
445  {
446  TLOG(TLVL_DEBUG + 36) << "Rejecting metric because queue full";
447  missed_metric_calls_++;
448  }
449  }
450  }
451  metric_cv_.notify_all();
452  }
453 }
454 
455 void artdaq::MetricManager::startMetricLoop_()
456 {
457  if (metric_sending_thread_.joinable())
458  {
459  metric_sending_thread_.join();
460  }
461  boost::thread::attributes attrs;
462  attrs.set_stack_size(4096 * 2000); // 8000 KB
463  TLOG(TLVL_INFO) << "Starting Metric Sending Thread";
464  try
465  {
466  metric_sending_thread_ = boost::thread(attrs, boost::bind(&MetricManager::sendMetricLoop_, this));
467 
468  char tname[16]; // Size 16 - see man page pthread_setname_np(3) and/or prctl(2)
469  snprintf(tname, sizeof(tname) - 1, "%s", "MetricSend"); // NOLINT
470  tname[sizeof(tname) - 1] = '\0'; // assure term. snprintf is not too evil :)
471  auto handle = metric_sending_thread_.native_handle();
472  pthread_setname_np(handle, tname);
473  }
474  catch (const boost::exception& e)
475  {
476  TLOG(TLVL_ERROR) << "Caught boost::exception starting Metric Sending thread: " << boost::diagnostic_information(e)
477  << ", errno=" << errno;
478  std::cerr << "Caught boost::exception starting Metric Sending thread: " << boost::diagnostic_information(e)
479  << ", errno=" << errno << std::endl;
480  exit(5);
481  }
482  TLOG(TLVL_INFO) << "Metric Sending thread started";
483 }
484 
486 {
487  std::lock_guard<std::mutex> lk(metric_cache_mutex_);
488  for (auto& cache_entry : metric_cache_)
489  {
490  if (cache_entry.second->DataPointCount > 0)
491  {
492  return false;
493  }
494  }
495 
496  return true;
497 }
498 
500 {
501  bool pluginsBusy = false;
502 
503  for (auto& p : metric_plugins_)
504  {
505  if (p->metricsPending())
506  {
507  pluginsBusy = true;
508  break;
509  }
510  }
511 
512  TLOG(TLVL_DEBUG + 33) << "Metric queue empty: " << metricQueueEmpty() << ", busy_: " << busy_ << ", Plugins busy: " << pluginsBusy;
513  return !metricQueueEmpty() || busy_ || pluginsBusy;
514 }
515 
516 size_t artdaq::MetricManager::metricQueueSize(std::string const& name)
517 {
518  std::lock_guard<std::mutex> lk(metric_cache_mutex_);
519  size_t size = 0;
520  if (name.empty())
521  {
522  for (auto& q : metric_cache_)
523  {
524  size += q.second->DataPointCount;
525  }
526  }
527  else
528  {
529  if (metric_cache_.count(name) != 0u)
530  {
531  size = metric_cache_[name]->DataPointCount;
532  }
533  }
534 
535  return size;
536 }
537 
538 void artdaq::MetricManager::sendMetricLoop_()
539 {
540  TLOG(TLVL_INFO) << "sendMetricLoop_ START";
541  auto last_send_time = std::chrono::steady_clock::time_point();
542  while (running_)
543  {
544  TLOG(TLVL_DEBUG + 34) << "sendMetricLoop_: Entering Metric input wait loop";
545  while (metricQueueEmpty() && running_)
546  {
547  std::unique_lock<std::mutex> lk(metric_mutex_);
548  metric_cv_.wait_for(lk, std::chrono::milliseconds(100));
549  auto now = std::chrono::steady_clock::now();
550  if (std::chrono::duration_cast<std::chrono::milliseconds>(now - last_send_time).count() >
551  metric_send_interval_ms_)
552  {
553  TLOG(TLVL_DEBUG + 34) << "sendMetricLoop_: Metric send interval exceeded: Sending metrics";
554  {
555  std::unique_lock<std::mutex> lk(metric_cache_mutex_); // last_metric_received_ is protected by metric_cache_mutex_
556  if (std::chrono::duration_cast<std::chrono::microseconds>(now - last_metric_received_).count() < metric_holdoff_us_)
557  {
558  lk.unlock();
559  usleep(metric_holdoff_us_);
560  }
561  }
562  for (auto& metric : metric_plugins_)
563  {
564  if (metric)
565  {
566  metric->sendMetrics();
567  }
568  }
569  last_send_time = now;
570  }
571  }
572  {
573  std::unique_lock<std::mutex> lk(metric_cache_mutex_); // last_metric_received_ is protected by metric_cache_mutex_
574  if (std::chrono::duration_cast<std::chrono::microseconds>(std::chrono::steady_clock::now() - last_metric_received_).count() < metric_holdoff_us_)
575  {
576  lk.unlock();
577  usleep(metric_holdoff_us_);
578  }
579  }
580 
581  TLOG(TLVL_DEBUG + 34) << "sendMetricLoop_: After Metric input wait loop";
582  busy_ = true;
583  auto processing_start = std::chrono::steady_clock::now();
584  auto temp_list = std::list<std::unique_ptr<MetricData>>();
585  {
586  std::lock_guard<std::mutex> lk(metric_cache_mutex_);
587 
588  for (auto& q : metric_cache_)
589  {
590  if (q.second != nullptr && q.second->DataPointCount > 0)
591  {
592  temp_list.emplace_back(new MetricData(*q.second));
593  q.second->Reset();
594  }
595  }
596  }
597 
598  auto calls = metric_calls_.exchange(0);
599  temp_list.emplace_back(
600  new MetricData("Metric Calls", calls, "metrics", 4, MetricMode::Accumulate | MetricMode::Rate, "", false));
601 
602  auto missed = missed_metric_calls_.exchange(0);
603  temp_list.emplace_back(
604  new MetricData("Missed Metric Calls", missed, "metrics", 4, MetricMode::Accumulate | MetricMode::Rate, "", false));
605 
606  TLOG(TLVL_DEBUG + 33) << "There are " << temp_list.size() << " Metrics to process (" << calls << " calls, " << missed
607  << " missed)";
608 
609  if (system_metric_collector_ != nullptr)
610  {
611  TLOG(TLVL_DEBUG + 33) << "Collecting System metrics (CPU, RAM, Network)";
612  auto systemMetrics = system_metric_collector_->SendMetrics();
613  for (auto& m : systemMetrics) { temp_list.emplace_back(std::move(m)); }
614  }
615 
616  TLOG(TLVL_DEBUG + 34) << "sendMetricLoop_: Before processing temp_list";
617  while (!temp_list.empty())
618  {
619  auto data_ = std::move(temp_list.front());
620  temp_list.pop_front();
621  if (data_->Type == MetricType::InvalidMetric)
622  {
623  continue;
624  }
625  if (!data_->UseNameOverride)
626  {
627  if (!data_->MetricPrefix.empty())
628  {
629  data_->Name = prefix_ + "." + data_->MetricPrefix + "." + data_->Name;
630  }
631  else
632  {
633  data_->Name = prefix_ + "." + data_->Name;
634  }
635  }
636 
637  for (auto& metric : metric_plugins_)
638  {
639  if (!metric)
640  {
641  continue;
642  }
643  if (metric->IsLevelEnabled(data_->Level))
644  {
645  try
646  {
647  metric->addMetricData(data_);
648  last_send_time = std::chrono::steady_clock::now();
649  }
650  catch (...)
651  {
652  TLOG(TLVL_ERROR) << "Error in MetricManager::sendMetric: error sending value to metric plugin with name "
653  << metric->getLibName();
654  }
655  }
656  }
657  }
658 
659  TLOG(TLVL_DEBUG + 34) << "sendMetricLoop_: Before sending metrics";
660  for (auto& metric : metric_plugins_)
661  {
662  if (!metric)
663  {
664  continue;
665  }
666  metric->sendMetrics(false, processing_start);
667  }
668 
669  // Limit rate of metrics going to plugins
670  TLOG(TLVL_DEBUG + 34) << "sendMetricLoop_: End of working loop";
671  busy_ = false;
672  usleep(10000);
673  }
674 
675  busy_ = true;
676  auto temp_list = std::list<std::unique_ptr<MetricData>>();
677  {
678  std::lock_guard<std::mutex> lk(metric_cache_mutex_);
679 
680  for (auto& q : metric_cache_)
681  {
682  if (q.second != nullptr && q.second->DataPointCount > 0)
683  {
684  temp_list.emplace_back(new MetricData(*q.second));
685  q.second->Reset();
686  }
687  }
688  // metric_cache_.clear();
689  }
690 
691  auto calls = metric_calls_.exchange(0);
692  temp_list.emplace_back(
693  new MetricData("Metric Calls", calls, "metrics", 4, MetricMode::Accumulate | MetricMode::Rate, "", false));
694 
695  auto missed = missed_metric_calls_.exchange(0);
696  temp_list.emplace_back(
697  new MetricData("Missed Metric Calls", missed, "metrics", 4, MetricMode::Accumulate | MetricMode::Rate, "", false));
698 
699  TLOG(TLVL_DEBUG + 33) << "There are " << temp_list.size() << " Metrics to process (" << calls << " calls, " << missed
700  << " missed)";
701 
702  while (!temp_list.empty())
703  {
704  auto data_ = std::move(temp_list.front());
705  temp_list.pop_front();
706  if (data_->Type == MetricType::InvalidMetric)
707  {
708  continue;
709  }
710  if (!data_->UseNameOverride)
711  {
712  if (!data_->MetricPrefix.empty())
713  {
714  data_->Name = prefix_ + "." + data_->MetricPrefix + "." + data_->Name;
715  }
716  else
717  {
718  data_->Name = prefix_ + "." + data_->Name;
719  }
720  }
721 
722  for (auto& metric : metric_plugins_)
723  {
724  if (!metric)
725  {
726  continue;
727  }
728  if (metric->IsLevelEnabled(data_->Level))
729  {
730  try
731  {
732  metric->addMetricData(data_);
733  last_send_time = std::chrono::steady_clock::now();
734  }
735  catch (...)
736  {
737  TLOG(TLVL_ERROR) << "Error in MetricManager::sendMetric: error sending value to metric plugin with name "
738  << metric->getLibName();
739  }
740  }
741  }
742  }
743 
744  for (auto& metric : metric_plugins_)
745  {
746  if (!metric)
747  {
748  continue;
749  }
750  try
751  {
752  metric->stopMetrics();
753  TLOG(TLVL_DEBUG + 32) << "Metric Plugin " << metric->getLibName() << " stopped.";
754  }
755  catch (...)
756  {
757  TLOG(TLVL_ERROR) << "Exception caught in MetricManager::do_stop(), error stopping plugin with name "
758  << metric->getLibName();
759  }
760  }
761  busy_ = false;
762  TLOG(TLVL_DEBUG + 32) << "MetricManager has been stopped.";
763 }
void shutdown()
Call the destructors for all configured MetricPlugin instances.
void initialize(fhicl::ParameterSet const &pset, std::string const &prefix="")
Initialize the MetricPlugin instances.
void sendMetric(std::string const &name, std::string const &value, std::string const &unit, int level, MetricMode mode, std::string const &metricPrefix="", bool useNameOverride=false)
Send a metric with the given parameters to any MetricPlugins with a threshold level &gt;= to level...
bool metricManagerBusy()
Determine whether the MetricManager or any of its plugins are currently processing metrics...
size_t metricQueueSize(std::string const &name="")
Return the size of the named metric queue
void reinitialize(fhicl::ParameterSet const &pset, std::string const &prefix="")
Reinitialize all MetricPlugin Instances.
Report the sum of all values. Use for counters to report accurate results.
MetricManager()
Construct an instance of the MetricManager class.
void do_start()
Perform startup actions for each configured MetricPlugin.
void do_stop()
Stop sending metrics to the MetricPlugin instances.
virtual ~MetricManager() noexcept
MetricManager destructor.
over. Use to create rates from counters.
Report only the last value recorded. Useful for event counters, run numbers, etc. ...
MetricMode
The Mode of the metric indicates how multiple metric values should be combined within a reporting int...
Definition: MetricData.hh:27
std::unique_ptr< MetricPlugin > makeMetricPlugin(std::string const &generator_plugin_spec, fhicl::ParameterSet const &ps, std::string const &app_name, std::string const &metric_name)
Load a given MetricPlugin and return a pointer to it.
Default, invalid value.
void do_resume()
Resume metric sending. Currently a No-Op.
bool metricQueueEmpty()
Returns whether the metric queue is completely empty
void do_pause()
Pause metric sending. Currently a No-Op.