artdaq_ganglia_plugin  v1_02_15
malloc_free.cc
1 // This file (malloc_free.cc) was created by Ron Rechenmacher <ron@fnal.gov> on
2 // Dec 2, 2015. "TERMS AND CONDITIONS" governing this file are in the README
3 // or COPYING file. If you do not have such a file, one can be obtained by
4 // contacting Ron or Fermi Lab in Batavia IL, 60510, phone: 630-840-3000.
5 // $RCSfile: malloc_free.cc,v $
6 static char const* rev = "$Revision: 1.12 $$Date: 2016/11/04 13:57:24 $";
7 
8 // This programm attemps to do a bunch of random mallocs and frees.
9 //
10 // make malloc_free CXX="g++ -std=c++0x"
11 // CXX="g++ -std=c++0x" make -e malloc_free # NOTE "-e"
12 
13 #include <getopt.h> // getopt_long, {no,required,optional}_argument, extern char *optarg; extern int opt{ind,err,opt}
14 #include <stdio.h> // printf
15 #include <stdlib.h> // malloc, rand - range [0, RAND_MAX] period at least 2**32
16 // random: 0 to RAND_MAX -- period 16 * ((2^31) - 1), has initstate() and setstate() functions
17 #include <libgen.h> // basename
18 #include <stdint.h> // uint64_t
19 #include <string.h> // memset
20 #include <sys/time.h> // gettimeofday
21 #include <time.h> // time()
22 #include <unistd.h> // getpid()
23 #include <boost/thread.hpp>
24 #include <mutex>
25 #include <set>
26 #include <tuple> // pair
27 #include <vector>
28 #include "send_gmetric.h"
29 
30 #define TRACE_NAME "malloc_free"
31 #if 1
32 #include "trace.h" // TRACE
33 #else
34 #define TRACE(vv, ...) \
35  do { \
36  if (opt_v >= vv) printf(__VA_ARGS__); \
37  } while (0)
38 #endif
39 
40 #define USAGE \
41  "\
42  usage: %s [-hv?]\n\
43 \n\
44 example: %s\n\
45 \n\
46 option:\n\
47  -V print version and exit\n\
48  --help\n\
49  --seed= random number generator seed\n\
50  --min= min alloc default=%u\n\
51  --max= max alloc default=%u\n\
52  -t --time= seconds\n\
53  --new use new/delete\n\
54  --no-seed \n\
55  --ave= default=%d\n\
56  --threads= default=%u\n\
57 ", \
58  basename(argv[0]), basename(argv[0]), opt_min_alloc, opt_max_alloc, opt_ave, opt_threads
59 
60 /* GLOBALS */
61 uint64_t opt_hi_water = 1 * (1024 * 1024 * 1024);
62 uint64_t opt_lo_water = 8 * (1024 * 1024);
63 unsigned opt_seed = 2, opt_seed_ = 1; // 2 variables -- to allow any possible seed value
64 unsigned opt_max_alloc = 128 * (1024);
65 unsigned opt_min_alloc = 1 * (1024);
66 int opt_v = 0;
67 int opt_new = 0;
68 unsigned opt_ave = 100000;
69 int opt_threads = 1;
70 int opt_time = 25;
71 int g_stop = 0;
72 
73 uint64_t human(char* optarg) {
74  char* endptr;
75  uint64_t ret = strtoul(optarg, &endptr, 0);
76  if (*endptr == 'k' || *endptr == 'K')
77  ret *= 1024;
78  else if (*endptr == 'm' || *endptr == 'M')
79  ret *= 1024 * 1024;
80  else if (*endptr == 'g' || *endptr == 'G')
81  ret *= 1024 * 1024 * 1024;
82  return ret;
83 }
84 
85 void parse_args(int argc, char* argv[]) {
86  // parse opt, optargs, and args
87  while (1) {
88  int opt;
89  static struct option long_options[] = {// name has_arg *flag val
90  {"help", no_argument, 0, 'h'},
91  {"seed", required_argument, 0, 's'},
92  {"hi", required_argument, 0, 1},
93  {"lo", required_argument, 0, 2},
94  {"min", required_argument, 0, 3},
95  {"max", required_argument, 0, 4},
96  {"no-seed", no_argument, 0, 5},
97  {"ave", required_argument, 0, 6},
98  {"threads", required_argument, 0, 7},
99  {"time", required_argument, 0, 't'},
100  {0, 0, 0, 0}};
101  opt = getopt_long(argc, argv, "?hvVs:t:", long_options, NULL);
102  if (opt == -1) break;
103  switch (opt) {
104  case '?':
105  case 'h':
106  printf(USAGE);
107  exit(0);
108  break;
109  case 'V':
110  printf("%s\n", rev);
111  exit(0);
112  break;
113  case 'v':
114  ++opt_v;
115  break;
116  case 's':
117  opt_seed = strtoul(optarg, NULL, 0);
118  opt_seed_ = 1;
119  break;
120  case 't':
121  opt_time = strtoul(optarg, NULL, 0);
122  break;
123  case 1:
124  opt_hi_water = human(optarg);
125  break;
126  case 2:
127  opt_lo_water = human(optarg);
128  break;
129  case 3:
130  opt_min_alloc = human(optarg);
131  break;
132  case 4:
133  opt_max_alloc = human(optarg);
134  break;
135  case 5:
136  opt_seed_ = 0;
137  break;
138  case 6:
139  opt_ave = strtoul(optarg, NULL, 0);
140  break;
141  case 7:
142  opt_threads = strtoul(optarg, NULL, 0);
143  break;
144  default:
145  printf("?? getopt returned character code 0%o ??\n", opt);
146  exit(1);
147  }
148  }
149 }
150 
151 uint64_t gettimeofday_us(void) {
152  struct timeval tv;
153  gettimeofday(&tv, NULL);
154  return (uint64_t)tv.tv_sec * 1000000 + tv.tv_usec;
155 }
156 
157 struct Stats {
158  Stats() : count_(0), running_tot_us_(0), running_tot_MB_(0.0) {}
159 
160  void update(double MB, int64_t delta_us) {
161  mtx_.lock();
162  running_tot_MB_ += MB;
163  running_tot_us_ += delta_us;
164  times_per_.insert((double)delta_us / MB);
165  ++count_;
166  mtx_.unlock();
167  }
168 
169  void get_clear(double& tot_MB, int64_t& tot_delta_us, int& count, double& min, double& max) {
170  mtx_.lock();
171  tot_MB = running_tot_MB_;
172  tot_delta_us = running_tot_us_;
173  count = count_;
174  if (count == 0)
175  min = max = 0;
176  else {
177  min = *(times_per_.begin());
178  max = *(times_per_.rbegin());
179  }
180  running_tot_MB_ = 0.0;
181  running_tot_us_ = 0;
182  times_per_.clear();
183  count_ = 0;
184  mtx_.unlock();
185  }
186 
187  private:
188  int count_;
189  int64_t running_tot_us_;
190  double running_tot_MB_;
191  std::multiset<double> times_per_;
192  std::mutex mtx_;
193 };
194 
195 struct Malloc {
196  Malloc(Stats& stats) : num_mallocs(0), stats_(stats) {}
197 
198 #define NUM_DELTAS opt_ave
199  int num_mallocs;
200  Stats& stats_;
201 
202  // alloc_bytes is an output -- This function returns a ptr to the memory
203  // and the num bytes allocated.
204  void* malloc_opt(unsigned* alloc_bytes) {
205  void* ptr;
206  if ((opt_max_alloc - opt_min_alloc) == 0)
207  *alloc_bytes = opt_max_alloc;
208  else
209  *alloc_bytes = (random() % (opt_max_alloc - opt_min_alloc)) + opt_min_alloc;
210 
211  uint64_t t0 = gettimeofday_us();
212 
213  ptr = malloc(*alloc_bytes);
214  if (ptr == NULL) {
215  perror("malloc");
216  exit(1);
217  }
218  unsigned ii;
219  for (ii = 0; ii < *alloc_bytes; ii += 4096) *(int*)((char*)ptr + ii) = 1;
220  ii /= 4096; // convert offset to pages
221 
222  int64_t delta_us = gettimeofday_us() - t0;
223 #define PAGES_PER_MB (0x100000 / 4096.0)
224  double MB = (double)ii / PAGES_PER_MB;
225  TLOG(TLVL_INFO) << "malloc " << *alloc_bytes << " bytes ii=" << ii << " MB=" << MB;
226 
227  stats_.update(MB, delta_us);
228 
229  return ptr;
230  }
231 };
232 
233 static void do_work(Stats* stats) {
234  std::vector<std::pair<void*, int>> ptr_vec;
235  uint64_t total_allocation = 0;
236  int direction; // 1 is up, 0 is down.
237  enum { up, down };
238 
239  Malloc mm(*stats);
240 
241  while (total_allocation < opt_lo_water) {
242  unsigned alloc_bytes;
243  int8_t* ptr = (int8_t*)mm.malloc_opt(&alloc_bytes);
244  std::pair<void*, int> xx(ptr, alloc_bytes);
245  ptr_vec.push_back(xx);
246  total_allocation += alloc_bytes;
247  }
248  direction = up;
249 
250  printf("ptr_vec.size()=%ld\n", ptr_vec.size());
251  while (!g_stop) {
252  long rr = random();
253  // for the "up" direction, higher probability for malloc
254  long tst = (direction == up) ? RAND_MAX / 4 * 3 : RAND_MAX / 4 * 1;
255  int malloc1_or_free0 = rr < tst;
256 
257  if (malloc1_or_free0) {
258  // do mallocs
259  unsigned alloc_bytes;
260  int8_t* ptr = (int8_t*)mm.malloc_opt(&alloc_bytes);
261  std::pair<void*, int> xx(ptr, alloc_bytes);
262  ptr_vec.push_back(xx);
263  total_allocation += alloc_bytes;
264  if (total_allocation > opt_hi_water && direction == up) {
265  TLOG(TLVL_WARNING) << "direction=down";
266  direction = down;
267  }
268  } else {
269  // do frees
270  if (ptr_vec.size() == 0) {
271  printf("size()==0 - continue\n");
272  continue;
273  }
274  unsigned free_idx = random() % ptr_vec.size();
275  std::pair<void*, int> xx(ptr_vec[free_idx]);
276  ptr_vec.erase(ptr_vec.begin() + free_idx);
277  TLOG(TLVL_INFO) << "free " << xx.second;
278  free(xx.first);
279  total_allocation -= xx.second;
280  if (total_allocation < opt_lo_water && direction == down) {
281  TLOG(TLVL_WARNING) << "direction=up";
282  direction = up;
283  }
284  }
285  }
286 }
287 
288 void send_metrics(double ave, double min, double max, double MBperalloc, int32_t threads) {
289 #define CLUSTER "mu2e DAQ"
290 #define GROUP "memory"
291  send_gmetric("ave", std::to_string(ave).c_str(), "double", "us/MB", "both", 30, 0, GROUP, CLUSTER, "ave us/MB",
292  "ave us/MB");
293  send_gmetric("min", std::to_string(min).c_str(), "double", "us/MB", "both", 30, 0, GROUP, CLUSTER, "min us/MB",
294  "min us/MB");
295  send_gmetric("max", std::to_string(max).c_str(), "double", "us/MB", "both", 30, 0, GROUP, CLUSTER, "max us/MB",
296  "max us/MB");
297  send_gmetric("ave_MBperalloc", std::to_string(MBperalloc).c_str(), "double", "MB/alloc", "both", 30, 0, GROUP,
298  CLUSTER, "ave MB/alloc", "ave MB/alloc");
299  send_gmetric("threads", std::to_string(threads).c_str(), "int32", "threads", "both", 30, 0, GROUP, CLUSTER,
300  "threads mallocing", "threads mallocing");
301 }
302 
303 int main(int argc, char* argv[]) {
304  parse_args(argc, argv);
305 
306  if (opt_hi_water < opt_lo_water) {
307  printf("changeing hi_water from %lu to %lu\n", opt_hi_water, opt_lo_water);
308  opt_hi_water = opt_lo_water;
309  }
310  if (opt_max_alloc < opt_min_alloc) {
311  printf("changeing max_alloc from %u to %u\n", opt_max_alloc, opt_min_alloc);
312  opt_max_alloc = opt_min_alloc;
313  }
314 
315  if (!opt_seed_) opt_seed = time(NULL) * getpid();
316  srandom(opt_seed);
317  printf("opt_seed=%u RAND_MAX=%d random=%ld\n", opt_seed, RAND_MAX, random());
318  printf("lo=%ld hi=%ld min=%u max=%u\n", opt_lo_water, opt_hi_water, opt_min_alloc, opt_max_alloc);
319 
320  init_gmetric("/etc/ganglia/gmond.conf");
321 
322  std::vector<boost::thread> threads(opt_threads);
323  std::vector<Stats> stats(opt_threads);
324  for (auto nn = 0; nn < opt_threads; ++nn) {
325  boost::thread::attributes attrs;
326  attrs.set_stack_size(4096 * 2000); // 8000 KB
327  try {
328  threads[nn] = boost::thread(attrs, boost::bind(&do_work, &stats[nn]));
329  } catch (const boost::exception& e) {
330  TLOG(TLVL_ERROR) << "Caught boost::exception starting malloc_free thread: " << boost::diagnostic_information(e)
331  << ", errno=" << errno;
332  std::cerr << "Caught boost::exception starting malloc_free thread: " << boost::diagnostic_information(e)
333  << ", errno=" << errno << std::endl;
334  exit(5);
335  }
336  }
337 
338  uint64_t start_us = gettimeofday_us();
339  uint64_t end_us = start_us + (1000000L * opt_time);
340  usleep(5000000);
341  for (auto cc = 2; 1; ++cc) {
342  double tot_ave_usperMB = 0.0, tot_min_usperMB = 0.0, tot_max_usperMB = 0.0, tot_MBperalloc = 0.0;
343  for (auto nn = 0; nn < opt_threads; ++nn) {
344  double tot_MB;
345  int64_t tot_us;
346  int count;
347  double min, max;
348  stats[nn].get_clear(tot_MB, tot_us, count, min, max);
349  tot_ave_usperMB += (tot_MB != 0.0) ? (double)tot_us / tot_MB : 0.0;
350  tot_min_usperMB += min;
351  tot_max_usperMB += max;
352  tot_MBperalloc += count ? tot_MB / count : 0;
353  TLOG(6) << "nn=" << nn << " tot_MB=" << std::setprecision(4) << tot_MB << " count=" << count;
354  }
355  if ((cc % 5) == 0)
356  TLOG(TLVL_ERROR) << opt_threads << " ave malloc usec/MB ave=" << std::setprecision(4)
357  << tot_ave_usperMB / opt_threads << " min=" << tot_min_usperMB / opt_threads
358  << " max=" << tot_max_usperMB / opt_threads << " MB/alloc=" << std::setprecision(1)
359  << tot_MBperalloc / opt_threads;
360 
361  send_metrics(tot_ave_usperMB / opt_threads, tot_min_usperMB / opt_threads, tot_max_usperMB / opt_threads,
362  tot_MBperalloc / opt_threads, opt_threads);
363 
364  uint64_t target = start_us + (5000000L * cc);
365  uint64_t now_us = gettimeofday_us();
366  int64_t slp = target - now_us;
367  if (slp > 0) {
368  if ((now_us + slp) > end_us) break;
369  usleep(slp);
370  } else {
371  printf("cc=%d slp=%ld\n", cc, slp);
372  if (now_us > end_us) break;
373  }
374  }
375  send_metrics(0.0, 0.0, 0.0, 0.0, 0);
376  g_stop = 1;
377  for (auto nn = 0; nn < opt_threads; ++nn) {
378  threads[nn].join();
379  }
380  printf("malloc_free finished.\n");
381  return (0);
382 } // main
int init_gmetric(const char *conf)
Initialize Ganglia.
Definition: send_gmetric.c:19
int send_gmetric(const char *name, const char *value, const char *type, const char *units, const char *slope, int tmax, int dmax, const char *group, const char *cluster, const char *desc, const char *title)
Send a metric to gmond.
Definition: send_gmetric.c:52