artdaq_ganglia_plugin  v1_02_17
malloc_free.cc
1 // This file (malloc_free.cc) was created by Ron Rechenmacher <ron@fnal.gov> on
2 // Dec 2, 2015. "TERMS AND CONDITIONS" governing this file are in the README
3 // or COPYING file. If you do not have such a file, one can be obtained by
4 // contacting Ron or Fermi Lab in Batavia IL, 60510, phone: 630-840-3000.
5 // $RCSfile: malloc_free.cc,v $
6 static char const* rev = "$Revision: 1.12 $$Date: 2016/11/04 13:57:24 $";
7 
8 // This programm attemps to do a bunch of random mallocs and frees.
9 //
10 // make malloc_free CXX="g++ -std=c++0x"
11 // CXX="g++ -std=c++0x" make -e malloc_free # NOTE "-e"
12 
13 #include <getopt.h> // getopt_long, {no,required,optional}_argument, extern char *optarg; extern int opt{ind,err,opt}
14 #include <stdio.h> // printf
15 #include <stdlib.h> // malloc, rand - range [0, RAND_MAX] period at least 2**32
16 // random: 0 to RAND_MAX -- period 16 * ((2^31) - 1), has initstate() and setstate() functions
17 #include <libgen.h> // basename
18 #include <stdint.h> // uint64_t
19 #include <string.h> // memset
20 #include <sys/time.h> // gettimeofday
21 #include <time.h> // time()
22 #include <unistd.h> // getpid()
23 #include <boost/thread.hpp>
24 #include <mutex>
25 #include <set>
26 #include <tuple> // pair
27 #include <vector>
28 #include "send_gmetric.h"
29 
30 #define TRACE_NAME "malloc_free"
31 #if 1
32 #include "trace.h" // TRACE
33 #else
34 #define TRACE(vv, ...) \
35  do \
36  { \
37  if (opt_v >= vv) printf(__VA_ARGS__); \
38  } while (0)
39 #endif
40 
41 #define USAGE \
42  "\
43  usage: %s [-hv?]\n\
44 \n\
45 example: %s\n\
46 \n\
47 option:\n\
48  -V print version and exit\n\
49  --help\n\
50  --seed= random number generator seed\n\
51  --min= min alloc default=%u\n\
52  --max= max alloc default=%u\n\
53  -t --time= seconds\n\
54  --new use new/delete\n\
55  --no-seed \n\
56  --ave= default=%d\n\
57  --threads= default=%u\n\
58 ", \
59  basename(argv[0]), basename(argv[0]), opt_min_alloc, opt_max_alloc, opt_ave, opt_threads
60 
61 /* GLOBALS */
62 uint64_t opt_hi_water = 1 * (1024 * 1024 * 1024);
63 uint64_t opt_lo_water = 8 * (1024 * 1024);
64 unsigned opt_seed = 2, opt_seed_ = 1; // 2 variables -- to allow any possible seed value
65 unsigned opt_max_alloc = 128 * (1024);
66 unsigned opt_min_alloc = 1 * (1024);
67 int opt_v = 0;
68 int opt_new = 0;
69 unsigned opt_ave = 100000;
70 int opt_threads = 1;
71 int opt_time = 25;
72 int g_stop = 0;
73 
74 uint64_t human(char* optarg)
75 {
76  char* endptr;
77  uint64_t ret = strtoul(optarg, &endptr, 0);
78  if (*endptr == 'k' || *endptr == 'K')
79  ret *= 1024;
80  else if (*endptr == 'm' || *endptr == 'M')
81  ret *= 1024 * 1024;
82  else if (*endptr == 'g' || *endptr == 'G')
83  ret *= 1024 * 1024 * 1024;
84  return ret;
85 }
86 
87 void parse_args(int argc, char* argv[])
88 {
89  // parse opt, optargs, and args
90  while (1)
91  {
92  int opt;
93  static struct option long_options[] = {// name has_arg *flag val
94  {"help", no_argument, 0, 'h'},
95  {"seed", required_argument, 0, 's'},
96  {"hi", required_argument, 0, 1},
97  {"lo", required_argument, 0, 2},
98  {"min", required_argument, 0, 3},
99  {"max", required_argument, 0, 4},
100  {"no-seed", no_argument, 0, 5},
101  {"ave", required_argument, 0, 6},
102  {"threads", required_argument, 0, 7},
103  {"time", required_argument, 0, 't'},
104  {0, 0, 0, 0}};
105  opt = getopt_long(argc, argv, "?hvVs:t:", long_options, NULL);
106  if (opt == -1) break;
107  switch (opt)
108  {
109  case '?':
110  case 'h':
111  printf(USAGE);
112  exit(0);
113  break;
114  case 'V':
115  printf("%s\n", rev);
116  exit(0);
117  break;
118  case 'v':
119  ++opt_v;
120  break;
121  case 's':
122  opt_seed = strtoul(optarg, NULL, 0);
123  opt_seed_ = 1;
124  break;
125  case 't':
126  opt_time = strtoul(optarg, NULL, 0);
127  break;
128  case 1:
129  opt_hi_water = human(optarg);
130  break;
131  case 2:
132  opt_lo_water = human(optarg);
133  break;
134  case 3:
135  opt_min_alloc = human(optarg);
136  break;
137  case 4:
138  opt_max_alloc = human(optarg);
139  break;
140  case 5:
141  opt_seed_ = 0;
142  break;
143  case 6:
144  opt_ave = strtoul(optarg, NULL, 0);
145  break;
146  case 7:
147  opt_threads = strtoul(optarg, NULL, 0);
148  break;
149  default:
150  printf("?? getopt returned character code 0%o ??\n", opt);
151  exit(1);
152  }
153  }
154 }
155 
156 uint64_t gettimeofday_us(void)
157 {
158  struct timeval tv;
159  gettimeofday(&tv, NULL);
160  return (uint64_t)tv.tv_sec * 1000000 + tv.tv_usec;
161 }
162 
163 struct Stats
164 {
165  Stats()
166  : count_(0), running_tot_us_(0), running_tot_MB_(0.0) {}
167 
168  void update(double MB, int64_t delta_us)
169  {
170  mtx_.lock();
171  running_tot_MB_ += MB;
172  running_tot_us_ += delta_us;
173  times_per_.insert((double)delta_us / MB);
174  ++count_;
175  mtx_.unlock();
176  }
177 
178  void get_clear(double& tot_MB, int64_t& tot_delta_us, int& count, double& min, double& max)
179  {
180  mtx_.lock();
181  tot_MB = running_tot_MB_;
182  tot_delta_us = running_tot_us_;
183  count = count_;
184  if (count == 0)
185  min = max = 0;
186  else
187  {
188  min = *(times_per_.begin());
189  max = *(times_per_.rbegin());
190  }
191  running_tot_MB_ = 0.0;
192  running_tot_us_ = 0;
193  times_per_.clear();
194  count_ = 0;
195  mtx_.unlock();
196  }
197 
198 private:
199  int count_;
200  int64_t running_tot_us_;
201  double running_tot_MB_;
202  std::multiset<double> times_per_;
203  std::mutex mtx_;
204 };
205 
206 struct Malloc
207 {
208  Malloc(Stats& stats)
209  : num_mallocs(0), stats_(stats) {}
210 
211 #define NUM_DELTAS opt_ave
212  int num_mallocs;
213  Stats& stats_;
214 
215  // alloc_bytes is an output -- This function returns a ptr to the memory
216  // and the num bytes allocated.
217  void* malloc_opt(unsigned* alloc_bytes)
218  {
219  void* ptr;
220  if ((opt_max_alloc - opt_min_alloc) == 0)
221  *alloc_bytes = opt_max_alloc;
222  else
223  *alloc_bytes = (random() % (opt_max_alloc - opt_min_alloc)) + opt_min_alloc;
224 
225  uint64_t t0 = gettimeofday_us();
226 
227  ptr = malloc(*alloc_bytes);
228  if (ptr == NULL)
229  {
230  perror("malloc");
231  exit(1);
232  }
233  unsigned ii;
234  for (ii = 0; ii < *alloc_bytes; ii += 4096) *(int*)((char*)ptr + ii) = 1;
235  ii /= 4096; // convert offset to pages
236 
237  int64_t delta_us = gettimeofday_us() - t0;
238 #define PAGES_PER_MB (0x100000 / 4096.0)
239  double MB = (double)ii / PAGES_PER_MB;
240  TLOG(TLVL_INFO) << "malloc " << *alloc_bytes << " bytes ii=" << ii << " MB=" << MB;
241 
242  stats_.update(MB, delta_us);
243 
244  return ptr;
245  }
246 };
247 
248 static void do_work(Stats* stats)
249 {
250  std::vector<std::pair<void*, int>> ptr_vec;
251  uint64_t total_allocation = 0;
252  int direction; // 1 is up, 0 is down.
253  enum
254  {
255  up,
256  down
257  };
258 
259  Malloc mm(*stats);
260 
261  while (total_allocation < opt_lo_water)
262  {
263  unsigned alloc_bytes;
264  int8_t* ptr = (int8_t*)mm.malloc_opt(&alloc_bytes);
265  std::pair<void*, int> xx(ptr, alloc_bytes);
266  ptr_vec.push_back(xx);
267  total_allocation += alloc_bytes;
268  }
269  direction = up;
270 
271  printf("ptr_vec.size()=%ld\n", ptr_vec.size());
272  while (!g_stop)
273  {
274  long rr = random();
275  // for the "up" direction, higher probability for malloc
276  long tst = (direction == up) ? RAND_MAX / 4 * 3 : RAND_MAX / 4 * 1;
277  int malloc1_or_free0 = rr < tst;
278 
279  if (malloc1_or_free0)
280  {
281  // do mallocs
282  unsigned alloc_bytes;
283  int8_t* ptr = (int8_t*)mm.malloc_opt(&alloc_bytes);
284  std::pair<void*, int> xx(ptr, alloc_bytes);
285  ptr_vec.push_back(xx);
286  total_allocation += alloc_bytes;
287  if (total_allocation > opt_hi_water && direction == up)
288  {
289  TLOG(TLVL_WARNING) << "direction=down";
290  direction = down;
291  }
292  }
293  else
294  {
295  // do frees
296  if (ptr_vec.size() == 0)
297  {
298  printf("size()==0 - continue\n");
299  continue;
300  }
301  unsigned free_idx = random() % ptr_vec.size();
302  std::pair<void*, int> xx(ptr_vec[free_idx]);
303  ptr_vec.erase(ptr_vec.begin() + free_idx);
304  TLOG(TLVL_INFO) << "free " << xx.second;
305  free(xx.first);
306  total_allocation -= xx.second;
307  if (total_allocation < opt_lo_water && direction == down)
308  {
309  TLOG(TLVL_WARNING) << "direction=up";
310  direction = up;
311  }
312  }
313  }
314 }
315 
316 void send_metrics(double ave, double min, double max, double MBperalloc, int32_t threads)
317 {
318 #define CLUSTER "mu2e DAQ"
319 #define GROUP "memory"
320  send_gmetric("ave", std::to_string(ave).c_str(), "double", "us/MB", "both", 30, 0, GROUP, CLUSTER, "ave us/MB",
321  "ave us/MB");
322  send_gmetric("min", std::to_string(min).c_str(), "double", "us/MB", "both", 30, 0, GROUP, CLUSTER, "min us/MB",
323  "min us/MB");
324  send_gmetric("max", std::to_string(max).c_str(), "double", "us/MB", "both", 30, 0, GROUP, CLUSTER, "max us/MB",
325  "max us/MB");
326  send_gmetric("ave_MBperalloc", std::to_string(MBperalloc).c_str(), "double", "MB/alloc", "both", 30, 0, GROUP,
327  CLUSTER, "ave MB/alloc", "ave MB/alloc");
328  send_gmetric("threads", std::to_string(threads).c_str(), "int32", "threads", "both", 30, 0, GROUP, CLUSTER,
329  "threads mallocing", "threads mallocing");
330 }
331 
332 int main(int argc, char* argv[])
333 {
334  parse_args(argc, argv);
335 
336  if (opt_hi_water < opt_lo_water)
337  {
338  printf("changeing hi_water from %lu to %lu\n", opt_hi_water, opt_lo_water);
339  opt_hi_water = opt_lo_water;
340  }
341  if (opt_max_alloc < opt_min_alloc)
342  {
343  printf("changeing max_alloc from %u to %u\n", opt_max_alloc, opt_min_alloc);
344  opt_max_alloc = opt_min_alloc;
345  }
346 
347  if (!opt_seed_) opt_seed = time(NULL) * getpid();
348  srandom(opt_seed);
349  printf("opt_seed=%u RAND_MAX=%d random=%ld\n", opt_seed, RAND_MAX, random());
350  printf("lo=%ld hi=%ld min=%u max=%u\n", opt_lo_water, opt_hi_water, opt_min_alloc, opt_max_alloc);
351 
352  init_gmetric("/etc/ganglia/gmond.conf");
353 
354  std::vector<boost::thread> threads(opt_threads);
355  std::vector<Stats> stats(opt_threads);
356  for (auto nn = 0; nn < opt_threads; ++nn)
357  {
358  boost::thread::attributes attrs;
359  attrs.set_stack_size(4096 * 2000); // 8000 KB
360  try
361  {
362  threads[nn] = boost::thread(attrs, boost::bind(&do_work, &stats[nn]));
363  }
364  catch (const boost::exception& e)
365  {
366  TLOG(TLVL_ERROR) << "Caught boost::exception starting malloc_free thread: " << boost::diagnostic_information(e)
367  << ", errno=" << errno;
368  std::cerr << "Caught boost::exception starting malloc_free thread: " << boost::diagnostic_information(e)
369  << ", errno=" << errno << std::endl;
370  exit(5);
371  }
372  }
373 
374  uint64_t start_us = gettimeofday_us();
375  uint64_t end_us = start_us + (1000000L * opt_time);
376  usleep(5000000);
377  for (auto cc = 2; 1; ++cc)
378  {
379  double tot_ave_usperMB = 0.0, tot_min_usperMB = 0.0, tot_max_usperMB = 0.0, tot_MBperalloc = 0.0;
380  for (auto nn = 0; nn < opt_threads; ++nn)
381  {
382  double tot_MB;
383  int64_t tot_us;
384  int count;
385  double min, max;
386  stats[nn].get_clear(tot_MB, tot_us, count, min, max);
387  tot_ave_usperMB += (tot_MB != 0.0) ? (double)tot_us / tot_MB : 0.0;
388  tot_min_usperMB += min;
389  tot_max_usperMB += max;
390  tot_MBperalloc += count ? tot_MB / count : 0;
391  TLOG(6) << "nn=" << nn << " tot_MB=" << std::setprecision(4) << tot_MB << " count=" << count;
392  }
393  if ((cc % 5) == 0)
394  TLOG(TLVL_ERROR) << opt_threads << " ave malloc usec/MB ave=" << std::setprecision(4)
395  << tot_ave_usperMB / opt_threads << " min=" << tot_min_usperMB / opt_threads
396  << " max=" << tot_max_usperMB / opt_threads << " MB/alloc=" << std::setprecision(1)
397  << tot_MBperalloc / opt_threads;
398 
399  send_metrics(tot_ave_usperMB / opt_threads, tot_min_usperMB / opt_threads, tot_max_usperMB / opt_threads,
400  tot_MBperalloc / opt_threads, opt_threads);
401 
402  uint64_t target = start_us + (5000000L * cc);
403  uint64_t now_us = gettimeofday_us();
404  int64_t slp = target - now_us;
405  if (slp > 0)
406  {
407  if ((now_us + slp) > end_us) break;
408  usleep(slp);
409  }
410  else
411  {
412  printf("cc=%d slp=%ld\n", cc, slp);
413  if (now_us > end_us) break;
414  }
415  }
416  send_metrics(0.0, 0.0, 0.0, 0.0, 0);
417  g_stop = 1;
418  for (auto nn = 0; nn < opt_threads; ++nn)
419  {
420  threads[nn].join();
421  }
422  printf("malloc_free finished.\n");
423  return (0);
424 } // main
int init_gmetric(const char *conf)
Initialize Ganglia.
Definition: send_gmetric.c:19
int send_gmetric(const char *name, const char *value, const char *type, const char *units, const char *slope, int tmax, int dmax, const char *group, const char *cluster, const char *desc, const char *title)
Send a metric to gmond.
Definition: send_gmetric.c:57