00001
00002
00003
00004
00005
00006 static char const* rev = "$Revision: 1.12 $$Date: 2016/11/04 13:57:24 $";
00007
00008
00009
00010
00011
00012
00013 #include <stdio.h>
00014 #include <getopt.h>
00015 #include <stdlib.h>
00016
00017 #include <libgen.h>
00018 #include <time.h>
00019 #include <sys/time.h>
00020 #include <unistd.h>
00021 #include <stdint.h>
00022 #include <string.h>
00023 #include <vector>
00024 #include <tuple>
00025 #include <set>
00026 #include <mutex>
00027 #include <boost/thread.hpp>
00028 #include "send_gmetric.h"
00029
00030 #define TRACE_NAME "malloc_free"
00031 #if 1
00032 # include "trace.h"
00033 #else
00034 # define TRACE( vv, ... ) do { if (opt_v>=vv) printf( __VA_ARGS__ ); } while (0)
00035 #endif
00036
00037 #define USAGE "\
00038 usage: %s [-hv?]\n\
00039 \n\
00040 example: %s\n\
00041 \n\
00042 option:\n\
00043 -V print version and exit\n\
00044 --help\n\
00045 --seed= random number generator seed\n\
00046 --min= min alloc default=%u\n\
00047 --max= max alloc default=%u\n\
00048 -t --time= seconds\n\
00049 --new use new/delete\n\
00050 --no-seed \n\
00051 --ave= default=%d\n\
00052 --threads= default=%u\n\
00053 ", basename(argv[0]), basename(argv[0]), opt_min_alloc, opt_max_alloc\
00054 , opt_ave, opt_threads
00055
00056
00057 uint64_t opt_hi_water = 1 * (1024 * 1024 * 1024);
00058 uint64_t opt_lo_water = 8 * (1024 * 1024);
00059 unsigned opt_seed = 2, opt_seed_ = 1;
00060 unsigned opt_max_alloc = 128 * (1024);
00061 unsigned opt_min_alloc = 1 * (1024);
00062 int opt_v = 0;
00063 int opt_new = 0;
00064 unsigned opt_ave = 100000;
00065 int opt_threads = 1;
00066 int opt_time = 25;
00067 int g_stop = 0;
00068
00069 uint64_t human(char* optarg)
00070 {
00071 char* endptr;
00072 uint64_t ret = strtoul(optarg, &endptr, 0);
00073 if (*endptr == 'k' || *endptr == 'K') ret *= 1024;
00074 else if (*endptr == 'm' || *endptr == 'M') ret *= 1024 * 1024;
00075 else if (*endptr == 'g' || *endptr == 'G') ret *= 1024 * 1024 * 1024;
00076 return ret;
00077 }
00078
00079 void parse_args(int argc, char* argv[])
00080 {
00081
00082 while (1)
00083 {
00084 int opt;
00085 static struct option long_options[] = {
00086
00087 {"help", no_argument, 0, 'h'},
00088 {"seed", required_argument,0, 's'},
00089 {"hi", required_argument,0, 1},
00090 {"lo", required_argument,0, 2},
00091 {"min", required_argument,0, 3},
00092 {"max", required_argument,0, 4},
00093 {"no-seed", no_argument, 0, 5},
00094 {"ave", required_argument,0, 6},
00095 {"threads", required_argument,0, 7},
00096 {"time", required_argument,0, 't'},
00097 {0, 0, 0, 0}
00098 };
00099 opt = getopt_long(argc, argv, "?hvVs:t:",
00100 long_options, NULL);
00101 if (opt == -1) break;
00102 switch (opt)
00103 {
00104 case '?': case 'h': printf(USAGE);
00105 exit(0);
00106 break;
00107 case 'V': printf("%s\n", rev);
00108 exit(0);
00109 break;
00110 case 'v': ++opt_v;
00111 break;
00112 case 's': opt_seed = strtoul(optarg, NULL, 0);
00113 opt_seed_ = 1;
00114 break;
00115 case 't': opt_time = strtoul(optarg, NULL, 0);
00116 break;
00117 case 1: opt_hi_water = human(optarg);
00118 break;
00119 case 2: opt_lo_water = human(optarg);
00120 break;
00121 case 3: opt_min_alloc = human(optarg);
00122 break;
00123 case 4: opt_max_alloc = human(optarg);
00124 break;
00125 case 5: opt_seed_ = 0;
00126 break;
00127 case 6: opt_ave = strtoul(optarg, NULL, 0);
00128 break;
00129 case 7: opt_threads = strtoul(optarg, NULL, 0);
00130 break;
00131 default:
00132 printf("?? getopt returned character code 0%o ??\n", opt);
00133 exit(1);
00134 }
00135 }
00136 }
00137
00138 uint64_t gettimeofday_us(void)
00139 {
00140 struct timeval tv;
00141 gettimeofday(&tv, NULL);
00142 return (uint64_t)tv.tv_sec * 1000000 + tv.tv_usec;
00143 }
00144
00145 struct Stats
00146 {
00147 Stats() : count_(0)
00148 , running_tot_us_(0)
00149 , running_tot_MB_(0.0) {}
00150
00151 void update(double MB, int64_t delta_us)
00152 {
00153 mtx_.lock();
00154 running_tot_MB_ += MB;
00155 running_tot_us_ += delta_us;
00156 times_per_.insert((double)delta_us / MB);
00157 ++count_;
00158 mtx_.unlock();
00159 }
00160
00161 void get_clear(double& tot_MB, int64_t& tot_delta_us, int& count, double& min, double& max)
00162 {
00163 mtx_.lock();
00164 tot_MB = running_tot_MB_;
00165 tot_delta_us = running_tot_us_;
00166 count = count_;
00167 if (count == 0)
00168 min = max = 0;
00169 else
00170 {
00171 min = *(times_per_.begin());
00172 max = *(times_per_.rbegin());
00173 }
00174 running_tot_MB_ = 0.0;
00175 running_tot_us_ = 0;
00176 times_per_.clear();
00177 count_ = 0;
00178 mtx_.unlock();
00179 }
00180
00181 private:
00182 int count_;
00183 int64_t running_tot_us_;
00184 double running_tot_MB_;
00185 std::multiset<double> times_per_;
00186 std::mutex mtx_;
00187 };
00188
00189 struct Malloc
00190 {
00191 Malloc(Stats& stats) : num_mallocs(0)
00192 , stats_(stats) {}
00193
00194 #define NUM_DELTAS opt_ave
00195 int num_mallocs;
00196 Stats& stats_;
00197
00198
00199
00200 void* malloc_opt(unsigned* alloc_bytes)
00201 {
00202 void* ptr;
00203 if ((opt_max_alloc - opt_min_alloc) == 0)
00204 *alloc_bytes = opt_max_alloc;
00205 else
00206 *alloc_bytes = (random() % (opt_max_alloc - opt_min_alloc)) + opt_min_alloc;
00207
00208 uint64_t t0 = gettimeofday_us();
00209
00210 ptr = malloc(*alloc_bytes);
00211 if (ptr == NULL)
00212 {
00213 perror("malloc");
00214 exit(1);
00215 }
00216 unsigned ii;
00217 for (ii = 0; ii < *alloc_bytes; ii += 4096)
00218 *(int*)((char*)ptr + ii) = 1;
00219 ii /= 4096;
00220
00221 int64_t delta_us = gettimeofday_us() - t0;
00222 # define PAGES_PER_MB (0x100000/4096.0)
00223 double MB = (double)ii / PAGES_PER_MB;
00224 TLOG(TLVL_INFO) << "malloc " << *alloc_bytes << " bytes ii=" << ii << " MB=" << MB;
00225
00226 stats_.update(MB, delta_us);
00227
00228 return ptr;
00229 }
00230 };
00231
00232
00233 static void do_work(Stats* stats)
00234 {
00235 std::vector<std::pair<void*, int>> ptr_vec;
00236 uint64_t total_allocation = 0;
00237 int direction;
00238 enum
00239 {
00240 up,
00241 down
00242 };
00243
00244 Malloc mm(*stats);
00245
00246 while (total_allocation < opt_lo_water)
00247 {
00248 unsigned alloc_bytes;
00249 int8_t* ptr = (int8_t*)mm.malloc_opt(&alloc_bytes);
00250 std::pair<void*, int> xx(ptr, alloc_bytes);
00251 ptr_vec.push_back(xx);
00252 total_allocation += alloc_bytes;
00253 }
00254 direction = up;
00255
00256 printf("ptr_vec.size()=%ld\n", ptr_vec.size());
00257 while (!g_stop)
00258 {
00259 long rr = random();
00260
00261 long tst = (direction == up) ? RAND_MAX / 4 * 3 : RAND_MAX / 4 * 1;
00262 int malloc1_or_free0 = rr < tst;
00263
00264 if (malloc1_or_free0)
00265 {
00266
00267 unsigned alloc_bytes;
00268 int8_t* ptr = (int8_t*)mm.malloc_opt(&alloc_bytes);
00269 std::pair<void*, int> xx(ptr, alloc_bytes);
00270 ptr_vec.push_back(xx);
00271 total_allocation += alloc_bytes;
00272 if (total_allocation > opt_hi_water && direction == up)
00273 {
00274 TLOG(TLVL_WARNING) << "direction=down";
00275 direction = down;
00276 }
00277 }
00278 else
00279 {
00280
00281 if (ptr_vec.size() == 0)
00282 {
00283 printf("size()==0 - continue\n");
00284 continue;
00285 }
00286 unsigned free_idx = random() % ptr_vec.size();
00287 std::pair<void*, int> xx(ptr_vec[free_idx]);
00288 ptr_vec.erase(ptr_vec.begin() + free_idx);
00289 TLOG(TLVL_INFO) << "free " << xx.second;
00290 free(xx.first);
00291 total_allocation -= xx.second;
00292 if (total_allocation < opt_lo_water && direction == down)
00293 {
00294 TLOG(TLVL_WARNING) << "direction=up";
00295 direction = up;
00296 }
00297 }
00298 }
00299 }
00300
00301 void send_metrics(double ave, double min, double max, double MBperalloc, int32_t threads)
00302 {
00303 # define CLUSTER "mu2e DAQ"
00304 # define GROUP "memory"
00305 send_gmetric("ave", std::to_string(ave).c_str(), "double", "us/MB"
00306 , "both", 30, 0, GROUP, CLUSTER, "ave us/MB", "ave us/MB");
00307 send_gmetric("min", std::to_string(min).c_str(), "double", "us/MB"
00308 , "both", 30, 0, GROUP, CLUSTER, "min us/MB", "min us/MB");
00309 send_gmetric("max", std::to_string(max).c_str(), "double", "us/MB"
00310 , "both", 30, 0, GROUP, CLUSTER, "max us/MB", "max us/MB");
00311 send_gmetric("ave_MBperalloc", std::to_string(MBperalloc).c_str(), "double", "MB/alloc"
00312 , "both", 30, 0, GROUP, CLUSTER, "ave MB/alloc", "ave MB/alloc");
00313 send_gmetric("threads", std::to_string(threads).c_str(), "int32", "threads"
00314 , "both", 30, 0, GROUP, CLUSTER, "threads mallocing", "threads mallocing");
00315 }
00316
00317
00318 int main(int argc, char* argv[])
00319 {
00320 parse_args(argc, argv);
00321
00322 if (opt_hi_water < opt_lo_water)
00323 {
00324 printf("changeing hi_water from %lu to %lu\n", opt_hi_water, opt_lo_water);
00325 opt_hi_water = opt_lo_water;
00326 }
00327 if (opt_max_alloc < opt_min_alloc)
00328 {
00329 printf("changeing max_alloc from %u to %u\n", opt_max_alloc, opt_min_alloc);
00330 opt_max_alloc = opt_min_alloc;
00331 }
00332
00333 if (!opt_seed_)
00334 opt_seed = time(NULL) * getpid();
00335 srandom(opt_seed);
00336 printf("opt_seed=%u RAND_MAX=%d random=%ld\n", opt_seed, RAND_MAX, random());
00337 printf("lo=%ld hi=%ld min=%u max=%u\n", opt_lo_water, opt_hi_water, opt_min_alloc, opt_max_alloc);
00338
00339 init_gmetric("/etc/ganglia/gmond.conf");
00340
00341
00342 std::vector<boost::thread> threads(opt_threads);
00343 std::vector<Stats> stats(opt_threads);
00344 for (auto nn = 0; nn < opt_threads; ++nn)
00345 {
00346 boost::thread::attributes attrs;
00347 attrs.set_stack_size(4096 * 200);
00348 threads[nn] = boost::thread(attrs, boost::bind(&do_work, &stats[nn]));
00349 }
00350
00351 uint64_t start_us = gettimeofday_us();
00352 uint64_t end_us = start_us + (1000000L * opt_time);
00353 usleep(5000000);
00354 for (auto cc = 2; 1; ++cc)
00355 {
00356 double tot_ave_usperMB = 0.0, tot_min_usperMB = 0.0, tot_max_usperMB = 0.0, tot_MBperalloc = 0.0;
00357 for (auto nn = 0; nn < opt_threads; ++nn)
00358 {
00359 double tot_MB;
00360 int64_t tot_us;
00361 int count;
00362 double min, max;
00363 stats[nn].get_clear(tot_MB, tot_us, count, min, max);
00364 tot_ave_usperMB += (tot_MB != 0.0) ? (double)tot_us / tot_MB : 0.0;
00365 tot_min_usperMB += min;
00366 tot_max_usperMB += max;
00367 tot_MBperalloc += count ? tot_MB / count : 0;
00368 TLOG(6) << "nn=" << nn << " tot_MB=" << std::setprecision(4) << tot_MB << " count=" << count;
00369 }
00370 if ((cc % 5) == 0)
00371 TLOG(TLVL_ERROR) << opt_threads
00372 << " ave malloc usec/MB ave=" << std::setprecision(4) << tot_ave_usperMB / opt_threads
00373 << " min=" << tot_min_usperMB / opt_threads << " max=" << tot_max_usperMB / opt_threads
00374 << " MB/alloc=" << std::setprecision(1) << tot_MBperalloc / opt_threads;
00375
00376 send_metrics(tot_ave_usperMB / opt_threads
00377 , tot_min_usperMB / opt_threads
00378 , tot_max_usperMB / opt_threads
00379 , tot_MBperalloc / opt_threads
00380 , opt_threads);
00381
00382 uint64_t target = start_us + (5000000L * cc);
00383 uint64_t now_us = gettimeofday_us();
00384 int64_t slp = target - now_us;
00385 if (slp > 0)
00386 {
00387 if ((now_us + slp) > end_us)
00388 break;
00389 usleep(slp);
00390 }
00391 else
00392 {
00393 printf("cc=%d slp=%ld\n", cc, slp);
00394 if (now_us > end_us)
00395 break;
00396 }
00397 }
00398 send_metrics(0.0, 0.0, 0.0, 0.0, 0);
00399 g_stop = 1;
00400 for (auto nn = 0; nn < opt_threads; ++nn)
00401 {
00402 threads[nn].join();
00403 }
00404 printf("malloc_free finished.\n");
00405 return (0);
00406 }