6 static const char*rev=
"$Revision: 1.19 $$Date: 2018/06/28 21:14:28 $";
12 usage: %s --cmd=<cmd>\n\
13 examples: %s --cmd='sleep 25' # this will take about a minute\n\
14 %s --cmd='taskset -c 4 dd if=/dev/zero of=/dev/sdf bs=50M count=500 oflag=direct' --disk=sdf\n\
15 %s --cmd='dd if=/dev/zero of=t.dat count=500' --stat='md*MB/s?/proc/diskstats?/md[0-3]/?$10?(1.0/2048)?yes'\n\
16 gnuplot -e png=0 `/bin/ls -t periodic_*_stats.out|head -1`\n\
17 For each cmd, record CPU info. Additionally, record total system CPU\n\
18 (can be >100 w/ multicore), file system Cached and Dirty %%, plus any\n\
19 other stats specified by --stats options\n\
21 --cmd= can have multiple\n\
22 --Cmd= run cmd, but dont graph CPU (can have multiple)\n\
23 --pid= graph comma or space sep. list of pids (getting cmd from /proc/)\n\
24 --disk= automatically build --stat for disk(s)\n\
25 --stat= desc?file?linespec?fieldspec?multiplier?rate\n\
26 builtin = CPUnode,Cached,Dirty\n\
27 cmd-builtin = CPUcmdN, CPU+cmdN\n\
28 --out-dir= output dir (for 2+ output files; stats and cmd+)\n\
30 --period= (float) default=%s\n\
31 --pre= the number of periods wait before exec of --cmd\n\
32 --post= the number of periods to loop after cmd exits\n\
33 --graph= add to graph any possibly included non-graphed metrics\n\
35 --no-defaults for no default stats (ie. cpu-parent,cpu-children)\n\
36 --init= default dropcache if root. NOT implemented yet\n\
43 --sys-iowait,-w include the system iowait on the graph\n\
44 --cmd-iowait include cmd iowait on the graph\n\
46 ", basename(argv[0]),basename(argv[0]),basename(argv[0]),basename(argv[0]),opt_period
48 enum { s_desc, s_file, s_linespec, s_fieldspec, s_multiplier, s_rate };
57 #include <sys/utsname.h>
65 # define TRACE_NAME "periodic_cmd_stats"
71 static void trace_ap(
const char*msg,va_list ap)
72 {
char m_[1024];
unsigned len=strlen(msg);
73 len=len<(
sizeof(m_)-2)?len:(
sizeof(m_)-2); memcpy(m_,msg,len+1);
74 if(m_[len-1]!=
'\n') memcpy(&(m_[len]),
"\n",2);
75 vprintf(m_,ap);va_end(ap);
77 static void trace_p(
const char*msg,...)__attribute__((format(printf,1,2)));
78 static
void trace_p(const
char*msg,...)
79 { va_list ap;va_start(ap,msg); trace_ap(msg ,ap); va_end(ap);
81 static void trace_p(
const std::string msg,...)
82 { va_list ap;va_start(ap,msg); trace_ap(msg.c_str(),ap); va_end(ap);
84 # define TRACE(lvl,...) do if(lvl<=1)trace_p(__VA_ARGS__);while(0)
85 # define TRACE_CNTL( xyzz,... )
91 std::vector<std::string> opt_cmd;
92 std::vector<std::string> opt_Cmd;
95 std::string opt_stats;
96 std::string opt_outdir(
"");
97 std::string opt_graph(
"CPUnode,Cached,Dirty,Free");
98 const char* opt_period=
"5.0";
99 std::string opt_comment;
107 int opt_sys_iowait=0;
108 int opt_cmd_iowait=0;
111 std::vector<pid_t> g_pid_vec;
113 void charreplace(
char* instr,
char oldc,
char newc )
123 void parse_args(
int argc,
char *argv[] )
129 static struct option long_options[] = {
131 {
"help", no_argument, 0,
'h' },
132 {
"init", required_argument,0,
'i' },
133 {
"cmd", required_argument,0,
'c' },
134 {
"Cmd", required_argument,0,
'C' },
135 {
"disk", required_argument,0,
'd' },
136 {
"stat", required_argument,0,
's' },
137 {
"out-dir", required_argument,0,
'o' },
138 {
"period", required_argument,0,
'p' },
139 {
"sys-iowait", no_argument, 0,
'w' },
140 {
"fault", no_argument, 0,
'f' },
141 {
"pid", required_argument,0,
'P' },
142 {
"ymax", required_argument,0, 1 },
143 {
"yincr", required_argument,0, 2 },
144 {
"y2max", required_argument,0, 3 },
145 {
"y2incr", required_argument,0, 4 },
146 {
"pre", required_argument,0, 5 },
147 {
"post", required_argument,0, 6 },
148 {
"graph", required_argument,0, 7 },
149 {
"yrange", required_argument,0, 8 },
150 {
"comment", required_argument,0, 9 },
151 {
"cmd-iowait", no_argument, 0, 10 },
154 opt = getopt_long( argc, argv,
"?hvqVi:c:C:d:s:o:p:P:wf",
155 long_options, NULL );
156 if (opt == -1)
break;
158 case '?':
case 'h': printf(USAGE); exit(0);
break;
159 case 'V': printf(
"%s\n",rev); exit(0);
break;
160 case 'v': ++opt_v;
break;
161 case 'q': --opt_v;
break;
162 case 'i': opt_init=optarg;
break;
163 case 'c': opt_cmd.push_back(optarg);
break;
164 case 'C': opt_Cmd.push_back(optarg);
break;
165 case 'd':
if(opt_disk.size())opt_disk=opt_disk+
","+optarg;
else opt_disk=optarg;
break;
166 case 's':
if(opt_stats.size())opt_stats=opt_stats+
","+optarg;
else opt_stats=optarg;
break;
167 case 'o': opt_outdir=std::string(optarg)+
"/";
break;
168 case 'p': opt_period=optarg;
break;
169 case 'w': opt_sys_iowait=1;
break;
170 case 'f': opt_fault=1;
break;
171 case 'P': charreplace(optarg,
' ',
',');
172 if(opt_pid.size())opt_pid=opt_pid+
","+optarg;
175 case 1: opt_ymax=strtoul(optarg,NULL,0);
break;
176 case 2: opt_yincr=strtoul(optarg,NULL,0);
break;
177 case 3: opt_y2max=strtoul(optarg,NULL,0);
break;
178 case 4: opt_y2incr=strtoul(optarg,NULL,0);
break;
179 case 5: opt_pre=strtoul(optarg,NULL,0);
break;
180 case 6: opt_post=strtoul(optarg,NULL,0);
break;
181 case 7: opt_graph+=std::string(
",")+optarg;
break;
182 case 8: opt_ymin =strtoul(optarg,NULL,0);
183 cp = strstr(optarg,
":")+1;
184 opt_ymax =strtoul(cp,NULL,0);
185 if ((cp=strstr(cp,
":"))) {
187 opt_yincr=strtoul(strstr(cp,
":")+1,NULL,0);
189 opt_yincr=(opt_ymax-opt_ymin)/5;
191 case 9: opt_comment=optarg;
break;
192 case 10: opt_cmd_iowait=1;
break;
194 printf(
"?? getopt returned character code 0%o ??\n", opt );
200 void perror_exit(
const char *msg, ... )
202 va_list ap;va_start(ap,msg);
203 vsnprintf( buf,
sizeof(buf), msg, ap );
205 TRACE( 0,
"%s", buf );
206 perror( buf ); exit(1);
216 pid_t fork_execv(
int close_start,
int close_cnt,
int sleepB4exec_us,
int iofd[3],
const char *cmd,
char *
const argv[],
char *
const env[] )
220 for (
auto ii=0; ii<3; ++ii) {
221 lcl_iofd[ii]=iofd[ii];
224 iofd[ii]=ii==0?pipes[ii][1]:pipes[ii][0];
228 if (pid < 0) perror_exit(
"fork");
230 if (lcl_iofd[0]==-1) {
232 int fd=dup2(pipes[0][0],0);
233 TRACE( 3,
"fork_execv dupped(%d) onto %d (should be 0)", pipes[0][0], fd );
236 if (sleepB4exec_us) {
239 usleep(sleepB4exec_us);
240 TRACE( 1,
"fork_execv sleep complete. sleepB4exec_us=%d sts=%d", sleepB4exec_us, 0 );
242 for (
auto ii=1; ii<3; ++ii) {
243 if (lcl_iofd[ii]==-1) {
245 int fd=dup2(pipes[ii][1],ii);
246 TRACE( 3,
"fork_execv dupped(%d) onto %d (should be %d)", pipes[ii][1], fd,ii );
248 }
else if (lcl_iofd[ii]!=ii) {
249 int fd=dup2(lcl_iofd[ii],ii);
250 TRACE( 3,
"fork_execv dupped(%d) onto %d (should be %d)", pipes[ii][1], fd,ii );
253 for (
auto ii=close_start; ii<(close_start+close_cnt); ++ii)
256 execve( cmd, argv, env );
261 for (
auto ii=0; ii<3; ++ii)
262 if (lcl_iofd[ii]==-1)
263 close(ii==0?pipes[ii][0]:pipes[ii][1]);
266 TRACE( 3,
"fork_execv pid=%d", pid );
270 uint64_t swapPtr(
void *X )
272 uint64_t x=(uint64_t)X;
273 x = (x & 0x00000000ffffffff) << 32 | (x & 0xffffffff00000000) >> 32;
274 x = (x & 0x0000ffff0000ffff) << 16 | (x & 0xfff0000fffff0000) >> 16;
275 x = (x & 0x00ff00ff00ff00ff) << 8 | (x & 0xff00ff00ff00ff00) >> 8;
298 static int g_devnullfd=-1;
301 std::string AWK( std::string
const &awk_cmd,
const char *file,
const char *input )
304 ssize_t bytes=0, tot_bytes=0;
305 char *
const argv_[4]={ (
char*)
"/bin/gawk",
306 (
char*)awk_cmd.c_str(),
311 if (g_devnullfd == -1)
312 g_devnullfd=open(
"/dev/null",O_WRONLY);
317 int iofd[3]={infd,-1,2};
318 TRACE( 3,
"AWK b4 fork_execv input=%p", (
void*)input );
321 pid=fork_execv(0,0,0,iofd,
"/bin/gawk",argv_,env);
323 int xx=strlen(input);
324 int sts=write(iofd[0],input,xx);
326 perror(
"write AWK stdin");
328 while ((bytes=read(iofd[1],&readbuf[tot_bytes],
sizeof(readbuf)-tot_bytes)) != 0) {
329 TRACE( 3,
"AWK while bytes=read > 0 bytes=%zd readbuf=0x%016lx errno=%d", bytes, swapPtr(&readbuf[tot_bytes]), errno );
331 if (errno == EINTR)
continue;
336 TRACE( 3,
"AWK after read tot="+std::to_string((
long long unsigned)tot_bytes)+
" bytes="+std::to_string((
long long unsigned)bytes)+
" input="+std::string(input) );
338 while ((bytes=read(iofd[1],&readbuf[tot_bytes],
sizeof(readbuf)-tot_bytes)) > 0)
340 TRACE( 3,
"AWK after read tot=%zd bytes=%zd [0]=0x%x input=%p", tot_bytes, bytes, readbuf[0], (
void*)input );
342 readbuf[tot_bytes>=0?tot_bytes:0]=
'\0';
344 TRACE( 3,
"AWK after close child stdout. child pid=%d", pid );
347 pid_t done_pid = waitpid(pid,&status,0);
348 TRACE( 3,
"AWK after wait pid=%d done_pid=%d status=%d(0x%x)"
349 , pid, done_pid, status, status );
351 return std::string(readbuf);
356 void string_addto_vector( std::string &instr, std::vector<std::string> &outvec,
char delim )
358 std::stringstream ss(instr);
360 { std::string substr;
361 std::getline( ss, substr, delim );
362 outvec.push_back( substr );
366 uint64_t gettimeofday_us(
void )
368 gettimeofday( &tv, NULL );
373 return (uint64_t)tv.tv_sec*1000000+tv.tv_usec;
376 #define DATA_START " DATA START"
377 #define GNUPLOT_PREFIX (const char *)"\
378 #!/usr/bin/env gnuplot\n\
381 # gnuplot -e 'ymin=400;ymax=1400' ./$0\n\
383 # gnuplot -e 'duration_s=35;set multiplot' ./gnuplot.gnuplot ./gnuplot.1.gnuplot -e 'set nomultiplot;pause -1'\n\
384 if(!exists('ARG0')) ARG0='' # for version 4, use: gnuplot -e ARG0=hello\n\
385 print 'ARG0=',ARG0 # ARG0.. automatically define in gnuplot version 5+\n\
386 if(!exists('ymin')) ymin=%d\n\
387 if(!exists('ymax')) ymax=%d\n\
388 if(!exists('yincr')) yincr=%d\n\
389 if(!exists('y2max')) y2max=%d\n\
390 if(!exists('y2incr')) y2incr=%d\n\
391 if(!exists('png')) png=1\n\
392 if(!exists('duration_s')) duration_s=0\n\
393 if(!exists('width')) width=512\n\
394 if(!exists('height')) height=384\n\
395 thisPid=system('echo `ps -p$$ -oppid=`')\n\
396 thisFile=system('ls -l /proc/'.thisPid.\"/fd | grep -v pipe: | tail -1 | sed -e 's/.*-> //'\")\n\
398 set title \"Disk Write Rate and %%CPU vs. time\\n%s %s %s%s\" # cmd and/or comment at end\n\
400 tfmt='%%Y-%%m-%%dT%%H:%%M:%%S' # try to use consistent format\n\
401 set timefmt '%%Y-%%m-%%dT%%H:%%M:%%S'\n\
403 set grid xtics back\n\
404 xstart=system(\"awk '/^....-..-..T/{print$1;exit}' \".thisFile)\n\
405 xend=system(\"awk 'END{print$1}' \".thisFile)\n\
406 print 'xstart='.xstart.' xend='.xend.' duration=',strptime(tfmt,xend)-strptime(tfmt,xstart)\n\
407 if(duration_s>0) end_t=strptime(tfmt,xstart)+duration_s; else end_t=strptime(tfmt,xend)\n\
408 set xrange [xstart:end_t]\n\
411 set ytics nomirror\n\
412 if(ymax==0) set yrange [ymin:*];\\\n\
413 else set yrange [ymin:ymax];set ytics yincr\n\
414 set grid ytics back\n\
416 set y2label '%%CPU, %%MemTotal'\n\
417 set y2tics autofreq\n\
418 if(y2max==0) set y2range [0:*];\\\n\
419 else set y2range [0:y2max];set y2tics y2incr\n\
422 if(png==1) set terminal png size width,height;\\\n\
423 pngfile=system( 'echo `basename '.thisFile.' .out`.png' );\\\n\
424 set output pngfile;\\\n\
425 else set terminal x11 size width,height\n\
427 plot \"< awk '/^#" DATA_START "/,/NEVER HAPPENS/' \".thisFile "
430 void sigchld_sigaction(
int signo, siginfo_t *info,
void *context __attribute__((__unused__)) )
433 for (
size_t ii=0; ii < g_pid_vec.size(); ++ii) {
434 pid_t pid=g_pid_vec[ii];
435 if (pid == info->si_pid) {
436 TRACE( 2,
"sigchld_sigaction signo=%d status=%d(0x%x) code=%d(0x%x) sending_pid=%d"
438 , info->si_status, info->si_status
439 , info->si_code, info->si_code
445 TRACE( 3,
"sigchld_sigaction signo=%d status=%d(0x%x) code=%d(0x%x) sending_pid=%d"
447 , info->si_status, info->si_status
448 , info->si_code, info->si_code
454 void read_proc_file(
const char *file,
char *buffer,
int buffer_size )
456 TRACE( 4,
"read_proc_file b4 open proc file"+std::string(file) );
457 int fd=open(file,O_RDONLY);
460 sts=read(fd,&buffer[offset],buffer_size-offset);
467 buffer[sts+offset]=
'\0';
469 TRACE( 4,
"read_proc_file after close "+std::string(file)+
" read=%d offset=%d",sts,offset );
473 pid_t check_pid_vec(
void )
475 for (
size_t ii=0; ii < g_pid_vec.size(); ) {
476 pid_t pid=g_pid_vec[ii];
478 pid_t pp = waitpid( pid, &status, WNOHANG );
479 TRACE( 3,
"check_pid_vec %d=waitpid(pid=%d) errno=%d", pp, pid, errno );
481 g_pid_vec.erase( g_pid_vec.begin()+ii );
483 if (errno == ECHILD && kill(pid,0)==0)
488 g_pid_vec.erase( g_pid_vec.begin()+ii );
493 if (g_pid_vec.size() == 0)
501 TRACE( 1,
"atexit cleanup g_pid_vec.size()=%zd\n", g_pid_vec.size() );
502 for (std::vector<pid_t>::iterator pid=g_pid_vec.begin(); pid!=g_pid_vec.end(); ++pid) {
503 kill( *pid, SIGHUP );
506 #if (defined(__cplusplus)&&(__cplusplus>=201103L)) || (defined(__STDC_VERSION__)&&(__STDC_VERSION__>=201112L))
507 # pragma GCC diagnostic push
508 # pragma GCC diagnostic ignored "-Wunused-parameter"
510 void sigint_sigaction(
int signo, siginfo_t *info,
void *context )
515 #if (defined(__cplusplus)&&(__cplusplus>=201103L)) || (defined(__STDC_VERSION__)&&(__STDC_VERSION__>=201112L))
516 # pragma GCC diagnostic pop
525 int post_periods_completed=0;
526 parse_args( argc, argv );
527 if ( (argc-optind)!=0
528 || ( opt_cmd.size()==0
529 && opt_pid.size()==0)) {
531 printf(
"unexpected argument(s) %d!=0\n", argc-optind );
532 for (ii=0; (optind+ii)<argc; ++ii)
533 printf(
"arg%d=%s\n",ii+1,argv[optind+ii]);
534 printf( USAGE ); exit( 0 );
537 std::vector<std::string> graphs;
538 string_addto_vector( opt_graph, graphs,
',' );
540 char motherboard[1024]={0};
542 FILE *fp=popen(
"dmidecode | grep -m2 'Product Name:' | tail -1",
"r" );
543 fread( motherboard, 1,
sizeof(motherboard), fp );
546 TRACE( 1,
"main - motherboard="+std::string(motherboard) );
554 struct sigaction sigaction_s;
556 # define DO_SIGCHLD 1
559 sigaction_s.sa_sigaction = sigchld_sigaction;
560 sigaction_s.sa_flags = SA_SIGINFO|SA_NOCLDWAIT;
562 sigaction_s.sa_handler = SIG_IGN;
563 sigaction_s.sa_flags = SA_NOCLDWAIT;
565 sigemptyset(&sigaction_s.sa_mask);
566 sigaction( SIGCHLD, &sigaction_s, NULL );
568 sigaction_s.sa_sigaction = sigint_sigaction;
569 sigaction_s.sa_flags = SA_SIGINFO;
570 sigaction( SIGINT, &sigaction_s, NULL );
574 long long unsigned concurentThreadsSupported = sysconf(_SC_NPROCESSORS_ONLN);
576 TRACE( 0,
"main concurentThreadsSupported=%u opt_stats="+opt_stats, concurentThreadsSupported );
579 gettimeofday( &tv, NULL );
580 strftime( run_time,
sizeof(run_time),
"%FT%H%M%S", localtime(&tv.tv_sec) );
581 TRACE( 0,
"main run_time="+std::string(run_time) );
587 if ((dot=strchr(ubuf.nodename,
'.')) != NULL)
589 std::string hostname(ubuf.nodename);
590 TRACE( 1,
"release="+std::string(ubuf.release)+
" version="+std::string(ubuf.version) );
593 std::string memKB=AWK(
"NR==1{print$2;exit}",
"/proc/meminfo",NULL );
594 memKB = memKB.substr(0,memKB.size()-1);
596 std::string dat_file_out(opt_outdir+
"periodic_"+run_time+
"_"+hostname+
"_stats.out");
598 double period=atof(opt_period);
602 std::vector<std::string> pidfile;
604 std::vector<std::string> stats;
608 for (
size_t ii=0; ii<opt_cmd.size(); ++ii) {
609 char cmd_file_out[1024];
610 snprintf( cmd_file_out,
sizeof(cmd_file_out),
"%speriodic_%s_%s_cmd%zd.out"
611 , opt_outdir.c_str(), run_time, hostname.c_str(), ii );
612 int fd=open( cmd_file_out, O_WRONLY|O_CREAT,0666 );
613 TRACE( 0,
"main fd=%d opt_cmd="+opt_cmd[ii]+
" cmd_file_out="+std::string(cmd_file_out), fd );
614 int iofd[3]={0,fd,fd};
615 char *
const argv_[4]={ (
char*)
"/bin/sh",
617 (
char*)opt_cmd[ii].c_str(),
619 g_pid_vec.push_back( fork_execv(0,0,(
int)(period*opt_pre*1e6),iofd,
"/bin/sh",argv_,NULL) );
621 std::string pidstr=std::to_string((
long long int)g_pid_vec[ii]);
622 pidfile.push_back(
"/proc/"+pidstr+
"/stat" );
624 char desc[128], ss[1024];
626 snprintf( ss,
sizeof(ss),
"CPUcmd%zd?%s?NR==1?$14+$15?1?yes", ii, pidfile[ii].c_str() );
627 stats.push_back( ss );
629 snprintf( desc,
sizeof(desc),
"CPU+cmd%zd", ii );
630 graphs.push_back( desc );
631 snprintf( ss,
sizeof(ss),
"%s?%s?NR==1?$14+$15+16+$17?1?yes", desc, pidfile[ii].c_str() );
632 stats.push_back( ss );
634 snprintf( desc,
sizeof(desc),
"WaitBlkIOcmd%zd", ii );
635 if (opt_cmd_iowait) graphs.push_back( desc );
636 snprintf( ss,
sizeof(ss),
"%s?%s?NR==1?$42?1?yes", desc, pidfile[ii].c_str() );
637 stats.push_back( ss );
639 snprintf( desc,
sizeof(desc),
"Faultcmd%zd", ii );
640 if (opt_fault) graphs.push_back( desc );
641 snprintf( ss,
sizeof(ss),
"%s?%s?NR==1?$10+$11+$12+$13?4096.0/1048576?yes", desc, pidfile[ii].c_str() );
642 stats.push_back( ss );
644 for (
size_t ii=0; ii<opt_Cmd.size(); ++ii) {
645 char cmd_file_out[1024];
646 snprintf( cmd_file_out,
sizeof(cmd_file_out),
"%speriodic_%s_%s_cmd%zd.out"
647 , opt_outdir.c_str(), run_time, hostname.c_str(), ii+opt_cmd.size() );
648 int fd=open( cmd_file_out, O_WRONLY|O_CREAT,0666 );
649 TRACE( 0,
"main fd=%d opt_Cmd="+opt_Cmd[ii]+
" cmd_file_out="+std::string(cmd_file_out), fd );
650 int iofd[3]={0,fd,fd};
651 char *
const argv_[4]={ (
char*)
"/bin/sh",
653 (
char*)opt_Cmd[ii].c_str(),
655 g_pid_vec.push_back( fork_execv(0,0,(
int)(period*opt_pre*1e6),iofd,
"/bin/sh",argv_,NULL) );
657 std::string pidstr=std::to_string((
long long int)g_pid_vec[ii]);
658 pidfile.push_back(
"/proc/"+pidstr+
"/stat" );
660 char desc[128], ss[1024];
661 snprintf( desc,
sizeof(desc),
"CPU+cmd%zd", ii+opt_cmd.size() );
662 snprintf( ss,
sizeof(ss),
"CPUcmd%zd?%s?NR==1?$14+$15?1?yes", ii+opt_cmd.size(), pidfile[ii].c_str() );
663 stats.push_back( ss );
664 snprintf( ss,
sizeof(ss),
"CPU+cmd%zd?%s?NR==1?$14+$15+16+$17?1?yes", ii+opt_cmd.size(), pidfile[ii].c_str() );
665 stats.push_back( ss );
668 std::vector<std::string> pids;
670 string_addto_vector( opt_pid, pids,
',' );
671 for (
size_t ii=0; ii<pids.size(); ++ii) {
672 g_pid_vec.push_back( std::stoi(pids[ii]) );
673 TRACE( 1,
"pid=%s g_pid_vec.size()=%ld", pids[ii].c_str(), g_pid_vec.size() );
674 pidfile.push_back(
"/proc/"+pids[ii]+
"/stat" );
675 char desc[128], ss[1024];
677 snprintf( ss,
sizeof(ss),
"CPUpid%zd?%s?NR==1?$14+$15?1?yes", ii, pidfile[ii].c_str() );
678 stats.push_back( ss );
680 std::ifstream t(
"/proc/"+pids[ii]+
"/comm");
681 std::string comm((std::istreambuf_iterator<char>(t)),
682 std::istreambuf_iterator<char>());
683 comm = comm.substr(0,comm.size()-1);
685 snprintf( desc,
sizeof(desc),
"CPU+pid%zd_%s", ii, comm.c_str() );
686 graphs.push_back( desc );
687 snprintf( ss,
sizeof(ss),
"%s?%s?NR==1?$14+$15+16+$17?1?yes", desc, pidfile[ii].c_str() );
688 stats.push_back( ss );
690 snprintf( desc,
sizeof(desc),
"WaitBlkIOpid%zd", ii );
691 if (opt_cmd_iowait) graphs.push_back( desc );
692 snprintf( ss,
sizeof(ss),
"%s?%s?NR==1?$42?1?yes", desc, pidfile[ii].c_str() );
693 stats.push_back( ss );
695 snprintf( desc,
sizeof(desc),
"Faultpid%zd", ii );
696 if (opt_fault) graphs.push_back( desc );
697 snprintf( ss,
sizeof(ss),
"%s?%s?NR==1?$10+$11+$12+$13?4096.0/1048576?yes", desc, pidfile[ii].c_str() );
698 stats.push_back( ss );
702 stats.push_back(
"CPUnode");
703 stats.push_back(
"IOWait");
if (opt_sys_iowait) { graphs.push_back(
"IOWait"); }
704 stats.push_back(
"Cached");
705 stats.push_back(
"Dirty");
706 stats.push_back(
"Free");
708 if (opt_disk.size()) {
709 std::vector<std::string> tmp;
710 string_addto_vector( opt_disk, tmp,
',' );
711 for (std::vector<std::string>::iterator dk=tmp.begin(); dk!=tmp.end(); ++dk) {
715 std::string statstr=*dk+
"_wrMB/s?/proc/diskstats?/"+*dk+
"/?$10?(1.0/2048)?yes";
716 stats.push_back(statstr);
717 std::vector<std::string> stat_spec;
718 string_addto_vector( statstr, stat_spec,
'?' );
719 graphs.push_back( stat_spec[s_desc] );
721 statstr=*dk+
"_rdMB/s?/proc/diskstats?/"+*dk+
"/?$6?(1.0/2048)?yes";
722 stats.push_back(statstr);
724 string_addto_vector( statstr, stat_spec,
'?' );
729 if (opt_stats.size()) {
730 std::vector<std::string> tmp_stats;
731 string_addto_vector( opt_stats, tmp_stats,
',' );
732 for (std::vector<std::string>::iterator st=tmp_stats.begin();
733 st!=tmp_stats.end(); ++st) {
734 stats.push_back(*st);
735 std::vector<std::string> stat_spec;
736 string_addto_vector( *st, stat_spec,
'?' );
737 graphs.push_back( stat_spec[s_desc] );
741 std::vector<long> pre_vals;
742 std::vector<double> multipliers;
743 std::vector<std::vector<std::string>> spec2(stats.size());
744 std::vector<std::string> awkCmd;
746 std::string header_str(
"#" DATA_START
"\n#_______time_______" );
748 int outfd=open(dat_file_out.c_str(),O_WRONLY|O_CREAT,0777);
750 FILE *outfp = fdopen(outfd,
"w");
752 std::string cmd_comment(
"");
754 cmd_comment +=
"\\ncmd: "+opt_cmd[0];
755 if (opt_comment.size())
756 cmd_comment +=
"\\ncomment: " + opt_comment;
757 fprintf( outfp, GNUPLOT_PREFIX, opt_ymin, opt_ymax, opt_yincr, opt_y2max, opt_y2incr
758 , run_time, hostname.c_str(), ubuf.release
759 , cmd_comment.c_str()
760 ,
"disk write MB/s" );
762 uint64_t t_start=gettimeofday_us();
765 bool first_graph_spec_added=
false;
766 for (
size_t ii=0; ii<stats.size(); ++ii) {
767 std::vector<std::string> stat_spec;
768 string_addto_vector( stats[ii], stat_spec,
'?' );
769 if (stat_spec[s_desc]==
"CPUnode" && stat_spec.size()==1)
771 stats[ii]+=
"?/proc/stat?/^cpu[^0-9]/?$2+$3+$4+$6+$7+$8+$9?1.0/"+std::to_string(concurentThreadsSupported)+
"?yes";
772 else if (stat_spec[s_desc]==
"IOWait" && stat_spec.size()==1)
773 stats[ii]+=
"?/proc/stat?/^cpu[^0-9]/?$6?1.0/"+std::to_string(concurentThreadsSupported)+
"?yes";
774 else if (stat_spec[s_desc]==
"Cached" && stat_spec.size()==1)
775 stats[ii]+=
"?/proc/meminfo?/^(Cached|Buffers):/?$2?1?no";
776 else if (stat_spec[s_desc]==
"Dirty" && stat_spec.size()==1)
777 stats[ii]+=
"?/proc/meminfo?/^Dirty:/?$2?1?no";
778 else if (stat_spec[s_desc]==
"Free" && stat_spec.size()==1)
779 stats[ii]+=
"?/proc/meminfo?/^MemFree:/?$2?1?no";
781 header_str +=
" "+stat_spec[s_desc];
783 string_addto_vector( stats[ii], spec2[ii],
'?' );
785 snprintf( awk_cmd,
sizeof(awk_cmd),
"%s{vv+=%s}END{print vv}"
787 , spec2[ii][s_linespec].c_str(), spec2[ii][s_fieldspec].c_str() );
788 awkCmd.push_back(awk_cmd);
790 std::string stat=AWK( awkCmd.back(), spec2[ii][s_file].c_str(), NULL );
792 pre_vals.push_back(atol(stat.c_str()));
793 multipliers.push_back(atof(AWK(
"BEGIN{print "+spec2[ii][s_multiplier]+
"}",
"/dev/null",NULL ).c_str()) );
795 for (
size_t jj=0; jj<graphs.size(); ++jj)
796 if (graphs[jj] == stat_spec[s_desc]) {
797 if (first_graph_spec_added) fprintf( outfp,
",\\\n '' " );
798 if (strncmp(stat_spec[s_desc].c_str(),
"CPU",3)==0)
799 fprintf( outfp,
"using 1:%zd title '%s' w linespoints axes x1y2", ii+2, stat_spec[s_desc].c_str() );
800 else if (stat_spec[s_desc] ==
"Cached" || stat_spec[s_desc] ==
"Dirty" || stat_spec[s_desc] ==
"Free")
801 fprintf( outfp,
"using 1:($%zd/%s*100) title '%s%%' w linespoints axes x1y2", ii+2, memKB.c_str(), stat_spec[s_desc].c_str() );
802 else if (stat_spec[s_desc].substr(0,6) ==
"CPUcmd" || stat_spec[s_desc].substr(0,6) ==
"CPU+cm")
803 fprintf( outfp,
"using 1:%zd title '%s' w linespoints axes x1y2", ii+2, stat_spec[s_desc].c_str() );
804 else if (stat_spec[s_desc].substr(0,12) ==
"WaitBlkIOcmd" )
805 fprintf( outfp,
"using 1:%zd title '%s' w linespoints axes x1y2", ii+2, stat_spec[s_desc].c_str() );
807 fprintf( outfp,
"using 1:%zd title '%s' w linespoints axes x1y1", ii+2, stat_spec[s_desc].c_str() );
808 first_graph_spec_added=
true;
811 header_str +=
" #\n";
813 fprintf( outfp,
"\nif(png==0) pause -1 'Press Enter/Return or ^C to finish'\n\
817 fprintf( outfp,
"cmds:\n" );
818 for (
size_t ii=0; ii<opt_cmd.size(); ++ii) {
819 std::string ss=opt_cmd[ii]+
"\n";
820 fprintf( outfp,
"%s", ss.c_str() );
824 fprintf( outfp,
"stats:\n" );
825 for (
size_t ii=0; ii<stats.size(); ++ii) {
826 std::string ss=stats[ii]+
"\n";
827 fprintf( outfp,
"%s", ss.c_str() );
831 fprintf( outfp,
"%s", header_str.c_str() ); fflush( outfp );
833 std::string tmpdbg(
"main lp=%d done stat%zd=%ld rate=%f ");
835 char proc_stats[8192];
842 int64_t t_sleep=(t_start+(uint64_t)(period*1e6))-gettimeofday_us();
844 int sts=usleep( t_sleep );
845 TRACE( 3,
"main usleep sts=%d errno=%d",sts, errno );
851 for (lp=2; lp<MAX_LP; ++lp) {
853 gettimeofday( &tv, NULL );
854 strftime( str,
sizeof(str),
"%FT%T", localtime(&tv.tv_sec) );
856 fprintf(outfp,
"%s", str );
857 std::string prv_file(
"");
858 for (
size_t ii=0; ii<stats.size(); ++ii) {
859 TRACE( 3,
"main lp=%d start stat%zd", lp, ii );
860 char const *awk_file;
861 if (ii < (2*opt_cmd.size())) {
865 read_proc_file( pidfile[ii/2].c_str(),proc_stats,
sizeof(proc_stats) );
867 awk_in=proc_stats; awk_file=NULL;
868 }
else if (spec2[ii][s_file] != prv_file) {
869 prv_file = spec2[ii][s_file];
870 read_proc_file( spec2[ii][s_file].c_str(), proc_stats,
sizeof(proc_stats) );
871 awk_in=proc_stats; awk_file=NULL;
875 std::string stat_str=AWK( awkCmd[ii], awk_file, awk_in );
877 long stat=atol(stat_str.c_str());
879 if (spec2[ii][s_rate] ==
"yes") {
882 rate=(stat-pre_vals[ii])*multipliers[ii]/period;
885 TRACE( 3, tmpdbg+
"stat_str[0]=0x%x stat_str.size()=%zd", lp,ii,stat,rate, stat_str[0], stat_str.size() );
886 fprintf(outfp,
" %.2f",rate );
887 if (rate < 0.0 && spec2[ii][s_file] ==
"/proc/diskstats") {
888 TRACE( 0,
"main stat:"+spec2[ii][s_desc]+
" rate=%f pre_val=%ld stat=%ld stat_str=\""+stat_str\
889 +
"\" awkCmd="+awkCmd[ii]+
" proc_diskstats="+proc_stats
890 , rate, pre_vals[ii], stat );
895 TRACE( 3,
"main lp=%d done stat%zd=%ld", lp, ii, stat );
896 fprintf(outfp,
" %.2f",stat*multipliers[ii] );
899 fprintf(outfp,
"\n"); fflush(outfp);
901 int64_t t_sleep=(t_start+(uint64_t)(period*lp*1000000))-gettimeofday_us();
903 int sts=usleep( t_sleep );
904 TRACE( 3,
"main usleep sts=%d errno=%d",sts,errno );
908 pp = check_pid_vec();
909 TRACE( 2,
"main pp=%d t_sleep=%ld", pp, t_sleep );
911 if (post_periods_completed == 0)
912 TRACE( 1,
"main processes complete - waiting %d post periods", opt_post );
913 if (post_periods_completed++ == opt_post)
918 fprintf(outfp,
"# MAX_LP abort\n" );
924 TRACE( 0,
"main done/complete/returning" );