00001
00002
00003
00004
00005
00006 static const char*rev="$Revision: 1.19 $$Date: 2018/06/28 21:14:28 $";
00007
00008
00009
00010
00011 #define USAGE "\
00012 usage: %s --cmd=<cmd>\n\
00013 examples: %s --cmd='sleep 25' # this will take about a minute\n\
00014 %s --cmd='taskset -c 4 dd if=/dev/zero of=/dev/sdf bs=50M count=500 oflag=direct' --disk=sdf\n\
00015 %s --cmd='dd if=/dev/zero of=t.dat count=500' --stat='md*MB/s?/proc/diskstats?/md[0-3]/?$10?(1.0/2048)?yes'\n\
00016 gnuplot -e png=0 `/bin/ls -t periodic_*_stats.out|head -1`\n\
00017 For each cmd, record CPU info. Additionally, record total system CPU\n\
00018 (can be >100 w/ multicore), file system Cached and Dirty %%, plus any\n\
00019 other stats specified by --stats options\n\
00020 options:\n\
00021 --cmd= can have multiple\n\
00022 --Cmd= run cmd, but dont graph CPU (can have multiple)\n\
00023 --pid= graph comma or space sep. list of pids (getting cmd from /proc/)\n\
00024 --disk= automatically build --stat for disk(s)\n\
00025 --stat= desc?file?linespec?fieldspec?multiplier?rate\n\
00026 builtin = CPUnode,Cached,Dirty\n\
00027 cmd-builtin = CPUcmdN, CPU+cmdN\n\
00028 --out-dir= output dir (for 2+ output files; stats and cmd+)\n\
00029 \n\
00030 --period= (float) default=%s\n\
00031 --pre= the number of periods wait before exec of --cmd\n\
00032 --post= the number of periods to loop after cmd exits\n\
00033 --graph= add to graph any possibly included non-graphed metrics\n\
00034 \n\
00035 --no-defaults for no default stats (ie. cpu-parent,cpu-children)\n\
00036 --init= default dropcache if root. NOT implemented yet\n\
00037 --duration=\n\
00038 --comment=\n\
00039 --yrange=400:1400\n\
00040 --y2max=\n\
00041 --y2incr=\n\
00042 --pause=0\n\
00043 --sys-iowait,-w include the system iowait on the graph\n\
00044 --cmd-iowait include cmd iowait on the graph\n\
00045 --fault,-f\n\
00046 ", basename(argv[0]),basename(argv[0]),basename(argv[0]),basename(argv[0]),opt_period
00047
00048 enum { s_desc, s_file, s_linespec, s_fieldspec, s_multiplier, s_rate };
00049
00050 #include <stdio.h>
00051 #include <getopt.h>
00052 #include <unistd.h>
00053 #include <sys/wait.h>
00054 #include <fcntl.h>
00055 #include <sys/time.h>
00056 #include <signal.h>
00057 #include <sys/utsname.h>
00058 #include <string>
00059 #include <sstream>
00060 #include <vector>
00061 #include <thread>
00062 #include <fstream>
00063
00064 #ifdef DO_TRACE
00065 # define TRACE_NAME "periodic_cmd_stats"
00066 # include "trace.h"
00067 #else
00068 # include <stdarg.h>
00069 # include <string.h>
00070 # include <string>
00071 static void trace_ap(const char*msg,va_list ap)
00072 { char m_[1024]; unsigned len=strlen(msg);
00073 len=len<(sizeof(m_)-2)?len:(sizeof(m_)-2); memcpy(m_,msg,len+1);
00074 if(m_[len-1]!='\n') memcpy(&(m_[len]),"\n",2);
00075 vprintf(m_,ap);va_end(ap);
00076 }
00077 static void trace_p(const char*msg,...)__attribute__((format(printf,1,2)));
00078 static void trace_p(const char*msg,...)
00079 { va_list ap;va_start(ap,msg); trace_ap(msg ,ap); va_end(ap);
00080 }
00081 static void trace_p(const std::string msg,...)
00082 { va_list ap;va_start(ap,msg); trace_ap(msg.c_str(),ap); va_end(ap);
00083 }
00084 # define TRACE(lvl,...) do if(lvl<=1)trace_p(__VA_ARGS__);while(0)
00085 # define TRACE_CNTL( xyzz,... )
00086 #endif
00087
00088
00089 int opt_v=1;
00090 char* opt_init=NULL;
00091 std::vector<std::string> opt_cmd;
00092 std::vector<std::string> opt_Cmd;
00093 std::string opt_pid;
00094 std::string opt_disk;
00095 std::string opt_stats;
00096 std::string opt_outdir("");
00097 std::string opt_graph("CPUnode,Cached,Dirty,Free");
00098 const char* opt_period="5.0";
00099 std::string opt_comment;
00100 int opt_pre=6;
00101 int opt_post=6;
00102 int opt_ymin=0;
00103 int opt_ymax=2000;
00104 int opt_yincr=200;
00105 int opt_y2max=200;
00106 int opt_y2incr=20;
00107 int opt_sys_iowait=0;
00108 int opt_cmd_iowait=0;
00109 int opt_fault=0;
00110
00111 std::vector<pid_t> g_pid_vec;
00112
00113 void charreplace( char* instr, char oldc, char newc )
00114 {
00115 while (*instr) {
00116 if (*instr == oldc)
00117 *instr=newc;
00118 ++instr;
00119 }
00120 }
00121
00122
00123 void parse_args( int argc, char *argv[] )
00124 {
00125 char *cp;
00126
00127 while (1) {
00128 int opt;
00129 static struct option long_options[] = {
00130
00131 { "help", no_argument, 0, 'h' },
00132 { "init", required_argument,0, 'i' },
00133 { "cmd", required_argument,0, 'c' },
00134 { "Cmd", required_argument,0, 'C' },
00135 { "disk", required_argument,0, 'd' },
00136 { "stat", required_argument,0, 's' },
00137 { "out-dir", required_argument,0, 'o' },
00138 { "period", required_argument,0, 'p' },
00139 { "sys-iowait", no_argument, 0, 'w' },
00140 { "fault", no_argument, 0, 'f' },
00141 { "pid", required_argument,0, 'P' },
00142 { "ymax", required_argument,0, 1 },
00143 { "yincr", required_argument,0, 2 },
00144 { "y2max", required_argument,0, 3 },
00145 { "y2incr", required_argument,0, 4 },
00146 { "pre", required_argument,0, 5 },
00147 { "post", required_argument,0, 6 },
00148 { "graph", required_argument,0, 7 },
00149 { "yrange", required_argument,0, 8 },
00150 { "comment", required_argument,0, 9 },
00151 { "cmd-iowait", no_argument, 0, 10 },
00152 { 0, 0, 0, 0 }
00153 };
00154 opt = getopt_long( argc, argv, "?hvqVi:c:C:d:s:o:p:P:wf",
00155 long_options, NULL );
00156 if (opt == -1) break;
00157 switch (opt) {
00158 case '?': case 'h': printf(USAGE); exit(0); break;
00159 case 'V': printf("%s\n",rev); exit(0); break;
00160 case 'v': ++opt_v; break;
00161 case 'q': --opt_v; break;
00162 case 'i': opt_init=optarg; break;
00163 case 'c': opt_cmd.push_back(optarg); break;
00164 case 'C': opt_Cmd.push_back(optarg); break;
00165 case 'd': if(opt_disk.size())opt_disk=opt_disk+","+optarg;else opt_disk=optarg;break;
00166 case 's': if(opt_stats.size())opt_stats=opt_stats+","+optarg;else opt_stats=optarg;break;
00167 case 'o': opt_outdir=std::string(optarg)+"/"; break;
00168 case 'p': opt_period=optarg; break;
00169 case 'w': opt_sys_iowait=1; break;
00170 case 'f': opt_fault=1; break;
00171 case 'P': charreplace(optarg,' ',',');
00172 if(opt_pid.size())opt_pid=opt_pid+","+optarg;
00173 else opt_pid=optarg;
00174 break;
00175 case 1: opt_ymax=strtoul(optarg,NULL,0); break;
00176 case 2: opt_yincr=strtoul(optarg,NULL,0); break;
00177 case 3: opt_y2max=strtoul(optarg,NULL,0); break;
00178 case 4: opt_y2incr=strtoul(optarg,NULL,0); break;
00179 case 5: opt_pre=strtoul(optarg,NULL,0); break;
00180 case 6: opt_post=strtoul(optarg,NULL,0); break;
00181 case 7: opt_graph+=std::string(",")+optarg; break;
00182 case 8: opt_ymin =strtoul(optarg,NULL,0);
00183 cp = strstr(optarg,":")+1;
00184 opt_ymax =strtoul(cp,NULL,0);
00185 if ((cp=strstr(cp,":"))) {
00186 ++cp;
00187 opt_yincr=strtoul(strstr(cp,":")+1,NULL,0);
00188 } else
00189 opt_yincr=(opt_ymax-opt_ymin)/5;
00190 break;
00191 case 9: opt_comment=optarg; break;
00192 case 10: opt_cmd_iowait=1; break;
00193 default:
00194 printf( "?? getopt returned character code 0%o ??\n", opt );
00195 exit( 1 );
00196 }
00197 }
00198 }
00199
00200 void perror_exit( const char *msg, ... )
00201 { char buf[1024];
00202 va_list ap;va_start(ap,msg);
00203 vsnprintf( buf, sizeof(buf), msg, ap );
00204 va_end(ap);
00205 TRACE( 0, "%s", buf );
00206 perror( buf ); exit(1);
00207 }
00208
00209
00210
00211
00212
00213
00214
00215
00216 pid_t fork_execv( int close_start, int close_cnt, int sleepB4exec_us, int iofd[3], const char *cmd, char *const argv[], char *const env[] )
00217 {
00218 int pipes[3][2];
00219 int lcl_iofd[3];
00220 for (auto ii=0; ii<3; ++ii) {
00221 lcl_iofd[ii]=iofd[ii];
00222 if (iofd[ii]==-1) {
00223 pipe(pipes[ii]);
00224 iofd[ii]=ii==0?pipes[ii][1]:pipes[ii][0];
00225 }
00226 }
00227 pid_t pid=fork();
00228 if (pid < 0) perror_exit("fork");
00229 else if (pid == 0) {
00230 if (lcl_iofd[0]==-1) {
00231 close(pipes[0][1]);
00232 int fd=dup2(pipes[0][0],0);
00233 TRACE( 3, "fork_execv dupped(%d) onto %d (should be 0)", pipes[0][0], fd );
00234 close(pipes[0][0]);
00235 }
00236 if (sleepB4exec_us) {
00237
00238
00239 usleep(sleepB4exec_us);
00240 TRACE( 1, "fork_execv sleep complete. sleepB4exec_us=%d sts=%d", sleepB4exec_us, 0 );
00241 }
00242 for (auto ii=1; ii<3; ++ii) {
00243 if (lcl_iofd[ii]==-1) {
00244 close(pipes[ii][0]);
00245 int fd=dup2(pipes[ii][1],ii);
00246 TRACE( 3, "fork_execv dupped(%d) onto %d (should be %d)", pipes[ii][1], fd,ii );
00247 close(pipes[ii][1]);
00248 } else if (lcl_iofd[ii]!=ii) {
00249 int fd=dup2(lcl_iofd[ii],ii);
00250 TRACE( 3, "fork_execv dupped(%d) onto %d (should be %d)", pipes[ii][1], fd,ii );
00251 }
00252 }
00253 for (auto ii=close_start; ii<(close_start+close_cnt); ++ii)
00254 close(ii);
00255 if (env)
00256 execve( cmd, argv, env );
00257 else
00258 execv( cmd, argv );
00259 exit(1);
00260 } else {
00261 for (auto ii=0; ii<3; ++ii)
00262 if (lcl_iofd[ii]==-1)
00263 close(ii==0?pipes[ii][0]:pipes[ii][1]);
00264 }
00265
00266 TRACE( 3, "fork_execv pid=%d", pid );
00267 return pid;
00268 }
00269
00270 uint64_t swapPtr( void *X )
00271 {
00272 uint64_t x=(uint64_t)X;
00273 x = (x & 0x00000000ffffffff) << 32 | (x & 0xffffffff00000000) >> 32;
00274 x = (x & 0x0000ffff0000ffff) << 16 | (x & 0xfff0000fffff0000) >> 16;
00275 x = (x & 0x00ff00ff00ff00ff) << 8 | (x & 0xff00ff00ff00ff00) >> 8;
00276 return x;
00277 }
00278
00279
00280
00281
00282
00283
00284
00285
00286
00287
00288
00289
00290
00291
00292
00293
00294
00295
00296
00297
00298 static int g_devnullfd=-1;
00299
00300
00301 std::string AWK( std::string const &awk_cmd, const char *file, const char *input )
00302 {
00303 char readbuf[1024];
00304 ssize_t bytes=0, tot_bytes=0;
00305 char *const argv_[4]={ (char*)"/bin/gawk",
00306 (char*)awk_cmd.c_str(),
00307 (char*)file,
00308 NULL };
00309 pid_t pid;;
00310 int infd=0;
00311 if (g_devnullfd == -1)
00312 g_devnullfd=open("/dev/null",O_WRONLY);
00313 if (input != NULL) {
00314 infd=-1;
00315 }
00316
00317 int iofd[3]={infd,-1,2};
00318 TRACE( 3, "AWK b4 fork_execv input=%p", (void*)input );
00319 char *env[1];
00320 env[0]=NULL;
00321 pid=fork_execv(0,0,0,iofd,"/bin/gawk",argv_,env);
00322 if(input) {
00323 int xx=strlen(input);
00324 int sts=write(iofd[0],input,xx);
00325 if (sts != xx)
00326 perror("write AWK stdin");
00327 close(iofd[0]);
00328 while ((bytes=read(iofd[1],&readbuf[tot_bytes],sizeof(readbuf)-tot_bytes)) != 0) {
00329 TRACE( 3, "AWK while bytes=read > 0 bytes=%zd readbuf=0x%016lx errno=%d", bytes, swapPtr(&readbuf[tot_bytes]), errno );
00330 if (bytes == -1) {
00331 if (errno == EINTR) continue;
00332 break;
00333 }
00334 tot_bytes+=bytes;
00335 }
00336 TRACE( 3, "AWK after read tot="+std::to_string((long long unsigned)tot_bytes)+" bytes="+std::to_string((long long unsigned)bytes)+" input="+std::string(input) );
00337 } else {
00338 while ((bytes=read(iofd[1],&readbuf[tot_bytes],sizeof(readbuf)-tot_bytes)) > 0)
00339 tot_bytes+=bytes;
00340 TRACE( 3, "AWK after read tot=%zd bytes=%zd [0]=0x%x input=%p", tot_bytes, bytes, readbuf[0], (void*)input );
00341 }
00342 readbuf[tot_bytes>=0?tot_bytes:0]='\0';
00343 close(iofd[1]);
00344 TRACE( 3, "AWK after close child stdout. child pid=%d", pid );
00345 #if 0
00346 int status;
00347 pid_t done_pid = waitpid(pid,&status,0);
00348 TRACE( 3, "AWK after wait pid=%d done_pid=%d status=%d(0x%x)"
00349 , pid, done_pid, status, status );
00350 #endif
00351 return std::string(readbuf);
00352 }
00353
00354
00355
00356 void string_addto_vector( std::string &instr, std::vector<std::string> &outvec, char delim )
00357 {
00358 std::stringstream ss(instr);
00359 while( ss.good() )
00360 { std::string substr;
00361 std::getline( ss, substr, delim );
00362 outvec.push_back( substr );
00363 }
00364 }
00365
00366 uint64_t gettimeofday_us( void )
00367 { struct timeval tv;
00368 gettimeofday( &tv, NULL );
00369
00370
00371
00372
00373 return (uint64_t)tv.tv_sec*1000000+tv.tv_usec;
00374 }
00375
00376 #define DATA_START " DATA START"
00377 #define GNUPLOT_PREFIX (const char *)"\
00378 #!/usr/bin/env gnuplot\n\
00379 # ./$0\n\
00380 # OR\n\
00381 # gnuplot -e 'ymin=400;ymax=1400' ./$0\n\
00382 # OR try\n\
00383 # gnuplot -e 'duration_s=35;set multiplot' ./gnuplot.gnuplot ./gnuplot.1.gnuplot -e 'set nomultiplot;pause -1'\n\
00384 if(!exists('ARG0')) ARG0='' # for version 4, use: gnuplot -e ARG0=hello\n\
00385 print 'ARG0=',ARG0 # ARG0.. automatically define in gnuplot version 5+\n\
00386 if(!exists('ymin')) ymin=%d\n\
00387 if(!exists('ymax')) ymax=%d\n\
00388 if(!exists('yincr')) yincr=%d\n\
00389 if(!exists('y2max')) y2max=%d\n\
00390 if(!exists('y2incr')) y2incr=%d\n\
00391 if(!exists('png')) png=1\n\
00392 if(!exists('duration_s')) duration_s=0\n\
00393 if(!exists('width')) width=512\n\
00394 if(!exists('height')) height=384\n\
00395 thisPid=system('echo `ps -p$$ -oppid=`')\n\
00396 thisFile=system('ls -l /proc/'.thisPid.\"/fd | grep -v pipe: | tail -1 | sed -e 's/.*-> //'\")\n\
00397 \n\
00398 set title \"Disk Write Rate and %%CPU vs. time\\n%s %s %s%s\" # cmd and/or comment at end\n\
00399 set xdata time\n\
00400 tfmt='%%Y-%%m-%%dT%%H:%%M:%%S' # try to use consistent format\n\
00401 set timefmt '%%Y-%%m-%%dT%%H:%%M:%%S'\n\
00402 set xlabel 'time'\n\
00403 set grid xtics back\n\
00404 xstart=system(\"awk '/^....-..-..T/{print$1;exit}' \".thisFile)\n\
00405 xend=system(\"awk 'END{print$1}' \".thisFile)\n\
00406 print 'xstart='.xstart.' xend='.xend.' duration=',strptime(tfmt,xend)-strptime(tfmt,xstart)\n\
00407 if(duration_s>0) end_t=strptime(tfmt,xstart)+duration_s; else end_t=strptime(tfmt,xend)\n\
00408 set xrange [xstart:end_t]\n\
00409 \n\
00410 set ylabel '%s'\n\
00411 set ytics nomirror\n\
00412 if(ymax==0) set yrange [ymin:*];\\\n\
00413 else set yrange [ymin:ymax];set ytics yincr\n\
00414 set grid ytics back\n\
00415 \n\
00416 set y2label '%%CPU, %%MemTotal'\n\
00417 set y2tics autofreq\n\
00418 if(y2max==0) set y2range [0:*];\\\n\
00419 else set y2range [0:y2max];set y2tics y2incr\n\
00420 set pointsize .6\n\
00421 \n\
00422 if(png==1) set terminal png size width,height;\\\n\
00423 pngfile=system( 'echo `basename '.thisFile.' .out`.png' );\\\n\
00424 set output pngfile;\\\n\
00425 else set terminal x11 size width,height\n\
00426 \n\
00427 plot \"< awk '/^#" DATA_START "/,/NEVER HAPPENS/' \".thisFile "
00428
00429
00430 void sigchld_sigaction( int signo, siginfo_t *info, void *context __attribute__((__unused__)) )
00431 {
00432
00433 for (size_t ii=0; ii < g_pid_vec.size(); ++ii) {
00434 pid_t pid=g_pid_vec[ii];
00435 if (pid == info->si_pid) {
00436 TRACE( 2, "sigchld_sigaction signo=%d status=%d(0x%x) code=%d(0x%x) sending_pid=%d"
00437 , signo
00438 , info->si_status, info->si_status
00439 , info->si_code, info->si_code
00440 , info->si_pid
00441 );
00442 return;
00443 }
00444 }
00445 TRACE( 3, "sigchld_sigaction signo=%d status=%d(0x%x) code=%d(0x%x) sending_pid=%d"
00446 , signo
00447 , info->si_status, info->si_status
00448 , info->si_code, info->si_code
00449 , info->si_pid
00450 );
00451 }
00452
00453
00454 void read_proc_file( const char *file, char *buffer, int buffer_size )
00455 {
00456 TRACE( 4, "read_proc_file b4 open proc file"+std::string(file) );
00457 int fd=open(file,O_RDONLY);
00458 int offset=0, sts=0;
00459 while (1) {
00460 sts=read(fd,&buffer[offset],buffer_size-offset);
00461 if (sts<=0) {
00462 sts=0;
00463 break;
00464 }
00465 offset+=sts;
00466 }
00467 buffer[sts+offset]='\0';
00468 close(fd);
00469 TRACE( 4, "read_proc_file after close "+std::string(file)+" read=%d offset=%d",sts,offset );
00470 }
00471
00472
00473 pid_t check_pid_vec( void )
00474 {
00475 for (size_t ii=0; ii < g_pid_vec.size(); ) {
00476 pid_t pid=g_pid_vec[ii];
00477 int status;
00478 pid_t pp = waitpid( pid, &status, WNOHANG );
00479 TRACE( 3, "check_pid_vec %d=waitpid(pid=%d) errno=%d", pp, pid, errno );
00480 if (pp > 0)
00481 g_pid_vec.erase( g_pid_vec.begin()+ii );
00482 else if (pp == -1) {
00483 if (errno == ECHILD && kill(pid,0)==0)
00484
00485 ++ii;
00486 else
00487
00488 g_pid_vec.erase( g_pid_vec.begin()+ii );
00489 }
00490 else
00491 ++ii;
00492 }
00493 if (g_pid_vec.size() == 0)
00494 return -1;
00495 else
00496 return 0;
00497 }
00498
00499 void cleanup( void )
00500 {
00501 TRACE( 1, "atexit cleanup g_pid_vec.size()=%zd\n", g_pid_vec.size() );
00502 for (std::vector<pid_t>::iterator pid=g_pid_vec.begin(); pid!=g_pid_vec.end(); ++pid) {
00503 kill( *pid, SIGHUP );
00504 }
00505 }
00506 #if (defined(__cplusplus)&&(__cplusplus>=201103L)) || (defined(__STDC_VERSION__)&&(__STDC_VERSION__>=201112L))
00507 # pragma GCC diagnostic push
00508 # pragma GCC diagnostic ignored "-Wunused-parameter"
00509 #endif
00510 void sigint_sigaction( int signo, siginfo_t *info, void *context )
00511 {
00512 cleanup();
00513 exit( 1 );
00514 }
00515 #if (defined(__cplusplus)&&(__cplusplus>=201103L)) || (defined(__STDC_VERSION__)&&(__STDC_VERSION__>=201112L))
00516 # pragma GCC diagnostic pop
00517 #endif
00518
00519
00520 int
00521 main( int argc
00522 , char *argv[] )
00523 {
00524 struct timeval tv;
00525 int post_periods_completed=0;
00526 parse_args( argc, argv );
00527 if ( (argc-optind)!=0
00528 || ( opt_cmd.size()==0
00529 && opt_pid.size()==0)) {
00530 int ii;
00531 printf( "unexpected argument(s) %d!=0\n", argc-optind );
00532 for (ii=0; (optind+ii)<argc; ++ii)
00533 printf("arg%d=%s\n",ii+1,argv[optind+ii]);
00534 printf( USAGE ); exit( 0 );
00535 }
00536
00537 std::vector<std::string> graphs;
00538 string_addto_vector( opt_graph, graphs, ',' );
00539
00540 char motherboard[1024]={0};
00541 if (getuid() == 0) {
00542 FILE *fp=popen( "dmidecode | grep -m2 'Product Name:' | tail -1", "r" );
00543 fread( motherboard, 1, sizeof(motherboard), fp );
00544 pclose( fp );
00545 }
00546 TRACE( 1, "main - motherboard="+std::string(motherboard) );
00547
00548
00549
00550
00551
00552
00553
00554 struct sigaction sigaction_s;
00555 #ifndef DO_SIGCHLD
00556 # define DO_SIGCHLD 1
00557 #endif
00558 #if DO_SIGCHLD
00559 sigaction_s.sa_sigaction = sigchld_sigaction;
00560 sigaction_s.sa_flags = SA_SIGINFO|SA_NOCLDWAIT;
00561 #else
00562 sigaction_s.sa_handler = SIG_IGN;
00563 sigaction_s.sa_flags = SA_NOCLDWAIT;
00564 #endif
00565 sigemptyset(&sigaction_s.sa_mask);
00566 sigaction( SIGCHLD, &sigaction_s, NULL );
00567
00568 sigaction_s.sa_sigaction = sigint_sigaction;
00569 sigaction_s.sa_flags = SA_SIGINFO;
00570 sigaction( SIGINT, &sigaction_s, NULL );
00571
00572
00573
00574 long long unsigned concurentThreadsSupported = sysconf(_SC_NPROCESSORS_ONLN);
00575
00576 TRACE( 0, "main concurentThreadsSupported=%u opt_stats="+opt_stats, concurentThreadsSupported );
00577
00578 char run_time[80];
00579 gettimeofday( &tv, NULL );
00580 strftime( run_time, sizeof(run_time), "%FT%H%M%S", localtime(&tv.tv_sec) );
00581 TRACE( 0, "main run_time="+std::string(run_time) );
00582
00583
00584 struct utsname ubuf;
00585 uname( &ubuf );
00586 char *dot;
00587 if ((dot=strchr(ubuf.nodename,'.')) != NULL)
00588 *dot = '\0';
00589 std::string hostname(ubuf.nodename);
00590 TRACE( 1,"release="+std::string(ubuf.release)+" version="+std::string(ubuf.version) );
00591
00592
00593 std::string memKB=AWK( "NR==1{print$2;exit}","/proc/meminfo",NULL );
00594 memKB = memKB.substr(0,memKB.size()-1);
00595
00596 std::string dat_file_out(opt_outdir+"periodic_"+run_time+"_"+hostname+"_stats.out");
00597
00598 double period=atof(opt_period);
00599
00600 atexit( cleanup );
00601 pid_t pp;
00602 std::vector<std::string> pidfile;
00603
00604 std::vector<std::string> stats;
00605
00606
00607
00608 for (size_t ii=0; ii<opt_cmd.size(); ++ii) {
00609 char cmd_file_out[1024];
00610 snprintf( cmd_file_out, sizeof(cmd_file_out), "%speriodic_%s_%s_cmd%zd.out"
00611 , opt_outdir.c_str(), run_time, hostname.c_str(), ii );
00612 int fd=open( cmd_file_out, O_WRONLY|O_CREAT,0666 );
00613 TRACE( 0, "main fd=%d opt_cmd="+opt_cmd[ii]+" cmd_file_out="+std::string(cmd_file_out), fd );
00614 int iofd[3]={0,fd,fd};
00615 char *const argv_[4]={ (char*)"/bin/sh",
00616 (char*)"-c",
00617 (char*)opt_cmd[ii].c_str(),
00618 NULL };
00619 g_pid_vec.push_back( fork_execv(0,0,(int)(period*opt_pre*1e6),iofd,"/bin/sh",argv_,NULL) );
00620 close(fd);
00621 std::string pidstr=std::to_string((long long int)g_pid_vec[ii]);
00622 pidfile.push_back( "/proc/"+pidstr+"/stat" );
00623
00624 char desc[128], ss[1024];
00625
00626 snprintf( ss, sizeof(ss), "CPUcmd%zd?%s?NR==1?$14+$15?1?yes", ii, pidfile[ii].c_str() );
00627 stats.push_back( ss );
00628
00629 snprintf( desc, sizeof(desc), "CPU+cmd%zd", ii );
00630 graphs.push_back( desc );
00631 snprintf( ss, sizeof(ss), "%s?%s?NR==1?$14+$15+16+$17?1?yes", desc, pidfile[ii].c_str() );
00632 stats.push_back( ss );
00633
00634 snprintf( desc, sizeof(desc), "WaitBlkIOcmd%zd", ii );
00635 if (opt_cmd_iowait) graphs.push_back( desc );
00636 snprintf( ss, sizeof(ss), "%s?%s?NR==1?$42?1?yes", desc, pidfile[ii].c_str() );
00637 stats.push_back( ss );
00638
00639 snprintf( desc, sizeof(desc), "Faultcmd%zd", ii );
00640 if (opt_fault) graphs.push_back( desc );
00641 snprintf( ss, sizeof(ss), "%s?%s?NR==1?$10+$11+$12+$13?4096.0/1048576?yes", desc, pidfile[ii].c_str() );
00642 stats.push_back( ss );
00643 }
00644 for (size_t ii=0; ii<opt_Cmd.size(); ++ii) {
00645 char cmd_file_out[1024];
00646 snprintf( cmd_file_out, sizeof(cmd_file_out), "%speriodic_%s_%s_cmd%zd.out"
00647 , opt_outdir.c_str(), run_time, hostname.c_str(), ii+opt_cmd.size() );
00648 int fd=open( cmd_file_out, O_WRONLY|O_CREAT,0666 );
00649 TRACE( 0, "main fd=%d opt_Cmd="+opt_Cmd[ii]+" cmd_file_out="+std::string(cmd_file_out), fd );
00650 int iofd[3]={0,fd,fd};
00651 char *const argv_[4]={ (char*)"/bin/sh",
00652 (char*)"-c",
00653 (char*)opt_Cmd[ii].c_str(),
00654 NULL };
00655 g_pid_vec.push_back( fork_execv(0,0,(int)(period*opt_pre*1e6),iofd,"/bin/sh",argv_,NULL) );
00656 close(fd);
00657 std::string pidstr=std::to_string((long long int)g_pid_vec[ii]);
00658 pidfile.push_back( "/proc/"+pidstr+"/stat" );
00659
00660 char desc[128], ss[1024];
00661 snprintf( desc, sizeof(desc), "CPU+cmd%zd", ii+opt_cmd.size() );
00662 snprintf( ss, sizeof(ss), "CPUcmd%zd?%s?NR==1?$14+$15?1?yes", ii+opt_cmd.size(), pidfile[ii].c_str() );
00663 stats.push_back( ss );
00664 snprintf( ss, sizeof(ss), "CPU+cmd%zd?%s?NR==1?$14+$15+16+$17?1?yes", ii+opt_cmd.size(), pidfile[ii].c_str() );
00665 stats.push_back( ss );
00666
00667 }
00668 std::vector<std::string> pids;
00669 if (opt_pid.size())
00670 string_addto_vector( opt_pid, pids, ',' );
00671 for (size_t ii=0; ii<pids.size(); ++ii) {
00672 g_pid_vec.push_back( std::stoi(pids[ii]) );
00673 TRACE( 1, "pid=%s g_pid_vec.size()=%ld", pids[ii].c_str(), g_pid_vec.size() );
00674 pidfile.push_back( "/proc/"+pids[ii]+"/stat" );
00675 char desc[128], ss[1024];
00676
00677 snprintf( ss, sizeof(ss), "CPUpid%zd?%s?NR==1?$14+$15?1?yes", ii, pidfile[ii].c_str() );
00678 stats.push_back( ss );
00679
00680 std::ifstream t("/proc/"+pids[ii]+"/comm");
00681 std::string comm((std::istreambuf_iterator<char>(t)),
00682 std::istreambuf_iterator<char>());
00683 comm = comm.substr(0,comm.size()-1);
00684
00685 snprintf( desc, sizeof(desc), "CPU+pid%zd_%s", ii, comm.c_str() );
00686 graphs.push_back( desc );
00687 snprintf( ss, sizeof(ss), "%s?%s?NR==1?$14+$15+16+$17?1?yes", desc, pidfile[ii].c_str() );
00688 stats.push_back( ss );
00689
00690 snprintf( desc, sizeof(desc), "WaitBlkIOpid%zd", ii );
00691 if (opt_cmd_iowait) graphs.push_back( desc );
00692 snprintf( ss, sizeof(ss), "%s?%s?NR==1?$42?1?yes", desc, pidfile[ii].c_str() );
00693 stats.push_back( ss );
00694
00695 snprintf( desc, sizeof(desc), "Faultpid%zd", ii );
00696 if (opt_fault) graphs.push_back( desc );
00697 snprintf( ss, sizeof(ss), "%s?%s?NR==1?$10+$11+$12+$13?4096.0/1048576?yes", desc, pidfile[ii].c_str() );
00698 stats.push_back( ss );
00699 }
00700
00701
00702 stats.push_back("CPUnode");
00703 stats.push_back("IOWait"); if (opt_sys_iowait) { graphs.push_back("IOWait"); }
00704 stats.push_back("Cached");
00705 stats.push_back("Dirty");
00706 stats.push_back("Free");
00707
00708 if (opt_disk.size()) {
00709 std::vector<std::string> tmp;
00710 string_addto_vector( opt_disk, tmp, ',' );
00711 for (std::vector<std::string>::iterator dk=tmp.begin(); dk!=tmp.end(); ++dk) {
00712
00713
00714
00715 std::string statstr=*dk+"_wrMB/s?/proc/diskstats?/"+*dk+"/?$10?(1.0/2048)?yes";
00716 stats.push_back(statstr);
00717 std::vector<std::string> stat_spec;
00718 string_addto_vector( statstr, stat_spec, '?' );
00719 graphs.push_back( stat_spec[s_desc] );
00720
00721 statstr=*dk+"_rdMB/s?/proc/diskstats?/"+*dk+"/?$6?(1.0/2048)?yes";
00722 stats.push_back(statstr);
00723 stat_spec.clear();
00724 string_addto_vector( statstr, stat_spec, '?' );
00725
00726 }
00727 }
00728
00729 if (opt_stats.size()) {
00730 std::vector<std::string> tmp_stats;
00731 string_addto_vector( opt_stats, tmp_stats, ',' );
00732 for (std::vector<std::string>::iterator st=tmp_stats.begin();
00733 st!=tmp_stats.end(); ++st) {
00734 stats.push_back(*st);
00735 std::vector<std::string> stat_spec;
00736 string_addto_vector( *st, stat_spec, '?' );
00737 graphs.push_back( stat_spec[s_desc] );
00738 }
00739 }
00740
00741 std::vector<long> pre_vals;
00742 std::vector<double> multipliers;
00743 std::vector<std::vector<std::string>> spec2(stats.size());
00744 std::vector<std::string> awkCmd;
00745
00746 std::string header_str( "#" DATA_START "\n#_______time_______" );
00747
00748 int outfd=open(dat_file_out.c_str(),O_WRONLY|O_CREAT,0777);
00749
00750 FILE *outfp = fdopen(outfd,"w");
00751
00752 std::string cmd_comment("");
00753 if (opt_cmd.size())
00754 cmd_comment += "\\ncmd: "+opt_cmd[0];
00755 if (opt_comment.size())
00756 cmd_comment += "\\ncomment: " + opt_comment;
00757 fprintf( outfp, GNUPLOT_PREFIX, opt_ymin, opt_ymax, opt_yincr, opt_y2max, opt_y2incr
00758 , run_time, hostname.c_str(), ubuf.release
00759 , cmd_comment.c_str()
00760 , "disk write MB/s" );
00761
00762 uint64_t t_start=gettimeofday_us();
00763
00764
00765 bool first_graph_spec_added=false;
00766 for (size_t ii=0; ii<stats.size(); ++ii) {
00767 std::vector<std::string> stat_spec;
00768 string_addto_vector( stats[ii], stat_spec, '?' );
00769 if (stat_spec[s_desc]=="CPUnode" && stat_spec.size()==1)
00770
00771 stats[ii]+="?/proc/stat?/^cpu[^0-9]/?$2+$3+$4+$6+$7+$8+$9?1.0/"+std::to_string(concurentThreadsSupported)+"?yes";
00772 else if (stat_spec[s_desc]=="IOWait" && stat_spec.size()==1)
00773 stats[ii]+="?/proc/stat?/^cpu[^0-9]/?$6?1.0/"+std::to_string(concurentThreadsSupported)+"?yes";
00774 else if (stat_spec[s_desc]=="Cached" && stat_spec.size()==1)
00775 stats[ii]+="?/proc/meminfo?/^(Cached|Buffers):/?$2?1?no";
00776 else if (stat_spec[s_desc]=="Dirty" && stat_spec.size()==1)
00777 stats[ii]+="?/proc/meminfo?/^Dirty:/?$2?1?no";
00778 else if (stat_spec[s_desc]=="Free" && stat_spec.size()==1)
00779 stats[ii]+="?/proc/meminfo?/^MemFree:/?$2?1?no";
00780
00781 header_str += " "+stat_spec[s_desc];
00782
00783 string_addto_vector( stats[ii], spec2[ii], '?' );
00784 char awk_cmd[1024];
00785 snprintf( awk_cmd, sizeof(awk_cmd), "%s{vv+=%s}END{print vv}"
00786
00787 , spec2[ii][s_linespec].c_str(), spec2[ii][s_fieldspec].c_str() );
00788 awkCmd.push_back(awk_cmd);
00789
00790 std::string stat=AWK( awkCmd.back(), spec2[ii][s_file].c_str(), NULL );
00791
00792 pre_vals.push_back(atol(stat.c_str()));
00793 multipliers.push_back(atof(AWK( "BEGIN{print "+spec2[ii][s_multiplier]+"}","/dev/null",NULL ).c_str()) );
00794
00795 for (size_t jj=0; jj<graphs.size(); ++jj)
00796 if (graphs[jj] == stat_spec[s_desc]) {
00797 if (first_graph_spec_added) fprintf( outfp, ",\\\n '' " );
00798 if (strncmp(stat_spec[s_desc].c_str(),"CPU",3)==0)
00799 fprintf( outfp, "using 1:%zd title '%s' w linespoints axes x1y2", ii+2, stat_spec[s_desc].c_str() );
00800 else if (stat_spec[s_desc] == "Cached" || stat_spec[s_desc] == "Dirty" || stat_spec[s_desc] == "Free")
00801 fprintf( outfp, "using 1:($%zd/%s*100) title '%s%%' w linespoints axes x1y2", ii+2, memKB.c_str(), stat_spec[s_desc].c_str() );
00802 else if (stat_spec[s_desc].substr(0,6) == "CPUcmd" || stat_spec[s_desc].substr(0,6) == "CPU+cm")
00803 fprintf( outfp, "using 1:%zd title '%s' w linespoints axes x1y2", ii+2, stat_spec[s_desc].c_str() );
00804 else if (stat_spec[s_desc].substr(0,12) == "WaitBlkIOcmd" )
00805 fprintf( outfp, "using 1:%zd title '%s' w linespoints axes x1y2", ii+2, stat_spec[s_desc].c_str() );
00806 else
00807 fprintf( outfp, "using 1:%zd title '%s' w linespoints axes x1y1", ii+2, stat_spec[s_desc].c_str() );
00808 first_graph_spec_added=true;
00809 }
00810 }
00811 header_str += " #\n";
00812
00813 fprintf( outfp, "\nif(png==0) pause -1 'Press Enter/Return or ^C to finish'\n\
00814 exit\n" );
00815
00816
00817 fprintf( outfp, "cmds:\n" );
00818 for (size_t ii=0; ii<opt_cmd.size(); ++ii) {
00819 std::string ss=opt_cmd[ii]+"\n";
00820 fprintf( outfp, "%s", ss.c_str() );
00821 }
00822
00823
00824 fprintf( outfp, "stats:\n" );
00825 for (size_t ii=0; ii<stats.size(); ++ii) {
00826 std::string ss=stats[ii]+"\n";
00827 fprintf( outfp, "%s", ss.c_str() );
00828 }
00829
00830
00831 fprintf( outfp, "%s", header_str.c_str() ); fflush( outfp );
00832
00833 std::string tmpdbg("main lp=%d done stat%zd=%ld rate=%f ");
00834
00835 char proc_stats[8192];
00836 char *awk_in;
00837 int lp;
00838
00839
00840
00841 eintr1:
00842 int64_t t_sleep=(t_start+(uint64_t)(period*1e6))-gettimeofday_us();
00843 if (t_sleep > 0) {
00844 int sts=usleep( t_sleep );
00845 TRACE( 3,"main usleep sts=%d errno=%d",sts, errno );
00846 if(errno == EINTR)
00847 goto eintr1;
00848 }
00849
00850 # define MAX_LP 600
00851 for (lp=2; lp<MAX_LP; ++lp) {
00852 char str[80];
00853 gettimeofday( &tv, NULL );
00854 strftime( str, sizeof(str), "%FT%T", localtime(&tv.tv_sec) );
00855
00856 fprintf(outfp, "%s", str );
00857 std::string prv_file("");
00858 for (size_t ii=0; ii<stats.size(); ++ii) {
00859 TRACE( 3, "main lp=%d start stat%zd", lp, ii );
00860 char const *awk_file;
00861 if (ii < (2*opt_cmd.size())) {
00862
00863
00864 if ((ii&1)==0) {
00865 read_proc_file( pidfile[ii/2].c_str(),proc_stats, sizeof(proc_stats) );
00866 }
00867 awk_in=proc_stats; awk_file=NULL;
00868 } else if (spec2[ii][s_file] != prv_file) {
00869 prv_file = spec2[ii][s_file];
00870 read_proc_file( spec2[ii][s_file].c_str(), proc_stats, sizeof(proc_stats) );
00871 awk_in=proc_stats; awk_file=NULL;
00872 }
00873
00874
00875 std::string stat_str=AWK( awkCmd[ii], awk_file, awk_in );
00876
00877 long stat=atol(stat_str.c_str());
00878
00879 if (spec2[ii][s_rate] == "yes") {
00880 double rate;
00881 if (stat_str!="\n")
00882 rate=(stat-pre_vals[ii])*multipliers[ii]/period;
00883 else
00884 rate=0.0;
00885 TRACE( 3, tmpdbg+"stat_str[0]=0x%x stat_str.size()=%zd", lp,ii,stat,rate, stat_str[0], stat_str.size() );
00886 fprintf(outfp, " %.2f",rate );
00887 if (rate < 0.0 && spec2[ii][s_file] == "/proc/diskstats") {
00888 TRACE( 0, "main stat:"+spec2[ii][s_desc]+" rate=%f pre_val=%ld stat=%ld stat_str=\""+stat_str\
00889 +"\" awkCmd="+awkCmd[ii]+" proc_diskstats="+proc_stats
00890 , rate, pre_vals[ii], stat );
00891
00892 }
00893 pre_vals[ii] = stat;
00894 } else {
00895 TRACE( 3, "main lp=%d done stat%zd=%ld", lp, ii, stat );
00896 fprintf(outfp, " %.2f",stat*multipliers[ii] );
00897 }
00898 }
00899 fprintf(outfp,"\n"); fflush(outfp);
00900 eintr2:
00901 int64_t t_sleep=(t_start+(uint64_t)(period*lp*1000000))-gettimeofday_us();
00902 if (t_sleep > 0) {
00903 int sts=usleep( t_sleep );
00904 TRACE( 3,"main usleep sts=%d errno=%d",sts,errno );
00905 if(errno == EINTR)
00906 goto eintr2;
00907 }
00908 pp = check_pid_vec();
00909 TRACE( 2, "main pp=%d t_sleep=%ld", pp, t_sleep );
00910 if (pp == -1) {
00911 if (post_periods_completed == 0)
00912 TRACE( 1, "main processes complete - waiting %d post periods", opt_post );
00913 if (post_periods_completed++ == opt_post)
00914 break;
00915 }
00916 }
00917 if (lp==MAX_LP) {
00918 fprintf(outfp,"# MAX_LP abort\n" );
00919 }
00920
00921
00922
00923
00924 TRACE( 0, "main done/complete/returning" );
00925
00926 return (0);
00927 }