$treeview $search $mathjax $extrastylesheet
artdaq
v3_04_00
$projectbrief
|
$projectbrief
|
$searchbox |
00001 // This file (periodic_cmd_stats.cc) was created by Ron Rechenmacher <ron@fnal.gov> on 00002 // Jan 5, 2017. "TERMS AND CONDITIONS" governing this file are in the README 00003 // or COPYING file. If you do not have such a file, one can be obtained by 00004 // contacting Ron or Fermi Lab in Batavia IL, 60510, phone: 630-840-3000. 00005 // $RCSfile: periodic_cmd_stats.cc,v $ 00006 static const char*rev="$Revision: 1.19 $$Date: 2018/06/28 21:14:28 $"; 00007 00008 // make periodic_cmd_stats 00009 // OR make periodic_cmd_stats CXX="g++ -std=c++0x -DDO_TRACE" 00010 00011 #define USAGE "\ 00012 usage: %s --cmd=<cmd>\n\ 00013 examples: %s --cmd='sleep 25' # this will take about a minute\n\ 00014 %s --cmd='taskset -c 4 dd if=/dev/zero of=/dev/sdf bs=50M count=500 oflag=direct' --disk=sdf\n\ 00015 %s --cmd='dd if=/dev/zero of=t.dat count=500' --stat='md*MB/s?/proc/diskstats?/md[0-3]/?$10?(1.0/2048)?yes'\n\ 00016 gnuplot -e png=0 `/bin/ls -t periodic_*_stats.out|head -1`\n\ 00017 For each cmd, record CPU info. Additionally, record total system CPU\n\ 00018 (can be >100 w/ multicore), file system Cached and Dirty %%, plus any\n\ 00019 other stats specified by --stats options\n\ 00020 options:\n\ 00021 --cmd= can have multiple\n\ 00022 --Cmd= run cmd, but dont graph CPU (can have multiple)\n\ 00023 --pid= graph comma or space sep. list of pids (getting cmd from /proc/)\n\ 00024 --disk= automatically build --stat for disk(s)\n\ 00025 --stat= desc?file?linespec?fieldspec?multiplier?rate\n\ 00026 builtin = CPUnode,Cached,Dirty\n\ 00027 cmd-builtin = CPUcmdN, CPU+cmdN\n\ 00028 --out-dir= output dir (for 2+ output files; stats and cmd+)\n\ 00029 \n\ 00030 --period= (float) default=%s\n\ 00031 --pre= the number of periods wait before exec of --cmd\n\ 00032 --post= the number of periods to loop after cmd exits\n\ 00033 --graph= add to graph any possibly included non-graphed metrics\n\ 00034 \n\ 00035 --no-defaults for no default stats (ie. cpu-parent,cpu-children)\n\ 00036 --init= default dropcache if root. NOT implemented yet\n\ 00037 --duration=\n\ 00038 --comment=\n\ 00039 --yrange=400:1400\n\ 00040 --y2max=\n\ 00041 --y2incr=\n\ 00042 --pause=0\n\ 00043 --sys-iowait,-w include the system iowait on the graph\n\ 00044 --cmd-iowait include cmd iowait on the graph\n\ 00045 --fault,-f\n\ 00046 ", basename(argv[0]),basename(argv[0]),basename(argv[0]),basename(argv[0]),opt_period 00047 00048 enum { s_desc, s_file, s_linespec, s_fieldspec, s_multiplier, s_rate }; 00049 00050 #include <stdio.h> // printf 00051 #include <getopt.h> // getopt_long, {no,required,optional}_argument, extern char *optarg; extern int opt{ind,err,opt} 00052 #include <unistd.h> // getpid, sysconf 00053 #include <sys/wait.h> // wait 00054 #include <fcntl.h> // O_WRONLY|O_CREAT, open 00055 #include <sys/time.h> /* gettimeofday, timeval */ 00056 #include <signal.h> /* sigaction, siginfo_t, sigset_t */ 00057 #include <sys/utsname.h> // uname 00058 #include <string> 00059 #include <sstream> // std::stringstream 00060 #include <vector> 00061 #include <thread> 00062 #include <fstream> // std::ifstream 00063 00064 #ifdef DO_TRACE 00065 # define TRACE_NAME "periodic_cmd_stats" 00066 # include "trace.h" 00067 #else 00068 # include <stdarg.h> /* va_list */ 00069 # include <string.h> /* memcpy */ 00070 # include <string> 00071 static void trace_ap(const char*msg,va_list ap) 00072 { char m_[1024]; unsigned len=strlen(msg); 00073 len=len<(sizeof(m_)-2)?len:(sizeof(m_)-2); memcpy(m_,msg,len+1); 00074 if(m_[len-1]!='\n') memcpy(&(m_[len]),"\n",2); 00075 vprintf(m_,ap);va_end(ap); 00076 } 00077 static void trace_p(const char*msg,...)__attribute__((format(printf,1,2))); 00078 static void trace_p(const char*msg,...) 00079 { va_list ap;va_start(ap,msg); trace_ap(msg ,ap); va_end(ap); 00080 } 00081 static void trace_p(const std::string msg,...) 00082 { va_list ap;va_start(ap,msg); trace_ap(msg.c_str(),ap); va_end(ap); 00083 } 00084 # define TRACE(lvl,...) do if(lvl<=1)trace_p(__VA_ARGS__);while(0) 00085 # define TRACE_CNTL( xyzz,... ) 00086 #endif 00087 00088 /* GLOBALS */ 00089 int opt_v=1; 00090 char* opt_init=NULL; 00091 std::vector<std::string> opt_cmd; 00092 std::vector<std::string> opt_Cmd; 00093 std::string opt_pid; 00094 std::string opt_disk; 00095 std::string opt_stats; 00096 std::string opt_outdir(""); 00097 std::string opt_graph("CPUnode,Cached,Dirty,Free"); // CPU+ will always be graphed 00098 const char* opt_period="5.0"; 00099 std::string opt_comment; 00100 int opt_pre=6; // number of periods to sleepB4exec 00101 int opt_post=6; 00102 int opt_ymin=0; 00103 int opt_ymax=2000; 00104 int opt_yincr=200; 00105 int opt_y2max=200; 00106 int opt_y2incr=20; 00107 int opt_sys_iowait=0; 00108 int opt_cmd_iowait=0; 00109 int opt_fault=0; 00110 00111 std::vector<pid_t> g_pid_vec; 00112 00113 void charreplace( char* instr, char oldc, char newc ) 00114 { 00115 while (*instr) { 00116 if (*instr == oldc) 00117 *instr=newc; 00118 ++instr; 00119 } 00120 } 00121 00122 00123 void parse_args( int argc, char *argv[] ) 00124 { 00125 char *cp; 00126 // parse opt, optargs, and args 00127 while (1) { 00128 int opt; 00129 static struct option long_options[] = { 00130 // name has_arg *flag val 00131 { "help", no_argument, 0, 'h' }, 00132 { "init", required_argument,0, 'i' }, 00133 { "cmd", required_argument,0, 'c' }, 00134 { "Cmd", required_argument,0, 'C' }, 00135 { "disk", required_argument,0, 'd' }, 00136 { "stat", required_argument,0, 's' }, 00137 { "out-dir", required_argument,0, 'o' }, 00138 { "period", required_argument,0, 'p' }, 00139 { "sys-iowait", no_argument, 0, 'w' }, 00140 { "fault", no_argument, 0, 'f' }, 00141 { "pid", required_argument,0, 'P' }, 00142 { "ymax", required_argument,0, 1 }, 00143 { "yincr", required_argument,0, 2 }, 00144 { "y2max", required_argument,0, 3 }, 00145 { "y2incr", required_argument,0, 4 }, 00146 { "pre", required_argument,0, 5 }, 00147 { "post", required_argument,0, 6 }, 00148 { "graph", required_argument,0, 7 }, 00149 { "yrange", required_argument,0, 8 }, 00150 { "comment", required_argument,0, 9 }, 00151 { "cmd-iowait", no_argument, 0, 10 }, 00152 { 0, 0, 0, 0 } 00153 }; 00154 opt = getopt_long( argc, argv, "?hvqVi:c:C:d:s:o:p:P:wf", 00155 long_options, NULL ); 00156 if (opt == -1) break; 00157 switch (opt) { 00158 case '?': case 'h': printf(USAGE); exit(0); break; 00159 case 'V': printf("%s\n",rev); exit(0); break; 00160 case 'v': ++opt_v; break; 00161 case 'q': --opt_v; break; 00162 case 'i': opt_init=optarg; break; 00163 case 'c': opt_cmd.push_back(optarg); break; 00164 case 'C': opt_Cmd.push_back(optarg); break; 00165 case 'd': if(opt_disk.size())opt_disk=opt_disk+","+optarg;else opt_disk=optarg;break; 00166 case 's': if(opt_stats.size())opt_stats=opt_stats+","+optarg;else opt_stats=optarg;break; 00167 case 'o': opt_outdir=std::string(optarg)+"/"; break; 00168 case 'p': opt_period=optarg; break; 00169 case 'w': opt_sys_iowait=1; break; 00170 case 'f': opt_fault=1; break; 00171 case 'P': charreplace(optarg,' ',','); 00172 if(opt_pid.size())opt_pid=opt_pid+","+optarg; 00173 else opt_pid=optarg; 00174 break; 00175 case 1: opt_ymax=strtoul(optarg,NULL,0); break; 00176 case 2: opt_yincr=strtoul(optarg,NULL,0); break; 00177 case 3: opt_y2max=strtoul(optarg,NULL,0); break; 00178 case 4: opt_y2incr=strtoul(optarg,NULL,0); break; 00179 case 5: opt_pre=strtoul(optarg,NULL,0); break; 00180 case 6: opt_post=strtoul(optarg,NULL,0); break; 00181 case 7: opt_graph+=std::string(",")+optarg; break; 00182 case 8: opt_ymin =strtoul(optarg,NULL,0); 00183 cp = strstr(optarg,":")+1; 00184 opt_ymax =strtoul(cp,NULL,0); 00185 if ((cp=strstr(cp,":"))) { 00186 ++cp; 00187 opt_yincr=strtoul(strstr(cp,":")+1,NULL,0); 00188 } else 00189 opt_yincr=(opt_ymax-opt_ymin)/5; 00190 break; 00191 case 9: opt_comment=optarg; break; 00192 case 10: opt_cmd_iowait=1; break; 00193 default: 00194 printf( "?? getopt returned character code 0%o ??\n", opt ); 00195 exit( 1 ); 00196 } 00197 } 00198 } /* parse_args */ 00199 00200 void perror_exit( const char *msg, ... ) 00201 { char buf[1024]; 00202 va_list ap;va_start(ap,msg); 00203 vsnprintf( buf, sizeof(buf), msg, ap ); 00204 va_end(ap); 00205 TRACE( 0, "%s", buf ); 00206 perror( buf ); exit(1); 00207 } 00208 00209 //void atfork_trace(void) { TRACE( 3, "process %d forking", getpid() ); } 00210 /* iofd is in/out 00211 if iofd[x]==-1 then create a pipe for that index, x, and return the appropriate pipe fd in iofd[x] 00212 else if iofd[x]!=x, dup2(iofd[x],x) 00213 else inherit 00214 Could add ==-2, then close??? 00215 */ 00216 pid_t fork_execv( int close_start, int close_cnt, int sleepB4exec_us, int iofd[3], const char *cmd, char *const argv[], char *const env[] ) 00217 { 00218 int pipes[3][2]; 00219 int lcl_iofd[3]; 00220 for (auto ii=0; ii<3; ++ii) { 00221 lcl_iofd[ii]=iofd[ii]; 00222 if (iofd[ii]==-1) { 00223 pipe(pipes[ii]); /* pipes[ii][0] refers to the read end */ 00224 iofd[ii]=ii==0?pipes[ii][1]:pipes[ii][0]; 00225 } 00226 } 00227 pid_t pid=fork(); 00228 if (pid < 0) perror_exit("fork"); 00229 else if (pid == 0) { /* child */ 00230 if (lcl_iofd[0]==-1) { // deal with child stdin 00231 close(pipes[0][1]); // child closes write end of pipe which will be it's stdin 00232 int fd=dup2(pipes[0][0],0); 00233 TRACE( 3, "fork_execv dupped(%d) onto %d (should be 0)", pipes[0][0], fd ); 00234 close(pipes[0][0]); 00235 } 00236 if (sleepB4exec_us) { 00237 // Do sleep before dealing with stdout/err incase we want TRACE to go to console 00238 //int sts=pthread_atfork( atfork_trace, NULL, NULL ); 00239 usleep(sleepB4exec_us); 00240 TRACE( 1, "fork_execv sleep complete. sleepB4exec_us=%d sts=%d", sleepB4exec_us, 0/*sts*/ ); 00241 } 00242 for (auto ii=1; ii<3; ++ii) { // deal with child stdout/err 00243 if (lcl_iofd[ii]==-1) { 00244 close(pipes[ii][0]); 00245 int fd=dup2(pipes[ii][1],ii); 00246 TRACE( 3, "fork_execv dupped(%d) onto %d (should be %d)", pipes[ii][1], fd,ii ); 00247 close(pipes[ii][1]); 00248 } else if (lcl_iofd[ii]!=ii) { 00249 int fd=dup2(lcl_iofd[ii],ii); 00250 TRACE( 3, "fork_execv dupped(%d) onto %d (should be %d)", pipes[ii][1], fd,ii ); 00251 } 00252 } 00253 for (auto ii=close_start; ii<(close_start+close_cnt); ++ii) 00254 close(ii); 00255 if (env) 00256 execve( cmd, argv, env ); 00257 else 00258 execv( cmd, argv ); 00259 exit(1); 00260 } else { // parent 00261 for (auto ii=0; ii<3; ++ii) 00262 if (lcl_iofd[ii]==-1) 00263 close(ii==0?pipes[ii][0]:pipes[ii][1]); 00264 } 00265 00266 TRACE( 3, "fork_execv pid=%d", pid ); 00267 return pid; 00268 } // fork_execv 00269 00270 uint64_t swapPtr( void *X ) 00271 { 00272 uint64_t x=(uint64_t)X; 00273 x = (x & 0x00000000ffffffff) << 32 | (x & 0xffffffff00000000) >> 32; 00274 x = (x & 0x0000ffff0000ffff) << 16 | (x & 0xfff0000fffff0000) >> 16; 00275 x = (x & 0x00ff00ff00ff00ff) << 8 | (x & 0xff00ff00ff00ff00) >> 8; 00276 return x; 00277 } 00278 00279 /* 00280 * Input to AWK can either be a file spec or a string. 00281 * If input is string, the fork_execv call is told to create pipe for input. 00282 * 00283 * The run time duration of the AWK prooces can be determined via TRACE: 00284 /home/ron/src 00285 mu2edaq01 :^) tshow|egrep 'AWK b4 |AWK after read' |tdelta -d 1 -post /b4/ -stats | tail 00286 1013 1489724640538688 2047 1116418481 13521 0 6 3 . AWK b4 fork_execv input=(nil) 00287 1018 1489724640536624 1969 1111669678 13521 0 6 3 . AWK b4 fork_execv input=(nil) 00288 1023 1489724640534717 1866 1107283893 13521 0 6 3 . AWK b4 fork_execv input=(nil) 00289 1032 1489724640531756 2289 1100474359 13521 0 13 3 . AWK b4 fork_execv input=(nil) 00290 cpu="0" 00291 min 1821 00292 max 49210 00293 tot 293610 00294 ave 2645.1351 00295 cnt 111 00296 --2017-03-17_08:13:23-- 00297 */ 00298 static int g_devnullfd=-1; 00299 00300 // Run the awk script specified in awk_cmd on the file 00301 std::string AWK( std::string const &awk_cmd, const char *file, const char *input ) 00302 { 00303 char readbuf[1024]; 00304 ssize_t bytes=0, tot_bytes=0; 00305 char *const argv_[4]={ (char*)"/bin/gawk", 00306 (char*)awk_cmd.c_str(), 00307 (char*)file, 00308 NULL }; 00309 pid_t pid;; 00310 int infd=0; 00311 if (g_devnullfd == -1) 00312 g_devnullfd=open("/dev/null",O_WRONLY); 00313 if (input != NULL) { 00314 infd=-1; 00315 } 00316 //int iofd[3]={infd,-1,g_devnullfd}; 00317 int iofd[3]={infd,-1,2};// make stdin=infd, create pipr for stdout, inherit stderr 00318 TRACE( 3, "AWK b4 fork_execv input=%p", (void*)input ); 00319 char *env[1]; 00320 env[0]=NULL; // mainly do not want big LD_LIBRARY_PATH 00321 pid=fork_execv(0,0/*closeCnt*/,0,iofd,"/bin/gawk",argv_,env); 00322 if(input/*||iofd[0]!=0*/) { 00323 int xx=strlen(input); 00324 int sts=write(iofd[0],input,xx); 00325 if (sts != xx) 00326 perror("write AWK stdin"); 00327 close(iofd[0]); 00328 while ((bytes=read(iofd[1],&readbuf[tot_bytes],sizeof(readbuf)-tot_bytes)) != 0) { 00329 TRACE( 3, "AWK while bytes=read > 0 bytes=%zd readbuf=0x%016lx errno=%d", bytes, swapPtr(&readbuf[tot_bytes]), errno ); 00330 if (bytes == -1) { 00331 if (errno == EINTR) continue; 00332 break; 00333 } 00334 tot_bytes+=bytes; 00335 } 00336 TRACE( 3, "AWK after read tot="+std::to_string((long long unsigned)tot_bytes)+" bytes="+std::to_string((long long unsigned)bytes)+" input="+std::string(input) ); 00337 } else { 00338 while ((bytes=read(iofd[1],&readbuf[tot_bytes],sizeof(readbuf)-tot_bytes)) > 0) 00339 tot_bytes+=bytes; 00340 TRACE( 3, "AWK after read tot=%zd bytes=%zd [0]=0x%x input=%p", tot_bytes, bytes, readbuf[0], (void*)input ); 00341 } 00342 readbuf[tot_bytes>=0?tot_bytes:0]='\0'; 00343 close(iofd[1]); 00344 TRACE( 3, "AWK after close child stdout. child pid=%d", pid ); 00345 #if 0 00346 int status; 00347 pid_t done_pid = waitpid(pid,&status,0); 00348 TRACE( 3, "AWK after wait pid=%d done_pid=%d status=%d(0x%x)" 00349 , pid, done_pid, status, status ); 00350 #endif 00351 return std::string(readbuf); 00352 } // AWK 00353 00354 00355 // separate string and _add_to_ vector 00356 void string_addto_vector( std::string &instr, std::vector<std::string> &outvec, char delim ) 00357 { 00358 std::stringstream ss(instr); 00359 while( ss.good() ) 00360 { std::string substr; 00361 std::getline( ss, substr, delim ); 00362 outvec.push_back( substr ); 00363 } 00364 } 00365 00366 uint64_t gettimeofday_us( void ) //struct timespec *ts ) 00367 { struct timeval tv; 00368 gettimeofday( &tv, NULL ); 00369 // if (ts) { 00370 // ts->tv_sec = tv.tv_sec; 00371 // ts->tv_nsec = tv.tv_usec * 1000; 00372 // } 00373 return (uint64_t)tv.tv_sec*1000000+tv.tv_usec; 00374 } /* gettimeofday_us */ 00375 00376 #define DATA_START " DATA START" 00377 #define GNUPLOT_PREFIX (const char *)"\ 00378 #!/usr/bin/env gnuplot\n\ 00379 # ./$0\n\ 00380 # OR\n\ 00381 # gnuplot -e 'ymin=400;ymax=1400' ./$0\n\ 00382 # OR try\n\ 00383 # gnuplot -e 'duration_s=35;set multiplot' ./gnuplot.gnuplot ./gnuplot.1.gnuplot -e 'set nomultiplot;pause -1'\n\ 00384 if(!exists('ARG0')) ARG0='' # for version 4, use: gnuplot -e ARG0=hello\n\ 00385 print 'ARG0=',ARG0 # ARG0.. automatically define in gnuplot version 5+\n\ 00386 if(!exists('ymin')) ymin=%d\n\ 00387 if(!exists('ymax')) ymax=%d\n\ 00388 if(!exists('yincr')) yincr=%d\n\ 00389 if(!exists('y2max')) y2max=%d\n\ 00390 if(!exists('y2incr')) y2incr=%d\n\ 00391 if(!exists('png')) png=1\n\ 00392 if(!exists('duration_s')) duration_s=0\n\ 00393 if(!exists('width')) width=512\n\ 00394 if(!exists('height')) height=384\n\ 00395 thisPid=system('echo `ps -p$$ -oppid=`')\n\ 00396 thisFile=system('ls -l /proc/'.thisPid.\"/fd | grep -v pipe: | tail -1 | sed -e 's/.*-> //'\")\n\ 00397 \n\ 00398 set title \"Disk Write Rate and %%CPU vs. time\\n%s %s %s%s\" # cmd and/or comment at end\n\ 00399 set xdata time\n\ 00400 tfmt='%%Y-%%m-%%dT%%H:%%M:%%S' # try to use consistent format\n\ 00401 set timefmt '%%Y-%%m-%%dT%%H:%%M:%%S'\n\ 00402 set xlabel 'time'\n\ 00403 set grid xtics back\n\ 00404 xstart=system(\"awk '/^....-..-..T/{print$1;exit}' \".thisFile)\n\ 00405 xend=system(\"awk 'END{print$1}' \".thisFile)\n\ 00406 print 'xstart='.xstart.' xend='.xend.' duration=',strptime(tfmt,xend)-strptime(tfmt,xstart)\n\ 00407 if(duration_s>0) end_t=strptime(tfmt,xstart)+duration_s; else end_t=strptime(tfmt,xend)\n\ 00408 set xrange [xstart:end_t]\n\ 00409 \n\ 00410 set ylabel '%s'\n\ 00411 set ytics nomirror\n\ 00412 if(ymax==0) set yrange [ymin:*];\\\n\ 00413 else set yrange [ymin:ymax];set ytics yincr\n\ 00414 set grid ytics back\n\ 00415 \n\ 00416 set y2label '%%CPU, %%MemTotal'\n\ 00417 set y2tics autofreq\n\ 00418 if(y2max==0) set y2range [0:*];\\\n\ 00419 else set y2range [0:y2max];set y2tics y2incr\n\ 00420 set pointsize .6\n\ 00421 \n\ 00422 if(png==1) set terminal png size width,height;\\\n\ 00423 pngfile=system( 'echo `basename '.thisFile.' .out`.png' );\\\n\ 00424 set output pngfile;\\\n\ 00425 else set terminal x11 size width,height\n\ 00426 \n\ 00427 plot \"< awk '/^#" DATA_START "/,/NEVER HAPPENS/' \".thisFile " 00428 00429 00430 void sigchld_sigaction( int signo, siginfo_t *info, void *context __attribute__((__unused__)) ) 00431 { 00432 /* see man sigaction for description of siginfo_t */ 00433 for (size_t ii=0; ii < g_pid_vec.size(); ++ii) { 00434 pid_t pid=g_pid_vec[ii]; 00435 if (pid == info->si_pid) { 00436 TRACE( 2, "sigchld_sigaction signo=%d status=%d(0x%x) code=%d(0x%x) sending_pid=%d" 00437 , signo 00438 , info->si_status, info->si_status 00439 , info->si_code, info->si_code 00440 , info->si_pid 00441 ); 00442 return; 00443 } 00444 } 00445 TRACE( 3, "sigchld_sigaction signo=%d status=%d(0x%x) code=%d(0x%x) sending_pid=%d" 00446 , signo 00447 , info->si_status, info->si_status 00448 , info->si_code, info->si_code 00449 , info->si_pid 00450 ); 00451 } 00452 00453 00454 void read_proc_file( const char *file, char *buffer, int buffer_size ) 00455 { 00456 TRACE( 4, "read_proc_file b4 open proc file"+std::string(file) ); 00457 int fd=open(file,O_RDONLY); 00458 int offset=0, sts=0; 00459 while (1) { 00460 sts=read(fd,&buffer[offset],buffer_size-offset); 00461 if (sts<=0) { 00462 sts=0; 00463 break; 00464 } 00465 offset+=sts; 00466 } 00467 buffer[sts+offset]='\0'; 00468 close(fd); 00469 TRACE( 4, "read_proc_file after close "+std::string(file)+" read=%d offset=%d",sts,offset ); 00470 } 00471 00472 00473 pid_t check_pid_vec( void ) 00474 { 00475 for (size_t ii=0; ii < g_pid_vec.size(); ) { 00476 pid_t pid=g_pid_vec[ii]; 00477 int status; 00478 pid_t pp = waitpid( pid, &status, WNOHANG ); 00479 TRACE( 3, "check_pid_vec %d=waitpid(pid=%d) errno=%d", pp, pid, errno ); 00480 if (pp > 0) 00481 g_pid_vec.erase( g_pid_vec.begin()+ii ); 00482 else if (pp == -1) { 00483 if (errno == ECHILD && kill(pid,0)==0) 00484 // there is a process, but not my child process 00485 ++ii; 00486 else 00487 // some other error 00488 g_pid_vec.erase( g_pid_vec.begin()+ii ); 00489 } 00490 else 00491 ++ii; 00492 } 00493 if (g_pid_vec.size() == 0) 00494 return -1; 00495 else 00496 return 0; 00497 } 00498 00499 void cleanup( void ) 00500 { 00501 TRACE( 1, "atexit cleanup g_pid_vec.size()=%zd\n", g_pid_vec.size() ); 00502 for (std::vector<pid_t>::iterator pid=g_pid_vec.begin(); pid!=g_pid_vec.end(); ++pid) { 00503 kill( *pid, SIGHUP ); 00504 } 00505 } 00506 #if (defined(__cplusplus)&&(__cplusplus>=201103L)) || (defined(__STDC_VERSION__)&&(__STDC_VERSION__>=201112L)) 00507 # pragma GCC diagnostic push 00508 # pragma GCC diagnostic ignored "-Wunused-parameter" /* b/c of TRACE_XTRA_UNUSED */ 00509 #endif 00510 void sigint_sigaction( int signo, siginfo_t *info, void *context ) 00511 { 00512 cleanup(); 00513 exit( 1 ); 00514 } 00515 #if (defined(__cplusplus)&&(__cplusplus>=201103L)) || (defined(__STDC_VERSION__)&&(__STDC_VERSION__>=201112L)) 00516 # pragma GCC diagnostic pop 00517 #endif 00518 00519 00520 int 00521 main( int argc 00522 , char *argv[] ) 00523 { 00524 struct timeval tv; 00525 int post_periods_completed=0; 00526 parse_args( argc, argv ); 00527 if ( (argc-optind)!=0 00528 || ( opt_cmd.size()==0 00529 && opt_pid.size()==0)) { //(argc-optind) is the number of non-opt args supplied. 00530 int ii; 00531 printf( "unexpected argument(s) %d!=0\n", argc-optind ); 00532 for (ii=0; (optind+ii)<argc; ++ii) 00533 printf("arg%d=%s\n",ii+1,argv[optind+ii]); 00534 printf( USAGE ); exit( 0 ); 00535 } 00536 00537 std::vector<std::string> graphs; 00538 string_addto_vector( opt_graph, graphs, ',' ); 00539 00540 char motherboard[1024]={0}; 00541 if (getuid() == 0) { 00542 FILE *fp=popen( "dmidecode | grep -m2 'Product Name:' | tail -1", "r" ); 00543 fread( motherboard, 1, sizeof(motherboard), fp ); 00544 pclose( fp ); 00545 } 00546 TRACE( 1, "main - motherboard="+std::string(motherboard) ); 00547 00548 /* Note, when doing "waitpid" the wait would sometimes take a "long" 00549 time (10's to 100's milliseconds; rcu???) If signal is generated 00550 (i.e SA_NOCLDWAIT w/ sigchld_sigaction (not SIG_IGN)), it would 00551 sometimes effect the read or write calls for the following AWK forks. 00552 So, use SIG_IGN+SA_NOCLDWAIT. 00553 */ 00554 struct sigaction sigaction_s; 00555 #ifndef DO_SIGCHLD 00556 # define DO_SIGCHLD 1 00557 #endif 00558 #if DO_SIGCHLD 00559 sigaction_s.sa_sigaction = sigchld_sigaction; 00560 sigaction_s.sa_flags = SA_SIGINFO|SA_NOCLDWAIT; 00561 #else 00562 sigaction_s.sa_handler = SIG_IGN; 00563 sigaction_s.sa_flags = SA_NOCLDWAIT; 00564 #endif 00565 sigemptyset(&sigaction_s.sa_mask); 00566 sigaction( SIGCHLD, &sigaction_s, NULL ); 00567 00568 sigaction_s.sa_sigaction = sigint_sigaction; 00569 sigaction_s.sa_flags = SA_SIGINFO; 00570 sigaction( SIGINT, &sigaction_s, NULL ); 00571 00572 //may return 0 when not able to detect 00573 //long long unsigned concurentThreadsSupported = std::thread::hardware_concurrency(); 00574 long long unsigned concurentThreadsSupported = sysconf(_SC_NPROCESSORS_ONLN); 00575 //TRACE_CNTL( "reset" ); TRACE_CNTL( "modeM", 1L ); 00576 TRACE( 0, "main concurentThreadsSupported=%u opt_stats="+opt_stats, concurentThreadsSupported ); 00577 00578 char run_time[80]; 00579 gettimeofday( &tv, NULL ); 00580 strftime( run_time, sizeof(run_time), "%FT%H%M%S", localtime(&tv.tv_sec) ); 00581 TRACE( 0, "main run_time="+std::string(run_time) ); 00582 00583 // get hostname 00584 struct utsname ubuf; 00585 uname( &ubuf ); 00586 char *dot; 00587 if ((dot=strchr(ubuf.nodename,'.')) != NULL) 00588 *dot = '\0'; 00589 std::string hostname(ubuf.nodename); 00590 TRACE( 1,"release="+std::string(ubuf.release)+" version="+std::string(ubuf.version) ); 00591 00592 // get system mem (KB) 00593 std::string memKB=AWK( "NR==1{print$2;exit}","/proc/meminfo",NULL ); 00594 memKB = memKB.substr(0,memKB.size()-1); // remove trailing newline 00595 00596 std::string dat_file_out(opt_outdir+"periodic_"+run_time+"_"+hostname+"_stats.out"); 00597 00598 double period=atof(opt_period); 00599 00600 atexit( cleanup ); 00601 pid_t pp; 00602 std::vector<std::string> pidfile; 00603 00604 std::vector<std::string> stats; 00605 00606 // For each cmd: create out file, fork process (with delay param), 00607 // add to stats vec to get CPU info, add to graphs vec to plot cmd CPU 00608 for (size_t ii=0; ii<opt_cmd.size(); ++ii) { 00609 char cmd_file_out[1024]; 00610 snprintf( cmd_file_out, sizeof(cmd_file_out), "%speriodic_%s_%s_cmd%zd.out" 00611 , opt_outdir.c_str(), run_time, hostname.c_str(), ii ); 00612 int fd=open( cmd_file_out, O_WRONLY|O_CREAT,0666 ); 00613 TRACE( 0, "main fd=%d opt_cmd="+opt_cmd[ii]+" cmd_file_out="+std::string(cmd_file_out), fd ); 00614 int iofd[3]={0,fd,fd}; // redirect stdout/err to the cmd-out-file 00615 char *const argv_[4]={ (char*)"/bin/sh", 00616 (char*)"-c", 00617 (char*)opt_cmd[ii].c_str(), 00618 NULL }; 00619 g_pid_vec.push_back( fork_execv(0,0,(int)(period*opt_pre*1e6),iofd,"/bin/sh",argv_,NULL) ); 00620 close(fd); // the output file has been given to the subprocess 00621 std::string pidstr=std::to_string((long long int)g_pid_vec[ii]); 00622 pidfile.push_back( "/proc/"+pidstr+"/stat" ); 00623 //pidfile.push_back( "/proc/"+pidstr+"/task/"+pidstr+"/stat" ); 00624 char desc[128], ss[1024]; 00625 // field 14-17: Documentation/filesystems/proc.txt Table 1-4: utime stime cutime cstime 00626 snprintf( ss, sizeof(ss), "CPUcmd%zd?%s?NR==1?$14+$15?1?yes", ii, pidfile[ii].c_str() ); 00627 stats.push_back( ss ); 00628 00629 snprintf( desc, sizeof(desc), "CPU+cmd%zd", ii ); 00630 graphs.push_back( desc ); // cmd0 is in the GNUPLOT_PREFIX 00631 snprintf( ss, sizeof(ss), "%s?%s?NR==1?$14+$15+16+$17?1?yes", desc, pidfile[ii].c_str() ); 00632 stats.push_back( ss ); 00633 00634 snprintf( desc, sizeof(desc), "WaitBlkIOcmd%zd", ii ); 00635 if (opt_cmd_iowait) graphs.push_back( desc ); 00636 snprintf( ss, sizeof(ss), "%s?%s?NR==1?$42?1?yes", desc, pidfile[ii].c_str() ); 00637 stats.push_back( ss ); 00638 00639 snprintf( desc, sizeof(desc), "Faultcmd%zd", ii ); 00640 if (opt_fault) graphs.push_back( desc ); 00641 snprintf( ss, sizeof(ss), "%s?%s?NR==1?$10+$11+$12+$13?4096.0/1048576?yes", desc, pidfile[ii].c_str() ); 00642 stats.push_back( ss ); 00643 } 00644 for (size_t ii=0; ii<opt_Cmd.size(); ++ii) { 00645 char cmd_file_out[1024]; 00646 snprintf( cmd_file_out, sizeof(cmd_file_out), "%speriodic_%s_%s_cmd%zd.out" 00647 , opt_outdir.c_str(), run_time, hostname.c_str(), ii+opt_cmd.size() ); 00648 int fd=open( cmd_file_out, O_WRONLY|O_CREAT,0666 ); 00649 TRACE( 0, "main fd=%d opt_Cmd="+opt_Cmd[ii]+" cmd_file_out="+std::string(cmd_file_out), fd ); 00650 int iofd[3]={0,fd,fd}; // redirect stdout/err to the cmd-out-file 00651 char *const argv_[4]={ (char*)"/bin/sh", 00652 (char*)"-c", 00653 (char*)opt_Cmd[ii].c_str(), 00654 NULL }; 00655 g_pid_vec.push_back( fork_execv(0,0,(int)(period*opt_pre*1e6),iofd,"/bin/sh",argv_,NULL) ); 00656 close(fd); // the output file has been given to the subprocess 00657 std::string pidstr=std::to_string((long long int)g_pid_vec[ii]); 00658 pidfile.push_back( "/proc/"+pidstr+"/stat" ); 00659 //pidfile.push_back( "/proc/"+pidstr+"/task/"+pidstr+"/stat" ); 00660 char desc[128], ss[1024]; 00661 snprintf( desc, sizeof(desc), "CPU+cmd%zd", ii+opt_cmd.size() ); 00662 snprintf( ss, sizeof(ss), "CPUcmd%zd?%s?NR==1?$14+$15?1?yes", ii+opt_cmd.size(), pidfile[ii].c_str() ); 00663 stats.push_back( ss ); 00664 snprintf( ss, sizeof(ss), "CPU+cmd%zd?%s?NR==1?$14+$15+16+$17?1?yes", ii+opt_cmd.size(), pidfile[ii].c_str() ); 00665 stats.push_back( ss ); 00666 // JUST DONT ADD THESE TO graphs 00667 } 00668 std::vector<std::string> pids; 00669 if (opt_pid.size()) 00670 string_addto_vector( opt_pid, pids, ',' ); 00671 for (size_t ii=0; ii<pids.size(); ++ii) { 00672 g_pid_vec.push_back( std::stoi(pids[ii]) ); 00673 TRACE( 1, "pid=%s g_pid_vec.size()=%ld", pids[ii].c_str(), g_pid_vec.size() ); 00674 pidfile.push_back( "/proc/"+pids[ii]+"/stat" ); 00675 char desc[128], ss[1024]; 00676 // field 14-17: Documentation/filesystems/proc.txt Table 1-4: utime stime cutime cstime 00677 snprintf( ss, sizeof(ss), "CPUpid%zd?%s?NR==1?$14+$15?1?yes", ii, pidfile[ii].c_str() ); 00678 stats.push_back( ss ); 00679 00680 std::ifstream t("/proc/"+pids[ii]+"/comm"); 00681 std::string comm((std::istreambuf_iterator<char>(t)), 00682 std::istreambuf_iterator<char>()); 00683 comm = comm.substr(0,comm.size()-1); // strip nl 00684 00685 snprintf( desc, sizeof(desc), "CPU+pid%zd_%s", ii, comm.c_str() ); 00686 graphs.push_back( desc ); // cmd0 is in the GNUPLOT_PREFIX 00687 snprintf( ss, sizeof(ss), "%s?%s?NR==1?$14+$15+16+$17?1?yes", desc, pidfile[ii].c_str() ); 00688 stats.push_back( ss ); 00689 00690 snprintf( desc, sizeof(desc), "WaitBlkIOpid%zd", ii ); 00691 if (opt_cmd_iowait) graphs.push_back( desc ); 00692 snprintf( ss, sizeof(ss), "%s?%s?NR==1?$42?1?yes", desc, pidfile[ii].c_str() ); 00693 stats.push_back( ss ); 00694 00695 snprintf( desc, sizeof(desc), "Faultpid%zd", ii ); 00696 if (opt_fault) graphs.push_back( desc ); 00697 snprintf( ss, sizeof(ss), "%s?%s?NR==1?$10+$11+$12+$13?4096.0/1048576?yes", desc, pidfile[ii].c_str() ); 00698 stats.push_back( ss ); 00699 } 00700 00701 00702 stats.push_back("CPUnode"); 00703 stats.push_back("IOWait"); if (opt_sys_iowait) { graphs.push_back("IOWait"); } 00704 stats.push_back("Cached"); 00705 stats.push_back("Dirty"); 00706 stats.push_back("Free"); 00707 00708 if (opt_disk.size()) { 00709 std::vector<std::string> tmp; 00710 string_addto_vector( opt_disk, tmp, ',' ); 00711 for (std::vector<std::string>::iterator dk=tmp.begin(); dk!=tmp.end(); ++dk) { 00712 // /proc/diskstat has 11 field after an initial 3 (14 total) for each device 00713 // The 7th field after the device name (the 10th field total) is # of sectors written. 00714 // Sectors appear to be 512 bytes. So, deviding by 2048 converts to MBs. 00715 std::string statstr=*dk+"_wrMB/s?/proc/diskstats?/"+*dk+"/?$10?(1.0/2048)?yes"; 00716 stats.push_back(statstr); 00717 std::vector<std::string> stat_spec; 00718 string_addto_vector( statstr, stat_spec, '?' ); 00719 graphs.push_back( stat_spec[s_desc] ); 00720 00721 statstr=*dk+"_rdMB/s?/proc/diskstats?/"+*dk+"/?$6?(1.0/2048)?yes"; 00722 stats.push_back(statstr); 00723 stat_spec.clear(); 00724 string_addto_vector( statstr, stat_spec, '?' ); 00725 //graphs.push_back( stat_spec[s_desc] ); // don't add read by default -- can be added with --graph 00726 } 00727 } 00728 00729 if (opt_stats.size()) { 00730 std::vector<std::string> tmp_stats; 00731 string_addto_vector( opt_stats, tmp_stats, ',' ); 00732 for (std::vector<std::string>::iterator st=tmp_stats.begin(); 00733 st!=tmp_stats.end(); ++st) { 00734 stats.push_back(*st); 00735 std::vector<std::string> stat_spec; 00736 string_addto_vector( *st, stat_spec, '?' ); 00737 graphs.push_back( stat_spec[s_desc] ); 00738 } 00739 } 00740 00741 std::vector<long> pre_vals; 00742 std::vector<double> multipliers; 00743 std::vector<std::vector<std::string>> spec2(stats.size()); 00744 std::vector<std::string> awkCmd; 00745 00746 std::string header_str( "#" DATA_START "\n#_______time_______" ); 00747 00748 int outfd=open(dat_file_out.c_str(),O_WRONLY|O_CREAT,0777); 00749 //FILE *outfp=stdout; 00750 FILE *outfp = fdopen(outfd,"w"); 00751 00752 std::string cmd_comment(""); 00753 if (opt_cmd.size()) 00754 cmd_comment += "\\ncmd: "+opt_cmd[0]; 00755 if (opt_comment.size()) 00756 cmd_comment += "\\ncomment: " + opt_comment; 00757 fprintf( outfp, GNUPLOT_PREFIX, opt_ymin, opt_ymax, opt_yincr, opt_y2max, opt_y2incr 00758 , run_time, hostname.c_str(), ubuf.release 00759 , cmd_comment.c_str() 00760 , "disk write MB/s" ); 00761 00762 uint64_t t_start=gettimeofday_us(); 00763 00764 // build header string and get initial values for "rate" stats 00765 bool first_graph_spec_added=false; 00766 for (size_t ii=0; ii<stats.size(); ++ii) { 00767 std::vector<std::string> stat_spec; 00768 string_addto_vector( stats[ii], stat_spec, '?' ); 00769 if (stat_spec[s_desc]=="CPUnode" && stat_spec.size()==1) 00770 // Ref. Documentation/filesystems/proc.txt: user+nice+system (skip idle) +iowait+irq+softirq+steal (skip guest) 00771 stats[ii]+="?/proc/stat?/^cpu[^0-9]/?$2+$3+$4+$6+$7+$8+$9?1.0/"+std::to_string(concurentThreadsSupported)+"?yes"; 00772 else if (stat_spec[s_desc]=="IOWait" && stat_spec.size()==1) 00773 stats[ii]+="?/proc/stat?/^cpu[^0-9]/?$6?1.0/"+std::to_string(concurentThreadsSupported)+"?yes"; 00774 else if (stat_spec[s_desc]=="Cached" && stat_spec.size()==1) 00775 stats[ii]+="?/proc/meminfo?/^(Cached|Buffers):/?$2?1?no"; 00776 else if (stat_spec[s_desc]=="Dirty" && stat_spec.size()==1) 00777 stats[ii]+="?/proc/meminfo?/^Dirty:/?$2?1?no"; 00778 else if (stat_spec[s_desc]=="Free" && stat_spec.size()==1) 00779 stats[ii]+="?/proc/meminfo?/^MemFree:/?$2?1?no"; 00780 00781 header_str += " "+stat_spec[s_desc]; 00782 00783 string_addto_vector( stats[ii], spec2[ii], '?' ); 00784 char awk_cmd[1024]; 00785 snprintf( awk_cmd, sizeof(awk_cmd), "%s{vv+=%s}END{print vv}" 00786 //snprintf( awk_cmd, sizeof(awk_cmd), "%s{vv+=%s;print \"vv now\",vv > \"/dev/stderr\";}END{print vv}" 00787 , spec2[ii][s_linespec].c_str(), spec2[ii][s_fieldspec].c_str() ); 00788 awkCmd.push_back(awk_cmd); 00789 00790 std::string stat=AWK( awkCmd.back(), spec2[ii][s_file].c_str(), NULL ); 00791 00792 pre_vals.push_back(atol(stat.c_str())); 00793 multipliers.push_back(atof(AWK( "BEGIN{print "+spec2[ii][s_multiplier]+"}","/dev/null",NULL ).c_str()) ); 00794 //fprintf( stderr, " l=%s", spec2[ii][s_linespec].c_str() ); 00795 for (size_t jj=0; jj<graphs.size(); ++jj) 00796 if (graphs[jj] == stat_spec[s_desc]) { 00797 if (first_graph_spec_added) fprintf( outfp, ",\\\n '' " ); 00798 if (strncmp(stat_spec[s_desc].c_str(),"CPU",3)==0) 00799 fprintf( outfp, "using 1:%zd title '%s' w linespoints axes x1y2", ii+2, stat_spec[s_desc].c_str() ); 00800 else if (stat_spec[s_desc] == "Cached" || stat_spec[s_desc] == "Dirty" || stat_spec[s_desc] == "Free") 00801 fprintf( outfp, "using 1:($%zd/%s*100) title '%s%%' w linespoints axes x1y2", ii+2, memKB.c_str(), stat_spec[s_desc].c_str() ); 00802 else if (stat_spec[s_desc].substr(0,6) == "CPUcmd" || stat_spec[s_desc].substr(0,6) == "CPU+cm") 00803 fprintf( outfp, "using 1:%zd title '%s' w linespoints axes x1y2", ii+2, stat_spec[s_desc].c_str() ); 00804 else if (stat_spec[s_desc].substr(0,12) == "WaitBlkIOcmd" ) 00805 fprintf( outfp, "using 1:%zd title '%s' w linespoints axes x1y2", ii+2, stat_spec[s_desc].c_str() ); 00806 else 00807 fprintf( outfp, "using 1:%zd title '%s' w linespoints axes x1y1", ii+2, stat_spec[s_desc].c_str() ); 00808 first_graph_spec_added=true; 00809 } 00810 } 00811 header_str += " #\n"; 00812 00813 fprintf( outfp, "\nif(png==0) pause -1 'Press Enter/Return or ^C to finish'\n\ 00814 exit\n" ); 00815 00816 // print the cmds 00817 fprintf( outfp, "cmds:\n" ); 00818 for (size_t ii=0; ii<opt_cmd.size(); ++ii) { 00819 std::string ss=opt_cmd[ii]+"\n"; 00820 fprintf( outfp, "%s", ss.c_str() ); 00821 } 00822 00823 // print the specs 00824 fprintf( outfp, "stats:\n" ); 00825 for (size_t ii=0; ii<stats.size(); ++ii) { 00826 std::string ss=stats[ii]+"\n"; 00827 fprintf( outfp, "%s", ss.c_str() ); 00828 } 00829 00830 // now print header 00831 fprintf( outfp, "%s", header_str.c_str() ); fflush( outfp ); 00832 00833 std::string tmpdbg("main lp=%d done stat%zd=%ld rate=%f "); 00834 //char tmpdbgbuf[128]; 00835 char proc_stats[8192]; 00836 char *awk_in; 00837 int lp; 00838 00839 // - - - - - - - - - - - - - - - - - - - - - - - - 00840 // wait a period and then start collecting the stats 00841 eintr1: 00842 int64_t t_sleep=(t_start+(uint64_t)(period*1e6))-gettimeofday_us(); 00843 if (t_sleep > 0) { 00844 int sts=usleep( t_sleep ); 00845 TRACE( 3,"main usleep sts=%d errno=%d",sts, errno ); 00846 if(errno == EINTR) 00847 goto eintr1; 00848 } 00849 00850 # define MAX_LP 600 00851 for (lp=2; lp<MAX_LP; ++lp) { 00852 char str[80]; 00853 gettimeofday( &tv, NULL ); 00854 strftime( str, sizeof(str), "%FT%T", localtime(&tv.tv_sec) ); 00855 //fprintf(outfp, "%s.%ld", str, tv.tv_usec/100000 ); 00856 fprintf(outfp, "%s", str ); 00857 std::string prv_file(""); 00858 for (size_t ii=0; ii<stats.size(); ++ii) { 00859 TRACE( 3, "main lp=%d start stat%zd", lp, ii ); 00860 char const *awk_file; 00861 if (ii < (2*opt_cmd.size())) { // For each cmd, the 00862 // /proc/<pid>/stat file 00863 // will be referenced twice. 00864 if ((ii&1)==0) { 00865 read_proc_file( pidfile[ii/2].c_str(),proc_stats, sizeof(proc_stats) ); 00866 } 00867 awk_in=proc_stats; awk_file=NULL; 00868 } else if (spec2[ii][s_file] != prv_file) { 00869 prv_file = spec2[ii][s_file]; 00870 read_proc_file( spec2[ii][s_file].c_str(), proc_stats, sizeof(proc_stats) ); 00871 awk_in=proc_stats; awk_file=NULL; 00872 } 00873 00874 00875 std::string stat_str=AWK( awkCmd[ii], awk_file, awk_in ); 00876 00877 long stat=atol(stat_str.c_str()); 00878 00879 if (spec2[ii][s_rate] == "yes") { 00880 double rate; 00881 if (stat_str!="\n") 00882 rate=(stat-pre_vals[ii])*multipliers[ii]/period; 00883 else 00884 rate=0.0; 00885 TRACE( 3, tmpdbg+"stat_str[0]=0x%x stat_str.size()=%zd", lp,ii,stat,rate, stat_str[0], stat_str.size() ); 00886 fprintf(outfp, " %.2f",rate ); 00887 if (rate < 0.0 && spec2[ii][s_file] == "/proc/diskstats") { 00888 TRACE( 0, "main stat:"+spec2[ii][s_desc]+" rate=%f pre_val=%ld stat=%ld stat_str=\""+stat_str\ 00889 +"\" awkCmd="+awkCmd[ii]+" proc_diskstats="+proc_stats 00890 , rate, pre_vals[ii], stat ); 00891 //TRACE_CNTL( "modeM", 0L ); 00892 } 00893 pre_vals[ii] = stat; 00894 } else { 00895 TRACE( 3, "main lp=%d done stat%zd=%ld", lp, ii, stat ); 00896 fprintf(outfp, " %.2f",stat*multipliers[ii] ); 00897 } 00898 } 00899 fprintf(outfp,"\n"); fflush(outfp); 00900 eintr2: 00901 int64_t t_sleep=(t_start+(uint64_t)(period*lp*1000000))-gettimeofday_us(); 00902 if (t_sleep > 0) { 00903 int sts=usleep( t_sleep ); 00904 TRACE( 3,"main usleep sts=%d errno=%d",sts,errno ); 00905 if(errno == EINTR) 00906 goto eintr2; 00907 } 00908 pp = check_pid_vec(); 00909 TRACE( 2, "main pp=%d t_sleep=%ld", pp, t_sleep ); 00910 if (pp == -1) { 00911 if (post_periods_completed == 0) 00912 TRACE( 1, "main processes complete - waiting %d post periods", opt_post ); 00913 if (post_periods_completed++ == opt_post) 00914 break; 00915 } 00916 } 00917 if (lp==MAX_LP) { 00918 fprintf(outfp,"# MAX_LP abort\n" ); 00919 } 00920 00921 //TRACE( 0, "main waiting for pid=%d", pid ); 00922 //wait(&status); 00923 //TRACE( 0, "main status=%d",status ); 00924 TRACE( 0, "main done/complete/returning" ); 00925 //TRACE_CNTL( "modeM", 0L ); 00926 return (0); 00927 } // main