artdaq  v3_09_00
periodic_cmd_stats.cc
1 // This file (periodic_cmd_stats.cc) was created by Ron Rechenmacher <ron@fnal.gov> on
2 // Jan 5, 2017. "TERMS AND CONDITIONS" governing this file are in the README
3 // or COPYING file. If you do not have such a file, one can be obtained by
4 // contacting Ron or Fermi Lab in Batavia IL, 60510, phone: 630-840-3000.
5 // $RCSfile: periodic_cmd_stats.cc,v $
6 static const char *rev = "$Revision: 1.19 $$Date: 2018/06/28 21:14:28 $";
7 
8 // make periodic_cmd_stats
9 // OR make periodic_cmd_stats CXX="g++ -std=c++0x -DDO_TRACE"
10 
11 #define USAGE \
12  "\
13  usage: %s --cmd=<cmd>\n\
14 examples: %s --cmd='sleep 25' # this will take about a minute\n\
15  %s --cmd='taskset -c 4 dd if=/dev/zero of=/dev/sdf bs=50M count=500 oflag=direct' --disk=sdf\n\
16  %s --cmd='dd if=/dev/zero of=t.dat count=500' --stat='md*MB/s?/proc/diskstats?/md[0-3]/?$10?(1.0/2048)?yes'\n\
17  gnuplot -e png=0 `/bin/ls -t periodic_*_stats.out|head -1`\n\
18 For each cmd, record CPU info. Additionally, record total system CPU\n\
19 (can be >100 w/ multicore), file system Cached and Dirty %%, plus any\n\
20 other stats specified by --stats options\n\
21 options:\n\
22 --cmd= can have multiple\n\
23 --Cmd= run cmd, but dont graph CPU (can have multiple)\n\
24 --pid= graph comma or space sep. list of pids (getting cmd from /proc/)\n\
25 --disk= automatically build --stat for disk(s)\n\
26 --stat= desc?file?linespec?fieldspec?multiplier?rate\n\
27  builtin = CPUnode,Cached,Dirty\n\
28  cmd-builtin = CPUcmdN, CPU+cmdN\n\
29 --out-dir= output dir (for 2+ output files; stats and cmd+)\n\
30 \n\
31 --period= (float) default=%s\n\
32 --pre= the number of periods wait before exec of --cmd\n\
33 --post= the number of periods to loop after cmd exits\n\
34 --graph= add to graph any possibly included non-graphed metrics\n\
35 \n\
36 --no-defaults for no default stats (ie. cpu-parent,cpu-children)\n\
37 --init= default dropcache if root. NOT implemented yet\n\
38 --duration=\n\
39 --comment=\n\
40 --yrange=400:1400\n\
41 --y2max=\n\
42 --y2incr=\n\
43 --pause=0\n\
44 --sys-iowait,-w include the system iowait on the graph\n\
45 --cmd-iowait include cmd iowait on the graph\n\
46 --fault,-f\n\
47 ", \
48  basename(argv[0]), basename(argv[0]), basename(argv[0]), basename(argv[0]), opt_period
49 
50 enum
51 {
52  s_desc,
53  s_file,
54  s_linespec,
55  s_fieldspec,
56  s_multiplier,
57  s_rate
58 };
59 
60 #include <fcntl.h> // O_WRONLY|O_CREAT, open
61 #include <getopt.h> // getopt_long, {no,required,optional}_argument, extern char *optarg; extern int opt{ind,err,opt}
62 #include <sys/time.h> /* gettimeofday, timeval */
63 #include <sys/utsname.h> // uname
64 #include <sys/wait.h> // wait
65 #include <unistd.h> // getpid, sysconf
66 #include <csignal> /* sigaction, siginfo_t, sigset_t */
67 #include <cstdio> // printf
68 #include <fstream> // std::ifstream
69 #include <sstream> // std::stringstream
70 #include <string>
71 #include <thread>
72 #include <vector>
73 
74 #ifdef DO_TRACE
75 #define TRACE_NAME "periodic_cmd_stats"
76 #include "trace.h"
77 #else
78 #include <cstdarg> /* va_list */
79 #include <cstring> /* memcpy */
80 #include <string>
81 static void trace_ap(const char *msg, va_list ap)
82 {
83  char m_[1024];
84  unsigned len = strlen(msg);
85  len = len < (sizeof(m_) - 2) ? len : (sizeof(m_) - 2);
86  memcpy(m_, msg, len + 1);
87  if (m_[len - 1] != '\n')
88  {
89  memcpy(&(m_[len]), "\n", 2);
90  }
91  vprintf(m_, ap);
92  va_end(ap);
93 }
94 static void trace_p(const char *msg, ...) __attribute__((format(printf, 1, 2)));
95 static void trace_p(const char *msg, ...)
96 {
97  va_list ap;
98  va_start(ap, msg); // NOLINT
99  trace_ap(msg, ap); // NOLINT
100  va_end(ap); // NOLINT
101 }
102 static void trace_p(const std::string msg, ...)
103 {
104  va_list ap;
105  va_start(ap, msg);
106  trace_ap(msg.c_str(), ap);
107  va_end(ap); // NOLINT
108 }
109 #define TRACE(lvl, ...) \
110  do \
111  if (lvl <= 1) trace_p(__VA_ARGS__); \
112  while (0)
113 #define TRACE_CNTL(xyzz, ...)
114 #endif
115 
116 /* GLOBALS */
117 int opt_v = 1;
118 char *opt_init = nullptr;
119 std::vector<std::string> opt_cmd;
120 std::vector<std::string> opt_Cmd;
121 std::string opt_pid;
122 std::string opt_disk;
123 std::string opt_stats;
124 std::string opt_outdir;
125 std::string opt_graph("CPUnode,Cached,Dirty,Free"); // CPU+ will always be graphed
126 const char *opt_period = "5.0";
127 std::string opt_comment;
128 int opt_pre = 6; // number of periods to sleepB4exec
129 int opt_post = 6;
130 int opt_ymin = 0;
131 int opt_ymax = 2000;
132 int opt_yincr = 200;
133 int opt_y2max = 200;
134 int opt_y2incr = 20;
135 int opt_sys_iowait = 0;
136 int opt_cmd_iowait = 0;
137 int opt_fault = 0;
138 
139 std::vector<pid_t> g_pid_vec;
140 
141 void charreplace(char *instr, char oldc, char newc)
142 {
143  while (*instr != 0)
144  {
145  if (*instr == oldc)
146  {
147  *instr = newc;
148  }
149  ++instr;
150  }
151 }
152 
153 void parse_args(int argc, char *argv[])
154 {
155  char *cp;
156  // parse opt, optargs, and args
157  while (true)
158  {
159  int opt;
160  static struct option long_options[] = {
161  // name has_arg *flag val
162  {"help", no_argument, nullptr, 'h'},
163  {"init", required_argument, nullptr, 'i'},
164  {"cmd", required_argument, nullptr, 'c'},
165  {"Cmd", required_argument, nullptr, 'C'},
166  {"disk", required_argument, nullptr, 'd'},
167  {"stat", required_argument, nullptr, 's'},
168  {"out-dir", required_argument, nullptr, 'o'},
169  {"period", required_argument, nullptr, 'p'},
170  {"sys-iowait", no_argument, nullptr, 'w'},
171  {"fault", no_argument, nullptr, 'f'},
172  {"pid", required_argument, nullptr, 'P'},
173  {"ymax", required_argument, nullptr, 1},
174  {"yincr", required_argument, nullptr, 2},
175  {"y2max", required_argument, nullptr, 3},
176  {"y2incr", required_argument, nullptr, 4},
177  {"pre", required_argument, nullptr, 5},
178  {"post", required_argument, nullptr, 6},
179  {"graph", required_argument, nullptr, 7},
180  {"yrange", required_argument, nullptr, 8},
181  {"comment", required_argument, nullptr, 9},
182  {"cmd-iowait", no_argument, nullptr, 10},
183  {nullptr, 0, nullptr, 0}};
184  opt = getopt_long(argc, argv, "?hvqVi:c:C:d:s:o:p:P:wf",
185  long_options, nullptr);
186  if (opt == -1)
187  {
188  break;
189  }
190  switch (opt)
191  {
192  case '?':
193  case 'h':
194  printf(USAGE);
195  exit(0);
196  break;
197  case 'V':
198  printf("%s\n", rev);
199  exit(0);
200  break;
201  case 'v':
202  ++opt_v;
203  break;
204  case 'q':
205  --opt_v;
206  break;
207  case 'i':
208  opt_init = optarg;
209  break;
210  case 'c':
211  opt_cmd.emplace_back(optarg);
212  break;
213  case 'C':
214  opt_Cmd.emplace_back(optarg);
215  break;
216  case 'd':
217  if (!opt_disk.empty())
218  {
219  opt_disk = opt_disk + "," + optarg;
220  }
221  else
222  {
223  opt_disk = optarg;
224  }
225  break;
226  case 's':
227  if (!opt_stats.empty())
228  {
229  opt_stats += std::string(",") + optarg;
230  }
231  else
232  {
233  opt_stats = optarg;
234  }
235  break;
236  case 'o':
237  opt_outdir = std::string(optarg) + "/";
238  break;
239  case 'p':
240  opt_period = optarg;
241  break;
242  case 'w':
243  opt_sys_iowait = 1;
244  break;
245  case 'f':
246  opt_fault = 1;
247  break;
248  case 'P':
249  charreplace(optarg, ' ', ',');
250  if (!opt_pid.empty() != 0u)
251  {
252  opt_pid = opt_pid + "," + optarg;
253  }
254  else
255  {
256  opt_pid = optarg;
257  }
258  break;
259  case 1:
260  opt_ymax = strtoul(optarg, nullptr, 0);
261  break;
262  case 2:
263  opt_yincr = strtoul(optarg, nullptr, 0);
264  break;
265  case 3:
266  opt_y2max = strtoul(optarg, nullptr, 0);
267  break;
268  case 4:
269  opt_y2incr = strtoul(optarg, nullptr, 0);
270  break;
271  case 5:
272  opt_pre = strtoul(optarg, nullptr, 0);
273  break;
274  case 6:
275  opt_post = strtoul(optarg, nullptr, 0);
276  break;
277  case 7:
278  opt_graph += std::string(",") + optarg;
279  break;
280  case 8:
281  opt_ymin = strtoul(optarg, nullptr, 0);
282  cp = strstr(optarg, ":") + 1;
283  opt_ymax = strtoul(cp, nullptr, 0);
284  if ((cp = strstr(cp, ":")) != nullptr)
285  {
286  ++cp;
287  opt_yincr = strtoul(strstr(cp, ":") + 1, nullptr, 0);
288  }
289  else
290  {
291  opt_yincr = (opt_ymax - opt_ymin) / 5;
292  }
293  break;
294  case 9:
295  opt_comment = optarg;
296  break;
297  case 10:
298  opt_cmd_iowait = 1;
299  break;
300  default:
301  printf("?? getopt returned character code 0%o ??\n", opt);
302  exit(1);
303  }
304  }
305 } /* parse_args */
306 
307 void perror_exit(const char *msg, ...)
308 {
309  char buf[1024];
310  va_list ap;
311  va_start(ap, msg);
312  vsnprintf(buf, sizeof(buf), msg, ap);
313  va_end(ap);
314  TRACE(0, "%s", buf);
315  perror(buf);
316  exit(1);
317 }
318 
319 //void atfork_trace(void) { TRACE( 3, "process %d forking", getpid() ); }
320 /* iofd is in/out
321  if iofd[x]==-1 then create a pipe for that index, x, and return the appropriate pipe fd in iofd[x]
322  else if iofd[x]!=x, dup2(iofd[x],x)
323  else inherit
324  Could add ==-2, then close???
325  */
326 pid_t fork_execv(int close_start, int close_cnt, int sleepB4exec_us, int iofd[3], const char *cmd, char *const argv[], char *const env[])
327 {
328  int pipes[3][2];
329  int lcl_iofd[3];
330  for (auto ii = 0; ii < 3; ++ii)
331  {
332  lcl_iofd[ii] = iofd[ii];
333  if (iofd[ii] == -1)
334  {
335  pipe(pipes[ii]); /* pipes[ii][0] refers to the read end */
336  iofd[ii] = ii == 0 ? pipes[ii][1] : pipes[ii][0];
337  }
338  }
339  pid_t pid = fork();
340  if (pid < 0)
341  {
342  perror_exit("fork");
343  }
344  else if (pid == 0)
345  { /* child */
346  if (lcl_iofd[0] == -1)
347  { // deal with child stdin
348  close(pipes[0][1]); // child closes write end of pipe which will be it's stdin
349  int fd = dup2(pipes[0][0], 0); // NOLINT
350  TRACE(3, "fork_execv dupped(%d) onto %d (should be 0)", pipes[0][0], fd);
351  close(pipes[0][0]);
352  }
353  if (sleepB4exec_us != 0)
354  {
355  // Do sleep before dealing with stdout/err incase we want TRACE to go to console
356  //int sts=pthread_atfork( atfork_trace, NULL, NULL );
357  usleep(sleepB4exec_us);
358  TRACE(1, "fork_execv sleep complete. sleepB4exec_us=%d sts=%d", sleepB4exec_us, 0 /*sts*/);
359  }
360  for (auto ii = 1; ii < 3; ++ii)
361  { // deal with child stdout/err
362  if (lcl_iofd[ii] == -1)
363  {
364  close(pipes[ii][0]);
365  int fd = dup2(pipes[ii][1], ii); // NOLINT
366  TRACE(3, "fork_execv dupped(%d) onto %d (should be %d)", pipes[ii][1], fd, ii);
367  close(pipes[ii][1]);
368  }
369  else if (lcl_iofd[ii] != ii)
370  {
371  int fd = dup2(lcl_iofd[ii], ii); // NOLINT
372  TRACE(3, "fork_execv dupped(%d) onto %d (should be %d)", pipes[ii][1], fd, ii);
373  }
374  }
375  for (auto ii = close_start; ii < (close_start + close_cnt); ++ii)
376  {
377  close(ii);
378  }
379  if (env != nullptr)
380  {
381  execve(cmd, argv, env);
382  }
383  else
384  {
385  execv(cmd, argv);
386  }
387  exit(1);
388  }
389  else
390  { // parent
391  for (auto ii = 0; ii < 3; ++ii)
392  {
393  if (lcl_iofd[ii] == -1)
394  {
395  close(ii == 0 ? pipes[ii][0] : pipes[ii][1]);
396  }
397  }
398  }
399 
400  TRACE(3, "fork_execv pid=%d", pid);
401  return pid;
402 } // fork_execv
403 
404 uint64_t swapPtr(void *X)
405 {
406  auto x = (uint64_t)X;
407  x = (x & 0x00000000ffffffff) << 32 | (x & 0xffffffff00000000) >> 32;
408  x = (x & 0x0000ffff0000ffff) << 16 | (x & 0xfff0000fffff0000) >> 16;
409  x = (x & 0x00ff00ff00ff00ff) << 8 | (x & 0xff00ff00ff00ff00) >> 8;
410  return x;
411 }
412 
413 /*
414  * Input to AWK can either be a file spec or a string.
415  * If input is string, the fork_execv call is told to create pipe for input.
416  *
417  * The run time duration of the AWK prooces can be determined via TRACE:
418 /home/ron/src
419 mu2edaq01 :^) tshow|egrep 'AWK b4 |AWK after read' |tdelta -d 1 -post /b4/ -stats | tail
420 1013 1489724640538688 2047 1116418481 13521 0 6 3 . AWK b4 fork_execv input=(nil)
421 1018 1489724640536624 1969 1111669678 13521 0 6 3 . AWK b4 fork_execv input=(nil)
422 1023 1489724640534717 1866 1107283893 13521 0 6 3 . AWK b4 fork_execv input=(nil)
423 1032 1489724640531756 2289 1100474359 13521 0 13 3 . AWK b4 fork_execv input=(nil)
424 cpu="0"
425  min 1821
426  max 49210
427  tot 293610
428  ave 2645.1351
429  cnt 111
430 --2017-03-17_08:13:23--
431  */
432 static int g_devnullfd = -1;
433 
434 // Run the awk script specified in awk_cmd on the file
435 std::string AWK(std::string const &awk_cmd, const char *file, const char *input)
436 {
437  char readbuf[1024];
438  ssize_t bytes = 0, tot_bytes = 0;
439  char *const argv_[4] = {(char *)"/bin/gawk",
440  (char *)awk_cmd.c_str(),
441  (char *)file,
442  nullptr};
443  pid_t pid;
444  ;
445  int infd = 0;
446  if (g_devnullfd == -1)
447  {
448  g_devnullfd = open("/dev/null", O_WRONLY);
449  }
450  if (input != nullptr)
451  {
452  infd = -1;
453  }
454  //int iofd[3]={infd,-1,g_devnullfd};
455  int iofd[3] = {infd, -1, 2}; // make stdin=infd, create pipr for stdout, inherit stderr
456  TRACE(3, "AWK b4 fork_execv input=%p", (void *)input);
457  char *env[1];
458  env[0] = nullptr; // mainly do not want big LD_LIBRARY_PATH
459  pid = fork_execv(0, 0 /*closeCnt*/, 0, iofd, "/bin/gawk", argv_, env); // NOLINT
460  if (input != nullptr /*||iofd[0]!=0*/)
461  {
462  int xx = strlen(input);
463  int sts = write(iofd[0], input, xx);
464  if (sts != xx)
465  {
466  perror("write AWK stdin");
467  }
468  close(iofd[0]);
469  while ((bytes = read(iofd[1], &readbuf[tot_bytes], sizeof(readbuf) - tot_bytes)) != 0)
470  {
471  TRACE(3, "AWK while bytes=read > 0 bytes=%zd readbuf=0x%016lx errno=%d", bytes, swapPtr(&readbuf[tot_bytes]), errno);
472  if (bytes == -1)
473  {
474  if (errno == EINTR)
475  {
476  continue;
477  }
478  break;
479  }
480  tot_bytes += bytes;
481  }
482  TRACE(3, "AWK after read tot=" + std::to_string((long long unsigned)tot_bytes) + " bytes=" + std::to_string((long long unsigned)bytes) + " input=" + std::string(input));
483  }
484  else
485  {
486  while ((bytes = read(iofd[1], &readbuf[tot_bytes], sizeof(readbuf) - tot_bytes)) > 0)
487  {
488  tot_bytes += bytes;
489  }
490  TRACE(3, "AWK after read tot=%zd bytes=%zd [0]=0x%x input=%p", tot_bytes, bytes, readbuf[0], (void *)input);
491  }
492  readbuf[tot_bytes >= 0 ? tot_bytes : 0] = '\0';
493  close(iofd[1]);
494  TRACE(3, "AWK after close child stdout. child pid=%d", pid);
495 #if 0
496  int status;
497  pid_t done_pid = waitpid(pid,&status,0);
498  TRACE( 3, "AWK after wait pid=%d done_pid=%d status=%d(0x%x)"
499  , pid, done_pid, status, status );
500 #endif
501  return std::string(readbuf);
502 } // AWK
503 
504 // separate string and _add_to_ vector
505 void string_addto_vector(std::string &instr, std::vector<std::string> &outvec, char delim)
506 {
507  std::stringstream ss(instr);
508  while (ss.good())
509  {
510  std::string substr;
511  std::getline(ss, substr, delim);
512  outvec.push_back(substr);
513  }
514 }
515 
516 uint64_t gettimeofday_us() //struct timespec *ts )
517 {
518  struct timeval tv;
519  gettimeofday(&tv, nullptr);
520  // if (ts) {
521  // ts->tv_sec = tv.tv_sec;
522  // ts->tv_nsec = tv.tv_usec * 1000;
523  // }
524  return (uint64_t)tv.tv_sec * 1000000 + tv.tv_usec;
525 } /* gettimeofday_us */
526 
527 #define DATA_START " DATA START"
528 #define GNUPLOT_PREFIX (const char *) \
529  "\
530 #!/usr/bin/env gnuplot\n\
531 # ./$0\n\
532 # OR\n\
533 # gnuplot -e 'ymin=400;ymax=1400' ./$0\n\
534 # OR try\n\
535 # gnuplot -e 'duration_s=35;set multiplot' ./gnuplot.gnuplot ./gnuplot.1.gnuplot -e 'set nomultiplot;pause -1'\n\
536 if(!exists('ARG0')) ARG0='' # for version 4, use: gnuplot -e ARG0=hello\n\
537 print 'ARG0=',ARG0 # ARG0.. automatically define in gnuplot version 5+\n\
538 if(!exists('ymin')) ymin=%d\n\
539 if(!exists('ymax')) ymax=%d\n\
540 if(!exists('yincr')) yincr=%d\n\
541 if(!exists('y2max')) y2max=%d\n\
542 if(!exists('y2incr')) y2incr=%d\n\
543 if(!exists('png')) png=1\n\
544 if(!exists('duration_s')) duration_s=0\n\
545 if(!exists('width')) width=512\n\
546 if(!exists('height')) height=384\n\
547 thisPid=system('echo `ps -p$$ -oppid=`')\n\
548 thisFile=system('ls -l /proc/'.thisPid.\"/fd | grep -v pipe: | tail -1 | sed -e 's/.*-> //'\")\n\
549 \n\
550 set title \"Disk Write Rate and %%CPU vs. time\\n%s %s %s%s\" # cmd and/or comment at end\n\
551 set xdata time\n\
552 tfmt='%%Y-%%m-%%dT%%H:%%M:%%S' # try to use consistent format\n\
553 set timefmt '%%Y-%%m-%%dT%%H:%%M:%%S'\n\
554 set xlabel 'time'\n\
555 set grid xtics back\n\
556 xstart=system(\"awk '/^....-..-..T/{print$1;exit}' \".thisFile)\n\
557 xend=system(\"awk 'END{print$1}' \".thisFile)\n\
558 print 'xstart='.xstart.' xend='.xend.' duration=',strptime(tfmt,xend)-strptime(tfmt,xstart)\n\
559 if(duration_s>0) end_t=strptime(tfmt,xstart)+duration_s; else end_t=strptime(tfmt,xend)\n\
560 set xrange [xstart:end_t]\n\
561 \n\
562 set ylabel '%s'\n\
563 set ytics nomirror\n\
564 if(ymax==0) set yrange [ymin:*];\\\n\
565 else set yrange [ymin:ymax];set ytics yincr\n\
566 set grid ytics back\n\
567 \n\
568 set y2label '%%CPU, %%MemTotal'\n\
569 set y2tics autofreq\n\
570 if(y2max==0) set y2range [0:*];\\\n\
571 else set y2range [0:y2max];set y2tics y2incr\n\
572 set pointsize .6\n\
573 \n\
574 if(png==1) set terminal png size width,height;\\\n\
575  pngfile=system( 'echo `basename '.thisFile.' .out`.png' );\\\n\
576  set output pngfile;\\\n\
577 else set terminal x11 size width,height\n\
578 \n\
579 plot \"< awk '/^#" DATA_START "/,/NEVER HAPPENS/' \".thisFile "
580 
581 void sigchld_sigaction(int signo, siginfo_t *info, void *context __attribute__((__unused__)))
582 {
583  /* see man sigaction for description of siginfo_t */
584  for (int pid : g_pid_vec)
585  {
586  if (pid == info->si_pid)
587  {
588  TRACE(2, "sigchld_sigaction signo=%d status=%d(0x%x) code=%d(0x%x) sending_pid=%d", signo, info->si_status, info->si_status, info->si_code, info->si_code, info->si_pid);
589  return;
590  }
591  }
592  TRACE(3, "sigchld_sigaction signo=%d status=%d(0x%x) code=%d(0x%x) sending_pid=%d", signo, info->si_status, info->si_status, info->si_code, info->si_code, info->si_pid);
593 }
594 
595 void read_proc_file(const char *file, char *buffer, int buffer_size)
596 {
597  TRACE(4, "read_proc_file b4 open proc file" + std::string(file));
598  int fd = open(file, O_RDONLY);
599  int offset = 0, sts = 0;
600  while (true)
601  {
602  sts = read(fd, &buffer[offset], buffer_size - offset);
603  if (sts <= 0)
604  {
605  sts = 0;
606  break;
607  }
608  offset += sts;
609  }
610  buffer[sts + offset] = '\0';
611  close(fd);
612  TRACE(4, "read_proc_file after close " + std::string(file) + " read=%d offset=%d", sts, offset);
613 }
614 
615 pid_t check_pid_vec()
616 {
617  for (size_t ii = 0; ii < g_pid_vec.size();)
618  {
619  pid_t pid = g_pid_vec[ii];
620  int status;
621  pid_t pp = waitpid(pid, &status, WNOHANG);
622  TRACE(3, "check_pid_vec %d=waitpid(pid=%d) errno=%d", pp, pid, errno);
623  if (pp > 0)
624  {
625  g_pid_vec.erase(g_pid_vec.begin() + ii);
626  }
627  else if (pp == -1)
628  {
629  if (errno == ECHILD && kill(pid, 0) == 0)
630  {
631  // there is a process, but not my child process
632  ++ii;
633  }
634  else
635  {
636  // some other error
637  g_pid_vec.erase(g_pid_vec.begin() + ii);
638  }
639  }
640  else
641  {
642  ++ii;
643  }
644  }
645  if (g_pid_vec.empty())
646  {
647  return -1;
648  }
649  {
650  return 0;
651  }
652 }
653 
654 void cleanup()
655 {
656  TRACE(1, "atexit cleanup g_pid_vec.size()=%zd\n", g_pid_vec.size());
657  for (int &pid : g_pid_vec)
658  {
659  kill(pid, SIGHUP);
660  }
661 }
662 #if (defined(__cplusplus) && (__cplusplus >= 201103L)) || (defined(__STDC_VERSION__) && (__STDC_VERSION__ >= 201112L))
663 #pragma GCC diagnostic push
664 #pragma GCC diagnostic ignored "-Wunused-parameter" /* b/c of TRACE_XTRA_UNUSED */
665 #endif
666 void sigint_sigaction(int signo, siginfo_t *info, void *context)
667 {
668  cleanup();
669  exit(1);
670 }
671 #if (defined(__cplusplus) && (__cplusplus >= 201103L)) || (defined(__STDC_VERSION__) && (__STDC_VERSION__ >= 201112L))
672 #pragma GCC diagnostic pop
673 #endif
674 
675 int main(int argc, char *argv[])
676 {
677  struct timeval tv;
678  int post_periods_completed = 0;
679  parse_args(argc, argv);
680  if ((argc - optind) != 0 || (opt_cmd.empty() && opt_pid.empty()))
681  { //(argc-optind) is the number of non-opt args supplied.
682  int ii;
683  printf("unexpected argument(s) %d!=0\n", argc - optind);
684  for (ii = 0; (optind + ii) < argc; ++ii)
685  {
686  printf("arg%d=%s\n", ii + 1, argv[optind + ii]);
687  }
688  printf(USAGE);
689  exit(0);
690  }
691 
692  std::vector<std::string> graphs;
693  string_addto_vector(opt_graph, graphs, ',');
694 
695  char motherboard[1024] = {0};
696  if (getuid() == 0)
697  {
698  FILE *fp = popen("dmidecode | grep -m2 'Product Name:' | tail -1", "r");
699  fread(motherboard, 1, sizeof(motherboard), fp);
700  pclose(fp);
701  }
702  TRACE(1, "main - motherboard=" + std::string(motherboard));
703 
704  /* Note, when doing "waitpid" the wait would sometimes take a "long"
705  time (10's to 100's milliseconds; rcu???) If signal is generated
706  (i.e SA_NOCLDWAIT w/ sigchld_sigaction (not SIG_IGN)), it would
707  sometimes effect the read or write calls for the following AWK forks.
708  So, use SIG_IGN+SA_NOCLDWAIT.
709  */
710  struct sigaction sigaction_s;
711 #ifndef DO_SIGCHLD
712 #define DO_SIGCHLD 1
713 #endif
714 #if DO_SIGCHLD
715  sigaction_s.sa_sigaction = sigchld_sigaction;
716  sigaction_s.sa_flags = SA_SIGINFO | SA_NOCLDWAIT;
717 #else
718  sigaction_s.sa_handler = SIG_IGN;
719  sigaction_s.sa_flags = SA_NOCLDWAIT;
720 #endif
721  sigemptyset(&sigaction_s.sa_mask);
722  sigaction(SIGCHLD, &sigaction_s, nullptr);
723 
724  sigaction_s.sa_sigaction = sigint_sigaction;
725  sigaction_s.sa_flags = SA_SIGINFO;
726  sigaction(SIGINT, &sigaction_s, nullptr);
727 
728  //may return 0 when not able to detect
729  //long long unsigned concurentThreadsSupported = std::thread::hardware_concurrency();
730  long long unsigned concurentThreadsSupported = sysconf(_SC_NPROCESSORS_ONLN);
731  //TRACE_CNTL( "reset" ); TRACE_CNTL( "modeM", 1L );
732  TRACE(0, "main concurentThreadsSupported=%u opt_stats=" + opt_stats, concurentThreadsSupported);
733 
734  char run_time[80];
735  gettimeofday(&tv, nullptr);
736  strftime(run_time, sizeof(run_time), "%FT%H%M%S", localtime(&tv.tv_sec));
737  TRACE(0, "main run_time=" + std::string(run_time));
738 
739  // get hostname
740  struct utsname ubuf;
741  uname(&ubuf);
742  char *dot;
743  if ((dot = strchr(ubuf.nodename, '.')) != nullptr)
744  {
745  *dot = '\0';
746  }
747  std::string hostname(ubuf.nodename);
748  TRACE(1, "release=" + std::string(ubuf.release) + " version=" + std::string(ubuf.version));
749 
750  // get system mem (KB)
751  std::string memKB = AWK("NR==1{print$2;exit}", "/proc/meminfo", nullptr);
752  memKB = memKB.substr(0, memKB.size() - 1); // remove trailing newline
753 
754  std::string dat_file_out(opt_outdir + "periodic_" + run_time + "_" + hostname + "_stats.out");
755 
756  double period = atof(opt_period);
757 
758  atexit(cleanup);
759  pid_t pp;
760  std::vector<std::string> pidfile;
761 
762  std::vector<std::string> stats;
763 
764  // For each cmd: create out file, fork process (with delay param),
765  // add to stats vec to get CPU info, add to graphs vec to plot cmd CPU
766  for (size_t ii = 0; ii < opt_cmd.size(); ++ii)
767  {
768  char cmd_file_out[1024];
769  snprintf(cmd_file_out, sizeof(cmd_file_out), "%speriodic_%s_%s_cmd%zd.out", opt_outdir.c_str(), run_time, hostname.c_str(), ii);
770  int fd = open(cmd_file_out, O_WRONLY | O_CREAT, 0666);
771  TRACE(0, "main fd=%d opt_cmd=" + opt_cmd[ii] + " cmd_file_out=" + std::string(cmd_file_out), fd);
772  int iofd[3] = {0, fd, fd}; // redirect stdout/err to the cmd-out-file
773  char *const argv_[4] = {(char *)"/bin/sh",
774  (char *)"-c",
775  (char *)opt_cmd[ii].c_str(),
776  nullptr};
777  g_pid_vec.push_back(fork_execv(0, 0, (int)(period * opt_pre * 1e6), iofd, "/bin/sh", argv_, nullptr));
778  close(fd); // the output file has been given to the subprocess
779  std::string pidstr = std::to_string((long long int)g_pid_vec[ii]);
780  pidfile.push_back("/proc/" + pidstr + "/stat");
781  //pidfile.push_back( "/proc/"+pidstr+"/task/"+pidstr+"/stat" );
782  char desc[128], ss[1024];
783  // field 14-17: Documentation/filesystems/proc.txt Table 1-4: utime stime cutime cstime
784  snprintf(ss, sizeof(ss), "CPUcmd%zd?%s?NR==1?$14+$15?1?yes", ii, pidfile[ii].c_str());
785  stats.emplace_back(ss);
786 
787  snprintf(desc, sizeof(desc), "CPU+cmd%zd", ii);
788  graphs.emplace_back(desc); // cmd0 is in the GNUPLOT_PREFIX
789  snprintf(ss, sizeof(ss), "%s?%s?NR==1?$14+$15+16+$17?1?yes", desc, pidfile[ii].c_str());
790  stats.emplace_back(ss);
791 
792  snprintf(desc, sizeof(desc), "WaitBlkIOcmd%zd", ii);
793  if (opt_cmd_iowait != 0)
794  {
795  graphs.emplace_back(desc);
796  }
797  snprintf(ss, sizeof(ss), "%s?%s?NR==1?$42?1?yes", desc, pidfile[ii].c_str());
798  stats.emplace_back(ss);
799 
800  snprintf(desc, sizeof(desc), "Faultcmd%zd", ii);
801  if (opt_fault != 0)
802  {
803  graphs.emplace_back(desc);
804  }
805  snprintf(ss, sizeof(ss), "%s?%s?NR==1?$10+$11+$12+$13?4096.0/1048576?yes", desc, pidfile[ii].c_str());
806  stats.emplace_back(ss);
807  }
808  for (size_t ii = 0; ii < opt_Cmd.size(); ++ii)
809  {
810  char cmd_file_out[1024];
811  snprintf(cmd_file_out, sizeof(cmd_file_out), "%speriodic_%s_%s_cmd%zd.out", opt_outdir.c_str(), run_time, hostname.c_str(), ii + opt_cmd.size());
812  int fd = open(cmd_file_out, O_WRONLY | O_CREAT, 0666);
813  TRACE(0, "main fd=%d opt_Cmd=" + opt_Cmd[ii] + " cmd_file_out=" + std::string(cmd_file_out), fd);
814  int iofd[3] = {0, fd, fd}; // redirect stdout/err to the cmd-out-file
815  char *const argv_[4] = {(char *)"/bin/sh",
816  (char *)"-c",
817  (char *)opt_Cmd[ii].c_str(),
818  nullptr};
819  g_pid_vec.push_back(fork_execv(0, 0, (int)(period * opt_pre * 1e6), iofd, "/bin/sh", argv_, nullptr));
820  close(fd); // the output file has been given to the subprocess
821  std::string pidstr = std::to_string((long long int)g_pid_vec[ii]);
822  pidfile.push_back("/proc/" + pidstr + "/stat");
823  //pidfile.push_back( "/proc/"+pidstr+"/task/"+pidstr+"/stat" );
824  char desc[128], ss[1024];
825  snprintf(desc, sizeof(desc), "CPU+cmd%zd", ii + opt_cmd.size());
826  snprintf(ss, sizeof(ss), "CPUcmd%zd?%s?NR==1?$14+$15?1?yes", ii + opt_cmd.size(), pidfile[ii].c_str());
827  stats.emplace_back(ss);
828  snprintf(ss, sizeof(ss), "CPU+cmd%zd?%s?NR==1?$14+$15+16+$17?1?yes", ii + opt_cmd.size(), pidfile[ii].c_str());
829  stats.emplace_back(ss);
830  // JUST DONT ADD THESE TO graphs
831  }
832  std::vector<std::string> pids;
833  if (!opt_pid.empty() != 0u)
834  {
835  string_addto_vector(opt_pid, pids, ',');
836  }
837  for (size_t ii = 0; ii < pids.size(); ++ii)
838  {
839  g_pid_vec.push_back(std::stoi(pids[ii]));
840  TRACE(1, "pid=%s g_pid_vec.size()=%ld", pids[ii].c_str(), g_pid_vec.size());
841  pidfile.push_back("/proc/" + pids[ii] + "/stat");
842  char desc[128], ss[1024];
843  // field 14-17: Documentation/filesystems/proc.txt Table 1-4: utime stime cutime cstime
844  snprintf(ss, sizeof(ss), "CPUpid%zd?%s?NR==1?$14+$15?1?yes", ii, pidfile[ii].c_str());
845  stats.emplace_back(ss);
846 
847  std::ifstream t("/proc/" + pids[ii] + "/comm");
848  std::string comm((std::istreambuf_iterator<char>(t)),
849  std::istreambuf_iterator<char>());
850  comm = comm.substr(0, comm.size() - 1); // strip nl
851 
852  snprintf(desc, sizeof(desc), "CPU+pid%zd_%s", ii, comm.c_str());
853  graphs.emplace_back(desc); // cmd0 is in the GNUPLOT_PREFIX
854  snprintf(ss, sizeof(ss), "%s?%s?NR==1?$14+$15+16+$17?1?yes", desc, pidfile[ii].c_str());
855  stats.emplace_back(ss);
856 
857  snprintf(desc, sizeof(desc), "WaitBlkIOpid%zd", ii);
858  if (opt_cmd_iowait != 0)
859  {
860  graphs.emplace_back(desc);
861  }
862  snprintf(ss, sizeof(ss), "%s?%s?NR==1?$42?1?yes", desc, pidfile[ii].c_str());
863  stats.emplace_back(ss);
864 
865  snprintf(desc, sizeof(desc), "Faultpid%zd", ii);
866  if (opt_fault != 0)
867  {
868  graphs.emplace_back(desc);
869  }
870  snprintf(ss, sizeof(ss), "%s?%s?NR==1?$10+$11+$12+$13?4096.0/1048576?yes", desc, pidfile[ii].c_str());
871  stats.emplace_back(ss);
872  }
873 
874  stats.emplace_back("CPUnode");
875  stats.emplace_back("IOWait");
876  if (opt_sys_iowait != 0) { graphs.emplace_back("IOWait"); }
877  stats.emplace_back("Cached");
878  stats.emplace_back("Dirty");
879  stats.emplace_back("Free");
880 
881  if (!opt_disk.empty() != 0u)
882  {
883  std::vector<std::string> tmp;
884  string_addto_vector(opt_disk, tmp, ',');
885  for (auto &dk : tmp)
886  {
887  // /proc/diskstat has 11 field after an initial 3 (14 total) for each device
888  // The 7th field after the device name (the 10th field total) is # of sectors written.
889  // Sectors appear to be 512 bytes. So, deviding by 2048 converts to MBs.
890  std::string statstr = dk + "_wrMB/s?/proc/diskstats?/" + dk + "/?$10?(1.0/2048)?yes";
891  stats.push_back(statstr);
892  std::vector<std::string> stat_spec;
893  string_addto_vector(statstr, stat_spec, '?');
894  graphs.push_back(stat_spec[s_desc]);
895 
896  statstr = dk + "_rdMB/s?/proc/diskstats?/" + dk + "/?$6?(1.0/2048)?yes";
897  stats.push_back(statstr);
898  stat_spec.clear();
899  string_addto_vector(statstr, stat_spec, '?');
900  //graphs.push_back( stat_spec[s_desc] ); // don't add read by default -- can be added with --graph
901  }
902  }
903 
904  if (!opt_stats.empty() != 0u)
905  {
906  std::vector<std::string> tmp_stats;
907  string_addto_vector(opt_stats, tmp_stats, ',');
908  for (auto &tmp_stat : tmp_stats)
909  {
910  stats.push_back(tmp_stat);
911  std::vector<std::string> stat_spec;
912  string_addto_vector(tmp_stat, stat_spec, '?');
913  graphs.push_back(stat_spec[s_desc]);
914  }
915  }
916 
917  std::vector<long> pre_vals;
918  std::vector<double> multipliers;
919  std::vector<std::vector<std::string>> spec2(stats.size());
920  std::vector<std::string> awkCmd;
921 
922  std::string header_str("#" DATA_START "\n#_______time_______");
923 
924  int outfd = open(dat_file_out.c_str(), O_WRONLY | O_CREAT, 0777);
925  //FILE *outfp=stdout;
926  FILE *outfp = fdopen(outfd, "w");
927 
928  std::string cmd_comment;
929  if (!opt_cmd.empty() != 0u)
930  {
931  cmd_comment += "\\ncmd: " + opt_cmd[0];
932  }
933  if (!opt_comment.empty() != 0u)
934  {
935  cmd_comment += "\\ncomment: " + opt_comment;
936  }
937  fprintf(outfp, GNUPLOT_PREFIX, opt_ymin, opt_ymax, opt_yincr, opt_y2max, opt_y2incr, run_time, hostname.c_str(), ubuf.release, cmd_comment.c_str(), "disk write MB/s");
938 
939  uint64_t t_start = gettimeofday_us();
940 
941  // build header string and get initial values for "rate" stats
942  bool first_graph_spec_added = false;
943  for (size_t ii = 0; ii < stats.size(); ++ii)
944  {
945  std::vector<std::string> stat_spec;
946  string_addto_vector(stats[ii], stat_spec, '?');
947  if (stat_spec[s_desc] == "CPUnode" && stat_spec.size() == 1)
948  {
949  // Ref. Documentation/filesystems/proc.txt: user+nice+system (skip idle) +iowait+irq+softirq+steal (skip guest)
950  stats[ii] += "?/proc/stat?/^cpu[^0-9]/?$2+$3+$4+$6+$7+$8+$9?1.0/" + std::to_string(concurentThreadsSupported) + "?yes";
951  }
952  else if (stat_spec[s_desc] == "IOWait" && stat_spec.size() == 1)
953  {
954  stats[ii] += "?/proc/stat?/^cpu[^0-9]/?$6?1.0/" + std::to_string(concurentThreadsSupported) + "?yes";
955  }
956  else if (stat_spec[s_desc] == "Cached" && stat_spec.size() == 1)
957  {
958  stats[ii] += "?/proc/meminfo?/^(Cached|Buffers):/?$2?1?no";
959  }
960  else if (stat_spec[s_desc] == "Dirty" && stat_spec.size() == 1)
961  {
962  stats[ii] += "?/proc/meminfo?/^Dirty:/?$2?1?no";
963  }
964  else if (stat_spec[s_desc] == "Free" && stat_spec.size() == 1)
965  {
966  stats[ii] += "?/proc/meminfo?/^MemFree:/?$2?1?no";
967  }
968 
969  header_str += " " + stat_spec[s_desc];
970 
971  string_addto_vector(stats[ii], spec2[ii], '?');
972  char awk_cmd[1024];
973  snprintf(awk_cmd, sizeof(awk_cmd), "%s{vv+=%s}END{print vv}"
974  //snprintf( awk_cmd, sizeof(awk_cmd), "%s{vv+=%s;print \"vv now\",vv > \"/dev/stderr\";}END{print vv}"
975  ,
976  spec2[ii][s_linespec].c_str(), spec2[ii][s_fieldspec].c_str());
977  awkCmd.emplace_back(awk_cmd);
978 
979  std::string stat = AWK(awkCmd.back(), spec2[ii][s_file].c_str(), nullptr);
980 
981  pre_vals.push_back(atol(stat.c_str()));
982  multipliers.push_back(atof(AWK("BEGIN{print " + spec2[ii][s_multiplier] + "}", "/dev/null", nullptr).c_str()));
983  //fprintf( stderr, " l=%s", spec2[ii][s_linespec].c_str() );
984  for (const auto &graph : graphs)
985  {
986  if (graph == stat_spec[s_desc])
987  {
988  if (first_graph_spec_added)
989  {
990  fprintf(outfp, ",\\\n '' ");
991  }
992  if (strncmp(stat_spec[s_desc].c_str(), "CPU", 3) == 0)
993  {
994  fprintf(outfp, "using 1:%zd title '%s' w linespoints axes x1y2", ii + 2, stat_spec[s_desc].c_str());
995  }
996  else if (stat_spec[s_desc] == "Cached" || stat_spec[s_desc] == "Dirty" || stat_spec[s_desc] == "Free")
997  {
998  fprintf(outfp, "using 1:($%zd/%s*100) title '%s%%' w linespoints axes x1y2", ii + 2, memKB.c_str(), stat_spec[s_desc].c_str());
999  }
1000  else if (stat_spec[s_desc].substr(0, 6) == "CPUcmd" || stat_spec[s_desc].substr(0, 6) == "CPU+cm")
1001  {
1002  fprintf(outfp, "using 1:%zd title '%s' w linespoints axes x1y2", ii + 2, stat_spec[s_desc].c_str());
1003  }
1004  else if (stat_spec[s_desc].substr(0, 12) == "WaitBlkIOcmd")
1005  {
1006  fprintf(outfp, "using 1:%zd title '%s' w linespoints axes x1y2", ii + 2, stat_spec[s_desc].c_str());
1007  }
1008  else
1009  {
1010  fprintf(outfp, "using 1:%zd title '%s' w linespoints axes x1y1", ii + 2, stat_spec[s_desc].c_str());
1011  }
1012  first_graph_spec_added = true;
1013  }
1014  }
1015  }
1016  header_str += " #\n";
1017 
1018  fprintf(outfp,
1019  "\nif(png==0) pause -1 'Press Enter/Return or ^C to finish'\n\
1020 exit\n");
1021 
1022  // print the cmds
1023  fprintf(outfp, "cmds:\n");
1024  for (const auto &ii : opt_cmd)
1025  {
1026  std::string ss = ii + "\n";
1027  fprintf(outfp, "%s", ss.c_str());
1028  }
1029 
1030  // print the specs
1031  fprintf(outfp, "stats:\n");
1032  for (const auto &stat : stats)
1033  {
1034  std::string ss = stat + "\n";
1035  fprintf(outfp, "%s", ss.c_str());
1036  }
1037 
1038  // now print header
1039  fprintf(outfp, "%s", header_str.c_str());
1040  fflush(outfp);
1041 
1042  std::string tmpdbg("main lp=%d done stat%zd=%ld rate=%f ");
1043  //char tmpdbgbuf[128];
1044  char proc_stats[8192];
1045  char *awk_in;
1046  int lp;
1047 
1048  // - - - - - - - - - - - - - - - - - - - - - - - -
1049  // wait a period and then start collecting the stats
1050 eintr1:
1051  int64_t t_sleep = (t_start + (uint64_t)(period * 1e6)) - gettimeofday_us();
1052  if (t_sleep > 0)
1053  {
1054  int sts = usleep(t_sleep); // NOLINT
1055  TRACE(3, "main usleep sts=%d errno=%d", sts, errno);
1056  if (errno == EINTR)
1057  {
1058  goto eintr1;
1059  }
1060  }
1061 
1062 #define MAX_LP 600
1063  for (lp = 2; lp < MAX_LP; ++lp)
1064  {
1065  char str[80];
1066  gettimeofday(&tv, nullptr);
1067  strftime(str, sizeof(str), "%FT%T", localtime(&tv.tv_sec));
1068  //fprintf(outfp, "%s.%ld", str, tv.tv_usec/100000 );
1069  fprintf(outfp, "%s", str);
1070  std::string prv_file;
1071  for (size_t ii = 0; ii < stats.size(); ++ii)
1072  {
1073  TRACE(3, "main lp=%d start stat%zd", lp, ii);
1074  char const *awk_file;
1075  if (ii < (2 * opt_cmd.size()))
1076  { // For each cmd, the
1077  // /proc/<pid>/stat file
1078  // will be referenced twice.
1079  if ((ii & 1) == 0)
1080  {
1081  read_proc_file(pidfile[ii / 2].c_str(), proc_stats, sizeof(proc_stats));
1082  }
1083  awk_in = proc_stats;
1084  awk_file = nullptr;
1085  }
1086  else if (spec2[ii][s_file] != prv_file)
1087  {
1088  prv_file = spec2[ii][s_file];
1089  read_proc_file(spec2[ii][s_file].c_str(), proc_stats, sizeof(proc_stats));
1090  awk_in = proc_stats;
1091  awk_file = nullptr;
1092  }
1093 
1094  std::string stat_str = AWK(awkCmd[ii], awk_file, awk_in); // NOLINT
1095 
1096  long stat = atol(stat_str.c_str());
1097 
1098  if (spec2[ii][s_rate] == "yes")
1099  {
1100  double rate;
1101  if (stat_str != "\n")
1102  {
1103  rate = (stat - pre_vals[ii]) * multipliers[ii] / period;
1104  }
1105  else
1106  {
1107  rate = 0.0;
1108  }
1109  TRACE(3, tmpdbg + "stat_str[0]=0x%x stat_str.size()=%zd", lp, ii, stat, rate, stat_str[0], stat_str.size());
1110  fprintf(outfp, " %.2f", rate);
1111  if (rate < 0.0 && spec2[ii][s_file] == "/proc/diskstats")
1112  {
1113  TRACE(0, "main stat:" + spec2[ii][s_desc] + " rate=%f pre_val=%ld stat=%ld stat_str=\"" + stat_str + "\" awkCmd=" + awkCmd[ii] + " proc_diskstats=" + proc_stats, rate, pre_vals[ii], stat);
1114  //TRACE_CNTL( "modeM", 0L );
1115  }
1116  pre_vals[ii] = stat;
1117  }
1118  else
1119  {
1120  TRACE(3, "main lp=%d done stat%zd=%ld", lp, ii, stat);
1121  fprintf(outfp, " %.2f", stat * multipliers[ii]);
1122  }
1123  }
1124  fprintf(outfp, "\n");
1125  fflush(outfp);
1126  eintr2:
1127  int64_t t_sleep = (t_start + (uint64_t)(period * lp * 1000000)) - gettimeofday_us();
1128  if (t_sleep > 0)
1129  {
1130  int sts = usleep(t_sleep); // NOLINT
1131  TRACE(3, "main usleep sts=%d errno=%d", sts, errno);
1132  if (errno == EINTR)
1133  {
1134  goto eintr2;
1135  }
1136  }
1137  pp = check_pid_vec();
1138  TRACE(2, "main pp=%d t_sleep=%ld", pp, t_sleep);
1139  if (pp == -1)
1140  {
1141  if (post_periods_completed == 0)
1142  {
1143  TRACE(1, "main processes complete - waiting %d post periods", opt_post);
1144  }
1145  if (post_periods_completed++ == opt_post)
1146  {
1147  break;
1148  }
1149  }
1150  }
1151  if (lp == MAX_LP)
1152  {
1153  fprintf(outfp, "# MAX_LP abort\n");
1154  }
1155 
1156  //TRACE( 0, "main waiting for pid=%d", pid );
1157  //wait(&status);
1158  //TRACE( 0, "main status=%d",status );
1159  TRACE(0, "main done/complete/returning");
1160  //TRACE_CNTL( "modeM", 0L );
1161  return (0);
1162 } // main