artdaq  v3_12_02
periodic_cmd_stats.cc
1 // This file (periodic_cmd_stats.cc) was created by Ron Rechenmacher <ron@fnal.gov> on
2 // Jan 5, 2017. "TERMS AND CONDITIONS" governing this file are in the README
3 // or COPYING file. If you do not have such a file, one can be obtained by
4 // contacting Ron or Fermi Lab in Batavia IL, 60510, phone: 630-840-3000.
5 // $RCSfile: periodic_cmd_stats.cc,v $
6 static const char *rev = "$Revision: 1.19 $$Date: 2018/06/28 21:14:28 $";
7 
8 // make periodic_cmd_stats
9 // OR make periodic_cmd_stats CXX="g++ -std=c++0x -DDO_TRACE"
10 
11 #define USAGE \
12  "\
13  usage: %s --cmd=<cmd>\n\
14 examples: %s --cmd='sleep 25' # this will take about a minute\n\
15  %s --cmd='taskset -c 4 dd if=/dev/zero of=/dev/sdf bs=50M count=500 oflag=direct' --disk=sdf\n\
16  %s --cmd='dd if=/dev/zero of=t.dat count=500' --stat='md*MB/s?/proc/diskstats?/md[0-3]/?$10?(1.0/2048)?yes'\n\
17  gnuplot -e png=0 `/bin/ls -t periodic_*_stats.out|head -1`\n\
18 For each cmd, record CPU info. Additionally, record total system CPU\n\
19 (can be >100 w/ multicore), file system Cached and Dirty %%, plus any\n\
20 other stats specified by --stats options\n\
21 options:\n\
22 --cmd= can have multiple\n\
23 --Cmd= run cmd, but dont graph CPU (can have multiple)\n\
24 --pid= graph comma or space sep. list of pids (getting cmd from /proc/)\n\
25 --disk= automatically build --stat for disk(s)\n\
26 --stat= desc?file?linespec?fieldspec?multiplier?rate\n\
27  builtin = CPUnode,Cached,Dirty\n\
28  cmd-builtin = CPUcmdN, CPU+cmdN\n\
29 --out-dir= output dir (for 2+ output files; stats and cmd+)\n\
30 \n\
31 --period= (float) default=%s\n\
32 --pre= the number of periods wait before exec of --cmd\n\
33 --post= the number of periods to loop after cmd exits\n\
34 --graph= add to graph any possibly included non-graphed metrics\n\
35 \n\
36 --no-defaults for no default stats (ie. cpu-parent,cpu-children)\n\
37 --init= default dropcache if root. NOT implemented yet\n\
38 --duration=\n\
39 --comment=\n\
40 --yrange=400:1400\n\
41 --y2max=\n\
42 --y2incr=\n\
43 --pause=0\n\
44 --sys-iowait,-w include the system iowait on the graph\n\
45 --cmd-iowait include cmd iowait on the graph\n\
46 --fault,-f\n\
47 ", \
48  basename(argv[0]), basename(argv[0]), basename(argv[0]), basename(argv[0]), opt_period
49 
50 enum
51 {
52  s_desc,
53  s_file,
54  s_linespec,
55  s_fieldspec,
56  s_multiplier,
57  s_rate
58 };
59 
60 #include <fcntl.h> // O_WRONLY|O_CREAT, open
61 #include <getopt.h> // getopt_long, {no,required,optional}_argument, extern char *optarg; extern int opt{ind,err,opt}
62 #include <sys/time.h> /* gettimeofday, timeval */
63 #include <sys/utsname.h> // uname
64 #include <sys/wait.h> // wait
65 #include <unistd.h> // getpid, sysconf
66 #include <csignal> /* sigaction, siginfo_t, sigset_t */
67 #include <cstdio> // printf
68 #include <fstream> // std::ifstream
69 #include <sstream> // std::stringstream
70 #include <string>
71 #include <thread>
72 #include <vector>
73 
74 #ifdef DO_TRACE
75 #define TRACE_NAME "periodic_cmd_stats"
76 #include "trace.h"
77 #else
78 #include <cstdarg> /* va_list */
79 #include <cstring> /* memcpy */
80 #include <string>
81 static void trace_ap(const char *msg, va_list ap)
82 {
83  char m_[1024];
84  unsigned len = strlen(msg);
85  len = len < (sizeof(m_) - 2) ? len : (sizeof(m_) - 2);
86  memcpy(m_, msg, len + 1);
87  if (m_[len - 1] != '\n')
88  {
89  memcpy(&(m_[len]), "\n", 2);
90  }
91  vprintf(m_, ap);
92  va_end(ap);
93 }
94 static void trace_p(const char *msg, ...) __attribute__((format(printf, 1, 2)));
95 static void trace_p(const char *msg, ...)
96 {
97  va_list ap;
98  va_start(ap, msg); // NOLINT
99  trace_ap(msg, ap); // NOLINT
100  va_end(ap); // NOLINT
101 }
102 static void trace_p(const std::string msg, ...)
103 {
104  va_list ap;
105  va_start(ap, msg);
106  trace_ap(msg.c_str(), ap);
107  va_end(ap); // NOLINT
108 }
109 enum
110 {
111  TLVL_ERROR,
112  TLVL_WARNING,
113  TLVL_INFO,
114  TLVL_DEBUG
115 };
116 #define TRACE(lvl, ...) \
117  do \
118  if (lvl <= TLVL_WARNING) trace_p(__VA_ARGS__); \
119  while (0)
120 #define TRACE_CNTL(xyzz, ...)
121 #endif
122 
123 /* GLOBALS */
124 int opt_v = 1;
125 char *opt_init = nullptr;
126 std::vector<std::string> opt_cmd;
127 std::vector<std::string> opt_Cmd;
128 std::string opt_pid;
129 std::string opt_disk;
130 std::string opt_stats;
131 std::string opt_outdir;
132 std::string opt_graph("CPUnode,Cached,Dirty,Free"); // CPU+ will always be graphed
133 const char *opt_period = "5.0";
134 std::string opt_comment;
135 int opt_pre = 6; // number of periods to sleepB4exec
136 int opt_post = 6;
137 int opt_ymin = 0;
138 int opt_ymax = 2000;
139 int opt_yincr = 200;
140 int opt_y2max = 200;
141 int opt_y2incr = 20;
142 int opt_sys_iowait = 0;
143 int opt_cmd_iowait = 0;
144 int opt_fault = 0;
145 
146 std::vector<pid_t> g_pid_vec;
147 
148 void charreplace(char *instr, char oldc, char newc)
149 {
150  while (*instr != 0)
151  {
152  if (*instr == oldc)
153  {
154  *instr = newc;
155  }
156  ++instr;
157  }
158 }
159 
160 void parse_args(int argc, char *argv[])
161 {
162  char *cp;
163  // parse opt, optargs, and args
164  while (true)
165  {
166  int opt;
167  static struct option long_options[] = {
168  // name has_arg *flag val
169  {"help", no_argument, nullptr, 'h'},
170  {"init", required_argument, nullptr, 'i'},
171  {"cmd", required_argument, nullptr, 'c'},
172  {"Cmd", required_argument, nullptr, 'C'},
173  {"disk", required_argument, nullptr, 'd'},
174  {"stat", required_argument, nullptr, 's'},
175  {"out-dir", required_argument, nullptr, 'o'},
176  {"period", required_argument, nullptr, 'p'},
177  {"sys-iowait", no_argument, nullptr, 'w'},
178  {"fault", no_argument, nullptr, 'f'},
179  {"pid", required_argument, nullptr, 'P'},
180  {"ymax", required_argument, nullptr, 1},
181  {"yincr", required_argument, nullptr, 2},
182  {"y2max", required_argument, nullptr, 3},
183  {"y2incr", required_argument, nullptr, 4},
184  {"pre", required_argument, nullptr, 5},
185  {"post", required_argument, nullptr, 6},
186  {"graph", required_argument, nullptr, 7},
187  {"yrange", required_argument, nullptr, 8},
188  {"comment", required_argument, nullptr, 9},
189  {"cmd-iowait", no_argument, nullptr, 10},
190  {nullptr, 0, nullptr, 0}};
191  opt = getopt_long(argc, argv, "?hvqVi:c:C:d:s:o:p:P:wf",
192  long_options, nullptr);
193  if (opt == -1)
194  {
195  break;
196  }
197  switch (opt)
198  {
199  case '?':
200  case 'h':
201  printf(USAGE);
202  exit(0);
203  break;
204  case 'V':
205  printf("%s\n", rev);
206  exit(0);
207  break;
208  case 'v':
209  ++opt_v;
210  break;
211  case 'q':
212  --opt_v;
213  break;
214  case 'i':
215  opt_init = optarg;
216  break;
217  case 'c':
218  opt_cmd.emplace_back(optarg);
219  break;
220  case 'C':
221  opt_Cmd.emplace_back(optarg);
222  break;
223  case 'd':
224  if (!opt_disk.empty())
225  {
226  opt_disk = opt_disk + "," + optarg;
227  }
228  else
229  {
230  opt_disk = optarg;
231  }
232  break;
233  case 's':
234  if (!opt_stats.empty())
235  {
236  opt_stats += std::string(",") + optarg;
237  }
238  else
239  {
240  opt_stats = optarg;
241  }
242  break;
243  case 'o':
244  opt_outdir = std::string(optarg) + "/";
245  break;
246  case 'p':
247  opt_period = optarg;
248  break;
249  case 'w':
250  opt_sys_iowait = 1;
251  break;
252  case 'f':
253  opt_fault = 1;
254  break;
255  case 'P':
256  charreplace(optarg, ' ', ',');
257  if (!opt_pid.empty() != 0u)
258  {
259  opt_pid = opt_pid + "," + optarg;
260  }
261  else
262  {
263  opt_pid = optarg;
264  }
265  break;
266  case 1:
267  opt_ymax = strtoul(optarg, nullptr, 0);
268  break;
269  case 2:
270  opt_yincr = strtoul(optarg, nullptr, 0);
271  break;
272  case 3:
273  opt_y2max = strtoul(optarg, nullptr, 0);
274  break;
275  case 4:
276  opt_y2incr = strtoul(optarg, nullptr, 0);
277  break;
278  case 5:
279  opt_pre = strtoul(optarg, nullptr, 0);
280  break;
281  case 6:
282  opt_post = strtoul(optarg, nullptr, 0);
283  break;
284  case 7:
285  opt_graph += std::string(",") + optarg;
286  break;
287  case 8:
288  opt_ymin = strtoul(optarg, nullptr, 0);
289  cp = strstr(optarg, ":") + 1;
290  opt_ymax = strtoul(cp, nullptr, 0);
291  if ((cp = strstr(cp, ":")) != nullptr)
292  {
293  ++cp;
294  opt_yincr = strtoul(strstr(cp, ":") + 1, nullptr, 0);
295  }
296  else
297  {
298  opt_yincr = (opt_ymax - opt_ymin) / 5;
299  }
300  break;
301  case 9:
302  opt_comment = optarg;
303  break;
304  case 10:
305  opt_cmd_iowait = 1;
306  break;
307  default:
308  printf("?? getopt returned character code 0%o ??\n", opt);
309  exit(1);
310  }
311  }
312 } /* parse_args */
313 
314 void perror_exit(const char *msg, ...)
315 {
316  char buf[1024];
317  va_list ap;
318  va_start(ap, msg);
319  vsnprintf(buf, sizeof(buf), msg, ap);
320  va_end(ap);
321  TRACE(TLVL_ERROR, "%s", buf);
322  perror(buf);
323  exit(1);
324 }
325 
326 // void atfork_trace(void) { TRACE( 3, "process %d forking", getpid() ); }
327 /* iofd is in/out
328  if iofd[x]==-1 then create a pipe for that index, x, and return the appropriate pipe fd in iofd[x]
329  else if iofd[x]!=x, dup2(iofd[x],x)
330  else inherit
331  Could add ==-2, then close???
332  */
333 pid_t fork_execv(int close_start, int close_cnt, int sleepB4exec_us, int iofd[3], const char *cmd, char *const argv[], char *const env[])
334 {
335  int pipes[3][2];
336  int lcl_iofd[3];
337  for (auto ii = 0; ii < 3; ++ii)
338  {
339  lcl_iofd[ii] = iofd[ii];
340  if (iofd[ii] == -1)
341  {
342  pipe(pipes[ii]); /* pipes[ii][0] refers to the read end */
343  iofd[ii] = ii == 0 ? pipes[ii][1] : pipes[ii][0];
344  }
345  }
346  pid_t pid = fork();
347  if (pid < 0)
348  {
349  perror_exit("fork");
350  }
351  else if (pid == 0)
352  { /* child */
353  if (lcl_iofd[0] == -1)
354  { // deal with child stdin
355  close(pipes[0][1]); // child closes write end of pipe which will be it's stdin
356  int fd = dup2(pipes[0][0], 0); // NOLINT
357  TRACE(TLVL_DEBUG + 0, "fork_execv dupped(%d) onto %d (should be 0)", pipes[0][0], fd);
358  close(pipes[0][0]);
359  }
360  if (sleepB4exec_us != 0)
361  {
362  // Do sleep before dealing with stdout/err incase we want TRACE to go to console
363  // int sts=pthread_atfork( atfork_trace, NULL, NULL );
364  usleep(sleepB4exec_us);
365  TRACE(TLVL_WARNING, "fork_execv sleep complete. sleepB4exec_us=%d sts=%d", sleepB4exec_us, 0 /*sts*/);
366  }
367  for (auto ii = 1; ii < 3; ++ii)
368  { // deal with child stdout/err
369  if (lcl_iofd[ii] == -1)
370  {
371  close(pipes[ii][0]);
372  int fd = dup2(pipes[ii][1], ii); // NOLINT
373  TRACE(TLVL_DEBUG + 0, "fork_execv dupped(%d) onto %d (should be %d)", pipes[ii][1], fd, ii);
374  close(pipes[ii][1]);
375  }
376  else if (lcl_iofd[ii] != ii)
377  {
378  int fd = dup2(lcl_iofd[ii], ii); // NOLINT
379  TRACE(TLVL_DEBUG + 0, "fork_execv dupped(%d) onto %d (should be %d)", pipes[ii][1], fd, ii);
380  }
381  }
382  for (auto ii = close_start; ii < (close_start + close_cnt); ++ii)
383  {
384  close(ii);
385  }
386  if (env != nullptr)
387  {
388  execve(cmd, argv, env);
389  }
390  else
391  {
392  execv(cmd, argv);
393  }
394  exit(1);
395  }
396  else
397  { // parent
398  for (auto ii = 0; ii < 3; ++ii)
399  {
400  if (lcl_iofd[ii] == -1)
401  {
402  close(ii == 0 ? pipes[ii][0] : pipes[ii][1]);
403  }
404  }
405  }
406 
407  TRACE(TLVL_DEBUG + 0, "fork_execv pid=%d", pid);
408  return pid;
409 } // fork_execv
410 
411 uint64_t swapPtr(void *X)
412 {
413  auto x = (uint64_t)X;
414  x = (x & 0x00000000ffffffff) << 32 | (x & 0xffffffff00000000) >> 32;
415  x = (x & 0x0000ffff0000ffff) << 16 | (x & 0xfff0000fffff0000) >> 16;
416  x = (x & 0x00ff00ff00ff00ff) << 8 | (x & 0xff00ff00ff00ff00) >> 8;
417  return x;
418 }
419 
420 /*
421  * Input to AWK can either be a file spec or a string.
422  * If input is string, the fork_execv call is told to create pipe for input.
423  *
424  * The run time duration of the AWK prooces can be determined via TRACE:
425 /home/ron/src
426 mu2edaq01 :^) tshow|egrep 'AWK b4 |AWK after read' |tdelta -d 1 -post /b4/ -stats | tail
427 1013 1489724640538688 2047 1116418481 13521 0 6 3 . AWK b4 fork_execv input=(nil)
428 1018 1489724640536624 1969 1111669678 13521 0 6 3 . AWK b4 fork_execv input=(nil)
429 1023 1489724640534717 1866 1107283893 13521 0 6 3 . AWK b4 fork_execv input=(nil)
430 1032 1489724640531756 2289 1100474359 13521 0 13 3 . AWK b4 fork_execv input=(nil)
431 cpu="0"
432  min 1821
433  max 49210
434  tot 293610
435  ave 2645.1351
436  cnt 111
437 --2017-03-17_08:13:23--
438  */
439 static int g_devnullfd = -1;
440 
441 // Run the awk script specified in awk_cmd on the file
442 std::string AWK(std::string const &awk_cmd, const char *file, const char *input)
443 {
444  char readbuf[1024];
445  ssize_t bytes = 0, tot_bytes = 0;
446  char *const argv_[4] = {(char *)"/bin/gawk",
447  (char *)awk_cmd.c_str(),
448  (char *)file,
449  nullptr};
450  pid_t pid;
451  ;
452  int infd = 0;
453  if (g_devnullfd == -1)
454  {
455  g_devnullfd = open("/dev/null", O_WRONLY);
456  }
457  if (input != nullptr)
458  {
459  infd = -1;
460  }
461  // int iofd[3]={infd,-1,g_devnullfd};
462  int iofd[3] = {infd, -1, 2}; // make stdin=infd, create pipr for stdout, inherit stderr
463  TRACE(TLVL_DEBUG + 0, "AWK b4 fork_execv input=%p", (void *)input);
464  char *env[1];
465  env[0] = nullptr; // mainly do not want big LD_LIBRARY_PATH
466  pid = fork_execv(0, 0 /*closeCnt*/, 0, iofd, "/bin/gawk", argv_, env); // NOLINT
467  if (input != nullptr /*||iofd[0]!=0*/)
468  {
469  int xx = strlen(input);
470  int sts = write(iofd[0], input, xx);
471  if (sts != xx)
472  {
473  perror("write AWK stdin");
474  }
475  close(iofd[0]);
476  while ((bytes = read(iofd[1], &readbuf[tot_bytes], sizeof(readbuf) - tot_bytes)) != 0)
477  {
478  TRACE(TLVL_DEBUG + 0, "AWK while bytes=read > 0 bytes=%zd readbuf=0x%016lx errno=%d", bytes, swapPtr(&readbuf[tot_bytes]), errno);
479  if (bytes == -1)
480  {
481  if (errno == EINTR)
482  {
483  continue;
484  }
485  break;
486  }
487  tot_bytes += bytes;
488  }
489  TRACE(TLVL_DEBUG + 0, "AWK after read tot=" + std::to_string((long long unsigned)tot_bytes) + " bytes=" + std::to_string((long long unsigned)bytes) + " input=" + std::string(input));
490  }
491  else
492  {
493  while ((bytes = read(iofd[1], &readbuf[tot_bytes], sizeof(readbuf) - tot_bytes)) > 0)
494  {
495  tot_bytes += bytes;
496  }
497  TRACE(TLVL_DEBUG + 0, "AWK after read tot=%zd bytes=%zd [0]=0x%x input=%p", tot_bytes, bytes, readbuf[0], (void *)input);
498  }
499  readbuf[tot_bytes >= 0 ? tot_bytes : 0] = '\0';
500  close(iofd[1]);
501  TRACE(TLVL_DEBUG + 0, "AWK after close child stdout. child pid=%d", pid);
502 #if 0
503  int status;
504  pid_t done_pid = waitpid(pid,&status,0);
505  TRACE( 3, "AWK after wait pid=%d done_pid=%d status=%d(0x%x)"
506  , pid, done_pid, status, status );
507 #endif
508  return std::string(readbuf);
509 } // AWK
510 
511 // separate string and _add_to_ vector
512 void string_addto_vector(std::string &instr, std::vector<std::string> &outvec, char delim)
513 {
514  std::stringstream ss(instr);
515  while (ss.good())
516  {
517  std::string substr;
518  std::getline(ss, substr, delim);
519  outvec.push_back(substr);
520  }
521 }
522 
523 uint64_t gettimeofday_us() // struct timespec *ts )
524 {
525  struct timeval tv;
526  gettimeofday(&tv, nullptr);
527  // if (ts) {
528  // ts->tv_sec = tv.tv_sec;
529  // ts->tv_nsec = tv.tv_usec * 1000;
530  // }
531  return (uint64_t)tv.tv_sec * 1000000 + tv.tv_usec;
532 } /* gettimeofday_us */
533 
534 #define DATA_START " DATA START"
535 #define GNUPLOT_PREFIX (const char *) \
536  "\
537 #!/usr/bin/env gnuplot\n\
538 # ./$0\n\
539 # OR\n\
540 # gnuplot -e 'ymin=400;ymax=1400' ./$0\n\
541 # OR try\n\
542 # gnuplot -e 'duration_s=35;set multiplot' ./gnuplot.gnuplot ./gnuplot.1.gnuplot -e 'set nomultiplot;pause -1'\n\
543 if(!exists('ARG0')) ARG0='' # for version 4, use: gnuplot -e ARG0=hello\n\
544 print 'ARG0=',ARG0 # ARG0.. automatically define in gnuplot version 5+\n\
545 if(!exists('ymin')) ymin=%d\n\
546 if(!exists('ymax')) ymax=%d\n\
547 if(!exists('yincr')) yincr=%d\n\
548 if(!exists('y2max')) y2max=%d\n\
549 if(!exists('y2incr')) y2incr=%d\n\
550 if(!exists('png')) png=1\n\
551 if(!exists('duration_s')) duration_s=0\n\
552 if(!exists('width')) width=512\n\
553 if(!exists('height')) height=384\n\
554 thisPid=system('echo `ps -p$$ -oppid=`')\n\
555 thisFile=system('ls -l /proc/'.thisPid.\"/fd | grep -v pipe: | tail -1 | sed -e 's/.*-> //'\")\n\
556 \n\
557 set title \"Disk Write Rate and %%CPU vs. time\\n%s %s %s%s\" # cmd and/or comment at end\n\
558 set xdata time\n\
559 tfmt='%%Y-%%m-%%dT%%H:%%M:%%S' # try to use consistent format\n\
560 set timefmt '%%Y-%%m-%%dT%%H:%%M:%%S'\n\
561 set xlabel 'time'\n\
562 set grid xtics back\n\
563 xstart=system(\"awk '/^....-..-..T/{print$1;exit}' \".thisFile)\n\
564 xend=system(\"awk 'END{print$1}' \".thisFile)\n\
565 print 'xstart='.xstart.' xend='.xend.' duration=',strptime(tfmt,xend)-strptime(tfmt,xstart)\n\
566 if(duration_s>0) end_t=strptime(tfmt,xstart)+duration_s; else end_t=strptime(tfmt,xend)\n\
567 set xrange [xstart:end_t]\n\
568 \n\
569 set ylabel '%s'\n\
570 set ytics nomirror\n\
571 if(ymax==0) set yrange [ymin:*];\\\n\
572 else set yrange [ymin:ymax];set ytics yincr\n\
573 set grid ytics back\n\
574 \n\
575 set y2label '%%CPU, %%MemTotal'\n\
576 set y2tics autofreq\n\
577 if(y2max==0) set y2range [0:*];\\\n\
578 else set y2range [0:y2max];set y2tics y2incr\n\
579 set pointsize .6\n\
580 \n\
581 if(png==1) set terminal png size width,height;\\\n\
582  pngfile=system( 'echo `basename '.thisFile.' .out`.png' );\\\n\
583  set output pngfile;\\\n\
584 else set terminal x11 size width,height\n\
585 \n\
586 plot \"< awk '/^#" DATA_START "/,/NEVER HAPPENS/' \".thisFile "
587 
588 void sigchld_sigaction(int signo, siginfo_t *info, void *context __attribute__((__unused__)))
589 {
590  /* see man sigaction for description of siginfo_t */
591  for (int pid : g_pid_vec)
592  {
593  if (pid == info->si_pid)
594  {
595  TRACE(TLVL_INFO, "sigchld_sigaction signo=%d status=%d(0x%x) code=%d(0x%x) sending_pid=%d", signo, info->si_status, info->si_status, info->si_code, info->si_code, info->si_pid);
596  return;
597  }
598  }
599  TRACE(TLVL_DEBUG + 0, "sigchld_sigaction signo=%d status=%d(0x%x) code=%d(0x%x) sending_pid=%d", signo, info->si_status, info->si_status, info->si_code, info->si_code, info->si_pid);
600 }
601 
602 void read_proc_file(const char *file, char *buffer, int buffer_size)
603 {
604  TRACE(TLVL_DEBUG + 1, "read_proc_file b4 open proc file" + std::string(file));
605  int fd = open(file, O_RDONLY);
606  int offset = 0, sts = 0;
607  while (true)
608  {
609  sts = read(fd, &buffer[offset], buffer_size - offset);
610  if (sts <= 0)
611  {
612  sts = 0;
613  break;
614  }
615  offset += sts;
616  }
617  buffer[sts + offset] = '\0';
618  close(fd);
619  TRACE(TLVL_DEBUG + 1, "read_proc_file after close " + std::string(file) + " read=%d offset=%d", sts, offset);
620 }
621 
622 pid_t check_pid_vec()
623 {
624  for (size_t ii = 0; ii < g_pid_vec.size();)
625  {
626  pid_t pid = g_pid_vec[ii];
627  int status;
628  pid_t pp = waitpid(pid, &status, WNOHANG);
629  TRACE(TLVL_DEBUG + 0, "check_pid_vec %d=waitpid(pid=%d) errno=%d", pp, pid, errno);
630  if (pp > 0)
631  {
632  g_pid_vec.erase(g_pid_vec.begin() + ii);
633  }
634  else if (pp == -1)
635  {
636  if (errno == ECHILD && kill(pid, 0) == 0)
637  {
638  // there is a process, but not my child process
639  ++ii;
640  }
641  else
642  {
643  // some other error
644  g_pid_vec.erase(g_pid_vec.begin() + ii);
645  }
646  }
647  else
648  {
649  ++ii;
650  }
651  }
652  if (g_pid_vec.empty())
653  {
654  return -1;
655  }
656  {
657  return 0;
658  }
659 }
660 
661 void cleanup()
662 {
663  TRACE(TLVL_WARNING, "atexit cleanup g_pid_vec.size()=%zd\n", g_pid_vec.size());
664  for (int &pid : g_pid_vec)
665  {
666  kill(pid, SIGHUP);
667  }
668 }
669 #if (defined(__cplusplus) && (__cplusplus >= 201103L)) || (defined(__STDC_VERSION__) && (__STDC_VERSION__ >= 201112L))
670 #pragma GCC diagnostic push
671 #pragma GCC diagnostic ignored "-Wunused-parameter" /* b/c of TRACE_XTRA_UNUSED */
672 #endif
673 void sigint_sigaction(int signo, siginfo_t *info, void *context)
674 {
675  cleanup();
676  exit(1);
677 }
678 #if (defined(__cplusplus) && (__cplusplus >= 201103L)) || (defined(__STDC_VERSION__) && (__STDC_VERSION__ >= 201112L))
679 #pragma GCC diagnostic pop
680 #endif
681 
682 int main(int argc, char *argv[])
683 {
684  struct timeval tv;
685  int post_periods_completed = 0;
686  parse_args(argc, argv);
687  if ((argc - optind) != 0 || (opt_cmd.empty() && opt_pid.empty()))
688  { //(argc-optind) is the number of non-opt args supplied.
689  int ii;
690  printf("unexpected argument(s) %d!=0\n", argc - optind);
691  for (ii = 0; (optind + ii) < argc; ++ii)
692  {
693  printf("arg%d=%s\n", ii + 1, argv[optind + ii]);
694  }
695  printf(USAGE);
696  exit(0);
697  }
698 
699  std::vector<std::string> graphs;
700  string_addto_vector(opt_graph, graphs, ',');
701 
702  char motherboard[1024] = {0};
703  if (getuid() == 0)
704  {
705  FILE *fp = popen("dmidecode | grep -m2 'Product Name:' | tail -1", "r");
706  fread(motherboard, 1, sizeof(motherboard), fp);
707  pclose(fp);
708  }
709  TRACE(TLVL_WARNING, "main - motherboard=" + std::string(motherboard));
710 
711  /* Note, when doing "waitpid" the wait would sometimes take a "long"
712  time (10's to 100's milliseconds; rcu???) If signal is generated
713  (i.e SA_NOCLDWAIT w/ sigchld_sigaction (not SIG_IGN)), it would
714  sometimes effect the read or write calls for the following AWK forks.
715  So, use SIG_IGN+SA_NOCLDWAIT.
716  */
717  struct sigaction sigaction_s;
718 #ifndef DO_SIGCHLD
719 #define DO_SIGCHLD 1
720 #endif
721 #if DO_SIGCHLD
722  sigaction_s.sa_sigaction = sigchld_sigaction;
723  sigaction_s.sa_flags = SA_SIGINFO | SA_NOCLDWAIT;
724 #else
725  sigaction_s.sa_handler = SIG_IGN;
726  sigaction_s.sa_flags = SA_NOCLDWAIT;
727 #endif
728  sigemptyset(&sigaction_s.sa_mask);
729  sigaction(SIGCHLD, &sigaction_s, nullptr);
730 
731  sigaction_s.sa_sigaction = sigint_sigaction;
732  sigaction_s.sa_flags = SA_SIGINFO;
733  sigaction(SIGINT, &sigaction_s, nullptr);
734 
735  // may return 0 when not able to detect
736  // long long unsigned concurentThreadsSupported = std::thread::hardware_concurrency();
737  long long unsigned concurentThreadsSupported = sysconf(_SC_NPROCESSORS_ONLN);
738  // TRACE_CNTL( "reset" ); TRACE_CNTL( "modeM", 1L );
739  TRACE(TLVL_ERROR, "main concurentThreadsSupported=%u opt_stats=" + opt_stats, concurentThreadsSupported);
740 
741  char run_time[80];
742  gettimeofday(&tv, nullptr);
743  strftime(run_time, sizeof(run_time), "%FT%H%M%S", localtime(&tv.tv_sec));
744  TRACE(TLVL_ERROR, "main run_time=" + std::string(run_time));
745 
746  // get hostname
747  struct utsname ubuf;
748  uname(&ubuf);
749  char *dot;
750  if ((dot = strchr(ubuf.nodename, '.')) != nullptr)
751  {
752  *dot = '\0';
753  }
754  std::string hostname(ubuf.nodename);
755  TRACE(TLVL_WARNING, "release=" + std::string(ubuf.release) + " version=" + std::string(ubuf.version));
756 
757  // get system mem (KB)
758  std::string memKB = AWK("NR==1{print$2;exit}", "/proc/meminfo", nullptr);
759  memKB = memKB.substr(0, memKB.size() - 1); // remove trailing newline
760 
761  std::string dat_file_out(opt_outdir + "periodic_" + run_time + "_" + hostname + "_stats.out");
762 
763  double period = atof(opt_period);
764 
765  atexit(cleanup);
766  pid_t pp;
767  std::vector<std::string> pidfile;
768 
769  std::vector<std::string> stats;
770 
771  // For each cmd: create out file, fork process (with delay param),
772  // add to stats vec to get CPU info, add to graphs vec to plot cmd CPU
773  for (size_t ii = 0; ii < opt_cmd.size(); ++ii)
774  {
775  char cmd_file_out[1024];
776  snprintf(cmd_file_out, sizeof(cmd_file_out), "%speriodic_%s_%s_cmd%zd.out", opt_outdir.c_str(), run_time, hostname.c_str(), ii);
777  int fd = open(cmd_file_out, O_WRONLY | O_CREAT, 0666);
778  TRACE(TLVL_ERROR, "main fd=%d opt_cmd=" + opt_cmd[ii] + " cmd_file_out=" + std::string(cmd_file_out), fd);
779  int iofd[3] = {0, fd, fd}; // redirect stdout/err to the cmd-out-file
780  char *const argv_[4] = {(char *)"/bin/sh",
781  (char *)"-c",
782  (char *)opt_cmd[ii].c_str(),
783  nullptr};
784  g_pid_vec.push_back(fork_execv(0, 0, (int)(period * opt_pre * 1e6), iofd, "/bin/sh", argv_, nullptr));
785  close(fd); // the output file has been given to the subprocess
786  std::string pidstr = std::to_string((long long int)g_pid_vec[ii]);
787  pidfile.push_back("/proc/" + pidstr + "/stat");
788  // pidfile.push_back( "/proc/"+pidstr+"/task/"+pidstr+"/stat" );
789  char desc[128], ss[1024];
790  // field 14-17: Documentation/filesystems/proc.txt Table 1-4: utime stime cutime cstime
791  snprintf(ss, sizeof(ss), "CPUcmd%zd?%s?NR==1?$14+$15?1?yes", ii, pidfile[ii].c_str());
792  stats.emplace_back(ss);
793 
794  snprintf(desc, sizeof(desc), "CPU+cmd%zd", ii);
795  graphs.emplace_back(desc); // cmd0 is in the GNUPLOT_PREFIX
796  snprintf(ss, sizeof(ss), "%s?%s?NR==1?$14+$15+16+$17?1?yes", desc, pidfile[ii].c_str());
797  stats.emplace_back(ss);
798 
799  snprintf(desc, sizeof(desc), "WaitBlkIOcmd%zd", ii);
800  if (opt_cmd_iowait != 0)
801  {
802  graphs.emplace_back(desc);
803  }
804  snprintf(ss, sizeof(ss), "%s?%s?NR==1?$42?1?yes", desc, pidfile[ii].c_str());
805  stats.emplace_back(ss);
806 
807  snprintf(desc, sizeof(desc), "Faultcmd%zd", ii);
808  if (opt_fault != 0)
809  {
810  graphs.emplace_back(desc);
811  }
812  snprintf(ss, sizeof(ss), "%s?%s?NR==1?$10+$11+$12+$13?4096.0/1048576?yes", desc, pidfile[ii].c_str());
813  stats.emplace_back(ss);
814  }
815  for (size_t ii = 0; ii < opt_Cmd.size(); ++ii)
816  {
817  char cmd_file_out[1024];
818  snprintf(cmd_file_out, sizeof(cmd_file_out), "%speriodic_%s_%s_cmd%zd.out", opt_outdir.c_str(), run_time, hostname.c_str(), ii + opt_cmd.size());
819  int fd = open(cmd_file_out, O_WRONLY | O_CREAT, 0666);
820  TRACE(TLVL_ERROR, "main fd=%d opt_Cmd=" + opt_Cmd[ii] + " cmd_file_out=" + std::string(cmd_file_out), fd);
821  int iofd[3] = {0, fd, fd}; // redirect stdout/err to the cmd-out-file
822  char *const argv_[4] = {(char *)"/bin/sh",
823  (char *)"-c",
824  (char *)opt_Cmd[ii].c_str(),
825  nullptr};
826  g_pid_vec.push_back(fork_execv(0, 0, (int)(period * opt_pre * 1e6), iofd, "/bin/sh", argv_, nullptr));
827  close(fd); // the output file has been given to the subprocess
828  std::string pidstr = std::to_string((long long int)g_pid_vec[ii]);
829  pidfile.push_back("/proc/" + pidstr + "/stat");
830  // pidfile.push_back( "/proc/"+pidstr+"/task/"+pidstr+"/stat" );
831  char desc[128], ss[1024];
832  snprintf(desc, sizeof(desc), "CPU+cmd%zd", ii + opt_cmd.size());
833  snprintf(ss, sizeof(ss), "CPUcmd%zd?%s?NR==1?$14+$15?1?yes", ii + opt_cmd.size(), pidfile[ii].c_str());
834  stats.emplace_back(ss);
835  snprintf(ss, sizeof(ss), "CPU+cmd%zd?%s?NR==1?$14+$15+16+$17?1?yes", ii + opt_cmd.size(), pidfile[ii].c_str());
836  stats.emplace_back(ss);
837  // JUST DONT ADD THESE TO graphs
838  }
839  std::vector<std::string> pids;
840  if (!opt_pid.empty() != 0u)
841  {
842  string_addto_vector(opt_pid, pids, ',');
843  }
844  for (size_t ii = 0; ii < pids.size(); ++ii)
845  {
846  g_pid_vec.push_back(std::stoi(pids[ii]));
847  TRACE(TLVL_WARNING, "pid=%s g_pid_vec.size()=%ld", pids[ii].c_str(), g_pid_vec.size());
848  pidfile.push_back("/proc/" + pids[ii] + "/stat");
849  char desc[128], ss[1024];
850  // field 14-17: Documentation/filesystems/proc.txt Table 1-4: utime stime cutime cstime
851  snprintf(ss, sizeof(ss), "CPUpid%zd?%s?NR==1?$14+$15?1?yes", ii, pidfile[ii].c_str());
852  stats.emplace_back(ss);
853 
854  std::ifstream t("/proc/" + pids[ii] + "/comm");
855  std::string comm((std::istreambuf_iterator<char>(t)),
856  std::istreambuf_iterator<char>());
857  comm = comm.substr(0, comm.size() - 1); // strip nl
858 
859  snprintf(desc, sizeof(desc), "CPU+pid%zd_%s", ii, comm.c_str());
860  graphs.emplace_back(desc); // cmd0 is in the GNUPLOT_PREFIX
861  snprintf(ss, sizeof(ss), "%s?%s?NR==1?$14+$15+16+$17?1?yes", desc, pidfile[ii].c_str());
862  stats.emplace_back(ss);
863 
864  snprintf(desc, sizeof(desc), "WaitBlkIOpid%zd", ii);
865  if (opt_cmd_iowait != 0)
866  {
867  graphs.emplace_back(desc);
868  }
869  snprintf(ss, sizeof(ss), "%s?%s?NR==1?$42?1?yes", desc, pidfile[ii].c_str());
870  stats.emplace_back(ss);
871 
872  snprintf(desc, sizeof(desc), "Faultpid%zd", ii);
873  if (opt_fault != 0)
874  {
875  graphs.emplace_back(desc);
876  }
877  snprintf(ss, sizeof(ss), "%s?%s?NR==1?$10+$11+$12+$13?4096.0/1048576?yes", desc, pidfile[ii].c_str());
878  stats.emplace_back(ss);
879  }
880 
881  stats.emplace_back("CPUnode");
882  stats.emplace_back("IOWait");
883  if (opt_sys_iowait != 0) { graphs.emplace_back("IOWait"); }
884  stats.emplace_back("Cached");
885  stats.emplace_back("Dirty");
886  stats.emplace_back("Free");
887 
888  if (!opt_disk.empty() != 0u)
889  {
890  std::vector<std::string> tmp;
891  string_addto_vector(opt_disk, tmp, ',');
892  for (auto &dk : tmp)
893  {
894  // /proc/diskstat has 11 field after an initial 3 (14 total) for each device
895  // The 7th field after the device name (the 10th field total) is # of sectors written.
896  // Sectors appear to be 512 bytes. So, deviding by 2048 converts to MBs.
897  std::string statstr = dk + "_wrMB/s?/proc/diskstats?/" + dk + "/?$10?(1.0/2048)?yes";
898  stats.push_back(statstr);
899  std::vector<std::string> stat_spec;
900  string_addto_vector(statstr, stat_spec, '?');
901  graphs.push_back(stat_spec[s_desc]);
902 
903  statstr = dk + "_rdMB/s?/proc/diskstats?/" + dk + "/?$6?(1.0/2048)?yes";
904  stats.push_back(statstr);
905  stat_spec.clear();
906  string_addto_vector(statstr, stat_spec, '?');
907  // graphs.push_back( stat_spec[s_desc] ); // don't add read by default -- can be added with --graph
908  }
909  }
910 
911  if (!opt_stats.empty() != 0u)
912  {
913  std::vector<std::string> tmp_stats;
914  string_addto_vector(opt_stats, tmp_stats, ',');
915  for (auto &tmp_stat : tmp_stats)
916  {
917  stats.push_back(tmp_stat);
918  std::vector<std::string> stat_spec;
919  string_addto_vector(tmp_stat, stat_spec, '?');
920  graphs.push_back(stat_spec[s_desc]);
921  }
922  }
923 
924  std::vector<long> pre_vals;
925  std::vector<double> multipliers;
926  std::vector<std::vector<std::string>> spec2(stats.size());
927  std::vector<std::string> awkCmd;
928 
929  std::string header_str("#" DATA_START "\n#_______time_______");
930 
931  int outfd = open(dat_file_out.c_str(), O_WRONLY | O_CREAT, 0777);
932  // FILE *outfp=stdout;
933  FILE *outfp = fdopen(outfd, "w");
934 
935  std::string cmd_comment;
936  if (!opt_cmd.empty() != 0u)
937  {
938  cmd_comment += "\\ncmd: " + opt_cmd[0];
939  }
940  if (!opt_comment.empty() != 0u)
941  {
942  cmd_comment += "\\ncomment: " + opt_comment;
943  }
944  fprintf(outfp, GNUPLOT_PREFIX, opt_ymin, opt_ymax, opt_yincr, opt_y2max, opt_y2incr, run_time, hostname.c_str(), ubuf.release, cmd_comment.c_str(), "disk write MB/s");
945 
946  uint64_t t_start = gettimeofday_us();
947 
948  // build header string and get initial values for "rate" stats
949  bool first_graph_spec_added = false;
950  for (size_t ii = 0; ii < stats.size(); ++ii)
951  {
952  std::vector<std::string> stat_spec;
953  string_addto_vector(stats[ii], stat_spec, '?');
954  if (stat_spec[s_desc] == "CPUnode" && stat_spec.size() == 1)
955  {
956  // Ref. Documentation/filesystems/proc.txt: user+nice+system (skip idle) +iowait+irq+softirq+steal (skip guest)
957  stats[ii] += "?/proc/stat?/^cpu[^0-9]/?$2+$3+$4+$6+$7+$8+$9?1.0/" + std::to_string(concurentThreadsSupported) + "?yes";
958  }
959  else if (stat_spec[s_desc] == "IOWait" && stat_spec.size() == 1)
960  {
961  stats[ii] += "?/proc/stat?/^cpu[^0-9]/?$6?1.0/" + std::to_string(concurentThreadsSupported) + "?yes";
962  }
963  else if (stat_spec[s_desc] == "Cached" && stat_spec.size() == 1)
964  {
965  stats[ii] += "?/proc/meminfo?/^(Cached|Buffers):/?$2?1?no";
966  }
967  else if (stat_spec[s_desc] == "Dirty" && stat_spec.size() == 1)
968  {
969  stats[ii] += "?/proc/meminfo?/^Dirty:/?$2?1?no";
970  }
971  else if (stat_spec[s_desc] == "Free" && stat_spec.size() == 1)
972  {
973  stats[ii] += "?/proc/meminfo?/^MemFree:/?$2?1?no";
974  }
975 
976  header_str += " " + stat_spec[s_desc];
977 
978  string_addto_vector(stats[ii], spec2[ii], '?');
979  char awk_cmd[1024];
980  snprintf(awk_cmd, sizeof(awk_cmd), "%s{vv+=%s}END{print vv}"
981  // snprintf( awk_cmd, sizeof(awk_cmd), "%s{vv+=%s;print \"vv now\",vv > \"/dev/stderr\";}END{print vv}"
982  ,
983  spec2[ii][s_linespec].c_str(), spec2[ii][s_fieldspec].c_str());
984  awkCmd.emplace_back(awk_cmd);
985 
986  std::string stat = AWK(awkCmd.back(), spec2[ii][s_file].c_str(), nullptr);
987 
988  pre_vals.push_back(atol(stat.c_str()));
989  multipliers.push_back(atof(AWK("BEGIN{print " + spec2[ii][s_multiplier] + "}", "/dev/null", nullptr).c_str()));
990  // fprintf( stderr, " l=%s", spec2[ii][s_linespec].c_str() );
991  for (const auto &graph : graphs)
992  {
993  if (graph == stat_spec[s_desc])
994  {
995  if (first_graph_spec_added)
996  {
997  fprintf(outfp, ",\\\n '' ");
998  }
999  if (strncmp(stat_spec[s_desc].c_str(), "CPU", 3) == 0)
1000  {
1001  fprintf(outfp, "using 1:%zd title '%s' w linespoints axes x1y2", ii + 2, stat_spec[s_desc].c_str());
1002  }
1003  else if (stat_spec[s_desc] == "Cached" || stat_spec[s_desc] == "Dirty" || stat_spec[s_desc] == "Free")
1004  {
1005  fprintf(outfp, "using 1:($%zd/%s*100) title '%s%%' w linespoints axes x1y2", ii + 2, memKB.c_str(), stat_spec[s_desc].c_str());
1006  }
1007  else if (stat_spec[s_desc].substr(0, 6) == "CPUcmd" || stat_spec[s_desc].substr(0, 6) == "CPU+cm")
1008  {
1009  fprintf(outfp, "using 1:%zd title '%s' w linespoints axes x1y2", ii + 2, stat_spec[s_desc].c_str());
1010  }
1011  else if (stat_spec[s_desc].substr(0, 12) == "WaitBlkIOcmd")
1012  {
1013  fprintf(outfp, "using 1:%zd title '%s' w linespoints axes x1y2", ii + 2, stat_spec[s_desc].c_str());
1014  }
1015  else
1016  {
1017  fprintf(outfp, "using 1:%zd title '%s' w linespoints axes x1y1", ii + 2, stat_spec[s_desc].c_str());
1018  }
1019  first_graph_spec_added = true;
1020  }
1021  }
1022  }
1023  header_str += " #\n";
1024 
1025  fprintf(outfp,
1026  "\nif(png==0) pause -1 'Press Enter/Return or ^C to finish'\n\
1027 exit\n");
1028 
1029  // print the cmds
1030  fprintf(outfp, "cmds:\n");
1031  for (const auto &ii : opt_cmd)
1032  {
1033  std::string ss = ii + "\n";
1034  fprintf(outfp, "%s", ss.c_str());
1035  }
1036 
1037  // print the specs
1038  fprintf(outfp, "stats:\n");
1039  for (const auto &stat : stats)
1040  {
1041  std::string ss = stat + "\n";
1042  fprintf(outfp, "%s", ss.c_str());
1043  }
1044 
1045  // now print header
1046  fprintf(outfp, "%s", header_str.c_str());
1047  fflush(outfp);
1048 
1049  std::string tmpdbg("main lp=%d done stat%zd=%ld rate=%f ");
1050  // char tmpdbgbuf[128];
1051  char proc_stats[8192];
1052  char *awk_in;
1053  int lp;
1054 
1055  // - - - - - - - - - - - - - - - - - - - - - - - -
1056  // wait a period and then start collecting the stats
1057 eintr1:
1058  int64_t t_sleep = (t_start + (uint64_t)(period * 1e6)) - gettimeofday_us();
1059  if (t_sleep > 0)
1060  {
1061  int sts = usleep(t_sleep); // NOLINT
1062  TRACE(TLVL_DEBUG + 0, "main usleep sts=%d errno=%d", sts, errno);
1063  if (errno == EINTR)
1064  {
1065  goto eintr1;
1066  }
1067  }
1068 
1069 #define MAX_LP 600
1070  for (lp = 2; lp < MAX_LP; ++lp)
1071  {
1072  char str[80];
1073  gettimeofday(&tv, nullptr);
1074  strftime(str, sizeof(str), "%FT%T", localtime(&tv.tv_sec));
1075  // fprintf(outfp, "%s.%ld", str, tv.tv_usec/100000 );
1076  fprintf(outfp, "%s", str);
1077  std::string prv_file;
1078  for (size_t ii = 0; ii < stats.size(); ++ii)
1079  {
1080  TRACE(TLVL_DEBUG + 0, "main lp=%d start stat%zd", lp, ii);
1081  char const *awk_file;
1082  if (ii < (2 * opt_cmd.size()))
1083  { // For each cmd, the
1084  // /proc/<pid>/stat file
1085  // will be referenced twice.
1086  if ((ii & 1) == 0)
1087  {
1088  read_proc_file(pidfile[ii / 2].c_str(), proc_stats, sizeof(proc_stats));
1089  }
1090  awk_in = proc_stats;
1091  awk_file = nullptr;
1092  }
1093  else if (spec2[ii][s_file] != prv_file)
1094  {
1095  prv_file = spec2[ii][s_file];
1096  read_proc_file(spec2[ii][s_file].c_str(), proc_stats, sizeof(proc_stats));
1097  awk_in = proc_stats;
1098  awk_file = nullptr;
1099  }
1100 
1101  std::string stat_str = AWK(awkCmd[ii], awk_file, awk_in); // NOLINT
1102 
1103  long stat = atol(stat_str.c_str());
1104 
1105  if (spec2[ii][s_rate] == "yes")
1106  {
1107  double rate;
1108  if (stat_str != "\n")
1109  {
1110  rate = (stat - pre_vals[ii]) * multipliers[ii] / period;
1111  }
1112  else
1113  {
1114  rate = 0.0;
1115  }
1116  TRACE(TLVL_DEBUG + 0, tmpdbg + "stat_str[0]=0x%x stat_str.size()=%zd", lp, ii, stat, rate, stat_str[0], stat_str.size());
1117  fprintf(outfp, " %.2f", rate);
1118  if (rate < 0.0 && spec2[ii][s_file] == "/proc/diskstats")
1119  {
1120  TRACE(TLVL_ERROR, "main stat:" + spec2[ii][s_desc] + " rate=%f pre_val=%ld stat=%ld stat_str=\"" + stat_str + "\" awkCmd=" + awkCmd[ii] + " proc_diskstats=" + proc_stats, rate, pre_vals[ii], stat);
1121  // TRACE_CNTL( "modeM", 0L );
1122  }
1123  pre_vals[ii] = stat;
1124  }
1125  else
1126  {
1127  TRACE(TLVL_DEBUG + 0, "main lp=%d done stat%zd=%ld", lp, ii, stat);
1128  fprintf(outfp, " %.2f", stat * multipliers[ii]);
1129  }
1130  }
1131  fprintf(outfp, "\n");
1132  fflush(outfp);
1133  eintr2:
1134  int64_t t_sleep = (t_start + (uint64_t)(period * lp * 1000000)) - gettimeofday_us();
1135  if (t_sleep > 0)
1136  {
1137  int sts = usleep(t_sleep); // NOLINT
1138  TRACE(TLVL_DEBUG + 0, "main usleep sts=%d errno=%d", sts, errno);
1139  if (errno == EINTR)
1140  {
1141  goto eintr2;
1142  }
1143  }
1144  pp = check_pid_vec();
1145  TRACE(TLVL_INFO, "main pp=%d t_sleep=%ld", pp, t_sleep);
1146  if (pp == -1)
1147  {
1148  if (post_periods_completed == 0)
1149  {
1150  TRACE(TLVL_WARNING, "main processes complete - waiting %d post periods", opt_post);
1151  }
1152  if (post_periods_completed++ == opt_post)
1153  {
1154  break;
1155  }
1156  }
1157  }
1158  if (lp == MAX_LP)
1159  {
1160  fprintf(outfp, "# MAX_LP abort\n");
1161  }
1162 
1163  // TRACE( 0, "main waiting for pid=%d", pid );
1164  // wait(&status);
1165  // TRACE( 0, "main status=%d",status );
1166  TRACE(TLVL_ERROR, "main done/complete/returning");
1167  // TRACE_CNTL( "modeM", 0L );
1168  return (0);
1169 } // main