6 static const char *rev =
"$Revision: 1.19 $$Date: 2018/06/28 21:14:28 $";
13 usage: %s --cmd=<cmd>\n\
14 examples: %s --cmd='sleep 25' # this will take about a minute\n\
15 %s --cmd='taskset -c 4 dd if=/dev/zero of=/dev/sdf bs=50M count=500 oflag=direct' --disk=sdf\n\
16 %s --cmd='dd if=/dev/zero of=t.dat count=500' --stat='md*MB/s?/proc/diskstats?/md[0-3]/?$10?(1.0/2048)?yes'\n\
17 gnuplot -e png=0 `/bin/ls -t periodic_*_stats.out|head -1`\n\
18 For each cmd, record CPU info. Additionally, record total system CPU\n\
19 (can be >100 w/ multicore), file system Cached and Dirty %%, plus any\n\
20 other stats specified by --stats options\n\
22 --cmd= can have multiple\n\
23 --Cmd= run cmd, but dont graph CPU (can have multiple)\n\
24 --pid= graph comma or space sep. list of pids (getting cmd from /proc/)\n\
25 --disk= automatically build --stat for disk(s)\n\
26 --stat= desc?file?linespec?fieldspec?multiplier?rate\n\
27 builtin = CPUnode,Cached,Dirty\n\
28 cmd-builtin = CPUcmdN, CPU+cmdN\n\
29 --out-dir= output dir (for 2+ output files; stats and cmd+)\n\
31 --period= (float) default=%s\n\
32 --pre= the number of periods wait before exec of --cmd\n\
33 --post= the number of periods to loop after cmd exits\n\
34 --graph= add to graph any possibly included non-graphed metrics\n\
36 --no-defaults for no default stats (ie. cpu-parent,cpu-children)\n\
37 --init= default dropcache if root. NOT implemented yet\n\
44 --sys-iowait,-w include the system iowait on the graph\n\
45 --cmd-iowait include cmd iowait on the graph\n\
48 basename(argv[0]), basename(argv[0]), basename(argv[0]), basename(argv[0]), opt_period
63 #include <sys/utsname.h>
75 #define TRACE_NAME "periodic_cmd_stats"
81 static void trace_ap(
const char *msg, va_list ap)
84 unsigned len = strlen(msg);
85 len = len < (
sizeof(m_) - 2) ? len : (
sizeof(m_) - 2);
86 memcpy(m_, msg, len + 1);
87 if (m_[len - 1] !=
'\n')
89 memcpy(&(m_[len]),
"\n", 2);
94 static void trace_p(
const char *msg, ...) __attribute__((format(printf, 1, 2)));
95 static
void trace_p(const
char *msg, ...)
102 static void trace_p(
const std::string msg, ...)
106 trace_ap(msg.c_str(), ap);
116 #define TRACE(lvl, ...) \
118 if (lvl <= TLVL_WARNING) trace_p(__VA_ARGS__); \
120 #define TRACE_CNTL(xyzz, ...)
125 char *opt_init =
nullptr;
126 std::vector<std::string> opt_cmd;
127 std::vector<std::string> opt_Cmd;
129 std::string opt_disk;
130 std::string opt_stats;
131 std::string opt_outdir;
132 std::string opt_graph(
"CPUnode,Cached,Dirty,Free");
133 const char *opt_period =
"5.0";
134 std::string opt_comment;
142 int opt_sys_iowait = 0;
143 int opt_cmd_iowait = 0;
146 std::vector<pid_t> g_pid_vec;
148 void charreplace(
char *instr,
char oldc,
char newc)
160 void parse_args(
int argc,
char *argv[])
167 static struct option long_options[] = {
169 {
"help", no_argument,
nullptr,
'h'},
170 {
"init", required_argument,
nullptr,
'i'},
171 {
"cmd", required_argument,
nullptr,
'c'},
172 {
"Cmd", required_argument,
nullptr,
'C'},
173 {
"disk", required_argument,
nullptr,
'd'},
174 {
"stat", required_argument,
nullptr,
's'},
175 {
"out-dir", required_argument,
nullptr,
'o'},
176 {
"period", required_argument,
nullptr,
'p'},
177 {
"sys-iowait", no_argument,
nullptr,
'w'},
178 {
"fault", no_argument,
nullptr,
'f'},
179 {
"pid", required_argument,
nullptr,
'P'},
180 {
"ymax", required_argument,
nullptr, 1},
181 {
"yincr", required_argument,
nullptr, 2},
182 {
"y2max", required_argument,
nullptr, 3},
183 {
"y2incr", required_argument,
nullptr, 4},
184 {
"pre", required_argument,
nullptr, 5},
185 {
"post", required_argument,
nullptr, 6},
186 {
"graph", required_argument,
nullptr, 7},
187 {
"yrange", required_argument,
nullptr, 8},
188 {
"comment", required_argument,
nullptr, 9},
189 {
"cmd-iowait", no_argument,
nullptr, 10},
190 {
nullptr, 0,
nullptr, 0}};
191 opt = getopt_long(argc, argv,
"?hvqVi:c:C:d:s:o:p:P:wf",
192 long_options,
nullptr);
218 opt_cmd.emplace_back(optarg);
221 opt_Cmd.emplace_back(optarg);
224 if (!opt_disk.empty())
226 opt_disk = opt_disk +
"," + optarg;
234 if (!opt_stats.empty())
236 opt_stats += std::string(
",") + optarg;
244 opt_outdir = std::string(optarg) +
"/";
256 charreplace(optarg,
' ',
',');
257 if (!opt_pid.empty() != 0u)
259 opt_pid = opt_pid +
"," + optarg;
267 opt_ymax = strtoul(optarg,
nullptr, 0);
270 opt_yincr = strtoul(optarg,
nullptr, 0);
273 opt_y2max = strtoul(optarg,
nullptr, 0);
276 opt_y2incr = strtoul(optarg,
nullptr, 0);
279 opt_pre = strtoul(optarg,
nullptr, 0);
282 opt_post = strtoul(optarg,
nullptr, 0);
285 opt_graph += std::string(
",") + optarg;
288 opt_ymin = strtoul(optarg,
nullptr, 0);
289 cp = strstr(optarg,
":") + 1;
290 opt_ymax = strtoul(cp,
nullptr, 0);
291 if ((cp = strstr(cp,
":")) !=
nullptr)
294 opt_yincr = strtoul(strstr(cp,
":") + 1,
nullptr, 0);
298 opt_yincr = (opt_ymax - opt_ymin) / 5;
302 opt_comment = optarg;
308 printf(
"?? getopt returned character code 0%o ??\n", opt);
314 void perror_exit(
const char *msg, ...)
319 vsnprintf(buf,
sizeof(buf), msg, ap);
321 TRACE(TLVL_ERROR,
"%s", buf);
333 pid_t fork_execv(
int close_start,
int close_cnt,
int sleepB4exec_us,
int iofd[3],
const char *cmd,
char *
const argv[],
char *
const env[])
337 for (
auto ii = 0; ii < 3; ++ii)
339 lcl_iofd[ii] = iofd[ii];
343 iofd[ii] = ii == 0 ? pipes[ii][1] : pipes[ii][0];
353 if (lcl_iofd[0] == -1)
356 int fd = dup2(pipes[0][0], 0);
357 TRACE(TLVL_DEBUG + 0,
"fork_execv dupped(%d) onto %d (should be 0)", pipes[0][0], fd);
360 if (sleepB4exec_us != 0)
364 usleep(sleepB4exec_us);
365 TRACE(TLVL_WARNING,
"fork_execv sleep complete. sleepB4exec_us=%d sts=%d", sleepB4exec_us, 0 );
367 for (
auto ii = 1; ii < 3; ++ii)
369 if (lcl_iofd[ii] == -1)
372 int fd = dup2(pipes[ii][1], ii);
373 TRACE(TLVL_DEBUG + 0,
"fork_execv dupped(%d) onto %d (should be %d)", pipes[ii][1], fd, ii);
376 else if (lcl_iofd[ii] != ii)
378 int fd = dup2(lcl_iofd[ii], ii);
379 TRACE(TLVL_DEBUG + 0,
"fork_execv dupped(%d) onto %d (should be %d)", pipes[ii][1], fd, ii);
382 for (
auto ii = close_start; ii < (close_start + close_cnt); ++ii)
388 execve(cmd, argv, env);
398 for (
auto ii = 0; ii < 3; ++ii)
400 if (lcl_iofd[ii] == -1)
402 close(ii == 0 ? pipes[ii][0] : pipes[ii][1]);
407 TRACE(TLVL_DEBUG + 0,
"fork_execv pid=%d", pid);
411 uint64_t swapPtr(
void *X)
413 auto x = (uint64_t)X;
414 x = (x & 0x00000000ffffffff) << 32 | (x & 0xffffffff00000000) >> 32;
415 x = (x & 0x0000ffff0000ffff) << 16 | (x & 0xfff0000fffff0000) >> 16;
416 x = (x & 0x00ff00ff00ff00ff) << 8 | (x & 0xff00ff00ff00ff00) >> 8;
439 static int g_devnullfd = -1;
442 std::string AWK(std::string
const &awk_cmd,
const char *file,
const char *input)
445 ssize_t bytes = 0, tot_bytes = 0;
446 char *
const argv_[4] = {(
char *)
"/bin/gawk",
447 (
char *)awk_cmd.c_str(),
453 if (g_devnullfd == -1)
455 g_devnullfd = open(
"/dev/null", O_WRONLY);
457 if (input !=
nullptr)
462 int iofd[3] = {infd, -1, 2};
463 TRACE(TLVL_DEBUG + 0,
"AWK b4 fork_execv input=%p", (
void *)input);
466 pid = fork_execv(0, 0 , 0, iofd,
"/bin/gawk", argv_, env);
467 if (input !=
nullptr )
469 int xx = strlen(input);
470 int sts = write(iofd[0], input, xx);
473 perror(
"write AWK stdin");
476 while ((bytes = read(iofd[1], &readbuf[tot_bytes],
sizeof(readbuf) - tot_bytes)) != 0)
478 TRACE(TLVL_DEBUG + 0,
"AWK while bytes=read > 0 bytes=%zd readbuf=0x%016lx errno=%d", bytes, swapPtr(&readbuf[tot_bytes]), errno);
489 TRACE(TLVL_DEBUG + 0,
"AWK after read tot=" + std::to_string((
long long unsigned)tot_bytes) +
" bytes=" + std::to_string((
long long unsigned)bytes) +
" input=" + std::string(input));
493 while ((bytes = read(iofd[1], &readbuf[tot_bytes],
sizeof(readbuf) - tot_bytes)) > 0)
497 TRACE(TLVL_DEBUG + 0,
"AWK after read tot=%zd bytes=%zd [0]=0x%x input=%p", tot_bytes, bytes, readbuf[0], (
void *)input);
499 readbuf[tot_bytes >= 0 ? tot_bytes : 0] =
'\0';
501 TRACE(TLVL_DEBUG + 0,
"AWK after close child stdout. child pid=%d", pid);
504 pid_t done_pid = waitpid(pid,&status,0);
505 TRACE( 3,
"AWK after wait pid=%d done_pid=%d status=%d(0x%x)"
506 , pid, done_pid, status, status );
508 return std::string(readbuf);
512 void string_addto_vector(std::string &instr, std::vector<std::string> &outvec,
char delim)
514 std::stringstream ss(instr);
518 std::getline(ss, substr, delim);
519 outvec.push_back(substr);
523 uint64_t gettimeofday_us()
526 gettimeofday(&tv,
nullptr);
531 return (uint64_t)tv.tv_sec * 1000000 + tv.tv_usec;
534 #define DATA_START " DATA START"
535 #define GNUPLOT_PREFIX (const char *) \
537 #!/usr/bin/env gnuplot\n\
540 # gnuplot -e 'ymin=400;ymax=1400' ./$0\n\
542 # gnuplot -e 'duration_s=35;set multiplot' ./gnuplot.gnuplot ./gnuplot.1.gnuplot -e 'set nomultiplot;pause -1'\n\
543 if(!exists('ARG0')) ARG0='' # for version 4, use: gnuplot -e ARG0=hello\n\
544 print 'ARG0=',ARG0 # ARG0.. automatically define in gnuplot version 5+\n\
545 if(!exists('ymin')) ymin=%d\n\
546 if(!exists('ymax')) ymax=%d\n\
547 if(!exists('yincr')) yincr=%d\n\
548 if(!exists('y2max')) y2max=%d\n\
549 if(!exists('y2incr')) y2incr=%d\n\
550 if(!exists('png')) png=1\n\
551 if(!exists('duration_s')) duration_s=0\n\
552 if(!exists('width')) width=512\n\
553 if(!exists('height')) height=384\n\
554 thisPid=system('echo `ps -p$$ -oppid=`')\n\
555 thisFile=system('ls -l /proc/'.thisPid.\"/fd | grep -v pipe: | tail -1 | sed -e 's/.*-> //'\")\n\
557 set title \"Disk Write Rate and %%CPU vs. time\\n%s %s %s%s\" # cmd and/or comment at end\n\
559 tfmt='%%Y-%%m-%%dT%%H:%%M:%%S' # try to use consistent format\n\
560 set timefmt '%%Y-%%m-%%dT%%H:%%M:%%S'\n\
562 set grid xtics back\n\
563 xstart=system(\"awk '/^....-..-..T/{print$1;exit}' \".thisFile)\n\
564 xend=system(\"awk 'END{print$1}' \".thisFile)\n\
565 print 'xstart='.xstart.' xend='.xend.' duration=',strptime(tfmt,xend)-strptime(tfmt,xstart)\n\
566 if(duration_s>0) end_t=strptime(tfmt,xstart)+duration_s; else end_t=strptime(tfmt,xend)\n\
567 set xrange [xstart:end_t]\n\
570 set ytics nomirror\n\
571 if(ymax==0) set yrange [ymin:*];\\\n\
572 else set yrange [ymin:ymax];set ytics yincr\n\
573 set grid ytics back\n\
575 set y2label '%%CPU, %%MemTotal'\n\
576 set y2tics autofreq\n\
577 if(y2max==0) set y2range [0:*];\\\n\
578 else set y2range [0:y2max];set y2tics y2incr\n\
581 if(png==1) set terminal png size width,height;\\\n\
582 pngfile=system( 'echo `basename '.thisFile.' .out`.png' );\\\n\
583 set output pngfile;\\\n\
584 else set terminal x11 size width,height\n\
586 plot \"< awk '/^#" DATA_START "/,/NEVER HAPPENS/' \".thisFile "
588 void sigchld_sigaction(
int signo, siginfo_t *info,
void *context __attribute__((__unused__)))
591 for (
int pid : g_pid_vec)
593 if (pid == info->si_pid)
595 TRACE(TLVL_INFO,
"sigchld_sigaction signo=%d status=%d(0x%x) code=%d(0x%x) sending_pid=%d", signo, info->si_status, info->si_status, info->si_code, info->si_code, info->si_pid);
599 TRACE(TLVL_DEBUG + 0,
"sigchld_sigaction signo=%d status=%d(0x%x) code=%d(0x%x) sending_pid=%d", signo, info->si_status, info->si_status, info->si_code, info->si_code, info->si_pid);
602 void read_proc_file(
const char *file,
char *buffer,
int buffer_size)
604 TRACE(TLVL_DEBUG + 1,
"read_proc_file b4 open proc file" + std::string(file));
605 int fd = open(file, O_RDONLY);
606 int offset = 0, sts = 0;
609 sts = read(fd, &buffer[offset], buffer_size - offset);
617 buffer[sts + offset] =
'\0';
619 TRACE(TLVL_DEBUG + 1,
"read_proc_file after close " + std::string(file) +
" read=%d offset=%d", sts, offset);
622 pid_t check_pid_vec()
624 for (
size_t ii = 0; ii < g_pid_vec.size();)
626 pid_t pid = g_pid_vec[ii];
628 pid_t pp = waitpid(pid, &status, WNOHANG);
629 TRACE(TLVL_DEBUG + 0,
"check_pid_vec %d=waitpid(pid=%d) errno=%d", pp, pid, errno);
632 g_pid_vec.erase(g_pid_vec.begin() + ii);
636 if (errno == ECHILD && kill(pid, 0) == 0)
644 g_pid_vec.erase(g_pid_vec.begin() + ii);
652 if (g_pid_vec.empty())
663 TRACE(TLVL_WARNING,
"atexit cleanup g_pid_vec.size()=%zd\n", g_pid_vec.size());
664 for (
int &pid : g_pid_vec)
669 #if (defined(__cplusplus) && (__cplusplus >= 201103L)) || (defined(__STDC_VERSION__) && (__STDC_VERSION__ >= 201112L))
670 #pragma GCC diagnostic push
671 #pragma GCC diagnostic ignored "-Wunused-parameter"
673 void sigint_sigaction(
int signo, siginfo_t *info,
void *context)
678 #if (defined(__cplusplus) && (__cplusplus >= 201103L)) || (defined(__STDC_VERSION__) && (__STDC_VERSION__ >= 201112L))
679 #pragma GCC diagnostic pop
682 int main(
int argc,
char *argv[])
685 int post_periods_completed = 0;
686 parse_args(argc, argv);
687 if ((argc - optind) != 0 || (opt_cmd.empty() && opt_pid.empty()))
690 printf(
"unexpected argument(s) %d!=0\n", argc - optind);
691 for (ii = 0; (optind + ii) < argc; ++ii)
693 printf(
"arg%d=%s\n", ii + 1, argv[optind + ii]);
699 std::vector<std::string> graphs;
700 string_addto_vector(opt_graph, graphs,
',');
702 char motherboard[1024] = {0};
705 FILE *fp = popen(
"dmidecode | grep -m2 'Product Name:' | tail -1",
"r");
706 fread(motherboard, 1,
sizeof(motherboard), fp);
709 TRACE(TLVL_WARNING,
"main - motherboard=" + std::string(motherboard));
717 struct sigaction sigaction_s;
722 sigaction_s.sa_sigaction = sigchld_sigaction;
723 sigaction_s.sa_flags = SA_SIGINFO | SA_NOCLDWAIT;
725 sigaction_s.sa_handler = SIG_IGN;
726 sigaction_s.sa_flags = SA_NOCLDWAIT;
728 sigemptyset(&sigaction_s.sa_mask);
729 sigaction(SIGCHLD, &sigaction_s,
nullptr);
731 sigaction_s.sa_sigaction = sigint_sigaction;
732 sigaction_s.sa_flags = SA_SIGINFO;
733 sigaction(SIGINT, &sigaction_s,
nullptr);
737 long long unsigned concurentThreadsSupported = sysconf(_SC_NPROCESSORS_ONLN);
739 TRACE(TLVL_ERROR,
"main concurentThreadsSupported=%u opt_stats=" + opt_stats, concurentThreadsSupported);
742 gettimeofday(&tv,
nullptr);
743 strftime(run_time,
sizeof(run_time),
"%FT%H%M%S", localtime(&tv.tv_sec));
744 TRACE(TLVL_ERROR,
"main run_time=" + std::string(run_time));
750 if ((dot = strchr(ubuf.nodename,
'.')) !=
nullptr)
754 std::string hostname(ubuf.nodename);
755 TRACE(TLVL_WARNING,
"release=" + std::string(ubuf.release) +
" version=" + std::string(ubuf.version));
758 std::string memKB = AWK(
"NR==1{print$2;exit}",
"/proc/meminfo",
nullptr);
759 memKB = memKB.substr(0, memKB.size() - 1);
761 std::string dat_file_out(opt_outdir +
"periodic_" + run_time +
"_" + hostname +
"_stats.out");
763 double period = atof(opt_period);
767 std::vector<std::string> pidfile;
769 std::vector<std::string> stats;
773 for (
size_t ii = 0; ii < opt_cmd.size(); ++ii)
775 char cmd_file_out[1024];
776 snprintf(cmd_file_out,
sizeof(cmd_file_out),
"%speriodic_%s_%s_cmd%zd.out", opt_outdir.c_str(), run_time, hostname.c_str(), ii);
777 int fd = open(cmd_file_out, O_WRONLY | O_CREAT, 0666);
778 TRACE(TLVL_ERROR,
"main fd=%d opt_cmd=" + opt_cmd[ii] +
" cmd_file_out=" + std::string(cmd_file_out), fd);
779 int iofd[3] = {0, fd, fd};
780 char *
const argv_[4] = {(
char *)
"/bin/sh",
782 (
char *)opt_cmd[ii].c_str(),
784 g_pid_vec.push_back(fork_execv(0, 0, (
int)(period * opt_pre * 1e6), iofd,
"/bin/sh", argv_,
nullptr));
786 std::string pidstr = std::to_string((
long long int)g_pid_vec[ii]);
787 pidfile.push_back(
"/proc/" + pidstr +
"/stat");
789 char desc[128], ss[1024];
791 snprintf(ss,
sizeof(ss),
"CPUcmd%zd?%s?NR==1?$14+$15?1?yes", ii, pidfile[ii].c_str());
792 stats.emplace_back(ss);
794 snprintf(desc,
sizeof(desc),
"CPU+cmd%zd", ii);
795 graphs.emplace_back(desc);
796 snprintf(ss,
sizeof(ss),
"%s?%s?NR==1?$14+$15+16+$17?1?yes", desc, pidfile[ii].c_str());
797 stats.emplace_back(ss);
799 snprintf(desc,
sizeof(desc),
"WaitBlkIOcmd%zd", ii);
800 if (opt_cmd_iowait != 0)
802 graphs.emplace_back(desc);
804 snprintf(ss,
sizeof(ss),
"%s?%s?NR==1?$42?1?yes", desc, pidfile[ii].c_str());
805 stats.emplace_back(ss);
807 snprintf(desc,
sizeof(desc),
"Faultcmd%zd", ii);
810 graphs.emplace_back(desc);
812 snprintf(ss,
sizeof(ss),
"%s?%s?NR==1?$10+$11+$12+$13?4096.0/1048576?yes", desc, pidfile[ii].c_str());
813 stats.emplace_back(ss);
815 for (
size_t ii = 0; ii < opt_Cmd.size(); ++ii)
817 char cmd_file_out[1024];
818 snprintf(cmd_file_out,
sizeof(cmd_file_out),
"%speriodic_%s_%s_cmd%zd.out", opt_outdir.c_str(), run_time, hostname.c_str(), ii + opt_cmd.size());
819 int fd = open(cmd_file_out, O_WRONLY | O_CREAT, 0666);
820 TRACE(TLVL_ERROR,
"main fd=%d opt_Cmd=" + opt_Cmd[ii] +
" cmd_file_out=" + std::string(cmd_file_out), fd);
821 int iofd[3] = {0, fd, fd};
822 char *
const argv_[4] = {(
char *)
"/bin/sh",
824 (
char *)opt_Cmd[ii].c_str(),
826 g_pid_vec.push_back(fork_execv(0, 0, (
int)(period * opt_pre * 1e6), iofd,
"/bin/sh", argv_,
nullptr));
828 std::string pidstr = std::to_string((
long long int)g_pid_vec[ii]);
829 pidfile.push_back(
"/proc/" + pidstr +
"/stat");
831 char desc[128], ss[1024];
832 snprintf(desc,
sizeof(desc),
"CPU+cmd%zd", ii + opt_cmd.size());
833 snprintf(ss,
sizeof(ss),
"CPUcmd%zd?%s?NR==1?$14+$15?1?yes", ii + opt_cmd.size(), pidfile[ii].c_str());
834 stats.emplace_back(ss);
835 snprintf(ss,
sizeof(ss),
"CPU+cmd%zd?%s?NR==1?$14+$15+16+$17?1?yes", ii + opt_cmd.size(), pidfile[ii].c_str());
836 stats.emplace_back(ss);
839 std::vector<std::string> pids;
840 if (!opt_pid.empty() != 0u)
842 string_addto_vector(opt_pid, pids,
',');
844 for (
size_t ii = 0; ii < pids.size(); ++ii)
846 g_pid_vec.push_back(std::stoi(pids[ii]));
847 TRACE(TLVL_WARNING,
"pid=%s g_pid_vec.size()=%ld", pids[ii].c_str(), g_pid_vec.size());
848 pidfile.push_back(
"/proc/" + pids[ii] +
"/stat");
849 char desc[128], ss[1024];
851 snprintf(ss,
sizeof(ss),
"CPUpid%zd?%s?NR==1?$14+$15?1?yes", ii, pidfile[ii].c_str());
852 stats.emplace_back(ss);
854 std::ifstream t(
"/proc/" + pids[ii] +
"/comm");
855 std::string comm((std::istreambuf_iterator<char>(t)),
856 std::istreambuf_iterator<char>());
857 comm = comm.substr(0, comm.size() - 1);
859 snprintf(desc,
sizeof(desc),
"CPU+pid%zd_%s", ii, comm.c_str());
860 graphs.emplace_back(desc);
861 snprintf(ss,
sizeof(ss),
"%s?%s?NR==1?$14+$15+16+$17?1?yes", desc, pidfile[ii].c_str());
862 stats.emplace_back(ss);
864 snprintf(desc,
sizeof(desc),
"WaitBlkIOpid%zd", ii);
865 if (opt_cmd_iowait != 0)
867 graphs.emplace_back(desc);
869 snprintf(ss,
sizeof(ss),
"%s?%s?NR==1?$42?1?yes", desc, pidfile[ii].c_str());
870 stats.emplace_back(ss);
872 snprintf(desc,
sizeof(desc),
"Faultpid%zd", ii);
875 graphs.emplace_back(desc);
877 snprintf(ss,
sizeof(ss),
"%s?%s?NR==1?$10+$11+$12+$13?4096.0/1048576?yes", desc, pidfile[ii].c_str());
878 stats.emplace_back(ss);
881 stats.emplace_back(
"CPUnode");
882 stats.emplace_back(
"IOWait");
883 if (opt_sys_iowait != 0) { graphs.emplace_back(
"IOWait"); }
884 stats.emplace_back(
"Cached");
885 stats.emplace_back(
"Dirty");
886 stats.emplace_back(
"Free");
888 if (!opt_disk.empty() != 0u)
890 std::vector<std::string> tmp;
891 string_addto_vector(opt_disk, tmp,
',');
897 std::string statstr = dk +
"_wrMB/s?/proc/diskstats?/" + dk +
"/?$10?(1.0/2048)?yes";
898 stats.push_back(statstr);
899 std::vector<std::string> stat_spec;
900 string_addto_vector(statstr, stat_spec,
'?');
901 graphs.push_back(stat_spec[s_desc]);
903 statstr = dk +
"_rdMB/s?/proc/diskstats?/" + dk +
"/?$6?(1.0/2048)?yes";
904 stats.push_back(statstr);
906 string_addto_vector(statstr, stat_spec,
'?');
911 if (!opt_stats.empty() != 0u)
913 std::vector<std::string> tmp_stats;
914 string_addto_vector(opt_stats, tmp_stats,
',');
915 for (
auto &tmp_stat : tmp_stats)
917 stats.push_back(tmp_stat);
918 std::vector<std::string> stat_spec;
919 string_addto_vector(tmp_stat, stat_spec,
'?');
920 graphs.push_back(stat_spec[s_desc]);
924 std::vector<long> pre_vals;
925 std::vector<double> multipliers;
926 std::vector<std::vector<std::string>> spec2(stats.size());
927 std::vector<std::string> awkCmd;
929 std::string header_str(
"#" DATA_START
"\n#_______time_______");
931 int outfd = open(dat_file_out.c_str(), O_WRONLY | O_CREAT, 0777);
933 FILE *outfp = fdopen(outfd,
"w");
935 std::string cmd_comment;
936 if (!opt_cmd.empty() != 0u)
938 cmd_comment +=
"\\ncmd: " + opt_cmd[0];
940 if (!opt_comment.empty() != 0u)
942 cmd_comment +=
"\\ncomment: " + opt_comment;
944 fprintf(outfp, GNUPLOT_PREFIX, opt_ymin, opt_ymax, opt_yincr, opt_y2max, opt_y2incr, run_time, hostname.c_str(), ubuf.release, cmd_comment.c_str(),
"disk write MB/s");
946 uint64_t t_start = gettimeofday_us();
949 bool first_graph_spec_added =
false;
950 for (
size_t ii = 0; ii < stats.size(); ++ii)
952 std::vector<std::string> stat_spec;
953 string_addto_vector(stats[ii], stat_spec,
'?');
954 if (stat_spec[s_desc] ==
"CPUnode" && stat_spec.size() == 1)
957 stats[ii] +=
"?/proc/stat?/^cpu[^0-9]/?$2+$3+$4+$6+$7+$8+$9?1.0/" + std::to_string(concurentThreadsSupported) +
"?yes";
959 else if (stat_spec[s_desc] ==
"IOWait" && stat_spec.size() == 1)
961 stats[ii] +=
"?/proc/stat?/^cpu[^0-9]/?$6?1.0/" + std::to_string(concurentThreadsSupported) +
"?yes";
963 else if (stat_spec[s_desc] ==
"Cached" && stat_spec.size() == 1)
965 stats[ii] +=
"?/proc/meminfo?/^(Cached|Buffers):/?$2?1?no";
967 else if (stat_spec[s_desc] ==
"Dirty" && stat_spec.size() == 1)
969 stats[ii] +=
"?/proc/meminfo?/^Dirty:/?$2?1?no";
971 else if (stat_spec[s_desc] ==
"Free" && stat_spec.size() == 1)
973 stats[ii] +=
"?/proc/meminfo?/^MemFree:/?$2?1?no";
976 header_str +=
" " + stat_spec[s_desc];
978 string_addto_vector(stats[ii], spec2[ii],
'?');
980 snprintf(awk_cmd,
sizeof(awk_cmd),
"%s{vv+=%s}END{print vv}"
983 spec2[ii][s_linespec].c_str(), spec2[ii][s_fieldspec].c_str());
984 awkCmd.emplace_back(awk_cmd);
986 std::string stat = AWK(awkCmd.back(), spec2[ii][s_file].c_str(),
nullptr);
988 pre_vals.push_back(atol(stat.c_str()));
989 multipliers.push_back(atof(AWK(
"BEGIN{print " + spec2[ii][s_multiplier] +
"}",
"/dev/null",
nullptr).c_str()));
991 for (
const auto &graph : graphs)
993 if (graph == stat_spec[s_desc])
995 if (first_graph_spec_added)
997 fprintf(outfp,
",\\\n '' ");
999 if (strncmp(stat_spec[s_desc].c_str(),
"CPU", 3) == 0)
1001 fprintf(outfp,
"using 1:%zd title '%s' w linespoints axes x1y2", ii + 2, stat_spec[s_desc].c_str());
1003 else if (stat_spec[s_desc] ==
"Cached" || stat_spec[s_desc] ==
"Dirty" || stat_spec[s_desc] ==
"Free")
1005 fprintf(outfp,
"using 1:($%zd/%s*100) title '%s%%' w linespoints axes x1y2", ii + 2, memKB.c_str(), stat_spec[s_desc].c_str());
1007 else if (stat_spec[s_desc].substr(0, 6) ==
"CPUcmd" || stat_spec[s_desc].substr(0, 6) ==
"CPU+cm")
1009 fprintf(outfp,
"using 1:%zd title '%s' w linespoints axes x1y2", ii + 2, stat_spec[s_desc].c_str());
1011 else if (stat_spec[s_desc].substr(0, 12) ==
"WaitBlkIOcmd")
1013 fprintf(outfp,
"using 1:%zd title '%s' w linespoints axes x1y2", ii + 2, stat_spec[s_desc].c_str());
1017 fprintf(outfp,
"using 1:%zd title '%s' w linespoints axes x1y1", ii + 2, stat_spec[s_desc].c_str());
1019 first_graph_spec_added =
true;
1023 header_str +=
" #\n";
1026 "\nif(png==0) pause -1 'Press Enter/Return or ^C to finish'\n\
1030 fprintf(outfp,
"cmds:\n");
1031 for (
const auto &ii : opt_cmd)
1033 std::string ss = ii +
"\n";
1034 fprintf(outfp,
"%s", ss.c_str());
1038 fprintf(outfp,
"stats:\n");
1039 for (
const auto &stat : stats)
1041 std::string ss = stat +
"\n";
1042 fprintf(outfp,
"%s", ss.c_str());
1046 fprintf(outfp,
"%s", header_str.c_str());
1049 std::string tmpdbg(
"main lp=%d done stat%zd=%ld rate=%f ");
1051 char proc_stats[8192];
1058 int64_t t_sleep = (t_start + (uint64_t)(period * 1e6)) - gettimeofday_us();
1061 int sts = usleep(t_sleep);
1062 TRACE(TLVL_DEBUG + 0,
"main usleep sts=%d errno=%d", sts, errno);
1070 for (lp = 2; lp < MAX_LP; ++lp)
1073 gettimeofday(&tv,
nullptr);
1074 strftime(str,
sizeof(str),
"%FT%T", localtime(&tv.tv_sec));
1076 fprintf(outfp,
"%s", str);
1077 std::string prv_file;
1078 for (
size_t ii = 0; ii < stats.size(); ++ii)
1080 TRACE(TLVL_DEBUG + 0,
"main lp=%d start stat%zd", lp, ii);
1081 char const *awk_file;
1082 if (ii < (2 * opt_cmd.size()))
1088 read_proc_file(pidfile[ii / 2].c_str(), proc_stats,
sizeof(proc_stats));
1090 awk_in = proc_stats;
1093 else if (spec2[ii][s_file] != prv_file)
1095 prv_file = spec2[ii][s_file];
1096 read_proc_file(spec2[ii][s_file].c_str(), proc_stats,
sizeof(proc_stats));
1097 awk_in = proc_stats;
1101 std::string stat_str = AWK(awkCmd[ii], awk_file, awk_in);
1103 long stat = atol(stat_str.c_str());
1105 if (spec2[ii][s_rate] ==
"yes")
1108 if (stat_str !=
"\n")
1110 rate = (stat - pre_vals[ii]) * multipliers[ii] / period;
1116 TRACE(TLVL_DEBUG + 0, tmpdbg +
"stat_str[0]=0x%x stat_str.size()=%zd", lp, ii, stat, rate, stat_str[0], stat_str.size());
1117 fprintf(outfp,
" %.2f", rate);
1118 if (rate < 0.0 && spec2[ii][s_file] ==
"/proc/diskstats")
1120 TRACE(TLVL_ERROR,
"main stat:" + spec2[ii][s_desc] +
" rate=%f pre_val=%ld stat=%ld stat_str=\"" + stat_str +
"\" awkCmd=" + awkCmd[ii] +
" proc_diskstats=" + proc_stats, rate, pre_vals[ii], stat);
1123 pre_vals[ii] = stat;
1127 TRACE(TLVL_DEBUG + 0,
"main lp=%d done stat%zd=%ld", lp, ii, stat);
1128 fprintf(outfp,
" %.2f", stat * multipliers[ii]);
1131 fprintf(outfp,
"\n");
1134 int64_t t_sleep = (t_start + (uint64_t)(period * lp * 1000000)) - gettimeofday_us();
1137 int sts = usleep(t_sleep);
1138 TRACE(TLVL_DEBUG + 0,
"main usleep sts=%d errno=%d", sts, errno);
1144 pp = check_pid_vec();
1145 TRACE(TLVL_INFO,
"main pp=%d t_sleep=%ld", pp, t_sleep);
1148 if (post_periods_completed == 0)
1150 TRACE(TLVL_WARNING,
"main processes complete - waiting %d post periods", opt_post);
1152 if (post_periods_completed++ == opt_post)
1160 fprintf(outfp,
"# MAX_LP abort\n");
1166 TRACE(TLVL_ERROR,
"main done/complete/returning");