6 static const char *rev =
"$Revision: 1.19 $$Date: 2018/06/28 21:14:28 $";
13 usage: %s --cmd=<cmd>\n\
14 examples: %s --cmd='sleep 25' # this will take about a minute\n\
15 %s --cmd='taskset -c 4 dd if=/dev/zero of=/dev/sdf bs=50M count=500 oflag=direct' --disk=sdf\n\
16 %s --cmd='dd if=/dev/zero of=t.dat count=500' --stat='md*MB/s?/proc/diskstats?/md[0-3]/?$10?(1.0/2048)?yes'\n\
17 gnuplot -e png=0 `/bin/ls -t periodic_*_stats.out|head -1`\n\
18 For each cmd, record CPU info. Additionally, record total system CPU\n\
19 (can be >100 w/ multicore), file system Cached and Dirty %%, plus any\n\
20 other stats specified by --stats options\n\
22 --cmd= can have multiple\n\
23 --Cmd= run cmd, but dont graph CPU (can have multiple)\n\
24 --pid= graph comma or space sep. list of pids (getting cmd from /proc/)\n\
25 --disk= automatically build --stat for disk(s)\n\
26 --stat= desc?file?linespec?fieldspec?multiplier?rate\n\
27 builtin = CPUnode,Cached,Dirty\n\
28 cmd-builtin = CPUcmdN, CPU+cmdN\n\
29 --out-dir= output dir (for 2+ output files; stats and cmd+)\n\
31 --period= (float) default=%s\n\
32 --pre= the number of periods wait before exec of --cmd\n\
33 --post= the number of periods to loop after cmd exits\n\
34 --graph= add to graph any possibly included non-graphed metrics\n\
36 --no-defaults for no default stats (ie. cpu-parent,cpu-children)\n\
37 --init= default dropcache if root. NOT implemented yet\n\
44 --sys-iowait,-w include the system iowait on the graph\n\
45 --cmd-iowait include cmd iowait on the graph\n\
48 basename(argv[0]), basename(argv[0]), basename(argv[0]), basename(argv[0]), opt_period
63 #include <sys/utsname.h>
75 #define TRACE_NAME "periodic_cmd_stats"
81 static void trace_ap(
const char *msg, va_list ap)
84 unsigned len = strlen(msg);
85 len = len < (
sizeof(m_) - 2) ? len : (
sizeof(m_) - 2);
86 memcpy(m_, msg, len + 1);
87 if (m_[len - 1] !=
'\n')
89 memcpy(&(m_[len]),
"\n", 2);
94 static void trace_p(
const char *msg, ...) __attribute__((format(printf, 1, 2)));
95 static
void trace_p(const
char *msg, ...)
102 static void trace_p(
const std::string msg, ...)
106 trace_ap(msg.c_str(), ap);
109 #define TRACE(lvl, ...) \
111 if (lvl <= 1) trace_p(__VA_ARGS__); \
113 #define TRACE_CNTL(xyzz, ...)
118 char *opt_init =
nullptr;
119 std::vector<std::string> opt_cmd;
120 std::vector<std::string> opt_Cmd;
122 std::string opt_disk;
123 std::string opt_stats;
124 std::string opt_outdir;
125 std::string opt_graph(
"CPUnode,Cached,Dirty,Free");
126 const char *opt_period =
"5.0";
127 std::string opt_comment;
135 int opt_sys_iowait = 0;
136 int opt_cmd_iowait = 0;
139 std::vector<pid_t> g_pid_vec;
141 void charreplace(
char *instr,
char oldc,
char newc)
153 void parse_args(
int argc,
char *argv[])
160 static struct option long_options[] = {
162 {
"help", no_argument,
nullptr,
'h'},
163 {
"init", required_argument,
nullptr,
'i'},
164 {
"cmd", required_argument,
nullptr,
'c'},
165 {
"Cmd", required_argument,
nullptr,
'C'},
166 {
"disk", required_argument,
nullptr,
'd'},
167 {
"stat", required_argument,
nullptr,
's'},
168 {
"out-dir", required_argument,
nullptr,
'o'},
169 {
"period", required_argument,
nullptr,
'p'},
170 {
"sys-iowait", no_argument,
nullptr,
'w'},
171 {
"fault", no_argument,
nullptr,
'f'},
172 {
"pid", required_argument,
nullptr,
'P'},
173 {
"ymax", required_argument,
nullptr, 1},
174 {
"yincr", required_argument,
nullptr, 2},
175 {
"y2max", required_argument,
nullptr, 3},
176 {
"y2incr", required_argument,
nullptr, 4},
177 {
"pre", required_argument,
nullptr, 5},
178 {
"post", required_argument,
nullptr, 6},
179 {
"graph", required_argument,
nullptr, 7},
180 {
"yrange", required_argument,
nullptr, 8},
181 {
"comment", required_argument,
nullptr, 9},
182 {
"cmd-iowait", no_argument,
nullptr, 10},
183 {
nullptr, 0,
nullptr, 0}};
184 opt = getopt_long(argc, argv,
"?hvqVi:c:C:d:s:o:p:P:wf",
185 long_options,
nullptr);
211 opt_cmd.emplace_back(optarg);
214 opt_Cmd.emplace_back(optarg);
217 if (!opt_disk.empty())
219 opt_disk = opt_disk +
"," + optarg;
227 if (!opt_stats.empty())
229 opt_stats += std::string(
",") + optarg;
237 opt_outdir = std::string(optarg) +
"/";
249 charreplace(optarg,
' ',
',');
250 if (!opt_pid.empty() != 0u)
252 opt_pid = opt_pid +
"," + optarg;
260 opt_ymax = strtoul(optarg,
nullptr, 0);
263 opt_yincr = strtoul(optarg,
nullptr, 0);
266 opt_y2max = strtoul(optarg,
nullptr, 0);
269 opt_y2incr = strtoul(optarg,
nullptr, 0);
272 opt_pre = strtoul(optarg,
nullptr, 0);
275 opt_post = strtoul(optarg,
nullptr, 0);
278 opt_graph += std::string(
",") + optarg;
281 opt_ymin = strtoul(optarg,
nullptr, 0);
282 cp = strstr(optarg,
":") + 1;
283 opt_ymax = strtoul(cp,
nullptr, 0);
284 if ((cp = strstr(cp,
":")) !=
nullptr)
287 opt_yincr = strtoul(strstr(cp,
":") + 1,
nullptr, 0);
291 opt_yincr = (opt_ymax - opt_ymin) / 5;
295 opt_comment = optarg;
301 printf(
"?? getopt returned character code 0%o ??\n", opt);
307 void perror_exit(
const char *msg, ...)
312 vsnprintf(buf,
sizeof(buf), msg, ap);
326 pid_t fork_execv(
int close_start,
int close_cnt,
int sleepB4exec_us,
int iofd[3],
const char *cmd,
char *
const argv[],
char *
const env[])
330 for (
auto ii = 0; ii < 3; ++ii)
332 lcl_iofd[ii] = iofd[ii];
336 iofd[ii] = ii == 0 ? pipes[ii][1] : pipes[ii][0];
346 if (lcl_iofd[0] == -1)
349 int fd = dup2(pipes[0][0], 0);
350 TRACE(3,
"fork_execv dupped(%d) onto %d (should be 0)", pipes[0][0], fd);
353 if (sleepB4exec_us != 0)
357 usleep(sleepB4exec_us);
358 TRACE(1,
"fork_execv sleep complete. sleepB4exec_us=%d sts=%d", sleepB4exec_us, 0 );
360 for (
auto ii = 1; ii < 3; ++ii)
362 if (lcl_iofd[ii] == -1)
365 int fd = dup2(pipes[ii][1], ii);
366 TRACE(3,
"fork_execv dupped(%d) onto %d (should be %d)", pipes[ii][1], fd, ii);
369 else if (lcl_iofd[ii] != ii)
371 int fd = dup2(lcl_iofd[ii], ii);
372 TRACE(3,
"fork_execv dupped(%d) onto %d (should be %d)", pipes[ii][1], fd, ii);
375 for (
auto ii = close_start; ii < (close_start + close_cnt); ++ii)
381 execve(cmd, argv, env);
391 for (
auto ii = 0; ii < 3; ++ii)
393 if (lcl_iofd[ii] == -1)
395 close(ii == 0 ? pipes[ii][0] : pipes[ii][1]);
400 TRACE(3,
"fork_execv pid=%d", pid);
404 uint64_t swapPtr(
void *X)
406 auto x = (uint64_t)X;
407 x = (x & 0x00000000ffffffff) << 32 | (x & 0xffffffff00000000) >> 32;
408 x = (x & 0x0000ffff0000ffff) << 16 | (x & 0xfff0000fffff0000) >> 16;
409 x = (x & 0x00ff00ff00ff00ff) << 8 | (x & 0xff00ff00ff00ff00) >> 8;
432 static int g_devnullfd = -1;
435 std::string AWK(std::string
const &awk_cmd,
const char *file,
const char *input)
438 ssize_t bytes = 0, tot_bytes = 0;
439 char *
const argv_[4] = {(
char *)
"/bin/gawk",
440 (
char *)awk_cmd.c_str(),
446 if (g_devnullfd == -1)
448 g_devnullfd = open(
"/dev/null", O_WRONLY);
450 if (input !=
nullptr)
455 int iofd[3] = {infd, -1, 2};
456 TRACE(3,
"AWK b4 fork_execv input=%p", (
void *)input);
459 pid = fork_execv(0, 0 , 0, iofd,
"/bin/gawk", argv_, env);
460 if (input !=
nullptr )
462 int xx = strlen(input);
463 int sts = write(iofd[0], input, xx);
466 perror(
"write AWK stdin");
469 while ((bytes = read(iofd[1], &readbuf[tot_bytes],
sizeof(readbuf) - tot_bytes)) != 0)
471 TRACE(3,
"AWK while bytes=read > 0 bytes=%zd readbuf=0x%016lx errno=%d", bytes, swapPtr(&readbuf[tot_bytes]), errno);
482 TRACE(3,
"AWK after read tot=" + std::to_string((
long long unsigned)tot_bytes) +
" bytes=" + std::to_string((
long long unsigned)bytes) +
" input=" + std::string(input));
486 while ((bytes = read(iofd[1], &readbuf[tot_bytes],
sizeof(readbuf) - tot_bytes)) > 0)
490 TRACE(3,
"AWK after read tot=%zd bytes=%zd [0]=0x%x input=%p", tot_bytes, bytes, readbuf[0], (
void *)input);
492 readbuf[tot_bytes >= 0 ? tot_bytes : 0] =
'\0';
494 TRACE(3,
"AWK after close child stdout. child pid=%d", pid);
497 pid_t done_pid = waitpid(pid,&status,0);
498 TRACE( 3,
"AWK after wait pid=%d done_pid=%d status=%d(0x%x)"
499 , pid, done_pid, status, status );
501 return std::string(readbuf);
505 void string_addto_vector(std::string &instr, std::vector<std::string> &outvec,
char delim)
507 std::stringstream ss(instr);
511 std::getline(ss, substr, delim);
512 outvec.push_back(substr);
516 uint64_t gettimeofday_us()
519 gettimeofday(&tv,
nullptr);
524 return (uint64_t)tv.tv_sec * 1000000 + tv.tv_usec;
527 #define DATA_START " DATA START"
528 #define GNUPLOT_PREFIX (const char *) \
530 #!/usr/bin/env gnuplot\n\
533 # gnuplot -e 'ymin=400;ymax=1400' ./$0\n\
535 # gnuplot -e 'duration_s=35;set multiplot' ./gnuplot.gnuplot ./gnuplot.1.gnuplot -e 'set nomultiplot;pause -1'\n\
536 if(!exists('ARG0')) ARG0='' # for version 4, use: gnuplot -e ARG0=hello\n\
537 print 'ARG0=',ARG0 # ARG0.. automatically define in gnuplot version 5+\n\
538 if(!exists('ymin')) ymin=%d\n\
539 if(!exists('ymax')) ymax=%d\n\
540 if(!exists('yincr')) yincr=%d\n\
541 if(!exists('y2max')) y2max=%d\n\
542 if(!exists('y2incr')) y2incr=%d\n\
543 if(!exists('png')) png=1\n\
544 if(!exists('duration_s')) duration_s=0\n\
545 if(!exists('width')) width=512\n\
546 if(!exists('height')) height=384\n\
547 thisPid=system('echo `ps -p$$ -oppid=`')\n\
548 thisFile=system('ls -l /proc/'.thisPid.\"/fd | grep -v pipe: | tail -1 | sed -e 's/.*-> //'\")\n\
550 set title \"Disk Write Rate and %%CPU vs. time\\n%s %s %s%s\" # cmd and/or comment at end\n\
552 tfmt='%%Y-%%m-%%dT%%H:%%M:%%S' # try to use consistent format\n\
553 set timefmt '%%Y-%%m-%%dT%%H:%%M:%%S'\n\
555 set grid xtics back\n\
556 xstart=system(\"awk '/^....-..-..T/{print$1;exit}' \".thisFile)\n\
557 xend=system(\"awk 'END{print$1}' \".thisFile)\n\
558 print 'xstart='.xstart.' xend='.xend.' duration=',strptime(tfmt,xend)-strptime(tfmt,xstart)\n\
559 if(duration_s>0) end_t=strptime(tfmt,xstart)+duration_s; else end_t=strptime(tfmt,xend)\n\
560 set xrange [xstart:end_t]\n\
563 set ytics nomirror\n\
564 if(ymax==0) set yrange [ymin:*];\\\n\
565 else set yrange [ymin:ymax];set ytics yincr\n\
566 set grid ytics back\n\
568 set y2label '%%CPU, %%MemTotal'\n\
569 set y2tics autofreq\n\
570 if(y2max==0) set y2range [0:*];\\\n\
571 else set y2range [0:y2max];set y2tics y2incr\n\
574 if(png==1) set terminal png size width,height;\\\n\
575 pngfile=system( 'echo `basename '.thisFile.' .out`.png' );\\\n\
576 set output pngfile;\\\n\
577 else set terminal x11 size width,height\n\
579 plot \"< awk '/^#" DATA_START "/,/NEVER HAPPENS/' \".thisFile "
581 void sigchld_sigaction(
int signo, siginfo_t *info,
void *context __attribute__((__unused__)))
584 for (
int pid : g_pid_vec)
586 if (pid == info->si_pid)
588 TRACE(2,
"sigchld_sigaction signo=%d status=%d(0x%x) code=%d(0x%x) sending_pid=%d", signo, info->si_status, info->si_status, info->si_code, info->si_code, info->si_pid);
592 TRACE(3,
"sigchld_sigaction signo=%d status=%d(0x%x) code=%d(0x%x) sending_pid=%d", signo, info->si_status, info->si_status, info->si_code, info->si_code, info->si_pid);
595 void read_proc_file(
const char *file,
char *buffer,
int buffer_size)
597 TRACE(4,
"read_proc_file b4 open proc file" + std::string(file));
598 int fd = open(file, O_RDONLY);
599 int offset = 0, sts = 0;
602 sts = read(fd, &buffer[offset], buffer_size - offset);
610 buffer[sts + offset] =
'\0';
612 TRACE(4,
"read_proc_file after close " + std::string(file) +
" read=%d offset=%d", sts, offset);
615 pid_t check_pid_vec()
617 for (
size_t ii = 0; ii < g_pid_vec.size();)
619 pid_t pid = g_pid_vec[ii];
621 pid_t pp = waitpid(pid, &status, WNOHANG);
622 TRACE(3,
"check_pid_vec %d=waitpid(pid=%d) errno=%d", pp, pid, errno);
625 g_pid_vec.erase(g_pid_vec.begin() + ii);
629 if (errno == ECHILD && kill(pid, 0) == 0)
637 g_pid_vec.erase(g_pid_vec.begin() + ii);
645 if (g_pid_vec.empty())
656 TRACE(1,
"atexit cleanup g_pid_vec.size()=%zd\n", g_pid_vec.size());
657 for (
int &pid : g_pid_vec)
662 #if (defined(__cplusplus) && (__cplusplus >= 201103L)) || (defined(__STDC_VERSION__) && (__STDC_VERSION__ >= 201112L))
663 #pragma GCC diagnostic push
664 #pragma GCC diagnostic ignored "-Wunused-parameter"
666 void sigint_sigaction(
int signo, siginfo_t *info,
void *context)
671 #if (defined(__cplusplus) && (__cplusplus >= 201103L)) || (defined(__STDC_VERSION__) && (__STDC_VERSION__ >= 201112L))
672 #pragma GCC diagnostic pop
675 int main(
int argc,
char *argv[])
678 int post_periods_completed = 0;
679 parse_args(argc, argv);
680 if ((argc - optind) != 0 || (opt_cmd.empty() && opt_pid.empty()))
683 printf(
"unexpected argument(s) %d!=0\n", argc - optind);
684 for (ii = 0; (optind + ii) < argc; ++ii)
686 printf(
"arg%d=%s\n", ii + 1, argv[optind + ii]);
692 std::vector<std::string> graphs;
693 string_addto_vector(opt_graph, graphs,
',');
695 char motherboard[1024] = {0};
698 FILE *fp = popen(
"dmidecode | grep -m2 'Product Name:' | tail -1",
"r");
699 fread(motherboard, 1,
sizeof(motherboard), fp);
702 TRACE(1,
"main - motherboard=" + std::string(motherboard));
710 struct sigaction sigaction_s;
715 sigaction_s.sa_sigaction = sigchld_sigaction;
716 sigaction_s.sa_flags = SA_SIGINFO | SA_NOCLDWAIT;
718 sigaction_s.sa_handler = SIG_IGN;
719 sigaction_s.sa_flags = SA_NOCLDWAIT;
721 sigemptyset(&sigaction_s.sa_mask);
722 sigaction(SIGCHLD, &sigaction_s,
nullptr);
724 sigaction_s.sa_sigaction = sigint_sigaction;
725 sigaction_s.sa_flags = SA_SIGINFO;
726 sigaction(SIGINT, &sigaction_s,
nullptr);
730 long long unsigned concurentThreadsSupported = sysconf(_SC_NPROCESSORS_ONLN);
732 TRACE(0,
"main concurentThreadsSupported=%u opt_stats=" + opt_stats, concurentThreadsSupported);
735 gettimeofday(&tv,
nullptr);
736 strftime(run_time,
sizeof(run_time),
"%FT%H%M%S", localtime(&tv.tv_sec));
737 TRACE(0,
"main run_time=" + std::string(run_time));
743 if ((dot = strchr(ubuf.nodename,
'.')) !=
nullptr)
747 std::string hostname(ubuf.nodename);
748 TRACE(1,
"release=" + std::string(ubuf.release) +
" version=" + std::string(ubuf.version));
751 std::string memKB = AWK(
"NR==1{print$2;exit}",
"/proc/meminfo",
nullptr);
752 memKB = memKB.substr(0, memKB.size() - 1);
754 std::string dat_file_out(opt_outdir +
"periodic_" + run_time +
"_" + hostname +
"_stats.out");
756 double period = atof(opt_period);
760 std::vector<std::string> pidfile;
762 std::vector<std::string> stats;
766 for (
size_t ii = 0; ii < opt_cmd.size(); ++ii)
768 char cmd_file_out[1024];
769 snprintf(cmd_file_out,
sizeof(cmd_file_out),
"%speriodic_%s_%s_cmd%zd.out", opt_outdir.c_str(), run_time, hostname.c_str(), ii);
770 int fd = open(cmd_file_out, O_WRONLY | O_CREAT, 0666);
771 TRACE(0,
"main fd=%d opt_cmd=" + opt_cmd[ii] +
" cmd_file_out=" + std::string(cmd_file_out), fd);
772 int iofd[3] = {0, fd, fd};
773 char *
const argv_[4] = {(
char *)
"/bin/sh",
775 (
char *)opt_cmd[ii].c_str(),
777 g_pid_vec.push_back(fork_execv(0, 0, (
int)(period * opt_pre * 1e6), iofd,
"/bin/sh", argv_,
nullptr));
779 std::string pidstr = std::to_string((
long long int)g_pid_vec[ii]);
780 pidfile.push_back(
"/proc/" + pidstr +
"/stat");
782 char desc[128], ss[1024];
784 snprintf(ss,
sizeof(ss),
"CPUcmd%zd?%s?NR==1?$14+$15?1?yes", ii, pidfile[ii].c_str());
785 stats.emplace_back(ss);
787 snprintf(desc,
sizeof(desc),
"CPU+cmd%zd", ii);
788 graphs.emplace_back(desc);
789 snprintf(ss,
sizeof(ss),
"%s?%s?NR==1?$14+$15+16+$17?1?yes", desc, pidfile[ii].c_str());
790 stats.emplace_back(ss);
792 snprintf(desc,
sizeof(desc),
"WaitBlkIOcmd%zd", ii);
793 if (opt_cmd_iowait != 0)
795 graphs.emplace_back(desc);
797 snprintf(ss,
sizeof(ss),
"%s?%s?NR==1?$42?1?yes", desc, pidfile[ii].c_str());
798 stats.emplace_back(ss);
800 snprintf(desc,
sizeof(desc),
"Faultcmd%zd", ii);
803 graphs.emplace_back(desc);
805 snprintf(ss,
sizeof(ss),
"%s?%s?NR==1?$10+$11+$12+$13?4096.0/1048576?yes", desc, pidfile[ii].c_str());
806 stats.emplace_back(ss);
808 for (
size_t ii = 0; ii < opt_Cmd.size(); ++ii)
810 char cmd_file_out[1024];
811 snprintf(cmd_file_out,
sizeof(cmd_file_out),
"%speriodic_%s_%s_cmd%zd.out", opt_outdir.c_str(), run_time, hostname.c_str(), ii + opt_cmd.size());
812 int fd = open(cmd_file_out, O_WRONLY | O_CREAT, 0666);
813 TRACE(0,
"main fd=%d opt_Cmd=" + opt_Cmd[ii] +
" cmd_file_out=" + std::string(cmd_file_out), fd);
814 int iofd[3] = {0, fd, fd};
815 char *
const argv_[4] = {(
char *)
"/bin/sh",
817 (
char *)opt_Cmd[ii].c_str(),
819 g_pid_vec.push_back(fork_execv(0, 0, (
int)(period * opt_pre * 1e6), iofd,
"/bin/sh", argv_,
nullptr));
821 std::string pidstr = std::to_string((
long long int)g_pid_vec[ii]);
822 pidfile.push_back(
"/proc/" + pidstr +
"/stat");
824 char desc[128], ss[1024];
825 snprintf(desc,
sizeof(desc),
"CPU+cmd%zd", ii + opt_cmd.size());
826 snprintf(ss,
sizeof(ss),
"CPUcmd%zd?%s?NR==1?$14+$15?1?yes", ii + opt_cmd.size(), pidfile[ii].c_str());
827 stats.emplace_back(ss);
828 snprintf(ss,
sizeof(ss),
"CPU+cmd%zd?%s?NR==1?$14+$15+16+$17?1?yes", ii + opt_cmd.size(), pidfile[ii].c_str());
829 stats.emplace_back(ss);
832 std::vector<std::string> pids;
833 if (!opt_pid.empty() != 0u)
835 string_addto_vector(opt_pid, pids,
',');
837 for (
size_t ii = 0; ii < pids.size(); ++ii)
839 g_pid_vec.push_back(std::stoi(pids[ii]));
840 TRACE(1,
"pid=%s g_pid_vec.size()=%ld", pids[ii].c_str(), g_pid_vec.size());
841 pidfile.push_back(
"/proc/" + pids[ii] +
"/stat");
842 char desc[128], ss[1024];
844 snprintf(ss,
sizeof(ss),
"CPUpid%zd?%s?NR==1?$14+$15?1?yes", ii, pidfile[ii].c_str());
845 stats.emplace_back(ss);
847 std::ifstream t(
"/proc/" + pids[ii] +
"/comm");
848 std::string comm((std::istreambuf_iterator<char>(t)),
849 std::istreambuf_iterator<char>());
850 comm = comm.substr(0, comm.size() - 1);
852 snprintf(desc,
sizeof(desc),
"CPU+pid%zd_%s", ii, comm.c_str());
853 graphs.emplace_back(desc);
854 snprintf(ss,
sizeof(ss),
"%s?%s?NR==1?$14+$15+16+$17?1?yes", desc, pidfile[ii].c_str());
855 stats.emplace_back(ss);
857 snprintf(desc,
sizeof(desc),
"WaitBlkIOpid%zd", ii);
858 if (opt_cmd_iowait != 0)
860 graphs.emplace_back(desc);
862 snprintf(ss,
sizeof(ss),
"%s?%s?NR==1?$42?1?yes", desc, pidfile[ii].c_str());
863 stats.emplace_back(ss);
865 snprintf(desc,
sizeof(desc),
"Faultpid%zd", ii);
868 graphs.emplace_back(desc);
870 snprintf(ss,
sizeof(ss),
"%s?%s?NR==1?$10+$11+$12+$13?4096.0/1048576?yes", desc, pidfile[ii].c_str());
871 stats.emplace_back(ss);
874 stats.emplace_back(
"CPUnode");
875 stats.emplace_back(
"IOWait");
876 if (opt_sys_iowait != 0) { graphs.emplace_back(
"IOWait"); }
877 stats.emplace_back(
"Cached");
878 stats.emplace_back(
"Dirty");
879 stats.emplace_back(
"Free");
881 if (!opt_disk.empty() != 0u)
883 std::vector<std::string> tmp;
884 string_addto_vector(opt_disk, tmp,
',');
890 std::string statstr = dk +
"_wrMB/s?/proc/diskstats?/" + dk +
"/?$10?(1.0/2048)?yes";
891 stats.push_back(statstr);
892 std::vector<std::string> stat_spec;
893 string_addto_vector(statstr, stat_spec,
'?');
894 graphs.push_back(stat_spec[s_desc]);
896 statstr = dk +
"_rdMB/s?/proc/diskstats?/" + dk +
"/?$6?(1.0/2048)?yes";
897 stats.push_back(statstr);
899 string_addto_vector(statstr, stat_spec,
'?');
904 if (!opt_stats.empty() != 0u)
906 std::vector<std::string> tmp_stats;
907 string_addto_vector(opt_stats, tmp_stats,
',');
908 for (
auto &tmp_stat : tmp_stats)
910 stats.push_back(tmp_stat);
911 std::vector<std::string> stat_spec;
912 string_addto_vector(tmp_stat, stat_spec,
'?');
913 graphs.push_back(stat_spec[s_desc]);
917 std::vector<long> pre_vals;
918 std::vector<double> multipliers;
919 std::vector<std::vector<std::string>> spec2(stats.size());
920 std::vector<std::string> awkCmd;
922 std::string header_str(
"#" DATA_START
"\n#_______time_______");
924 int outfd = open(dat_file_out.c_str(), O_WRONLY | O_CREAT, 0777);
926 FILE *outfp = fdopen(outfd,
"w");
928 std::string cmd_comment;
929 if (!opt_cmd.empty() != 0u)
931 cmd_comment +=
"\\ncmd: " + opt_cmd[0];
933 if (!opt_comment.empty() != 0u)
935 cmd_comment +=
"\\ncomment: " + opt_comment;
937 fprintf(outfp, GNUPLOT_PREFIX, opt_ymin, opt_ymax, opt_yincr, opt_y2max, opt_y2incr, run_time, hostname.c_str(), ubuf.release, cmd_comment.c_str(),
"disk write MB/s");
939 uint64_t t_start = gettimeofday_us();
942 bool first_graph_spec_added =
false;
943 for (
size_t ii = 0; ii < stats.size(); ++ii)
945 std::vector<std::string> stat_spec;
946 string_addto_vector(stats[ii], stat_spec,
'?');
947 if (stat_spec[s_desc] ==
"CPUnode" && stat_spec.size() == 1)
950 stats[ii] +=
"?/proc/stat?/^cpu[^0-9]/?$2+$3+$4+$6+$7+$8+$9?1.0/" + std::to_string(concurentThreadsSupported) +
"?yes";
952 else if (stat_spec[s_desc] ==
"IOWait" && stat_spec.size() == 1)
954 stats[ii] +=
"?/proc/stat?/^cpu[^0-9]/?$6?1.0/" + std::to_string(concurentThreadsSupported) +
"?yes";
956 else if (stat_spec[s_desc] ==
"Cached" && stat_spec.size() == 1)
958 stats[ii] +=
"?/proc/meminfo?/^(Cached|Buffers):/?$2?1?no";
960 else if (stat_spec[s_desc] ==
"Dirty" && stat_spec.size() == 1)
962 stats[ii] +=
"?/proc/meminfo?/^Dirty:/?$2?1?no";
964 else if (stat_spec[s_desc] ==
"Free" && stat_spec.size() == 1)
966 stats[ii] +=
"?/proc/meminfo?/^MemFree:/?$2?1?no";
969 header_str +=
" " + stat_spec[s_desc];
971 string_addto_vector(stats[ii], spec2[ii],
'?');
973 snprintf(awk_cmd,
sizeof(awk_cmd),
"%s{vv+=%s}END{print vv}"
976 spec2[ii][s_linespec].c_str(), spec2[ii][s_fieldspec].c_str());
977 awkCmd.emplace_back(awk_cmd);
979 std::string stat = AWK(awkCmd.back(), spec2[ii][s_file].c_str(),
nullptr);
981 pre_vals.push_back(atol(stat.c_str()));
982 multipliers.push_back(atof(AWK(
"BEGIN{print " + spec2[ii][s_multiplier] +
"}",
"/dev/null",
nullptr).c_str()));
984 for (
const auto &graph : graphs)
986 if (graph == stat_spec[s_desc])
988 if (first_graph_spec_added)
990 fprintf(outfp,
",\\\n '' ");
992 if (strncmp(stat_spec[s_desc].c_str(),
"CPU", 3) == 0)
994 fprintf(outfp,
"using 1:%zd title '%s' w linespoints axes x1y2", ii + 2, stat_spec[s_desc].c_str());
996 else if (stat_spec[s_desc] ==
"Cached" || stat_spec[s_desc] ==
"Dirty" || stat_spec[s_desc] ==
"Free")
998 fprintf(outfp,
"using 1:($%zd/%s*100) title '%s%%' w linespoints axes x1y2", ii + 2, memKB.c_str(), stat_spec[s_desc].c_str());
1000 else if (stat_spec[s_desc].substr(0, 6) ==
"CPUcmd" || stat_spec[s_desc].substr(0, 6) ==
"CPU+cm")
1002 fprintf(outfp,
"using 1:%zd title '%s' w linespoints axes x1y2", ii + 2, stat_spec[s_desc].c_str());
1004 else if (stat_spec[s_desc].substr(0, 12) ==
"WaitBlkIOcmd")
1006 fprintf(outfp,
"using 1:%zd title '%s' w linespoints axes x1y2", ii + 2, stat_spec[s_desc].c_str());
1010 fprintf(outfp,
"using 1:%zd title '%s' w linespoints axes x1y1", ii + 2, stat_spec[s_desc].c_str());
1012 first_graph_spec_added =
true;
1016 header_str +=
" #\n";
1019 "\nif(png==0) pause -1 'Press Enter/Return or ^C to finish'\n\
1023 fprintf(outfp,
"cmds:\n");
1024 for (
const auto &ii : opt_cmd)
1026 std::string ss = ii +
"\n";
1027 fprintf(outfp,
"%s", ss.c_str());
1031 fprintf(outfp,
"stats:\n");
1032 for (
const auto &stat : stats)
1034 std::string ss = stat +
"\n";
1035 fprintf(outfp,
"%s", ss.c_str());
1039 fprintf(outfp,
"%s", header_str.c_str());
1042 std::string tmpdbg(
"main lp=%d done stat%zd=%ld rate=%f ");
1044 char proc_stats[8192];
1051 int64_t t_sleep = (t_start + (uint64_t)(period * 1e6)) - gettimeofday_us();
1054 int sts = usleep(t_sleep);
1055 TRACE(3,
"main usleep sts=%d errno=%d", sts, errno);
1063 for (lp = 2; lp < MAX_LP; ++lp)
1066 gettimeofday(&tv,
nullptr);
1067 strftime(str,
sizeof(str),
"%FT%T", localtime(&tv.tv_sec));
1069 fprintf(outfp,
"%s", str);
1070 std::string prv_file;
1071 for (
size_t ii = 0; ii < stats.size(); ++ii)
1073 TRACE(3,
"main lp=%d start stat%zd", lp, ii);
1074 char const *awk_file;
1075 if (ii < (2 * opt_cmd.size()))
1081 read_proc_file(pidfile[ii / 2].c_str(), proc_stats,
sizeof(proc_stats));
1083 awk_in = proc_stats;
1086 else if (spec2[ii][s_file] != prv_file)
1088 prv_file = spec2[ii][s_file];
1089 read_proc_file(spec2[ii][s_file].c_str(), proc_stats,
sizeof(proc_stats));
1090 awk_in = proc_stats;
1094 std::string stat_str = AWK(awkCmd[ii], awk_file, awk_in);
1096 long stat = atol(stat_str.c_str());
1098 if (spec2[ii][s_rate] ==
"yes")
1101 if (stat_str !=
"\n")
1103 rate = (stat - pre_vals[ii]) * multipliers[ii] / period;
1109 TRACE(3, tmpdbg +
"stat_str[0]=0x%x stat_str.size()=%zd", lp, ii, stat, rate, stat_str[0], stat_str.size());
1110 fprintf(outfp,
" %.2f", rate);
1111 if (rate < 0.0 && spec2[ii][s_file] ==
"/proc/diskstats")
1113 TRACE(0,
"main stat:" + spec2[ii][s_desc] +
" rate=%f pre_val=%ld stat=%ld stat_str=\"" + stat_str +
"\" awkCmd=" + awkCmd[ii] +
" proc_diskstats=" + proc_stats, rate, pre_vals[ii], stat);
1116 pre_vals[ii] = stat;
1120 TRACE(3,
"main lp=%d done stat%zd=%ld", lp, ii, stat);
1121 fprintf(outfp,
" %.2f", stat * multipliers[ii]);
1124 fprintf(outfp,
"\n");
1127 int64_t t_sleep = (t_start + (uint64_t)(period * lp * 1000000)) - gettimeofday_us();
1130 int sts = usleep(t_sleep);
1131 TRACE(3,
"main usleep sts=%d errno=%d", sts, errno);
1137 pp = check_pid_vec();
1138 TRACE(2,
"main pp=%d t_sleep=%ld", pp, t_sleep);
1141 if (post_periods_completed == 0)
1143 TRACE(1,
"main processes complete - waiting %d post periods", opt_post);
1145 if (post_periods_completed++ == opt_post)
1153 fprintf(outfp,
"# MAX_LP abort\n");
1159 TRACE(0,
"main done/complete/returning");