From 8eeebc69f554b580e4a56fc4f0e9e06110ed7c4f Mon Sep 17 00:00:00 2001 From: Jared Still Date: Thu, 3 Dec 2020 14:56:51 -0800 Subject: [PATCH 1/6] changed "#!/usr/bin/env python" to "#!/usr/bin/env python2" --- bin/psn | 2 +- bin/schedlat | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/bin/psn b/bin/psn index dfaca6e..99eceea 100755 --- a/bin/psn +++ b/bin/psn @@ -1,4 +1,4 @@ -#!/usr/bin/env python +#!/usr/bin/env python2 # use the following line on RHEL5 (if you have the additional python26 package installed from EPEL): ##!/usr/bin/env python26 diff --git a/bin/schedlat b/bin/schedlat index 147d65b..6df3368 100755 --- a/bin/schedlat +++ b/bin/schedlat @@ -1,4 +1,4 @@ -#!/usr/bin/env python +#!/usr/bin/env python2 # Copyright 2020 Tanel Poder # Licensed under the Apache License, Version 2.0 (the "License") From 7961dbf13df9a6e996ee63f8e8e0999ff0549a0f Mon Sep 17 00:00:00 2001 From: Jared Still Date: Thu, 3 Dec 2020 15:26:18 -0800 Subject: [PATCH 2/6] added directory check --- bin/run_xcapture.sh | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/bin/run_xcapture.sh b/bin/run_xcapture.sh index 2597433..eb47f65 100755 --- a/bin/run_xcapture.sh +++ b/bin/run_xcapture.sh @@ -21,6 +21,11 @@ if [ $# -ne 1 ]; then exit 1 fi +if [ ! -d "$1" ]; then + echo "Directory '$1' does not exist" + exit 2 +fi + SUDO=sudo # change to empty string if running without sudo NICE=-5 # set to 0 if don't want to increase priority SLEEP=60 @@ -28,7 +33,7 @@ SLEEP=60 logger "$0 Starting up outdir=$1 nice=$NICE" while true ; do - $SUDO nice -n $NICE xcapture -o $1 -c exe,cmdline,kstack + $SUDO nice -n $NICE ./xcapture -o $1 -c exe,cmdline,kstack # we only get here should xcapture be terminated, try to restart logger "$0 terminated with $?, attempting to restart in $SLEEP seconds" From 9c9fcbb9836ce01d530979351be329cd444baffa Mon Sep 17 00:00:00 2001 From: Jared Still Date: Fri, 4 Dec 2020 07:23:00 -0800 Subject: [PATCH 3/6] examples for querying the CSV files --- Query-Examples.md | 89 +++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 89 insertions(+) create mode 100644 Query-Examples.md diff --git a/Query-Examples.md b/Query-Examples.md new file mode 100644 index 0000000..6adf467 --- /dev/null +++ b/Query-Examples.md @@ -0,0 +1,89 @@ + +Query Examples +============== + +As per [Always-on Profiling for Production Systems](https://0x.tools/), the primary tools for querying the CSV files are: + +- grep +- awk +- cut +- sort +- uniq + +While grep is primarily used for filtering, this can be done with awk as well. + +Both awk and cut can be used to project columns. + +The uniq utility is useful for group by, but also for sorting when counts are involved. + + +This command gets the count of occurrences of oraagent.bin, showing which is the most active. + +```text +# awk -F, '/oraagent.bin/ { print $2 }' /var/log/xcapture/2020-12-03.15.csv | sort -n | uniq -c | sort -n +1 26652 +80 3131 +101 1543 +123 13048 +``` + +This same thing may also be done in awk: + +```text +# awk -F, '/oraagent.bin/ { freq[$2]++ }END{for (key in freq){print freq[key] ": " key}}' /var/log/xcapture/2020-12-03.15.csv | sort -n +1: 26652 +80: 3131 +101: 1543 +122: 13048 +``` + +Here we can see the uniq list of call stacks for PID, along with the count for that stack: + +```text +# awk -F, '/oraagent.bin/ && $2==13048 && $7=="[running]" && $11 != "" { print $11 }' /var/log/xcapture/2020-12-03.15.csv | sort | uniq -c | sort -n + 1 ->retint_user()->prepare_exit_to_usermode()->exit_to_usermode_loop() + 3 ->SyS_nanosleep()->hrtimer_nanosleep() + 3 ->SyS_poll()->do_sys_poll()->poll_schedule_timeout() + 6 ->page_fault()->do_page_fault()->__do_page_fault()->call_rwsem_down_read_failed() + 8 ->SyS_read()->vfs_read()->__vfs_read()->pipe_read()->pipe_wait() + 90 ->SyS_futex()->do_futex()->futex_wait()->futex_wait_queue_me() +``` + +Breakdown for this command line + +`awk -F, '/oraagent.bin/ && $2==13048 && $7=="[running]" && $11 != "" { print $11 }' /var/log/xcapture/2020-12-03.15.csv | sort | uniq -c | sort -n` + +get specify the field separator as a comma + `awk -F, ` + +get only lines that contain oraagent.bin + `'/oraagent.bin/ ` + +continue if the PID = 13048 + `&& $2==13048 ` + +continue if the SYSCALL = '[running]' + `&& $7=="[running]"` + +continue if KSTACK is not empty + `&& $11 != "" ` + +print the KSTACK + `{ print $11 }' ` + +this is the input file + `/var/log/xcapture/2020-12-03.15.csv ` + +sort the output + `| sort` + +get a count per unique output + `| uniq -c ` + +sort by the count of items + `| sort -n` + + +Should you want to do more complex transformations to the data, you may want to use Perl or similar. + + From 281585d03a5bbce958cdea0a95cf8741064452c2 Mon Sep 17 00:00:00 2001 From: Jared Still Date: Fri, 4 Dec 2020 07:50:04 -0800 Subject: [PATCH 4/6] check for directory existence --- bin/run_xcpu.sh | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/bin/run_xcpu.sh b/bin/run_xcpu.sh index 934f4bd..c379520 100755 --- a/bin/run_xcpu.sh +++ b/bin/run_xcpu.sh @@ -27,6 +27,11 @@ if [ $# -ne 1 ]; then exit 1 fi +if [ ! -d "$1" ]; then + echo "Directory '$1' does not exist" + exit 2 +fi + logger "$0 Starting up outdir=$1 nice=$NICE" while true ; do From 3ec9c4ebbbea1e8062adff05bdcdaa89a46ba9fa Mon Sep 17 00:00:00 2001 From: Jared Still Date: Fri, 4 Dec 2020 12:09:04 -0800 Subject: [PATCH 5/6] dynamically locate path to perf, exit if not installed --- bin/run_xcpu.sh | 10 +++++++++- 1 file changed, 9 insertions(+), 1 deletion(-) diff --git a/bin/run_xcpu.sh b/bin/run_xcpu.sh index c379520..162c55f 100755 --- a/bin/run_xcpu.sh +++ b/bin/run_xcpu.sh @@ -20,7 +20,15 @@ FREQUENCY=1 # 1 Hz sampling SUDO=sudo # change to empty string if running without sudo NICE=-5 SLEEP=60 -PERF=/usr/bin/perf +PERF=$(which perf) + +[ -x "$PERF" ] || { + echo + echo "perf is not installed" + echo + exit 1 +} + if [ $# -ne 1 ]; then echo "Usage: $0 output_dir" From b3b4926f303fee4655a695be03e97778fae7bcc2 Mon Sep 17 00:00:00 2001 From: Jared Still Date: Tue, 28 Mar 2023 16:59:59 -0700 Subject: [PATCH 6/6] check for dir, use python2 --- bin/psn | 7 +++++-- bin/run_xcapture.sh | 5 +++++ 2 files changed, 10 insertions(+), 2 deletions(-) mode change 100644 => 100755 bin/psn mode change 100644 => 100755 bin/run_xcapture.sh diff --git a/bin/psn b/bin/psn old mode 100644 new mode 100755 index 1aaee50..8bfdbe2 --- a/bin/psn +++ b/bin/psn @@ -1,5 +1,8 @@ -#!/usr/bin/env python -# +#!/usr/bin/env python2 + +# use the following line on RHEL5 (if you have the additional python26 package installed from EPEL): +##!/usr/bin/env python26 + # psn -- Linux Process Snapper by Tanel Poder [https://0x.tools] # Copyright 2019-2021 Tanel Poder # diff --git a/bin/run_xcapture.sh b/bin/run_xcapture.sh old mode 100644 new mode 100755 index 7506312..3f23df5 --- a/bin/run_xcapture.sh +++ b/bin/run_xcapture.sh @@ -22,6 +22,11 @@ if [ $# -ne 1 ]; then exit 1 fi +if [ ! -d "$1" ]; then + echo "Directory '$1' does not exist" + exit 2 +fi + SUDO=sudo # change to empty string if running without sudo NICE=-5 # set to 0 if don't want to increase priority SLEEP=60