Skip to content

Commit 0264531

Browse files
gitstergitster
committed
Merge branch 'cc/perf-bisect'
Performance measuring framework in t/perf learned to help bisecting performance regressions. * cc/perf-bisect: t/perf: add scripts to bisect performance regressions perf/run: add --subsection option
2 parents 7a79d7e + 297e685 commit 0264531

File tree

3 files changed

+166
-10
lines changed

3 files changed

+166
-10
lines changed

‎t/perf/bisect_regression

Lines changed: 73 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,73 @@
1+
#!/bin/sh
2+
3+
# Read a line coming from `./aggregate.perl --sort-by regression ...`
4+
# and automatically bisect to find the commit responsible for the
5+
# performance regression.
6+
#
7+
# Lines from `./aggregate.perl --sort-by regression ...` look like:
8+
#
9+
# +100.0% p7821-grep-engines-fixed.1 0.04(0.10+0.03) 0.08(0.11+0.08) v2.14.3 v2.15.1
10+
# +33.3% p7820-grep-engines.1 0.03(0.08+0.02) 0.04(0.08+0.02) v2.14.3 v2.15.1
11+
#
12+
13+
die () {
14+
echo >&2 "error: $*"
15+
exit 1
16+
}
17+
18+
while [ $# -gt 0 ]; do
19+
arg="$1"
20+
case "$arg" in
21+
--help)
22+
echo "usage: $0 [--config file] [--subsection subsection]"
23+
exit 0
24+
;;
25+
--config)
26+
shift
27+
GIT_PERF_CONFIG_FILE=$(cd "$(dirname "$1")"; pwd)/$(basename "$1")
28+
export GIT_PERF_CONFIG_FILE
29+
shift ;;
30+
--subsection)
31+
shift
32+
GIT_PERF_SUBSECTION="$1"
33+
export GIT_PERF_SUBSECTION
34+
shift ;;
35+
--*)
36+
die "unrecognised option: '$arg'" ;;
37+
*)
38+
die "unknown argument '$arg'"
39+
;;
40+
esac
41+
done
42+
43+
read -r regression subtest oldtime newtime oldrev newrev
44+
45+
test_script=$(echo "$subtest" | sed -e 's/\(.*\)\.[0-9]*$/\1.sh/')
46+
test_number=$(echo "$subtest" | sed -e 's/.*\.\([0-9]*\)$/\1/')
47+
48+
# oldtime and newtime are decimal number, not integers
49+
50+
oldtime=$(echo "$oldtime" | sed -e 's/^\([0-9]\+\.[0-9]\+\).*$/\1/')
51+
newtime=$(echo "$newtime" | sed -e 's/^\([0-9]\+\.[0-9]\+\).*$/\1/')
52+
53+
test $(echo "$newtime" "$oldtime" | awk '{ print ($1 > $2) }') = 1 ||
54+
die "New time '$newtime' shoud be greater than old time '$oldtime'"
55+
56+
tmpdir=$(mktemp -d -t bisect_regression_XXXXXX) || die "Failed to create temp directory"
57+
echo "$oldtime" >"$tmpdir/oldtime" || die "Failed to write to '$tmpdir/oldtime'"
58+
echo "$newtime" >"$tmpdir/newtime" || die "Failed to write to '$tmpdir/newtime'"
59+
60+
# Bisecting must be performed from the top level directory (even with --no-checkout)
61+
(
62+
toplevel_dir=$(git rev-parse --show-toplevel) || die "Failed to find top level directory"
63+
cd "$toplevel_dir" || die "Failed to cd into top level directory '$toplevel_dir'"
64+
65+
git bisect start --no-checkout "$newrev" "$oldrev" || die "Failed to start bisecting"
66+
67+
git bisect run t/perf/bisect_run_script "$test_script" "$test_number" "$tmpdir"
68+
res="$?"
69+
70+
git bisect reset
71+
72+
exit "$res"
73+
)

‎t/perf/bisect_run_script

Lines changed: 47 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,47 @@
1+
#!/bin/sh
2+
3+
script="$1"
4+
test_number="$2"
5+
info_dir="$3"
6+
7+
# This aborts the bisection immediately
8+
die () {
9+
echo >&2 "error: $*"
10+
exit 255
11+
}
12+
13+
bisect_head=$(git rev-parse --verify BISECT_HEAD) || die "Failed to find BISECT_HEAD ref"
14+
15+
script_number=$(echo "$script" | sed -e "s/^p\([0-9]*\).*\$/\1/") || die "Failed to get script number for '$script'"
16+
17+
oldtime=$(cat "$info_dir/oldtime") || die "Failed to access '$info_dir/oldtime'"
18+
newtime=$(cat "$info_dir/newtime") || die "Failed to access '$info_dir/newtime'"
19+
20+
cd t/perf || die "Failed to cd into 't/perf'"
21+
22+
result_file="$info_dir/perf_${script_number}_${bisect_head}_results.txt"
23+
24+
GIT_PERF_DIRS_OR_REVS="$bisect_head"
25+
export GIT_PERF_DIRS_OR_REVS
26+
27+
./run "$script" >"$result_file" 2>&1 || die "Failed to run perf test '$script'"
28+
29+
rtime=$(sed -n "s/^$script_number\.$test_number:.*\([0-9]\+\.[0-9]\+\)(.*).*\$/\1/p" "$result_file")
30+
31+
echo "newtime: $newtime"
32+
echo "rtime: $rtime"
33+
echo "oldtime: $oldtime"
34+
35+
# Compare ($newtime - $rtime) with ($rtime - $oldtime)
36+
# Times are decimal number, not integers
37+
38+
if test $(echo "$newtime" "$rtime" "$oldtime" | awk '{ print ($1 - $2 > $2 - $3) }') = 1
39+
then
40+
# Current commit is considered "good/old"
41+
echo "$rtime" >"$info_dir/oldtime"
42+
exit 0
43+
else
44+
# Current commit is considered "bad/new"
45+
echo "$rtime" >"$info_dir/newtime"
46+
exit 1
47+
fi

‎t/perf/run

Lines changed: 46 additions & 10 deletions
Original file line numberDiff line numberDiff line change
@@ -1,21 +1,34 @@
11
#!/bin/sh
22

3-
case "$1" in
3+
die () {
4+
echo >&2 "error: $*"
5+
exit 1
6+
}
7+
8+
while [ $# -gt 0 ]; do
9+
arg="$1"
10+
case "$arg" in
11+
--)
12+
break ;;
413
--help)
5-
echo "usage: $0 [--config file] [other_git_tree...] [--] [test_scripts]"
6-
exit 0
7-
;;
14+
echo "usage: $0 [--config file] [--subsection subsec] [other_git_tree...] [--] [test_scripts]"
15+
exit 0 ;;
816
--config)
917
shift
1018
GIT_PERF_CONFIG_FILE=$(cd "$(dirname "$1")"; pwd)/$(basename "$1")
1119
export GIT_PERF_CONFIG_FILE
1220
shift ;;
13-
esac
14-
15-
die () {
16-
echo >&2 "error: $*"
17-
exit 1
18-
}
21+
--subsection)
22+
shift
23+
GIT_PERF_SUBSECTION="$1"
24+
export GIT_PERF_SUBSECTION
25+
shift ;;
26+
--*)
27+
die "unrecognised option: '$arg'" ;;
28+
*)
29+
break ;;
30+
esac
31+
done
1932

2033
run_one_dir () {
2134
if test $# -eq 0; then
@@ -172,9 +185,32 @@ get_subsections "perf" >test-results/run_subsections.names
172185

173186
if test $(wc -l <test-results/run_subsections.names) -eq 0
174187
then
188+
if test -n "$GIT_PERF_SUBSECTION"
189+
then
190+
if test -n "$GIT_PERF_CONFIG_FILE"
191+
then
192+
die "no subsections are defined in config file '$GIT_PERF_CONFIG_FILE'"
193+
else
194+
die "subsection '$GIT_PERF_SUBSECTION' defined without a config file"
195+
fi
196+
fi
175197
(
176198
run_subsection "$@"
177199
)
200+
elif test -n "$GIT_PERF_SUBSECTION"
201+
then
202+
egrep "^$GIT_PERF_SUBSECTION\$" test-results/run_subsections.names >/dev/null ||
203+
die "subsection '$GIT_PERF_SUBSECTION' not found in '$GIT_PERF_CONFIG_FILE'"
204+
205+
egrep "^$GIT_PERF_SUBSECTION\$" test-results/run_subsections.names | while read -r subsec
206+
do
207+
(
208+
GIT_PERF_SUBSECTION="$subsec"
209+
export GIT_PERF_SUBSECTION
210+
echo "======== Run for subsection '$GIT_PERF_SUBSECTION' ========"
211+
run_subsection "$@"
212+
)
213+
done
178214
else
179215
while read -r subsec
180216
do

0 commit comments

Comments
 (0)