Skip to content

Commit 6fedcd8

Browse files
gitstergitster
committed
Merge branch 'as/test-tweaks'
Output from the tests is coloured using "green is okay, yellow is questionable, red is bad and blue is informative" scheme. * as/test-tweaks: tests: paint unexpectedly fixed known breakages in bold red tests: test the test framework more thoroughly tests: refactor mechanics of testing in a sub test-lib tests: change info messages from yellow/brown to cyan tests: paint skipped tests in blue tests: paint known breakages in yellow tests: test number comes first in 'not ok $count - $message'
2 parents fbe8aa7 + b73d9a2 commit 6fedcd8

File tree

2 files changed

+184
-59
lines changed

2 files changed

+184
-59
lines changed

‎t/t0000-basic.sh

Lines changed: 166 additions & 48 deletions
Original file line numberDiff line numberDiff line change
@@ -45,39 +45,176 @@ test_expect_failure 'pretend we have a known breakage' '
4545
false
4646
'
4747

48-
test_expect_success 'pretend we have fixed a known breakage (run in sub test-lib)' "
49-
mkdir passing-todo &&
50-
(cd passing-todo &&
51-
cat >passing-todo.sh <<-EOF &&
52-
#!$SHELL_PATH
53-
54-
test_description='A passing TODO test
48+
run_sub_test_lib_test () {
49+
name="$1" descr="$2" # stdin is the body of the test code
50+
mkdir "$name" &&
51+
(
52+
cd "$name" &&
53+
cat >"$name.sh" <<-EOF &&
54+
#!$SHELL_PATH
55+
56+
test_description='$descr (run in sub test-lib)
57+
58+
This is run in a sub test-lib so that we do not get incorrect
59+
passing metrics
60+
'
61+
62+
# Point to the t/test-lib.sh, which isn't in ../ as usual
63+
. "\$TEST_DIRECTORY"/test-lib.sh
64+
EOF
65+
cat >>"$name.sh" &&
66+
chmod +x "$name.sh" &&
67+
export TEST_DIRECTORY &&
68+
./"$name.sh" >out 2>err
69+
)
70+
}
5571

56-
This is run in a sub test-lib so that we do not get incorrect
57-
passing metrics
58-
'
72+
check_sub_test_lib_test () {
73+
name="$1" # stdin is the expected output from the test
74+
(
75+
cd "$name" &&
76+
! test -s err &&
77+
sed -e 's/^> //' -e 's/Z$//' >expect &&
78+
test_cmp expect out
79+
)
80+
}
81+
82+
test_expect_success 'pretend we have a fully passing test suite' "
83+
run_sub_test_lib_test full-pass '3 passing tests' <<-\\EOF &&
84+
for i in 1 2 3
85+
do
86+
test_expect_success \"passing test #\$i\" 'true'
87+
done
88+
test_done
89+
EOF
90+
check_sub_test_lib_test full-pass <<-\\EOF
91+
> ok 1 - passing test #1
92+
> ok 2 - passing test #2
93+
> ok 3 - passing test #3
94+
> # passed all 3 test(s)
95+
> 1..3
96+
EOF
97+
"
5998

60-
# Point to the t/test-lib.sh, which isn't in ../ as usual
61-
TEST_DIRECTORY=\"$TEST_DIRECTORY\"
62-
. \"\$TEST_DIRECTORY\"/test-lib.sh
99+
test_expect_success 'pretend we have a partially passing test suite' "
100+
test_must_fail run_sub_test_lib_test \
101+
partial-pass '2/3 tests passing' <<-\\EOF &&
102+
test_expect_success 'passing test #1' 'true'
103+
test_expect_success 'failing test #2' 'false'
104+
test_expect_success 'passing test #3' 'true'
105+
test_done
106+
EOF
107+
check_sub_test_lib_test partial-pass <<-\\EOF
108+
> ok 1 - passing test #1
109+
> not ok 2 - failing test #2
110+
# false
111+
> ok 3 - passing test #3
112+
> # failed 1 among 3 test(s)
113+
> 1..3
114+
EOF
115+
"
63116

64-
test_expect_failure 'pretend we have fixed a known breakage' '
65-
:
66-
'
117+
test_expect_success 'pretend we have a known breakage' "
118+
run_sub_test_lib_test failing-todo 'A failing TODO test' <<-\\EOF &&
119+
test_expect_success 'passing test' 'true'
120+
test_expect_failure 'pretend we have a known breakage' 'false'
121+
test_done
122+
EOF
123+
check_sub_test_lib_test failing-todo <<-\\EOF
124+
> ok 1 - passing test
125+
> not ok 2 - pretend we have a known breakage # TODO known breakage
126+
> # still have 1 known breakage(s)
127+
> # passed all remaining 1 test(s)
128+
> 1..2
129+
EOF
130+
"
67131

132+
test_expect_success 'pretend we have fixed a known breakage' "
133+
run_sub_test_lib_test passing-todo 'A passing TODO test' <<-\\EOF &&
134+
test_expect_failure 'pretend we have fixed a known breakage' 'true'
68135
test_done
69136
EOF
70-
chmod +x passing-todo.sh &&
71-
./passing-todo.sh >out 2>err &&
72-
! test -s err &&
73-
sed -e 's/^> //' >expect <<-\\EOF &&
74-
> ok 1 - pretend we have fixed a known breakage # TODO known breakage
75-
> # fixed 1 known breakage(s)
76-
> # passed all 1 test(s)
137+
check_sub_test_lib_test passing-todo <<-\\EOF
138+
> ok 1 - pretend we have fixed a known breakage # TODO known breakage vanished
139+
> # 1 known breakage(s) vanished; please update test(s)
77140
> 1..1
78141
EOF
79-
test_cmp expect out)
80142
"
143+
144+
test_expect_success 'pretend we have fixed one of two known breakages (run in sub test-lib)' "
145+
run_sub_test_lib_test partially-passing-todos \
146+
'2 TODO tests, one passing' <<-\\EOF &&
147+
test_expect_failure 'pretend we have a known breakage' 'false'
148+
test_expect_success 'pretend we have a passing test' 'true'
149+
test_expect_failure 'pretend we have fixed another known breakage' 'true'
150+
test_done
151+
EOF
152+
check_sub_test_lib_test partially-passing-todos <<-\\EOF
153+
> not ok 1 - pretend we have a known breakage # TODO known breakage
154+
> ok 2 - pretend we have a passing test
155+
> ok 3 - pretend we have fixed another known breakage # TODO known breakage vanished
156+
> # 1 known breakage(s) vanished; please update test(s)
157+
> # still have 1 known breakage(s)
158+
> # passed all remaining 1 test(s)
159+
> 1..3
160+
EOF
161+
"
162+
163+
test_expect_success 'pretend we have a pass, fail, and known breakage' "
164+
test_must_fail run_sub_test_lib_test \
165+
mixed-results1 'mixed results #1' <<-\\EOF &&
166+
test_expect_success 'passing test' 'true'
167+
test_expect_success 'failing test' 'false'
168+
test_expect_failure 'pretend we have a known breakage' 'false'
169+
test_done
170+
EOF
171+
check_sub_test_lib_test mixed-results1 <<-\\EOF
172+
> ok 1 - passing test
173+
> not ok 2 - failing test
174+
> # false
175+
> not ok 3 - pretend we have a known breakage # TODO known breakage
176+
> # still have 1 known breakage(s)
177+
> # failed 1 among remaining 2 test(s)
178+
> 1..3
179+
EOF
180+
"
181+
182+
test_expect_success 'pretend we have a mix of all possible results' "
183+
test_must_fail run_sub_test_lib_test \
184+
mixed-results2 'mixed results #2' <<-\\EOF &&
185+
test_expect_success 'passing test' 'true'
186+
test_expect_success 'passing test' 'true'
187+
test_expect_success 'passing test' 'true'
188+
test_expect_success 'passing test' 'true'
189+
test_expect_success 'failing test' 'false'
190+
test_expect_success 'failing test' 'false'
191+
test_expect_success 'failing test' 'false'
192+
test_expect_failure 'pretend we have a known breakage' 'false'
193+
test_expect_failure 'pretend we have a known breakage' 'false'
194+
test_expect_failure 'pretend we have fixed a known breakage' 'true'
195+
test_done
196+
EOF
197+
check_sub_test_lib_test mixed-results2 <<-\\EOF
198+
> ok 1 - passing test
199+
> ok 2 - passing test
200+
> ok 3 - passing test
201+
> ok 4 - passing test
202+
> not ok 5 - failing test
203+
> # false
204+
> not ok 6 - failing test
205+
> # false
206+
> not ok 7 - failing test
207+
> # false
208+
> not ok 8 - pretend we have a known breakage # TODO known breakage
209+
> not ok 9 - pretend we have a known breakage # TODO known breakage
210+
> ok 10 - pretend we have fixed a known breakage # TODO known breakage vanished
211+
> # 1 known breakage(s) vanished; please update test(s)
212+
> # still have 2 known breakage(s)
213+
> # failed 3 among remaining 7 test(s)
214+
> 1..10
215+
EOF
216+
"
217+
81218
test_set_prereq HAVEIT
82219
haveit=no
83220
test_expect_success HAVEIT 'test runs if prerequisite is satisfied' '
@@ -159,19 +296,8 @@ then
159296
fi
160297

161298
test_expect_success 'tests clean up even on failures' "
162-
mkdir failing-cleanup &&
163-
(
164-
cd failing-cleanup &&
165-
166-
cat >failing-cleanup.sh <<-EOF &&
167-
#!$SHELL_PATH
168-
169-
test_description='Failing tests with cleanup commands'
170-
171-
# Point to the t/test-lib.sh, which isn't in ../ as usual
172-
TEST_DIRECTORY=\"$TEST_DIRECTORY\"
173-
. \"\$TEST_DIRECTORY\"/test-lib.sh
174-
299+
test_must_fail run_sub_test_lib_test \
300+
failing-cleanup 'Failing tests with cleanup commands' <<-\\EOF &&
175301
test_expect_success 'tests clean up even after a failure' '
176302
touch clean-after-failure &&
177303
test_when_finished rm clean-after-failure &&
@@ -181,29 +307,21 @@ test_expect_success 'tests clean up even on failures' "
181307
test_when_finished \"(exit 2)\"
182308
'
183309
test_done
184-
185310
EOF
186-
187-
chmod +x failing-cleanup.sh &&
188-
test_must_fail ./failing-cleanup.sh >out 2>err &&
189-
! test -s err &&
190-
! test -f \"trash directory.failing-cleanup/clean-after-failure\" &&
191-
sed -e 's/Z$//' -e 's/^> //' >expect <<-\\EOF &&
192-
> not ok - 1 tests clean up even after a failure
311+
check_sub_test_lib_test failing-cleanup <<-\\EOF
312+
> not ok 1 - tests clean up even after a failure
193313
> # Z
194314
> # touch clean-after-failure &&
195315
> # test_when_finished rm clean-after-failure &&
196316
> # (exit 1)
197317
> # Z
198-
> not ok - 2 failure to clean up causes the test to fail
318+
> not ok 2 - failure to clean up causes the test to fail
199319
> # Z
200320
> # test_when_finished \"(exit 2)\"
201321
> # Z
202322
> # failed 2 among 2 test(s)
203323
> 1..2
204324
EOF
205-
test_cmp expect out
206-
)
207325
"
208326

209327
################################################################

‎t/test-lib.sh

Lines changed: 18 additions & 11 deletions
Original file line numberDiff line numberDiff line change
@@ -212,11 +212,13 @@ then
212212
error)
213213
tput bold; tput setaf 1;; # bold red
214214
skip)
215-
tput bold; tput setaf 2;; # bold green
215+
tput setaf 4;; # blue
216+
warn)
217+
tput setaf 3;; # brown/yellow
216218
pass)
217-
tput setaf 2;; # green
219+
tput setaf 2;; # green
218220
info)
219-
tput setaf 3;; # brown
221+
tput setaf 6;; # cyan
220222
*)
221223
test -n "$quiet" && return;;
222224
esac
@@ -298,20 +300,20 @@ test_ok_ () {
298300

299301
test_failure_ () {
300302
test_failure=$(($test_failure + 1))
301-
say_color error "not ok - $test_count $1"
303+
say_color error "not ok $test_count - $1"
302304
shift
303305
echo "$@" | sed -e 's/^/# /'
304306
test "$immediate" = "" || { GIT_EXIT_OK=t; exit 1; }
305307
}
306308

307309
test_known_broken_ok_ () {
308310
test_fixed=$(($test_fixed+1))
309-
say_color "" "ok $test_count - $@ # TODO known breakage"
311+
say_color error "ok $test_count - $@ # TODO known breakage vanished"
310312
}
311313

312314
test_known_broken_failure_ () {
313315
test_broken=$(($test_broken+1))
314-
say_color skip "not ok $test_count - $@ # TODO known breakage"
316+
say_color warn "not ok $test_count - $@ # TODO known breakage"
315317
}
316318

317319
test_debug () {
@@ -404,13 +406,18 @@ test_done () {
404406

405407
if test "$test_fixed" != 0
406408
then
407-
say_color pass "# fixed $test_fixed known breakage(s)"
409+
say_color error "# $test_fixed known breakage(s) vanished; please update test(s)"
408410
fi
409411
if test "$test_broken" != 0
410412
then
411-
say_color error "# still have $test_broken known breakage(s)"
412-
msg="remaining $(($test_count-$test_broken)) test(s)"
413+
say_color warn "# still have $test_broken known breakage(s)"
414+
fi
415+
if test "$test_broken" != 0 || test "$test_fixed" != 0
416+
then
417+
test_remaining=$(( $test_count - $test_broken - $test_fixed ))
418+
msg="remaining $test_remaining test(s)"
413419
else
420+
test_remaining=$test_count
414421
msg="$test_count test(s)"
415422
fi
416423
case "$test_failure" in
@@ -424,7 +431,7 @@ test_done () {
424431

425432
if test $test_external_has_tap -eq 0
426433
then
427-
if test $test_count -gt 0
434+
if test $test_remaining -gt 0
428435
then
429436
say_color pass "# passed all $msg"
430437
fi
@@ -615,7 +622,7 @@ for skp in $GIT_SKIP_TESTS
615622
do
616623
case "$this_test" in
617624
$skp)
618-
say_color skip >&3 "skipping test $this_test altogether"
625+
say_color info >&3 "skipping test $this_test altogether"
619626
skip_all="skip all tests in $this_test"
620627
test_done
621628
esac

0 commit comments

Comments
 (0)