Skip to content

Commit 2daf49f

Browse files
authored
Merge pull request #613 from TypedDevs/fix/dx-silent-test-passes
fix(runner): improve DX for silent test failures
2 parents d2ed146 + 8af0db9 commit 2daf49f

21 files changed

+317
-20
lines changed

CHANGELOG.md

Lines changed: 6 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -2,8 +2,14 @@
22

33
## Unreleased
44

5+
### Added
6+
- Add risky test detection for tests with zero assertions (shown as warning, does not fail)
7+
58
### Fixed
69
- Fix `source` of non-existent file in `set_up()` silently passing all tests (#611)
10+
- Fix `set_up` running before strict mode — unbound variables in hooks now detected with `--strict`
11+
- Fix `source` failure in `tear_down()`, `set_up_before_script()`, and `tear_down_after_script()` silently passing
12+
- Add missing runtime error patterns: ambiguous redirect, integer expression expected, too many arguments, value too great, not a valid identifier, unexpected EOF
713

814
## [0.34.0](https://github.com/TypedDevs/bashunit/compare/0.33.0...0.34.0) - 2026-03-17
915

src/assert_dates.sh

Lines changed: 19 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -12,13 +12,31 @@ function bashunit::date::to_epoch() {
1212
;;
1313
esac
1414

15+
# Normalize ISO 8601: replace T with space, strip Z suffix, strip tz offset
16+
local normalized="$input"
17+
normalized="${normalized/T/ }"
18+
normalized="${normalized%Z}"
19+
# Strip timezone offset (+HHMM or -HHMM) at end for initial parsing
20+
case "$normalized" in
21+
*[+-][0-9][0-9][0-9][0-9])
22+
normalized="${normalized%[+-][0-9][0-9][0-9][0-9]}"
23+
;;
24+
esac
25+
1526
# Format conversion (GNU vs BSD date)
1627
local epoch
17-
# Try GNU date first (-d flag)
28+
# Try GNU date first (-d flag) with original input
1829
epoch=$(date -d "$input" +%s 2>/dev/null) && {
1930
echo "$epoch"
2031
return 0
2132
}
33+
# Try GNU date with normalized (space-separated) input
34+
if [[ "$normalized" != "$input" ]]; then
35+
epoch=$(date -d "$normalized" +%s 2>/dev/null) && {
36+
echo "$epoch"
37+
return 0
38+
}
39+
fi
2240
# Try BSD date (-j -f flag) with ISO 8601 datetime + timezone offset
2341
epoch=$(date -j -f "%Y-%m-%dT%H:%M:%S%z" "$input" +%s 2>/dev/null) && {
2442
echo "$epoch"

src/colors.sh

Lines changed: 4 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -27,11 +27,13 @@ if bashunit::env::is_no_color_enabled; then
2727
_BASHUNIT_COLOR_SKIPPED=""
2828
_BASHUNIT_COLOR_INCOMPLETE=""
2929
_BASHUNIT_COLOR_SNAPSHOT=""
30+
_BASHUNIT_COLOR_RISKY=""
3031
_BASHUNIT_COLOR_RETURN_ERROR=""
3132
_BASHUNIT_COLOR_RETURN_SUCCESS=""
3233
_BASHUNIT_COLOR_RETURN_SKIPPED=""
3334
_BASHUNIT_COLOR_RETURN_INCOMPLETE=""
3435
_BASHUNIT_COLOR_RETURN_SNAPSHOT=""
36+
_BASHUNIT_COLOR_RETURN_RISKY=""
3537
_BASHUNIT_COLOR_DEFAULT=""
3638
else
3739
_BASHUNIT_COLOR_BOLD="$(bashunit::sgr 1)"
@@ -42,10 +44,12 @@ else
4244
_BASHUNIT_COLOR_SKIPPED="$(bashunit::sgr 33)"
4345
_BASHUNIT_COLOR_INCOMPLETE="$(bashunit::sgr 36)"
4446
_BASHUNIT_COLOR_SNAPSHOT="$(bashunit::sgr 34)"
47+
_BASHUNIT_COLOR_RISKY="$(bashunit::sgr 35)"
4548
_BASHUNIT_COLOR_RETURN_ERROR="$(bashunit::sgr 41)$_BASHUNIT_COLOR_BLACK$_BASHUNIT_COLOR_BOLD"
4649
_BASHUNIT_COLOR_RETURN_SUCCESS="$(bashunit::sgr 42)$_BASHUNIT_COLOR_BLACK$_BASHUNIT_COLOR_BOLD"
4750
_BASHUNIT_COLOR_RETURN_SKIPPED="$(bashunit::sgr 43)$_BASHUNIT_COLOR_BLACK$_BASHUNIT_COLOR_BOLD"
4851
_BASHUNIT_COLOR_RETURN_INCOMPLETE="$(bashunit::sgr 46)$_BASHUNIT_COLOR_BLACK$_BASHUNIT_COLOR_BOLD"
4952
_BASHUNIT_COLOR_RETURN_SNAPSHOT="$(bashunit::sgr 44)$_BASHUNIT_COLOR_BLACK$_BASHUNIT_COLOR_BOLD"
53+
_BASHUNIT_COLOR_RETURN_RISKY="$(bashunit::sgr 45)$_BASHUNIT_COLOR_BLACK$_BASHUNIT_COLOR_BOLD"
5054
_BASHUNIT_COLOR_DEFAULT="$(bashunit::sgr 0)"
5155
fi

src/console_results.sh

Lines changed: 50 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -30,6 +30,7 @@ function bashunit::console_results::render_result() {
3030
local tests_incomplete=$_BASHUNIT_TESTS_INCOMPLETE
3131
local tests_snapshot=$_BASHUNIT_TESTS_SNAPSHOT
3232
local tests_failed=$_BASHUNIT_TESTS_FAILED
33+
local tests_risky=$_BASHUNIT_TESTS_RISKY
3334
local assertions_passed=$_BASHUNIT_ASSERTIONS_PASSED
3435
local assertions_skipped=$_BASHUNIT_ASSERTIONS_SKIPPED
3536
local assertions_incomplete=$_BASHUNIT_ASSERTIONS_INCOMPLETE
@@ -42,6 +43,7 @@ function bashunit::console_results::render_result() {
4243
total_tests=$((total_tests + tests_incomplete))
4344
total_tests=$((total_tests + tests_snapshot))
4445
total_tests=$((total_tests + tests_failed))
46+
total_tests=$((total_tests + tests_risky))
4547

4648
local total_assertions=0
4749
total_assertions=$((total_assertions + assertions_passed))
@@ -66,6 +68,9 @@ function bashunit::console_results::render_result() {
6668
if [[ "$tests_failed" -gt 0 ]] || [[ "$assertions_failed" -gt 0 ]]; then
6769
printf " %s%s failed%s," "$_BASHUNIT_COLOR_FAILED" "$tests_failed" "$_BASHUNIT_COLOR_DEFAULT"
6870
fi
71+
if [[ "$tests_risky" -gt 0 ]]; then
72+
printf " %s%s risky%s," "$_BASHUNIT_COLOR_RISKY" "$tests_risky" "$_BASHUNIT_COLOR_DEFAULT"
73+
fi
6974
printf " %s total\n" "$total_tests"
7075

7176
printf "%sAssertions:%s" "$_BASHUNIT_COLOR_FAINT" "$_BASHUNIT_COLOR_DEFAULT"
@@ -92,6 +97,12 @@ function bashunit::console_results::render_result() {
9297
return 1
9398
fi
9499

100+
if [[ "$tests_risky" -gt 0 ]]; then
101+
printf "\n%s%s%s\n" "$_BASHUNIT_COLOR_RETURN_RISKY" " Some tests risky (no assertions) " "$_BASHUNIT_COLOR_DEFAULT"
102+
bashunit::console_results::print_execution_time
103+
return 0
104+
fi
105+
95106
if [[ "$tests_incomplete" -gt 0 ]]; then
96107
printf "\n%s%s%s\n" "$_BASHUNIT_COLOR_RETURN_INCOMPLETE" " Some tests incomplete " "$_BASHUNIT_COLOR_DEFAULT"
97108
bashunit::console_results::print_execution_time
@@ -359,6 +370,23 @@ function bashunit::console_results::print_snapshot_test() {
359370
bashunit::state::print_line "snapshot" "$line"
360371
}
361372

373+
function bashunit::console_results::print_risky_test() {
374+
local test_name=$1
375+
local duration=${2:-"0"}
376+
377+
local line
378+
line=$(printf "%s⚠ Risky%s: %s" "$_BASHUNIT_COLOR_RISKY" "$_BASHUNIT_COLOR_DEFAULT" "$test_name")
379+
380+
local full_line=$line
381+
if bashunit::env::is_show_execution_time_enabled; then
382+
local time_display
383+
time_display=$(bashunit::console_results::format_duration "$duration")
384+
full_line="$(printf "%s\n" "$(bashunit::str::rpad "$line" "$time_display")")"
385+
fi
386+
387+
bashunit::state::print_line "risky" "$full_line"
388+
}
389+
362390
function bashunit::console_results::print_error_test() {
363391
local function_name=$1
364392
local error="$2"
@@ -447,3 +475,25 @@ function bashunit::console_results::print_incomplete_tests_and_reset() {
447475
echo ""
448476
fi
449477
}
478+
479+
function bashunit::console_results::print_risky_tests_and_reset() {
480+
if [[ -s "$RISKY_OUTPUT_PATH" ]]; then
481+
local total_risky
482+
total_risky=$(bashunit::state::get_tests_risky)
483+
484+
if bashunit::env::is_simple_output_enabled; then
485+
printf "\n"
486+
fi
487+
488+
if [[ "$total_risky" -eq 1 ]]; then
489+
echo -e "${_BASHUNIT_COLOR_BOLD}There was 1 risky test:${_BASHUNIT_COLOR_DEFAULT}\n"
490+
else
491+
echo -e "${_BASHUNIT_COLOR_BOLD}There were $total_risky risky tests:${_BASHUNIT_COLOR_DEFAULT}\n"
492+
fi
493+
494+
tr -d '\r' <"$RISKY_OUTPUT_PATH" | sed '/^[[:space:]]*$/d' | sed 's/^/|/'
495+
rm "$RISKY_OUTPUT_PATH"
496+
497+
echo ""
498+
fi
499+
}

src/env.sh

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -276,6 +276,7 @@ MKTEMP="$(command -v mktemp)"
276276
FAILURES_OUTPUT_PATH=$("$MKTEMP")
277277
SKIPPED_OUTPUT_PATH=$("$MKTEMP")
278278
INCOMPLETE_OUTPUT_PATH=$("$MKTEMP")
279+
RISKY_OUTPUT_PATH=$("$MKTEMP")
279280

280281
# Initialize temp directory once at startup for performance
281282
BASHUNIT_TEMP_DIR="${TMPDIR:-/tmp}/bashunit/tmp"

src/main.sh

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -657,6 +657,7 @@ function bashunit::main::exec_tests() {
657657

658658
if ! bashunit::env::is_tap_output_enabled; then
659659
bashunit::console_results::print_failing_tests_and_reset
660+
bashunit::console_results::print_risky_tests_and_reset
660661
bashunit::console_results::print_incomplete_tests_and_reset
661662
bashunit::console_results::print_skipped_tests_and_reset
662663
fi
@@ -750,6 +751,7 @@ function bashunit::main::cleanup() {
750751
function bashunit::main::handle_stop_on_failure_sync() {
751752
printf "\n%sStop on failure enabled...%s\n" "${_BASHUNIT_COLOR_SKIPPED}" "${_BASHUNIT_COLOR_DEFAULT}"
752753
bashunit::console_results::print_failing_tests_and_reset
754+
bashunit::console_results::print_risky_tests_and_reset
753755
bashunit::console_results::print_incomplete_tests_and_reset
754756
bashunit::console_results::print_skipped_tests_and_reset
755757
bashunit::console_results::render_result

src/parallel.sh

Lines changed: 7 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -87,6 +87,13 @@ function bashunit::parallel::aggregate_test_results() {
8787
continue
8888
fi
8989

90+
# Check for risky test (zero assertions, no error)
91+
local total_for_test=$((failed + passed + skipped + incomplete + snapshot))
92+
if [ "$total_for_test" -eq 0 ] && [ "${exit_code:-0}" -eq 0 ]; then
93+
bashunit::state::add_tests_risky
94+
continue
95+
fi
96+
9097
bashunit::state::add_tests_passed
9198
done
9299
done

src/reports.sh

Lines changed: 7 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -24,6 +24,10 @@ function bashunit::reports::add_test_passed() {
2424
bashunit::reports::add_test "$1" "$2" "$3" "$4" "passed"
2525
}
2626

27+
function bashunit::reports::add_test_risky() {
28+
bashunit::reports::add_test "$1" "$2" "$3" "$4" "risky"
29+
}
30+
2731
function bashunit::reports::add_test_failed() {
2832
bashunit::reports::add_test "$1" "$2" "$3" "$4" "failed" "$5"
2933
}
@@ -94,6 +98,8 @@ function bashunit::reports::generate_junit_xml() {
9498
local escaped_message
9599
escaped_message=$(bashunit::reports::__xml_escape "$failure_message")
96100
echo " <failure message=\"Test failed\">$escaped_message</failure>"
101+
elif [[ "$status" == "risky" ]]; then
102+
echo " <skipped message=\"Test has no assertions (risky)\"/>"
97103
elif [[ "$status" == "skipped" ]]; then
98104
echo " <skipped/>"
99105
elif [[ "$status" == "incomplete" ]]; then
@@ -151,6 +157,7 @@ function bashunit::reports::generate_report_html() {
151157
echo " .skipped { background-color: #fcf8e3; }"
152158
echo " .incomplete { background-color: #d9edf7; }"
153159
echo " .snapshot { background-color: #dfe6e9; }"
160+
echo " .risky { background-color: #f5e6f5; }"
154161
echo " </style>"
155162
echo "</head>"
156163
echo "<body>"

src/runner.sh

Lines changed: 55 additions & 17 deletions
Original file line numberDiff line numberDiff line change
@@ -691,7 +691,10 @@ function bashunit::runner::run_test() {
691691
"division by 0" "cannot allocate memory" "bad file descriptor" \
692692
"segmentation fault" "illegal option" "argument list too long" \
693693
"readonly variable" "missing keyword" "killed" \
694-
"cannot execute binary file" "invalid arithmetic operator"; do
694+
"cannot execute binary file" "invalid arithmetic operator" \
695+
"ambiguous redirect" "integer expression expected" \
696+
"too many arguments" "value too great" \
697+
"not a valid identifier" "unexpected EOF"; do
695698
if [[ "$runtime_output" == *"$error"* ]]; then
696699
runtime_error="${runtime_output#*: }" # Remove everything up to and including ": "
697700
runtime_error=${runtime_error//$'\n'/} # Remove all newlines using parameter expansion
@@ -814,6 +817,18 @@ function bashunit::runner::run_test() {
814817
return
815818
fi
816819

820+
# Check for risky test (zero assertions)
821+
if [[ "$total_assertions" -eq 0 ]]; then
822+
bashunit::state::add_tests_risky
823+
if ! bashunit::env::is_failures_only_enabled; then
824+
bashunit::console_results::print_risky_test "${label}" "$duration"
825+
fi
826+
bashunit::reports::add_test_risky "$test_file" "$label" "$duration" "$total_assertions"
827+
bashunit::runner::write_risky_result_output "$test_file" "$fn_name"
828+
bashunit::internal_log "Test risky" "$label"
829+
return
830+
fi
831+
817832
# In failures-only mode, suppress successful test output
818833
if ! bashunit::env::is_failures_only_enabled; then
819834
if [[ "$fn_name" == "$interpolated_fn_name" ]]; then
@@ -1066,6 +1081,21 @@ function bashunit::runner::write_incomplete_result_output() {
10661081
echo -e "$test_nr) $test_file:$line_number\n$output_msg" >>"$INCOMPLETE_OUTPUT_PATH"
10671082
}
10681083

1084+
function bashunit::runner::write_risky_result_output() {
1085+
local test_file=$1
1086+
local fn_name=$2
1087+
1088+
local line_number
1089+
line_number=$(bashunit::helper::get_function_line_number "$fn_name")
1090+
1091+
local test_nr="*"
1092+
if ! bashunit::parallel::is_enabled; then
1093+
test_nr=$(bashunit::state::get_tests_risky)
1094+
fi
1095+
1096+
echo -e "$test_nr) $test_file:$line_number\nTest has no assertions (risky)" >>"$RISKY_OUTPUT_PATH"
1097+
}
1098+
10691099
function bashunit::runner::record_file_hook_failure() {
10701100
local hook_name="$1"
10711101
local test_file="$2"
@@ -1103,15 +1133,19 @@ function bashunit::runner::execute_file_hook() {
11031133
local hook_output_file
11041134
hook_output_file=$(bashunit::temp_file "${hook_name}_output")
11051135

1106-
# Enable errexit and errtrace to catch any failing command in the hook.
1107-
# The ERR trap saves the exit status to a global variable (since return value
1108-
# from trap doesn't propagate properly), disables errexit (to prevent caller
1109-
# from exiting) and returns from the hook function, preventing subsequent
1110-
# commands from executing.
1136+
# Enable errtrace to catch any failing command in the hook.
1137+
# Using -E (errtrace) without -e (errexit) prevents the main process from
1138+
# exiting on source failures (Bash 3.2 doesn't trigger ERR trap with -eE).
1139+
# The ERR trap saves the exit status to a global variable, cleans up shell
1140+
# options, and returns from the hook function to prevent subsequent commands
1141+
# from executing.
11111142
# Variables set before the failure are preserved since we don't use a subshell.
11121143
_BASHUNIT_HOOK_ERR_STATUS=0
1113-
set -eE
1114-
trap '_BASHUNIT_HOOK_ERR_STATUS=$?; set +eE; trap - ERR; return $_BASHUNIT_HOOK_ERR_STATUS' ERR
1144+
set -E
1145+
if bashunit::env::is_strict_mode_enabled; then
1146+
set -uo pipefail
1147+
fi
1148+
trap '_BASHUNIT_HOOK_ERR_STATUS=$?; set +Eu +o pipefail; trap - ERR; return $_BASHUNIT_HOOK_ERR_STATUS' ERR
11151149

11161150
{
11171151
"$hook_name"
@@ -1120,7 +1154,7 @@ function bashunit::runner::execute_file_hook() {
11201154
# Capture exit status from global variable and clean up
11211155
status=$_BASHUNIT_HOOK_ERR_STATUS
11221156
trap - ERR
1123-
set +eE
1157+
set +Eu +o pipefail
11241158

11251159
if [[ -f "$hook_output_file" ]]; then
11261160
hook_output=""
@@ -1204,15 +1238,19 @@ function bashunit::runner::execute_test_hook() {
12041238
local hook_output_file
12051239
hook_output_file=$(bashunit::temp_file "${hook_name}_output")
12061240

1207-
# Enable errexit and errtrace to catch any failing command in the hook.
1208-
# The ERR trap saves the exit status to a global variable (since return value
1209-
# from trap doesn't propagate properly), disables errexit (to prevent caller
1210-
# from exiting) and returns from the hook function, preventing subsequent
1211-
# commands from executing.
1241+
# Enable errtrace to catch any failing command in the hook.
1242+
# Using -E (errtrace) without -e (errexit) prevents the subshell from
1243+
# exiting on source failures (Bash 3.2 doesn't trigger ERR trap with -eE).
1244+
# The ERR trap saves the exit status to a global variable, cleans up shell
1245+
# options, and returns from the hook function to prevent subsequent commands
1246+
# from executing.
12121247
# Variables set before the failure are preserved since we don't use a subshell.
12131248
_BASHUNIT_HOOK_ERR_STATUS=0
1214-
set -eE
1215-
trap '_BASHUNIT_HOOK_ERR_STATUS=$?; set +eE; trap - ERR; return $_BASHUNIT_HOOK_ERR_STATUS' ERR
1249+
set -E
1250+
if bashunit::env::is_strict_mode_enabled; then
1251+
set -uo pipefail
1252+
fi
1253+
trap '_BASHUNIT_HOOK_ERR_STATUS=$?; set +Eu +o pipefail; trap - ERR; return $_BASHUNIT_HOOK_ERR_STATUS' ERR
12161254

12171255
{
12181256
"$hook_name"
@@ -1221,7 +1259,7 @@ function bashunit::runner::execute_test_hook() {
12211259
# Capture exit status from global variable and clean up
12221260
status=$_BASHUNIT_HOOK_ERR_STATUS
12231261
trap - ERR
1224-
set +eE
1262+
set +Eu +o pipefail
12251263

12261264
if [[ -f "$hook_output_file" ]]; then
12271265
hook_output=""

src/state.sh

Lines changed: 14 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -12,6 +12,7 @@ _BASHUNIT_TESTS_FAILED=0
1212
_BASHUNIT_TESTS_SKIPPED=0
1313
_BASHUNIT_TESTS_INCOMPLETE=0
1414
_BASHUNIT_TESTS_SNAPSHOT=0
15+
_BASHUNIT_TESTS_RISKY=0
1516
_BASHUNIT_ASSERTIONS_PASSED=0
1617
_BASHUNIT_ASSERTIONS_FAILED=0
1718
_BASHUNIT_ASSERTIONS_SKIPPED=0
@@ -68,6 +69,14 @@ function bashunit::state::add_tests_snapshot() {
6869
((_BASHUNIT_TESTS_SNAPSHOT++)) || true
6970
}
7071

72+
function bashunit::state::get_tests_risky() {
73+
echo "$_BASHUNIT_TESTS_RISKY"
74+
}
75+
76+
function bashunit::state::add_tests_risky() {
77+
((_BASHUNIT_TESTS_RISKY++)) || true
78+
}
79+
7180
function bashunit::state::get_assertions_passed() {
7281
echo "$_BASHUNIT_ASSERTIONS_PASSED"
7382
}
@@ -298,6 +307,7 @@ function bashunit::state::print_line() {
298307
skipped) char="${_BASHUNIT_COLOR_SKIPPED}S${_BASHUNIT_COLOR_DEFAULT}" ;;
299308
incomplete) char="${_BASHUNIT_COLOR_INCOMPLETE}I${_BASHUNIT_COLOR_DEFAULT}" ;;
300309
snapshot) char="${_BASHUNIT_COLOR_SNAPSHOT}N${_BASHUNIT_COLOR_DEFAULT}" ;;
310+
risky) char="${_BASHUNIT_COLOR_RISKY}R${_BASHUNIT_COLOR_DEFAULT}" ;;
301311
error) char="${_BASHUNIT_COLOR_FAILED}E${_BASHUNIT_COLOR_DEFAULT}" ;;
302312
*) char="?" && bashunit::log "warning" "unknown test type '$type'" ;;
303313
esac
@@ -364,6 +374,10 @@ function bashunit::state::print_tap_line() {
364374
printf "ok %d - %s # snapshot\n" \
365375
"$_BASHUNIT_TOTAL_TESTS_COUNT" "$test_name"
366376
;;
377+
risky)
378+
printf "ok %d - %s # RISKY no assertions\n" \
379+
"$_BASHUNIT_TOTAL_TESTS_COUNT" "$test_name"
380+
;;
367381
*)
368382
printf "not ok %d - %s\n" \
369383
"$_BASHUNIT_TOTAL_TESTS_COUNT" "$test_name"

0 commit comments

Comments
 (0)