Skip to content

Commit 28d5b4a

Browse files
jonsimjonsim
committed
Prevent failing tests printing suite failures
If tests fail in a suite, only print a suite failure if a suite-level keyword also failed. Prior to this commit, a confusing 'SUITE FAILED' trace with all passing keyword would be output after the failing test.
1 parent 2e7a3cf commit 28d5b4a

File tree

6 files changed

+95
-41
lines changed

6 files changed

+95
-41
lines changed

‎robot_trace/RobotTrace.py‎

Lines changed: 7 additions & 9 deletions
Original file line numberDiff line numberDiff line change
@@ -132,13 +132,15 @@ def __init__(self):
132132
self._stack: list[str] = []
133133
self.has_warnings: bool = False
134134
self.has_errors: bool = False
135+
self.has_failures: bool = False
135136

136137
def clear(self):
137138
self._trace = ""
138139
self._depth = 0
139140
self._stack.clear()
140141
self.has_warnings = False
141142
self.has_errors = False
143+
self.has_failures = False
142144

143145
@property
144146
def _indent(self) -> str:
@@ -464,17 +466,11 @@ def end_suite(self, name, attributes):
464466
status = attributes["status"]
465467
status_text = ""
466468
if trace:
467-
should_print = False
469+
should_print = self.print_passed
468470
status_text = "SUITE " + self._past_tense(status)
469471
status_color = None
470-
if status == "PASS":
471-
should_print = self.print_passed
472-
status_color = ANSI.Fore.GREEN
473-
elif status == "SKIP":
474-
should_print = self.print_skipped
475-
status_color = ANSI.Fore.YELLOW
476-
elif status == "FAIL":
477-
should_print = self.print_failed
472+
if self.suite_trace_stack.has_failures:
473+
should_print |= self.print_failed
478474
status_color = ANSI.Fore.RED
479475
if self.suite_trace_stack.has_errors:
480476
should_print |= self.print_errored
@@ -646,6 +642,8 @@ def log_message(self, attributes):
646642
elif level == "WARN":
647643
self.stats.warnings += 1
648644
stack.has_warnings = True
645+
elif level == "FAIL":
646+
stack.has_failures = True
649647

650648
stack.append_trace("\n".join(lines))
651649

‎tests/minimal_testcases/suite_setup_failing_testcase.robot‎

Lines changed: 16 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,16 @@
1+
*** Settings ***
2+
Suite Setup Do Suite Setup
3+
4+
5+
*** Test Cases ***
6+
Test Case 1
7+
Log In Test Case 1
8+
Fail Test case always fails
9+
10+
11+
*** Keywords ***
12+
Do Suite Setup
13+
Log Doing suite setup
14+
${value}= Evaluate 1 + 2
15+
Should Be Equal As Numbers ${value} 3
16+
Log Suite setup completed

‎tests/minimal_testcases/suite_setup_failing_testcase.trace.normal‎

Lines changed: 11 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,11 @@
1+
TEST FAILED: Suite Setup Failing Testcase.Test Case 1
2+
═════════════════════════════════════════════════════
3+
▶ BuiltIn.Log('In Test Case 1')
4+
I In Test Case 1
5+
✓ PASS 0s
6+
▶ BuiltIn.Fail('Test case always fails')
7+
F Test case always fails
8+
✗ FAIL 0s
9+
10+
RUN COMPLETE: 1 test, 1 completed (0 passed, 0 skipped, 1 failed).
11+
Total elapsed: 0s.

‎tests/minimal_testcases/suite_teardown_failing_testcase.robot‎

Lines changed: 16 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,16 @@
1+
*** Settings ***
2+
Suite Teardown Do Suite Teardown
3+
4+
5+
*** Test Cases ***
6+
Test Case 1
7+
Log In Test Case 1
8+
Fail Test case always fails
9+
10+
11+
*** Keywords ***
12+
Do Suite Teardown
13+
Log Doing suite teardown
14+
${value}= Evaluate 1 + 2
15+
Should Be Equal As Numbers ${value} 3
16+
Log Suite teardown completed

‎tests/minimal_testcases/suite_teardown_failing_testcase.trace.normal‎

Lines changed: 11 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,11 @@
1+
TEST FAILED: Suite Teardown Failing Testcase.Test Case 1
2+
════════════════════════════════════════════════════════
3+
▶ BuiltIn.Log('In Test Case 1')
4+
I In Test Case 1
5+
✓ PASS 0s
6+
▶ BuiltIn.Fail('Test case always fails')
7+
F Test case always fails
8+
✗ FAIL 0s
9+
10+
RUN COMPLETE: 1 test, 1 completed (0 passed, 0 skipped, 1 failed).
11+
Total elapsed: 0s.

‎tests/system/minimal_testcases.robot‎

Lines changed: 34 additions & 32 deletions
Original file line numberDiff line numberDiff line change
@@ -14,38 +14,40 @@ ${TESTCASE_DIR} ${CURDIR}${/}..${/}minimal_testcases
1414

1515
*** Test Cases ***
1616
# robocop: off=NAME07 It's useful to have the test case names match the file.
17-
basic_failing basic_failing expected_rc=1
18-
basic_passing basic_passing
19-
basic_skipping basic_skipping
20-
for_failing for_failing expected_rc=1
21-
for_passing for_passing
22-
if_else_failing if_else_failing expected_rc=1
23-
if_else_passing if_else_passing
24-
invalid_syntax invalid_syntax expected_rc=1
25-
log_debug log_debug --loglevel=DEBUG
26-
log_error log_error
27-
log_multiline log_multiline
28-
log_warning log_warning
29-
multiple_failing multiple_failing expected_rc=2
30-
multiple_passing multiple_passing
31-
nested_keywords_1 nested_keywords_1
32-
nested_keywords_2 nested_keywords_2
33-
nested_keywords_3 nested_keywords_3
34-
nested_keywords_failing nested_keywords_failing expected_rc=1
35-
suite_setup_failing suite_setup_failing expected_rc=1
36-
suite_setup_passing suite_setup_passing
37-
suite_setup_warning suite_setup_warning
38-
suite_teardown_failing suite_teardown_failing expected_rc=1
39-
suite_teardown_passing suite_teardown_passing
40-
suite_teardown_warning suite_teardown_warning
41-
test_setup_failing test_setup_failing expected_rc=1
42-
test_setup_passing test_setup_passing
43-
test_setup_warning test_setup_warning
44-
test_teardown_failing test_teardown_failing expected_rc=1
45-
test_teardown_passing test_teardown_passing
46-
test_teardown_warning test_teardown_warning
47-
try_catch_failing try_catch_failing
48-
try_catch_passing try_catch_passing
17+
basic_failing basic_failing expected_rc=1
18+
basic_passing basic_passing
19+
basic_skipping basic_skipping
20+
for_failing for_failing expected_rc=1
21+
for_passing for_passing
22+
if_else_failing if_else_failing expected_rc=1
23+
if_else_passing if_else_passing
24+
invalid_syntax invalid_syntax expected_rc=1
25+
log_debug log_debug --loglevel=DEBUG
26+
log_error log_error
27+
log_multiline log_multiline
28+
log_warning log_warning
29+
multiple_failing multiple_failing expected_rc=2
30+
multiple_passing multiple_passing
31+
nested_keywords_1 nested_keywords_1
32+
nested_keywords_2 nested_keywords_2
33+
nested_keywords_3 nested_keywords_3
34+
nested_keywords_failing nested_keywords_failing expected_rc=1
35+
suite_setup_failing suite_setup_failing expected_rc=1
36+
suite_setup_failing_testcase suite_setup_failing_testcase expected_rc=1
37+
suite_setup_passing suite_setup_passing
38+
suite_setup_warning suite_setup_warning
39+
suite_teardown_failing suite_teardown_failing expected_rc=1
40+
suite_teardown_failing_testcase suite_teardown_failing_testcase expected_rc=1
41+
suite_teardown_passing suite_teardown_passing
42+
suite_teardown_warning suite_teardown_warning
43+
test_setup_failing test_setup_failing expected_rc=1
44+
test_setup_passing test_setup_passing
45+
test_setup_warning test_setup_warning
46+
test_teardown_failing test_teardown_failing expected_rc=1
47+
test_teardown_passing test_teardown_passing
48+
test_teardown_warning test_teardown_warning
49+
try_catch_failing try_catch_failing
50+
try_catch_passing try_catch_passing
4951

5052

5153
*** Keywords ***

0 commit comments

Comments
 (0)