Skip to content

Commit 83ffb2a

Browse files
hadleyhadley
committed
Repeat failures at end of ProgressReporter
1 parent b6676ab commit 83ffb2a

File tree

3 files changed

+211
-1
lines changed

3 files changed

+211
-1
lines changed

‎NEWS.md‎

Lines changed: 2 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -18,7 +18,8 @@
1818
you to verify that the correct tests are skipped (#1801).
1919

2020
* `ProgressReporter` only reports the run time of test files that take longer
21-
than 1s, rather than 0.1s. (#1806).
21+
than 1s, rather than 0.1s. (#1806) and re-displays all failures at the end
22+
of the results.
2223

2324
* New `is_snapshot()` returns `TRUE` if code is running inside a snapshot test
2425
(#1796) and `is_checking()` returns `TRUE` if test is running inside of

‎R/reporter-progress.R‎

Lines changed: 13 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -25,6 +25,7 @@ ProgressReporter <- R6::R6Class("ProgressReporter",
2525
update_interval = NULL,
2626

2727
skips = NULL,
28+
problems = NULL,
2829

2930
max_fail = NULL,
3031
n_ok = 0,
@@ -58,6 +59,7 @@ ProgressReporter <- R6::R6Class("ProgressReporter",
5859
self$update_interval <- update_interval
5960

6061
self$skips <- Stack$new()
62+
self$problems <- Stack$new()
6163
self$ctxt_issues <- Stack$new()
6264

6365
# Capture at init so not affected by test settings
@@ -217,6 +219,7 @@ ProgressReporter <- R6::R6Class("ProgressReporter",
217219
self$n_fail <- self$n_fail + 1
218220
self$ctxt_n_fail <- self$ctxt_n_fail + 1
219221
self$ctxt_issues$push(result)
222+
self$problems$push(result)
220223
} else if (expectation_skip(result)) {
221224
self$n_skip <- self$n_skip + 1
222225
self$ctxt_n_skip <- self$ctxt_n_skip + 1
@@ -249,6 +252,15 @@ ProgressReporter <- R6::R6Class("ProgressReporter",
249252

250253
skip_report(self)
251254

255+
if (self$problems$size() > 0) {
256+
problems <- self$problems$as_list()
257+
self$rule("Failed tests", line = 1)
258+
for (problem in problems) {
259+
self$cat_line(issue_summary(problem))
260+
self$cat_line()
261+
}
262+
}
263+
252264
status <- summary_line(self$n_fail, self$n_warn, self$n_skip, self$n_ok)
253265
self$cat_line(status)
254266

@@ -457,6 +469,7 @@ ParallelProgressReporter <- R6::R6Class("ParallelProgressReporter",
457469
self$n_fail <- self$n_fail + 1
458470
self$files[[file]]$n_fail <- self$files[[file]]$n_fail + 1L
459471
self$files[[file]]$issues$push(result)
472+
self$problems$push(result)
460473
} else if (expectation_skip(result)) {
461474
self$n_skip <- self$n_skip + 1
462475
self$files[[file]]$n_skip <- self$files[[file]]$n_skip + 1L

‎tests/testthat/_snaps/reporter-progress.md‎

Lines changed: 196 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -18,6 +18,17 @@
1818
--------------------------------------------------------------------------------
1919

2020
== Results =====================================================================
21+
-- Failed tests ----------------------------------------------------------------
22+
Error ('reporters/error-setup.R:6:1'): (code run outside of `test_that()`)
23+
Error in `h()`: !
24+
Backtrace:
25+
x
26+
1. +-testthat::setup(f())
27+
2. | \-rlang::eval_tidy(enquo(code), env = env)
28+
3. \-f()
29+
4. \-g()
30+
5. \-h()
31+
2132
[ FAIL 1 | WARN 0 | SKIP 0 | PASS 0 ]
2233

2334
I believe in you!
@@ -124,6 +135,73 @@
124135
i Increase this number with (e.g.) `testthat::set_max_fails(Inf)`
125136

126137
== Results =====================================================================
138+
-- Failed tests ----------------------------------------------------------------
139+
Failure ('reporters/fail-many.R:3:5'): Example
140+
FALSE is not TRUE
141+
142+
`actual`: FALSE
143+
`expected`: TRUE
144+
145+
Failure ('reporters/fail-many.R:3:5'): Example
146+
FALSE is not TRUE
147+
148+
`actual`: FALSE
149+
`expected`: TRUE
150+
151+
Failure ('reporters/fail-many.R:3:5'): Example
152+
FALSE is not TRUE
153+
154+
`actual`: FALSE
155+
`expected`: TRUE
156+
157+
Failure ('reporters/fail-many.R:3:5'): Example
158+
FALSE is not TRUE
159+
160+
`actual`: FALSE
161+
`expected`: TRUE
162+
163+
Failure ('reporters/fail-many.R:3:5'): Example
164+
FALSE is not TRUE
165+
166+
`actual`: FALSE
167+
`expected`: TRUE
168+
169+
Failure ('reporters/fail-many.R:3:5'): Example
170+
FALSE is not TRUE
171+
172+
`actual`: FALSE
173+
`expected`: TRUE
174+
175+
Failure ('reporters/fail-many.R:3:5'): Example
176+
FALSE is not TRUE
177+
178+
`actual`: FALSE
179+
`expected`: TRUE
180+
181+
Failure ('reporters/fail-many.R:3:5'): Example
182+
FALSE is not TRUE
183+
184+
`actual`: FALSE
185+
`expected`: TRUE
186+
187+
Failure ('reporters/fail-many.R:3:5'): Example
188+
FALSE is not TRUE
189+
190+
`actual`: FALSE
191+
`expected`: TRUE
192+
193+
Failure ('reporters/fail-many.R:3:5'): Example
194+
FALSE is not TRUE
195+
196+
`actual`: FALSE
197+
`expected`: TRUE
198+
199+
Failure ('reporters/fail-many.R:3:5'): Example
200+
FALSE is not TRUE
201+
202+
`actual`: FALSE
203+
`expected`: TRUE
204+
127205
[ FAIL 11 | WARN 0 | SKIP 0 | PASS 0 ]
128206
== Terminated early ============================================================
129207

@@ -303,6 +381,124 @@
303381
--------------------------------------------------------------------------------
304382

305383
== Results =====================================================================
384+
-- Failed tests ----------------------------------------------------------------
385+
Error ('reporters/backtraces.R:6:3'): errors thrown at block level are entraced
386+
Error in `g()`: foo
387+
Backtrace:
388+
x
389+
1. \-f()
390+
2. \-g()
391+
392+
Error ('reporters/backtraces.R:11:3'): errors thrown from a quasi-labelled argument are entraced
393+
Error in `foo()`: foo
394+
Backtrace:
395+
x
396+
1. +-testthat::expect_s3_class(foo(), "foo")
397+
2. | \-testthat::quasi_label(enquo(object), arg = "object")
398+
3. | \-rlang::eval_bare(expr, quo_get_env(quo))
399+
4. \-foo()
400+
401+
Error ('reporters/backtraces.R:18:3'): errors thrown from a quasi-labelled argument are entraced (deep case)
402+
Error in `foo()`: foo
403+
Backtrace:
404+
x
405+
1. +-testthat::expect_s3_class(f(), "foo")
406+
2. | \-testthat::quasi_label(enquo(object), arg = "object")
407+
3. | \-rlang::eval_bare(expr, quo_get_env(quo))
408+
4. \-f()
409+
5. \-g()
410+
6. +-testthat::expect_s3_class(foo(), "foo")
411+
7. | \-testthat::quasi_label(enquo(object), arg = "object")
412+
8. | \-rlang::eval_bare(expr, quo_get_env(quo))
413+
9. \-foo()
414+
415+
Error ('reporters/backtraces.R:28:3'): errors thrown from a quasi-labelled argument are entraced (deep deep case)
416+
Error in `bar()`: foobar
417+
Backtrace:
418+
x
419+
1. \-f()
420+
2. \-g()
421+
3. +-testthat::expect_s3_class(foo(), "foo")
422+
4. | \-testthat::quasi_label(enquo(object), arg = "object")
423+
5. | \-rlang::eval_bare(expr, quo_get_env(quo))
424+
6. \-foo()
425+
7. \-bar()
426+
427+
Error ('reporters/backtraces.R:35:3'): failed expect_error() prints a backtrace
428+
Error in `signaller()`: bar
429+
Backtrace:
430+
x
431+
1. +-testthat::expect_error(f(), "foo")
432+
2. | \-testthat:::expect_condition_matching(...)
433+
3. | \-testthat:::quasi_capture(...)
434+
4. | +-testthat (local) .capture(...)
435+
5. | | \-base::withCallingHandlers(...)
436+
6. | \-rlang::eval_bare(quo_get_expr(.quo), quo_get_env(.quo))
437+
7. \-f()
438+
8. \-signaller()
439+
440+
Error ('reporters/backtraces.R:43:3'): Errors are inspected with `conditionMessage()`
441+
<foobar/rlang_error/error/condition>
442+
Error in `eval(code, test_env)`: dispatched
443+
Backtrace:
444+
x
445+
1. \-rlang::abort("Wrong message", "foobar")
446+
447+
Error ('reporters/backtraces.R:58:3'): deep stacks are shown
448+
Error in `f(x - 1)`: This is deep
449+
Backtrace:
450+
x
451+
1. \-f(25)
452+
2. \-f(x - 1)
453+
3. \-f(x - 1)
454+
4. \-f(x - 1)
455+
5. \-f(x - 1)
456+
6. \-f(x - 1)
457+
7. \-f(x - 1)
458+
8. \-f(x - 1)
459+
9. \-f(x - 1)
460+
10. \-f(x - 1)
461+
11. \-f(x - 1)
462+
12. \-f(x - 1)
463+
13. \-f(x - 1)
464+
14. \-f(x - 1)
465+
15. \-f(x - 1)
466+
16. \-f(x - 1)
467+
17. \-f(x - 1)
468+
18. \-f(x - 1)
469+
19. \-f(x - 1)
470+
20. \-f(x - 1)
471+
21. \-f(x - 1)
472+
22. \-f(x - 1)
473+
23. \-f(x - 1)
474+
24. \-f(x - 1)
475+
25. \-f(x - 1)
476+
26. \-f(x - 1)
477+
478+
Failure ('reporters/backtraces.R:66:1'): (code run outside of `test_that()`)
479+
FALSE is not TRUE
480+
481+
`actual`: FALSE
482+
`expected`: TRUE
483+
Backtrace:
484+
x
485+
1. \-f()
486+
2. \-g()
487+
3. \-h()
488+
4. \-testthat::expect_true(FALSE)
489+
490+
Failure ('reporters/backtraces.R:69:3'): nested expectations get backtraces
491+
FALSE is not TRUE
492+
493+
`actual`: FALSE
494+
`expected`: TRUE
495+
Backtrace:
496+
x
497+
1. \-f()
498+
2. \-g()
499+
3. \-h()
500+
4. \-testthat::expect_true(FALSE)
501+
306502
[ FAIL 9 | WARN 1 | SKIP 0 | PASS 1 ]
307503

308504
I believe in you!

0 commit comments

Comments
 (0)