Use pass() and fail() in testthat expectations (#2129)

Instead of using `expect()`, we now prefer early returns with `fail()` and a final return with `pass()`. The main thing I (re?) discovered here is that you must return `fail()`, as while it throws interactively, it returns inside of `test_that()`.

Author: hadley

.Rbuildignore

@@ -23,3 +23,5 @@
 ^[\.]?air\.toml$
 ^\.vscode$
 ^\.git-blame-ignore-rev$
+^CLAUDE\.md$
+^\.claude$

.claude/settings.local.json

@@ -0,0 +1,8 @@
+{
+  "permissions": {
+    "allow": [
+      "Bash(find:*)"
+    ],
+    "deny": []
+  }
+}

CLAUDE.md

@@ -0,0 +1,89 @@
+# CLAUDE.md
+
+This file provides guidance to Claude Code (claude.ai/code) when working with code in this repository.
+
+## About This Project
+
+testthat is R's most popular unit testing framework, used by thousands of CRAN packages. It provides functions to make testing R code as fun and addictive as possible, with clear expectations, visual progress indicators, and seamless integration with R package development workflows.
+
+## Key Development Commands
+
+### Testing
+- `devtools::test()` or `Ctrl/Cmd+Shift+T` in RStudio - Run all tests
+- `devtools::test_file("tests/testthat/test-filename.R")` - Run tests in a specific file
+- `testthat::test_local()` - Run tests for local source package
+- `testthat::test_package("testthat")` - Run tests for installed package
+- `R CMD check` - Full package check including tests
+
+### Building and Installation
+- `devtools::load_all()` or `Ctrl/Cmd+Shift+L` - Load package for development
+- `devtools::document()` - Generate documentation
+- `devtools::check()` - Run R CMD check
+- `devtools::install()` - Install package locally
+
+## Core Architecture
+
+### Main Components
+
+1. **Core Testing Functions** (`R/test-that.R`, `R/test-package.R`):
+   - `test_that()` - The fundamental testing function
+   - `test_local()`, `test_package()`, `test_check()` - Different ways to run test suites
+
+2. **Expectations** (`R/expect-*.R`):
+   - Modular expectation functions (equality, conditions, types, etc.)
+   - Each expectation type has its own file following the pattern `expect-[type].R`
+
+3. **Reporters** (`R/reporter*.R`):
+   - Different output formats for test results
+   - Object-oriented design with base `Reporter` class
+   - Includes check, debug, progress, summary, JUnit, TAP formats
+
+4. **Snapshot Testing** (`R/snapshot*.R`):
+   - Value snapshots, file snapshots, output snapshots
+   - Automatic management and comparison of expected outputs
+
+5. **Parallel Testing** (`R/parallel*.R`):
+   - Multi-core test execution
+   - Configuration via `Config/testthat/parallel: true` in DESCRIPTION
+
+6. **Mocking** (`R/mock*.R`, `R/mock2*.R`):
+   - Function mocking capabilities
+   - Both legacy (`mock.R`) and modern (`mock2*.R`) implementations
+
+### Key Design Patterns
+
+- **Editions**: testthat has different "editions" with varying behavior, controlled by `Config/testthat/edition`
+- **Reporters**: Extensible reporting system using R6 classes
+- **Lazy Evaluation**: Expectations use substitute() and lazy evaluation for better error messages
+- **C++ Integration**: Core functionality implemented in C++ for performance
+
+### File Organization
+
+- `R/` - All R source code, organized by functionality
+- `src/` - C++ source code and Makevars
+- `inst/include/testthat/` - C++ headers for other packages to use
+- `tests/testthat/` - Package's own comprehensive test suite
+- `vignettes/` - Documentation on testing concepts and workflows
+
+### Important Configuration
+
+The package uses several DESCRIPTION fields for configuration:
+- `Config/testthat/edition: 3` - Sets testthat edition
+- `Config/testthat/parallel: true` - Enables parallel testing
+- `Config/testthat/start-first` - Tests to run first in parallel mode
+
+### C++ Testing Infrastructure
+
+testthat provides C++ testing capabilities via Catch framework:
+- Headers in `inst/include/testthat/`
+- Test runner infrastructure in `src/test-runner.cpp`
+- Integration with R's testing system
+
+### Snapshot Testing Workflow
+
+- Snapshots stored in `tests/testthat/_snaps/`
+- Different snapshot types: values, files, output
+- Version-specific snapshots for different R versions
+- Use `testthat::snapshot_accept()` to update snapshots
+
+This codebase prioritizes backward compatibility, comprehensive testing, and clear, descriptive error messages to help R developers write better tests.

R/expect-comparison.R

@@ -34,18 +34,17 @@ expect_compare <- function(operator = c("<", "<=", ">", ">="), act, exp) {
   if (length(cmp) != 1 || !is.logical(cmp)) {
     abort("Result of comparison must be a single logical value")
   }
-  expect(
-    if (!is.na(cmp)) cmp else FALSE,
-    sprintf(
+  if (!isTRUE(cmp)) {
+    msg <- sprintf(
       "%s is %s %s. Difference: %.3g",
       act$lab,
       msg,
       exp$lab,
       act$val - exp$val
-    ),
-    trace_env = caller_env()
-  )
-  invisible(act$val)
+    )
+    return(fail(msg, trace_env = caller_env()))
+  }
+  pass(act$val)
 }
 #' @export
 #' @rdname comparison-expectations

R/expect-condition.R

@@ -138,8 +138,10 @@ expect_error <- function(
 
     # Access error fields with `[[` rather than `$` because the
     # `$.Throwable` from the rJava package throws with unknown fields
-    expect(is.null(msg), msg, info = info, trace = act$cap[["trace"]])
-    invisible(act$val %||% act$cap)
+    if (!is.null(msg)) {
+      return(fail(msg, info = info, trace = act$cap[["trace"]]))
+    }
+    pass(act$val %||% act$cap)
   }
 }
 
@@ -186,9 +188,10 @@ expect_warning <- function(
       ...,
       cond_type = "warnings"
     )
-    expect(is.null(msg), msg, info = info)
-
-    invisible(act$val)
+    if (!is.null(msg)) {
+      return(fail(msg, info = info))
+    }
+    pass(act$val)
   }
 }
 
@@ -218,9 +221,10 @@ expect_message <- function(
   } else {
     act <- quasi_capture(enquo(object), label, capture_messages)
     msg <- compare_messages(act$cap, act$lab, regexp = regexp, all = all, ...)
-    expect(is.null(msg), msg, info = info)
-
-    invisible(act$val)
+    if (!is.null(msg)) {
+      return(fail(msg, info = info))
+    }
+    pass(act$val)
   }
 }
 
@@ -262,9 +266,10 @@ expect_condition <- function(
       inherit = inherit,
       cond_type = "condition"
     )
-    expect(is.null(msg), msg, info = info, trace = act$cap[["trace"]])
-
-    invisible(act$val %||% act$cap)
+    if (!is.null(msg)) {
+      return(fail(msg, info = info, trace = act$cap[["trace"]]))
+    }
+    pass(act$val %||% act$cap)
   }
 }
 
@@ -303,17 +308,12 @@ expect_condition_matching <- function(
 
   # Access error fields with `[[` rather than `$` because the
   # `$.Throwable` from the rJava package throws with unknown fields
-  expect(
-    is.null(msg),
-    msg,
-    info = info,
-    trace = act$cap[["trace"]],
-    trace_env = trace_env
-  )
-
+  if (!is.null(msg)) {
+    return(fail(msg, info = info, trace = act$cap[["trace"]], trace_env = trace_env))
+  }
   # If a condition was expected, return it. Otherwise return the value
   # of the expression.
-  invisible(if (expected) act$cap else act$val)
+  pass(if (expected) act$cap else act$val)
 }
 
 # -------------------------------------------------------------------------

R/expect-constant.R

@@ -56,7 +56,6 @@ expect_false <- function(object, info = NULL, label = NULL) {
 #' show_failure(expect_null(y))
 expect_null <- function(object, info = NULL, label = NULL) {
   act <- quasi_label(enquo(object), label, arg = "object")
-
   expect_waldo_constant(act, NULL, info = info)
 }
 
@@ -71,17 +70,15 @@ expect_waldo_constant <- function(act, constant, info, ...) {
     ...
   )
 
-  expect(
-    length(comp) == 0,
-    sprintf(
+  if (length(comp) != 0) {
+    msg <- sprintf(
       "%s is not %s\n\n%s",
       act$lab,
       deparse(constant),
       paste0(comp, collapse = "\n\n")
-    ),
-    info = info,
-    trace_env = caller_env()
-  )
+    )
+    return(fail(msg, info = info, trace_env = caller_env()))
+  }
 
-  invisible(act$val)
+  pass(act$val)
 }

R/expect-equality.R

@@ -75,12 +75,11 @@ expect_equal <- function(
       comp <- compare(act$val, exp$val, ...)
     }
 
-    expect(
-      comp$equal,
-      sprintf("%s not equal to %s.\n%s", act$lab, exp$lab, comp$message),
-      info = info
-    )
-    invisible(act$val)
+    if (!comp$equal) {
+      msg <- sprintf("%s not equal to %s.\n%s", act$lab, exp$lab, comp$message)
+      return(fail(msg, info = info))
+    }
+    pass(act$val)
   }
 }
 
@@ -112,12 +111,11 @@ expect_identical <- function(
       }
     }
 
-    expect(
-      ident,
-      sprintf("%s not identical to %s.\n%s", act$lab, exp$lab, msg),
-      info = info
-    )
-    invisible(act$val)
+    if (!ident) {
+      msg <- sprintf("%s not identical to %s.\n%s", act$lab, exp$lab, msg)
+      return(fail(msg, info = info))
+    }
+    pass(act$val)
   }
 }
 
@@ -129,22 +127,19 @@ expect_waldo_equal <- function(type, act, exp, info, ...) {
     x_arg = "actual",
     y_arg = "expected"
   )
-  expect(
-    length(comp) == 0,
-    sprintf(
+  if (length(comp) != 0) {
+    msg <- sprintf(
       "%s (%s) not %s to %s (%s).\n\n%s",
       act$lab,
       "`actual`",
       type,
       exp$lab,
       "`expected`",
       paste0(comp, collapse = "\n\n")
-    ),
-    info = info,
-    trace_env = caller_env()
-  )
-
-  invisible(act$val)
+    )
+    return(fail(msg, info = info, trace_env = caller_env()))
+  }
+  pass(act$val)
 }
 
 #' Is an object equal to the expected value, ignoring attributes?
@@ -188,12 +183,16 @@ expect_equivalent <- function(
   )
 
   comp <- compare(act$val, exp$val, ..., check.attributes = FALSE)
-  expect(
-    comp$equal,
-    sprintf("%s not equivalent to %s.\n%s", act$lab, exp$lab, comp$message),
-    info = info
-  )
-  invisible(act$val)
+  if (!comp$equal) {
+    msg <- sprintf(
+      "%s not equivalent to %s.\n%s",
+      act$lab,
+      exp$lab,
+      comp$message
+    )
+    return(fail(msg, info = info))
+  }
+  pass(act$val)
 }
 
 
@@ -225,12 +224,11 @@ expect_reference <- function(
   act <- quasi_label(enquo(object), label, arg = "object")
   exp <- quasi_label(enquo(expected), expected.label, arg = "expected")
 
-  expect(
-    is_reference(act$val, exp$val),
-    sprintf("%s not a reference to %s.", act$lab, exp$lab),
-    info = info
-  )
-  invisible(act$val)
+  if (!is_reference(act$val, exp$val)) {
+    msg <- sprintf("%s not a reference to %s.", act$lab, exp$lab)
+    return(fail(msg, info = info))
+  }
+  pass(act$val)
 }
 
 # expect_reference() needs dev version of rlang