Make new snapshots fail on CI

Fixes #1461

Author: hadley

NEWS.md

@@ -1,5 +1,6 @@
 # testthat (development version)
 
+* `expect_snapshot()` and friends will now fail when creating a new snapshot on CI. This is usually a signal that you've forgotten to run it locally before committing (#1461).
 * `expect_snapshot_value()` can now handle expressions that generate `-` (#1678) or zero length atomic vectors (#2042).
 * `expect_matches()` failures should be a little easier to read (#2135).
 * New `local_on_cran(TRUE)` allows you to simulate how your tests will run on CRAN (#2112).

R/test-files.R

@@ -311,7 +311,7 @@ test_files_reporter <- function(reporter, .env = parent.frame()) {
   reporters <- list(
     find_reporter(reporter),
     lister, # track data
-    local_snapshotter("_snaps", fail_on_new = FALSE, .env = .env)
+    local_snapshotter("_snaps", fail_on_new = on_ci(), .env = .env)
   )
   list(
     multi = MultiReporter$new(reporters = compact(reporters)),

tests/testthat/test-snapshot-file.R

@@ -54,7 +54,7 @@ test_that("expect_snapshot_file works with variant", {
 })
 
 test_that("basic workflow", {
-  snapper <- local_snapshotter()
+  snapper <- local_snapshotter(fail_on_new = FALSE)
 
   # warns on first run
   snapper$start_file("snapshot-6", "test")
@@ -79,7 +79,7 @@ test_that("basic workflow", {
 })
 
 test_that("can announce snapshot file", {
-  snapper <- local_snapshotter()
+  snapper <- local_snapshotter(fail_on_new = FALSE)
   snapper$start_file("snapshot-announce", "test")
   announce_snapshot_file(name = "bar.svg")
   expect_equal(snapper$snap_file_seen, "snapshot-announce/bar.svg")

tests/testthat/test-snapshot-reporter.R

@@ -1,5 +1,5 @@
 test_that("can establish local snapshotter for testing", {
-  snapper <- local_snapshotter()
+  snapper <- local_snapshotter(fail_on_new = FALSE)
 
   snapper$start_file("snapshot-1", "test")
   expect_true(snapper$is_active())
@@ -9,7 +9,7 @@ test_that("can establish local snapshotter for testing", {
 
 test_that("basic workflow", {
   path <- withr::local_tempdir()
-  snapper <- local_snapshotter(path)
+  snapper <- local_snapshotter(path, fail_on_new = FALSE)
   snapper$start_file("snapshot-2")
   # output if not active (because test not set here)
   expect_snapshot_output("x") |>
@@ -39,7 +39,7 @@ test_that("basic workflow", {
 })
 
 test_that("only create new files for changed variants", {
-  snapper <- local_snapshotter()
+  snapper <- local_snapshotter(fail_on_new = FALSE)
   snapper$start_file("variants", "test")
   expect_warning(expect_snapshot_output("x"), "Adding new")
   expect_warning(expect_snapshot_output("x", variant = "a"), "Adding new")
@@ -75,7 +75,7 @@ test_that("only create new files for changed variants", {
 })
 
 test_that("only reverting change in variant deletes .new", {
-  snapper <- local_snapshotter()
+  snapper <- local_snapshotter(fail_on_new = FALSE)
   snapper$start_file("v", "test")
   expect_warning(expect_snapshot_output("x", variant = "a"), "Adding new")
   expect_warning(expect_snapshot_output("x", variant = "b"), "Adding new")
@@ -98,7 +98,7 @@ test_that("only reverting change in variant deletes .new", {
 
 test_that("removing tests removes snap file", {
   path <- withr::local_tempdir()
-  snapper <- local_snapshotter(path)
+  snapper <- local_snapshotter(path, fail_on_new = FALSE)
   snapper$start_file("snapshot-3", "test")
   expect_warning(expect_snapshot_output("x"), "Adding new")
   snapper$end_file()
@@ -110,7 +110,7 @@ test_that("removing tests removes snap file", {
 })
 
 test_that("errors in test doesn't change snapshot", {
-  snapper <- local_snapshotter()
+  snapper <- local_snapshotter(fail_on_new = FALSE)
 
   # First run
   snapper$start_file("snapshot-5", "test")

tests/testthat/test-snapshot-value.R

@@ -32,7 +32,7 @@ test_that("reparse handles common cases", {
 })
 
 test_that("errors if can't roundtrip", {
-  snapper <- local_snapshotter()
+  snapper <- local_snapshotter(fail_on_new = FALSE)
   snapper$start_file("snapshot-4", "test")
 
   expect_error(expect_snapshot_value(NULL), "safely serialized")

vignettes/custom-expectation.Rmd

@@ -12,7 +12,7 @@ library(testthat)
 knitr::opts_chunk$set(collapse = TRUE, comment = "#>")
 
 # Pretend we're snapsotting
-snapper <- local_snapshotter()
+snapper <- local_snapshotter(fail_on_new = FALSE)
 snapper$start_file("snapshotting.Rmd", "test")
 ```
 

vignettes/snapshotting.Rmd

@@ -36,7 +36,7 @@ library(testthat)
 ```
 
 ```{r include = FALSE}
-snapper <- local_snapshotter()
+snapper <- local_snapshotter(fail_on_new = FALSE)
 snapper$start_file("snapshotting.Rmd", "test")
 ```