Set INSTA_UPDATE

Previously, INSTA_UPDATE=always would cause any Insta tests in the
program under test to update their snapshots and return success. This
has two problems:

1. The tests succeed when they should fail, so mutants are counted as
   missed when they may actually have decent coverage.

2. The source tree is polluted by updates from Insta.

Now cargo-mutants specifically knows about Insta and sets the env
variable.

Author: sourcefrog

Cargo.lock

@@ -82,6 +82,7 @@ exclude = [
     "testdata/tree/factorial",
     "testdata/tree/hang_avoided_by_attr/",
     "testdata/tree/hang_when_mutated",
+    "testdata/tree/insta",
     "testdata/tree/integration_tests",
     "testdata/tree/override_dependency",
     "testdata/tree/patch_dependency",

Cargo.toml

@@ -6,6 +6,8 @@
 
 - Changed: Find mutants before doing a baseline test, so that you will find out earlier if there's nothing to test.
 
+- New: Set `INSTA_UPDATE=no` so that tests that use the Insta library (as cargo-mutants does for its own tests) don't write updates back into the source directory, and don't falsely pass.
+
 ## 1.0.1
 
 Released 2022-09-12

NEWS.md

@@ -61,6 +61,10 @@ pub fn run_cargo(
     // <https://doc.rust-lang.org/rustc/lints/levels.html#capping-lints>
     // TODO: Maybe this should append instead of overwriting it...?
     env.push(("RUSTFLAGS".into(), "--cap-lints=allow".into()));
+    // The tests might use Insta <https://insta.rs>, and we don't want it to write
+    // updates to the source tree, and we *certainly* don't want it to write
+    // updates and then let the test pass.
+    env.push(("INSTA_UPDATE".into(), "no".into()));
 
     let message = format!("run {}", argv.join(" "),);
     log_file.message(&message);

src/cargo.rs

@@ -0,0 +1,10 @@
+[package]
+name = "cargo-mutants-testdata-insta"
+version = "0.0.0"
+edition = "2018"
+authors = ["Martin Pool"]
+publish = false
+
+[dependencies.insta]
+version = "1.19.1"
+default-features = false

testdata/tree/insta/Cargo.toml

@@ -0,0 +1,5 @@
+# cargo-mutants `insta` test tree
+
+An example of a crate that uses the [Insta](https://insta.rs) test framework.
+
+Insta in some modes will either write `.snap.new` files into the source directory, or update existing snapshots. We don't want either of those to happen when testing mutants.

testdata/tree/insta/README.md

@@ -0,0 +1,3 @@
+pub fn say_hello(name: &str) -> String {
+    format!("Hello, {name}!")
+}

testdata/tree/insta/src/lib.rs

@@ -0,0 +1,7 @@
+use cargo_mutants_testdata_insta::say_hello;
+
+#[test]
+fn say_hello_vs_insta_snapshot() {
+    let name = "Robin";
+    insta::assert_snapshot!(say_hello(name));
+}

testdata/tree/insta/tests/insta.rs

@@ -0,0 +1,5 @@
+---
+source: tests/insta.rs
+expression: say_hello(name)
+---
+Hello, Robin!

testdata/tree/insta/tests/snapshots/insta__say_hello_vs_insta_snapshot.snap

@@ -1145,6 +1145,25 @@ fn strict_warnings_about_unused_variables_are_disabled_so_mutants_compile() {
         .stdout(contains("1 mutant tested: 1 caught"));
 }
 
+/// `INSTA_UPDATE=always` in the environment will cause Insta to update
+/// the snaphots, so the tests will pass, so mutants will not be caught.
+/// This test checks that cargo-mutants sets the environment variable
+/// so that mutants are caught properly.
+#[test]
+fn insta_test_failures_are_detected() {
+    for insta_update in ["auto", "always"] {
+        println!("INSTA_UPDATE={insta_update}");
+        let tmp_src_dir = copy_of_testdata("insta");
+        run_assert_cmd()
+            .arg("mutants")
+            .args(["--no-times", "--no-shuffle", "--caught"])
+            .env("INSTA_UPDATE", insta_update)
+            .current_dir(tmp_src_dir.path())
+            .assert()
+            .success();
+    }
+}
+
 fn check_text_list_output(dir: &Path, test_name: &str) {
     // There is a `missed.txt` file with the right content, etc.
     for name in ["missed", "caught", "timeout", "unviable"] {