diff --git a/Project.toml b/Project.toml index 858fab0..26a641c 100644 --- a/Project.toml +++ b/Project.toml @@ -1,7 +1,7 @@ name = "SnapshotTesting" uuid = "cf213a48-8697-4c15-82cb-081f2086cf9e" authors = ["Nathan Daly and contributors"] -version = "0.1.1" +version = "0.2.0" [deps] DeepDiffs = "ab62b9b5-e342-54a8-a765-a90f495de1a6" diff --git a/src/snapshots.jl b/src/snapshots.jl index 3dd146c..c4e2add 100644 --- a/src/snapshots.jl +++ b/src/snapshots.jl @@ -16,9 +16,17 @@ function create_expectation_snapshot(func, expected_dir, subpath) func(snapshot_dir) end -function test_snapshot(func, expected_dir, subpath; allow_additions = true, regenerate = false) - if regenerate - create_expectation_snapshot(func, expected_dir, subpath) +function test_snapshot(func, expected_dir, subpath; allow_additions = true) + # We're testing against the expected files + expected_path = joinpath(expected_dir, subpath) + + if !isdir(expected_path) + mkpath(expected_path) + func(expected_path) + @info """Snapshot for \"$subpath\" did not exist. It has been created at: + $expected_path + """ + @info "Please run the tests again for any changes to take effect" return nothing end @@ -29,14 +37,34 @@ function test_snapshot(func, expected_dir, subpath; allow_additions = true, rege mkpath(snapshot_dir) func(snapshot_dir) - # Test against the expected files - expected_path = joinpath(expected_dir, subpath) @testset "$subpath" begin - _recursive_diff_dirs(expected_path, snapshot_dir; allow_additions) + has_failures = _recursive_diff_dirs(expected_path, snapshot_dir; allow_additions) + + if has_failures + if isinteractive() || force_update() + if force_update() || input_bool("Replace snapshot with actual result (in $subpath)?") + rm(expected_path; recursive=true, force=true) + cp(snapshot_dir, expected_path; force=true) + @info "Snapshot updated at $expected_path" + @info "Please run the tests again for any changes to take effect" + end + else + @error """ + Snapshot test failed for \"$subpath\". + To update the snapshots either run the tests interactively with 'include(\"test/runtests.jl\")', + or to force-update all failing snapshots set the environment variable `JULIA_SNAPSHOTTESTS_UPDATE` + to "true" and re-run the tests via Pkg. + """ + end + end end end -# Diff all the files in the output directory against the expected directory + +# Diff all the files in the output directory against the expected directory, returning +# whether or not we found any failures function _recursive_diff_dirs(expected_dir, new_dir; allow_additions) + has_failures = false + # Collect new files new_files = Set(String[]) for (root, _, files) in walkdir(new_dir) @@ -54,6 +82,7 @@ function _recursive_diff_dirs(expected_dir, new_dir; allow_additions) subpath = _chopprefix(_chopprefix(expected_path, expected_dir), "/") @test subpath in new_files if !(subpath in new_files) + has_failures = true @error("New snapshot is missing file `$subpath`. Expected contents:\n", expected_content) else @@ -62,6 +91,7 @@ function _recursive_diff_dirs(expected_dir, new_dir; allow_additions) new_content = read(new_path, String) @test new_content == expected_content if new_content != expected_content + has_failures = true println("Found non-matching content in `$file`.") display(DeepDiffs.deepdiff(expected_content, new_content)) end @@ -74,6 +104,7 @@ function _recursive_diff_dirs(expected_dir, new_dir; allow_additions) if !allow_additions # Report test failures for the new files if requested if !isempty(new_files) + has_failures = true @error("New snapshot contains unexpected files. If this is not an error in your case, pass `allow_additions = true`.") for path in new_files @@ -86,6 +117,8 @@ function _recursive_diff_dirs(expected_dir, new_dir; allow_additions) end end end + + return has_failures end @@ -102,3 +135,32 @@ function _chopprefix(s::AbstractString, prefix::AbstractString) i, j = iterate(s, k), iterate(prefix, j[2]) end end + +""" + force_update() + +Check if the environment variable `JULIA_SNAPSHOTTESTS_UPDATE` is set to "true". +When true, all failing snapshot tests will automatically update their references. +""" +force_update() = tryparse(Bool, get(ENV, "JULIA_SNAPSHOTTESTS_UPDATE", "false")) === true + +""" + input_bool(prompt) + +Display an interactive y/n prompt and return true for 'y', false for 'n'. +Loops until a valid response is given. +""" +function input_bool(prompt) + while true + println(prompt, " [y/n]") + response = readline() + length(response) == 0 && continue + reply = lowercase(first(strip(response))) + if reply == 'y' + return true + elseif reply == 'n' + return false + end + # Otherwise loop and repeat the prompt + end +end diff --git a/test/runtests.jl b/test/runtests.jl index 6eeb934..3a03c52 100644 --- a/test/runtests.jl +++ b/test/runtests.jl @@ -5,4 +5,7 @@ using Test @testset "snapshots" begin include("snapshots.jl") end + @testset "update modes" begin + include("test_update_modes.jl") + end end diff --git a/test/test_update_modes.jl b/test/test_update_modes.jl new file mode 100644 index 0000000..8e32530 --- /dev/null +++ b/test/test_update_modes.jl @@ -0,0 +1,113 @@ +using SnapshotTesting +using Test + +@testset "Snapshot Update Modes" begin + + @testset "Environment variable force_update" begin + # Test that force_update returns false by default (clear env var first) + withenv("JULIA_SNAPSHOTTESTS_UPDATE" => nothing) do + @test SnapshotTesting.force_update() == false + end + + # Test that force_update returns true when env var is set + withenv("JULIA_SNAPSHOTTESTS_UPDATE" => "true") do + @test SnapshotTesting.force_update() == true + end + + # Test that it returns false for other values + withenv("JULIA_SNAPSHOTTESTS_UPDATE" => "false") do + @test SnapshotTesting.force_update() == false + end + + withenv("JULIA_SNAPSHOTTESTS_UPDATE" => "1") do + @test SnapshotTesting.force_update() == true # "1" parses as true + end + + withenv("JULIA_SNAPSHOTTESTS_UPDATE" => "0") do + @test SnapshotTesting.force_update() == false # "0" parses as false + end + + withenv("JULIA_SNAPSHOTTESTS_UPDATE" => "invalid") do + @test SnapshotTesting.force_update() == false # invalid string returns false + end + end + + @testset "Interactive input_bool" begin + # Helper to test input_bool with simulated input + function test_input(input_string) + # Create a temporary file with the input + mktempdir() do tmpdir + input_file = joinpath(tmpdir, "input.txt") + write(input_file, input_string) + + open(input_file, "r") do input_io + redirect_stdin(input_io) do + redirect_stdout(devnull) do + SnapshotTesting.input_bool("Test prompt") + end + end + end + end + end + + # Test 'y' response + @test test_input("y\n") == true + + # Test 'n' response + @test test_input("n\n") == false + + # Test case insensitivity with uppercase Y + @test test_input("Y\n") == true + + # Test case insensitivity with uppercase N + @test test_input("N\n") == false + + # Test that invalid input is retried + @test test_input("invalid\ny\n") == true + + # Test empty input is retried + @test test_input("\n\nn\n") == false + end + + @testset "Auto-creation of missing snapshots" begin + mktempdir() do tmpdir + expected = joinpath(tmpdir, "expected") + mkpath(expected) + + test_path = joinpath(expected, "autocreate") + @test !isdir(test_path) + + # Run test - should auto-create snapshot + redirect_stdout(devnull) do + SnapshotTesting.test_snapshot(expected, "autocreate") do dir + write(joinpath(dir, "newfile.txt"), "auto-created content") + end + end + + # Verify snapshot was created + @test isdir(test_path) + @test isfile(joinpath(test_path, "newfile.txt")) + @test read(joinpath(test_path, "newfile.txt"), String) == "auto-created content" + end + end + + @testset "Successful snapshot test (no changes needed)" begin + mktempdir() do tmpdir + expected = joinpath(tmpdir, "expected") + mkpath(expected) + + # Create initial snapshot + test_path = joinpath(expected, "nochange") + mkpath(test_path) + write(joinpath(test_path, "file.txt"), "same content") + + # Run test with same content - should pass without prompts + @testset "matching content" begin + SnapshotTesting.test_snapshot(expected, "nochange") do dir + write(joinpath(dir, "file.txt"), "same content") + end + end + end + end + +end