Skip to content

Commit dbafef5

Browse files
Otto-AAOtto-AA
committed
Use inline-snapshot for regression tests
1 parent 810d761 commit dbafef5

13 files changed

+349
-178
lines changed

‎CONTRIBUTING.rst‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -21,7 +21,7 @@ Running the tests
2121
2222
uv run pytest
2323
24-
This also runs E2E tests that verify that `mutmut run` produces the same output as before. If your code changes should change the output of `mutmut run` and this test fails, try to delete the `snapshots/*.json` files (as described in the test errors).
24+
We use `inline-snapshot` for E2E and integration tests, to prevent unexpected changes in the output. If the output _should_ change, you can use `uv run pytest --inline-snapshot=fix` to update the snapshots.
2525

2626
If pytest terminates before reporting the test failures, it likely hit a case where mutmut calls `os._exit(...)`. Try looking at these calls first for troubleshooting.
2727

‎pyproject.toml‎

Lines changed: 4 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -55,10 +55,14 @@ source-include = ["HISTORY.rst"]
5555
dev = [
5656
"inline-snapshot>=0.32.0",
5757
"pytest-asyncio>=1.0.0",
58+
"ruff>=0.15.1",
5859
]
5960

6061
[tool.pytest.ini_options]
6162
testpaths = [
6263
"tests",
6364
]
6465
asyncio_default_fixture_loop_scope = "function"
66+
67+
[tool.inline-snapshot]
68+
format-command="ruff format --stdin-filename {filename}"

tests/e2e/test_e2e_result_snapshots.py renamed to tests/e2e/e2e_utils.py

Lines changed: 4 additions & 37 deletions
Original file line numberDiff line numberDiff line change
@@ -4,8 +4,6 @@
44
from contextlib import contextmanager
55
from pathlib import Path
66
from typing import Any
7-
import pytest
8-
import sys
97

108
import mutmut
119
from mutmut.__main__ import SourceFileMutationData, _run, ensure_config_loaded, walk_source_files
@@ -47,8 +45,10 @@ def write_json_file(path: Path, data: Any):
4745
json.dump(data, file, indent=2)
4846

4947

50-
def asserts_results_did_not_change(project: str):
48+
def run_mutmut_on_project(project: str) -> dict:
5149
"""Runs mutmut on this project and verifies that the results stay the same for all mutations."""
50+
mutmut._reset_globals()
51+
5252
project_path = Path("..").parent / "e2e_projects" / project
5353

5454
mutants_path = project_path / "mutants"
@@ -58,37 +58,4 @@ def asserts_results_did_not_change(project: str):
5858
with change_cwd(project_path):
5959
_run([], None)
6060

61-
results = read_all_stats_for_project(project_path)
62-
63-
snapshot_path = Path("tests") / "e2e" / "snapshots" / (project + ".json")
64-
65-
if snapshot_path.exists():
66-
# compare results against previous snapshot
67-
previous_snapshot = read_json_file(snapshot_path)
68-
69-
err_msg = f'Mutmut results changed for the E2E project \'{project}\'. If this change was on purpose, delete {snapshot_path} and rerun the tests.'
70-
assert results == previous_snapshot, err_msg
71-
else:
72-
# create the first snapshot
73-
write_json_file(snapshot_path, results)
74-
75-
76-
def test_my_lib_result_snapshot():
77-
mutmut._reset_globals()
78-
asserts_results_did_not_change("my_lib")
79-
80-
81-
def test_config_result_snapshot():
82-
mutmut._reset_globals()
83-
asserts_results_did_not_change("config")
84-
85-
86-
def test_mutate_only_covered_lines_result_snapshot():
87-
mutmut._reset_globals()
88-
asserts_results_did_not_change("mutate_only_covered_lines")
89-
90-
91-
@pytest.mark.skipif(sys.version_info < (3, 14), reason="Can only test python 3.14 features on 3.14")
92-
def test_python_3_14_result_snapshot():
93-
mutmut._reset_globals()
94-
asserts_results_did_not_change("py3_14_features")
61+
return read_all_stats_for_project(project_path)

‎tests/e2e/snapshots/config.json‎

Lines changed: 0 additions & 18 deletions
This file was deleted.

‎tests/e2e/snapshots/mutate_only_covered_lines.json‎

Lines changed: 0 additions & 39 deletions
This file was deleted.

‎tests/e2e/snapshots/my_lib.json‎

Lines changed: 0 additions & 75 deletions
This file was deleted.

‎tests/e2e/snapshots/py3_14_features.json‎

Lines changed: 0 additions & 8 deletions
This file was deleted.

‎tests/e2e/test_e2e_config.py‎

Lines changed: 26 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,26 @@
1+
from inline_snapshot import snapshot
2+
3+
from tests.e2e.e2e_utils import run_mutmut_on_project
4+
5+
6+
def test_config_result_snapshot():
7+
assert run_mutmut_on_project("config") == snapshot(
8+
{
9+
"mutants/config_pkg/__init__.py.meta": {
10+
"config_pkg.x_hello__mutmut_1": 1,
11+
"config_pkg.x_hello__mutmut_2": 1,
12+
"config_pkg.x_hello__mutmut_3": 1,
13+
},
14+
"mutants/config_pkg/math.py.meta": {
15+
"config_pkg.math.x_add__mutmut_1": 0,
16+
"config_pkg.math.x_call_depth_two__mutmut_1": 1,
17+
"config_pkg.math.x_call_depth_two__mutmut_2": 1,
18+
"config_pkg.math.x_call_depth_three__mutmut_1": 1,
19+
"config_pkg.math.x_call_depth_three__mutmut_2": 1,
20+
"config_pkg.math.x_call_depth_four__mutmut_1": 33,
21+
"config_pkg.math.x_call_depth_four__mutmut_2": 33,
22+
"config_pkg.math.x_call_depth_five__mutmut_1": 33,
23+
"config_pkg.math.x_func_with_no_tests__mutmut_1": 33,
24+
},
25+
}
26+
)

‎tests/e2e/test_e2e_coverage.py‎

Lines changed: 47 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,47 @@
1+
from inline_snapshot import snapshot
2+
3+
from tests.e2e.e2e_utils import run_mutmut_on_project
4+
5+
6+
def test_mutate_only_covered_lines_result_snapshot():
7+
assert run_mutmut_on_project("mutate_only_covered_lines") == snapshot(
8+
{
9+
"mutants/src/mutate_only_covered_lines/__init__.py.meta": {
10+
"mutate_only_covered_lines.x_hello_mutate_only_covered_lines__mutmut_1": 1,
11+
"mutate_only_covered_lines.x_hello_mutate_only_covered_lines__mutmut_2": 1,
12+
"mutate_only_covered_lines.x_hello_mutate_only_covered_lines__mutmut_3": 1,
13+
"mutate_only_covered_lines.x_mutate_only_covered_lines_multiline__mutmut_1": 1,
14+
"mutate_only_covered_lines.x_mutate_only_covered_lines_multiline__mutmut_2": 1,
15+
"mutate_only_covered_lines.x_mutate_only_covered_lines_multiline__mutmut_3": 1,
16+
"mutate_only_covered_lines.x_mutate_only_covered_lines_multiline__mutmut_4": 1,
17+
"mutate_only_covered_lines.x_mutate_only_covered_lines_multiline__mutmut_5": 0,
18+
"mutate_only_covered_lines.x_mutate_only_covered_lines_multiline__mutmut_6": 0,
19+
"mutate_only_covered_lines.x_mutate_only_covered_lines_multiline__mutmut_7": 0,
20+
"mutate_only_covered_lines.x_mutate_only_covered_lines_multiline__mutmut_8": 0,
21+
"mutate_only_covered_lines.x_mutate_only_covered_lines_multiline__mutmut_9": 0,
22+
"mutate_only_covered_lines.x_mutate_only_covered_lines_multiline__mutmut_10": 0,
23+
"mutate_only_covered_lines.x_mutate_only_covered_lines_multiline__mutmut_11": 0,
24+
"mutate_only_covered_lines.x_mutate_only_covered_lines_multiline__mutmut_12": 0,
25+
"mutate_only_covered_lines.x_mutate_only_covered_lines_multiline__mutmut_13": 0,
26+
"mutate_only_covered_lines.x_mutate_only_covered_lines_multiline__mutmut_14": 0,
27+
"mutate_only_covered_lines.x_mutate_only_covered_lines_multiline__mutmut_15": 0,
28+
"mutate_only_covered_lines.x_mutate_only_covered_lines_multiline__mutmut_16": 0,
29+
"mutate_only_covered_lines.x_mutate_only_covered_lines_multiline__mutmut_17": 0,
30+
"mutate_only_covered_lines.x_mutate_only_covered_lines_multiline__mutmut_18": 0,
31+
"mutate_only_covered_lines.x_mutate_only_covered_lines_multiline__mutmut_19": 0,
32+
"mutate_only_covered_lines.x_mutate_only_covered_lines_multiline__mutmut_20": 0,
33+
"mutate_only_covered_lines.x_mutate_only_covered_lines_multiline__mutmut_21": 0,
34+
"mutate_only_covered_lines.x_mutate_only_covered_lines_multiline__mutmut_22": 0,
35+
"mutate_only_covered_lines.x_mutate_only_covered_lines_multiline__mutmut_23": 0,
36+
"mutate_only_covered_lines.x_mutate_only_covered_lines_multiline__mutmut_24": 1,
37+
"mutate_only_covered_lines.x_mutate_only_covered_lines_multiline__mutmut_25": 1,
38+
"mutate_only_covered_lines.x_mutate_only_covered_lines_multiline__mutmut_26": 1,
39+
"mutate_only_covered_lines.x_mutate_only_covered_lines_multiline__mutmut_27": 1,
40+
"mutate_only_covered_lines.x_mutate_only_covered_lines_multiline__mutmut_28": 1,
41+
"mutate_only_covered_lines.x_mutate_only_covered_lines_multiline__mutmut_29": 1,
42+
"mutate_only_covered_lines.x_mutate_only_covered_lines_multiline__mutmut_30": 1,
43+
"mutate_only_covered_lines.x_mutate_only_covered_lines_multiline__mutmut_31": 1,
44+
"mutate_only_covered_lines.x_mutate_only_covered_lines_multiline__mutmut_32": 1,
45+
}
46+
}
47+
)

‎tests/e2e/test_e2e_my_lib.py‎

Lines changed: 83 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,83 @@
1+
from inline_snapshot import snapshot
2+
3+
from tests.e2e.e2e_utils import run_mutmut_on_project
4+
5+
6+
def test_my_lib_result_snapshot():
7+
assert run_mutmut_on_project("my_lib") == snapshot(
8+
{
9+
"mutants/src/my_lib/__init__.py.meta": {
10+
"my_lib.x_hello__mutmut_1": 1,
11+
"my_lib.x_hello__mutmut_2": 1,
12+
"my_lib.x_hello__mutmut_3": 1,
13+
"my_lib.x_badly_tested__mutmut_1": 0,
14+
"my_lib.x_badly_tested__mutmut_2": 0,
15+
"my_lib.x_badly_tested__mutmut_3": 0,
16+
"my_lib.x_untested__mutmut_1": 33,
17+
"my_lib.x_untested__mutmut_2": 33,
18+
"my_lib.x_untested__mutmut_3": 33,
19+
"my_lib.x_make_greeter__mutmut_1": 1,
20+
"my_lib.x_make_greeter__mutmut_2": 1,
21+
"my_lib.x_make_greeter__mutmut_3": 1,
22+
"my_lib.x_make_greeter__mutmut_4": 1,
23+
"my_lib.x_make_greeter__mutmut_5": 0,
24+
"my_lib.x_make_greeter__mutmut_6": 0,
25+
"my_lib.x_make_greeter__mutmut_7": 0,
26+
"my_lib.x_fibonacci__mutmut_1": 1,
27+
"my_lib.x_fibonacci__mutmut_2": 0,
28+
"my_lib.x_fibonacci__mutmut_3": 0,
29+
"my_lib.x_fibonacci__mutmut_4": 0,
30+
"my_lib.x_fibonacci__mutmut_5": 0,
31+
"my_lib.x_fibonacci__mutmut_6": 0,
32+
"my_lib.x_fibonacci__mutmut_7": 0,
33+
"my_lib.x_fibonacci__mutmut_8": 0,
34+
"my_lib.x_fibonacci__mutmut_9": 0,
35+
"my_lib.x_async_consumer__mutmut_1": 1,
36+
"my_lib.x_async_consumer__mutmut_2": 1,
37+
"my_lib.x_async_generator__mutmut_1": 1,
38+
"my_lib.x_async_generator__mutmut_2": 1,
39+
"my_lib.x_simple_consumer__mutmut_1": 1,
40+
"my_lib.x_simple_consumer__mutmut_2": 1,
41+
"my_lib.x_simple_consumer__mutmut_3": 1,
42+
"my_lib.x_simple_consumer__mutmut_4": 1,
43+
"my_lib.x_simple_consumer__mutmut_5": 1,
44+
"my_lib.x_simple_consumer__mutmut_6": 0,
45+
"my_lib.x_simple_consumer__mutmut_7": 1,
46+
"my_lib.x_double_generator__mutmut_1": 1,
47+
"my_lib.x_double_generator__mutmut_2": 1,
48+
"my_lib.x_double_generator__mutmut_3": 0,
49+
"my_lib.x_double_generator__mutmut_4": 0,
50+
"my_lib.xǁPointǁ__init____mutmut_1": 1,
51+
"my_lib.xǁPointǁ__init____mutmut_2": 1,
52+
"my_lib.xǁPointǁabs__mutmut_1": 33,
53+
"my_lib.xǁPointǁabs__mutmut_2": 33,
54+
"my_lib.xǁPointǁabs__mutmut_3": 33,
55+
"my_lib.xǁPointǁabs__mutmut_4": 33,
56+
"my_lib.xǁPointǁabs__mutmut_5": 33,
57+
"my_lib.xǁPointǁabs__mutmut_6": 33,
58+
"my_lib.xǁPointǁadd__mutmut_1": 0,
59+
"my_lib.xǁPointǁadd__mutmut_2": 1,
60+
"my_lib.xǁPointǁadd__mutmut_3": 1,
61+
"my_lib.xǁPointǁadd__mutmut_4": 0,
62+
"my_lib.xǁPointǁto_origin__mutmut_1": 1,
63+
"my_lib.xǁPointǁto_origin__mutmut_2": 1,
64+
"my_lib.xǁPointǁto_origin__mutmut_3": 0,
65+
"my_lib.xǁPointǁto_origin__mutmut_4": 0,
66+
"my_lib.xǁPointǁ__len____mutmut_1": 33,
67+
"my_lib.x_escape_sequences__mutmut_1": 1,
68+
"my_lib.x_escape_sequences__mutmut_2": 0,
69+
"my_lib.x_escape_sequences__mutmut_3": 1,
70+
"my_lib.x_escape_sequences__mutmut_4": 0,
71+
"my_lib.x_escape_sequences__mutmut_5": 0,
72+
"my_lib.x_create_a_segfault_when_mutated__mutmut_1": -11,
73+
"my_lib.x_create_a_segfault_when_mutated__mutmut_2": 0,
74+
"my_lib.x_create_a_segfault_when_mutated__mutmut_3": 0,
75+
"my_lib.x_some_func__mutmut_1": 0,
76+
"my_lib.x_some_func__mutmut_2": 0,
77+
"my_lib.x_some_func__mutmut_3": 1,
78+
"my_lib.x_func_with_star__mutmut_1": 1,
79+
"my_lib.x_func_with_star__mutmut_2": 1,
80+
"my_lib.x_func_with_arbitrary_args__mutmut_1": 1,
81+
}
82+
}
83+
)

0 commit comments

Comments
 (0)