Improve stepwise to not forget failed tests

Now `--stepwise` will remember the last failed test, even if the previous pytest invocations did not pass `--stepwise`.

Previously it would always clear the cache if not active, which hinders certain interactive workflows, which is the prime use cases for the flag.

Author: nicoddemus

changelog/13122.improvement.rst

@@ -0,0 +1,11 @@
+Improve the ``--stepwise`` flag to not forget the last failed test in case pytest is executed later without the flag.
+
+This enables the following workflow:
+
+1. Execute pytest with ``--stepwise``, pytest then stops at the first failing test;
+2. User iteratively updates the code and runs the test in isolation, without the ``--stepwise`` flag
+   (for example in an IDE), until it is fixed.
+3. At this point, the user can execute pytest with ``--stepwise`` again and pytest will continue from the previously
+   failed test, and if it passes, continue with the next tests.
+
+Previously, at step 3, pytest would start from the beginning, forgetting the failed tests.

src/_pytest/stepwise.py

@@ -47,8 +47,6 @@ def pytest_sessionfinish(session: Session) -> None:
             # Do not update cache if this process is a xdist worker to prevent
             # race conditions (#10641).
             return
-        # Clear the list of failing tests if the plugin is not active.
-        session.config.cache.set(STEPWISE_CACHE_DIR, [])
 
 
 class StepwisePlugin:

testing/test_cacheprovider.py

@@ -104,7 +104,7 @@ def test_cache_failure_warns(
         pytester.makepyfile("def test_error(): raise Exception")
         result = pytester.runpytest()
         assert result.ret == 1
-        # warnings from nodeids, lastfailed, and stepwise
+        # warnings from nodeids and lastfailed
         result.stdout.fnmatch_lines(
             [
                 # Validate location/stacklevel of warning from cacheprovider.
@@ -113,7 +113,7 @@ def test_cache_failure_warns(
                 "  */cacheprovider.py:*: PytestCacheWarning: could not create cache path "
                 f"{unwritable_cache_dir}/v/cache/nodeids: *",
                 '    config.cache.set("cache/nodeids", sorted(self.cached_nodeids))',
-                "*1 failed, 3 warnings in*",
+                "*1 failed, 2 warnings in*",
             ]
         )
 

testing/test_stepwise.py

@@ -358,3 +358,41 @@ def test_one():
     with stepwise_cache_file.open(encoding="utf-8") as file_handle:
         observed_value = file_handle.readlines()
     assert [expected_value] == observed_value
+
+
+def test_do_not_clear_cache_if_disabled(pytester: Pytester) -> None:
+    """
+    If pytest is run without --step-wise, do not clear the stepwise cache.
+
+    Keeping the cache around is important for this workflow:
+
+    1. Run tests with --stepwise
+    2. Stop at the failing test, and iterate over it changing the code and running it in isolation
+    (in the IDE for example).
+    3. Run tests with --stepwise again - at this point we expect to start from the failing test, which should now pass,
+       and continue with the next tests.
+    """
+    pytester.makepyfile(
+        """
+        def test_1():
+            pass
+        def test_2():
+            assert False
+        def test_3():
+            pass
+        """
+    )
+    result = pytester.runpytest("--stepwise")
+    result.stdout.fnmatch_lines(
+        ["*::test_2 - assert False*", "*failed, continuing from this test next run*"]
+    )
+
+    # Run a specific test without passing `--stepwise`.
+    result = pytester.runpytest("-k", "test_1")
+    result.stdout.fnmatch_lines(["*1 passed*"])
+
+    # Running with `--stepwise` should continue from the last failing test.
+    result = pytester.runpytest("--stepwise")
+    result.stdout.fnmatch_lines(
+        ["*::test_2 - assert False*", "*failed, continuing from this test next run*"]
+    )