fix(ci-insights): Filter existing tests that are not related to the session (#259)

The goal is to only keep tests related to the current pytest session. We
need to do this because we are using the length of the existing tests to
determine the budget to allocate to flaky detection.

Author: remyduthu

pytest_mergify/__init__.py

@@ -139,6 +139,10 @@ def pytest_sessionstart(self, session: _pytest.main.Session) -> None:
         if self.mergify_ci.flaky_detector:
             self.mergify_ci.flaky_detector.set_deadline()
 
+    def pytest_collection_finish(self, session: _pytest.main.Session) -> None:
+        if self.mergify_ci.flaky_detector:
+            self.mergify_ci.flaky_detector.filter_existing_tests_with_session(session)
+
     @pytest.hookimpl(hookwrapper=True)
     def pytest_sessionfinish(
         self,

pytest_mergify/flaky_detection.py

@@ -4,7 +4,7 @@
 import typing
 
 import _pytest
-import _pytest.nodes
+import _pytest.main
 import _pytest.reports
 import requests
 
@@ -123,6 +123,12 @@ def detect_from_report(self, report: _pytest.reports.TestReport) -> bool:
 
         return True
 
+    def filter_existing_tests_with_session(self, session: _pytest.main.Session) -> None:
+        session_tests = {item.nodeid for item in session.items}
+        self._context.existing_test_names = [
+            test for test in self._context.existing_test_names if test in session_tests
+        ]
+
     def get_retry_count_for_new_test(self, test: str) -> int:
         metrics = self._new_test_metrics.get(test)
         if not metrics:

tests/test_ci_insights.py

@@ -3,6 +3,7 @@
 import typing
 
 import _pytest.nodes
+import _pytest.pytester
 import _pytest.reports
 import pytest
 import responses
@@ -413,6 +414,42 @@ def test_new():
     )
 
 
+@responses.activate
+def test_flaky_detector_filter_existing_tests_with_session(
+    monkeypatch: pytest.MonkeyPatch,
+    pytester: _pytest.pytester.Pytester,
+) -> None:
+    _set_test_environment(monkeypatch)
+    _make_quarantine_mock()
+    _make_flaky_detection_context_mock(
+        existing_test_names=[
+            "test_flaky_detector_filter_existing_tests_with_session.py::test_foo",
+            "test_flaky_detector_filter_existing_tests_with_session.py::test_bar",
+            "test_flaky_detector_filter_existing_tests_with_session.py::test_baz",  # Unknown test, should be filtered.
+        ]
+    )
+
+    pytester.makepyfile(
+        """
+        def test_foo():
+            assert True
+
+        def test_bar():
+            assert True
+        """
+    )
+
+    plugin = pytest_mergify.PytestMergify()
+
+    result = pytester.runpytest_inprocess(plugins=[plugin])
+    result.assert_outcomes(passed=2)  # 2 existing tests.
+
+    assert plugin.mergify_ci.flaky_detector is not None
+
+    # Unknown test should have been filtered out after collection.
+    assert len(plugin.mergify_ci.flaky_detector._context.existing_test_names) == 2
+
+
 def _get_span_counts(
     spans: typing.Dict[str, trace.ReadableSpan],
 ) -> typing.Dict[str, int]: