fix(ci-insights): Properly detect if we can rerun a test (#304)

The goal of this change is to use the initial duration to detect if we
can really rerun a test entirely. It's to mitigate the issues when a
timeout occurs during a run. Thus, we should not start any new execution
if it's not feasible based on the initial duration of the test.

Fixes: MRGFY-6282

Author: remyduthu

pytest_mergify/__init__.py

@@ -235,9 +235,7 @@ def pytest_runtest_protocol(
             for _ in range(
                 self.mergify_ci.flaky_detector.get_rerun_count_for_test(item.nodeid)
             ):
-                if self.mergify_ci.flaky_detector.is_test_deadline_exceeded(
-                    item.nodeid
-                ):
+                if self.mergify_ci.flaky_detector.should_abort_reruns(item.nodeid):
                     break
 
                 for report in self._reruntestprotocol(item, nextitem):
@@ -317,7 +315,7 @@ def pytest_runtest_teardown(
 
         # The goal here is to keep only function-scoped finalizers during
         # reruns and restore higher-scoped finalizers only on the last one.
-        if self.mergify_ci.flaky_detector.is_test_deadline_exceeded(
+        if self.mergify_ci.flaky_detector.should_abort_reruns(
             item.nodeid
         ) or self.mergify_ci.flaky_detector.is_last_rerun_for_test(item.nodeid):
             self.mergify_ci.flaky_detector.restore_item_finalizers(item)

pytest_mergify/flaky_detection.py

@@ -234,12 +234,22 @@ def _get_normalized_rerun_count(
 
         return result
 
-    def is_test_deadline_exceeded(self, test: str) -> bool:
+    def should_abort_reruns(self, test: str) -> bool:
+        """
+        Determines if a test can be rerun within its deadline.
+
+        We must ensure there's enough time remaining before the deadline to
+        complete another full test execution. This prevents starting a rerun
+        that would exceed the deadline and potentially timeout.
+        """
         metrics = self._test_metrics.get(test)
         if not metrics or not metrics.deadline:
             return False
 
-        return datetime.datetime.now(datetime.timezone.utc) >= metrics.deadline
+        projected_completion = (
+            datetime.datetime.now(datetime.timezone.utc) + metrics.initial_duration
+        )
+        return projected_completion >= metrics.deadline
 
     def make_report(self) -> str:
         result = "🐛 Flaky detection"

tests/test_flaky_detection.py

@@ -4,6 +4,7 @@
 import _pytest
 import _pytest.reports
 import freezegun
+import pytest
 
 from pytest_mergify import flaky_detection
 
@@ -201,3 +202,64 @@ def test_flaky_detector_get_rerun_count_for_test_with_fast_test() -> None:
     detector.set_deadline()
 
     assert detector.get_rerun_count_for_test("foo") == 1000
+
+
+@freezegun.freeze_time(
+    time_to_freeze=datetime.datetime.fromisoformat("2025-01-01T00:00:00+00:00")
+)
+@pytest.mark.parametrize(
+    argnames=("metrics", "test", "expected"),
+    argvalues=[
+        pytest.param({}, "foo", False, id="Metrics not found"),
+        pytest.param(
+            {"foo": flaky_detection._TestMetrics()}, "foo", False, id="Deadline not set"
+        ),
+        pytest.param(
+            {
+                "foo": flaky_detection._TestMetrics(
+                    deadline=datetime.datetime.fromisoformat(
+                        "2025-01-02T00:00:00+00:00"
+                    ),
+                    initial_call_duration=datetime.timedelta(seconds=1),
+                ),
+            },
+            "foo",
+            False,
+            id="Not aborted",
+        ),
+        pytest.param(
+            {
+                "foo": flaky_detection._TestMetrics(
+                    deadline=datetime.datetime.fromisoformat(
+                        "2025-01-01T00:00:00+00:00"
+                    ),
+                    initial_call_duration=datetime.timedelta(),
+                ),
+            },
+            "foo",
+            True,
+            id="Aborted by deadline",
+        ),
+        pytest.param(
+            {
+                "foo": flaky_detection._TestMetrics(
+                    deadline=datetime.datetime.fromisoformat(
+                        "2025-01-01T00:00:00+00:00"
+                    ),
+                    initial_call_duration=datetime.timedelta(minutes=2),
+                ),
+            },
+            "foo",
+            True,
+            id="Aborted by initial duration",
+        ),
+    ],
+)
+def test_flaky_detector_should_abort_reruns(
+    metrics: typing.Dict[str, flaky_detection._TestMetrics],
+    test: str,
+    expected: bool,
+) -> None:
+    detector = InitializedFlakyDetector()
+    detector._test_metrics = metrics
+    assert detector.should_abort_reruns(test) == expected