feat: add quarantine report in Mergify CI section (#181)

related to MRGFY-5846

Author: Greesb

pytest_mergify/__init__.py

@@ -65,13 +65,15 @@ def pytest_terminal_summary(
 
         if self.mergify_ci.quarantined_tests is None:
             terminalreporter.write_line("CI Insights Quarantine could not be setup")
-        elif (
-            self.mergify_ci.quarantined_tests is not None
-            and self.mergify_ci.quarantined_tests.init_error_msg
-        ):
-            terminalreporter.write_line(
-                self.mergify_ci.quarantined_tests.init_error_msg, yellow=True
-            )
+        elif self.mergify_ci.quarantined_tests is not None:
+            if self.mergify_ci.quarantined_tests.init_error_msg:
+                terminalreporter.write_line(
+                    self.mergify_ci.quarantined_tests.init_error_msg, yellow=True
+                )
+            else:
+                terminalreporter.write_line(
+                    self.mergify_ci.quarantined_tests.quarantined_tests_report()
+                )
 
         # CI Insights Traces upload logs
         if self.mergify_ci.tracer_provider is None:

pytest_mergify/quarantine.py

@@ -1,4 +1,5 @@
 import dataclasses
+import os
 import pytest
 import _pytest.nodes
 import requests
@@ -14,6 +15,9 @@ class Quarantine:
     quarantined_tests: typing.List[str] = dataclasses.field(
         init=False, default_factory=list
     )
+    quarantine_used_by_tests: typing.Set[str] = dataclasses.field(
+        init=False, default_factory=set
+    )
     init_error_msg: typing.Optional[str] = dataclasses.field(init=False, default=None)
 
     def __post_init__(self) -> None:
@@ -51,8 +55,31 @@ def __post_init__(self) -> None:
     def __contains__(self, item: _pytest.nodes.Item) -> bool:
         return item.nodeid in self.quarantined_tests
 
-    @staticmethod
-    def mark_test_as_quarantined(test_item: _pytest.nodes.Item) -> None:
+    def quarantined_tests_report(self) -> str:
+        report_str = f"""🛡️ Quarantine
+- Repository: {self.repo_name}
+- Branch: {self.branch_name}
+- Quarantined tests fetched from API: {len(self.quarantined_tests)}
+"""
+
+        if self.quarantine_used_by_tests:
+            report_str += f"""
+- 🔒 Quarantined:
+    · {f"{os.linesep}    · ".join(sorted(self.quarantine_used_by_tests))}
+"""
+
+        unused_quarantined_tests = (
+            set(self.quarantined_tests) - self.quarantine_used_by_tests
+        )
+        if unused_quarantined_tests:
+            report_str += f"""
+- Unused quarantined tests:
+    · {f"{os.linesep}    · ".join(sorted(unused_quarantined_tests))}
+"""
+
+        return report_str
+
+    def mark_test_as_quarantined(self, test_item: _pytest.nodes.Item) -> None:
         test_item.add_marker(
             pytest.mark.xfail(
                 reason="Test is quarantined from Mergify CI Insights",
@@ -62,3 +89,4 @@ def mark_test_as_quarantined(test_item: _pytest.nodes.Item) -> None:
             ),
             append=True,
         )
+        self.quarantine_used_by_tests.add(test_item.nodeid)

tests/test_spans_quarantine.py

@@ -30,6 +30,7 @@ def test_my_very_flaky_success_test():
         quarantined_tests=[
             "test_spans_quarantine.py::test_my_very_flaky_failure_test",
             "test_spans_quarantine.py::test_my_very_flaky_success_test",
+            "test_spans_quarantine.py::some_other_test",
         ],
     )
     assert spans is not None
@@ -95,3 +96,16 @@ def test_my_very_flaky_success_test():
     assert spans[
         "test_spans_quarantine.py::test_my_very_flaky_success_test"
     ].attributes["cicd.test.quarantined"]
+
+    assert """🛡️ Quarantine
+- Repository: foo/bar
+- Branch: main
+- Quarantined tests fetched from API: 3
+
+- 🔒 Quarantined:
+    · test_spans_quarantine.py::test_my_very_flaky_failure_test
+    · test_spans_quarantine.py::test_my_very_flaky_success_test
+
+- Unused quarantined tests:
+    · test_spans_quarantine.py::some_other_test
+""" in result.stdout.str()