Reporters can now take the output of run_all_with_progress (#84)

* Reports can operate on top of the two test outputs

* Add headless API

Author: bruno-f-cruz

src/contraqctor/qc/base.py

@@ -792,6 +792,7 @@ def run_all(self) -> t.Generator[Result, None, None]:
         Yields:
             Result: Result objects produced by all test methods.
         """
+
         for test in self.get_tests():
             yield from self.run_test(test)
 
@@ -1256,14 +1257,39 @@ def _update_suite_progress(
         summary_line = f"[cyan]{suite_name:<{suite_name_width}} | {status_bar} | {stats.get_status_summary()}"
         progress.update(suite_task, description=summary_line)
 
+    def run_all(self) -> t.Dict[t.Optional[str], t.List[Result]]:
+        """Run all tests in all suites without progress display.
+
+        Executes all tests and collects results without visual progress reporting.
+
+        Returns:
+            Dict[Optional[str], List[Result]]: Results grouped by test group name.
+        """
+        collected_tests = self._collect_tests()
+        collected_results: t.List[_TaggedResult] = []
+
+        for group, tests_in_group in _TaggedTest.group_by_group(collected_tests):
+            for suite, tests_in_suite in _TaggedTest.group_by_suite(tests_in_group):
+                results: t.List[Result] = []
+                for test in tests_in_suite:
+                    results.extend(suite.run_test(test.test))
+                for result in results:
+                    collected_results.append(
+                        _TaggedResult(suite=suite, group=group, result=result, test=result.test_reference)
+                    )
+
+        self._results = collected_results
+
+        out: t.Dict[t.Optional[str], t.List[Result]] = {}
+        for group, grouped_results in _TaggedResult.group_by_group(collected_results):
+            out[group] = [tagged_result.result for tagged_result in grouped_results]
+        return out
+
     def run_all_with_progress(
         self,
         *,
         reporter: t.Optional["Reporter"] = None,
-        render_context: bool = True,
-        render_description: bool = True,
-        render_traceback: bool = True,
-        render_message: bool = True,
+        **reporter_kwargs: t.Any,
     ) -> t.Dict[t.Optional[str], t.List[Result]]:
         """Run all tests in all suites with a rich progress display.
 
@@ -1272,10 +1298,6 @@ def run_all_with_progress(
 
         Args:
             reporter: Optional reporter to use for output. If None, uses ConsoleReporter.
-            render_context: Whether to render test context in result output.
-            render_description: Whether to render test descriptions in result output.
-            render_traceback: Whether to render tracebacks for errors in result output.
-            render_message: Whether to render test result messages in result output.
 
         Returns:
             Dict[Optional[str], List[Result]]: Results grouped by test group name.
@@ -1386,14 +1408,9 @@ def run_all_with_progress(
 
         self._results = collected_results
         if self._results:
-            total_stats = ResultsStatistics.from_results([tr.result for tr in self._results])
             reporter.report_results(
                 self._results,
-                total_stats,
-                render_description=render_description,
-                render_traceback=render_traceback,
-                render_message=render_message,
-                render_context=render_context,
+                **reporter_kwargs,
             )
 
         out: t.Dict[t.Optional[str], t.List[Result]] = {}

src/contraqctor/qc/reporters.py

@@ -37,8 +37,7 @@ class Reporter(abc.ABC):
     @abc.abstractmethod
     def report_results(
         self,
-        results: t.List[_TaggedResult],
-        statistics: ResultsStatistics,
+        results: dict[str | None, list[Result]] | list[_TaggedResult],
         *,
         render_context: bool = True,
         render_description: bool = True,
@@ -82,8 +81,7 @@ def __init__(
 
     def report_results(
         self,
-        results: t.List[_TaggedResult],
-        statistics: ResultsStatistics,
+        results: dict[str | None, list[Result]] | list[_TaggedResult],
         *,
         render_context: bool = True,
         render_description: bool = True,
@@ -108,6 +106,7 @@ def report_results(
         if not results:
             return
 
+        results = _normalize_results(results)
         # Setup serializer and serialize ALL results if needed
         # (not just the ones being displayed)
         serializer = None
@@ -238,7 +237,7 @@ class HtmlReporter(Reporter):
 
     def __init__(
         self,
-        output_path: t.Union[str, Path],
+        output_path: t.Union[str, Path] = "report.html",
         template_dir: t.Optional[t.Union[str, Path]] = None,
         default_group_name: str = "Ungrouped",
         serializer: t.Optional[ContextExportableObjSerializer] = None,
@@ -258,14 +257,13 @@ def __init__(
 
     def report_results(
         self,
-        results: t.List[_TaggedResult],
-        statistics: ResultsStatistics,
+        results: dict[str | None, list[Result]] | list[_TaggedResult],
         *,
         render_context: bool = True,
         render_description: bool = True,
         render_traceback: bool = True,
         render_message: bool = True,
-        serialize_context_exportable_obj: bool = False,
+        serialize_context_exportable_obj: bool = True,
         **kwargs,
     ) -> None:
         """Generate HTML report of test results.
@@ -282,6 +280,8 @@ def report_results(
         """
         template = self.env.get_template("report.html")
 
+        results = _normalize_results(results)
+
         grouped_results = []
         for group, test_results in _TaggedResult.group_by_group(results):
             group_name = group or self.default_group_name
@@ -327,7 +327,7 @@ def report_results(
 
         html_content = template.render(
             groups=grouped_results,
-            statistics=statistics,
+            statistics=ResultsStatistics.from_results([tr.result for tr in results]),
             status_color=STATUS_COLOR,
             render_context=render_context,
             render_description=render_description,
@@ -339,3 +339,28 @@ def report_results(
         )
 
         self.output_path.write_text(html_content, encoding="utf-8")
+
+
+def _normalize_results(
+    results: dict[str | None, list[Result]] | list[_TaggedResult],
+) -> list[_TaggedResult]:
+    """Normalize results to a list of _TaggedResult instances.
+
+    Args:
+        results: Either a dict mapping group names to lists of Results,
+                 or a list of _TaggedResult instances.
+
+    Returns:
+        A list of _TaggedResult instances.
+    """
+    if isinstance(results, dict):
+        normalized_results = []
+        for group, test_results in results.items():
+            for result in test_results:
+                assert result.suite_reference is not None, "Result must have suite_reference set"
+                normalized_results.append(
+                    _TaggedResult(suite=result.suite_reference, result=result, group=group, test=result.test_reference)
+                )
+        return normalized_results
+    else:
+        return results

tests/test_qc/test_runner.py

@@ -123,3 +123,30 @@ def test_run_all_with_progress(self, mock_progress):
             assert stats[Status.ERROR] == 2
             assert stats[Status.SKIPPED] == 2
             assert stats[Status.WARNING] == 2
+
+    def test_run_all(self):
+        """Test running all tests without progress reporting."""
+
+        runner = Runner()
+        suite1 = MockSuite()
+        suite2 = MockSuite()
+
+        runner.add_suite(suite1)  # Default group (None)
+        runner.add_suite(suite2, group="TestGroup")
+
+        grouped_results = runner.run_all()
+
+        assert None in grouped_results
+        assert "TestGroup" in grouped_results
+
+        assert len(grouped_results[None]) == 6  # All tests in MockSuite
+        assert len(grouped_results["TestGroup"]) == 6  # All tests in MockSuite
+
+        all_results = grouped_results[None] + grouped_results["TestGroup"]
+        stats = ResultsStatistics.from_results(all_results)
+
+        assert stats[Status.PASSED] == 4
+        assert stats[Status.FAILED] == 2
+        assert stats[Status.ERROR] == 2
+        assert stats[Status.SKIPPED] == 2
+        assert stats[Status.WARNING] == 2

tests/test_qc/test_serializers.py

@@ -6,7 +6,7 @@
 import pytest
 
 from contraqctor.qc._context_extensions import ContextExportableObj
-from contraqctor.qc.base import ResultsStatistics, Suite, _TaggedResult
+from contraqctor.qc.base import Suite, _TaggedResult
 from contraqctor.qc.reporters import ConsoleReporter, HtmlReporter
 from contraqctor.qc.serializers import (
     ContextExportableObjSerializer,
@@ -293,13 +293,9 @@ def test_fail_with_image(self):
             result = next(iter(results_iter))
             tagged_results.append(_TaggedResult(suite=suite, group="Test", result=result, test=test_method))
 
-        stats = ResultsStatistics.from_results([tr.result for tr in tagged_results])
-
         with tempfile.TemporaryDirectory() as tmpdir:
             reporter = ConsoleReporter()
-            reporter.report_results(
-                tagged_results, stats, serialize_context_exportable_obj=True, asset_output_dir=tmpdir
-            )
+            reporter.report_results(tagged_results, serialize_context_exportable_obj=True, asset_output_dir=tmpdir)
 
             files = list(Path(tmpdir).glob("*.png"))
             assert len(files) == 2
@@ -324,16 +320,14 @@ def test_fail_with_image(self):
             result = next(iter(results_iter))
             tagged_results.append(_TaggedResult(suite=suite, group="Test", result=result, test=test_method))
 
-        stats = ResultsStatistics.from_results([tr.result for tr in tagged_results])
-
         default_dir = Path("./report/assets")
         if default_dir.exists():
             import shutil
 
             shutil.rmtree(default_dir)
 
         reporter = ConsoleReporter()
-        reporter.report_results(tagged_results, stats, serialize_context_exportable_obj=True)
+        reporter.report_results(tagged_results, serialize_context_exportable_obj=True)
 
         assert default_dir.exists()
         files = list(default_dir.glob("*.png"))
@@ -367,12 +361,10 @@ def test_with_image(self):
             result = next(iter(results_iter))
             tagged_results.append(_TaggedResult(suite=suite, group="Test", result=result, test=test_method))
 
-        stats = ResultsStatistics.from_results([tr.result for tr in tagged_results])
-
         with tempfile.TemporaryDirectory() as tmpdir:
             output_path = Path(tmpdir) / "test_report.html"
             reporter = HtmlReporter(output_path)
-            reporter.report_results(tagged_results, stats, serialize_context_exportable_obj=True)
+            reporter.report_results(tagged_results, serialize_context_exportable_obj=True)
 
             assert output_path.exists()
             content = output_path.read_text()
@@ -398,12 +390,10 @@ def test_with_image(self):
             result = next(iter(results_iter))
             tagged_results.append(_TaggedResult(suite=suite, group="Test", result=result, test=test_method))
 
-        stats = ResultsStatistics.from_results([tr.result for tr in tagged_results])
-
         with tempfile.TemporaryDirectory() as tmpdir:
             output_path = Path(tmpdir) / "test_report.html"
             reporter = HtmlReporter(output_path)
-            reporter.report_results(tagged_results, stats, serialize_context_exportable_obj=False)
+            reporter.report_results(tagged_results, serialize_context_exportable_obj=False)
 
             assert output_path.exists()
             content = output_path.read_text()