Split and delete runner.py

Author: ZzEeKkAa

dpbench/console/report.py

@@ -45,8 +45,7 @@ def execute_report(args: Namespace, conn: sqlalchemy.Engine):
         )
 
     import dpbench.config as cfg
-    from dpbench.infrastructure.reporter import update_run_id
-    from dpbench.infrastructure.runner import print_report
+    from dpbench.infrastructure.reporter import print_report, update_run_id
 
     cfg.GLOBAL = cfg.read_configs(
         benchmarks=args.benchmarks,

dpbench/infrastructure/benchmark_results.py

@@ -95,3 +95,45 @@ def Result(self, run_id: int, benchmark_name, framework_version) -> Result:
             if self.validation_state == ValidationStatusCodes.SUCCESS
             else "Fail",
         )
+
+    def _format_ns(self, time_in_ns: int):
+        time = int(time_in_ns)
+        assert time >= 0
+        suff = [
+            ("s", 1000_000_000),
+            ("ms", 1000_000),
+            ("\u03BCs", 1000),
+            ("ns", 0),
+        ]
+        for s, scale in suff:
+            if time >= scale:
+                scaled_time = float(time) / scale if scale > 0 else time
+                return f"{scaled_time}{s} ({time} ns)"
+
+    def print(self, framework_name: str = "", framework_version: str = ""):
+        print(
+            "================ implementation "
+            + self.impl_postfix
+            + " ========================\n"
+            + "implementation:",
+            self.impl_postfix,
+        )
+
+        if self.error_state == ErrorCodes.SUCCESS:
+            print("framework:", framework_name)
+            print("framework version:", framework_version)
+            print("setup time:", self._format_ns(self.setup_time))
+            print("warmup time:", self._format_ns(self.warmup_time))
+            print("teardown time:", self._format_ns(self.teardown_time))
+            print("max execution times:", self._format_ns(self.max_exec_time))
+            print("min execution times:", self._format_ns(self.min_exec_time))
+            print(
+                "median execution times:",
+                self._format_ns(self.median_exec_time),
+            )
+            print("repeats:", self.repeats)
+            print("preset:", self.preset)
+            print("validated:", self.validation_state)
+        else:
+            print("error states:", self.error_state)
+            print("error msg:", self.error_msg)

dpbench/infrastructure/benchmark_runner.py

@@ -20,7 +20,6 @@
 from dpbench.infrastructure.enums import ErrorCodes, ValidationStatusCodes
 from dpbench.infrastructure.frameworks import Framework
 from dpbench.infrastructure.frameworks.fabric import build_framework
-from dpbench.infrastructure.runner import _print_results
 from dpbench.infrastructure.timer import timer
 
 """
@@ -219,7 +218,7 @@ def runner(c: mpc.Connection) -> None:
                 framework,
             )
 
-            _print_results(benchmark_results, framework)
+            benchmark_results.print(framework.fname, framework.version())
 
             c.send(benchmark_results)
 
@@ -312,7 +311,7 @@ def run_benchmark_in_sub_process(
             results.error_state = ErrorCodes.FAILED_EXECUTION
             results.error_msg = "Expected failure"
 
-            _print_results(results, None)
+            results.print()
 
             return results
 
@@ -323,7 +322,7 @@ def run_benchmark_in_sub_process(
             results.error_state = ErrorCodes.UNIMPLEMENTED
             results.error_msg = "Unimplemented"
 
-            _print_results(results, None)
+            results.print()
 
             return results
 
@@ -348,14 +347,14 @@ def run_benchmark_in_sub_process(
                 results.error_state = ErrorCodes.FAILED_EXECUTION
                 results.error_msg = "Core dump"
 
-                _print_results(results, None)
+                results.print()
                 self.kill_process(rc.framework)
         else:
             results = BenchmarkResults(0, rc.implementation, rc.preset)
             results.error_state = ErrorCodes.EXECUTION_TIMEOUT
             results.error_msg = "Execution timed out"
 
-            _print_results(results, None)
+            results.print()
             self.kill_process(rc.framework)
 
         return results

dpbench/infrastructure/reporter.py

@@ -326,3 +326,42 @@ def get_unexpected_failures(
         )
 
     return failures.difference(expected_failures)
+
+
+def print_report(
+    conn: sqlalchemy.Engine,
+    run_id: int,
+    implementations: set[str],
+    comparison_pairs: list[tuple[str, str]] = [],
+):
+    if not implementations:
+        implementations = {impl.postfix for impl in cfg.GLOBAL.implementations}
+
+    implementations = list(implementations)
+    implementations.sort()
+
+    generate_impl_summary_report(
+        conn, run_id=run_id, implementations=implementations
+    )
+
+    generate_performance_report(
+        conn,
+        run_id=run_id,
+        implementations=implementations,
+        headless=True,
+    )
+
+    generate_comparison_report(
+        conn,
+        run_id=run_id,
+        implementations=implementations,
+        comparison_pairs=comparison_pairs,
+        headless=True,
+    )
+
+    unexpected_failures = get_unexpected_failures(conn, run_id=run_id)
+
+    if len(unexpected_failures) > 0:
+        raise ValueError(
+            f"Unexpected benchmark implementations failed: {unexpected_failures}.",
+        )