Add comparison report

Author: ZzEeKkAa

dpbench/console/_namespace.py

@@ -28,3 +28,26 @@ class Namespace(argparse.Namespace):
     timeout: float
     precision: Union[str, None]
     program: str
+    comparisons: list[str]
+
+
+class CommaSeparateStringAction(argparse.Action):
+    """Action that reads comma separated string into set of strings.
+
+    This action supposed to be used in argparse argument.
+    """
+
+    def __call__(self, _, namespace, values, __):
+        """Split values into set of strings."""
+        setattr(namespace, self.dest, set(values.split(",")))
+
+
+class CommaSeparateStringListAction(argparse.Action):
+    """Action that reads comma separated string into set of strings.
+
+    This action supposed to be used in argparse argument.
+    """
+
+    def __call__(self, _, namespace, values, __):
+        """Split values into list of strings."""
+        setattr(namespace, self.dest, values.split(","))

dpbench/console/entry.py

@@ -7,22 +7,11 @@
 import argparse
 from importlib.metadata import version
 
-from ._namespace import Namespace
+from ._namespace import CommaSeparateStringAction, Namespace
 from .report import add_report_arguments, execute_report
 from .run import add_run_arguments, execute_run
 
 
-class CommaSeparateStringAction(argparse.Action):
-    """Action that reads comma separated string into set of strings.
-
-    This action supposed to be used in argparse argument.
-    """
-
-    def __call__(self, _, namespace, values, __):
-        """Split values into set of strings."""
-        setattr(namespace, self.dest, set(values.split(",")))
-
-
 def parse_args() -> Namespace:
     """Parse console arguments into dpbench Namespace."""
     parser = argparse.ArgumentParser()

dpbench/console/report.py

@@ -8,7 +8,7 @@
 
 import sqlalchemy
 
-from ._namespace import Namespace
+from ._namespace import CommaSeparateStringListAction, Namespace
 
 
 def add_report_arguments(parser: argparse.ArgumentParser):
@@ -17,7 +17,16 @@ def add_report_arguments(parser: argparse.ArgumentParser):
     Args:
         parser: argument parser where arguments will be populated.
     """
-    pass
+    parser.add_argument(
+        "-c",
+        "--comparisons",
+        type=str,
+        action=CommaSeparateStringListAction,
+        nargs="?",
+        default=[],
+        help="Comma separated list of implementation pairs that need to be"
+        + " compared.",
+    )
 
 
 def execute_report(args: Namespace, conn: sqlalchemy.Engine):
@@ -29,6 +38,12 @@ def execute_report(args: Namespace, conn: sqlalchemy.Engine):
         args: object with all input arguments.
         conn: database connection.
     """
+    if len(args.comparisons) % 2 != 0:
+        raise ValueError(
+            "--comparisons must contain pairs, but odd number of"
+            + " elements was provided"
+        )
+
     import dpbench.config as cfg
     from dpbench.infrastructure.reporter import update_run_id
     from dpbench.infrastructure.runner import print_report
@@ -39,9 +54,15 @@ def execute_report(args: Namespace, conn: sqlalchemy.Engine):
         load_implementations=False,
     )
 
+    comparison_pairs = [
+        tuple(args.comparisons[i : i + 2])
+        for i in range(0, len(args.comparisons), 2)
+    ]
+
     args.run_id = update_run_id(conn, args.run_id)
     print_report(
         conn=conn,
         run_id=args.run_id,
         implementations=args.implementations,
+        comparison_pairs=comparison_pairs,
     )

dpbench/infrastructure/__init__.py

@@ -21,6 +21,7 @@
     NumbaMlirFramework,
 )
 from .reporter import (
+    generate_comparison_report,
     generate_impl_summary_report,
     generate_performance_report,
     get_unexpected_failures,
@@ -45,6 +46,7 @@
     "store_results",
     "generate_impl_summary_report",
     "generate_performance_report",
+    "generate_comparison_report",
     "get_unexpected_failures",
     "validate",
 ]

dpbench/infrastructure/reporter.py

@@ -25,6 +25,7 @@
 __all__ = [
     "generate_impl_summary_report",
     "generate_performance_report",
+    "generate_comparison_report",
 ]
 
 
@@ -216,6 +217,74 @@ def generate_performance_report(
     generate_summary(df)
 
 
+def generate_comparison_report(
+    conn: sqlalchemy.Engine,
+    run_id: int,
+    implementations: list[str],
+    comparison_pairs: list[tuple[str, str]],
+    headless=False,
+):
+    """generate comparison report with median times for each benchmark"""
+    if len(comparison_pairs) == 0:
+        return
+
+    legends = read_legends()
+
+    if not headless:
+        generate_header(conn, run_id)
+        generate_legend(legends)
+
+    columns = [
+        dm.Result.input_size_human.label("input_size"),
+        dm.Result.benchmark,
+        dm.Result.problem_preset,
+    ]
+
+    for impl in implementations:
+        columns.append(
+            func.ifnull(
+                func.max(
+                    case(
+                        (
+                            dm.Result.implementation == impl,
+                            dm.Result.median_exec_time,
+                        ),
+                    )
+                ),
+                None,
+            ).label(impl),
+        )
+
+    sql = (
+        sqlalchemy.select(*columns)
+        .group_by(
+            dm.Result.benchmark,
+            dm.Result.problem_preset,
+        )
+        .where(dm.Result.run_id == run_id)
+    )
+
+    df = pd.read_sql_query(
+        sql=sql,
+        con=conn.connect(),
+    )
+
+    for index, row in df.iterrows():
+        for target, reference in comparison_pairs:
+            if row[reference] == 0 or row[target] == 0:
+                boost = "n/a"
+            else:
+                boost = (
+                    str(round((row[target] / row[reference]) * 100, 2)) + "%"
+                )
+            df.at[index, target + "_to_" + reference] = boost
+
+    for impl in implementations:
+        df = df.drop(impl, axis=1)
+
+    generate_summary(df)
+
+
 def get_failures_from_results(
     results_db: Union[str, sqlalchemy.Engine] = "results.db",
     run_id: int = None,

dpbench/infrastructure/runner.py

@@ -213,6 +213,7 @@ def print_report(
     conn: sqlalchemy.Engine,
     run_id: int,
     implementations: set[str],
+    comparison_pairs: list[tuple[str, str]] = [],
 ):
     if not implementations:
         implementations = {impl.postfix for impl in cfg.GLOBAL.implementations}
@@ -231,6 +232,14 @@ def print_report(
         headless=True,
     )
 
+    dpbi.generate_comparison_report(
+        conn,
+        run_id=run_id,
+        implementations=implementations,
+        comparison_pairs=comparison_pairs,
+        headless=True,
+    )
+
     unexpected_failures = dpbi.get_unexpected_failures(conn, run_id=run_id)
 
     if len(unexpected_failures) > 0: