Merge pull request #230 from IntelPython/feature/allow_run_specific_implementations

Allow run specific implementations

Author: ZzEeKkAa

dpbench/infrastructure/reporter.py

@@ -105,7 +105,7 @@ def generate_summary(data: pd.DataFrame):
 def generate_impl_summary_report(
     results_db: Union[str, sqlalchemy.Engine] = "results.db",
     run_id: int = None,
-    postfixes: list[str] = None,
+    implementations: list[str] = None,
 ):
     """generate implementation summary report with status of each benchmark"""
     conn = update_connection(results_db=results_db)
@@ -121,10 +121,10 @@ def generate_impl_summary_report(
         dm.Result.problem_preset,
     ]
 
-    if postfixes is None:
-        postfixes = [impl.postfix for impl in cfg.GLOBAL.implementations]
+    if implementations is None:
+        implementations = [impl.postfix for impl in cfg.GLOBAL.implementations]
 
-    for impl in postfixes:
+    for impl in implementations:
         columns.append(
             func.ifnull(
                 func.max(
@@ -161,23 +161,28 @@ def generate_impl_summary_report(
 def generate_performance_report(
     results_db: Union[str, sqlalchemy.Engine] = "results.db",
     run_id: int = None,
+    implementations: list[str] = None,
+    headless=False,
 ):
     """generate performance report with median times for each benchmark"""
     conn = update_connection(results_db=results_db)
     run_id = update_run_id(conn, run_id)
     legends = read_legends()
 
-    generate_header(conn, run_id)
-    generate_legend(legends)
+    if not headless:
+        generate_header(conn, run_id)
+        generate_legend(legends)
 
     columns = [
         dm.Result.input_size_human.label("input_size"),
         dm.Result.benchmark,
         dm.Result.problem_preset,
     ]
 
-    for _, row in legends.iterrows():
-        impl = row["impl_postfix"]
+    if implementations is None:
+        implementations = [impl.postfix for impl in cfg.GLOBAL.implementations]
+
+    for impl in implementations:
         columns.append(
             func.ifnull(
                 func.max(
@@ -207,9 +212,7 @@ def generate_performance_report(
     )
 
     for index, row in df.iterrows():
-        for _, legend_row in legends.iterrows():
-            impl = legend_row["impl_postfix"]
-
+        for impl in implementations:
             time = row[impl]
             if time:
                 NANOSECONDS_IN_MILISECONDS: Final[float] = 1000 * 1000.0

dpbench/runner.py

@@ -78,6 +78,28 @@ def run_benchmark(
     print_results=True,
     run_id: int = None,
 ):
+    """Run specific benchmark.
+
+    Args:
+        bname (str, semi-optional): Name of the benchmark. Either name, either
+            configuration must be provided.
+        benchmark (Benchmark, semi-optional): Benchmark configuration. Either
+            name, either configuration must be provided.
+        implementation_postfix: (str, optional): Implementation postfixes
+            to be executed. If not provided, all possible implementations will
+            be executed.
+        preset (str, optional): Problem size. Defaults to "S".
+        repeat (int, optional): Number of repetitions. Defaults to 1.
+        validate (bool, optional): Whether to validate against NumPy.
+            Defaults to True.
+        timeout (float, optional): Timeout setting. Defaults to 10.0.
+        conn: connection to database. If not provided results won't be stored.
+        print_results (bool, optional): Either print results. Defaults to True.
+        run_id (int, optional): Either store result to specific run_id.
+            If not provided, new run_id will be created.
+
+    Returns: nothing.
+    """
     bench_cfg = get_benchmark(benchmark=benchmark, benchmark_name=bname)
     bname = bench_cfg.name
     print("")
@@ -119,37 +141,45 @@ def run_benchmarks(
     repeat=10,
     validate=True,
     timeout=200.0,
-    dbfile=None,
     print_results=True,
     run_id=None,
+    implementations: list[str] = None,
 ):
     """Run all benchmarks in the dpbench benchmark directory
+
     Args:
-        bconfig_path (str, optional): Path to benchmark configurations.
-        Defaults to None.
         preset (str, optional): Problem size. Defaults to "S".
         repeat (int, optional): Number of repetitions. Defaults to 1.
         validate (bool, optional): Whether to validate against NumPy.
-        Defaults to True.
+            Defaults to True.
         timeout (float, optional): Timeout setting. Defaults to 10.0.
+        print_results (bool, optional): Either print results. Defaults to True.
+        run_id (int, optional): Either store result to specific run_id.
+            If not provided, new run_id will be created.
+        implementations: (list[str], optional): List of implementation postfixes
+            to be executed. If not provided, all possible implementations will
+            be executed.
+
+    Returns: nothing.
     """
 
     print("===============================================================")
     print("")
     print("***Start Running DPBench***")
-    if not dbfile:
-        dbfile = "results.db"
 
     dpbi.create_results_table()
-    conn = dpbi.create_connection(db_file=dbfile)
+    conn = dpbi.create_connection(db_file="results.db")
     if run_id is None:
         run_id = dpbi.create_run(conn)
 
+    if implementations is None:
+        implementations = [impl.postfix for impl in cfg.GLOBAL.implementations]
+
     for b in cfg.GLOBAL.benchmarks:
-        for impl in cfg.GLOBAL.implementations:
+        for impl in implementations:
             run_benchmark(
                 benchmark=b,
-                implementation_postfix=impl.postfix,
+                implementation_postfix=impl,
                 preset=preset,
                 repeat=repeat,
                 validate=validate,
@@ -167,6 +197,14 @@ def run_benchmarks(
     print("===============================================================")
     print("")
 
-    dpbi.generate_impl_summary_report(conn, run_id=run_id)
+    if print_results:
+        dpbi.generate_impl_summary_report(
+            conn, run_id=run_id, implementations=implementations
+        )
 
-    return dbfile
+        dpbi.generate_performance_report(
+            conn,
+            run_id=run_id,
+            implementations=implementations,
+            headless=True,
+        )