Move ref impl to config

Author: ZzEeKkAa

dpbench/config/benchmark.py

@@ -88,6 +88,7 @@ class Benchmark:
     array_args: List[str] = field(default_factory=list)
     output_args: List[str] = field(default_factory=list)
     implementations: List[BenchmarkImplementation] = field(default_factory=list)
+    reference_implementation_postfix: str = None
     expected_failure_implementations: List[str] = field(default_factory=list)
 
     @staticmethod
@@ -108,6 +109,9 @@ def from_dict(obj: Any) -> "Benchmark":
         _array_args = obj.get("array_args") or []
         _output_args = obj.get("output_args") or []
         _implementations = obj.get("implementations") or []
+        _reference_implementation_postfix = obj.get(
+            "reference_implementation_postfix"
+        )
         _expected_failure_implementations = (
             obj.get("expected_failure_implementations") or []
         )
@@ -126,5 +130,6 @@ def from_dict(obj: Any) -> "Benchmark":
             _array_args,
             _output_args,
             _implementations,
+            _reference_implementation_postfix,
             _expected_failure_implementations,
         )

dpbench/config/reader.py

@@ -19,6 +19,8 @@
 from .implementation_postfix import Implementation
 from .module import Module
 
+_REFERENCE_IMPLEMENTATIONS = {"numpy", "python"}
+
 
 def read_configs(  # noqa: C901: TODO: move modules into config
     benchmarks: set[str] = None,
@@ -207,6 +209,7 @@ def read_frameworks(
                 postfix
                 for postfix in framework.postfixes
                 if postfix.postfix in implementations
+                or postfix.postfix in _REFERENCE_IMPLEMENTATIONS
             ]
 
         if len(framework.postfixes) == 0:
@@ -215,24 +218,6 @@ def read_frameworks(
         config.frameworks.append(framework)
 
 
-def read_implementation_postfixes(
-    config: Config, impl_postfix_file: str
-) -> None:
-    """Read and populate implementation postfix configuration file.
-
-    Args:
-        config: Configuration object where settings should be populated.
-        impl_postfix_file: Path to the configuration file.
-    """
-    with open(impl_postfix_file) as file:
-        file_contents = file.read()
-
-    implementation_postfixes = tomli.loads(file_contents)
-    for impl in implementation_postfixes["implementations"]:
-        implementation = Implementation.from_dict(impl)
-        config.implementations.append(implementation)
-
-
 def read_precision_dtypes(config: Config, precision_dtypes_file: str) -> None:
     """Read and populate dtype_obj data types file.
 
@@ -341,13 +326,15 @@ def read_benchmark_implementations(
     ]
 
     setup_init(config, modules)
+    set_default_reference_implementation_postfix(config, modules)
 
     for module in modules:
         module_name, postfix = discover_module_name_and_postfix(module, config)
 
         if (
             implementations
             and (postfix not in implementations)
+            and (postfix != config.reference_implementation_postfix)
             or (config.init and config.init.module_name.endswith(module_name))
         ):
             continue
@@ -382,6 +369,34 @@ def read_benchmark_implementations(
         )
 
 
+def set_default_reference_implementation_postfix(
+    config: Benchmark,
+    modules: set[str] = None,
+):
+    """Sets reference implementation postfix if not set.
+
+    It will set it to 'numpy' or 'python' with priority to 'numpy' depending on
+    the available modules.
+
+    Args:
+        config: Benchmark configuration object where settings should be
+            populated.
+        modules: List of modules in benchmark implementation dir.
+    """
+    if config.reference_implementation_postfix:
+        return
+
+    postfixes = {
+        discover_module_name_and_postfix(module, config)[1]
+        for module in modules
+    }
+
+    for postfix in _REFERENCE_IMPLEMENTATIONS:
+        if postfix in postfixes:
+            config.reference_implementation_postfix = postfix
+            break
+
+
 def get_benchmark_index(configs: list[Benchmark], module_name: str) -> int:
     """Finds configuration index by module name."""
     return next(

dpbench/infrastructure/benchmark.py

@@ -363,21 +363,19 @@ def _set_reference_implementation(self) -> BenchmarkImplFn:
 
         """
 
-        ref_impl = None
-
-        python_impl = [
-            impl for impl in self.impl_fnlist if "python" in impl.name
+        reference_implementations = [
+            impl
+            for impl in self.impl_fnlist
+            if self.info.reference_implementation_postfix in impl.name
         ]
-        numpy_impl = [impl for impl in self.impl_fnlist if "numpy" in impl.name]
 
-        if numpy_impl:
-            ref_impl = numpy_impl[0]
-        elif python_impl:
-            ref_impl = python_impl[0]
-        else:
+        print(self.impl_fnlist)
+        print(reference_implementations)
+
+        if len(reference_implementations) == 0:
             raise RuntimeError("No reference implementation")
 
-        return ref_impl
+        return reference_implementations[0]
 
     def _set_impl_to_framework_map(self) -> dict[str, Framework]:
         """Create a dictionary mapping each implementation function name to a