Merge pull request #236 from IntelPython/feature/structure_framework

ZzEeKkAa · web-flow · commit 42ca2fd8c6bf · 2023-04-28T13:14:11.000-04:00
Structure frameworks
diff --git a/dpbench/config/config.py b/dpbench/config/config.py
@@ -4,7 +4,7 @@
 
 """Configuration root related configuration classes."""
 
-from dataclasses import dataclass
+from dataclasses import dataclass, field
 
 from .benchmark import Benchmark
 from .framework import Framework
@@ -15,7 +15,8 @@
 class Config:
     """Root of the configuration."""
 
-    frameworks: list[Framework]
-    benchmarks: list[Benchmark]
-    implementations: list[Implementation]
-    dtypes: dict[str, dict[str, str]]
+    frameworks: list[Framework] = field(default_factory=list)
+    benchmarks: list[Benchmark] = field(default_factory=list)
+    # deprecated. Use frameworks.postfixes instead
+    implementations: list[Implementation] = field(default_factory=list)
+    dtypes: dict[str, dict[str, str]] = field(default_factory=dict)
diff --git a/dpbench/config/framework.py b/dpbench/config/framework.py
@@ -4,8 +4,10 @@
 
 """Framework related configuration classes."""
 
-from dataclasses import dataclass
-from typing import Any
+from dataclasses import dataclass, field
+from typing import Any, List
+
+from .implementation_postfix import Implementation
 
 
 @dataclass
@@ -18,6 +20,8 @@ class Framework:
     class_: str
     arch: str
     sycl_device: str
+    dpcpp_version: str
+    postfixes: List[Implementation] = field(default_factory=list)
 
     @staticmethod
     def from_dict(obj: Any) -> "Framework":
@@ -28,6 +32,18 @@ def from_dict(obj: Any) -> "Framework":
         _class = str(obj.get("class") or "")
         _arch = str(obj.get("arch") or "")
         _sycl_device = str(obj.get("sycl_device") or "")
+        _dpcpp_version = str(obj.get("dpcpp_version") or "")
+        _postfixes = obj.get("postfixes") or []
+        for i, _postfix in enumerate(_postfixes):
+            _postfixes[i] = Implementation.from_dict(_postfix)
+        _postfixes = list[Implementation](_postfixes)
         return Framework(
-            _simple_name, _full_name, _prefix, _class, _arch, _sycl_device
+            _simple_name,
+            _full_name,
+            _prefix,
+            _class,
+            _arch,
+            _sycl_device,
+            _dpcpp_version,
+            _postfixes,
         )
diff --git a/dpbench/config/module.py b/dpbench/config/module.py
@@ -14,7 +14,6 @@ class Module:
     benchmark_configs_path: str = ""
     benchmark_configs_recursive: bool = False
     framework_configs_path: str = ""
-    impl_postfix_path: str = ""
     precision_dtypes_path: str = ""
 
     benchmarks_module: str = ""
diff --git a/dpbench/config/reader.py b/dpbench/config/reader.py
@@ -10,7 +10,6 @@
 import pkgutil
 import re
 import sys
-from typing import Callable
 
 import tomli
 
@@ -34,7 +33,7 @@ def read_configs(  # noqa: C901: TODO: move modules into config
     Returns:
         Configuration object with populated configurations.
     """
-    config: Config = Config([], [], [], None)
+    config: Config = Config()
 
     dirname: str = os.path.dirname(__file__)
 
@@ -48,9 +47,6 @@ def read_configs(  # noqa: C901: TODO: move modules into config
             framework_configs_path=os.path.join(
                 dirname, "../configs/framework_info"
             ),
-            impl_postfix_path=os.path.join(
-                dirname, "../configs/impl_postfix.toml"
-            ),
             precision_dtypes_path=os.path.join(
                 dirname, "../configs/precision_dtypes.toml"
             ),
@@ -97,13 +93,14 @@ def read_configs(  # noqa: C901: TODO: move modules into config
             )
         if mod.framework_configs_path != "":
             read_frameworks(config, mod.framework_configs_path)
-        if mod.impl_postfix_path != "":
-            read_implementation_postfixes(config, mod.impl_postfix_path)
         if mod.precision_dtypes_path != "":
             read_precision_dtypes(config, mod.precision_dtypes_path)
         if mod.path != "":
             sys.path.append(mod.path)
 
+    for framework in config.frameworks:
+        config.implementations += framework.postfixes
+
     if implementations is None:
         implementations = [impl.postfix for impl in config.implementations]
 
diff --git a/dpbench/configs/framework_info/dpcpp.toml b/dpbench/configs/framework_info/dpcpp.toml
@@ -10,3 +10,8 @@ postfix = "dpcpp"
 class = "DpcppFramework"
 arch = "cpu"
 sycl_device = "cpu"
+dpcpp_version = "IntelLLVM 2023.1.0"
+
+[[framework.postfixes]]
+impl_postfix = "sycl"
+description = "DPC++ native ext."
diff --git a/dpbench/configs/framework_info/dpcpp_version.toml b/dpbench/configs/framework_info/dpcpp_version.toml
diff --git a/dpbench/configs/framework_info/dpnp.toml b/dpbench/configs/framework_info/dpnp.toml
@@ -7,6 +7,10 @@ simple_name = "dpnp"
 full_name = "dpnp"
 prefix = "dp"
 postfix = "dpnp"
-class = "Framework"
+class = "DpnpFramework"
 arch = "cpu"
 sycl_device = "cpu"
+
+[[framework.postfixes]]
+impl_postfix = "dpnp"
+description = "dpnp"
diff --git a/dpbench/configs/framework_info/numba.toml b/dpbench/configs/framework_info/numba.toml
@@ -8,3 +8,15 @@ full_name = "Numba"
 prefix = "nb"
 class = "NumbaFramework"
 arch = "cpu"
+
+[[framework.postfixes]]
+impl_postfix = "numba_n"
+description = "Numba nopython"
+
+[[framework.postfixes]]
+impl_postfix = "numba_np"
+description = "Numba nopython, Parallel=True"
+
+[[framework.postfixes]]
+impl_postfix = "numba_npr"
+description = "Numba nopython, Parallel=True, prange"
diff --git a/dpbench/configs/framework_info/numba_dpex.toml b/dpbench/configs/framework_info/numba_dpex.toml
@@ -9,3 +9,15 @@ prefix = "dpex"
 class = "NumbaDpexFramework"
 arch = "cpu"
 sycl_device = "cpu"
+
+[[framework.postfixes]]
+impl_postfix = "numba_dpex_k"
+description = "Numba-dpex kernel"
+
+[[framework.postfixes]]
+impl_postfix = "numba_dpex_p"
+description = "Numba-dpex prange"
+
+[[framework.postfixes]]
+impl_postfix = "numba_dpex_n"
+description = "Numba-dpex NumPy API"
diff --git a/dpbench/configs/framework_info/numpy.toml b/dpbench/configs/framework_info/numpy.toml
@@ -9,3 +9,7 @@ prefix = "np"
 postfix = "numpy"
 class = "Framework"
 arch = "cpu"
+
+[[framework.postfixes]]
+impl_postfix = "numpy"
+description = "NumPy"
diff --git a/dpbench/configs/framework_info/python.toml b/dpbench/configs/framework_info/python.toml
@@ -9,3 +9,7 @@ prefix = "python"
 postfix = "python"
 class = "Framework"
 arch = "cpu"
+
+[[framework.postfixes]]
+impl_postfix = "python"
+description = "Python"
diff --git a/dpbench/configs/impl_postfix.toml b/dpbench/configs/impl_postfix.toml
diff --git a/dpbench/infrastructure/__init__.py b/dpbench/infrastructure/__init__.py
@@ -12,11 +12,13 @@
     create_run,
     store_results,
 )
-from .dpcpp_framework import DpcppFramework
-from .dpnp_framework import DpnpFramework
-from .framework import Framework
-from .numba_dpex_framework import NumbaDpexFramework
-from .numba_framework import NumbaFramework
+from .frameworks import (
+    DpcppFramework,
+    DpnpFramework,
+    Framework,
+    NumbaDpexFramework,
+    NumbaFramework,
+)
 from .reporter import (
     generate_impl_summary_report,
     generate_performance_report,
diff --git a/dpbench/infrastructure/benchmark.py b/dpbench/infrastructure/benchmark.py
@@ -24,12 +24,8 @@
 
 from . import timeout_decorator as tout
 from .datamodel import Result, store_results
-from .dpcpp_framework import DpcppFramework
-from .dpnp_framework import DpnpFramework
 from .enums import ErrorCodes, ValidationStatusCodes
-from .framework import Framework
-from .numba_dpex_framework import NumbaDpexFramework
-from .numba_framework import NumbaFramework
+from .frameworks import Framework, build_framework_map
 
 # A global namedtuple to store a function implementing a benchmark along with
 # the name of the implementation.
@@ -697,56 +693,22 @@ def _set_reference_implementation(self) -> BenchmarkImplFn:
 
         return ref_impl
 
-    def _set_impl_to_framework_map(  # noqa: C901: refactor framework load
-        self,
-        impl_fnlist,
-    ) -> dict[str, Framework]:
+    def _set_impl_to_framework_map(self) -> dict[str, Framework]:
         """Create a dictionary mapping each implementation function name to a
         corresponding Framework object.
 
-        Args:
-            impl_fnlist : list of implementation functions
-
         Returns:
             Dict: Dictionary mapping implementation function to a Framework
         """
 
-        impl_to_fw_map = dict()
-
-        for bimpl in impl_fnlist:
-            if "_numba" in bimpl.name and "_dpex" not in bimpl.name:
-                impl_to_fw_map.update({bimpl.name: NumbaFramework("numba")})
-            elif "_numpy" in bimpl.name:
-                impl_to_fw_map.update({bimpl.name: Framework("numpy")})
-            elif "_python" in bimpl.name:
-                impl_to_fw_map.update({bimpl.name: Framework("python")})
-            elif "_dpex" in bimpl.name:
-                try:
-                    fw = NumbaDpexFramework("numba_dpex")
-                    impl_to_fw_map.update({bimpl.name: fw})
-                except Exception:
-                    logging.exception(
-                        "Framework could not be "
-                        + "created for numba_dpex due to:"
-                    )
-            elif "_sycl" in bimpl.name:
-                try:
-                    fw = DpcppFramework("dpcpp")
-                    impl_to_fw_map.update({bimpl.name: fw})
-                except Exception:
-                    logging.exception(
-                        "Framework could not be created for dpcpp due to:"
-                    )
-            elif "_dpnp" in bimpl.name:
-                try:
-                    fw = DpnpFramework("dpnp")
-                    impl_to_fw_map.update({bimpl.name: fw})
-                except Exception:
-                    logging.exception(
-                        "Framework could not be created for dpcpp due to:"
-                    )
+        framework_map = build_framework_map()
 
-        return impl_to_fw_map
+        return {
+            f"{self.info.module_name}_{impl.postfix}": framework_map[
+                impl.postfix
+            ]
+            for impl in self.info.implementations
+        }
 
     def _get_validation_data(self, preset, precision):
         if preset in self.refdata.keys():
@@ -827,7 +789,7 @@ def __init__(
 
         self.impl_fnlist = self._set_implementation_fn_list()
         self.ref_impl_fn = self._set_reference_implementation()
-        self.impl_to_fw_map = self._set_impl_to_framework_map(self.impl_fnlist)
+        self.impl_to_fw_map = self._set_impl_to_framework_map()
 
     def get_impl_fnlist(self) -> list[BenchmarkImplFn]:
         """Returns a list of function objects each for a single implementation
diff --git a/dpbench/infrastructure/frameworks/__init__.py b/dpbench/infrastructure/frameworks/__init__.py
@@ -0,0 +1,19 @@
+# SPDX-FileCopyrightText: 2022 - 2023 Intel Corporation
+#
+# SPDX-License-Identifier: Apache-2.0
+
+from .dpcpp_framework import DpcppFramework
+from .dpnp_framework import DpnpFramework
+from .fabric import build_framework_map
+from .framework import Framework
+from .numba_dpex_framework import NumbaDpexFramework
+from .numba_framework import NumbaFramework
+
+__all__ = [
+    "Framework",
+    "NumbaFramework",
+    "NumbaDpexFramework",
+    "DpnpFramework",
+    "DpcppFramework",
+    "build_framework_map",
+]
diff --git a/dpbench/infrastructure/frameworks/dpcpp_framework.py b/dpbench/infrastructure/frameworks/dpcpp_framework.py
@@ -83,14 +83,4 @@ def copy_from_func(self) -> Callable:
     def version(self) -> str:
         """Returns the framework version."""
         # hack the dpcpp version, need validate dpcpp available first
-        import pathlib
-
-        import tomli
-
-        parent_folder = pathlib.Path(__file__).parent.absolute()
-        version_file = parent_folder.joinpath(
-            "..", "configs", "framework_info", "dpcpp_version.toml"
-        )
-        with open(version_file, "rb") as toml_file:
-            version = tomli.load(toml_file)["dpcpp_version"]
-        return version
+        return self.info.dpcpp_version
diff --git a/dpbench/infrastructure/frameworks/dpnp_framework.py b/dpbench/infrastructure/frameworks/dpnp_framework.py
diff --git a/dpbench/infrastructure/frameworks/fabric.py b/dpbench/infrastructure/frameworks/fabric.py
diff --git a/dpbench/infrastructure/frameworks/framework.py b/dpbench/infrastructure/frameworks/framework.py
diff --git a/dpbench/infrastructure/frameworks/numba_dpex_framework.py b/dpbench/infrastructure/frameworks/numba_dpex_framework.py
diff --git a/dpbench/infrastructure/frameworks/numba_framework.py b/dpbench/infrastructure/frameworks/numba_framework.py
