Merge pull request #235 from IntelPython/feature/set_max_complexity_to_10

ZzEeKkAa · web-flow · commit 1bcde4d1354d · 2023-04-28T11:11:56.000-04:00
Set max complexity to 10
diff --git a/.flake8 b/.flake8
@@ -12,5 +12,4 @@ exclude =
     ./dpbench/benchmarks/pairwise_distance/pairwise_distance_numba_dpex_k.py
     ./dpbench/benchmarks/pairwise_distance/pairwise_distance_numba_dpex_p.py
     ./dpbench/benchmarks/pairwise_distance/pairwise_distance_numba_npr.py
-# TODO: simplify to 10
-max-complexity = 14
+max-complexity = 10
diff --git a/dpbench/benchmarks/dbscan/dbscan_numba_dpex_k.py b/dpbench/benchmarks/dbscan/dbscan_numba_dpex_k.py
@@ -74,7 +74,9 @@ def get_neighborhood(n, dim, data, eps, ind_lst, sz_lst, block_size, nblocks):
 
 
 @nb.njit(parallel=False, fastmath=True)
-def compute_clusters(n, min_pts, assignments, sizes, indices_list):
+def compute_clusters(  # noqa: C901: TODO: can we simplify logic?
+    n, min_pts, assignments, sizes, indices_list
+):
     nclusters = 0
     nnoise = 0
     for i in range(n):
diff --git a/dpbench/benchmarks/dbscan/dbscan_numba_dpex_p.py b/dpbench/benchmarks/dbscan/dbscan_numba_dpex_p.py
@@ -75,7 +75,9 @@ def get_neighborhood(n, dim, data, eps, ind_lst, sz_lst):
 
 
 @nb.njit(parallel=False, fastmath=True)
-def compute_clusters(n, min_pts, assignments, sizes, indices_list):
+def compute_clusters(  # noqa: C901: TODO: can we simplify logic?
+    n, min_pts, assignments, sizes, indices_list
+):
     nclusters = 0
     nnoise = 0
     for i in range(n):
diff --git a/dpbench/benchmarks/dbscan/dbscan_numba_n.py b/dpbench/benchmarks/dbscan/dbscan_numba_n.py
@@ -73,7 +73,9 @@ def get_neighborhood(n, dim, data, eps, ind_lst, sz_lst):
 
 
 @nb.njit(parallel=False, fastmath=True)
-def compute_clusters(n, min_pts, assignments, sizes, indices_list):
+def compute_clusters(  # noqa: C901: TODO: can we simplify logic?
+    n, min_pts, assignments, sizes, indices_list
+):
     nclusters = 0
     nnoise = 0
     for i in range(n):
diff --git a/dpbench/benchmarks/dbscan/dbscan_numba_npr.py b/dpbench/benchmarks/dbscan/dbscan_numba_npr.py
@@ -73,7 +73,9 @@ def get_neighborhood(n, dim, data, eps, ind_lst, sz_lst):
 
 
 @nb.njit(parallel=False, fastmath=True)
-def compute_clusters(n, min_pts, assignments, sizes, indices_list):
+def compute_clusters(  # noqa: C901: TODO: can we simplify logic?
+    n, min_pts, assignments, sizes, indices_list
+):
     nclusters = 0
     nnoise = 0
     for i in range(n):
diff --git a/dpbench/benchmarks/knn/knn_numba_dpex_k.py b/dpbench/benchmarks/knn/knn_numba_dpex_k.py
@@ -9,7 +9,7 @@
 
 
 @dpex.kernel
-def _knn_kernel(
+def _knn_kernel(  # noqa: C901: TODO: can we simplify logic?
     train,
     train_labels,
     test,
diff --git a/dpbench/benchmarks/knn/knn_numba_dpex_p.py b/dpbench/benchmarks/knn/knn_numba_dpex_p.py
@@ -8,7 +8,7 @@
 
 
 @dpjit
-def knn(
+def knn(  # noqa: C901: TODO: can we simplify logic?
     x_train,
     y_train,
     x_test,
diff --git a/dpbench/benchmarks/knn/knn_numba_npr.py b/dpbench/benchmarks/knn/knn_numba_npr.py
@@ -7,7 +7,7 @@
 
 
 @nb.njit(parallel=True, fastmath=True)
-def knn(
+def knn(  # noqa: C901: TODO: can we simplify logic?
     x_train,
     y_train,
     x_test,
diff --git a/dpbench/benchmarks/knn/knn_python.py b/dpbench/benchmarks/knn/knn_python.py
@@ -7,7 +7,7 @@
 import numpy as np
 
 
-def knn(
+def knn(  # noqa: C901: TODO: can we simplify logic?
     x_train,
     y_train,
     x_test,
diff --git a/dpbench/benchmarks/npbench/mandelbrot1/mandelbrot1_numba_n.py b/dpbench/benchmarks/npbench/mandelbrot1/mandelbrot1_numba_n.py
@@ -20,7 +20,9 @@ def linspace(start, stop, num, dtype):
 
 
 @nb.jit(nopython=True, parallel=False, fastmath=True)
-def mandelbrot(xmin, xmax, ymin, ymax, xn, yn, maxiter, horizon=2.0):
+def mandelbrot(  # noqa: C901: TODO: can we simplify logic?
+    xmin, xmax, ymin, ymax, xn, yn, maxiter, horizon=2.0
+):
     # Adapted from https://www.ibm.com/developerworks/community/blogs/jfp/...
     #              .../entry/How_To_Compute_Mandelbrodt_Set_Quickly?lang=en
     # X = np.linspace(xmin, xmax, xn, dtype=np.float64)
diff --git a/dpbench/benchmarks/npbench/mandelbrot1/mandelbrot1_numba_np.py b/dpbench/benchmarks/npbench/mandelbrot1/mandelbrot1_numba_np.py
@@ -20,7 +20,9 @@ def linspace(start, stop, num, dtype):
 
 
 @nb.jit(nopython=True, parallel=True, fastmath=True)
-def mandelbrot(xmin, xmax, ymin, ymax, xn, yn, maxiter, horizon=2.0):
+def mandelbrot(  # noqa: C901: TODO: can we simplify logic?
+    xmin, xmax, ymin, ymax, xn, yn, maxiter, horizon=2.0
+):
     # Adapted from https://www.ibm.com/developerworks/community/blogs/jfp/...
     #              .../entry/How_To_Compute_Mandelbrodt_Set_Quickly?lang=en
     # X = np.linspace(xmin, xmax, xn, dtype=np.float64)
diff --git a/dpbench/config/reader.py b/dpbench/config/reader.py
@@ -21,9 +21,9 @@
 from .module import Module
 
 
-def read_configs(
+def read_configs(  # noqa: C901: TODO: move modules into config
     benchmarks: list[str] = None,
-    postfixes: list[str] = None,
+    implementations: list[str] = None,
 ) -> Config:
     """Read all configuration files and populate those settings into Config.
 
@@ -104,14 +104,13 @@ def read_configs(
         if mod.path != "":
             sys.path.append(mod.path)
 
+    if implementations is None:
+        implementations = [impl.postfix for impl in config.implementations]
+
     for benchmark in config.benchmarks:
-        postfixes_tmp = postfixes
-        if postfixes_tmp is None:
-            postfixes_tmp = [impl.postfix for impl in config.implementations]
         read_benchmark_implementations(
             benchmark,
-            config.implementations,
-            postfixes=postfixes_tmp,
+            implementations,
         )
 
     return config
@@ -136,22 +135,20 @@ def read_benchmarks(
     for bench_info_file in os.listdir(bench_info_dir):
         bench_info_file_path = os.path.join(bench_info_dir, bench_info_file)
 
-        if not os.path.isfile(bench_info_file_path):
-            if recursive:
-                read_benchmarks(
-                    config=config,
-                    bench_info_dir=bench_info_file_path,
-                    recursive=recursive,
-                    parent_package=parent_package + "." + bench_info_file,
-                    benchmarks=benchmarks,
-                )
-
-            continue
-
-        if not bench_info_file.endswith(".toml"):
-            continue
+        if os.path.isdir(bench_info_file_path) and recursive:
+            read_benchmarks(
+                config=config,
+                bench_info_dir=bench_info_file_path,
+                recursive=recursive,
+                parent_package=parent_package + "." + bench_info_file,
+                benchmarks=benchmarks,
+            )
 
-        if benchmarks and not bench_info_file[:-5] in benchmarks:
+        if (
+            not os.path.isfile(bench_info_file_path)
+            or not bench_info_file.endswith(".toml")
+            or (benchmarks and not bench_info_file[:-5] in benchmarks)
+        ):
             continue
 
         with open(bench_info_file_path) as file:
@@ -290,17 +287,15 @@ def discover_module_name_and_postfix(module: str, config: Config):
 
 def read_benchmark_implementations(
     config: Benchmark,
-    known_implementations: list[Implementation],
-    postfixes: list[str] = None,
+    implementations: list[str] = None,
 ) -> None:
     """Read and discover implementation modules and functions.
 
     Args:
         config: Benchmark configuration object where settings should be
             populated.
-        postfixes: List of postfixes to import. Set it to None to import all
-            available implementations. It does not affect initialization import.
-        implementations: Prepopulated list of implementations.
+        implementations: List of postfixes to import. It does not affect
+            initialization import.
 
     Raises:
         RuntimeError: Implementation file does not match any known postfix.
@@ -326,14 +321,9 @@ def read_benchmark_implementations(
     for module in modules:
         module_name, postfix = discover_module_name_and_postfix(module, config)
 
-        if postfixes and postfix not in postfixes:
-            continue
-
-        if config.init and config.init.module_name.endswith(module_name):
-            continue
-
-        if postfix not in [impl.postfix for impl in known_implementations]:
-            logging.warning(f"Skipping postfix: {module}")
+        if (postfix not in implementations) or (
+            config.init and config.init.module_name.endswith(module_name)
+        ):
             continue
 
         func_name: str = None
diff --git a/dpbench/infrastructure/benchmark.py b/dpbench/infrastructure/benchmark.py
