Merge branch 'hyperparametertuning' of https://github.com/KernelTuner/kernel_tuner into hyperparametertuning

Author: fjwillemsen

.gitignore

@@ -2,7 +2,7 @@
 poetry.lock
 noxenv.txt
 noxsettings.toml
-hyperparamtuning/*
+hyperparamtuning*/*
 *.prof
 
 ### Python ###

doc/requirements.txt

@@ -61,26 +61,23 @@ def compile(self, kernel_instance):
         path.mkdir(exist_ok=True)
 
         # TODO get applications & GPUs args from benchmark
-        # gpus = ["RTX_3090", "RTX_2080_Ti"]
-        # applications = None
-
-        gpus = ["A100", "W6600"]
-        folder = "../../autotuning_methodology/benchmark_hub/kernels"
+        gpus = ["A100", "A4000", "MI250X"]
+        folder = "../autotuning_methodology/benchmark_hub/kernels"
         applications = [
             {
                 "name": "dedispersion_milo",
                 "folder": folder,
                 "input_file": "dedispersion_milo.json"
             },
             {
-                "name": "convolution_milo",
+                "name": "hotspot_milo",
                 "folder": folder,
-                "input_file": "convolution_milo.json"
+                "input_file": "hotspot_milo.json"
             },
             {
-                "name": "hotspot_milo",
+                "name": "convolution_milo",
                 "folder": folder,
-                "input_file": "hotspot_milo.json"
+                "input_file": "convolution_milo.json"
             },
             {
                 "name": "gemm_milo",
@@ -104,8 +101,9 @@ def compile(self, kernel_instance):
         # any additional settings
         override = { 
             "experimental_groups_defaults": { 
-                "repeats": 10,
-                "samples": self.iterations 
+                "repeats": 25,
+                "samples": self.iterations,
+                "minimum_fraction_of_budget_valid": 0.01, 
             }
         }
 

doc/requirements_test.txt

@@ -103,16 +103,16 @@ def put_if_not_present(target_dict, key, value):
         hyperparams = {
             'popsize': [10, 20, 30],
             'maxiter': [50, 100, 150],
-            'w': [0.25, 0.5, 0.75],
+            # 'w': [0.25, 0.5, 0.75],   # disabled due to low influence according to KW-test (H=0.0215) and mutual information
             'c1': [1.0, 2.0, 3.0],
             'c2': [0.5, 1.0, 1.5]
         }
     elif strategy_to_tune.lower() == "greedy_ils":
         hyperparams = {
             'neighbor': ['Hamming', 'adjacent'],
             'restart': [True, False],
-            'no_improvement': [1, 10, 25, 33, 50, 66, 75, 100, 200],
-            'random_walk': [0.01, 0.1, 0.2, 0.3, 0.4, 0.5, 0.6, 0.7, 0.8, 0.9, 0.99]
+            'no_improvement': [10, 25, 50, 75],
+            'random_walk': [0.1, 0.2, 0.3, 0.4, 0.5]
         }
     elif strategy_to_tune.lower() == "dual_annealing":
         hyperparams = {
@@ -127,7 +127,7 @@ def put_if_not_present(target_dict, key, value):
     elif strategy_to_tune.lower() == "basinhopping":
         hyperparams = {
             'method': ["Nelder-Mead", "Powell", "CG", "BFGS", "L-BFGS-B", "TNC", "COBYLA", "SLSQP"],
-            'T': [0.5, 1.0, 1.5],
+            'T': [0.1, 0.25, 0.5, 0.75, 1.0, 1.25, 1.5],
         }
     elif strategy_to_tune.lower() == "genetic_algorithm":
         hyperparams = {
@@ -136,12 +136,27 @@ def put_if_not_present(target_dict, key, value):
             'maxiter': [50, 100, 150],
             'mutation_chance': [5, 10, 20]
         }
-    elif strategy_to_tune.lower() == "mls":
+    elif strategy_to_tune.lower() == "greedy_mls":
         hyperparams = {
             'neighbor': ["Hamming", "adjacent"],
             'restart': [True, False],
             'randomize': [True, False]
         }
+    elif strategy_to_tune.lower() == "simulated_annealing":
+        hyperparams = {
+            'T': [0.5, 1.0, 1.5],
+            'T_min': [0.0001, 0.001, 0.01],
+            'alpha': [0.9925, 0.995, 0.9975],
+            'maxiter': [1, 2, 3]
+        }
+    elif strategy_to_tune.lower() == "bayes_opt":
+        hyperparams = {
+            # 'covariancekernel': ["constantrbf", "rbf", "matern32", "matern52"],
+            'covariancelengthscale': [1.0, 1.5, 2.0],
+            'method': ["poi", "ei", "lcb", "lcb-srinivas", "multi", "multi-advanced", "multi-fast", "multi-ultrafast"],
+            'samplingmethod': ["random", "LHS"],
+            'popsize': [10, 20, 30]
+        }
     else:
         raise ValueError(f"Invalid argument {strategy_to_tune=}")
 

kernel_tuner/backends/hypertuner.py

@@ -4,7 +4,7 @@
 
 from jsonschema import validate
 
-from kernel_tuner import util
+from kernel_tuner.util import get_instance_string, looks_like_a_filename, read_file
 
 #specifies for a number of pre-defined objectives whether
 #the objective should be minimized or maximized (boolean value denotes higher is better)
@@ -205,8 +205,8 @@ def top_result(item):
         meta["version_number"] = "1.0"
         meta["kernel_name"] = kernel_name
         if kernel_string and not callable(kernel_string) and not isinstance(kernel_string, list):
-            if util.looks_like_a_filename(kernel_string):
-                meta["kernel_string"] = util.read_file(kernel_string)
+            if looks_like_a_filename(kernel_string):
+                meta["kernel_string"] = read_file(kernel_string)
             else:
                 meta["kernel_string"] = kernel_string
         meta["objective"] = objective
@@ -337,7 +337,7 @@ def _select_best_common_config(results, objective, objective_higher_is_better):
     for config in results:
         params = config["tunable_parameters"]
 
-        config_str = util.get_instance_string(params)
+        config_str = get_instance_string(params)
         #count occurances
         results_table[config_str] = results_table.get(config_str,0) + 1
         #add to performance

kernel_tuner/hyper.py

@@ -50,11 +50,6 @@
 from kernel_tuner.strategies import (
     basinhopping,
     bayes_opt,
-    bayes_opt_alt_BOTorch,
-    bayes_opt_BOTorch,
-    bayes_opt_GPyTorch,
-    bayes_opt_GPyTorch_lean,
-    bayes_opt_old,
     brute_force,
     diff_evo,
     dual_annealing,
@@ -85,12 +80,7 @@
     "pso": pso,
     "simulated_annealing": simulated_annealing,
     "firefly_algorithm": firefly_algorithm,
-    "bayes_opt": bayes_opt,
-    "bayes_opt_old": bayes_opt_old,
-    "bayes_opt_GPyTorch": bayes_opt_GPyTorch,
-    "bayes_opt_GPyTorch_lean": bayes_opt_GPyTorch_lean,
-    "bayes_opt_BOTorch": bayes_opt_BOTorch,
-    "bayes_opt_BOTorch_alt": bayes_opt_alt_BOTorch,
+    "bayes_opt": bayes_opt
 }
 
 
@@ -886,6 +876,16 @@ def tune_kernel_T1(
     problem_size = kernelspec["ProblemSize"]
     device = kernelspec["Device"]["Name"]
     strategy = inputs["Search"]["Name"]
+    if "Attributes" in inputs["Search"]:
+        strategy_options = {}
+        for attribute in inputs["Search"]["Attributes"]:
+            strategy_options[attribute["Name"]] = attribute["Value"]
+    if "Budget" in inputs:
+        budget = inputs["Budget"][0]
+        assert budget["Type"] == "ConfigurationCount"
+        if strategy_options is None:
+            strategy_options = {}
+        strategy_options["max_fevals"] = budget["BudgetValue"]
 
     # set the cache path
     if cache_filepath is None and "SimulationInput" in kernelspec:
@@ -908,6 +908,8 @@ def tune_kernel_T1(
                 tune_param = eval(vals)
             else:
                 tune_param = literal_eval(vals)
+        if param["Type"] == "string":
+            tune_param = eval(param["Values"])
         if tune_param is not None:
             tune_params[param["Name"]] = tune_param
         else:

kernel_tuner/integration.py

@@ -4,6 +4,7 @@
 from pathlib import Path
 from random import choice, shuffle
 from typing import List, Union
+from warnings import warn
 
 import numpy as np
 from constraint import (
@@ -69,9 +70,7 @@ def __init__(
             ), "When `from_cache` is used, the positional arguments must be set to None."
             tune_params = from_cache["tune_params"]
         if from_cache is None:
-            assert (
-                tune_params is not None and max_threads is not None
-            ), "Must specify positional arguments."
+            assert tune_params is not None and max_threads is not None, "Must specify positional arguments."
 
         # set the object attributes using the arguments
         framework_l = framework.lower()
@@ -237,6 +236,7 @@ def __build_searchspace_bruteforce(self, block_size_names: list, max_threads: in
                     isinstance(self._modified_restrictions, list)
                     and block_size_restriction_spaced not in self._modified_restrictions
                 ):
+                    print(f"added default block size restriction '{block_size_restriction_spaced}'")
                     self._modified_restrictions.append(block_size_restriction_spaced)
                     if isinstance(self.restrictions, list):
                         self.restrictions.append(block_size_restriction_spaced)
@@ -864,6 +864,11 @@ def get_random_sample_indices(self, num_samples: int) -> np.ndarray:
 
     def get_random_sample(self, num_samples: int) -> List[tuple]:
         """Get the parameter configurations for a random, non-conflicting sample (caution: not unique in consecutive calls)."""
+        if self.size < num_samples:
+            warn(
+                f"Too many samples requested ({num_samples}), reducing the number of samples to the searchspace size ({self.size})"
+            )
+            num_samples = self.size
         return self.get_param_configs_at_indices(self.get_random_sample_indices(num_samples))
 
     def get_neighbors_indices_no_cache(self, param_config: tuple, neighbor_method=None) -> List[int]:

kernel_tuner/interface.py

@@ -1,7 +1,7 @@
 """The strategy that uses the basinhopping global optimization method."""
 import scipy.optimize
 
-from kernel_tuner import util
+from kernel_tuner.util import StopCriterionReached
 from kernel_tuner.searchspace import Searchspace
 from kernel_tuner.strategies import common
 from kernel_tuner.strategies.common import CostFunc, setup_method_arguments, setup_method_options
@@ -31,7 +31,7 @@ def tune(searchspace: Searchspace, runner, tuning_options):
     try:
         opt_result = scipy.optimize.basinhopping(cost_func, x0, T=T, stepsize=eps,
                                              minimizer_kwargs=minimizer_kwargs, disp=tuning_options.verbose)
-    except util.StopCriterionReached as e:
+    except StopCriterionReached as e:
         if tuning_options.verbose:
             print(e)
 

kernel_tuner/searchspace.py

@@ -149,7 +149,7 @@ def tune(searchspace: Searchspace, runner, tuning_options):
         if max_fevals - bo.fevals <= 0:
             raise ValueError("No function evaluations left for optimization after sampling")
         bo.optimize(max_fevals)
-    except util.StopCriterionReached as e:
+    except StopCriterionReached as e:
         if tuning_options.verbose:
             print(e)
 
@@ -238,7 +238,7 @@ def get_hyperparam(name: str, default, supported_values=list()):
         self.invalid_value = 1e20
         self.opt_direction = opt_direction
         if opt_direction == "min":
-            self.worst_value = np.PINF
+            self.worst_value = np.inf
             self.argopt = np.argmin
         elif opt_direction == "max":
             self.worst_value = np.NINF
@@ -265,7 +265,7 @@ def get_hyperparam(name: str, default, supported_values=list()):
         self.__visited_num = 0
         self.__visited_valid_num = 0
         self.__visited_searchspace_indices = [False] * self.searchspace_size
-        self.__observations = [np.NaN] * self.searchspace_size
+        self.__observations = [np.nan] * self.searchspace_size
         self.__valid_observation_indices = [False] * self.searchspace_size
         self.__valid_params = list()
         self.__valid_observations = list()
@@ -314,7 +314,7 @@ def is_not_visited(self, index: int) -> bool:
 
     def is_valid(self, observation: float) -> bool:
         """Returns whether an observation is valid."""
-        return not (observation is None or observation == self.invalid_value or observation == np.NaN)
+        return not (observation is None or observation == self.invalid_value or observation == np.nan)
 
     def get_af_by_name(self, name: str):
         """Get the basic acquisition functions by their name."""