Add initial support for scikit-optimize minimize methods (skopt)

Author: stijnh

kernel_tuner/interface.py

@@ -65,6 +65,7 @@
     pyatf_strategies,
     random_sample,
     simulated_annealing,
+    skopt
 )
 from kernel_tuner.strategies.wrapper import OptAlgWrapper
 
@@ -82,6 +83,7 @@
     "mls": mls,
     "pso": pso,
     "simulated_annealing": simulated_annealing,
+    "skopt": skopt,
     "firefly_algorithm": firefly_algorithm,
     "bayes_opt": bayes_opt,
     "pyatf_strategies": pyatf_strategies,
@@ -394,6 +396,7 @@ def __deepcopy__(self, _):
             * "pso" particle swarm optimization
             * "random_sample" takes a random sample of the search space
             * "simulated_annealing" simulated annealing strategy
+            * "skopt" uses the minimization methods from `skopt`
 
         Strategy-specific parameters and options are explained under strategy_options.
 
@@ -594,6 +597,7 @@ def tune_kernel(
 
     kernelsource = core.KernelSource(kernel_name, kernel_source, lang, defines)
 
+    print("block_size_names", block_size_names)
     _check_user_input(kernel_name, kernelsource, arguments, block_size_names)
 
     # default objective if none is specified
@@ -676,6 +680,7 @@ def preprocess_cache(filepath):
 
     # create search space
     tuning_options.restrictions_unmodified = deepcopy(restrictions)
+    print(searchspace_construction_options)
     searchspace = Searchspace(tune_params, restrictions, runner.dev.max_threads, **searchspace_construction_options)
     restrictions = searchspace._modified_restrictions
     tuning_options.restrictions = restrictions

kernel_tuner/strategies/common.py

@@ -73,6 +73,7 @@ def __init__(
         snap=True,
         return_invalid=False,
         return_raw=None,
+        invalid_value=sys.float_info.max,
     ):
         """An abstract method to handle evaluation of configurations.
 
@@ -100,6 +101,7 @@ def __init__(
             self.return_raw = f"{tuning_options['objective']}s"
         self.results = []
         self.budget_spent_fraction = 0.0
+        self.invalid_return_value = invalid_value
 
 
     def __call__(self, x, check_restrictions=True):
@@ -168,7 +170,7 @@ def __call__(self, x, check_restrictions=True):
         else:
             # this is not a valid configuration, replace with float max if needed
             if not self.return_invalid:
-                return_value = sys.float_info.max
+                return_value = self.invalid_return_value
 
         # include raw data in return if requested
         if self.return_raw is not None:

kernel_tuner/strategies/skopt.py

@@ -0,0 +1,72 @@
+"""The strategy that uses a minimizer method for searching through the parameter space."""
+
+from kernel_tuner.util import StopCriterionReached
+from kernel_tuner.searchspace import Searchspace
+from kernel_tuner.strategies.common import (
+    CostFunc,
+    get_options,
+    scale_from_params,
+    get_strategy_docstring,
+)
+
+supported_methods = ["forest", "gbrt", "gp", "dummy"]
+
+_options = dict(
+        method=(f"Local optimization algorithm to use, choose any from {supported_methods}", "gp"),
+        options=("Options passed to the skopt method as kwargs.", dict()),
+        popsize=("Number of initial samples. If `None`, let skopt choose the initial population", None),
+        maxiter=("Maximum number of times to repeat the method until the budget is exhausted.", 1),
+)
+
+def tune(searchspace: Searchspace, runner, tuning_options):
+    import skopt
+
+    cost_func = CostFunc(searchspace, tuning_options, runner, scaling=True, invalid_value=1e9)
+    bounds, _, eps = cost_func.get_bounds_x0_eps()
+
+    method, skopt_options, popsize, maxiter = get_options(tuning_options.strategy_options, _options)
+
+    # Get maximum number of evaluations
+    max_fevals = searchspace.size
+    if "max_fevals" in tuning_options:
+        max_fevals = min(tuning_options["max_fevals"], max_fevals)
+
+    # Set the maximum number of calls to 100 times the maximum number of evaluations.
+    # Not all calls by skopt will result in an evaluation, due to restrictions or
+    # since different calls might map to the same configuration.
+    if "n_calls" not in skopt_options:
+        skopt_options["n_calls"] = 100 * max_fevals
+
+    # If the initial population size is specified, we select `popsize` samples
+    # from the search space. This is more efficient than letting skopt select
+    # the samples as it is not aware of restrictions.
+    if popsize:
+        x0 = searchspace.get_random_sample(min(popsize, max_fevals))
+        skopt_options["x0"] = [list(scale_from_params(x, searchspace.tune_params, eps)) for x in x0]
+
+
+    opt_result = None
+
+    try:
+        for _ in range(maxiter):
+            if method == "dummy":
+                opt_result = skopt.dummy_minimize(cost_func, bounds, **skopt_options)
+            elif method == "forest":
+                opt_result = skopt.forest_minimize(cost_func, bounds, **skopt_options)
+            elif method == "gp":
+                opt_result = skopt.gp_minimize(cost_func, bounds, **skopt_options)
+            elif method == "gbrt":
+                opt_result = skopt.gbrt_minimize(cost_func, bounds, **skopt_options)
+            else:
+                raise ValueError(f"invalid skopt method: {method}")
+    except StopCriterionReached as e:
+        if tuning_options.verbose:
+            print(e)
+
+    if opt_result and tuning_options.verbose:
+        print(opt_result.message)
+
+    return cost_func.results
+
+
+tune.__doc__ = get_strategy_docstring("skopt minimize", _options)