re-add support for user-specified starting point

Author: benvanwerkhoven

doc/source/optimization.rst

@@ -46,6 +46,12 @@ cache files, serving a value from the cache for the first time in the run also c
 Only unique function evaluations are counted, so the second time a parameter configuration is selected by the strategy it is served from the 
 cache, but not counted as a unique function evaluation.
 
+All optimization algorithms, except for brute_force, random_sample, and bayes_opt, allow the user to specify an initial guess or 
+starting point for the optimization, called ``x0``. This can be passed to the strategy using the ``strategy_options=`` dictionary with ``"x0"`` as key and
+a list of values for each parameter in tune_params to note the starting point. For example, for a kernel that has parameters ``block_size_x`` (64, 128, 256)
+and ``tile_size_x`` (1,2,3), one could pass ``strategy_options=dict(x0=[128,2])`` to ``tune_kernel()`` to make sure the strategy starts from
+the configuration with ``block_size_x=128, tile_size_x=2``. The order in the ``x0`` list should match the order in the tunable parameters dictionary.
+
 Below all the strategies are listed with their strategy-specific options that can be passed in a dictionary to the ``strategy_options=`` argument
 of ``tune_kernel()``.
 

kernel_tuner/strategies/common.py

@@ -44,7 +44,7 @@ def make_strategy_options_doc(strategy_options):
 
 def get_options(strategy_options, options):
     """Get the strategy-specific options or their defaults from user-supplied strategy_options."""
-    accepted = list(options.keys()) + ["max_fevals", "time_limit"]
+    accepted = list(options.keys()) + ["max_fevals", "time_limit", "x0"]
     for key in strategy_options:
         if key not in accepted:
             raise ValueError(f"Unrecognized option {key} in strategy_options")
@@ -114,6 +114,11 @@ def __call__(self, x, check_restrictions=True):
 
         return return_value
 
+    def get_start_pos(self):
+        """Get starting position for optimization."""
+        _, x0, _ = self.get_bounds_x0_eps()
+        return x0
+
     def get_bounds_x0_eps(self):
         """Compute bounds, x0 (the initial guess), and eps."""
         values = list(self.searchspace.tune_params.values())
@@ -130,7 +135,7 @@ def get_bounds_x0_eps(self):
             bounds = [(0, eps * len(v)) for v in values]
             if x0:
                 # x0 has been supplied by the user, map x0 into [0, eps*len(v)]
-                x0 = scale_from_params(x0, self.tuning_options, eps)
+                x0 = scale_from_params(x0, self.searchspace.tune_params, eps)
             else:
                 # get a valid x0
                 pos = list(self.searchspace.get_random_sample(1)[0])

kernel_tuner/strategies/diff_evo.py

@@ -15,12 +15,11 @@
 
 def tune(searchspace: Searchspace, runner, tuning_options):
 
-
     method, popsize, maxiter = common.get_options(tuning_options.strategy_options, _options)
 
     # build a bounds array as needed for the optimizer
     cost_func = CostFunc(searchspace, tuning_options, runner)
-    bounds = cost_func.get_bounds()
+    bounds, x0, _ = cost_func.get_bounds_x0_eps()
 
     # ensure particles start from legal points
     population = list(list(p) for p in searchspace.get_random_sample(popsize))
@@ -29,7 +28,7 @@ def tune(searchspace: Searchspace, runner, tuning_options):
     opt_result = None
     try:
         opt_result = differential_evolution(cost_func, bounds, maxiter=maxiter, popsize=popsize, init=population,
-                                        polish=False, strategy=method, disp=tuning_options.verbose)
+                                        polish=False, strategy=method, disp=tuning_options.verbose, x0=x0)
     except util.StopCriterionReached as e:
         if tuning_options.verbose:
             print(e)

kernel_tuner/strategies/firefly_algorithm.py

@@ -21,7 +21,7 @@ def tune(searchspace: Searchspace, runner, tuning_options):
     cost_func = CostFunc(searchspace, tuning_options, runner, scaling=True)
 
     # using this instead of get_bounds because scaling is used
-    bounds, _, eps = cost_func.get_bounds_x0_eps()
+    bounds, x0, eps = cost_func.get_bounds_x0_eps()
 
     num_particles, maxiter, B0, gamma, alpha = common.get_options(tuning_options.strategy_options, _options)
 
@@ -38,6 +38,9 @@ def tune(searchspace: Searchspace, runner, tuning_options):
     for i, particle in enumerate(swarm):
         particle.position = scale_from_params(population[i], searchspace.tune_params, eps)
 
+    # include user provided starting point
+    swarm[0].position = x0
+
     # compute initial intensities
     for j in range(num_particles):
         try:

kernel_tuner/strategies/genetic_algorithm.py

@@ -27,6 +27,8 @@ def tune(searchspace: Searchspace, runner, tuning_options):
 
     population = list(list(p) for p in searchspace.get_random_sample(pop_size))
 
+    population[0] = cost_func.get_start_pos()
+
     for generation in range(generations):
 
         # determine fitness of population members

kernel_tuner/strategies/greedy_ils.py

@@ -31,7 +31,7 @@ def tune(searchspace: Searchspace, runner, tuning_options):
     cost_func = CostFunc(searchspace, tuning_options, runner)
 
     #while searching
-    candidate = searchspace.get_random_sample(1)[0]
+    candidate = cost_func.get_start_pos()
     best_score = cost_func(candidate, check_restrictions=False)
 
     last_improvement = 0

kernel_tuner/strategies/greedy_mls.py

@@ -24,9 +24,15 @@ def tune(searchspace: Searchspace, runner, tuning_options):
 
     fevals = 0
 
+    first_candidate = cost_func.get_start_pos()
+
     #while searching
     while fevals < max_fevals:
-        candidate = searchspace.get_random_sample(1)[0]
+        if first_candidate:
+            candidate = first_candidate
+            first_candidate = None
+        else:
+            candidate = searchspace.get_random_sample(1)[0]
 
         try:
             base_hillclimb(candidate, neighbor, max_fevals, searchspace, tuning_options, cost_func, restart=restart, randomize=randomize, order=order)

kernel_tuner/strategies/pso.py

@@ -21,8 +21,7 @@ def tune(searchspace: Searchspace, runner, tuning_options):
     cost_func = CostFunc(searchspace, tuning_options, runner, scaling=True)
 
     #using this instead of get_bounds because scaling is used
-    bounds, _, eps = cost_func.get_bounds_x0_eps()
-
+    bounds, x0, eps = cost_func.get_bounds_x0_eps()
 
     num_particles, maxiter, w, c1, c2 = common.get_options(tuning_options.strategy_options, _options)
 
@@ -39,6 +38,9 @@ def tune(searchspace: Searchspace, runner, tuning_options):
     for i, particle in enumerate(swarm):
         particle.position = scale_from_params(population[i], searchspace.tune_params, eps)
 
+    # include user provided starting point
+    swarm[0].position = x0
+
     # start optimization
     for i in range(maxiter):
         if tuning_options.verbose:

kernel_tuner/strategies/simulated_annealing.py

@@ -30,7 +30,7 @@ def tune(searchspace: Searchspace, runner, tuning_options):
     max_feval = tuning_options.strategy_options.get("max_fevals", max_iter)
 
     # get random starting point and evaluate cost
-    pos = list(searchspace.get_random_sample(1)[0])
+    pos = cost_func.get_start_pos()
     old_cost = cost_func(pos, check_restrictions=False)
 
     # main optimization loop

test/strategies/test_strategies.py

@@ -78,3 +78,17 @@ def test_strategies(vector_add, strategy):
         for expected_key, expected_type in expected_items.items():
             assert expected_key in res
             assert isinstance(res[expected_key], expected_type)
+
+    # check if strategy respects user-specified starting point (x0)
+    if not strategy in ["brute_force", "random_sample", "bayes_opt"]:
+        x0 = [256]
+        filter_options["x0"] = x0
+
+        results, _ = kernel_tuner.tune_kernel(*vector_add, strategy=strategy, strategy_options=filter_options,
+                                         verbose=False, cache=cache_filename, simulation_mode=True)
+
+        assert results[0]["block_size_x"] == x0[0]
+
+
+
+