Fix CostFunc returning results containing InvalidConfig

Author: stijnh

kernel_tuner/strategies/common.py

@@ -146,13 +146,14 @@ def _run_configs(self, xs, check_restrictions=True):
         batch_keys = [] # The keys of the configs to run
         pending_indices_by_key = dict()  # Maps key => where to store result in `final_results`
         final_results = []  # List returned to the user
+        legal_indices = []  # Indices in `final_results` that are legal
 
         # Loop over all configurations. For each configurations there are four cases:
         # 1. The configuration is invalid, we can skip it
         # 2. The configuration is in  `unique_results`, we can get it from there
         # 3. The configuration is in  `pending_indices_by_key`, it is duplicate in `xs`
         # 4. The configuration must be evaluated by the runner.
-        for x in xs:
+        for index, x in enumerate(xs):
             config, is_legal = self._normalize_and_validate_config(x, check_restrictions=check_restrictions)
             logging.debug("normalize config: %s -> %s (legal: %s)", str(x), str(config), is_legal)
             key = ",".join([str(i) for i in config])
@@ -166,18 +167,19 @@ def _run_configs(self, xs, check_restrictions=True):
             # 2. Attempt to retrieve from `unique_results` 
             elif key in self.unique_results:
                 result = dict(self.unique_results[key])
+                legal_indices.append(index)
                 final_results.append(result)
 
             # 3. We have already seen this config in the current batch
             elif key in pending_indices_by_key:
-                pending_indices_by_key[key].append(len(final_results))
+                pending_indices_by_key[key].append(index)
                 final_results.append(None)
 
             # 4. A new config, we must evaluate this
             else:
                 batch_keys.append(key)
                 batch_configs.append(config)
-                pending_indices_by_key[key] = [len(final_results)]
+                pending_indices_by_key[key] = [index]
                 final_results.append(None)
 
         # compile and benchmark the batch
@@ -190,6 +192,7 @@ def _run_configs(self, xs, check_restrictions=True):
 
             # set in the results array
             for index in pending_indices_by_key[key]:
+                legal_indices.append(index)
                 final_results[index] = dict(result)
 
                 # Disable the timings. Only the first result must get these.
@@ -200,10 +203,9 @@ def _run_configs(self, xs, check_restrictions=True):
             # Put result in `unique_results`
             self.unique_results[key] = result
 
-        for result in final_results:
-            # Skip if None. Result is missing if runner exhausted the budget
-            if result is not None:
-                self.results.append(result)
+        # Only things in `legal_indices` are valid results
+        for index in sorted(legal_indices):
+            self.results.append(final_results[index])
 
         # upon returning from this function control will be given back to the strategy, so reset the start time
         self.runner.last_strategy_start_time = perf_counter()

kernel_tuner/util.py

@@ -732,7 +732,7 @@ def process_metrics(params, metrics):
     :rtype: dict
 
     """
-    if metrics:
+    if metrics is not None:
         if not isinstance(metrics, dict):
             raise ValueError("metrics should be a dictionary to preserve order and support composability")
         for k, v in metrics.items():

test/strategies/test_strategies.py

@@ -53,6 +53,7 @@ def vector_add():
         strategies.append(pytest.param(s, marks=skip_if_no_pyatf))
     else:
         strategies.append(s)
+
 @pytest.mark.parametrize('strategy', strategies)
 def test_strategies(vector_add, strategy):
     options = dict(popsize=5, neighbor='adjacent')

test/test_common.py

@@ -4,6 +4,7 @@
 import pytest
 
 import kernel_tuner.strategies.common as common
+import kernel_tuner.util
 from kernel_tuner.interface import Options
 from kernel_tuner.searchspace import Searchspace
 
@@ -13,6 +14,7 @@ def tuning_options():
     tuning_options["strategy_options"] = {}
     tuning_options["objective"] = "time"
     tuning_options["objective_higher_is_better"] = False
+    tuning_options["budget"] = kernel_tuner.util.TuningBudget()
     return tuning_options