Remove sorting, gradient optimize only continuous params

Author: till-m

bayes_opt/acquisition.py

@@ -215,15 +215,22 @@ def _acq_min(
         if n_random == 0 and n_l_bfgs_b == 0:
             error_msg = "Either n_random or n_l_bfgs_b needs to be greater than 0."
             raise ValueError(error_msg)
-        x_min_r, min_acq_r = self._random_sample_minimize(acq, space, n_random=n_random)
-        x_min_l, min_acq_l = self._l_bfgs_b_minimize(acq, space, n_x_seeds=n_l_bfgs_b)
-        # Either n_random or n_l_bfgs_b is not 0 => at least one of x_min_r and x_min_l is not None
-        if min_acq_r < min_acq_l:
-            return x_min_r
-        return x_min_l
+        x_min_r, min_acq_r, x_seeds = self._random_sample_minimize(
+            acq, space, n_random=max(n_random, n_l_bfgs_b), n_x_seeds=n_l_bfgs_b
+        )
+        if n_l_bfgs_b:
+            x_min_l, min_acq_l = self._l_bfgs_b_minimize(acq, space, x_seeds=x_seeds)
+            # Either n_random or n_l_bfgs_b is not 0 => at least one of x_min_r and x_min_l is not None
+            if min_acq_r > min_acq_l:
+                return x_min_l
+        return x_min_r
 
     def _random_sample_minimize(
-        self, acq: Callable[[NDArray[Float]], NDArray[Float]], space: TargetSpace, n_random: int
+        self,
+        acq: Callable[[NDArray[Float]], NDArray[Float]],
+        space: TargetSpace,
+        n_random: int,
+        n_x_seeds: int = 0,
     ) -> tuple[NDArray[Float] | None, float]:
         """Random search to find the minimum of `acq` function.
 
@@ -238,6 +245,8 @@ def _random_sample_minimize(
         n_random : int
             Number of random samples to use.
 
+        n_x_seeds : int
+            Number of top points to return, for use as starting points for L-BFGS-B.
         Returns
         -------
         x_min : np.ndarray
@@ -252,10 +261,18 @@ def _random_sample_minimize(
         ys = acq(x_tries)
         x_min = x_tries[ys.argmin()]
         min_acq = ys.min()
-        return x_min, min_acq
+        if n_x_seeds != 0:
+            idxs = np.argsort(ys)[-n_x_seeds:]
+            x_seeds = x_tries[idxs]
+        else:
+            x_seeds = []
+        return x_min, min_acq, x_seeds
 
     def _l_bfgs_b_minimize(
-        self, acq: Callable[[NDArray[Float]], NDArray[Float]], space: TargetSpace, n_x_seeds: int = 10
+        self,
+        acq: Callable[[NDArray[Float]], NDArray[Float]],
+        space: TargetSpace,
+        x_seeds: NDArray[Float] | None = None,
     ) -> tuple[NDArray[Float] | None, float]:
         """Random search to find the minimum of `acq` function.
 
@@ -267,8 +284,8 @@ def _l_bfgs_b_minimize(
         space : TargetSpace
             The target space over which to optimize.
 
-        n_x_seeds : int
-            Number of starting points for the L-BFGS-B optimizer.
+        x_seeds : int
+            Starting points for the L-BFGS-B optimizer.
 
         Returns
         -------
@@ -278,24 +295,35 @@ def _l_bfgs_b_minimize(
         min_acq : float
             Acquisition function value at `x_min`
         """
-        if n_x_seeds == 0:
-            return None, np.inf
-        x_seeds = space.random_sample(n_x_seeds, random_state=self.random_state)
+        continuous_dimensions = space.continuous_dimensions
+        continuous_bounds = space.bounds[continuous_dimensions]
+
+        if not continuous_dimensions.any():
+            min_acq = np.inf
+            x_min = np.array([np.nan] * space.bounds.shape[0])
+            return x_min, min_acq
 
         min_acq: float | None = None
         x_try: NDArray[Float]
         x_min: NDArray[Float]
         for x_try in x_seeds:
-            # Find the minimum of minus the acquisition function
-            res: OptimizeResult = minimize(acq, x_try, bounds=space.bounds, method="L-BFGS-B")
 
+            def continuous_acq(x: NDArray[Float], x_try=x_try) -> NDArray[Float]:
+                x_try[continuous_dimensions] = x
+                return acq(x_try)
+
+            # Find the minimum of minus the acquisition function
+            res: OptimizeResult = minimize(
+                continuous_acq, x_try[continuous_dimensions], bounds=continuous_bounds, method="L-BFGS-B"
+            )
             # See if success
             if not res.success:
                 continue
 
             # Store it if better than previous minimum(maximum).
             if min_acq is None or np.squeeze(res.fun) >= min_acq:
-                x_min = res.x
+                x_try[continuous_dimensions] = res.x
+                x_min = x_try
                 min_acq = np.squeeze(res.fun)
 
         if min_acq is None:

bayes_opt/bayesian_optimization.py

@@ -14,6 +14,7 @@
 
 from bayes_opt import acquisition
 from bayes_opt.constraint import ConstraintModel
+from bayes_opt.domain_reduction import DomainTransformer
 from bayes_opt.event import DEFAULT_EVENTS, Events
 from bayes_opt.logger import _get_default_logger
 from bayes_opt.parameter import WrappedKernel
@@ -162,11 +163,10 @@ def __init__(
         self._verbose = verbose
         self._bounds_transformer = bounds_transformer
         if self._bounds_transformer:
-            try:
-                self._bounds_transformer.initialize(self._space)
-            except (AttributeError, TypeError) as exc:
-                error_msg = "The transformer must be an instance of DomainTransformer"
-                raise TypeError(error_msg) from exc
+            if not isinstance(self._bounds_transformer, DomainTransformer):
+                msg = "The transformer must be an instance of DomainTransformer"
+                raise TypeError(msg)
+            self._bounds_transformer.initialize(self._space)
 
         super().__init__(events=DEFAULT_EVENTS)
 
@@ -330,5 +330,7 @@ def set_bounds(self, new_bounds: BoundsMapping) -> None:
     def set_gp_params(self, **params: Any) -> None:
         """Set parameters of the internal Gaussian Process Regressor."""
         if "kernel" in params:
-            params["kernel"] = WrappedKernel(params["kernel"], self._space.kernel_transform)
+            params["kernel"] = WrappedKernel(
+                base_kernel=params["kernel"], transform=self._space.kernel_transform
+            )
         self._gp.set_params(**params)

bayes_opt/domain_reduction.py

@@ -75,13 +75,7 @@ def __init__(
         self.gamma_pan = gamma_pan
         self.eta = eta
 
-        self.minimum_window_value: NDArray[Float] | Sequence[float] | float
-        if isinstance(minimum_window, Mapping):
-            self.minimum_window_value = [
-                item[1] for item in sorted(minimum_window.items(), key=lambda x: x[0])
-            ]
-        else:
-            self.minimum_window_value = minimum_window
+        self.minimum_window_value = minimum_window
 
     def initialize(self, target_space: TargetSpace) -> None:
         """Initialize all of the parameters.
@@ -91,6 +85,11 @@ def initialize(self, target_space: TargetSpace) -> None:
         target_space : TargetSpace
             TargetSpace this DomainTransformer operates on.
         """
+        if isinstance(self.minimum_window_value, Mapping):
+            self.minimum_window_value = [self.minimum_window_value[key] for key in target_space.keys]
+        else:
+            self.minimum_window_value = self.minimum_window_value
+
         any_not_float = any([not isinstance(p, FloatParameter) for p in target_space._params_config.values()])
         if any_not_float:
             msg = "Domain reduction is only supported for all-FloatParameter optimization."

bayes_opt/parameter.py

@@ -59,6 +59,11 @@ def bounds(self) -> NDArray[Any]:
         """The bounds of the parameter in float space."""
         return self._bounds
 
+    @property
+    @abc.abstractmethod
+    def is_continuous(self) -> bool:
+        """Whether the parameter is continuous."""
+
     def random_sample(
         self, n_samples: int, random_state: np.random.RandomState | int | None
     ) -> NDArray[Float]:
@@ -161,6 +166,11 @@ class FloatParameter(BayesParameter):
     def __init__(self, name: str, bounds: tuple[float, float]) -> None:
         super().__init__(name, np.array(bounds))
 
+    @property
+    def is_continuous(self) -> bool:
+        """Whether the parameter is continuous."""
+        return True
+
     def to_float(self, value: float) -> float:
         """Convert a parameter value to a float.
 
@@ -248,6 +258,11 @@ class IntParameter(BayesParameter):
     def __init__(self, name: str, bounds: tuple[int, int]) -> None:
         super().__init__(name, np.array(bounds))
 
+    @property
+    def is_continuous(self) -> bool:
+        """Whether the parameter is continuous."""
+        return False
+
     def random_sample(
         self, n_samples: int, random_state: np.random.RandomState | int | None
     ) -> NDArray[Float]:
@@ -340,6 +355,11 @@ def __init__(self, name: str, categories: Sequence[Any]) -> None:
         bounds = np.vstack((lower, upper)).T
         super().__init__(name, bounds)
 
+    @property
+    def is_continuous(self) -> bool:
+        """Whether the parameter is continuous."""
+        return False
+
     def random_sample(
         self, n_samples: int, random_state: np.random.RandomState | int | None
     ) -> NDArray[Float]:
@@ -478,6 +498,7 @@ def __call__(self, X: NDArray[Float], Y: NDArray[Float] = None, eval_gradient: b
         K_gradient : ndarray of shape (n_samples_X, n_samples_X, n_dims)
         """
         X = self.transform(X)
+        Y = self.transform(Y) if Y is not None else None
         return self.base_kernel(X, Y, eval_gradient)
 
     def is_stationary(self):

bayes_opt/target_space.py

@@ -82,7 +82,7 @@ def __init__(
         self.target_func = target_func
 
         # Get the name of the parameters
-        self._keys: list[str] = sorted(pbounds)
+        self._keys: list[str] = list(pbounds.keys())
 
         self._params_config = self.make_params(pbounds)
         self._dim = sum([self._params_config[key].dim for key in self._keys])
@@ -180,6 +180,11 @@ def keys(self) -> list[str]:
         """
         return self._keys
 
+    @property
+    def params_config(self) -> dict[str, BayesParameter]:
+        """Get the parameters configuration."""
+        return self._params_config
+
     @property
     def bounds(self) -> NDArray[Float]:
         """Get the bounds of this TargetSpace.
@@ -210,6 +215,20 @@ def masks(self) -> dict[str, NDArray[np.bool_]]:
         """
         return self._masks
 
+    @property
+    def continuous_dimensions(self) -> NDArray[np.bool_]:
+        """Get the continuous parameters.
+
+        Returns
+        -------
+        dict
+        """
+        result = np.zeros(self.dim, dtype=bool)
+        masks = self.masks
+        for key in self.keys:
+            result[masks[key]] = self._params_config[key].is_continuous
+        return result
+
     def make_params(self, pbounds: BoundsMapping) -> dict[str, BayesParameter]:
         """Create a dictionary of parameters from a dictionary of bounds.
 
@@ -226,7 +245,7 @@ def make_params(self, pbounds: BoundsMapping) -> dict[str, BayesParameter]:
             parameter objects as values.
         """
         params: dict[str, BayesParameter] = {}
-        for key in sorted(pbounds):
+        for key in pbounds:
             pbound = pbounds[key]
 
             if isinstance(pbound, BayesParameter):
@@ -285,8 +304,7 @@ def params_to_array(self, params: Mapping[str, float | NDArray[Float]]) -> NDArr
         """
         if set(params) != set(self.keys):
             error_msg = (
-                f"Parameters' keys ({sorted(params)}) do "
-                f"not match the expected set of keys ({self.keys})."
+                f"Parameters' keys ({params}) do " f"not match the expected set of keys ({self.keys})."
             )
             raise ValueError(error_msg)
         return self._to_float(params)
@@ -337,9 +355,7 @@ def array_to_params(self, x: NDArray[Float]) -> dict[str, float | NDArray[Float]
 
     def _to_float(self, value: Mapping[str, float | NDArray[Float]]) -> NDArray[Float]:
         if set(value) != set(self.keys):
-            msg = (
-                f"Parameters' keys ({sorted(value)}) do " f"not match the expected set of keys ({self.keys})."
-            )
+            msg = f"Parameters' keys ({value}) do " f"not match the expected set of keys ({self.keys})."
             raise ValueError(msg)
         res = np.zeros(self._dim)
         for key in self._keys:
@@ -389,8 +405,7 @@ def _as_array(self, x: Any) -> NDArray[Float]:
         x = x.ravel()
         if x.size != self.dim:
             error_msg = (
-                f"Size of array ({len(x)}) is different than the "
-                f"expected number of parameters ({len(self.keys)})."
+                f"Size of array ({len(x)}) is different than the " f"expected number of ({len(self.dim)})."
             )
             raise ValueError(error_msg)
         return x
@@ -666,8 +681,7 @@ def set_bounds(self, new_bounds: BoundsMapping) -> None:
         new_bounds : dict
             A dictionary with the parameter name and its new bounds
         """
-        print(new_bounds)
-        new__params_config = self.make_params(new_bounds)
+        new_params_config = self.make_params(new_bounds)
 
         for key in self.keys:
             if key in new_bounds:
@@ -676,12 +690,12 @@ def set_bounds(self, new_bounds: BoundsMapping) -> None:
                 ) == set(new_bounds[key]):
                     msg = "Changing bounds of categorical parameters is not supported"
                     raise NotImplementedError(msg)
-                if not isinstance(new__params_config[key], type(self._params_config[key])):
+                if not isinstance(new_params_config[key], type(self._params_config[key])):
                     msg = (
-                        f"Parameter type {type(new__params_config[key])} of"
+                        f"Parameter type {type(new_params_config[key])} of"
                         " new bounds does not match parameter type"
                         f" {type(self._params_config[key])} of old bounds"
                     )
                     raise ValueError(msg)
-                self._params_config[key] = new__params_config[key]
+                self._params_config[key] = new_params_config[key]
         self._bounds = self.calculate_bounds()

examples/parameter_types.ipynb

@@ -2,5 +2,5 @@
 set -ex
 
 poetry run ruff format bayes_opt tests
-poetry run ruff check bayes_opt tests --fix
+poetry run ruff check bayes_opt --fix
 

scripts/format.sh

@@ -114,7 +114,7 @@ def fun(x):
         except IndexError:
             return np.nan
 
-    _, min_acq_l = acq._l_bfgs_b_minimize(fun, space=target_space, n_x_seeds=1)
+    _, min_acq_l = acq._l_bfgs_b_minimize(fun, space=target_space, x_seeds=np.array([[2.5, 0.5]]))
     assert min_acq_l == np.inf
 
 

tests/test_acquisition.py

@@ -38,7 +38,7 @@ def test_register():
     assert len(optimizer.res) == 1
     assert len(optimizer.space) == 1
 
-    optimizer.space.register(params={"p1": 5, "p2": 4}, target=9)
+    optimizer.space.register(params=np.array([5, 4]), target=9)
     assert len(optimizer.res) == 2
     assert len(optimizer.space) == 2
 
@@ -196,12 +196,12 @@ def test_set_bounds():
     # Ignore unknown keys
     optimizer.set_bounds({"other": (7, 8)})
     assert all(optimizer.space.bounds[:, 0] == np.array([0, 0, 0, 0]))
-    assert all(optimizer.space.bounds[:, 1] == np.array([1, 2, 3, 4]))
+    assert all(optimizer.space.bounds[:, 1] == np.array([1, 3, 2, 4]))
 
     # Update bounds accordingly
     optimizer.set_bounds({"p2": (1, 8)})
-    assert all(optimizer.space.bounds[:, 0] == np.array([0, 1, 0, 0]))
-    assert all(optimizer.space.bounds[:, 1] == np.array([1, 8, 3, 4]))
+    assert all(optimizer.space.bounds[:, 0] == np.array([0, 0, 1, 0]))
+    assert all(optimizer.space.bounds[:, 1] == np.array([1, 3, 8, 4]))
 
 
 def test_set_gp_params():