Update parameters, testing

Author: till-m

bayes_opt/bayesian_optimization.py

@@ -16,7 +16,7 @@
 from bayes_opt.constraint import ConstraintModel
 from bayes_opt.event import DEFAULT_EVENTS, Events
 from bayes_opt.logger import _get_default_logger
-from bayes_opt.parameter import wrap_kernel
+from bayes_opt.parameter import WrappedKernel
 from bayes_opt.target_space import TargetSpace
 from bayes_opt.util import ensure_rng
 
@@ -152,7 +152,7 @@ def __init__(
 
         # Internal GP regressor
         self._gp = GaussianProcessRegressor(
-            kernel=wrap_kernel(Matern(nu=2.5), transform=self._space.kernel_transform),
+            kernel=WrappedKernel(Matern(nu=2.5), transform=self._space.kernel_transform),
             alpha=1e-6,
             normalize_y=True,
             n_restarts_optimizer=5,
@@ -329,4 +329,6 @@ def set_bounds(self, new_bounds: BoundsMapping) -> None:
 
     def set_gp_params(self, **params: Any) -> None:
         """Set parameters of the internal Gaussian Process Regressor."""
+        if "kernel" in params:
+            params["kernel"] = WrappedKernel(params["kernel"], self._space.kernel_transform)
         self._gp.set_params(**params)

bayes_opt/constraint.py

@@ -9,7 +9,7 @@
 from sklearn.gaussian_process import GaussianProcessRegressor
 from sklearn.gaussian_process.kernels import Matern
 
-from bayes_opt.parameter import wrap_kernel
+from bayes_opt.parameter import WrappedKernel
 
 if TYPE_CHECKING:
     from collections.abc import Callable
@@ -71,7 +71,7 @@ def __init__(
 
         self._model = [
             GaussianProcessRegressor(
-                kernel=wrap_kernel(Matern(nu=2.5), transform) if transform is not None else Matern(nu=2.5),
+                kernel=WrappedKernel(Matern(nu=2.5), transform) if transform is not None else Matern(nu=2.5),
                 alpha=1e-6,
                 normalize_y=True,
                 n_restarts_optimizer=5,

bayes_opt/parameter.py

@@ -4,7 +4,6 @@
 
 import abc
 from collections.abc import Sequence
-from inspect import signature
 from numbers import Number
 from typing import TYPE_CHECKING, Any, Callable, Union
 
@@ -375,8 +374,6 @@ def to_float(self, value: Any) -> NDArray[Float]:
         """
         res = np.zeros(len(self.categories))
         one_hot_index = [i for i, val in enumerate(self.categories) if val == value]
-        if len(one_hot_index) != 1:
-            raise ValueError
         res[one_hot_index] = 1
         return res.astype(float)
 
@@ -432,7 +429,7 @@ def kernel_transform(self, value: NDArray[Float]) -> NDArray[Float]:
         """
         value = np.atleast_2d(value)
         res = np.zeros(value.shape)
-        res[np.argmax(value, axis=0)] = 1
+        res[:, np.argmax(value, axis=1)] = 1
         return res
 
     @property
@@ -441,52 +438,68 @@ def dim(self) -> int:
         return len(self.categories)
 
 
-def wrap_kernel(kernel: kernels.Kernel, transform: Callable[[Any], Any]) -> kernels.Kernel:
-    """Wrap a kernel to transform input data before passing it to the kernel.
+class WrappedKernel(kernels.Kernel):
+    """Wrap a kernel with a parameter transformation.
+
+    The transform function is applied to the input before passing it to the base kernel.
 
     Parameters
     ----------
-    kernel : kernels.Kernel
-        The kernel to wrap.
+    base_kernel : kernels.Kernel
 
-    transform : Callable
-        The transformation function to apply to the input data.
+    transform : Callable[[Any], Any]
+    """
 
-    Returns
-    -------
-    kernels.Kernel
-        The wrapped kernel.
+    def __init__(self, base_kernel: kernels.Kernel, transform: Callable[[Any], Any]) -> None:
+        super().__init__()
+        self.base_kernel = base_kernel
+        self.transform = transform
 
-    Notes
-    -----
-    See https://arxiv.org/abs/1805.03463 for more information.
-    """
-    kernel_type = type(kernel)
+    def __call__(self, X: NDArray[Float], Y: NDArray[Float] = None, eval_gradient: bool = False) -> Any:
+        """Return the kernel k(X, Y) and optionally its gradient after applying the transform.
 
-    class WrappedKernel(kernel_type):
-        @copy_signature(getattr(kernel_type.__init__, "deprecated_original", kernel_type.__init__))
-        def __init__(self, **kwargs: Any) -> None:
-            super().__init__(**kwargs)
+        For details, see the documentation of the base kernel.
 
-        def __call__(self, X: Any, Y: Any = None, eval_gradient: bool = False) -> Any:
-            X = transform(X)
-            return super().__call__(X, Y, eval_gradient)
+        Parameters
+        ----------
+        X :  ndarray of shape (n_samples_X, n_features)
+            Left argument of the returned kernel k(X, Y).
 
-        def __reduce__(self) -> str | tuple[Any, ...]:
-            return (wrap_kernel, (kernel, transform))
+        Y : ndarray of shape (n_samples_Y, n_features), default=None
+            Right argument of the returned kernel k(X, Y). If None, k(X, X) is evaluated.
 
-    return WrappedKernel(**kernel.get_params())
+        eval_gradient : bool, default=False
+            Determines whether the gradient with respect to the kernel hyperparameter is calculated.
 
+        Returns
+        -------
+        K : ndarray of shape (n_samples_X, n_samples_Y)
 
-def copy_signature(source_fct: Callable[..., Any]) -> Callable[[Callable[..., Any]], Callable[..., Any]]:
-    """Clones a signature from a source function to a target function.
+        K_gradient : ndarray of shape (n_samples_X, n_samples_X, n_dims)
+        """
+        X = self.transform(X)
+        return self.base_kernel(X, Y, eval_gradient)
 
-    via
-    https://stackoverflow.com/a/58989918/
-    """
+    def is_stationary(self):
+        """Return whether the kernel is stationary."""
+        return self.base_kernel.is_stationary()
 
-    def copy(target_fct: Callable[..., Any]) -> Callable[..., Any]:
-        target_fct.__signature__ = signature(source_fct)
-        return target_fct
+    def diag(self, X: NDArray[Float]) -> NDArray[Float]:
+        """Return the diagonal of k(X, X).
 
-    return copy
+        This method allows for more efficient calculations than calling
+        np.diag(self(X)).
+
+
+        Parameters
+        ----------
+        X : array-like of shape (n_samples,)
+            Left argument of the returned kernel k(X, Y)
+
+        Returns
+        -------
+        K_diag : ndarray of shape (n_samples_X,)
+            Diagonal of kernel k(X, X)
+        """
+        X = self.transform(X)
+        return self.base_kernel.diag(X)

examples/parameter_types.ipynb

@@ -49,15 +49,13 @@ def gboost(log_learning_rate, max_depth, min_samples_split):
 
     continuous_optimizer = BayesianOptimization(
         f=gboost,
-        acquisition_function=acquisition.ExpectedImprovement(1e-1),
         pbounds=continuous_pbounds,
         verbose=0,
         random_state=42,
     )
 
     discrete_optimizer = BayesianOptimization(
         f=gboost,
-        acquisition_function=acquisition.ExpectedImprovement(1e-1),
         pbounds=discrete_pbounds,
         verbose=0,
         random_state=42,

examples/typed_hyperparameter_tuning.py

@@ -1,7 +1,9 @@
 from __future__ import annotations
 
 import numpy as np
+import pytest
 
+from bayes_opt import BayesianOptimization
 from bayes_opt.parameter import CategoricalParameter, FloatParameter, IntParameter
 from bayes_opt.target_space import TargetSpace
 
@@ -38,6 +40,11 @@ def target_func(**kwargs):
 
     assert (space.target == np.array([target1, target2])).all()
 
+    p1 = space._params_config["p1"]
+    assert p1.to_float(0.2) == 0.2
+    assert p1.to_float(np.array(2.3)) == 2.3
+    assert p1.to_float(3) == 3.0
+
 
 def test_int_parameters():
     def target_func(**kwargs):
@@ -68,6 +75,15 @@ def target_func(**kwargs):
 
     assert (space.target == np.array([target1, target2])).all()
 
+    p1 = space._params_config["p1"]
+    assert p1.to_float(0) == 0.0
+    assert p1.to_float(np.array(2)) == 2.0
+    assert p1.to_float(3) == 3.0
+
+    assert p1.kernel_transform(0) == 0.0
+    assert p1.kernel_transform(2.3) == 2.0
+    assert p1.kernel_transform(np.array([1.3, 3.6, 7.2])) == pytest.approx(np.array([1, 4, 7]))
+
 
 def test_cat_parameters():
     fruit_ratings = {"apple": 1.0, "banana": 2.0, "mango": 5.0, "honeydew melon": -10.0, "strawberry": np.pi}
@@ -102,6 +118,23 @@ def target_func(fruit: str):
 
     assert (space.target == np.array([target1, target2])).all()
 
+    p1 = space._params_config["fruit"]
+    for i, fruit in enumerate(fruits):
+        assert (p1.to_float(fruit) == np.eye(5)[i]).all()
+
+    assert (p1.kernel_transform(np.array([0.8, 0.2, 0.3, 0.5, 0.78])) == np.array([1, 0, 0, 0, 0])).all()
+    assert (p1.kernel_transform(np.array([0.78, 0.2, 0.3, 0.5, 0.8])) == np.array([0, 0, 0, 0, 1.0])).all()
+
+
+def test_cateogrical_valid_bounds():
+    pbounds = {"fruit": ("apple", "banana", "mango", "honeydew melon", "banana", "strawberry")}
+    with pytest.raises(ValueError):
+        TargetSpace(None, pbounds)
+
+    pbounds = {"fruit": ("apple",)}
+    with pytest.raises(ValueError):
+        TargetSpace(None, pbounds)
+
 
 def test_to_string():
     pbounds = {"p1": (0, 1), "p2": (1, 2)}
@@ -133,3 +166,20 @@ def test_to_string():
     assert space._params_config["fruit"].to_string("mango", 5) == "mango"
     assert space._params_config["fruit"].to_string("honeydew melon", 10) == "honeyde..."
     assert space._params_config["fruit"].to_string("strawberry", 10) == "strawberry"
+
+
+def test_integration_mixed_optimization():
+    fruit_ratings = {"apple": 1.0, "banana": 2.0, "mango": 5.0, "honeydew melon": -10.0, "strawberry": np.pi}
+
+    pbounds = {
+        "p1": (0, 1),
+        "p2": (1, 2),
+        "p3": (-1, 3, int),
+        "fruit": ("apple", "banana", "mango", "honeydew melon", "strawberry"),
+    }
+
+    def target_func(p1, p2, p3, fruit):
+        return p1 + p2 + p3 + fruit_ratings[fruit]
+
+    optimizer = BayesianOptimization(target_func, pbounds)
+    optimizer.maximize(init_points=2, n_iter=10)