Add worst known feasible value to constrained problems (#3016)

Summary:
Pull Request resolved: #3016

Add the worst known feasible value to the constrained BoTorch test functions. This is useful when evaluating the performance of different optimization methods as this value can be assigned to all infeasible trials. This has the desirable property that any feasible trial has better performance than an infeasible trial.

Reviewed By: esantorella

Differential Revision: D82384518

fbshipit-source-id: ed8b9b7aeec174da11d5bbbe40caa867ec265e87

Author: David Eriksson

botorch/test_functions/base.py

@@ -239,6 +239,7 @@ class ConstrainedBaseTestProblem(BaseTestProblem, ABC):
     num_constraints: int
     _check_grad_at_opt: bool = False
     constraint_noise_std: None | float | list[float] = None
+    _worst_feasible_value: float | None = None
 
     def evaluate_slack(self, X: Tensor, noise: bool = True) -> Tensor:
         r"""Evaluate the constraint slack on a set of points.
@@ -313,6 +314,26 @@ def _evaluate_slack_true(self, X: Tensor) -> Tensor:
         """
         pass  # pragma: no cover
 
+    @property
+    def worst_feasible_value(self) -> float:
+        r"""The worst feasible value of the objective function. This is useful when
+        evaluating the performance of different optimization methods as this value
+        can be assigned to all infeasible trials. This has the desirable property that
+        any feasible trial has better performance than an infeasible trial.
+        """
+        if isinstance(self, MultiObjectiveTestProblem):
+            return 0.0  # Can return 0.0 for MOO since this is the smallest possible HV
+        elif self._worst_feasible_value is not None:
+            return (
+                -self._worst_feasible_value
+                if self.negate
+                else self._worst_feasible_value
+            )
+        raise NotImplementedError(
+            f"Problem {self.__class__.__name__} does not specify the "
+            "worst feasible value."
+        )
+
 
 class MultiObjectiveTestProblem(BaseTestProblem, ABC):
     r"""Base class for multi-objective test functions.

botorch/test_functions/synthetic.py

@@ -1064,6 +1064,7 @@ class ConstrainedGramacy(ConstrainedSyntheticTestFunction):
     _bounds = [(0.0, 1.0), (0.0, 1.0)]
     _optimizers = [(0.1954, 0.4044)]
     _optimal_value = 0.5998  # approximate from [Gramacy2016]_
+    _worst_feasible_value = 1.732051  # Computed from 100 SLSQP restarts
 
     def _evaluate_true(self, X: Tensor) -> Tensor:
         """
@@ -1122,6 +1123,10 @@ def __init__(
         self.constraint_noise_std = self._validate_constraint_noise(
             constraint_noise_std
         )
+        if dim == 3:
+            self._worst_feasible_value = -0.0002735  # Computed from 100 SLSQP restarts
+        elif dim == 6:
+            self._worst_feasible_value = -0.0001346  # Computed from 100 SLSQP restarts
 
     def _evaluate_slack_true(self, X: Tensor) -> Tensor:
         return -X.norm(dim=-1, keepdim=True) + 1
@@ -1167,6 +1172,10 @@ def __init__(
         self.constraint_noise_std = self._validate_constraint_noise(
             constraint_noise_std
         )
+        if dim == 3:
+            self._worst_feasible_value = -0.0002735  # Computed from 100 SLSQP restarts
+        elif dim == 6:
+            self._worst_feasible_value = -0.0001346  # Computed from 100 SLSQP restarts
 
     def _evaluate_slack_true(self, X: Tensor) -> Tensor:
         return -X.pow(2).sum(dim=-1, keepdim=True) + 1
@@ -1184,6 +1193,7 @@ class PressureVessel(ConstrainedSyntheticTestFunction):
     num_constraints = 4
     _bounds = [(0.0, 10.0), (0.0, 10.0), (10.0, 50.0), (150.0, 200.0)]
     _optimal_value = 6059.946341  # from [CoelloCoello2002constraint]
+    _worst_feasible_value = 240526.7248  # Computed from 100 SLSQP restarts
 
     def _evaluate_true(self, X: Tensor) -> Tensor:
         x1, x2, x3, x4 = X.unbind(-1)
@@ -1224,6 +1234,7 @@ class WeldedBeamSO(ConstrainedSyntheticTestFunction):
     num_constraints = 6
     _bounds = [(0.125, 10.0), (0.1, 10.0), (0.1, 10.0), (0.1, 10.0)]
     _optimal_value = 1.728226  # from [CoelloCoello2002constraint]
+    _worst_feasible_value = 19.01859  # Computed from 100 SLSQP restarts
 
     def _evaluate_true(self, X: Tensor) -> Tensor:
         x1, x2, x3, x4 = X.unbind(-1)
@@ -1279,6 +1290,7 @@ class TensionCompressionString(ConstrainedSyntheticTestFunction):
     num_constraints = 4
     _bounds = [(0.01, 1.0), (0.01, 1.0), (0.01, 20.0)]
     _optimal_value = 0.012681  # from [CoelloCoello2002constraint]
+    _worst_feasible_value = 0.306081  # Computed from 100 SLSQP restarts
 
     def _evaluate_true(self, X: Tensor) -> Tensor:
         x1, x2, x3 = X.unbind(-1)
@@ -1320,6 +1332,7 @@ class SpeedReducer(ConstrainedSyntheticTestFunction):
         (5.0, 5.5),
     ]
     _optimal_value = 2996.3482  # from [Lemonge2010constrained]
+    _worst_feasible_value = 6117.3420  # Computed from 100 SLSQP restarts
 
     def _evaluate_true(self, X: Tensor) -> Tensor:
         x1, x2, x3, x4, x5, x6, x7 = X.unbind(-1)
@@ -1377,6 +1390,7 @@ class KeaneBumpFunction(ConstrainedSyntheticTestFunction):
         40: -0.826624404,
         50: -0.83078783,
     }
+    _worst_feasible_value = 0.0  # Computed from 100 SLSQP restarts
 
     def __init__(
         self,

botorch/utils/testing.py

@@ -33,6 +33,7 @@
     BaseTestProblem,
     ConstrainedBaseTestProblem,
     CorruptedTestProblem,
+    MultiObjectiveTestProblem,
 )
 from botorch.test_functions.synthetic import Rosenbrock
 from botorch.utils.transforms import unnormalize
@@ -402,6 +403,25 @@ def test_evaluate_slack(self):
                 else:
                     self.assertTrue(is_equal.all().item())
 
+    def test_worst_feasible_value(self):
+        """Test that a function's worst_feasible_value is correctly computed,
+        and defined if it should be.
+        """
+        for dtype in (torch.float, torch.double):
+            for f in self.functions:
+                f.to(device=self.device, dtype=dtype)
+                if f._worst_feasible_value is None:
+                    self.assertTrue(isinstance(f, MultiObjectiveTestProblem))
+                    self.assertGreaterEqual(f.worst_feasible_value, 0.0)
+                else:
+                    worst_feas_val = f.worst_feasible_value
+                    worst_feas_val_exp = (
+                        -f._worst_feasible_value
+                        if f.negate
+                        else f._worst_feasible_value
+                    )
+                    self.assertEqual(worst_feas_val, worst_feas_val_exp)
+
     @property
     @abstractmethod
     def functions(self) -> Sequence[BaseTestProblem]:

test/test_functions/test_synthetic.py

@@ -144,6 +144,10 @@ def test_constraint_noise_length_validation(self):
         ):
             DummyConstrainedSyntheticTestFunction(constraint_noise_std=[0.1, 0.2])
 
+    def test_worst_feasible_value_not_implemented(self):
+        with self.assertRaises(NotImplementedError):
+            DummyConstrainedSyntheticTestFunction().worst_feasible_value
+
 
 class TestAckley(
     BotorchTestCase, BaseTestProblemTestCaseMixIn, SyntheticTestFunctionTestCaseMixin
@@ -387,13 +391,14 @@ class TestConstrainedHartmann(
     SyntheticTestFunctionTestCaseMixin,
     ConstrainedTestProblemTestCaseMixin,
 ):
-    functions = [
-        ConstrainedHartmann(dim=6, negate=True),
-        ConstrainedHartmann(noise_std=0.1, dim=6, negate=True),
-        ConstrainedHartmann(
-            noise_std=0.1, constraint_noise_std=0.2, dim=6, negate=True
-        ),
-    ]
+    for dim in [3, 6]:
+        functions = [
+            ConstrainedHartmann(dim=dim, negate=True),
+            ConstrainedHartmann(noise_std=0.1, dim=dim, negate=True),
+            ConstrainedHartmann(
+                noise_std=0.1, constraint_noise_std=0.2, dim=dim, negate=True
+            ),
+        ]
 
 
 class TestConstrainedHartmannSmooth(
@@ -402,12 +407,13 @@ class TestConstrainedHartmannSmooth(
     SyntheticTestFunctionTestCaseMixin,
     ConstrainedTestProblemTestCaseMixin,
 ):
-    functions = [
-        ConstrainedHartmannSmooth(dim=6, negate=True),
-        ConstrainedHartmannSmooth(
-            dim=6, noise_std=0.1, constraint_noise_std=0.2, negate=True
-        ),
-    ]
+    for dim in [3, 6]:
+        functions = [
+            ConstrainedHartmannSmooth(dim=dim, negate=True),
+            ConstrainedHartmannSmooth(
+                dim=dim, noise_std=0.1, constraint_noise_std=0.2, negate=True
+            ),
+        ]
 
 
 class TestPressureVessel(