Merge pull request #226 from automl/218-ensure-complete-display-with-high-dimensional-hyperparameter-configuration-spaces

218 ensure complete display with high dimensional hyperparameter configuration spaces

Author: mwever

CHANGELOG.md

@@ -1,3 +1,28 @@
+# Version 1.3.9
+
+## Parallel Coordinates
+- Set the margin of the graph so it gets dynamically computed depending on the length of the labels.
+
+# Version 1.3.6
+
+## New SMAC version compatibility
+- The smac3v2 converter can now also handle the output of the new smac version
+
+# Version 1.3.5
+
+## Converter
+- Added a new converter to handle RayTune runs
+- Added example RayTune runs to logs
+
+# Version 1.3.5
+
+## Access Specifier
+- Changed access specifier from '_fit' to 'fit' in the RF surrogate
+
+## Ablation tests
+- Added Polynomial "model" class for testing ablation path behavior
+- Added extra test case using the polynomial to validate ablation output correctness
+
 # Version 1.3.4
 
 ## Bug-Fixes

Makefile

@@ -61,6 +61,9 @@ install-optuna:
 install-bohb:
 	$(PIP) install -e ".[bohb]"
 
+install-raytune:
+	$(PIP) install -e ".[raytune]"
+
 check-black:
 	$(BLACK) ${SOURCE_DIR} --check || :
 	$(BLACK) ${EXAMPLES_DIR} --check || :

README.md

@@ -25,11 +25,12 @@ conda install -c anaconda swig
 pip install DeepCAVE
 ```
 
-To load runs created with Optuna or the BOHB optimizer, you need to install the
+To load runs created with Optuna, the BOHB optimizer or RayTune, you need to install the
 respective packages by running:
 ```bash
 pip install deepcave[optuna]
 pip install deepcave[bohb]
+pip install deepcave[raytune]
 ```
 
 To try the examples for recording your results in DeepCAVE format, run this after installing:

deepcave/config.py

@@ -156,7 +156,17 @@ def CONVERTERS(self) -> List[Type["Run"]]:
         from deepcave.runs.converters.dataframe import DataFrameRun
         from deepcave.runs.converters.deepcave import DeepCAVERun
         from deepcave.runs.converters.optuna import OptunaRun
+        from deepcave.runs.converters.raytune import RayTuneRun
         from deepcave.runs.converters.smac3v1 import SMAC3v1Run
         from deepcave.runs.converters.smac3v2 import SMAC3v2Run
 
-        return [AMLTKRun, BOHBRun, DeepCAVERun, OptunaRun, SMAC3v1Run, SMAC3v2Run, DataFrameRun]
+        return [
+            AMLTKRun,
+            BOHBRun,
+            DeepCAVERun,
+            OptunaRun,
+            RayTuneRun,
+            SMAC3v1Run,
+            SMAC3v2Run,
+            DataFrameRun,
+        ]

deepcave/custom_queue.py

@@ -103,7 +103,7 @@ def get_workers(self) -> List[Worker]:
         List[Worker]
             A list of the queued workers.
         """
-        return Worker.all(queue=self._queue)
+        return Worker.all(queue=self._queue)  # type: ignore
 
     def is_processed(self, job_id: str) -> bool:
         """

deepcave/evaluators/ablation.py

@@ -75,25 +75,24 @@ def calculate(
         self,
         objectives: Optional[Union[Objective, List[Objective]]],  # noqa
         budget: Optional[Union[int, float]] = None,  # noqa
-        n_trees: int = 50,  # noqa
-        seed: int = 0,  # noqa
+        model: Any = None,
     ) -> None:
         """
         Calculate the ablation path performances and improvements.
 
+        To use standard Random Forest surrogate do not pass a model.
+        The option to pass another model is just for testing purposes.
+
         Parameters
         ----------
         objectives : Optional[Union[Objective, List[Objective]]]
             The objective(s) to be considered.
         budget : Optional[Union[int, float]]
             The budget to be considered. If None, all budgets of the run are considered.
             Default is None.
-        n_trees : int
-            The number of trees for the surrogate model.
-            Default is 50.
-        seed : int
-            The seed for the surrogate model.
-            Default is 0.
+        model :
+            The surrogate model to use for the prediction of the perfromances.
+            By default None.
         """
         if isinstance(objectives, list) and len(objectives) > 1:
             raise ValueError("Only one objective is supported for ablation paths.")
@@ -103,25 +102,30 @@ def calculate(
         performances: OrderedDict = OrderedDict()
         improvements: OrderedDict = OrderedDict()
 
+        self._model = model
+
         df = self.run.get_encoded_data(objective, budget, specific=True)
 
         # Obtain all configurations with theirs costs
         df = df.dropna(subset=[objective.name])
         X = df[list(self.run.configspace.keys())].to_numpy()
         Y = df[objective.name].to_numpy()
 
-        # A Random Forest Regressor is used as surrogate model
-        self._model = RandomForestSurrogate(self.cs, seed=seed, n_trees=n_trees)
-        self._model._fit(X, Y)
-
         # Get the incumbent configuration
         incumbent_config, _ = self.run.get_incumbent(budget=budget, objectives=objective)
         incumbent_encode = self.run.encode_config(incumbent_config)
 
         # Get the default configuration
+
         self.default_config = self.cs.get_default_configuration()
         default_encode = self.run.encode_config(self.default_config)
 
+        # The default model is a RF Surrogate, but it cant be passed as parameter directly
+        # because it needs access to its config space
+        if self._model is None:
+            self._model = RandomForestSurrogate(self.cs, seed=0, n_trees=50)
+
+        self._model.fit(X, Y)
         # Obtain the predicted cost of the default and incumbent configuration
         def_cost, def_std = self._model.predict(np.array([default_encode]))
         def_cost, def_std = def_cost[0], def_std[0]
@@ -166,6 +170,7 @@ def calculate(
                     performances[max_hp] = (max_hp_cost, max_hp_std)
                 impr_std = np.sqrt(def_std**2 + max_hp_std**2)
                 improvements[max_hp] = ((def_cost - max_hp_cost), impr_std)
+
                 # New 'default' cost and std
                 def_cost = max_hp_cost
                 def_std = max_hp_std
@@ -248,7 +253,6 @@ def _ablation(
             if hp in incumbent_config.keys() and hp in self.default_config.keys():
                 config_copy = copy.copy(self.default_config)
                 config_copy[hp] = incumbent_config[hp]
-
                 new_cost, _ = self._model.predict(np.array([self.run.encode_config(config_copy)]))
                 if objective.optimize == "upper":
                     new_cost = -new_cost
@@ -262,6 +266,7 @@ def _ablation(
             else:
                 continue
         hp_count = len(list(self.cs.keys()))
+
         if max_hp != "":
             # For the maximum impact hyperparameter, switch the default with the incumbent value
             self.default_config[max_hp] = incumbent_config[max_hp]
@@ -270,6 +275,7 @@ def _ablation(
             )
             if objective.optimize == "upper":
                 max_hp_cost = -max_hp_cost
+
             return True, max_hp, max_hp_cost[0], max_hp_std[0]
         else:
             self.logger.info(

deepcave/evaluators/epm/random_forest_surrogate.py

@@ -78,3 +78,16 @@ def _fit(self, X: np.ndarray, y: np.ndarray) -> None:
             Corresponding target values.
         """
         self._model.train(X, y)
+
+    def fit(self, X: np.ndarray, y: np.ndarray) -> None:
+        """
+        Train the surrogate model.
+
+        Parameters
+        ----------
+        X : np.ndarray
+            Input data points.
+        y : np.ndarray
+            Corresponding target values.
+        """
+        self._fit(X, y)

deepcave/evaluators/mo_ablation.py

@@ -122,8 +122,7 @@ def calculate(
         self,
         objectives: Optional[Union[Objective, List[Objective]]],  # noqa
         budget: Optional[Union[int, float]] = None,  # noqa
-        n_trees: int = 50,  # noqa
-        seed: int = 0,  # noqa
+        model: Any = None,
     ) -> None:
         """
         Calculate the MO ablation path performances and improvements.
@@ -135,12 +134,9 @@ def calculate(
         budget : Optional[Union[int, float]]
             The budget to be considered. If None, all budgets of the run are considered.
             Default is None.
-        n_trees : int
-            The number of trees for the surrogate model.
-            Default is 50.
-        seed : int
-            The seed for the surrogate model.
-            Default is 0.
+        model : Any
+            For mo ablation this parameter does not do anything, except fit the head.
+            By default None.
         """
         assert isinstance(objectives, list)
         for objective in objectives:
@@ -166,8 +162,9 @@ def calculate(
 
             # train one model per objective
             Y = df[normed].to_numpy()
-            model = RandomForestSurrogate(self.cs, seed=seed, n_trees=n_trees)
-            model._fit(X, Y)
+            if model is None:
+                model = RandomForestSurrogate(self.cs, seed=0, n_trees=50)
+            model.fit(X, Y)
             self.models.append(model)
 
         weightings = get_weightings(objectives_normed, df)

deepcave/plugins/__init__.py

@@ -306,7 +306,6 @@ def plugin_input_update(pathname: str, *inputs_list: str) -> List[Optional[str]]
                     if passed_inputs is not None:
                         # First get normal inputs
                         inputs = self.load_inputs()
-
                         # Overwrite/set the passed inputs
                         update_dict(inputs, passed_inputs)
 
@@ -1096,7 +1095,6 @@ def load_run_inputs(
                     run_path = run.path
                     if run_path is not None:
                         run_name = run_path.parent.name + "/" + run.name
-
                     values.append(run.id)
                     labels.append(run_name)
                     disabled.append(False)

deepcave/plugins/budget/budget_correlation.py

@@ -198,6 +198,7 @@ def process(run: AbstractRun, inputs: Dict[str, int]) -> Dict[str, Any]:
                             objective_id
                         ]
                     ]
+
                     c2 += [
                         run.get_avg_costs(config_id, budget2, statuses=[Status.SUCCESS])[0][
                             objective_id