Merge pull request #36 from automl/feature/add-converter-for-smac-2.0

Feature/add converter for smac 2.0

Author: sarah-segel

.pre-commit-config.yaml

@@ -3,7 +3,7 @@
 # If things break on update, raise an issue
 repos:
   - repo: https://github.com/pycqa/isort
-    rev: 5.10.1
+    rev: 5.12.0
     hooks:
       - id: isort
         name: isort imports deepcave
@@ -14,7 +14,7 @@ repos:
         files: tests
 
   - repo: https://github.com/ambv/black
-    rev: 22.1.0
+    rev: 23.1.0
     hooks:
       - id: black
         name: black formatter deepcave
@@ -29,7 +29,7 @@ repos:
         files: examples
 
   - repo: https://github.com/pycqa/pydocstyle
-    rev: 6.1.1
+    rev: 6.3.0
     hooks:
       - id: pydocstyle
         files: deepcave
@@ -42,8 +42,8 @@ repos:
         name: mypy deepcave
         files: deepcave
 
-  - repo: https://gitlab.com/pycqa/flake8
-    rev: 4.0.1
+  - repo: https://github.com/PyCQA/flake8
+    rev: 6.0.0
     hooks:
       - id: flake8
         name: flake8 deepcave

CHANGELOG.md

@@ -1,3 +1,11 @@
+# Version 1.1
+
+## Converters
+- SMAC 2.0
+
+## Dependencies
+- Remove SMAC dependency by adding required function directly
+
 # Version 1.0.1
 
 ## Bug-Fixes

deepcave/__init__.py

@@ -17,7 +17,7 @@
     "Source Code": "https://github.com/automl/deepcave",
 }
 copyright = f"Copyright {datetime.date.today().strftime('%Y')}, {author}"
-version = "1.0.1"
+version = "1.1"
 
 _exec_file = sys.argv[0]
 _exec_files = ["server.py", "worker.py", "sphinx-build"]

deepcave/config.py

@@ -83,6 +83,7 @@ def PLUGINS(self) -> Dict[str, List["Plugin"]]:
     def CONVERTERS(self) -> List[Type["Run"]]:
         from deepcave.runs.converters.bohb import BOHBRun
         from deepcave.runs.converters.deepcave import DeepCAVERun
-        from deepcave.runs.converters.smac import SMACRun
+        from deepcave.runs.converters.smac3v1 import SMAC3v1Run
+        from deepcave.runs.converters.smac3v2 import SMAC3v2Run
 
-        return [DeepCAVERun, BOHBRun, SMACRun]
+        return [DeepCAVERun, BOHBRun, SMAC3v1Run, SMAC3v2Run]

deepcave/evaluators/epm/random_forest.py

@@ -16,7 +16,8 @@
 from sklearn.decomposition import PCA
 from sklearn.exceptions import NotFittedError
 from sklearn.preprocessing import MinMaxScaler
-from smac.epm.util_funcs import get_types
+
+from deepcave.evaluators.epm.utils import get_types
 
 VERY_SMALL_NUMBER = 1e-10
 PYRFR_MAPPING = {

deepcave/evaluators/epm/utils.py

@@ -0,0 +1,117 @@
+import typing
+
+import numpy as np
+from ConfigSpace import ConfigurationSpace
+from ConfigSpace.hyperparameters import (
+    BetaFloatHyperparameter,
+    BetaIntegerHyperparameter,
+    CategoricalHyperparameter,
+    Constant,
+    NormalFloatHyperparameter,
+    NormalIntegerHyperparameter,
+    OrdinalHyperparameter,
+    UniformFloatHyperparameter,
+    UniformIntegerHyperparameter,
+)
+
+
+def get_types(
+    config_space: ConfigurationSpace,
+    instance_features: typing.Optional[np.ndarray] = None,
+) -> typing.Tuple[typing.List[int], typing.List[typing.Tuple[float, float]]]:
+    """Return the types of the hyperparameters and the bounds of the
+    hyperparameters and instance features.
+    """
+    # Extract types vector for rf from config space and the bounds
+    types = [0] * len(config_space.get_hyperparameters())
+    bounds = [(np.nan, np.nan)] * len(types)
+
+    for i, param in enumerate(config_space.get_hyperparameters()):
+        parents = config_space.get_parents_of(param.name)
+        if len(parents) == 0:
+            can_be_inactive = False
+        else:
+            can_be_inactive = True
+
+        if isinstance(param, (CategoricalHyperparameter)):
+            n_cats = len(param.choices)
+            if can_be_inactive:
+                n_cats = len(param.choices) + 1
+            types[i] = n_cats
+            bounds[i] = (int(n_cats), np.nan)
+        elif isinstance(param, (OrdinalHyperparameter)):
+            n_cats = len(param.sequence)
+            types[i] = 0
+            if can_be_inactive:
+                bounds[i] = (0, int(n_cats))
+            else:
+                bounds[i] = (0, int(n_cats) - 1)
+        elif isinstance(param, Constant):
+            # for constants we simply set types to 0 which makes it a numerical
+            # parameter
+            if can_be_inactive:
+                bounds[i] = (2, np.nan)
+                types[i] = 2
+            else:
+                bounds[i] = (0, np.nan)
+                types[i] = 0
+            # and we leave the bounds to be 0 for now
+        elif isinstance(param, UniformFloatHyperparameter):
+            # Are sampled on the unit hypercube thus the bounds
+            # are always 0.0, 1.0
+            if can_be_inactive:
+                bounds[i] = (-1.0, 1.0)
+            else:
+                bounds[i] = (0, 1.0)
+        elif isinstance(param, UniformIntegerHyperparameter):
+            if can_be_inactive:
+                bounds[i] = (-1.0, 1.0)
+            else:
+                bounds[i] = (0, 1.0)
+        elif isinstance(param, NormalFloatHyperparameter):
+            if can_be_inactive:
+                raise ValueError(
+                    "Inactive parameters not supported for Beta and Normal Hyperparameters"
+                )
+
+            bounds[i] = (param._lower, param._upper)
+        elif isinstance(param, NormalIntegerHyperparameter):
+            if can_be_inactive:
+                raise ValueError(
+                    "Inactive parameters not supported for Beta and Normal Hyperparameters"
+                )
+
+            bounds[i] = (param.nfhp._lower, param.nfhp._upper)
+        elif isinstance(param, BetaFloatHyperparameter):
+            if can_be_inactive:
+                raise ValueError(
+                    "Inactive parameters not supported for Beta and Normal Hyperparameters"
+                )
+
+            bounds[i] = (param._lower, param._upper)
+        elif isinstance(param, BetaIntegerHyperparameter):
+            if can_be_inactive:
+                raise ValueError(
+                    "Inactive parameters not supported for Beta and Normal Hyperparameters"
+                )
+
+            bounds[i] = (param.bfhp._lower, param.bfhp._upper)
+        elif not isinstance(
+            param,
+            (
+                UniformFloatHyperparameter,
+                UniformIntegerHyperparameter,
+                OrdinalHyperparameter,
+                CategoricalHyperparameter,
+                NormalFloatHyperparameter,
+                NormalIntegerHyperparameter,
+                BetaFloatHyperparameter,
+                BetaIntegerHyperparameter,
+            ),
+        ):
+            raise TypeError("Unknown hyperparameter type %s" % type(param))
+
+    if instance_features is not None:
+        types = types + [0] * instance_features.shape[1]
+
+    return types, bounds

deepcave/runs/__init__.py

@@ -482,7 +482,6 @@ def get_status(self, config_id: int, budget: Optional[Union[int, float]] = None)
         # Unfortunately, we have to iterate through the history to find the status
         # TODO: Cache the stati
         for trial in self.history:
-
             if trial_key == trial.get_key():
                 return trial.status
 
@@ -893,7 +892,7 @@ def get_encoded_data(
 
 def check_equality(
     runs: List[AbstractRun],
-    meta: bool = True,
+    meta: bool = False,
     configspace: bool = True,
     objectives: bool = True,
     budgets: bool = True,

deepcave/runs/converters/smac.py deepcave/runs/converters/smac3v1.pydeepcave/runs/converters/smac.py renamed to deepcave/runs/converters/smac3v1.py

@@ -9,8 +9,8 @@
 from deepcave.utils.hash import file_to_hash
 
 
-class SMACRun(Run):
-    prefix = "SMAC"
+class SMAC3v1Run(Run):
+    prefix = "SMAC3v1"
     _initial_order = 2
 
     @property
@@ -65,7 +65,7 @@ def from_path(cls, path):
                     meta[arg] = value
 
         # Let's create a new run object
-        run = SMACRun(
+        run = SMAC3v1Run(
             name=path.stem, configspace=configspace, objectives=[objective1, objective2], meta=meta
         )
 

deepcave/runs/converters/smac3v2.py

@@ -0,0 +1,147 @@
+import json
+from pathlib import Path
+
+import numpy as np
+
+from deepcave.runs import Status
+from deepcave.runs.objective import Objective
+from deepcave.runs.run import Run
+from deepcave.utils.hash import file_to_hash
+
+
+class SMAC3v2Run(Run):
+    prefix = "SMAC3v2"
+    _initial_order = 2
+
+    @property
+    def hash(self):
+        if self.path is None:
+            return ""
+
+        # Use hash of history.json as id
+        return file_to_hash(self.path / "runhistory.json")
+
+    @property
+    def latest_change(self):
+        if self.path is None:
+            return 0
+
+        return Path(self.path / "runhistory.json").stat().st_mtime
+
+    @classmethod
+    def from_path(cls, path):
+        """
+        Based on working_dir/run_name/*, return a new trials object.
+        """
+        path = Path(path)
+
+        # Read configspace
+        from ConfigSpace.read_and_write import json as cs_json
+
+        with (path / "configspace.json").open("r") as f:
+            configspace = cs_json.read(f.read())
+
+        # Read objectives
+        # We have to define it ourselves, because we don't know the type of the objective
+        # Only lock lower
+        objective1 = Objective("Cost", lower=0)
+        objective2 = Objective("Time", lower=0)
+
+        # Read meta
+        with (path / "scenario.json").open() as json_file:
+            meta = json.load(json_file)
+            meta["run_objectives"] = meta.pop("objectives")
+
+        # Let's create a new run object
+        run = SMAC3v2Run(
+            name=path.stem, configspace=configspace, objectives=[objective1, objective2], meta=meta
+        )
+
+        # We have to set the path manually
+        run._path = path
+
+        # Iterate over the runhistory
+        with (path / "runhistory.json").open() as json_file:
+            all_data = json.load(json_file)
+            data = all_data["data"]
+            config_origins = all_data["config_origins"]
+            configs = all_data["configs"]
+
+        instance_ids = []
+
+        first_starttime = None
+        seeds = []
+        for (
+            config_id,
+            instance_id,
+            seed,
+            budget,
+            cost,
+            time,
+            status,
+            starttime,
+            endtime,
+            additional_info,
+        ) in data:
+            if instance_id not in instance_ids:
+                instance_ids += [instance_id]
+
+            if len(instance_ids) > 1:
+                raise RuntimeError("Instances are not supported.")
+
+            config_id = str(config_id)
+            config = configs[config_id]
+
+            if seed not in seeds:
+                seeds.append(seed)
+
+            if len(seeds) > 1:
+                raise RuntimeError("Multiple seeds are not supported.")
+
+            if first_starttime is None:
+                first_starttime = starttime
+
+            starttime = starttime - first_starttime
+            endtime = endtime - first_starttime
+
+            if status == 0:
+                # still running
+                continue
+            elif status == 1:
+                status = Status.SUCCESS
+            elif status == 3:
+                status = Status.TIMEOUT
+            elif status == 4:
+                status = Status.MEMORYOUT
+            else:
+                status = Status.CRASHED
+
+            if status != Status.SUCCESS:
+                # We don't want cost included which are failed
+                cost = None
+                time = None
+            else:
+                time = endtime - starttime
+
+            # Round budget
+            if budget:
+                budget = np.round(budget, 2)
+            else:
+                budget = 0.0
+
+            origin = None
+            if config_id in config_origins:
+                origin = config_origins[config_id]
+
+            run.add(
+                costs=[cost, time],
+                config=config,
+                budget=budget,
+                start_time=starttime,
+                end_time=endtime,
+                status=status,
+                origin=origin,
+                additional=additional_info,
+            )
+
+        return run