Merge pull request #223 from automl/222-compatibility-with-current-smac-version

222 compatibility with current smac version

Author: mwever

CHANGELOG.md

@@ -1,5 +1,10 @@
 # Version 1.3.6
 
+## New SMAC version compatibility
+- The smac3v2 converter can now also handle the output of the new smac version
+
+# Version 1.3.5
+
 ## Converter
 - Added a new converter to handle RayTune runs
 - Added example RayTune runs to logs

deepcave/custom_queue.py

@@ -103,7 +103,7 @@ def get_workers(self) -> List[Worker]:
         List[Worker]
             A list of the queued workers.
         """
-        return Worker.all(queue=self._queue)
+        return Worker.all(queue=self._queue)  # type: ignore
 
     def is_processed(self, job_id: str) -> bool:
         """

deepcave/runs/__init__.py

@@ -505,7 +505,6 @@ def get_config_id(self, config: Union[Configuration, Dict]) -> Optional[int]:
             config = dict(config)
         # Use same rounding as ConfigSpace does
         input_config_tuple = config_to_tuple(config, ROUND_PLACES)
-
         # Check if the input configuration tuple exists in the config id mapping
         if input_config_tuple in self.config_id_mapping:
             return self.config_id_mapping[input_config_tuple]
@@ -745,6 +744,7 @@ def get_avg_costs(
 
         # Budget might not be evaluated
         all_costs = self.get_all_costs(budget=budget, statuses=statuses)
+
         if config_id in all_costs:
             config_costs = all_costs[config_id]
         else:

deepcave/runs/converters/smac3v2.py

@@ -149,21 +149,28 @@ def from_path(cls, path: Union[Path, str]) -> "SMAC3v2Run":
 
         first_starttime = None
 
-        if isinstance(data, list):
-            import warnings
+        # The newer runhistory.json format needs to be transformed
+        if isinstance(data[0], dict):
+            data = [[d[k] for k in data[0].keys()] for d in data]
+
+        # There needs to be a slot for the cpu time parameter
+        # (which is absent in the older versions)
+        right_form = []
+        for d_list in data:
+            if len(d_list) == 10:
+                d_list.insert(6, 0.0)
+            right_form.append(d_list)
+        data = right_form
 
-            warnings.warn(
-                "The runhistory.json file is in an outdated format.",
-                DeprecationWarning,
-                stacklevel=2,  # Adjusts the stack level to point to the caller.
-            )
+        if isinstance(data, list):
             for (
                 config_id,
                 instance_id,
                 seed,
                 budget,
                 cost,
                 time,
+                cpu_time,
                 status,
                 starttime,
                 endtime,
@@ -176,6 +183,7 @@ def from_path(cls, path: Union[Path, str]) -> "SMAC3v2Run":
                     budget,
                     cost,
                     time,
+                    cpu_time,
                     status,
                     starttime,
                     endtime,
@@ -188,12 +196,20 @@ def from_path(cls, path: Union[Path, str]) -> "SMAC3v2Run":
                 if run_dict is not None:
                     run.add(**run_dict)
         elif isinstance(data, dict):
+            import warnings
+
+            warnings.warn(
+                "The runhistory.json file is in an outdated format.",
+                DeprecationWarning,
+                stacklevel=2,  # Adjusts the stack level to point to the caller.
+            )
             for config_id, config_data in data.items():
                 instance_id = config_data["instance"]
                 seed = config_data["seed"]
                 budget = config_data["budget"]
                 cost = config_data["cost"]
                 time = config_data["time"]
+                cpu_time = config_data["cpu_time"]
                 status = config_data["status"]
                 starttime = config_data["starttime"]
                 endtime = config_data["endtime"]
@@ -205,6 +221,7 @@ def from_path(cls, path: Union[Path, str]) -> "SMAC3v2Run":
                     budget,
                     cost,
                     time,
+                    cpu_time,
                     status,
                     starttime,
                     endtime,
@@ -228,6 +245,7 @@ def _process_data_entry(
         budget: Optional[float],
         cost: Optional[Union[List[Union[float, None]], float]],
         time: Optional[float],
+        cpu_time: Optional[float],
         status: int,
         starttime: float,
         endtime: float,
@@ -276,6 +294,9 @@ def _process_data_entry(
         else:
             budget = 0.0
 
+        if not cpu_time:
+            cpu_time = 0.0
+
         origin = None
         if config_id in config_origins:
             origin = config_origins[config_id]
@@ -285,6 +306,7 @@ def _process_data_entry(
             "config": config,
             "budget": budget,
             "seed": seed,
+            "cpu_time": cpu_time,
             "start_time": starttime,
             "end_time": endtime,
             "status": status,

deepcave/runs/run.py

@@ -235,6 +235,7 @@ def add(
         config: Union[Dict, Configuration],
         seed: int,
         budget: float = np.inf,
+        cpu_time: float = 0.0,
         start_time: float = 0.0,
         end_time: float = 0.0,
         status: Status = Status.SUCCESS,
@@ -258,6 +259,8 @@ def add(
             Seed of the run.
         budget : float, optional
             Budget of the run. By default np.inf
+        cpu_time : float, optional
+            The cpu time of the run. By default 0.0.
         start_time : float, optional
             Start time. By default, 0.0
         end_time : float, optional
@@ -333,6 +336,7 @@ def add(
             budget=budget,
             seed=seed,
             costs=costs,
+            cpu_time=cpu_time,
             start_time=np.round(start_time, 2),
             end_time=np.round(end_time, 2),
             status=status,
@@ -475,6 +479,10 @@ def load(self, path: Optional[Union[str, Path]] = None) -> None:
         with jsonlines.open(self.history_fn) as f:
             self.history = []
             for obj in f:
+                # Check wheter cpu time is provided, if not set default
+                if len(obj) != 11:
+                    obj.insert(6, 0.0)
+
                 # Create trial object here
                 trial = Trial(*obj)
                 self.history.append(trial)

deepcave/runs/trial.py

@@ -43,10 +43,12 @@ class Trial:
         The identificator of the configuration.
     budget : Union[int, float]
         The budget for the trial.
-    seed: int
+    seed : int
         The seed for the trial.
     costs : List[float]
         A list of the costs of the trial.
+    cpu_time : float
+        The cpu time of the trial.
     start_time : float
         The start time of the trial.
     end_time : float
@@ -59,6 +61,7 @@ class Trial:
     budget: Union[int, float]
     seed: int
     costs: List[float]
+    cpu_time: float
     start_time: float
     end_time: float
     status: Status
@@ -98,6 +101,7 @@ def to_json(self) -> List[Any]:
             self.budget,
             self.seed,
             self.costs,
+            self.cpu_time,
             self.start_time,
             self.end_time,
             self.status,

logs/SMAC3v2/mlp/run_4/configspace.json

@@ -0,0 +1 @@
+{"name": null, "hyperparameters": [{"type": "categorical", "name": "activation", "choices": ["logistic", "tanh", "relu"], "weights": null, "default_value": "tanh", "meta": null}, {"type": "uniform_int", "name": "n_layer", "lower": 1, "upper": 5, "default_value": 1, "log": false, "meta": null}, {"type": "uniform_int", "name": "n_neurons", "lower": 8, "upper": 256, "default_value": 10, "log": true, "meta": null}, {"type": "categorical", "name": "solver", "choices": ["lbfgs", "sgd", "adam"], "weights": null, "default_value": "adam", "meta": null}, {"type": "uniform_int", "name": "batch_size", "lower": 30, "upper": 300, "default_value": 200, "log": false, "meta": null}, {"type": "categorical", "name": "learning_rate", "choices": ["constant", "invscaling", "adaptive"], "weights": null, "default_value": "constant", "meta": null}, {"type": "uniform_float", "name": "learning_rate_init", "lower": 0.0001, "upper": 1.0, "default_value": 0.001, "log": true, "meta": null}], "conditions": [{"type": "IN", "child": "batch_size", "parent": "solver", "values": ["sgd", "adam"]}, {"type": "EQ", "child": "learning_rate", "parent": "solver", "value": "sgd"}, {"type": "IN", "child": "learning_rate_init", "parent": "solver", "values": ["sgd", "adam"]}], "forbiddens": [], "python_module_version": "1.2.0", "format_version": 0.4}