Changes required for considering multiple budgets

Author: Sarah Krebs

deepcave/evaluators/footprint.py

@@ -241,7 +241,7 @@ def get_surface(
             Steps to create the meshgrid. By default 0.5.
         performance : bool, optional
             Whether to get the surface from the performance or the valid areas.
-            Default is set to True.
+            Default is set to True (i.e. from performance).
 
         Returns
         -------

deepcave/plugins/budget/budget_correlation.py

@@ -171,17 +171,25 @@ def process(run: AbstractRun, inputs: Dict[str, int]) -> Dict[str, Any]:
                 budget2 = run.get_budget(budget2_id)
                 budget2_readable = run.get_budget(budget2_id, human=True)
 
-                costs1 = run.get_all_costs(budget1, statuses=[Status.SUCCESS])
-                costs2 = run.get_all_costs(budget2, statuses=[Status.SUCCESS])
+                config_ids1 = run.get_configs(budget1, statuses=[Status.SUCCESS]).keys()
+                config_ids2 = run.get_configs(budget2, statuses=[Status.SUCCESS]).keys()
 
                 # Combine config ids
                 # So it is guaranteed that there is the same number of configs for each budget
-                config_ids = set(costs1.keys()) & set(costs2.keys())
+                config_ids = set(config_ids1) & set(config_ids2)
 
                 c1, c2 = [], []
                 for config_id in config_ids:
-                    c1 += [costs1[config_id][objective_id]]
-                    c2 += [costs2[config_id][objective_id]]
+                    c1 += [
+                        run.get_avg_costs(config_id, budget1, statuses=[Status.SUCCESS])[0][
+                            objective_id
+                        ]
+                    ]
+                    c2 += [
+                        run.get_avg_costs(config_id, budget2, statuses=[Status.SUCCESS])[0][
+                            objective_id
+                        ]
+                    ]
 
                 correlation = round(stats.spearmanr(c1, c2).correlation, 2)
                 correlations_symmetric["Budget"][budget2_readable] = budget2_readable  # type: ignore # noqa: E501

deepcave/plugins/summary/configurations.py

@@ -24,6 +24,7 @@
 from deepcave.constants import VALUE_RANGE
 from deepcave.plugins.dynamic import DynamicPlugin
 from deepcave.runs import AbstractRun
+from deepcave.runs.status import Status
 from deepcave.utils.compression import deserialize, serialize
 from deepcave.utils.layout import create_table, get_slider_marks
 from deepcave.utils.styled_plotty import (
@@ -214,9 +215,12 @@ def process(run, inputs) -> Dict[str, Any]:  # type: ignore
                 for seed in seeds:
                     try:
                         cost.append(
-                            run.get_costs(config_id=selected_config_id, budget=budget, seed=seed)[
-                                seed
-                            ][objective_id]
+                            run.get_costs(
+                                config_id=selected_config_id,
+                                budget=budget,
+                                seed=seed,
+                                statuses=[Status.SUCCESS],
+                            )[seed][objective_id]
                         )
                         seeds_evaluated += 1
                     except Exception:
@@ -234,7 +238,7 @@ def process(run, inputs) -> Dict[str, Any]:  # type: ignore
                     else:
                         performances_table_data[objective.name] += [cost]
                 else:
-                    performances[objective.name][budget] = None
+                    performances[objective.name][budget] = np.nan
                     if len(seeds) > 1:
                         performances_table_data[f"{objective.name}"] += ["No seed evaluated"]
                     else:
@@ -358,13 +362,18 @@ def _get_objective_figure(
         """
         objective_data = []
         for i, (metric, values) in enumerate(outputs["performances"].items()):
+            mean_objective_values, std_objective_values = [], []
+            for objective_values in values.values():
+                mean_objective_values.append(np.nanmean(objective_values))
+                std_objective_values.append(np.nanstd(objective_values))
+
             trace_kwargs = {
                 "x": list(values.keys()),
-                "y": np.mean(np.array(list(values.values())), axis=1),
+                "y": mean_objective_values,
                 "error_y": dict(
                     type="data",
                     symmetric=True,
-                    array=np.std(np.array(list(values.values())), axis=1),
+                    array=std_objective_values,
                 ),
                 "name": metric,
                 "fill": "tozeroy",

deepcave/plugins/summary/overview.py

@@ -126,20 +126,17 @@ def load_outputs(run, *_: Any) -> List[Any]:  # type: ignore
             A list of the created tables of the overview.
         """
         # Get best cost across all objectives, highest budget
-        incumbent, _ = run.get_incumbent()
+        incumbent, _ = run.get_incumbent(statuses=[Status.SUCCESS])
         config_id = run.get_config_id(incumbent)
         objective_names = run.get_objective_names()
 
-        best_performance = {}
-
-        avg_costs = run.get_avg_costs(config_id)
-
-        for idx in range(len(objective_names)):
-            best_performance[objective_names[idx]] = avg_costs[idx]
+        avg_costs, std_costs = run.get_avg_costs(config_id)
 
         best_performances = []
-        for name, value in best_performance.items():
-            best_performances += [f"{round(value, 2)} ({name})"]
+        for idx in range(len(objective_names)):
+            best_performances += [
+                f"{round(avg_costs[idx], 2)} ± {round(std_costs[idx], 2)} ({objective_names[idx]})"
+            ]
 
         optimizer = run.prefix
         if isinstance(run, Group):

deepcave/runs/__init__.py

@@ -823,9 +823,6 @@ def get_all_costs(
         else:
             history = self.history
         for trial in history:
-            if trial.config_id not in results:
-                results[trial.config_id] = {}
-
             if statuses is not None:
                 if isinstance(statuses, Status):
                     statuses = [statuses]
@@ -844,12 +841,15 @@ def get_all_costs(
                 latest_budget = highest_evaluated_budget[trial.config_id]
                 # Only the highest budget is kept
                 if trial.budget >= latest_budget:
+                    if trial.config_id not in results:
+                        results[trial.config_id] = {}
                     results[trial.config_id][trial.seed] = trial.costs
             else:
                 if trial.budget is not None:
                     if trial.budget != budget:
                         continue
-
+                if trial.config_id not in results:
+                    results[trial.config_id] = {}
                 results[trial.config_id][trial.seed] = trial.costs
         return results
 
@@ -939,19 +939,21 @@ def get_incumbent(
         min_cost = np.inf
         best_config_id = None
 
-        results = self.get_all_costs(budget, statuses, seed, selected_ids)
+        results = self.get_all_costs(
+            budget=budget, statuses=statuses, seed=seed, selected_ids=selected_ids
+        )
 
         seed_count = {}
         for config_id, costs in results.items():
-            seed_count[config_id] = len(costs.values())
+            seed_count[config_id] = len(costs)
         max_seed_count = max(seed_count.values())
 
         for config_id, costs in results.items():
             # If there are multiple seeds, only configurations evaluated on all seeds are
             # considered. From these configurations, the one with the highest average cost
             # over the seeds is considered as the incumbent.
             if max_seed_count > 1:
-                if len(costs.values()) < max_seed_count:
+                if len(costs) < max_seed_count:
                     continue
 
                 # Get average over all seeds
@@ -1396,7 +1398,7 @@ def check_equality(
     configspace: bool = True,
     objectives: bool = True,
     budgets: bool = True,
-    seeds: bool = True,
+    seeds: bool = False,
 ) -> Dict[str, Any]:
     """
     Check the passed runs on equality based on the selected runs.
@@ -1408,15 +1410,15 @@ def check_equality(
     runs : list[AbstractRun]
         Runs to check for equality.
     meta : bool, optional
-        Meta-Data excluding objectives and budgets, by default True.
+        Meta-Data excluding objectives and budgets, by default False.
     configspace : bool, optional
         Wheter to include the configuration space, by default True.
     objectives : bool, optional
         Wheter to include the objectives, by default True.
     budgets : bool, optional
         Whether to include the budgets, by default True.
     seeds : bool, optional
-        Whether to include the seeds, by default True.
+        Whether to include the seeds, by default False.
 
     Returns
     -------
@@ -1517,7 +1519,7 @@ def check_equality(
         for run in runs:
             s2 = run.get_seeds(include_combined=False)
             if s1 != s2:
-                raise NotMergeableError("Seeds of runs are not equal.")
+                raise NotMergeableError("Seeds of runs are not equal.", RunInequality.INEQ_SEED)
 
         result["seeds"] = s1
         if meta:

deepcave/runs/exceptions.py

@@ -34,3 +34,4 @@ class RunInequality(Enum):
     INEQ_OBJECTIVE = 2
     INEQ_BUDGET = 3
     INEQ_CONFIGSPACE = 4
+    INEQ_SEED = 5

deepcave/runs/group.py

@@ -103,7 +103,7 @@ def __init__(self, name: str, runs: List[AbstractRun]):
                     # Deep copy trial
                     trial = deepcopy(trial)
 
-                    (config_id, budget) = trial.get_key()
+                    (config_id, budget, seed) = trial.get_key()
 
                     # Config id might have changed
                     new_config_id = config_mapping[config_id]

deepcave/utils/styled_plotty.py

@@ -474,12 +474,15 @@ def get_hovertext_from_config(run: AbstractRun, config_id: int) -> str:
     objectives = run.get_objectives()
     budget = run.get_highest_budget(config_id)
 
-    avg_costs = run.get_avg_costs(config_id)
+    avg_costs, std_costs = run.get_avg_costs(config_id, budget=budget)
 
     assert budget is not None
     string += f"<b>Objectives</b> (on highest found budget {round(budget, 2)})<br>"
-    for objective, cost in zip(objectives, avg_costs):
-        string += f"{objective.name}: {cost}<br>"
+    for objective, cost, std_cost in zip(objectives, avg_costs, std_costs):
+        if std_cost == 0.0:
+            string += f"{objective.name}: {cost}<br>"
+        else:
+            string += f"{objective.name}: {cost} ± {std_cost}<br>"
 
     string += "<br><b>Hyperparameters</b>:<br>"