Get hovertext per budget

Author: Sarah Krebs

CHANGELOG.md

@@ -17,6 +17,7 @@
 - Add exit button which first deletes running jobs and then terminates DeepCave.
 - Nicer handling of Keyboard Interrupt.
 - Disable debug mode.
+- Get hovertext per budget in Footprint, Config Cube, Cost over Time, and Pareto Front.
 
 ## Bug-Fixes
 - Fix missing objective specification in LPI evaluator (#71).

deepcave/plugins/objective/configuration_cube.py

@@ -326,6 +326,7 @@ def load_outputs(run, inputs, outputs) -> go.Figure:  # type: ignore
         n_configs = inputs["n_configs"]
         objective_id = inputs["objective_id"]
         objective = run.get_objective(objective_id)
+        budget = run.get_budget(inputs["budget_id"])
         df = df.groupby(df.columns.drop(objective.name).to_list(), as_index=False).mean()
         df.index = df.index.astype("str")
 
@@ -393,7 +394,9 @@ def load_outputs(run, inputs, outputs) -> go.Figure:  # type: ignore
                 "color": costs,
                 "colorbar": {"thickness": 30, "title": objective.name},
             },
-            "hovertext": [get_hovertext_from_config(run, config_id) for config_id in config_ids],
+            "hovertext": [
+                get_hovertext_from_config(run, config_id, budget) for config_id in config_ids
+            ],
             "meta": {"colorbar": costs},
             "hoverinfo": "text",
         }

deepcave/plugins/objective/cost_over_time.py

@@ -341,6 +341,7 @@ def load_outputs(runs, inputs, outputs) -> go.Figure:  # type: ignore
                 continue
 
             objective = run.get_objective(inputs["objective_id"])
+            budget = run.get_budget(inputs["budget_id"])
             config_ids = outputs[run.id]["config_ids"]
             x = outputs[run.id]["times"]
             if inputs["xaxis"] == "trials":
@@ -357,7 +358,9 @@ def load_outputs(runs, inputs, outputs) -> go.Figure:  # type: ignore
             symbol = None
             mode = "lines"
             if len(config_ids) > 0:
-                hovertext = [get_hovertext_from_config(run, config_id) for config_id in config_ids]
+                hovertext = [
+                    get_hovertext_from_config(run, config_id, budget) for config_id in config_ids
+                ]
                 hoverinfo = "text"
                 symbol = "circle"
                 mode = "lines+markers"

deepcave/plugins/objective/pareto_front.py

@@ -423,6 +423,7 @@ def load_outputs(runs, inputs, outputs) -> go.Figure:  # type: ignore
             points_avg = np.array(outputs[run.id]["points_avg"])
             points_std = np.array(outputs[run.id]["points_std"])
             config_ids = outputs[run.id]["config_ids"]
+            budget = run.get_budget(inputs["budget_id"])
             pareto_config_ids = []
 
             x, y, x_std, y_std = [], [], [], []
@@ -486,7 +487,7 @@ def load_outputs(runs, inputs, outputs) -> go.Figure:  # type: ignore
                 line_shape = "hv"
 
             hovertext = [
-                get_hovertext_from_config(run, config_id) for config_id in pareto_config_ids
+                get_hovertext_from_config(run, config_id, budget) for config_id in pareto_config_ids
             ]
 
             error_pareto_x = (

deepcave/plugins/summary/footprint.py

@@ -335,6 +335,7 @@ def load_outputs(run, inputs, outputs) -> List[Any]:  # type: ignore
             The plotly figure of the footprint performance and area.
         """
         objective = run.get_objective(inputs["objective_id"])
+        budget = run.get_budget(inputs["budget_id"])
         show_borders = inputs["show_borders"]
         show_supports = inputs["show_supports"]
 
@@ -401,7 +402,8 @@ def load_outputs(run, inputs, outputs) -> List[Any]:  # type: ignore
                     marker_symbol=marker_symbol,
                     marker={"size": size, "color": get_color(color_id)},
                     hovertext=[
-                        get_hovertext_from_config(run, config_id) for config_id in config_ids
+                        get_hovertext_from_config(run, config_id, budget)
+                        for config_id in config_ids
                     ],
                     hoverinfo="text",
                 )

deepcave/runs/__init__.py

@@ -722,7 +722,11 @@ def get_avg_costs(
         objectives = self.get_objectives()
 
         # Budget might not be evaluated
-        config_costs = self.get_all_costs(budget=budget, statuses=statuses)[config_id]
+        all_costs = self.get_all_costs(budget=budget, statuses=statuses)
+        if config_id in all_costs:
+            config_costs = all_costs[config_id]
+        else:
+            raise ValueError(f"No costs available for config_id {config_id}.")
 
         avg_costs, std_costs = [], []
         for idx in range(len(objectives)):
@@ -912,6 +916,9 @@ def get_incumbent(
             budget=budget, statuses=statuses, seed=seed, selected_ids=selected_ids
         )
 
+        if len(results) == 0:
+            raise RuntimeError("No data available, thus no incumbent found.")
+
         seed_count = {}
         for config_id, seed_costs_dict in results.items():
             seed_count[config_id] = len(seed_costs_dict)
@@ -1128,13 +1135,14 @@ def get_trajectory(
             trial = self.history[id]
 
             # Get the incumbent over all trials up to this point
-            _, cost = self.get_incumbent(
-                objectives=objective,
-                budget=budget,
-                seed=seed,
-                selected_ids=[selected_id for selected_id, _ in order[: i + 1]],
-            )
-            if cost is None:
+            try:
+                _, cost = self.get_incumbent(
+                    objectives=objective,
+                    budget=budget,
+                    seed=seed,
+                    selected_ids=[selected_id for selected_id, _ in order[: i + 1]],
+                )
+            except RuntimeError:
                 continue
 
             # Now it's important to check whether the cost was minimized or maximized

deepcave/utils/styled_plotty.py

@@ -440,7 +440,9 @@ def get_hyperparameter_ticks_from_values(
     return tickvals, ticktext
 
 
-def get_hovertext_from_config(run: AbstractRun, config_id: int) -> str:
+def get_hovertext_from_config(
+    run: AbstractRun, config_id: int, budget: Optional[Union[int, float]] = None
+) -> str:
     """
     Generate hover text with metrics for a configuration.
 
@@ -453,6 +455,9 @@ def get_hovertext_from_config(run: AbstractRun, config_id: int) -> str:
         The run instance
     config_id : int
         The id of the configuration
+    budget : Optional[Union[int, float]]
+            Budget to get the hovertext for. If no budget is given, the highest budget is chosen.
+            By default None.
 
     Returns
     -------
@@ -472,12 +477,21 @@ def get_hovertext_from_config(run: AbstractRun, config_id: int) -> str:
 
     # It's also nice to see the metrics
     objectives = run.get_objectives()
-    budget = run.get_highest_budget(config_id)
+    if budget is None or budget == -1:
+        highest_budget = run.get_highest_budget(config_id)
+        assert highest_budget is not None
+        string += f"<b>Objectives</b> (on highest found budget {round(highest_budget, 2)})<br>"
+    else:
+        string += f"<b>Objectives</b> (on budget {round(budget, 2)})<br>"
 
-    avg_costs, std_costs = run.get_avg_costs(config_id, budget=budget)
+    try:
+        avg_c, std_c = run.get_avg_costs(config_id, budget=budget)
+        avg_costs: List[Optional[float]] = list(avg_c)
+        std_costs: List[Optional[float]] = list(std_c)
+    except ValueError:
+        avg_costs = [None for _ in range(len(objectives))]
+        std_costs = [None for _ in range(len(objectives))]
 
-    assert budget is not None
-    string += f"<b>Objectives</b> (on highest found budget {round(budget, 2)})<br>"
     for objective, cost, std_cost in zip(objectives, avg_costs, std_costs):
         if std_cost == 0.0:
             string += f"{objective.name}: {cost}<br>"