Merge pull request #160 from automl/fix/remove-objective-merging-from-plugins

Fix/remove objective merging from plugins

Author: sarah-segel

CHANGELOG.md

@@ -4,8 +4,6 @@
 - Runs now get displayed with their parent directory for better distinguishability.
 - Increase plot font sizes.
 - Add a simple loading bar functionality for longer runs.
-- Show a run's hoover-text for the actual budget of a trial in Cost over Time with Combined budget (#154).
-- Use highest budget as default budget for Cost over Time instead of Combined.
 
 ## General
 - Seed is now required in the Recorder.
@@ -14,6 +12,13 @@
 - Use normalized LPI importance via variance instead of importance over mean (#152)
 - Return nan as importance values if variance is 0. for a hyperparameter / budget (#152)
 
+## Plugins
+- Show a run's hoover-text for the actual budget of a trial in Cost over Time with Combined budget (#154).
+- Use highest budget as default budget for Cost over Time instead of Combined.
+- Show best value / config for each objective instead of merged objective in Overview (#159).
+- Use chosen objective instead of merged objective to get the incumbent for the calculation of LPI importance (#159).
+- Add total runtime in overview (#155).
+
 # Version 1.2
 
 ## Plugins

deepcave/evaluators/lpi.py

@@ -95,7 +95,7 @@ def calculate(
 
         # Set variables
         self.continous_neighbors = continous_neighbors
-        self.incumbent, _ = self.run.get_incumbent(budget=budget)
+        self.incumbent, _ = self.run.get_incumbent(budget=budget, objectives=objectives)
         self.default = self.cs.get_default_configuration()
         self.incumbent_array = self.incumbent.get_array()
 

deepcave/plugins/summary/overview.py

@@ -37,7 +37,7 @@
 from deepcave.runs.status import Status
 from deepcave.utils.layout import create_table, help_button
 from deepcave.utils.styled_plotty import get_discrete_heatmap, save_image
-from deepcave.utils.util import get_latest_change
+from deepcave.utils.util import custom_round, get_latest_change
 
 
 class Overview(DynamicPlugin):
@@ -131,23 +131,38 @@ def load_outputs(run, *_: Any) -> List[Any]:  # type: ignore
         List[Any]
             A list of the created tables of the overview.
         """
-        # Get best cost across all objectives, highest budget
-        incumbent, _ = run.get_incumbent(statuses=[Status.SUCCESS])
-        config_id = run.get_config_id(incumbent)
-        objective_names = run.get_objective_names()
-
-        avg_costs, std_costs = run.get_avg_costs(config_id)
-
-        best_performances = []
-        for idx in range(len(objective_names)):
-            best_performances += [
-                f"{round(avg_costs[idx], 2)} ± {round(std_costs[idx], 2)} ({objective_names[idx]})"
-            ]
-
         optimizer = run.prefix
         if isinstance(run, Group):
             optimizer = run.get_runs()[0].prefix
 
+        performance_outputs = []
+        for idx, obj in enumerate(run.get_objectives()):
+            # Get best cost for the objective, highest budget
+            incumbent, _ = run.get_incumbent(objectives=obj, statuses=[Status.SUCCESS])
+            config_id = run.get_config_id(incumbent)
+            avg_costs, std_costs = run.get_avg_costs(config_id)
+
+            if len(run.get_seeds(include_combined=False)) > 1:
+                best_performance = (
+                    f"{custom_round(avg_costs[idx])} " f"± {custom_round(std_costs[idx])}"
+                )
+            else:
+                best_performance = f"{custom_round(avg_costs[idx])}"
+
+            performance_outputs.append(
+                html.Div(
+                    [
+                        html.Span(f"Best {obj.name}: {best_performance} "),
+                        html.A(
+                            "(See Configuration)",
+                            href=Configurations.get_link(run, config_id),
+                            style={"color": "white"},
+                        ),
+                    ],
+                    className="card-text",
+                ),
+            )
+
         # Design card for quick information here
         card = dbc.Card(
             [
@@ -162,15 +177,12 @@ def load_outputs(run, *_: Any) -> List[Any]:  # type: ignore
                             f"Latest change: {get_latest_change(run.latest_change)}",
                             className="card-text",
                         ),
+                        *performance_outputs,
                         html.Div(
                             [
                                 html.Span(
-                                    f"Best average performance: {', '.join(best_performances)} "
-                                ),
-                                html.A(
-                                    "(See Configuration)",
-                                    href=Configurations.get_link(run, config_id),
-                                    style={"color": "white"},
+                                    f"Total runtime [s]: "
+                                    f"{max(trial.end_time for trial in run.history)}"
                                 ),
                             ],
                             className="card-text",

deepcave/utils/util.py

@@ -145,3 +145,25 @@ def print_progress_bar(
 
     if iteration == total:
         print()
+
+
+def custom_round(number: float, min_decimals: int = 3, max_decimals: int = 10) -> float:
+    """
+    Round a number to the nearest decimal.
+
+    Parameters
+    ----------
+    number : float
+        The number to round.
+    min_decimals : int
+        The minimum number of decimals.
+        Default is 2.
+    max_decimals : int
+        The maximum number of decimals.
+        Default is 10.
+    """
+    for i in range(min_decimals, max_decimals + 1):
+        rounded = round(number, i)
+        if rounded != round(number, i - 1):
+            return rounded
+    return round(number, max_decimals)