Merge pull request #135 from automl/feature/multiple-seeds

Feature/multiple seeds

Author: sarah-segel

CHANGELOG.md

@@ -8,13 +8,17 @@
 
 ## Enhancements
 - Fix lower bounds of dependency versions.
-- Allow to load multi-objective SMAC3v2 and add example (#69)
+- Allow to load multi-objective SMAC3v2 and add example (#69).
+- Allow to load runs with multiple seeds and add examples (#70).
+- Correct incumbent calculation when single objective should be maximized.
+- Correct range of configuration cube slider for number of configs.
 - Do not disable existing loggers.
 - Update author email.
 - Add exit button which first deletes running jobs and then terminates DeepCave.
 - Nicer handling of Keyboard Interrupt.
 - Disable debug mode.
 - Save plotly plots in higher resolution upon download.
+- Get hovertext per budget in Footprint, Config Cube, Cost over Time, and Pareto Front.
 
 ## Bug-Fixes
 - Fix missing objective specification in LPI evaluator (#71).
@@ -29,6 +33,7 @@
 - For PCP, show hyperparameters with highest importance closest to the cost, i.e. right (#124).
 - Add init files to all test directories.
 - Correct LPI importance tests.
+- Free port when exiting via the exit button (#52).
 
 ## Documentation
 - Add How to Contribute section.

deepcave/constants.py

@@ -23,3 +23,4 @@
 RANDOM_CONFIG_ID = -2  # Used for random configs
 COMBINED_COST_NAME = "Combined Cost"
 COMBINED_BUDGET = -1
+COMBINED_SEED = -1

deepcave/evaluators/footprint.py

@@ -241,7 +241,7 @@ def get_surface(
             Steps to create the meshgrid. By default 0.5.
         performance : bool, optional
             Whether to get the surface from the performance or the valid areas.
-            Default is set to True.
+            Default is set to True (i.e. from performance).
 
         Returns
         -------

deepcave/layouts/header.py

@@ -17,10 +17,11 @@
 import time
 
 import dash_bootstrap_components as dbc
+import requests
 from dash import dcc, html
 from dash.dependencies import Input, Output
 
-from deepcave import app, c, queue
+from deepcave import app, c, config, queue
 from deepcave.layouts import Layout
 
 
@@ -114,6 +115,7 @@ def callback(n_clicks: Optional[int]) -> None:
             # Then we want to terminate DeepCAVE
             if n_clicks is not None:
                 time.sleep(1)
+                requests.post(f"http://localhost:{config.DASH_PORT}/shutdown")
                 os._exit(130)
 
     def __call__(self) -> html.Header:  # noqa: D102

deepcave/plugins/budget/budget_correlation.py

@@ -171,17 +171,25 @@ def process(run: AbstractRun, inputs: Dict[str, int]) -> Dict[str, Any]:
                 budget2 = run.get_budget(budget2_id)
                 budget2_readable = run.get_budget(budget2_id, human=True)
 
-                costs1 = run.get_all_costs(budget1, statuses=[Status.SUCCESS])
-                costs2 = run.get_all_costs(budget2, statuses=[Status.SUCCESS])
+                config_ids1 = run.get_configs(budget1, statuses=[Status.SUCCESS]).keys()
+                config_ids2 = run.get_configs(budget2, statuses=[Status.SUCCESS]).keys()
 
                 # Combine config ids
                 # So it is guaranteed that there is the same number of configs for each budget
-                config_ids = set(costs1.keys()) & set(costs2.keys())
+                config_ids = set(config_ids1) & set(config_ids2)
 
                 c1, c2 = [], []
                 for config_id in config_ids:
-                    c1 += [costs1[config_id][objective_id]]
-                    c2 += [costs2[config_id][objective_id]]
+                    c1 += [
+                        run.get_avg_costs(config_id, budget1, statuses=[Status.SUCCESS])[0][
+                            objective_id
+                        ]
+                    ]
+                    c2 += [
+                        run.get_avg_costs(config_id, budget2, statuses=[Status.SUCCESS])[0][
+                            objective_id
+                        ]
+                    ]
 
                 correlation = round(stats.spearmanr(c1, c2).correlation, 2)
                 correlations_symmetric["Budget"][budget2_readable] = budget2_readable  # type: ignore # noqa: E501

deepcave/plugins/objective/configuration_cube.py

@@ -211,10 +211,10 @@ def load_dependency_inputs(self, run, _, inputs) -> Dict[str, Any]:  # type: ign
         budget = run.get_budget(budget_value)
         configs = run.get_configs(budget=budget)
         if n_configs_value == 0:
-            n_configs_value = len(configs) - 1
+            n_configs_value = len(configs)
         else:
-            if n_configs_value > len(configs) - 1:
-                n_configs_value = len(configs) - 1
+            if n_configs_value > len(configs):
+                n_configs_value = len(configs)
 
         # Restrict to three hyperparameters
         selected_hps = inputs["hyperparameter_names"]["value"]
@@ -233,8 +233,8 @@ def load_dependency_inputs(self, run, _, inputs) -> Dict[str, Any]:  # type: ign
             },
             "n_configs": {
                 "min": 0,
-                "max": len(configs) - 1,
-                "marks": get_slider_marks(list(range(len(configs)))),
+                "max": len(configs),
+                "marks": get_slider_marks(list(range(0, len(configs) + 1))),
                 "value": n_configs_value,
             },
             "hyperparameter_names": {
@@ -332,6 +332,9 @@ def load_outputs(run, inputs, outputs) -> go.Figure:  # type: ignore
         n_configs = inputs["n_configs"]
         objective_id = inputs["objective_id"]
         objective = run.get_objective(objective_id)
+        budget = run.get_budget(inputs["budget_id"])
+        df = df.groupby(df.columns.drop(objective.name).to_list(), as_index=False).mean()
+        df.index = df.index.astype("str")
 
         # Limit to n_configs
         idx = [str(i) for i in range(n_configs, len(df))]
@@ -397,7 +400,9 @@ def load_outputs(run, inputs, outputs) -> go.Figure:  # type: ignore
                 "color": costs,
                 "colorbar": {"thickness": 30, "title": objective.name},
             },
-            "hovertext": [get_hovertext_from_config(run, config_id) for config_id in config_ids],
+            "hovertext": [
+                get_hovertext_from_config(run, config_id, budget) for config_id in config_ids
+            ],
             "meta": {"colorbar": costs},
             "hoverinfo": "text",
         }

deepcave/plugins/objective/cost_over_time.py

@@ -345,6 +345,7 @@ def load_outputs(runs, inputs, outputs) -> go.Figure:  # type: ignore
                 continue
 
             objective = run.get_objective(inputs["objective_id"])
+            budget = run.get_budget(inputs["budget_id"])
             config_ids = outputs[run.id]["config_ids"]
             x = outputs[run.id]["times"]
             if inputs["xaxis"] == "trials":
@@ -361,7 +362,9 @@ def load_outputs(runs, inputs, outputs) -> go.Figure:  # type: ignore
             symbol = None
             mode = "lines"
             if len(config_ids) > 0:
-                hovertext = [get_hovertext_from_config(run, config_id) for config_id in config_ids]
+                hovertext = [
+                    get_hovertext_from_config(run, config_id, budget) for config_id in config_ids
+                ]
                 hoverinfo = "text"
                 symbol = "circle"
                 mode = "lines+markers"

deepcave/plugins/objective/parallel_coordinates.py

@@ -290,7 +290,9 @@ def process(run, inputs) -> Dict[str, Any]:  # type: ignore
         """
         budget = run.get_budget(inputs["budget_id"])
         objective = run.get_objective(inputs["objective_id"])
-        df = serialize(run.get_encoded_data(objective, budget))
+        df = run.get_encoded_data(objective, budget)
+        df = df.groupby(df.columns.drop(objective.name).to_list(), as_index=False).mean()
+        df = serialize(df)
         result: Dict[str, Any] = {"df": df}
 
         if inputs["show_important_only"]:

deepcave/plugins/objective/pareto_front.py

@@ -183,20 +183,38 @@ def get_filter_layout(register: Callable) -> List[Any]:
             The layouts for the filter block.
         """
         return [
-            html.Div(
+            dbc.Row(
                 [
-                    dbc.Label("Show all configurations"),
-                    help_button(
-                        "Additionally to the pareto front, also the other configurations "
-                        "are displayed. This makes it easier to see the performance "
-                        "differences."
+                    dbc.Col(
+                        [
+                            dbc.Label("Show all configurations"),
+                            help_button(
+                                "Additionally to the pareto front, also the other configurations "
+                                "are displayed. This makes it easier to see the performance "
+                                "differences."
+                            ),
+                            dbc.Select(
+                                id=register("show_all", ["value", "options"]),
+                                placeholder="Select ...",
+                            ),
+                        ],
+                        md=6,
                     ),
-                    dbc.Select(
-                        id=register("show_all", ["value", "options"]),
-                        placeholder="Select ...",
+                    dbc.Col(
+                        [
+                            dbc.Label("Show error bars"),
+                            help_button(
+                                "Show error bars In the case of non-deterministic runs with "
+                                "multiple seeds evaluated per configuration."
+                            ),
+                            dbc.Select(
+                                id=register("show_error", ["value", "options"]),
+                                placeholder="Select ...",
+                            ),
+                        ],
+                        md=6,
                     ),
                 ],
-                className="mb-3",
             ),
             dbc.Row(
                 [
@@ -251,6 +269,7 @@ def load_inputs(self) -> Dict[str, Dict[str, Any]]:
                 "value": self.budget_options[-1]["value"],
             },
             "show_all": {"options": get_select_options(binary=True), "value": "false"},
+            "show_error": {"options": get_select_options(binary=True), "value": "false"},
             "show_runs": {"options": get_select_options(binary=True), "value": "true"},
             "show_groups": {"options": get_select_options(binary=True), "value": "true"},
         }
@@ -291,22 +310,29 @@ def process(run, inputs) -> Dict[str, Any]:  # type: ignore
         objective_id_2 = inputs["objective_id_2"]
         objective_2 = run.get_objective(objective_id_2)
 
-        points: Union[List, np.ndarray] = []
-        config_ids: Union[List, np.ndarray] = []
-        for config_id, costs in run.get_all_costs(budget, statuses=[Status.SUCCESS]).items():
-            points += [[costs[objective_id_1], costs[objective_id_2]]]
-            config_ids += [config_id]
+        points_avg: Union[List, np.ndarray] = []
+        points_std: Union[List, np.ndarray] = []
+        config_ids: Union[List, np.ndarray] = list(
+            run.get_configs(budget, statuses=[Status.SUCCESS]).keys()
+        )
+
+        for config_id in config_ids:
+            avg_costs, std_costs = run.get_avg_costs(config_id, budget, statuses=[Status.SUCCESS])
+            points_avg += [[avg_costs[objective_id_1], avg_costs[objective_id_2]]]
+            points_std += [[std_costs[objective_id_1], std_costs[objective_id_2]]]
 
-        points = np.array(points)
+        points_avg = np.array(points_avg)
+        points_std = np.array(points_std)
         config_ids = np.array(config_ids)
 
         # Sort the points s.t. x axis is monotonically increasing
-        sorted_idx = np.argsort(points[:, 0])
-        points = points[sorted_idx]
+        sorted_idx = np.argsort(points_avg[:, 0])
+        points_avg = points_avg[sorted_idx]
+        points_std = points_std[sorted_idx]
         config_ids = config_ids[sorted_idx]
 
-        is_front: np.ndarray = np.ones(points.shape[0], dtype=bool)
-        for point_idx, costs in enumerate(points):
+        is_front: np.ndarray = np.ones(points_avg.shape[0], dtype=bool)
+        for point_idx, costs in enumerate(points_avg):
             if is_front[point_idx]:
                 # Keep any point with a lower/upper cost
                 # This loop is a little bit complicated than
@@ -316,9 +342,9 @@ def process(run, inputs) -> Dict[str, Any]:  # type: ignore
                 select = None
                 for idx, (objective, cost) in enumerate(zip([objective_1, objective_2], costs)):
                     if objective.optimize == "upper":
-                        select2 = np.any(points[is_front][:, idx, np.newaxis] > [cost], axis=1)
+                        select2 = np.any(points_avg[is_front][:, idx, np.newaxis] > [cost], axis=1)
                     else:
-                        select2 = np.any(points[is_front][:, idx, np.newaxis] < [cost], axis=1)
+                        select2 = np.any(points_avg[is_front][:, idx, np.newaxis] < [cost], axis=1)
 
                     if select is None:
                         select = select2
@@ -331,7 +357,8 @@ def process(run, inputs) -> Dict[str, Any]:  # type: ignore
                 is_front[point_idx] = True
 
         return {
-            "points": points.tolist(),
+            "points_avg": points_avg.tolist(),
+            "points_std": points_std.tolist(),
             "pareto_points": is_front.tolist(),
             "config_ids": config_ids.tolist(),
         }
@@ -384,6 +411,7 @@ def load_outputs(runs, inputs, outputs) -> go.Figure:  # type: ignore
             The output figure.
         """
         show_all = inputs["show_all"]
+        show_error = inputs["show_error"]
 
         traces = []
         for idx, run in enumerate(runs):
@@ -396,31 +424,50 @@ def load_outputs(runs, inputs, outputs) -> go.Figure:  # type: ignore
             if run.prefix != "group" and not show_runs:
                 continue
 
-            points = np.array(outputs[run.id]["points"])
+            points_avg = np.array(outputs[run.id]["points_avg"])
+            points_std = np.array(outputs[run.id]["points_std"])
             config_ids = outputs[run.id]["config_ids"]
+            budget = run.get_budget(inputs["budget_id"])
             pareto_config_ids = []
 
-            x, y = [], []
-            x_pareto, y_pareto = [], []
+            x, y, x_std, y_std = [], [], [], []
+            x_pareto, y_pareto, x_pareto_std, y_pareto_std = [], [], [], []
 
             pareto_points = outputs[run.id]["pareto_points"]
             for point_idx, pareto in enumerate(pareto_points):
                 if pareto:
-                    x_pareto += [points[point_idx][0]]
-                    y_pareto += [points[point_idx][1]]
+                    x_pareto += [points_avg[point_idx][0]]
+                    y_pareto += [points_avg[point_idx][1]]
+                    x_pareto_std += [points_std[point_idx][0]]
+                    y_pareto_std += [points_std[point_idx][1]]
                     pareto_config_ids += [config_ids[point_idx]]
                 else:
-                    x += [points[point_idx][0]]
-                    y += [points[point_idx][1]]
+                    x += [points_avg[point_idx][0]]
+                    y += [points_avg[point_idx][1]]
+                    x_std += [points_std[point_idx][0]]
+                    y_std += [points_std[point_idx][1]]
 
             color = get_color(idx, alpha=0.1)
             color_pareto = get_color(idx)
 
             if show_all:
+                error_x = (
+                    dict(array=x_std, color="rgba(0, 0, 0, 0.3)")
+                    if show_error and not all(value == 0.0 for value in x_std)
+                    else None
+                )
+                error_y = (
+                    dict(array=y_std, color="rgba(0, 0, 0, 0.3)")
+                    if show_error and not all(value == 0.0 for value in y_std)
+                    else None
+                )
+
                 traces.append(
                     go.Scatter(
                         x=x,
                         y=y,
+                        error_x=error_x,
+                        error_y=error_y,
                         name=run.name,
                         mode="markers",
                         showlegend=False,
@@ -444,13 +491,26 @@ def load_outputs(runs, inputs, outputs) -> go.Figure:  # type: ignore
                 line_shape = "hv"
 
             hovertext = [
-                get_hovertext_from_config(run, config_id) for config_id in pareto_config_ids
+                get_hovertext_from_config(run, config_id, budget) for config_id in pareto_config_ids
             ]
 
+            error_pareto_x = (
+                dict(array=x_pareto_std, color="rgba(0, 0, 0, 0.3)")
+                if show_error and not all(value == 0.0 for value in x_pareto_std)
+                else None
+            )
+            error_pareto_y = (
+                dict(array=y_pareto_std, color="rgba(0, 0, 0, 0.3)")
+                if show_error and not all(value == 0.0 for value in y_pareto_std)
+                else None
+            )
+
             traces.append(
                 go.Scatter(
                     x=x_pareto,
                     y=y_pareto,
+                    error_x=error_pareto_x,
+                    error_y=error_pareto_y,
                     name=run.name,
                     line_shape=line_shape,
                     showlegend=True,