Merge pull request #132 from automl/save-higher-resolution-plots

sarah-segel · web-flow · commit 58143606a971 · 2024-04-24T09:22:34.000+02:00
Bug/fix-some-minor-problems
diff --git a/CHANGELOG.md b/CHANGELOG.md
@@ -14,6 +14,7 @@
 - Add exit button which first deletes running jobs and then terminates DeepCave.
 - Nicer handling of Keyboard Interrupt.
 - Disable debug mode.
+- Save plotly plots in higher resolution upon download.
 
 ## Bug-Fixes
 - Fix missing objective specification in LPI evaluator (#71).
@@ -25,6 +26,9 @@
 - When getting budget, objectives etc from multiple runs in Cost over Time and Pareto Front:
     - Instead of taking the first run as comparative value,
     - take the one with the lowest budget, else the index for the budgets could be out of bounds.
+- For PCP, show hyperparameters with highest importance closest to the cost, i.e. right (#124).
+- Add init files to all test directories.
+- Correct LPI importance tests.
 
 ## Documentation
 - Add How to Contribute section.
diff --git a/deepcave/config.py b/deepcave/config.py
@@ -56,6 +56,7 @@ class Config:
     SAVE_IMAGES = False  # The figure will be saved to the cache directory.
     FIGURE_MARGIN = dict(t=30, b=0, l=0, r=0)
     FIGURE_HEIGHT = "40vh"
+    FIGURE_DOWNLOAD_SCALE = 4.0
 
     # Redis settings
     REDIS_PORT: int = 6379
diff --git a/deepcave/plugins/budget/budget_correlation.py b/deepcave/plugins/budget/budget_correlation.py
@@ -220,7 +220,11 @@ def get_output_layout(register: Callable) -> List[Any]:
                 [
                     dbc.Tab(
                         dcc.Graph(
-                            id=register("graph", "figure"), style={"height": Config.FIGURE_HEIGHT}
+                            id=register("graph", "figure"),
+                            style={"height": Config.FIGURE_HEIGHT},
+                            config={
+                                "toImageButtonOptions": {"scale": Config.FIGURE_DOWNLOAD_SCALE}
+                            },
                         ),
                         label="Graph",
                     ),
diff --git a/deepcave/plugins/hyperparameter/importances.py b/deepcave/plugins/hyperparameter/importances.py
@@ -357,7 +357,11 @@ def get_output_layout(register: Callable) -> dcc.Graph:
         dcc.Graph
             Layout for the output block.
         """
-        return dcc.Graph(register("graph", "figure"), style={"height": Config.FIGURE_HEIGHT})
+        return dcc.Graph(
+            register("graph", "figure"),
+            style={"height": Config.FIGURE_HEIGHT},
+            config={"toImageButtonOptions": {"scale": Config.FIGURE_DOWNLOAD_SCALE}},
+        )
 
     @staticmethod
     def load_outputs(run, inputs, outputs) -> go.Figure:  # type: ignore
diff --git a/deepcave/plugins/hyperparameter/pdp.py b/deepcave/plugins/hyperparameter/pdp.py
@@ -366,7 +366,11 @@ def get_output_layout(register: Callable) -> dcc.Graph:
         dcc.Graph
             Layout for the output block.
         """
-        return dcc.Graph(register("graph", "figure"), style={"height": Config.FIGURE_HEIGHT})
+        return dcc.Graph(
+            register("graph", "figure"),
+            style={"height": Config.FIGURE_HEIGHT},
+            config={"toImageButtonOptions": {"scale": Config.FIGURE_DOWNLOAD_SCALE}},
+        )
 
     @staticmethod
     def get_pdp_figure(  # type: ignore
diff --git a/deepcave/plugins/hyperparameter/symbolic_explanations.py b/deepcave/plugins/hyperparameter/symbolic_explanations.py
@@ -491,8 +491,16 @@ def get_output_layout(register: Callable) -> List[dcc.Graph]:
             Layout for the output block.
         """
         return [
-            dcc.Graph(register("symb_graph", "figure"), style={"height": Config.FIGURE_HEIGHT}),
-            dcc.Graph(register("pdp_graph", "figure"), style={"height": Config.FIGURE_HEIGHT}),
+            dcc.Graph(
+                register("symb_graph", "figure"),
+                style={"height": Config.FIGURE_HEIGHT},
+                config={"toImageButtonOptions": {"scale": Config.FIGURE_DOWNLOAD_SCALE}},
+            ),
+            dcc.Graph(
+                register("pdp_graph", "figure"),
+                style={"height": Config.FIGURE_HEIGHT},
+                config={"toImageButtonOptions": {"scale": Config.FIGURE_DOWNLOAD_SCALE}},
+            ),
         ]
 
     @staticmethod
diff --git a/deepcave/plugins/objective/configuration_cube.py b/deepcave/plugins/objective/configuration_cube.py
@@ -294,7 +294,13 @@ def get_output_layout(register: Callable) -> Tuple[dcc.Graph,]:
         Tuple[dcc.Graph,]
             Layout for the output block.
         """
-        return (dcc.Graph(register("graph", "figure"), style={"height": Config.FIGURE_HEIGHT}),)
+        return (
+            dcc.Graph(
+                register("graph", "figure"),
+                style={"height": Config.FIGURE_HEIGHT},
+                config={"toImageButtonOptions": {"scale": Config.FIGURE_DOWNLOAD_SCALE}},
+            ),
+        )
 
     @staticmethod
     def load_outputs(run, inputs, outputs) -> go.Figure:  # type: ignore
diff --git a/deepcave/plugins/objective/cost_over_time.py b/deepcave/plugins/objective/cost_over_time.py
@@ -298,7 +298,11 @@ def get_output_layout(register: Callable) -> dcc.Graph:
         dcc.Graph
             The layouts for the output block.
         """
-        return dcc.Graph(register("graph", "figure"), style={"height": Config.FIGURE_HEIGHT})
+        return dcc.Graph(
+            register("graph", "figure"),
+            style={"height": Config.FIGURE_HEIGHT},
+            config={"toImageButtonOptions": {"scale": Config.FIGURE_DOWNLOAD_SCALE}},
+        )
 
     @staticmethod
     def load_outputs(runs, inputs, outputs) -> go.Figure:  # type: ignore
diff --git a/deepcave/plugins/objective/parallel_coordinates.py b/deepcave/plugins/objective/parallel_coordinates.py
@@ -299,7 +299,9 @@ def process(run, inputs) -> Dict[str, Any]:  # type: ignore
             evaluator.calculate(objective, budget, n_trees=10, seed=0)
             importances_dict = evaluator.get_importances()
             importances = {u: v[0] for u, v in importances_dict.items()}
-            important_hp_names = sorted(importances, key=lambda key: importances[key], reverse=True)
+            important_hp_names = sorted(
+                importances, key=lambda key: importances[key], reverse=False
+            )
             result["important_hp_names"] = important_hp_names
 
         return result
@@ -320,7 +322,11 @@ def get_output_layout(register: Callable) -> dcc.Graph:
         dcc.Graph
             The layouts for the output block.
         """
-        return dcc.Graph(register("graph", "figure"), style={"height": Config.FIGURE_HEIGHT})
+        return dcc.Graph(
+            register("graph", "figure"),
+            style={"height": Config.FIGURE_HEIGHT},
+            config={"toImageButtonOptions": {"scale": Config.FIGURE_DOWNLOAD_SCALE}},
+        )
 
     @staticmethod
     def load_outputs(run, inputs, outputs) -> go.Figure:  # type: ignore
diff --git a/deepcave/plugins/objective/pareto_front.py b/deepcave/plugins/objective/pareto_front.py
@@ -352,7 +352,11 @@ def get_output_layout(register: Callable) -> dcc.Graph:
         dcc.Graph
             The layout for the output block.
         """
-        return dcc.Graph(register("graph", "figure"), style={"height": Config.FIGURE_HEIGHT})
+        return dcc.Graph(
+            register("graph", "figure"),
+            style={"height": Config.FIGURE_HEIGHT},
+            config={"toImageButtonOptions": {"scale": Config.FIGURE_DOWNLOAD_SCALE}},
+        )
 
     @staticmethod
     def load_outputs(runs, inputs, outputs) -> go.Figure:  # type: ignore
diff --git a/deepcave/plugins/static.py b/deepcave/plugins/static.py
@@ -157,7 +157,7 @@ def plugin_process(n_clicks: int, _: Any, *inputs_list: str) -> Optional[Any]:
             raw_outputs = {}
             raw_outputs_available = True
             for run in runs:
-                raw_outputs[run.id] = rc.get(run, self.id, inputs_key)  # same problem
+                raw_outputs[run.id] = rc.get(run, self.id, inputs_key)
 
                 if raw_outputs[run.id] is None:
                     raw_outputs_available = False
@@ -186,7 +186,7 @@ def plugin_process(n_clicks: int, _: Any, *inputs_list: str) -> Optional[Any]:
 
                     # Check if processing is needed
                     for run in runs:
-                        job_id = self._get_job_id(run.id, inputs_key)  # same problem
+                        job_id = self._get_job_id(run.id, inputs_key)
 
                         # Results are already achieved or it was already processed
                         if raw_outputs[run.id] is not None or queue.is_processed(job_id):
diff --git a/deepcave/plugins/summary/configurations.py b/deepcave/plugins/summary/configurations.py
@@ -287,6 +287,9 @@ def get_output_layout(
                         dcc.Graph(
                             id=register("performance_graph", "figure"),
                             style={"height": Config.FIGURE_HEIGHT},
+                            config={
+                                "toImageButtonOptions": {"scale": Config.FIGURE_DOWNLOAD_SCALE}
+                            },
                         ),
                         label="Graph",
                     ),
@@ -301,6 +304,9 @@ def get_output_layout(
                         dcc.Graph(
                             id=register("configspace_graph", "figure"),
                             style={"height": Config.FIGURE_HEIGHT},
+                            config={
+                                "toImageButtonOptions": {"scale": Config.FIGURE_DOWNLOAD_SCALE}
+                            },
                         ),
                         label="Graph",
                     ),
diff --git a/deepcave/plugins/summary/footprint.py b/deepcave/plugins/summary/footprint.py
@@ -296,13 +296,17 @@ def get_output_layout(register: Callable) -> dbc.Tabs:
             [
                 dbc.Tab(
                     dcc.Graph(
-                        id=register("performance", "figure"), style={"height": Config.FIGURE_HEIGHT}
+                        id=register("performance", "figure"),
+                        style={"height": Config.FIGURE_HEIGHT},
+                        config={"toImageButtonOptions": {"scale": Config.FIGURE_DOWNLOAD_SCALE}},
                     ),
                     label="Performance",
                 ),
                 dbc.Tab(
                     dcc.Graph(
-                        id=register("area", "figure"), style={"height": Config.FIGURE_HEIGHT}
+                        id=register("area", "figure"),
+                        style={"height": Config.FIGURE_HEIGHT},
+                        config={"toImageButtonOptions": {"scale": Config.FIGURE_DOWNLOAD_SCALE}},
                     ),
                     label="Coverage",
                 ),
diff --git a/deepcave/plugins/summary/overview.py b/deepcave/plugins/summary/overview.py
@@ -84,13 +84,19 @@ def get_output_layout(register: Callable) -> List[Any]:
                         dcc.Graph(
                             id=register("status_statistics", "figure"),
                             style={"height": Config.FIGURE_HEIGHT},
+                            config={
+                                "toImageButtonOptions": {"scale": Config.FIGURE_DOWNLOAD_SCALE}
+                            },
                         ),
                         label="Barplot",
                     ),
                     dbc.Tab(
                         dcc.Graph(
                             id=register("config_statistics", "figure"),
                             style={"height": Config.FIGURE_HEIGHT},
+                            config={
+                                "toImageButtonOptions": {"scale": Config.FIGURE_DOWNLOAD_SCALE}
+                            },
                         ),
                         label="Heatmap",
                     ),
diff --git a/setup.py b/setup.py
@@ -35,7 +35,7 @@ def read_file(file_name):
 
 setuptools.setup(
     name="deepcave",
-    author_email="sass@tnt.uni-hannover.de",
+    author_email="s.segel@ai.uni-hannover.de",
     description="An interactive framework to visualize and analyze your AutoML process in real-time.",
     long_description=read_file("README.md"),
     long_description_content_type="text/markdown",
diff --git a/tests/test_evaluators/test_lpi.py b/tests/test_evaluators/test_lpi.py
@@ -2,13 +2,13 @@
 
 from deepcave.evaluators.lpi import LPI as Evaluator
 from deepcave.runs import AbstractRun
-from deepcave.runs.converters.smac3v1 import SMAC3v1Run
+from deepcave.runs.converters.smac3v2 import SMAC3v2Run
 
 
 class TestLPI(unittest.TestCase):
     def setUp(self):
         # Initiate run here
-        self.run: AbstractRun = SMAC3v1Run.from_path("logs/SMAC3v1/mlp/run_1")
+        self.run: AbstractRun = SMAC3v2Run.from_path("logs/SMAC3v2/mlp/run_1")
         self.hp_names = self.run.configspace.get_hyperparameter_names()
         self.evaluator = Evaluator(self.run)
 
@@ -17,10 +17,28 @@ def test(self):
         objective = self.run.get_objective(0)
 
         # Calculate
-        self.evaluator.calculate(objective, budget)
-        importance_dict = self.evaluator.get_importances(self.hp_names)
+        self.evaluator.calculate(objective, budget, seed=0)
+        importances = self.evaluator.get_importances(self.hp_names)
 
-        print(importance_dict)
+        self.evaluator.calculate(objective, budget, seed=42)
+        importances2 = self.evaluator.get_importances(self.hp_names)
+
+        # Different seed: Different results
+        assert importances["batch_size"][1] != importances2["batch_size"][1]
+
+    def test_seed(self):
+        budget = self.run.get_budget(0)
+        objective = self.run.get_objective(0)
+
+        # Calculate
+        self.evaluator.calculate(objective, budget, seed=0)
+        importances = self.evaluator.get_importances(self.hp_names)
+
+        self.evaluator.calculate(objective, budget, seed=0)
+        importances2 = self.evaluator.get_importances(self.hp_names)
+
+        # Same seed: Same results
+        assert importances["batch_size"][1] == importances2["batch_size"][1]
 
 
 if __name__ == "__main__":
diff --git a/tests/test_runs/__init__.py b/tests/test_runs/__init__.py
diff --git a/tests/test_utils/__init__.py b/tests/test_utils/__init__.py
