Merge pull request #183 from automl/development

Version 1.3

Author: sarah-segel

.pre-commit-config.yaml

@@ -48,22 +48,25 @@ repos:
             'absl-py>=1.0.0',
             'jsonlines>=3.0.0',
             'pandas>=1.3.4',
-            'numpy>=1.22.2',
-            'matplotlib>=3.5.1',
+            'numpy==2.0.1',
+            'matplotlib==3.9.0',
+            'seaborn>=0.13.0',
             'pyyaml>=6.0.1',
             'kaleido>=0.2.1',
             'gplearn>=0.4.2',
             'sympy>=1.12',
-            'ConfigSpace==0.6.1',
+            'requests>=2.31.0',
+            'ConfigSpace==1.2.0',
             'pyrfr>=0.9.0',
-            'hpbandster==0.7.4',
             'dash==2.0.0',
             'dash-extensions==0.0.71',
             'dash-bootstrap-components==1.0.3',
             'redis>=4.1.4',
             'rq>=1.10.1',
             'werkzeug==2.0.3',
-            'pyPDPPartitioner>=0.1.8'
+            'pyarrow==16.1.0',
+            'fastparquet==2024.5.0',
+            'pyPDPPartitioner>=0.1.9'
         ] # Needed for mypy, so that it knows the types to check
 
   - repo: https://github.com/PyCQA/flake8

CHANGELOG.md

@@ -1,3 +1,37 @@
+# Version 1.3
+
+## Converters
+- Add support for AMLTK.
+- Add support for Optuna.
+- Add support for adding a run in DataFrame format (#137).
+- BOHB: Consider missing state as valid state as state is not provided by default (#65).
+
+## Version-Updates
+- ConfigSpace from 0.6.1 to 1.2.0
+- Numpy from 1.26.4 to 2.0.1
+- pyPDPPartitioner from to 0.1.8 to 0.1.9
+
+## Ablation Paths Plugin
+- Added ablation paths as a plugin.
+- Added ablation as a evaluator to use for the plugin.
+- Added tests as well as documentation.
+- Modified the RandomForest surrogate model, so it can be passed the number of trees.
+
+## Move plugins
+- Moved the Configuration cube and the parallel coordinates plugin from objective to hyperparameter.
+- Changed layout accordingly as well as the documentation.
+
+## Bug-Fixes
+- Upgrade pyPDPPartitioner to 0.1.9 as a requirement which fixes a bug related to copying config space objects (#111).
+
+## Quality of Life
+- Update the documentation to reflect the new changes.
+- Disable warnings during RunHandler initialization to avoid showing warnings with respect to previously loaded runs.
+- Remove option to use docker (not yet supported).
+- Add SMAC3v2 multi-fidelity example runs.
+- Add more useful SMAC3v2 multi-objective example run.
+- Remove defective matplotlib button and associated functionalities.
+
 # Version 1.2.1
 
 ## Quality of Life

Makefile

@@ -2,7 +2,7 @@
 # are usually completed in github actions.
 
 SHELL := /bin/bash
-VERSION := 1.2.1
+VERSION := 1.3
 
 NAME := DeepCAVE
 PACKAGE_NAME := deepcave
@@ -52,6 +52,12 @@ install-dev:
 install-examples:
 	$(PIP) install -e ".[examples]"
 
+install-optuna:
+	$(PIP) install -e ".[optuna]"
+
+install-bohb:
+	$(PIP) install -e ".[bohb]"
+
 check-black:
 	$(BLACK) ${SOURCE_DIR} --check || :
 	$(BLACK) ${EXAMPLES_DIR} --check || :

README.md

@@ -2,30 +2,15 @@
 
 # DeepCAVE
 
-DeepCAVE is a visualization and analysis tool for AutoML (especially for the sub-problem
-hyperparameter optimization) runs. The framework is programmed on top of Dash and therefore
-entirely interactive. Multiple and diverse plugins make it possible to efficiently generate insights
-and bring the human back in the loop. Moreover, the powerful run interface and the modularized
-plugin structure allow extending the tool at any time effortlessly.
+DeepCAVE is a visualization and analysis tool for AutoML, with a particular focus on
+hyperparameter optimization (HPO). Built on the Dash framework, it offers a fully
+interactive experience. The tool features a variety of plugins that enable efficient insight
+generation, aiding in understanding and debugging the application of HPO.
+Additionally, the powerful run interface and the modularized plugin structure allow extending the 
+tool at any time effortlessly.
 
 ![Configuration Footprint](docs/images/plugins/configuration_footprint.png)
 
-Following features are provided:
-- Interactive Dashboard (completely written in Python) to self-analyze optimization runs/processes.
-- Analyzing while optimizing (run changes are automatically detected).
-- A large collection of plugins to explore multiple areas like performance, hyperparameter and
-budget analysis.
-- Save your runs using DeepCAVE's native recorder.
-- Support for many optimizers using converters (e.g., DeepCAVE, SMAC and BOHB).
-- Select runs directly from a working directory in the interface.
-- Select groups of runs for combined analysis.
-- Modularized plugin structure with access to selected runs/groups to provide maximal flexibility.
-- Asynchronous execution of expensive plugins and caching of their results.
-- Help buttons and integrated documentation in the interface helps you to understand the plugins.
-- Use the matplotlib mode to customize and save the plots for your publication.
-- The API mode gives you full access to the code, while you do not have to interact with the 
-interface. Otherwise, you can also make use of the raw data, provided by every plugin.
-
 
 ## Installation
 
@@ -50,41 +35,21 @@ conda install -c anaconda swig
 make install-dev
 ```
 
-If you want to use the given examples, run this after installing:
+If you want to try the examples for recording your results in DeepCAVE format, run this after installing:
 ```bash
 make install-examples
 ```
 
-Please visit the [documentation](https://automl.github.io/DeepCAVE/main/installation.html) to get
-further help (e.g. if you can not install redis server or you are on a mac).
-
-
-## Recording
-
-A minimal example is given to show the simplicity yet powerful API to record runs.
-However, existing optimizers like BOHB, SMAC, Auto-Sklearn, Auto-PyTorch are supported natively.
-
-```python
-import ConfigSpace as CS
-from deepcave import Recorder, Objective
-
-
-configspace = CS.ConfigurationSpace(seed=0)
-alpha = CS.hyperparameters.UniformFloatHyperparameter(
-    name='alpha', lower=0, upper=1)
-configspace.add_hyperparameter(alpha)
-
-accuracy = Objective("accuracy", lower=0, upper=1, optimize="upper")
-mse = Objective("mse", lower=0)
-
-with Recorder(configspace, objectives=[accuracy, mse]) as r:
-    for config in configspace.sample_configuration(100):
-        for budget in [20, 40, 60]:
-            r.start(config, budget)
-            # Your code goes here
-            r.end(costs=[0.5, 0.5])
+To load runs created with Optuna or the BOHB optimizer, you need to install the
+respective packages by running:
+```bash
+make install-optuna
+make install-bohb
 ```
 
+Please visit the [documentation](https://automl.github.io/DeepCAVE/main/installation.html) to get
+further help (e.g. if you cannot install redis server or if you are on MacOS).
+
 
 ## Visualizing and Evaluating
 
@@ -100,12 +65,56 @@ You can find more arguments and information (like using custom configurations) i
 
 ## Example runs
 
-DeepCAVE comes with some pre-evaluated runs to get a feel for what DeepCAVE can do.
+DeepCAVE comes with some pre-evaluated runs to get a feeling for what DeepCAVE can do.
 
-When you installed the package from GitHub via `git clone git@github.com:automl/DeepCAVE.git; pip install -e DeepCAVE`,
+If you cloned the repository from GitHub via `git clone https://github.com/automl/DeepCAVE.git`,
 you can try out some examples by exploring the `logs` directory inside the DeepCAVE dashboard.
 For example, if you navigate to `logs/DeepCAVE`, you can view the run `mnist_pytorch` if you hit
-the `+` button.
+the `+` button left to it.
+
+
+## Features
+
+### Interactive Interface
+- **Interactive Dashboard:**  
+  The dashboard runs in a webbrowser and allows you to self-analyze your optimization runs interactively.
+  
+- **Run Selection Interface:**  
+  Easily select runs from your working directory directly within the interface.
+  
+- **Integrated Help and Documentation:**  
+  Use help buttons and integrated documentation within the interface to better understand the plugins.
+
+### Comprehensive Analysis Tools
+- **Extensive Plugin Collection:**  
+  Explore a wide range of plugins for in-depth performance, hyperparameter, and budget analysis.
+
+- **Analysis of Running Processes:**  
+  Analyze and monitor optimization processes as they occur, with automatic detection of run changes.
+  
+- **Group Analysis:**  
+  Choose groups of runs for combined analysis to gain deeper insights.
+
+### Flexible and Modular Architecture
+- **Modular Plugin Architecture:**  
+  Benefit from a modularized plugin structure with access to selected runs and groups, offering you maximum flexibility.
+  
+- **Asynchronous Execution:**  
+  Utilize asynchronous execution of resource-intensive plugins and caching of results to improve performance.
+
+### Broad Optimizer Support
+- **Optimizer Support:**  
+  Work with many frameworks and optimizers using our converters, including converters for SMAC, BOHB, AMLTK, and Optuna.
+  
+- **Native Format Saving:**  
+  Save AutoML runs from various frameworks in DeepCAVE's native format using the built-in recorder.
+  
+- **Flexible Data Loading:**  
+  Alternatively, load AutoML runs from other frameworks by converting them into a Pandas DataFrame.
+
+### Developer and API Features
+- **API Mode:**  
+  Interact with the code directly through API mode, allowing you to bypass the graphical interface if preferred.
 
 
 ## Citation

deepcave/__init__.py

@@ -12,21 +12,22 @@
 import datetime
 import os
 import sys
+import warnings
 from functools import wraps
 from pathlib import Path
 
 name = "DeepCAVE"
 package_name = "deepcave"
 author = "R. Sass and E. Bergman and A. Biedenkapp and F. Hutter and M. Lindauer"
-author_email = "s.segel@ai.uni-hannover.de"
+author_email = "l.fehring@ai.uni-hannover.de"
 description = "An interactive framework to visualize and analyze your AutoML process in real-time."
 url = "automl.org"
 project_urls = {
     "Documentation": "https://automl.github.io/DeepCAVE/main",
     "Source Code": "https://github.com/automl/deepcave",
 }
 copyright = f"Copyright {datetime.date.today().strftime('%Y')}, {author}"
-version = "1.2.1"
+version = "1.3"
 
 _exec_file = sys.argv[0]
 _exec_files = ["server.py", "worker.py", "sphinx-build"]
@@ -115,8 +116,13 @@ def get_app(title: str) -> Any:
         # Run caches
         rc = RunCaches(config)
 
-        # Run Handler
-        run_handler = RunHandler(config, c, rc)
+        # Supress warnings during initializing run handler to avoid showing warnings with respect
+        # to previously loaded runs
+        with warnings.catch_warnings():
+            warnings.simplefilter("ignore")
+
+            # Run Handler
+            run_handler = RunHandler(config, c, rc)
 
         # Notifications
         notification = Notification()

deepcave/cli.py

@@ -25,7 +25,6 @@
     "How many workers should be specified. In your case, the maximum number of workers should be "
     f"{multiprocessing.cpu_count() - 1}.",
 )
-flags.DEFINE_boolean("docker", False, "Uses docker image to start DeepCAVE. Not supported yet.")
 flags.DEFINE_string("config", None, "Filename to a user-specific config.")
 flags.DEFINE_string(
     "get_config_value", None, "Prints the value of a given config key. Useful for bash scripts."
@@ -44,18 +43,14 @@ def execute(_: Any) -> None:
 
     HERE = Path(__file__).parent
 
-    if FLAGS.docker:
-        exit("The command is not supported yet.")
-        # subprocess.call('./start_docker.sh')
-    else:
-        start = HERE / "start.sh"
-        open = "true" if FLAGS.open else "false"
-        n_workers = str(FLAGS.n_workers)
+    start = HERE / "start.sh"
+    open = "true" if FLAGS.open else "false"
+    n_workers = str(FLAGS.n_workers)
 
-        if FLAGS.config is not None:
-            subprocess.call([start, open, n_workers, str(FLAGS.config)])
-        else:
-            subprocess.call([start, open, n_workers])
+    if FLAGS.config is not None:
+        subprocess.call([start, open, n_workers, str(FLAGS.config)])
+    else:
+        subprocess.call([start, open, n_workers])
 
 
 def main() -> None:

deepcave/config.py

@@ -69,7 +69,6 @@ class Config:
 
     # Default Meta information which are used across the platform
     META_DEFAULT: Dict[str, Any] = {
-        "matplotlib-mode": False,
         "working_dir": None,  # str(DEFAULT_WORKING_DIRECTORY),
         "selected_run_paths": [],
         "groups": {},  # {group_name: [run_path, ...]}
@@ -94,14 +93,17 @@ def SERVER_NAME(self) -> str:
     def PLUGINS(self) -> Dict[str, List[Any]]:
         """A list of available plugins per category."""
         from deepcave.plugins.budget.budget_correlation import BudgetCorrelation
+        from deepcave.plugins.hyperparameter.ablation_paths import AblationPaths
+        from deepcave.plugins.hyperparameter.configuration_cube import ConfigurationCube
         from deepcave.plugins.hyperparameter.importances import Importances
+        from deepcave.plugins.hyperparameter.parallel_coordinates import (
+            ParallelCoordinates,
+        )
         from deepcave.plugins.hyperparameter.pdp import PartialDependencies
         from deepcave.plugins.hyperparameter.symbolic_explanations import (
             SymbolicExplanations,
         )
-        from deepcave.plugins.objective.configuration_cube import ConfigurationCube
         from deepcave.plugins.objective.cost_over_time import CostOverTime
-        from deepcave.plugins.objective.parallel_coordinates import ParallelCoordinates
         from deepcave.plugins.objective.pareto_front import ParetoFront
         from deepcave.plugins.summary.configurations import Configurations
         from deepcave.plugins.summary.footprint import FootPrint
@@ -116,15 +118,16 @@ def PLUGINS(self) -> Dict[str, List[Any]]:
             ],
             "Objective Analysis": [
                 CostOverTime(),
-                ConfigurationCube(),
                 ParetoFront(),
-                ParallelCoordinates(),
             ],
             "Budget Analysis": [
                 BudgetCorrelation(),
             ],
             "Hyperparameter Analysis": [
                 Importances(),
+                AblationPaths(),
+                ConfigurationCube(),
+                ParallelCoordinates(),
                 PartialDependencies(),
                 SymbolicExplanations(),
             ],
@@ -134,9 +137,12 @@ def PLUGINS(self) -> Dict[str, List[Any]]:
     @property
     def CONVERTERS(self) -> List[Type["Run"]]:
         """Get a list of available run converters."""
+        from deepcave.runs.converters.amltk import AMLTKRun
         from deepcave.runs.converters.bohb import BOHBRun
+        from deepcave.runs.converters.dataframe import DataFrameRun
         from deepcave.runs.converters.deepcave import DeepCAVERun
+        from deepcave.runs.converters.optuna import OptunaRun
         from deepcave.runs.converters.smac3v1 import SMAC3v1Run
         from deepcave.runs.converters.smac3v2 import SMAC3v2Run
 
-        return [DeepCAVERun, BOHBRun, SMAC3v1Run, SMAC3v2Run]
+        return [AMLTKRun, BOHBRun, DeepCAVERun, OptunaRun, SMAC3v1Run, SMAC3v2Run, DataFrameRun]

deepcave/evaluators/__init__.py

@@ -10,4 +10,6 @@
     This module provides utilities for creating a footprint of a run.
 lpi
     This module provides utilities to calculate the local parameter importance (LPI).
+ablation
+    This module evaluates the ablation paths.
 """