Merge development and solve conflicts

Author: Sarah Krebs

.flake8

@@ -7,6 +7,6 @@ extend-exclude =
     build
 extend-ignore =
     # No whitespace before ':' in [x : y]
-    E203
+    E203,
     # No lambdas — too strict
-    E731
+    E731,

.github/workflows/docs.yml

@@ -1,5 +1,9 @@
 name: docs
 
+concurrency:
+  group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.ref }}
+  cancel-in-progress: true
+
 on:
   # Trigger manually
   workflow_dispatch:
@@ -22,19 +26,22 @@ env:
 jobs:
   build-and-deploy:
     runs-on: ubuntu-latest
+    defaults:
+      run:
+        shell: bash # Default to using bash on all
     steps:
     - name: Checkout
-      uses: actions/checkout@v2
+      uses: actions/checkout@v4
 
     - name: Setup Python
-      uses: actions/setup-python@v2
+      uses: actions/setup-python@v5
       with:
         python-version: 3.9
 
     - name: Install dependencies
       run: |
         pip install .
-        pip install .[dev]
+        pip install ".[dev]"
 
     - name: Make docs
       run: |

.github/workflows/pytest.yml

@@ -1,25 +1,50 @@
 name: tests
-on: [ push ]
+
+# Cancel previous tests for this branch if a new commit is pushed
+concurrency:
+  group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.ref }}
+  cancel-in-progress: true
+
+on:
+  # Allow manual trigger of workflow
+  workflow_dispatch:
+
+  # Run tests on push/merge to main
+  push:
+    branches:
+      - main
+
+  # Run tests on anything that has a pull request to main/development
+  pull_request:
+    branches:
+      - main
+      - development
+
 jobs:
   pytest:
-    name: PyTest
-    runs-on: ubuntu-latest
-    steps:
-      - uses: actions/checkout@v2
-      - name: Add conda to system path
-        run: |
-          # $CONDA is an environment variable pointing to the root of the miniconda directory
-          echo $CONDA/bin >> $GITHUB_PATH
+    name: Test ${{ matrix.python-version }}-${{ matrix.os }}
+    runs-on: ${{ matrix.os }}
+    defaults:
+      run:
+        shell: bash # Default to using bash on all
+
+    strategy:
+      fail-fast: false
+      matrix:
+        python-version: ["3.9", "3.10"] # "3.11" fails due to swig
+        os: ["ubuntu-latest"] # "macos-latest", "windows-latest" fails due to swig
 
+    steps:
+      - uses: actions/checkout@v4
       - name: Setup Python 3.9
-        uses: actions/setup-python@v2
+        uses: actions/setup-python@v5
         with:
-          python-version: 3.9
+          python-version: ${{ matrix.python-version }}
 
       - name: Install dependencies
         run: |
-          conda install -c anaconda swig
-          make install-dev
-          
-      - name: Run pytest 
+          pip install swig
+          python -m pip install ".[dev]"
+
+      - name: Run pytest
         run: pytest tests

CHANGELOG.md

@@ -2,6 +2,9 @@
 
 ## Plugins
 - Add symbolic explanations plugin (#46).
+- It is now possible to view multiple unequal runs at once in Cost over Time and Pareto (#93).
+- Runs with unequal objectives cannot be displayed together.
+- Added an enum for displaying according warning messages.
 
 ## Enhancements
 - Fix lower bounds of dependency versions.
@@ -20,10 +23,17 @@
 - Reset inputs to fix error when subsequently selecting runs with different configspaces, objectives or budgets (#106).
 - Fix errors due to changing inputs before runselection (#64).
 - For fANOVA, remove constant hyperparameters from configspace (#9).
+- When getting budget, objectives etc from multiple runs in Cost over Time and Pareto Front:
+    - Instead of taking the first run as comparative value,
+    - take the one with the lowest budget, else the index for the budgets could be out of bounds.
 - For PCP, show hyperparameters with highest importance closest to the cost, i.e. right (#124).
 - Add init files to all test directories.
 - Correct LPI importance tests.
 
+## Documentation
+- Add How to Contribute section.
+- Expand documentation for partial dependence plugin.
+
 ## Version-Updates
 - Black version from 23.1.0 to 23.3.0
 - Mypy from 0.930 to 1.5.1
@@ -39,6 +49,10 @@
 - Major overhaul of docstrings in various files
 - Removed unused imports and variables
 
+## Groups
+- Groups now get a default name, if no name was entered by the user.
+- Groups get sorted alphabetically to appear more ordered.
+
 ## Additional Changes
 - Added a "make install examples" in Makefile
 
@@ -68,7 +82,7 @@
 - SMAC 2.0
 
 ## Dependencies
-- Remove SMAC dependency by adding required function directly
+- Remove SMAC dependency by adding required function directly.
 
 # Version 1.0.1
 

README.md

@@ -1,3 +1,5 @@
+<img src="docs/images/DeepCAVE_Logo_wide.png" alt="Logo"/> 
+
 # DeepCAVE
 
 DeepCAVE is a visualization and analysis tool for AutoML (especially for the sub-problem
@@ -14,7 +16,7 @@ Following features are provided:
 - A large collection of plugins to explore multiple areas like performance, hyperparameter and
 budget analysis.
 - Save your runs using DeepCAVE's native recorder.
-- Support for many optimizers using converter (e.g., DeepCAVE, SMAC and BOHB).
+- Support for many optimizers using converters (e.g., DeepCAVE, SMAC and BOHB).
 - Select runs directly from a working directory in the interface.
 - Select groups of runs for combined analysis.
 - Modularized plugin structure with access to selected runs/groups to provide maximal flexibility.
@@ -41,6 +43,7 @@ pip install DeepCAVE
 If you want to contribute to DeepCAVE use the following steps instead:
 ```bash
 git clone https://github.com/automl/DeepCAVE.git
+cd DeepCAVE
 conda create -n DeepCAVE python=3.9
 conda activate DeepCAVE
 conda install -c anaconda swig

deepcave/layouts/general.py

@@ -337,6 +337,7 @@ def get_layout(
 
             # Load from cache if page is loaded
             children = []
+
             for name, paths in groups.items():
                 if name is None:
                     continue
@@ -368,10 +369,29 @@ def callback(group_names: List[str], all_run_paths, i):
                 self._refresh_groups = False
                 raise PreventUpdate()
 
+            # For the default group names, if no name was entered
+            group_counter = 0
+
             groups = {}
             for group_name, run_paths in zip(group_names, all_run_paths):
                 if group_name is None or group_name == "":
-                    continue
+                    # Set the default group name with a counter,
+                    # so the groups dont overwrite themselves
+                    group_name_unavailable = True
+
+                    groups_cache = c.get("groups")
+                    if groups_cache is None:
+                        continue
+
+                    # Check to see that no group name that already exists
+                    # gets picked
+                    while group_name_unavailable:
+                        group_name = f"Group {group_counter}"
+                        assert groups_cache is not None
+                        if group_name not in groups_cache.keys():
+                            group_name_unavailable = False
+                        else:
+                            group_counter += 1
 
                 if run_paths is None or len(run_paths) == 0:
                     continue
@@ -386,16 +406,20 @@ def callback(group_names: List[str], all_run_paths, i):
 
                 groups[group_name] = valid_run_paths
 
+            # Sort the groups alphabetically, so when added
+            # they appear ordered
+            sorted_groups = dict(sorted(groups.items()))
+
             try:
                 # Now save it
-                run_handler.update_groups(groups)
+                run_handler.update_groups(sorted_groups)
             except NotMergeableError:
                 notification.update("The selected runs are not mergeable.")
 
                 # This will automatically trigger the group display s.t. the selection is redo.
                 return i + 1
 
-            self.logger.debug(f"Groups: {groups}")
+            self.logger.debug(f"Groups: {sorted_groups}")
 
             raise PreventUpdate()
 

deepcave/plugins/__init__.py

@@ -917,7 +917,6 @@ def __call__(self, render_button: bool = False) -> List[Component]:
             ]
         else:
             components += [html.H1(self.name)]
-
         try:
             self.check_runs_compatibility(self.all_runs)
         except NotMergeableError as message:

deepcave/plugins/dynamic.py

@@ -95,6 +95,7 @@ def plugin_output_update(_: Any, *inputs_list: str) -> Any:
             runs = self.get_selected_runs(inputs)
 
             raw_outputs = {}
+            rc.clear()
             for run in runs:
                 run_outputs = rc.get(run, self.id, inputs_key)
                 if run_outputs is None:

deepcave/plugins/hyperparameter/importances.py

@@ -147,7 +147,10 @@ def get_filter_layout(register: Callable) -> List[html.Div]:
             html.Div(
                 [
                     dbc.Label("Budgets"),
-                    help_button("The hyperparameters are sorted by the highest budget."),
+                    help_button(
+                        "Budget refers to the multi-fidelity budget. "
+                        "The hyperparameters are sorted by the highest budget."
+                    ),
                     dbc.Checklist(id=register("budget_ids", ["value", "options"]), inline=True),
                 ]
             ),

deepcave/plugins/hyperparameter/pdp.py

@@ -85,8 +85,9 @@ def get_input_layout(register: Callable) -> List[dbc.Row]:
                         [
                             dbc.Label("Budget"),
                             help_button(
+                                "Budget refers to the multi-fidelity budget. "
                                 "Combined budget means that the trial on the highest"
-                                " evaluated budget is used.\n\n"
+                                " evaluated budget is used.  \n "
                                 "Note: Selecting combined budget might be misleading if"
                                 " a time objective is used. Often, higher budget take "
                                 " longer to evaluate, which might negatively influence "