Merge pull request #94 from automl/bug/pre-commit-hooks

Bug/pre commit hooks

Author: sarah-segel

.flake8

@@ -6,5 +6,7 @@ extend-exclude =
     .venv
     build
 extend-ignore =
-    E203  # No whitespace before ':' in [x : y]
-    E731  # No lambdas — too strict
+    # No whitespace before ':' in [x : y]
+    E203
+    # No lambdas — too strict
+    E731

.pre-commit-config.yaml

@@ -14,7 +14,7 @@ repos:
         files: tests
 
   - repo: https://github.com/ambv/black
-    rev: 23.1.0
+    rev: 23.3.0
     hooks:
       - id: black
         name: black formatter deepcave
@@ -33,14 +33,38 @@ repos:
     hooks:
       - id: pydocstyle
         files: deepcave
-        additional_dependencies: ["toml"] # Needed to parse pyproject.toml
+        additional_dependencies: ["tomli"] # Needed to parse pyproject.toml
 
   - repo: https://github.com/pre-commit/mirrors-mypy
-    rev: v0.930
+    rev: v1.5.1
     hooks:
       - id: mypy
         name: mypy deepcave
         files: deepcave
+        args: [--install-types, --non-interactive]
+        additional_dependencies: [
+            'wheel>=0.41.2',
+            'setuptools==68.2.2',
+            'absl-py>=1.0.0',
+            'jsonlines>=3.0.0',
+            'pandas>=1.3.4',
+            'numpy>=1.22.2',
+            'matplotlib>=3.5.1',
+            'pyyaml>=6.0.1',
+            'kaleido>=0.2.1',
+            'gplearn>=0.4.2',
+            'sympy>=1.12',
+            'ConfigSpace==0.6.1',
+            'pyrfr>=0.9.0',
+            'hpbandster==0.7.4',
+            'dash==2.0.0',
+            'dash-extensions==0.0.71',
+            'dash-bootstrap-components==1.0.3',
+            'redis>=4.1.4',
+            'rq>=1.10.1',
+            'werkzeug==2.0.3',
+            'pyPDPPartitioner>=0.1.8'
+        ] # Needed for mypy, so that it knows the types to check
 
   - repo: https://github.com/PyCQA/flake8
     rev: 6.0.0

CHANGELOG.md

@@ -20,6 +20,24 @@
 - Fix errors due to changing inputs before runselection (#64).
 - For fANOVA, remove constant hyperparameters from configspace (#9).
 
+## Version-Updates
+- Black version from 23.1.0 to 23.3.0
+- Mypy from 0.930 to 1.5.1
+
+## Mypy
+- Updated args so there are no missing imports
+- Updated additional dependencies, so mypy can check all types
+- Note: If the installs in requirements change, it has to be adapted in additional dependencies
+- Added many type annotations
+- In some cases internal refactoring for variables, due to typing
+
+## Pydocstyle and Linter
+- Major overhaul of docstrings in various files
+- Removed unused imports and variables
+
+## Additional Changes
+- Added a "make install examples" in Makefile
+
 # Version 1.1.3
 
 ## Bug-Fixes

Makefile

@@ -48,7 +48,10 @@ install:
 install-dev:
 	$(PIP) install -e ".[dev]"
 	pre-commit install
-
+	
+install-examples:
+	$(PIP) install -e ".[examples]"
+	
 check-black:
 	$(BLACK) ${SOURCE_DIR} --check || :
 	$(BLACK) ${EXAMPLES_DIR} --check || :
@@ -62,7 +65,7 @@ check-pydocstyle:
 	$(PYDOCSTYLE) ${SOURCE_DIR} || :
 
 check-mypy:
-	$(MYPY) ${SOURCE_DIR} || :
+	$(MYPY) --check-untyped-defs --install-types --non-interactive --ignore-missing-imports ${SOURCE_DIR} || :
 
 check-flake8:
 	$(FLAKE8) ${SOURCE_DIR} || :

README.md

@@ -47,6 +47,11 @@ conda install -c anaconda swig
 make install-dev
 ```
 
+If you want to use the given examples, run this after installing:
+```bash
+make install-examples
+```
+
 Please visit the [documentation](https://automl.github.io/DeepCAVE/main/installation.html) to get
 further help (e.g. if you can not install redis server or you are on a mac).
 

deepcave/__init__.py

@@ -1,3 +1,12 @@
+#  noqa: D400
+"""
+# DeepCAVE
+
+This module is used to initialize and set up the configuration for the DeepCAVE framework.
+
+The Dash application gets created.
+"""
+
 from typing import Any, Callable, TypeVar, cast
 
 import datetime
@@ -25,7 +34,20 @@
 ROOT_DIR = Path(__file__).parent
 
 
-def get_app(title: str):
+def get_app(title: str) -> Any:
+    """
+    Get the Dash Proxy.
+
+    Parameters
+    ----------
+    title : str
+        The title of the application.
+
+    Returns
+    -------
+    DashProxy
+        The dash proxy.
+    """
     import dash_bootstrap_components as dbc
     from dash_extensions.enrich import (
         DashProxy,
@@ -61,8 +83,8 @@ def get_app(title: str):
 if any(file in _exec_file for file in _exec_files):
     from deepcave.custom_queue import Queue
     from deepcave.runs.handler import RunHandler
-    from deepcave.runs.objective import Objective  # noqa
-    from deepcave.runs.recorder import Recorder  # noqa
+    from deepcave.runs.objective import Objective
+    from deepcave.runs.recorder import Recorder
     from deepcave.utils.cache import Cache
     from deepcave.utils.configs import parse_config
     from deepcave.utils.notification import Notification
@@ -129,8 +151,37 @@ def get_app(title: str):
 
 
 def interactive(func: F) -> F:
+    """
+    Define the interactive decorator.
+
+    Parameters
+    ----------
+    func : F
+        The function to be decorated.
+
+    Returns
+    -------
+    F
+        The decorated function.
+    """
+
     @wraps(func)
     def inner(*args: Any, **kwargs: Any) -> Any:
+        """
+        Inner function of the decorator.
+
+        Parameters
+        ----------
+        *args : Any
+            Arguments to be passed to the wrap function.
+        **kwargs : Any
+            Keyword arguments to be passed to the wrap function.
+
+        Returns
+        -------
+        Any
+            The result of the function.
+        """
         if _api_mode:
             return
 

deepcave/cli.py

@@ -1,3 +1,14 @@
+#  noqa: D400
+"""
+# CLI
+
+This module defines command-line options using flags.
+
+This includes the entry point for the programs execution.
+"""
+
+from typing import Any
+
 import multiprocessing
 import subprocess
 from pathlib import Path
@@ -21,7 +32,8 @@
 )
 
 
-def execute(_) -> None:
+def execute(_: Any) -> None:
+    """Entry point for the programs execution."""
     if (config_key := FLAGS.get_config_value) is not None:
         config = FLAGS.config
         if config is not None:
@@ -47,6 +59,7 @@ def execute(_) -> None:
 
 
 def main() -> None:
+    """Call the execute function."""
     try:
         app.run(execute)
     except KeyboardInterrupt:

deepcave/config.py

@@ -1,9 +1,50 @@
+#  noqa: D400
+"""
+# Config
+
+This module defines the config object and its constants.
+Also defines multiple constants for directories, the server name, available plugins and converters.
+
+## Classes
+    - Config: Describe the config object.
+"""
+
 from typing import Any, Dict, List, Type
 
 from pathlib import Path
 
+from deepcave.runs.run import Run
+
 
 class Config:
+    """
+    Describe the config object.
+
+    Also define the constants of the config object.
+    Includes multiple constants for directories, the server name, available plugins and converters.
+
+    Constants
+    ---------
+    TITLE : str
+    DEBUG: bool
+    REFRESH_RATE: int
+    SAVE_IMAGES: bool
+    FIGURE_MARGIN: Dict
+    FIGURE_HEIGHT: str
+    REDIS_PORT: int
+    REDIS_ADDRESS: str
+    DASH_PORT: int
+    DASH_ADDRESS: str
+    META_DEFAULT: Dict
+
+    Properties
+    ----------
+    DASH_ADRESS : str
+        The address of the server name.
+    DASH_PORT : int
+        The port of the server name.
+    """
+
     # General config
     TITLE: str = "DeepCAVE"
     DEBUG: bool = False
@@ -34,18 +75,22 @@ class Config:
 
     @property
     def DEFAULT_WORKING_DIRECTORY(self) -> Path:
+        """Specifies the default working directory."""
         return Path.cwd() / "logs"
 
     @property
     def CACHE_DIR(self) -> Path:
+        """Specifies the default cache directory."""
         return Path(__file__).parent / "cache"
 
     @property
     def SERVER_NAME(self) -> str:
+        """Specifies the server name, consisting of address and port."""
         return f"http://{self.DASH_ADDRESS}:{self.DASH_PORT}"
 
     @property
-    def PLUGINS(self) -> Dict[str, List["Plugin"]]:
+    def PLUGINS(self) -> Dict[str, List[Any]]:
+        """A list of available plugins per category."""
         from deepcave.plugins.budget.budget_correlation import BudgetCorrelation
         from deepcave.plugins.hyperparameter.importances import Importances
         from deepcave.plugins.hyperparameter.pdp import PartialDependencies
@@ -60,6 +105,7 @@ def PLUGINS(self) -> Dict[str, List["Plugin"]]:
         from deepcave.plugins.summary.footprint import FootPrint
         from deepcave.plugins.summary.overview import Overview
 
+        plugins: Dict[str, List[Any]] = {}
         plugins = {
             "Summary": [
                 Overview(),
@@ -85,6 +131,7 @@ def PLUGINS(self) -> Dict[str, List["Plugin"]]:
 
     @property
     def CONVERTERS(self) -> List[Type["Run"]]:
+        """Get a list of available run converters."""
         from deepcave.runs.converters.bohb import BOHBRun
         from deepcave.runs.converters.deepcave import DeepCAVERun
         from deepcave.runs.converters.smac3v1 import SMAC3v1Run

deepcave/constants.py

@@ -1,3 +1,20 @@
+#  noqa: D400
+"""
+# Constants
+
+This module defines the constants for the DeepCAVE framework.
+
+## Constants
+    NAN_VALUE: float
+    NAN_LABEL: str
+    VALUE_RANGE: List
+    CONSTANT_VALUE: float
+    BORDER_CONFIG_ID: int
+    RANDOM_CONFIG_ID: innt
+    COMBINED_COST_NAME: str
+    COMBINED_BUDGET: int
+"""
+
 NAN_VALUE = -0.2
 NAN_LABEL = "NaN"
 VALUE_RANGE = [NAN_VALUE, 1]