chore: add type validation to core commands (#3221)

Author: Panaetius

.github/workflows/test_deploy.yml

@@ -168,7 +168,7 @@ jobs:
       - name: Test mypy
         env:
           POETRY_VIRTUALENVS_CREATE: false
-        run: mypy
+        run: mypy renku
 
   test-distro:
     runs-on: ubuntu-latest

.pre-commit-config.yaml

@@ -37,10 +37,9 @@ repos:
       hooks:
           - id: shellcheck
     - repo: https://github.com/pre-commit/mirrors-mypy
-      rev: "v0.961"
+      rev: "v0.990"
       hooks:
           - id: mypy
-            language_version: '3.8'
             args:
             - --no-strict-optional
             - --ignore-missing-imports

poetry.lock

@@ -98,6 +98,7 @@ pluggy = "==1.0.0"
 poetry-dynamic-versioning = { version = ">=0.14,<0.18", optional = true }
 portalocker = ">=2.2.1,<2.5"
 psutil = ">=5.4.7,<5.9.2"
+pydantic = "==1.10.2"
 pydocstyle = { version = "<6.1.2,>=4.0.1", optional = true }
 pyjwt = ">=2.1.0,<2.5.0"
 pyld = "==2.0.3"
@@ -123,7 +124,7 @@ rdflib = "<7.0,>=6.0.0"
 redis = { version = ">=3.5.3,<4.2.0", optional = true }
 renku-sphinx-theme = { version = ">=0.2.0", optional = true }
 requests = ">=2.23.0,<2.28.2"
-responses = { version = ">=0.7.0,<=0.22.0", optional = true }
+responses = { version = ">=0.22.0,<0.23.0", optional = true }
 rich = ">=9.3.0,<12.6.0"
 rq = { version = "==1.11.0", optional = true }
 rq-scheduler = { version = "==0.11.0", optional = true }
@@ -227,6 +228,7 @@ all = [
     "plantweb",
     "poetry-dynamic-versioning",
     "ptvsd",
+    "pydantic",
     "pydocstyle",
     "pyte",
     "pytest",
@@ -344,6 +346,7 @@ files = [
     "tests/**/*.py"
 ]
 implicit_optional = true
+check_untyped_defs = true
 
 [[tool.mypy.overrides]]
 module = [
@@ -397,6 +400,10 @@ ignore_missing_imports = true
 module = "renku.core.migration.*"
 ignore_errors = true
 
+[[tool.mypy.overrides]]
+module = "renku.command.schema.calamus.*"
+ignore_errors = true
+
 [tool.coverage.run]
 omit = ["renku/conftest.py", "renku/data/*", "docs/*", "tests/*"]
 relative_files = true

pyproject.toml

@@ -20,6 +20,7 @@
 from __future__ import absolute_import, print_function
 
 import importlib
+import importlib.util
 import sys
 import warnings
 
@@ -55,9 +56,9 @@ def exec_module(self, module):
         sys.modules[self.additional_namespace] = module
 
     def __getattr__(self, name):
-        """Forward all calls to wrapped loaded except for one implemented here."""
+        """Forward all calls to wrapped loader except for one implemented here."""
         if name in ["exec_module", "create_module", "get_code"]:
-            object.__getattr__(self, name)
+            object.__getattribute__(self, name)
 
         return getattr(self.wrapped_loader, name)
 
@@ -88,6 +89,9 @@ def find_spec(self, fullname, path, target=None):
 
                 sys.meta_path = [x for x in sys.meta_path if x is not self]
                 spec = importlib.util.find_spec(target_name)
+                if spec is None:
+                    return None
+
                 spec.loader = LoaderWrapper(spec.loader, fullname, target_name)
             finally:
                 sys.meta_path.insert(0, self)

renku/__init__.py

@@ -25,7 +25,7 @@
 from renku.domain_model.project_context import project_context
 
 try:
-    import importlib_resources
+    import importlib_resources  # type: ignore[import]
 except ImportError:
     import importlib.resources as importlib_resources  # type: ignore
 

renku/command/checks/githooks.py

@@ -17,39 +17,46 @@
 # limitations under the License.
 """Clone a Renku repo along with all Renku-specific initializations."""
 
+from pathlib import Path
+from typing import Any, Dict, Optional, Union
+
+from git.remote import RemoteProgress
+from pydantic import validate_arguments
+
 from renku.command.command_builder.command import Command
 from renku.domain_model.project_context import project_context
 
 
+@validate_arguments(config=dict(arbitrary_types_allowed=True))
 def _project_clone(
-    url,
-    path=None,
-    install_githooks=True,
-    install_mergetool=True,
-    skip_smudge=True,
-    recursive=True,
-    depth=None,
-    progress=None,
-    config=None,
-    raise_git_except=False,
-    checkout_revision=None,
-    use_renku_credentials=False,
+    url: str,
+    path: Optional[Union[str, Path]] = None,
+    install_githooks: bool = True,
+    install_mergetool: bool = True,
+    skip_smudge: bool = True,
+    recursive: bool = True,
+    depth: Optional[int] = None,
+    progress: Optional[RemoteProgress] = None,
+    config: Optional[Dict[str, Any]] = None,
+    raise_git_except: bool = False,
+    checkout_revision: Optional[str] = None,
+    use_renku_credentials: bool = False,
 ):
     """Clone Renku project repo, install Git hooks and LFS.
 
     Args:
-        url: Git URL to clone.
-        path: Path to clone to (Default value = None).
-        install_githooks: Whether to install the pre-commit hook or not (Default value = True).
-        install_mergetool: Whether to install the renku metadata git mergetool or not (Default value = True).
-        skip_smudge: Whether to skip pulling files from LFS (Default value = True).
-        recursive: Recursively clone (Default value = True).
-        depth: Clone depth (commits from HEAD) (Default value = None).
-        progress: Git progress object (Default value = None).
-        config: Initial config (Default value = None).
-        raise_git_except: Whether to raise Git exceptions or not (Default value = False).
-        checkout_revision: Specific revision to check out (Default value = None).
-        use_renku_credentials: Whether to use credentials stored in renku (Default value = False).
+        url(str): Git URL to clone.
+        path(Optional[str]): Path to clone to (Default value = None).
+        install_githooks(bool): Whether to install the pre-commit hook or not (Default value = True).
+        install_mergetool(bool): Whether to install the renku metadata git mergetool or not (Default value = True).
+        skip_smudge(bool): Whether to skip pulling files from LFS (Default value = True).
+        recursive(bool): Recursively clone (Default value = True).
+        depth(Optional[int]): Clone depth (commits from HEAD) (Default value = None).
+        progress(Optional[RemoteProgress]): Git progress object (Default value = None).
+        config(Optional[Dict[str, Any]]): Initial config (Default value = None).
+        raise_git_except(bool): Whether to raise Git exceptions or not (Default value = False).
+        checkout_revision(Optional[str]): Specific revision to check out (Default value = None).
+        use_renku_credentials(bool): Whether to use credentials stored in renku (Default value = False).
 
     Returns:
         Tuple of cloned ``Repository`` and whether it's a Renku project or not.

renku/command/clone.py

@@ -357,6 +357,7 @@ def command(self, operation: Callable):
         Returns:
             Command: This command.
         """
+
         self._operation = operation
 
         return self

renku/command/command_builder/command.py

@@ -17,6 +17,10 @@
 # limitations under the License.
 """Get and set Renku repository or global options."""
 
+from typing import Dict, Optional
+
+from pydantic import validate_arguments
+
 from renku.command.command_builder.command import Command
 from renku.core import errors
 from renku.core.config import get_config, get_value, remove_value, set_value
@@ -39,13 +43,16 @@ def _split_section_and_key(key):
     return "renku", key
 
 
-def _update_multiple_config(values, global_only=False, commit_message=None):
+@validate_arguments(config=dict(arbitrary_types_allowed=True))
+def _update_multiple_config(
+    values: Dict[str, Optional[str]], global_only: bool = False, commit_message: Optional[str] = None
+):
     """Add, update, or remove multiple configuration values.
 
     Args:
-        values: Dictionary of config key values to update.
-        global_only: Whether to only update global config (Default value = False).
-        commit_message: Commit message for change (Default value = None).
+        values(Dict[str, str]): Dictionary of config key values to update.
+        global_only(bool): Whether to only update global config (Default value = False).
+        commit_message(Optional[str]): Commit message for change (Default value = None).
     """
     for k, v in values.items():
         if v is not None:
@@ -65,15 +72,23 @@ def update_multiple_config():
     )
 
 
-def _update_config(key, *, value=None, remove=False, global_only=False, commit_message=None):
+@validate_arguments(config=dict(arbitrary_types_allowed=True))
+def _update_config(
+    key: str,
+    *,
+    value: Optional[str] = None,
+    remove: bool = False,
+    global_only: bool = False,
+    commit_message: Optional[str] = None,
+):
     """Add, update, or remove configuration values.
 
     Args:
-        key: Config key.
-        value: Config value (Default value = None).
-        remove: Whether to remove values (Default value = False).
-        global_only: Whether to only update global config (Default value = False).
-        commit_message: Commit message for change (Default value = None).
+        key(str): Config key.
+        value(Optional[str]): Config value (Default value = None).
+        remove(bool): Whether to remove values (Default value = False).
+        global_only(bool): Whether to only update global config (Default value = False).
+        commit_message(Optional[str]): Commit message for change (Default value = None).
     Raises:
         errors.ParameterError: If key wasn't found.
     Returns:
@@ -100,7 +115,8 @@ def update_config():
     )
 
 
-def _read_config(key, config_filter=ConfigFilter.ALL, as_string=True):
+@validate_arguments(config=dict(arbitrary_types_allowed=True))
+def _read_config(key: Optional[str], config_filter: ConfigFilter = ConfigFilter.ALL, as_string: bool = True):
     """Read configuration.
 
     Args: