feat(cli): add renku template validate command (#2936)

Author: Panaetius

.pre-commit-config.yaml

@@ -1,36 +1,54 @@
+default_language_version:
+    python: python3.8
 repos:
--   repo: https://github.com/pre-commit/pre-commit-hooks
-    rev: v2.3.0
-    hooks:
-    -   id: check-yaml
-    -   id: end-of-file-fixer
-    -   id: trailing-whitespace
--   repo: https://github.com/psf/black
-    rev: 22.3.0
-    hooks:
-    -   id: black
-        additional_dependencies: ['click==8.0.4']
--   repo: https://github.com/pycqa/isort
-    rev: 5.10.1
-    hooks:
-    -   id: isort
-        name: isort (python)
--   repo: https://github.com/pycqa/flake8
-    rev: '3.9.2'
-    hooks:
-    -   id: flake8
-        exclude: ^docs/
-        args:
-        - "--max-line-length=120"
-        - "--show-source"
-        - "--ignore=E121,E126,E203,E226,E231,W503,W504"
--   repo: https://github.com/pycqa/pydocstyle
-    rev: 4.0.1
-    hooks:
-    -   id: pydocstyle
-        args:
-        - --ignore=D105,D107,D202,D203,D212,D213,D401,D406,D407,D410,D411,D413
--   repo: https://github.com/koalaman/shellcheck-precommit
-    rev: v0.8.0
-    hooks:
-    -   id: shellcheck
+    - repo: https://github.com/pre-commit/pre-commit-hooks
+      rev: v2.3.0
+      hooks:
+          - id: check-yaml
+          - id: end-of-file-fixer
+          - id: trailing-whitespace
+    - repo: https://github.com/psf/black
+      rev: 22.3.0
+      hooks:
+          - id: black
+            additional_dependencies: ["click==8.0.4"]
+    - repo: https://github.com/pycqa/isort
+      rev: 5.10.1
+      hooks:
+          - id: isort
+            name: isort (python)
+    - repo: https://github.com/pycqa/flake8
+      rev: "3.9.2"
+      hooks:
+          - id: flake8
+            exclude: ^docs/
+            args:
+                - "--max-line-length=120"
+                - "--show-source"
+                - "--ignore=E121,E126,E203,E226,E231,W503,W504"
+    - repo: https://github.com/pycqa/pydocstyle
+      rev: 4.0.1
+      hooks:
+          - id: pydocstyle
+            args:
+                - --ignore=D105,D107,D202,D203,D212,D213,D401,D406,D407,D410,D411,D413
+    - repo: https://github.com/koalaman/shellcheck-precommit
+      rev: v0.8.0
+      hooks:
+          - id: shellcheck
+    - repo: https://github.com/pre-commit/mirrors-mypy
+      rev: "v0.961"
+      hooks:
+          - id: mypy
+            language_version: '3.8'
+            args:
+            - --no-strict-optional
+            - --ignore-missing-imports
+            additional_dependencies:
+            - types-python-dateutil>=2.8.10
+            - types-PyYAML<6.1.0,>=5.4
+            - types-redis>=3.5.3,<4.1.0
+            - types-requests<2.27.2,>=2.23.0
+            - types-tabulate<0.8.10,>=0.7.7
+            - attrs<21.5.0,>=21.4.0
+            - filelock>=3.3.0,<3.6.1

poetry.lock

@@ -71,7 +71,7 @@ deepmerge = "==1.0.1"
 docker = "<6,>=3.7.2"
 environ-config = ">=18.2.0,<22.2.0"
 fakeredis = { version = ">=1.4.1,<1.7.2", optional = true }
-filelock = ">=3.0.0,<3.6.1"
+filelock = ">=3.3.0,<3.6.1"
 flake8 = { version = "<4.0,>=3.8", optional = true }                             #wait for https://github.com/flakehell/flakehell/pull/23 to be merged before bumping
 flakehell = { version = ">=0.9.0,<1.0.*", optional = true }
 flaky = { version = "==3.7.0", optional = true }
@@ -132,7 +132,7 @@ responses = { version = ">=0.7.0,<0.21.0", optional = true }
 rich = ">=9.3.0,<12.3.0"
 rq = { version = "==1.10.1", optional = true }
 rq-scheduler = { version = "==0.11.0", optional = true }
-sentry-sdk = { version = ">=1.0.0,<1.5.12,!=1.5.10", extras = ["flask"], optional = true }
+sentry-sdk = { version = ">=1.5.11,<1.5.12", extras = ["flask"], optional = true }
 shellingham = "1.4.0"
 sphinxcontrib-spelling = { version = "7.*", optional = true }
 sphinx-rtd-theme = { version = "<1.1,>=0.5.0", optional = true }

pyproject.toml

@@ -18,7 +18,7 @@
 """Template management commands."""
 
 from renku.command.command_builder.command import Command
-from renku.core.template.usecase import list_templates, set_template, show_template, update_template
+from renku.core.template.usecase import list_templates, set_template, show_template, update_template, validate_templates
 
 
 def list_templates_command():
@@ -55,3 +55,8 @@ def update_template_command():
         .with_database(write=True)
         .with_commit()
     )
+
+
+def validate_templates_command():
+    """Command to validate a template repository."""
+    return Command().command(validate_templates)

renku/command/template.py

@@ -150,16 +150,15 @@ def prepare_worktree(
     # TODO sys.argv
 
     if commit is NULL_TREE:
-        args = ["add", "--detach", path]
-        original_client.repository.run_git_command("worktree", *args)
+        original_client.repository.create_worktree(path, detach=True)
         client = attr.evolve(original_client, path=path)
         client.repository.run_git_command("checkout", "--orphan", branch_name)
         client.repository.remove("*", recursive=True, force=True)
     else:
-        args = ["add", "-b", branch_name, path]
+        revision = None
         if commit:
-            args.append(commit.hexsha)
-        original_client.repository.run_git_command("worktree", *args)
+            revision = commit.hexsha
+        original_client.repository.create_worktree(path, branch=branch_name, reference=revision)
         client = attr.evolve(original_client, path=path)
 
     client.repository.get_configuration = original_client.repository.get_configuration
@@ -207,7 +206,7 @@ def finalize_worktree(
         raise errors.FailedMerge(client.repository, branch_name, merge_args)
 
     if delete:
-        client.repository.run_git_command("worktree", "remove", path)
+        client.repository.remove_worktree(path)
 
         if new_branch:
             # delete the created temporary branch

renku/core/management/git.py

@@ -58,7 +58,7 @@ def _path_converter(path) -> Path:
 class PathMixin:
     """Define a default path attribute."""
 
-    path: Path = attr.ib(default=default_path, converter=_path_converter)
+    path: Path = attr.ib(default=default_path, converter=_path_converter)  # type: ignore
 
     @path.validator
     def _check_path(self, _, value):

renku/core/management/repository.py

@@ -407,8 +407,10 @@ class RepositoryTemplates(TemplatesSource):
     get available versions of templates.
     """
 
-    def __init__(self, path, source, reference, version, repository: Repository):
-        super().__init__(path=path, source=source, reference=reference, version=version)
+    def __init__(self, path, source, reference, version, repository: Repository, skip_validation: bool = False):
+        super().__init__(
+            path=path, source=source, reference=reference, version=version, skip_validation=skip_validation
+        )
         self.repository: Repository = repository
 
     @classmethod

renku/core/template/template.py

@@ -17,7 +17,10 @@
 # limitations under the License.
 """Template use cases."""
 
-from typing import Dict, List, NamedTuple, Optional, Tuple
+import os
+import tempfile
+from pathlib import Path
+from typing import Any, Dict, List, NamedTuple, Optional, Tuple, Union
 
 import click
 
@@ -29,6 +32,7 @@
 from renku.core.management.migrate import is_renku_project
 from renku.core.template.template import (
     FileAction,
+    RepositoryTemplates,
     TemplateAction,
     copy_template_to_client,
     fetch_templates_source,
@@ -38,6 +42,7 @@
 from renku.core.util import communication
 from renku.domain_model.tabulate import tabulate
 from renku.domain_model.template import RenderedTemplate, Template, TemplateMetadata, TemplatesSource
+from renku.infrastructure.repository import Repository
 
 
 def list_templates(source, reference) -> List[TemplateViewModel]:
@@ -242,3 +247,53 @@ def prompt_to_select_template():
         return templates_source.templates[0]
 
     return prompt_to_select_template()
+
+
+def validate_templates(
+    source: Optional[str] = None, reference: Optional[str] = None
+) -> Dict[str, Union[str, Dict[str, List[str]]]]:
+    """Validate a template repository.
+
+    Args:
+        source(str, optional): Remote repository URL to clone and check (Default value = None).
+        reference(str, optional): Git commit/branch/tag to check (Default value = None).
+    Returns:
+        Dict[str, Union[str, Dict[str, List[str]]]]: Dictionary containing errors and warnings for manifest and
+            templates, along with a ``valid`` field telling if all checks passed.
+    """
+
+    if source is not None:
+        path = Path(tempfile.mkdtemp())
+        repo = Repository.clone_from(path=path, url=source)
+        repo.checkout(reference=reference)
+    else:
+        path = Path(os.getcwd())
+        repo = Repository(path=path)
+
+        if reference is not None:
+            path = Path(tempfile.mkdtemp())
+            repo.create_worktree(path, reference=reference)
+            repo = Repository(path=path)
+
+    version = repo.head.commit.hexsha
+
+    result: Dict[str, Any] = {"manifest": None, "templates": {}, "warnings": [], "valid": True}
+
+    try:
+        template_source = RepositoryTemplates(
+            path=path, source=path, reference="", version=version, repository=repo, skip_validation=True
+        )
+        result["warnings"] = template_source.manifest.validate(manifest_only=True)
+    except errors.InvalidTemplateError as e:
+        result["manifest"] = e.args[0] if e.args else str(e)
+        result["valid"] = False
+        return result
+
+    for template in template_source.manifest.templates:
+        template.templates_source = template_source
+        issues = template.validate(skip_files=False, raise_errors=False)
+        if issues:
+            result["templates"][template.id] = issues
+            result["valid"] = False
+
+    return result