chore: update precommit (#3187)

Author: Panaetius

.pre-commit-config.yaml

@@ -1,53 +1,85 @@
 repos:
-    - repo: https://github.com/pre-commit/pre-commit-hooks
-      rev: v2.3.0
-      hooks:
-          - id: check-yaml
-          - id: end-of-file-fixer
-          - id: trailing-whitespace
-    - repo: https://github.com/psf/black
-      rev: 22.3.0
-      hooks:
-          - id: black
-            additional_dependencies: ["click==8.0.4"]
-    - repo: https://github.com/pycqa/isort
-      rev: 5.10.1
-      hooks:
-          - id: isort
-            name: isort (python)
-    - repo: https://github.com/pycqa/flake8
-      rev: "6.0.0"
-      hooks:
-          - id: flake8
-            exclude: ^docs/
-            additional_dependencies:
-            - Flake8-pyproject==1.2.2
-    - repo: https://github.com/pycqa/pydocstyle
-      rev: 4.0.1
-      hooks:
-          - id: pydocstyle
-            args:
-                - --ignore=D105,D107,D202,D203,D212,D213,D401,D406,D407,D410,D411,D413
-    - repo: https://github.com/koalaman/shellcheck-precommit
-      rev: v0.8.0
-      hooks:
-          - id: shellcheck
-    - repo: https://github.com/pre-commit/mirrors-mypy
-      rev: "v0.990"
-      hooks:
-          - id: mypy
-            args:
-            - --no-strict-optional
-            - --ignore-missing-imports
-            additional_dependencies:
-            - types-python-dateutil>=2.8.10
-            - types-PyYAML<6.1.0,>=5.4
-            - types-redis>=3.5.3,<4.1.0
-            - types-requests<2.27.2,>=2.23.0
-            - types-tabulate<0.8.10,>=0.7.7
-            - attrs<21.5.0,>=21.4.0
-            - filelock>=3.3.0,<3.6.1
-    - repo: https://github.com/hadolint/hadolint
-      rev: master
-      hooks:
-          - id: hadolint-docker
+  - repo: https://github.com/pre-commit/pre-commit-hooks
+    rev: v4.3.0
+    hooks:
+      - id: check-yaml
+        exclude: ^helm-chart/renku-core/templates
+      - id: check-added-large-files
+      - id: check-case-conflict
+      - id: check-json
+      - id: check-merge-conflict
+      - id: check-toml
+      - id: debug-statements
+      - id: end-of-file-fixer
+        types: [python]
+      - id: mixed-line-ending
+      - id: trailing-whitespace
+  - repo: https://github.com/psf/black
+    rev: 22.10.0
+    hooks:
+      - id: black
+        additional_dependencies: ["click==8.0.4"]
+  - repo: https://github.com/pycqa/isort
+    rev: 5.10.1
+    hooks:
+      - id: isort
+        name: isort (python)
+  - repo: https://github.com/pycqa/flake8
+    rev: "5.0.4"
+    hooks:
+      - id: flake8
+        exclude: ^docs/
+        args:
+          - "--max-line-length=120"
+          - "--show-source"
+          - "--ignore=E121,E126,E203,E226,E231,W503,W504"
+        additional_dependencies:
+          - Flake8-pyproject==1.2.2
+  - repo: https://github.com/pycqa/pydocstyle
+    rev: 6.1.1
+    hooks:
+      - id: pydocstyle
+        args:
+          - --ignore=D105,D107,D202,D203,D212,D213,D401,D406,D407,D410,D411,D413
+  - repo: https://github.com/koalaman/shellcheck-precommit
+    rev: v0.8.0
+    hooks:
+      - id: shellcheck
+  - repo: https://github.com/pre-commit/mirrors-mypy
+    rev: "v0.990"
+    hooks:
+      - id: mypy
+        args:
+          - --no-strict-optional
+          - --ignore-missing-imports
+        additional_dependencies:
+          - types-python-dateutil>=2.8.10
+          - types-PyYAML<6.1.0,>=5.4
+          - types-redis>=3.5.3,<4.1.0
+          - types-requests<2.27.2,>=2.23.0
+          - types-tabulate<0.8.10,>=0.7.7
+          - attrs<21.5.0,>=21.4.0
+          - filelock>=3.3.0,<3.6.1
+  - repo: https://github.com/hadolint/hadolint
+    rev: 'v2.10.0'
+    hooks:
+      - id: hadolint-docker
+  - repo: https://github.com/asottile/yesqa
+    rev: v1.4.0
+    hooks:
+      - id: yesqa
+  - repo: https://github.com/PyCQA/bandit
+    rev: '1.7.4'
+    hooks:
+    - id: bandit
+      args: [--configfile, pyproject.toml]
+      types: [python]
+      additional_dependencies: ["bandit[toml]"]
+  - repo: local
+    hooks:
+    - id: poetry-check
+      name: poetry check
+      entry: poetry check
+      language: system
+      files: pyproject.toml
+      pass_filenames: false

poetry.lock

@@ -54,7 +54,6 @@ python = "^3.7.1"
 appdirs = "<=1.4.4,>=1.4.3"
 attrs = ">=21.1.0,<22.2.0"
 bashlex = ">=0.16,<0.17"
-black = { version = "==22.6.0", optional = true }
 calamus = ">=0.3.13,<0.5"
 click = ">=8.0,<8.1.4"
 click-option-group = "<0.6.0,>=0.5.2"
@@ -111,7 +110,7 @@ apispec-webframeworks = { version = "<0.6,>=0.5.2", optional = true }
 circus = { version = "==0.17.1", optional = true }
 flask = { version = "==2.1.1", optional = true }
 gunicorn = { version = "*", optional = true }
-marshmallow = { version = ">=3.18.0", optional = true }
+marshmallow = { version = ">=3.18.0,<3.20.0", optional = true }
 marshmallow-oneofschema = { version = ">=3.0.1,<4.0.0", optional = true }
 pillow = { version = ">=9.0.0,<9.4", optional = true }
 python-dotenv = { version = ">=0.19.0,<0.21.0", optional = true }
@@ -122,7 +121,7 @@ sentry-sdk = { version = ">=1.5.11,<1.5.12", extras = ["flask"],  optional = tru
 walrus = { version = ">=0.8.2,<0.10.0", optional = true }
 
 [tool.poetry.group.dev.dependencies]
-black = "==22.6.0"
+black = "==22.10.0"
 flake8 = [
     { version = ">=6.0.0,<7.0.0", python = ">=3.8.1"},
     { version = ">=4.0.0,<5.0.0", python = "<3.8.1"}
@@ -222,6 +221,8 @@ exclude = '''
 [tool.pydocstyle]
 add_ignore = ["D105", "D107", "D202", "D401"]
 
+[tool.bandit]
+skips = ["B101", "B603", "B404"]
 
 [tool.isort]
 multi_line_output = 3
@@ -275,6 +276,11 @@ files = [
     "renku/**/*.py",
     "tests/**/*.py"
 ]
+pretty = true
+show_column_numbers = true
+show_error_codes = true
+show_error_context = true
+warn_unreachable = true
 implicit_optional = true
 check_untyped_defs = true
 

pyproject.toml

@@ -21,7 +21,7 @@
 import re
 from datetime import datetime
 from itertools import islice
-from typing import Dict, List, Optional, Tuple, cast
+from typing import TYPE_CHECKING, Dict, Generator, List, Optional, Tuple, cast
 
 from renku.command.command_builder.command import Command
 from renku.core.util import communication
@@ -30,6 +30,9 @@
 from renku.domain_model.provenance.activity import Activity
 from renku.domain_model.workflow.plan import AbstractPlan
 
+if TYPE_CHECKING:
+    from renku.infrastructure.repository import Commit
+
 CHECKPOINTS_PER_PAGE = 50
 
 
@@ -40,7 +43,7 @@ def rollback_command():
 
 def _rollback_command():
     """Perform a rollback of the repo."""
-    commits = project_context.repository.iterate_commits(project_context.metadata_path)
+    commits: Generator["Commit", None, None] = project_context.repository.iterate_commits(project_context.metadata_path)
 
     checkpoint = _prompt_for_checkpoint(commits)
 
@@ -304,7 +307,7 @@ def _checkpoint_iterator(commits):
     """
     transaction_pattern = re.compile(r"\n\nrenku-transaction:\s([0-9a-g]+)$")
 
-    current_checkpoint = None
+    current_checkpoint: Optional[Tuple[str, "Commit", str]] = None
 
     for commit in commits:
         commit_message = commit.message

renku/command/rollback.py

@@ -250,7 +250,7 @@ def get_full_repository_path(url: Optional[str]) -> str:
     Returns:
         The hostname plus path extracted from the URL.
     """
-    if str is None:
+    if url is None:
         return ""
 
     parsed_url = parse_git_url(url)

renku/core/util/git.py

@@ -21,7 +21,6 @@
 import os
 import re
 import tempfile
-from collections.abc import Iterable
 from pathlib import Path
 from typing import TYPE_CHECKING, List, Optional, Tuple, Union
 
@@ -42,9 +41,6 @@ def construct_creators(creators: List[Union[dict, str]], ignore_email=False) ->
     """Parse input and return a list of Person."""
     creators = creators or []
 
-    if not isinstance(creators, Iterable) or isinstance(creators, str):
-        raise errors.ParameterError("Invalid creators type")
-
     people = []
     no_email_warnings = []
     for creator in creators:

renku/core/util/metadata.py

@@ -411,7 +411,7 @@ def iterate_workflow(
     tag_separator = "@"
     index_pattern = re.compile(r"{iter_index}")
 
-    iter_params: Optional[Dict[str, Any]] = {"indexed": {}, "params": {}, "tagged": {}}
+    iter_params: Dict[str, Any] = {"indexed": {}, "params": {}, "tagged": {}}
     workflow_params = {}
     if mapping_path:
         mapping = safe_read_yaml(mapping_path)
@@ -423,42 +423,42 @@ def iterate_workflow(
             iter_params["indexed"][param_name] = param_value  # type: ignore
         else:
             try:
-                param_value = ast.literal_eval(param_value)
+                evaluated_param_value = ast.literal_eval(param_value)
             except Exception:
                 raise errors.ParameterError(
                     f"The value of '{param_name}' parameter is neither a list nor templated variable!"
                 )
 
-            if isinstance(param_value, list) and len(param_value) == 1:
+            if isinstance(evaluated_param_value, list) and len(evaluated_param_value) == 1:
                 communication.warn(
                     f"The parameter '{param_name}' has only one element '{param_value}', "
                     "changing it to be a fixed parameter!"
                 )
                 workflow_params[param_name] = param_value[0]
                 continue
-            elif not isinstance(param_value, list):
+            elif not isinstance(evaluated_param_value, list):
                 workflow_params[param_name] = param_value
                 continue
 
             if tag_separator in param_name:
                 name, tag = param_name.split(tag_separator, maxsplit=1)
                 if tag in iter_params["tagged"]:
-                    iter_params["tagged"][tag][name] = param_value
+                    iter_params["tagged"][tag][name] = evaluated_param_value
                 else:
-                    iter_params["tagged"][tag] = {name: param_value}
+                    iter_params["tagged"][tag] = {name: evaluated_param_value}
 
                 param_name = name
             else:
-                iter_params["params"][param_name] = param_value
+                iter_params["params"][param_name] = evaluated_param_value
 
         set_param = reduce(lambda x, y: {y: x}, reversed(param_name.split(".")), param_value)  # type: ignore
         workflow_params = always_merger.merge(workflow_params, set_param)
 
-    iter_params = _validate_iterate_parameters(workflow, workflow_params, cast(Dict[str, Any], iter_params))
-    if iter_params is None:
+    validated_iter_params = _validate_iterate_parameters(workflow, workflow_params, cast(Dict[str, Any], iter_params))
+    if validated_iter_params is None:
         return
 
-    plans, execute_plan = _build_iterations(workflow, workflow_params, iter_params, index_pattern)
+    plans, execute_plan = _build_iterations(workflow, workflow_params, validated_iter_params, index_pattern)
 
     communication.echo(f"\n\n{tabulate(execute_plan, execute_plan[0].keys())}")
     if not dry_run:

renku/core/workflow/execute.py

@@ -210,7 +210,7 @@ def get_elements(data: Union[Dict[str, Any], List[Dict[str, Any]]]) -> Generator
     """Get elements of a list of dict as a dict."""
     if isinstance(data, list):
         for element in data:
-            if isinstance(element, list):
+            if isinstance(element, list):  # type: ignore[unreachable]
                 raise errors.ParseError(f"Data cannot be a list: {element}")
             yield element
     elif not isinstance(data, dict):

renku/core/workflow/parser/renku.py

@@ -19,7 +19,7 @@
 
 import itertools
 import os
-from datetime import timedelta
+from datetime import datetime, timedelta
 from pathlib import Path
 from typing import Any, Dict, Generator, List, Optional, Set, Tuple, Union, cast, overload
 
@@ -79,12 +79,12 @@ def get_plan(
 
 
 @overload
-def get_latest_plan(plan: None = ..., plan_gateway: IPlanGateway = ...) -> None:  # noqa: D103
+def get_latest_plan(plan: None = ..., plan_gateway: IPlanGateway = ...) -> None:
     ...
 
 
 @overload
-def get_latest_plan(plan: AbstractPlan, plan_gateway: IPlanGateway = ...) -> AbstractPlan:  # noqa: D103
+def get_latest_plan(plan: AbstractPlan, plan_gateway: IPlanGateway = ...) -> AbstractPlan:
     ...
 
 
@@ -197,12 +197,12 @@ def show_workflow(name_or_id_or_path: str, activity_gateway: IActivityGateway, w
             if isinstance(workflow, Plan):
 
                 num_executions = 0
-                last_execution = None
+                last_execution: Optional[datetime] = None
 
                 for activity in relevant_activities:
                     num_executions += 1
 
-                    if not last_execution or last_execution < activity.ended_at_time:
+                    if last_execution is None or last_execution < activity.ended_at_time:
                         last_execution = activity.ended_at_time
 
                 workflow = cast(Plan, DynamicProxy(workflow))
@@ -810,7 +810,7 @@ def get_plans_with_metadata(activity_gateway: IActivityGateway, plan_gateway: IP
         latest_plan.touches_existing_files = False
         latest_plan.number_of_executions = 0
         latest_plan.created = latest_plan.date_created
-        latest_plan.last_executed = None
+        latest_plan.last_executed = cast(Optional[datetime], None)
         latest_plan.children = []
         duration = _get_plan_duration(latest_plan, duration_cache, activity_map)
 
@@ -819,7 +819,7 @@ def get_plans_with_metadata(activity_gateway: IActivityGateway, plan_gateway: IP
 
         if isinstance(plan_chain[0], Plan):
             for activity in activity_map.get(latest_plan.id, []):
-                if not latest_plan.last_executed or latest_plan.last_executed < activity.ended_at_time:
+                if latest_plan.last_executed is None or latest_plan.last_executed < activity.ended_at_time:
                     latest_plan.last_executed = activity.ended_at_time
 
                 latest_plan.number_of_executions += 1