Merge branch 'main' into fix/multiversion-build

Nicoretti · Nicoretti · commit 83a15cbe2a5a · 2024-11-15T15:22:29.000+01:00
diff --git a/.gitattributes b/.gitattributes
@@ -0,0 +1 @@
+poetry.lock linguist-generated=true
diff --git a/.github/workflows/cd.yml b/.github/workflows/cd.yml
@@ -16,3 +16,9 @@ jobs:
     uses: ./.github/workflows/build-and-publish.yml
     secrets:
       PYPI_TOKEN: ${{ secrets.PYPI_TOKEN }}
+
+  publish-docs:
+    needs: [ cd-job ]
+    name: Publish Documentation
+    uses: ./.github/workflows/gh-pages.yml
+
diff --git a/.github/workflows/gh-pages.yml b/.github/workflows/gh-pages.yml
@@ -1,6 +1,8 @@
 name: Publish Documentation
 
-on: workflow_call
+on: 
+  workflow_call:
+  workflow_dispatch:
 
 jobs:
 
diff --git a/.github/workflows/report.yml b/.github/workflows/report.yml
@@ -51,3 +51,4 @@ jobs:
           poetry run coverage report -- --format markdown >> $GITHUB_STEP_SUMMARY
           echo  -e "\n\n# Static Code Analysis\n" >> $GITHUB_STEP_SUMMARY
           cat .lint.txt >> $GITHUB_STEP_SUMMARY
+          poetry run tbx security pretty-print .security.json >> $GITHUB_STEP_SUMMARY
diff --git a/.import_linter_config b/.import_linter_config
@@ -0,0 +1,2 @@
+[importlinter]
+root_package = exasol.toolbox
diff --git a/doc/changes/unreleased.md b/doc/changes/unreleased.md
@@ -1 +1,11 @@
 # Unreleased
+
+## 🐞 Fixed
+
+* Fixed the issue with publishing new documentation after releasing a new version
+## ✨ Added
+
+* #149: Added nox task to lint imports
+* Added support to manually trigger documentation build
+* #248: Added security results to workflow summary
+* #233: Added nox task to verify dependency declarations
diff --git a/doc/developer_guide/modules/modules.rst b/doc/developer_guide/modules/modules.rst
@@ -6,5 +6,6 @@ Modules
 
     sphinx/sphinx
     nox
+    nox_tasks
     pre_commit_hooks
 
diff --git a/doc/developer_guide/modules/nox.rst b/doc/developer_guide/modules/nox.rst
@@ -1,5 +1,6 @@
 nox
 ===
+
 The nox package contains nox related functionalities like pre defined nox tasks.
 
 .. figure:: ../../_static/nothing-to-see-here.png
diff --git a/doc/developer_guide/modules/nox_tasks.rst b/doc/developer_guide/modules/nox_tasks.rst
@@ -0,0 +1,15 @@
+nox_tasks
+=========
+
+lint:import (experimental)
+__________________________
+
+`Import Linter <https://import-linter.readthedocs.io/en/stable/readme.html>`_
+allows you to define and enforce rules for the imports within and between Python packages.
+
+.. important::
+
+    First configure the linter in file :code:`.import_linter_config`, see
+    `import-linter top-level-configuration <https://import-linter.readthedocs.io/en/stable/usage.html#top-level-configuration>`_
+    and `import-linter contract types <https://import-linter.readthedocs.io/en/stable/contract_types.html>`_
+
diff --git a/doc/developer_guide/modules/pre_commit_hooks.rst b/doc/developer_guide/modules/pre_commit_hooks.rst
@@ -1,5 +1,6 @@
 pre_commit_hooks
 =================
+
 In the pre_commit_hook package contains git commit hooks and similar functionalities.
 
 .. figure:: ../../_static/nothing-to-see-here.png
diff --git a/doc/user_guide/getting_started.rst b/doc/user_guide/getting_started.rst
@@ -193,6 +193,7 @@ You are ready to use the toolbox. With *nox -l* you can list all available tasks
     - lint:code -> Runs the static code analyzer on the project
     - lint:typing -> Runs the type checker on the project
     - lint:security -> Runs the security linter on the project
+    - lint:dependencies -> Checks if only valid sources of dependencies are used
     - docs:multiversion -> Builds the multiversion project documentation
     - docs:build -> Builds the project documentation
     - docs:open -> Opens the built project documentation
diff --git a/exasol/toolbox/nox/_lint.py b/exasol/toolbox/nox/_lint.py
@@ -1,8 +1,17 @@
 from __future__ import annotations
 
-from typing import Iterable
+import argparse
+import sys
+from pathlib import Path
+from typing import (
+    Dict,
+    Iterable,
+    List,
+)
 
 import nox
+import rich.console
+import tomlkit
 from nox import Session
 
 from exasol.toolbox.nox._shared import python_files
@@ -65,6 +74,62 @@ def _security_lint(session: Session, files: Iterable[str]) -> None:
     )
 
 
+def _import_lint(session: Session, path: Path) -> None:
+    session.run("poetry", "run", "lint-imports", "--config", path)
+
+
+class Dependencies:
+    def __init__(self, illegal: dict[str, list[str]] | None):
+        self._illegal = illegal or {}
+
+    @staticmethod
+    def parse(pyproject_toml: str) -> Dependencies:
+        def _source_filter(version) -> bool:
+            ILLEGAL_SPECIFIERS = ["url", "git", "path"]
+            return any(specifier in version for specifier in ILLEGAL_SPECIFIERS)
+
+        def find_illegal(part) -> list[str]:
+            return [
+                f"{name} = {version}"
+                for name, version in part.items()
+                if _source_filter(version)
+            ]
+
+        illegal: dict[str, list[str]] = {}
+        toml = tomlkit.loads(pyproject_toml)
+        poetry = toml.get("tool", {}).get("poetry", {})
+
+        part = poetry.get("dependencies", {})
+        if illegal_group := find_illegal(part):
+            illegal["tool.poetry.dependencies"] = illegal_group
+
+        part = poetry.get("dev", {}).get("dependencies", {})
+        if illegal_group := find_illegal(part):
+            illegal["tool.poetry.dev.dependencies"] = illegal_group
+
+        part = poetry.get("group", {})
+        for group, content in part.items():
+            illegal_group = find_illegal(content.get("dependencies", {}))
+            if illegal_group:
+                illegal[f"tool.poetry.group.{group}.dependencies"] = illegal_group
+        return Dependencies(illegal)
+
+    @property
+    def illegal(self) -> dict[str, list[str]]:
+        return self._illegal
+
+
+def report_illegal(illegal: dict[str, list[str]], console: rich.console.Console):
+    count = sum(len(deps) for deps in illegal.values())
+    suffix = "y" if count == 1 else "ies"
+    console.print(f"{count} illegal dependenc{suffix}\n", style="red")
+    for section, dependencies in illegal.items():
+        console.print(f"\\[{section}]", style="red")
+        for dependency in dependencies:
+            console.print(dependency, style="red")
+        console.print("")
+
+
 @nox.session(name="lint:code", python=False)
 def lint(session: Session) -> None:
     "Runs the static code analyzer on the project"
@@ -84,3 +149,45 @@ def security_lint(session: Session) -> None:
     """Runs the security linter on the project"""
     py_files = [f"{file}" for file in python_files(PROJECT_CONFIG.root)]
     _security_lint(session, list(filter(lambda file: "test" not in file, py_files)))
+
+
+@nox.session(name="lint:dependencies", python=False)
+def dependency_check(session: Session) -> None:
+    """Checks if only valid sources of dependencies are used"""
+    content = Path(PROJECT_CONFIG.root, "pyproject.toml").read_text()
+    dependencies = Dependencies.parse(content)
+    console = rich.console.Console()
+    if illegal := dependencies.illegal:
+        report_illegal(illegal, console)
+        sys.exit(1)
+
+
+@nox.session(name="lint:import", python=False)
+def import_lint(session: Session) -> None:
+    """(experimental) Runs import linter on the project"""
+    parser = argparse.ArgumentParser(
+        usage="nox -s import-lint -- [options]",
+        description="Runs the import linter on the project",
+    )
+    parser.add_argument(
+        "-c",
+        "--config",
+        type=str,
+        help="path to the configuration file for the importlinter",
+        metavar="TEXT",
+    )
+
+    args: argparse.Namespace = parser.parse_args(args=session.posargs)
+    file: str = args.config
+    path: Path | None = None
+    if file is None:
+        path = getattr(
+            PROJECT_CONFIG, "import_linter_config", Path(".import_linter_config")
+        )
+    else:
+        path = Path(file)
+    if not path.exists():
+        session.error(
+            "Please make sure you have a configuration file for the importlinter"
+        )
+    _import_lint(session=session, path=path)
diff --git a/exasol/toolbox/nox/tasks.py b/exasol/toolbox/nox/tasks.py
@@ -68,5 +68,6 @@ def check(session: Session) -> None:
     python_files,
 )
 
+
 # isort: on
 # fmt: on
diff --git a/exasol/toolbox/templates/github/workflows/cd.yml b/exasol/toolbox/templates/github/workflows/cd.yml
@@ -16,3 +16,9 @@ jobs:
     uses: ./.github/workflows/build-and-publish.yml
     secrets:
       PYPI_TOKEN: ${{ secrets.PYPI_TOKEN }}
+
+  publish-docs:
+    needs: [ cd-job ]
+    name: Publish Documentation
+    uses: ./.github/workflows/gh-pages.yml
+
diff --git a/exasol/toolbox/templates/github/workflows/gh-pages.yml b/exasol/toolbox/templates/github/workflows/gh-pages.yml
@@ -1,6 +1,8 @@
 name: Publish Documentation
 
-on: workflow_call
+on: 
+  workflow_call:
+  workflow_dispatch:
 
 jobs:
 
diff --git a/exasol/toolbox/templates/github/workflows/report.yml b/exasol/toolbox/templates/github/workflows/report.yml
@@ -51,3 +51,4 @@ jobs:
           poetry run coverage report -- --format markdown >> $GITHUB_STEP_SUMMARY
           echo  -e "\n\n# Static Code Analysis\n" >> $GITHUB_STEP_SUMMARY
           cat .lint.txt >> $GITHUB_STEP_SUMMARY
+          poetry run tbx security pretty-print .security.json >> $GITHUB_STEP_SUMMARY
diff --git a/exasol/toolbox/tools/security.py b/exasol/toolbox/tools/security.py
@@ -11,6 +11,7 @@
 from enum import Enum
 from functools import partial
 from inspect import cleandoc
+from pathlib import Path
 from typing import (
     Generator,
     Iterable,
@@ -101,6 +102,63 @@ def from_maven(report: str) -> Iterable[Issue]:
             )
 
 
+@dataclass(frozen=True)
+class SecurityIssue:
+    file_name: str
+    line: int
+    column: int
+    cwe: str
+    test_id: str
+    description: str
+    references: tuple
+
+
+def from_json(report_str: str, prefix: Path) -> Iterable[SecurityIssue]:
+    report = json.loads(report_str)
+    issues = report.get("results", {})
+    for issue in issues:
+        references = []
+        if issue["more_info"]:
+            references.append(issue["more_info"])
+        if issue.get("issue_cwe", {}).get("link", None):
+            references.append(issue["issue_cwe"]["link"])
+        yield SecurityIssue(
+            file_name=issue["filename"].replace(str(prefix) + "/", ""),
+            line=issue["line_number"],
+            column=issue["col_offset"],
+            cwe=str(issue["issue_cwe"].get("id", "")),
+            test_id=issue["test_id"],
+            description=issue["issue_text"],
+            references=tuple(references),
+        )
+
+
+def issues_to_markdown(issues: Iterable[SecurityIssue]) -> str:
+    template = cleandoc(
+        """
+        {header}{rows}
+    """
+    )
+
+    def _header():
+        header = "# Security\n\n"
+        header += "|File|line/<br>column|Cwe|Test ID|Details|\n"
+        header += "|---|:-:|:-:|:-:|---|\n"
+        return header
+
+    def _row(issue):
+        row = "|" + issue.file_name + "|"
+        row += f"line: {issue.line}<br>column: {issue.column}|"
+        row += issue.cwe + "|"
+        row += issue.test_id + "|"
+        for element in issue.references:
+            row += element + " ,<br>"
+        row = row[:-5] + "|"
+        return row
+
+    return template.format(header=_header(), rows="\n".join(_row(i) for i in issues))
+
+
 def security_issue_title(issue: Issue) -> str:
     return f"🔐 {issue.cve}: {issue.coordinates}"
 
@@ -257,6 +315,23 @@ def create(
         stdout(format_jsonl(issue_url, issue))
 
 
+class PPrintFormats(str, Enum):
+    markdown = "markdown"
+
+
+@CLI.command(name="pretty-print")
+def json_issue_to_markdown(
+    json_file: typer.FileText = typer.Argument(
+        mode="r", help="json file with issues to convert"
+    ),
+    path: Path = typer.Argument(default=Path("."), help="path to project root"),
+) -> None:
+    content = json_file.read()
+    issues = from_json(content, path.absolute())
+    issues = sorted(issues, key=lambda i: (i.file_name, i.cwe, i.test_id))
+    print(issues_to_markdown(issues))
+
+
 def format_jsonl(issue_url: str, issue: Issue) -> str:
     issue_json = asdict(issue)
     issue_json["issue_url"] = issue_url.strip()
diff --git a/noxconfig.py b/noxconfig.py
@@ -35,6 +35,7 @@ class Config:
 
     root: Path = Path(__file__).parent
     doc: Path = Path(__file__).parent / "doc"
+    importlinter: Path = Path(__file__).parent / ".import_linter_config"
     version_file: Path = Path(__file__).parent / "exasol" / "toolbox" / "version.py"
     path_filters: Iterable[str] = (
         "dist",
diff --git a/poetry.lock b/poetry.lock
diff --git a/pyproject.toml b/pyproject.toml
diff --git a/test/integration/cli/security-pprint-emty.t b/test/integration/cli/security-pprint-emty.t
diff --git a/test/integration/cli/security-pprint.t b/test/integration/cli/security-pprint.t
diff --git a/test/unit/dependencies_check_test.py b/test/unit/dependencies_check_test.py
diff --git a/test/unit/security_test.py b/test/unit/security_test.py

Original file line number	Diff line number	Diff line change
`@@ -0,0 +1,2 @@`
	`1`	`+[importlinter]`
	`2`	`+root_package = exasol.toolbox`
-Original file line number
+Diff line change
@@ @@ -1,5 +1,6 @@ @@
 nox
 ===
++
 The nox package contains nox related functionalities like pre defined nox tasks.
 .. figure:: ../../_static/nothing-to-see-here.png