Merge branch 'main' into feature/add-support-for-manually-triggering-documentation-build

Author: Nicoretti

.github/workflows/report.yml

@@ -51,3 +51,4 @@ jobs:
           poetry run coverage report -- --format markdown >> $GITHUB_STEP_SUMMARY
           echo  -e "\n\n# Static Code Analysis\n" >> $GITHUB_STEP_SUMMARY
           cat .lint.txt >> $GITHUB_STEP_SUMMARY
+          poetry run tbx security pretty-print .security.json >> $GITHUB_STEP_SUMMARY

doc/changes/unreleased.md

@@ -1,4 +1,7 @@
 # Unreleased
 
 ## ✨ Added
+
 * Added support to manually trigger documentation build
+* #248: Added security results to workflow summary
+* #233: Added nox task to verify dependency declarations

doc/user_guide/getting_started.rst

@@ -193,6 +193,7 @@ You are ready to use the toolbox. With *nox -l* you can list all available tasks
     - lint:code -> Runs the static code analyzer on the project
     - lint:typing -> Runs the type checker on the project
     - lint:security -> Runs the security linter on the project
+    - lint:dependencies -> Checks if only valid sources of dependencies are used
     - docs:multiversion -> Builds the multiversion project documentation
     - docs:build -> Builds the project documentation
     - docs:open -> Opens the built project documentation

exasol/toolbox/nox/_lint.py

@@ -1,13 +1,22 @@
 from __future__ import annotations
 
-from typing import Iterable
+from typing import (
+    Iterable,
+    List,
+    Dict
+)
 
 import nox
 from nox import Session
 
 from exasol.toolbox.nox._shared import python_files
 from noxconfig import PROJECT_CONFIG
 
+from pathlib import Path
+import rich.console
+import tomlkit
+import sys
+
 
 def _pylint(session: Session, files: Iterable[str]) -> None:
     session.run(
@@ -65,6 +74,61 @@ def _security_lint(session: Session, files: Iterable[str]) -> None:
     )
 
 
+class Dependencies:
+    def __init__(self, illegal: Dict[str, List[str]] | None):
+        self._illegal = illegal or {}
+
+    @staticmethod
+    def parse(pyproject_toml: str) -> "Dependencies":
+        def _source_filter(version) -> bool:
+            ILLEGAL_SPECIFIERS = ['url', 'git', 'path']
+            return any(
+                specifier in version
+                for specifier in ILLEGAL_SPECIFIERS
+            )
+
+        def find_illegal(part) -> List[str]:
+            return [
+                f"{name} = {version}"
+                for name, version in part.items()
+                if _source_filter(version)
+            ]
+
+        illegal: Dict[str, List[str]] = {}
+        toml = tomlkit.loads(pyproject_toml)
+        poetry = toml.get("tool", {}).get("poetry", {})
+
+        part = poetry.get("dependencies", {})
+        if illegal_group := find_illegal(part):
+            illegal["tool.poetry.dependencies"] = illegal_group
+
+        part = poetry.get("dev", {}).get("dependencies", {})
+        if illegal_group := find_illegal(part):
+            illegal["tool.poetry.dev.dependencies"] = illegal_group
+
+        part = poetry.get("group", {})
+        for group, content in part.items():
+            illegal_group = find_illegal(content.get("dependencies", {}))
+            if illegal_group:
+                illegal[f"tool.poetry.group.{group}.dependencies"] = illegal_group
+        return Dependencies(illegal)
+
+    @property
+    def illegal(self) -> Dict[str, List[str]]:
+        return self._illegal
+
+
+def report_illegal(illegal: Dict[str, List[str]], console: rich.console.Console):
+    count = sum(len(deps) for deps in illegal.values())
+    suffix = "y" if count == 1 else "ies"
+    console.print(f"{count} illegal dependenc{suffix}\n", style="red")
+    for section, dependencies in illegal.items():
+        console.print(f"\\[{section}]", style="red")
+        for dependency in dependencies:
+            console.print(dependency, style="red")
+        console.print("")
+
+
 @nox.session(name="lint:code", python=False)
 def lint(session: Session) -> None:
     "Runs the static code analyzer on the project"
@@ -84,3 +148,14 @@ def security_lint(session: Session) -> None:
     """Runs the security linter on the project"""
     py_files = [f"{file}" for file in python_files(PROJECT_CONFIG.root)]
     _security_lint(session, list(filter(lambda file: "test" not in file, py_files)))
+
+
+@nox.session(name="lint:dependencies", python=False)
+def dependency_check(session: Session) -> None:
+    """Checks if only valid sources of dependencies are used"""
+    content = Path(PROJECT_CONFIG.root, "pyproject.toml").read_text()
+    dependencies = Dependencies.parse(content)
+    console = rich.console.Console()
+    if illegal := dependencies.illegal:
+        report_illegal(illegal, console)
+        sys.exit(1)

exasol/toolbox/nox/tasks.py

@@ -68,5 +68,6 @@ def check(session: Session) -> None:
     python_files,
 )
 
+
 # isort: on
 # fmt: on

exasol/toolbox/templates/github/workflows/report.yml

@@ -51,3 +51,4 @@ jobs:
           poetry run coverage report -- --format markdown >> $GITHUB_STEP_SUMMARY
           echo  -e "\n\n# Static Code Analysis\n" >> $GITHUB_STEP_SUMMARY
           cat .lint.txt >> $GITHUB_STEP_SUMMARY
+          poetry run tbx security pretty-print .security.json >> $GITHUB_STEP_SUMMARY

exasol/toolbox/tools/security.py

@@ -16,8 +16,8 @@
     Iterable,
     Tuple,
 )
-
 import typer
+from pathlib import Path
 
 stdout = print
 stderr = partial(print, file=sys.stderr)
@@ -101,6 +101,64 @@ def from_maven(report: str) -> Iterable[Issue]:
             )
 
 
+@dataclass(frozen=True)
+class SecurityIssue:
+    file_name: str
+    line: int
+    column: int
+    cwe: str
+    test_id: str
+    description: str
+    references: tuple
+
+
+def from_json(report_str: str, prefix: Path) -> Iterable[SecurityIssue]:
+    report = json.loads(report_str)
+    issues = report.get("results", {})
+    for issue in issues:
+        references = []
+        if issue["more_info"]:
+            references.append(issue["more_info"])
+        if issue.get("issue_cwe", {}).get("link", None):
+            references.append(issue["issue_cwe"]["link"])
+        yield SecurityIssue(
+            file_name=issue["filename"].replace(str(prefix) + "/", ""),
+            line=issue["line_number"],
+            column=issue["col_offset"],
+            cwe=str(issue["issue_cwe"].get("id", "")),
+            test_id=issue["test_id"],
+            description=issue["issue_text"],
+            references=tuple(references)
+        )
+
+
+def issues_to_markdown(issues: Iterable[SecurityIssue]) -> str:
+    template = cleandoc("""
+        {header}{rows}
+    """)
+
+    def _header():
+        header = "# Security\n\n"
+        header += "|File|line/<br>column|Cwe|Test ID|Details|\n"
+        header += "|---|:-:|:-:|:-:|---|\n"
+        return header
+
+    def _row(issue):
+        row = "|" + issue.file_name + "|"
+        row += f"line: {issue.line}<br>column: {issue.column}|"
+        row += issue.cwe + "|"
+        row += issue.test_id + "|"
+        for element in issue.references:
+            row += element + " ,<br>"
+        row = row[:-5] + "|"
+        return row
+
+    return template.format(
+        header=_header(),
+        rows="\n".join(_row(i) for i in issues)
+    )
+
+
 def security_issue_title(issue: Issue) -> str:
     return f"🔐 {issue.cve}: {issue.coordinates}"
 
@@ -159,7 +217,6 @@ def create_security_issue(issue: Issue, project="") -> Tuple[str, str]:
 CVE_CLI = typer.Typer()
 CLI.add_typer(CVE_CLI, name="cve", help="Work with CVE's")
 
-
 class Format(str, Enum):
     Maven = "maven"
 
@@ -257,6 +314,21 @@ def create(
         stdout(format_jsonl(issue_url, issue))
 
 
+class PPrintFormats(str, Enum):
+    markdown = "markdown"
+
+
+@CLI.command(name="pretty-print")
+def json_issue_to_markdown(
+        json_file: typer.FileText = typer.Argument(mode="r", help="json file with issues to convert"),
+        path: Path = typer.Argument(default=Path("."), help="path to project root")
+) -> None:
+    content = json_file.read()
+    issues = from_json(content, path.absolute())
+    issues = sorted(issues, key=lambda i: (i.file_name, i.cwe, i.test_id))
+    print(issues_to_markdown(issues))
+
+
 def format_jsonl(issue_url: str, issue: Issue) -> str:
     issue_json = asdict(issue)
     issue_json["issue_url"] = issue_url.strip()