diff --git a/.github/workflows/report.yml b/.github/workflows/report.yml index 8a5fb7df4..fc749f57d 100644 --- a/.github/workflows/report.yml +++ b/.github/workflows/report.yml @@ -27,14 +27,7 @@ jobs: path: ./artifacts - name: Copy Artifacts into Root Folder - working-directory: ./artifacts - run: | - poetry run -- coverage combine --keep coverage-python3.9*/.coverage - # Errors during copying are ignored because they are checked in the next step - cp .coverage ../ || true - cp lint-python3.9/.lint.txt ../ || true - cp lint-python3.9/.lint.json ../ || true - cp security-python3.9/.security.json ../ || true + run: poetry run -- nox -s artifacts:copy -- artifacts - name: Validate Artifacts run: poetry run -- nox -s artifacts:validate diff --git a/doc/changes/unreleased.md b/doc/changes/unreleased.md index 79e701b84..94fdd7479 100644 --- a/doc/changes/unreleased.md +++ b/doc/changes/unreleased.md @@ -1 +1,7 @@ # Unreleased + +## Summary + +## ✨ Features + +* #426: Allowed configuring the python version used for coverage diff --git a/doc/user_guide/collecting_metrics.rst b/doc/user_guide/collecting_metrics.rst new file mode 100644 index 000000000..47fe6dab7 --- /dev/null +++ b/doc/user_guide/collecting_metrics.rst @@ -0,0 +1,35 @@ +Collecting Metrics +================== + +PTB allows you to collect various metrics on the quality of your project +regarding Coverage, Security, and Static Code Analysis. + +For each metric, there is a dedicated nox task, generating one or multiple +files and based on a selected external Python tool. + ++-----------------------------+-----------------------------+--------------+ +| Nox Task | Generated Files | Based on | ++=============================+=============================+==============+ +| ``lint:code`` | ``lint.txt``, ``lint.json`` | ``pylint`` | ++-----------------------------+-----------------------------+--------------+ +| ``lint:security`` | ``.security.json`` | ``bandit`` | ++-----------------------------+-----------------------------+--------------+ +| ``test:unit -- --coverage`` | ``.coverage`` | ``coverage`` | ++-----------------------------+-----------------------------+--------------+ + +The metrics are computed for each point in your build matrix, e.g. for each +Python version defined in file ``noxconfig.py``: + +.. code-block:: python + + @dataclass(frozen=True) + class Config: + python_versions = ["3.9", "3.10", "3.11", "3.12", "3.13"] + +The GitHub workflows of your project can: + +* Use a build matrix, e.g. using different Python versions as shown above +* Define multiple test sessions, e.g. for distinguishing fast vs. slow or expensive tests. + +PTB combines the coverage data of all test sessions but using only the Python +version named first in attribute ``python_versions`` of class ``Config``. diff --git a/doc/user_guide/user_guide.rst b/doc/user_guide/user_guide.rst index 70d4b9077..8b47fe66c 100644 --- a/doc/user_guide/user_guide.rst +++ b/doc/user_guide/user_guide.rst @@ -12,3 +12,4 @@ customization migrating how_to_release + collecting_metrics diff --git a/exasol/toolbox/nox/_artifacts.py b/exasol/toolbox/nox/_artifacts.py index 71f4b6d1d..7bcfceb57 100644 --- a/exasol/toolbox/nox/_artifacts.py +++ b/exasol/toolbox/nox/_artifacts.py @@ -1,13 +1,16 @@ import json import pathlib import re +import shutil import sqlite3 import sys +from collections.abc import Iterable from pathlib import Path import nox from nox import Session +from exasol.toolbox.nox._shared import MINIMUM_PYTHON_VERSION from noxconfig import PROJECT_CONFIG @@ -118,3 +121,49 @@ def _validate_coverage(path: Path) -> str: f"Invalid database, the database is missing the following tables {missing}" ) return "" + + +@nox.session(name="artifacts:copy", python=False) +def copy_artifacts(session: Session) -> None: + """ + Copy artifacts to the current directory + """ + + dir = Path(session.posargs[0]) + suffix = _python_version_suffix() + _combine_coverage(session, dir, f"coverage{suffix}*/.coverage") + _copy_artifacts( + dir, + dir.parent, + [ + f"lint{suffix}/.lint.txt", + f"lint{suffix}/.lint.json", + f"security{suffix}/.security.json", + ], + ) + + +def _python_version_suffix() -> str: + versions = getattr(PROJECT_CONFIG, "python_versions", None) + pivot = versions[0] if versions else MINIMUM_PYTHON_VERSION + return f"-python{pivot}" + + +def _combine_coverage(session: Session, dir: Path, pattern: str): + """ + pattern: glob pattern, e.g. "*.coverage" + """ + if args := [f for f in dir.glob(pattern) if f.exists()]: + session.run("coverage", "combine", "--keep", *sorted(args)) + else: + print(f"Could not find any file {dir}/{pattern}", file=sys.stderr) + + +def _copy_artifacts(source: Path, dest: Path, files: Iterable[str]): + for file in files: + path = source / file + if path.exists(): + print(f"Copying file {path}", file=sys.stderr) + shutil.copy(path, dest) + else: + print(f"File not found {path}", file=sys.stderr) diff --git a/exasol/toolbox/nox/_shared.py b/exasol/toolbox/nox/_shared.py index b198904e6..f4473bde6 100644 --- a/exasol/toolbox/nox/_shared.py +++ b/exasol/toolbox/nox/_shared.py @@ -20,6 +20,8 @@ DEFAULT_PATH_FILTERS = {"dist", ".eggs", "venv", ".poetry"} DOCS_OUTPUT_DIR = ".html-documentation" +MINIMUM_PYTHON_VERSION = "3.9" + class Mode(Enum): Fix = auto() diff --git a/exasol/toolbox/templates/github/workflows/report.yml b/exasol/toolbox/templates/github/workflows/report.yml index 8409c1294..978b660d4 100644 --- a/exasol/toolbox/templates/github/workflows/report.yml +++ b/exasol/toolbox/templates/github/workflows/report.yml @@ -27,14 +27,7 @@ jobs: path: ./artifacts - name: Copy Artifacts into Root Folder - working-directory: ./artifacts - run: | - poetry run -- coverage combine --keep coverage-python3.9*/.coverage - # Errors during copying are ignored because they are checked in the next step - cp .coverage ../ || true - cp lint-python3.9/.lint.txt ../ || true - cp lint-python3.9/.lint.json ../ || true - cp security-python3.9/.security.json ../ || true + run: poetry run -- nox -s artifacts:copy -- artifacts - name: Validate Artifacts run: poetry run -- nox -s artifacts:validate diff --git a/test/unit/artifacts_test.py b/test/unit/artifacts_test.py new file mode 100644 index 000000000..5cf91679b --- /dev/null +++ b/test/unit/artifacts_test.py @@ -0,0 +1,89 @@ +import contextlib +import re +from dataclasses import dataclass +from inspect import cleandoc +from pathlib import Path +from unittest.mock import ( + Mock, + call, + patch, +) + +import pytest + +from exasol.toolbox.nox._artifacts import copy_artifacts + + +@contextlib.contextmanager +def mock_session(path: Path, python_version: str, *files: str): + with patch("exasol.toolbox.nox._artifacts.PROJECT_CONFIG") as config: + config.python_versions = [python_version] + for rel in files: + file = path / rel + file.parent.mkdir(parents=True, exist_ok=True) + file.write_text(rel) + yield Mock(posargs=[str(path)]) + + +def test_missing_files(tmp_path, capsys): + with mock_session(tmp_path, "9.9") as session: + copy_artifacts(session) + captured = capsys.readouterr() + assert re.match( + cleandoc( + f""" + Could not find any file .*/coverage-python9.9\\*/.coverage + File not found .*/lint-python9.9/.lint.txt + File not found .*/lint-python9.9/.lint.json + File not found .*/security-python9.9/.security.json + """ + ), + captured.err, + ) + + +@dataclass +class endswith: + """ + Assert that the str representation of the argument ends with the + specified suffix. + """ + + suffix: str + + def __eq__(self, actual): + return str(actual).endswith(self.suffix) + + +def test_all_files(tmp_path, capsys): + with mock_session( + tmp_path / "artifacts", + "9.9", + "coverage-python9.9-fast/.coverage", + "coverage-python9.9-slow/.coverage", + "lint-python9.9/.lint.txt", + "lint-python9.9/.lint.json", + "security-python9.9/.security.json", + ) as session: + copy_artifacts(session) + + captured = capsys.readouterr() + assert session.run.call_args == call( + "coverage", + "combine", + "--keep", + endswith("coverage-python9.9-fast/.coverage"), + endswith("coverage-python9.9-slow/.coverage"), + ) + assert re.match( + cleandoc( + f""" + Copying file .*/lint-python9.9/.lint.txt + Copying file .*/lint-python9.9/.lint.json + Copying file .*/security-python9.9/.security.json + """ + ), + captured.err, + ) + for f in [".lint.txt", ".lint.json", ".security.json"]: + assert (tmp_path / f).exists() diff --git a/test/unit/nox/_package_version_test.py b/test/unit/nox/_package_version_test.py index fe2783ab7..23bdf0b47 100644 --- a/test/unit/nox/_package_version_test.py +++ b/test/unit/nox/_package_version_test.py @@ -8,9 +8,7 @@ write_version_module, ) from exasol.toolbox.util.version import Version -from noxconfig import ( - Config, -) +from noxconfig import Config DEFAULT_VERSION = Version(major=0, minor=1, patch=0) ALTERNATE_VERSION = Version(major=0, minor=2, patch=0) diff --git a/test/unit/util/dependencies/licenses_test.py b/test/unit/util/dependencies/licenses_test.py index 5cf2ec183..bb3e33684 100644 --- a/test/unit/util/dependencies/licenses_test.py +++ b/test/unit/util/dependencies/licenses_test.py @@ -7,9 +7,7 @@ _packages_from_json, packages_to_markdown, ) -from exasol.toolbox.util.dependencies.poetry_dependencies import ( - PoetryGroup, -) +from exasol.toolbox.util.dependencies.poetry_dependencies import PoetryGroup from exasol.toolbox.util.dependencies.shared_models import Package MAIN_GROUP = PoetryGroup(name="main", toml_section="project.dependencies") diff --git a/test/unit/util/version_test.py b/test/unit/util/version_test.py index caa291855..9a2a4f326 100644 --- a/test/unit/util/version_test.py +++ b/test/unit/util/version_test.py @@ -1,7 +1,5 @@ import subprocess -from unittest.mock import ( - patch, -) +from unittest.mock import patch import pytest