diff --git a/doc/changes/unreleased.md b/doc/changes/unreleased.md index 235c0c794..d885782d5 100644 --- a/doc/changes/unreleased.md +++ b/doc/changes/unreleased.md @@ -5,9 +5,21 @@ With #441, please ensure that the location of the `version.py` is given for `Config.version_file`, which is specified in the `noxconfig.py` +With #449, it's possible to customize what arguments are being using with `pyupgrade` +via the `noxconfig.Config`: +```python +pyupgrade_args = ("--py310-plus",) +``` + ## 📚 Documentation * Updated getting_started.rst for allowing tag-based releases ## ✨ Features -* [#441](https://github.com/exasol/python-toolbox/issues/441): Switched nox task for `version:check` to use the config value of `version_file` to specify the location of the `version.py` \ No newline at end of file +* [#441](https://github.com/exasol/python-toolbox/issues/441): Switched nox task for `version:check` to use the config value of `version_file` to specify the location of the `version.py` + +## ⚒️ Refactorings + +* [#449](https://github.com/exasol/python-toolbox/issues/449): Refactored `dependency:licenses`: + * to use pydantic models & `poetry show` for dependencies + * to updated reading the `pyproject.toml` to be compatible with poetry 2.x+ \ No newline at end of file diff --git a/exasol/toolbox/nox/_dependencies.py b/exasol/toolbox/nox/_dependencies.py index a0af5ff40..95f827f0a 100644 --- a/exasol/toolbox/nox/_dependencies.py +++ b/exasol/toolbox/nox/_dependencies.py @@ -3,214 +3,19 @@ import argparse import json import subprocess -import tempfile -from dataclasses import dataclass -from inspect import cleandoc -from json import loads from pathlib import Path import nox -import tomlkit from nox import Session - -@dataclass(frozen=True) -class Package: - name: str - package_link: str - version: str - license: str - license_link: str - - -def _dependencies(toml_str: str) -> dict[str, list]: - toml = tomlkit.loads(toml_str) - poetry = toml.get("tool", {}).get("poetry", {}) - dependencies: dict[str, list] = {} - - packages = poetry.get("dependencies", {}) - if packages: - dependencies["project"] = [] - for package in packages: - dependencies["project"].append(package) - - packages = poetry.get("dev", {}).get("dependencies", {}) - if packages: - dependencies["dev"] = [] - for package in packages: - dependencies["dev"].append(package) - - groups = poetry.get("group", {}) - for group in groups: - packages = groups.get(group, {}).get("dependencies") - if packages and not dependencies.get(group, {}): - dependencies[group] = [] - for package in packages: - dependencies[group].append(package) - return dependencies - - -def _normalize(_license: str) -> str: - def is_multi_license(l): - return ";" in l - - def select_most_restrictive(licenses: list) -> str: - _max = 0 - lic = "Unknown" - _mapping = { - "Unknown": -1, - "Unlicensed": 0, - "BSD": 1, - "MIT": 2, - "MPLv2": 3, - "LGPLv2": 4, - "GPLv2": 5, - "GPLv3": 6, - } - for l in licenses: - if l in _mapping: - if _mapping[l] > _mapping[lic]: - lic = l - else: - return "
".join(licenses) - return lic - - mapping = { - "BSD License": "BSD", - "MIT License": "MIT", - "The Unlicensed (Unlicensed)": "Unlicensed", - "Mozilla Public License 2.0 (MPL 2.0)": "MPLv2", - "GNU General Public License (GPL)": "GPL", - "GNU Lesser General Public License v2 (LGPLv2)": "LGPLv2", - "GNU General Public License v2 (GPLv2)": "GPLv2", - "GNU General Public License v2 or later (GPLv2+)": "GPLv2+", - "GNU General Public License v3 (GPLv3)": "GPLv3", - "Apache Software License": "Apache", - } - - if is_multi_license(_license): - items = [] - for item in _license.split(";"): - item = str(item).strip() - if item in mapping: - items.append(mapping[item]) - else: - items.append(item) - return select_most_restrictive(items) - - if _license not in mapping: - return _license - - return mapping[_license] - - -def _packages_from_json(json: str) -> list[Package]: - packages = loads(json) - packages_list = [] - mapping = { - "GPLv1": "https://www.gnu.org/licenses/old-licenses/gpl-1.0.html", - "GPLv2": "https://www.gnu.org/licenses/old-licenses/gpl-2.0.html", - "LGPLv2": "https://www.gnu.org/licenses/old-licenses/lgpl-2.0.html", - "GPLv3": "https://www.gnu.org/licenses/gpl-3.0.html", - "LGPLv3": "https://www.gnu.org/licenses/lgpl-3.0.html", - "Apache": "https://www.apache.org/licenses/LICENSE-2.0", - "MIT": "https://mit-license.org/", - "BSD": "https://opensource.org/license/bsd-3-clause", - } - for package in packages: - package_license = _normalize(package["License"]) - packages_list.append( - Package( - name=package["Name"], - package_link="" if package["URL"] == "UNKNOWN" else package["URL"], - version=package["Version"], - license=package_license, - license_link=( - "" if package_license not in mapping else mapping[package_license] - ), - ) - ) - return packages_list - - -def _licenses() -> list[Package]: - with tempfile.NamedTemporaryFile() as file: - subprocess.run( - [ - "poetry", - "run", - "pip-licenses", - "--format=json", - "--output-file=" + file.name, - "--with-system", - "--with-urls", - ], - capture_output=True, - ) - result = _packages_from_json(file.read().decode()) - return result - - -def _packages_to_markdown( - dependencies: dict[str, list], packages: list[Package] -) -> str: - def heading(): - text = "# Dependencies\n" - return text - - def dependency(group: str, group_packages: list, packages: list[Package]) -> str: - def _header(_group: str): - _group = "".join([word.capitalize() for word in _group.strip().split()]) - text = f"## {_group} Dependencies\n" - text += "|Package|version|Licence|\n" - text += "|---|---|---|\n" - return text - - def _rows(_group_packages: list, _packages: list[Package]) -> str: - def _normalize_package_name(name: str) -> str: - _name = name.lower() - while "_" in _name: - _name = _name.replace("_", "-") - return _name - - text = "" - for package in _group_packages: - consistent = filter( - lambda elem: (_normalize_package_name(elem.name) == package), - _packages, - ) - for content in consistent: - if content.package_link: - text += f"|[{content.name}]({content.package_link})" - else: - text += f"|{content.name}" - text += f"|{content.version}" - if content.license_link: - text += f"|[{content.license}]({content.license_link})|\n" - else: - text += f"|{content.license}|\n" - text += "\n" - return text - - _template = cleandoc( - """ - {header}{rows} - """ - ) - return _template.format( - header=_header(group), rows=_rows(group_packages, packages) - ) - - template = cleandoc( - """ - {heading}{rows} - """ - ) - - rows = "" - for group in dependencies: - rows += dependency(group, dependencies[group], packages) - return template.format(heading=heading(), rows=rows) +from exasol.toolbox.util.dependencies.licenses import ( + licenses, + packages_to_markdown, +) +from exasol.toolbox.util.dependencies.poetry_dependencies import ( + PoetryDependencies, + PoetryToml, +) class Audit: @@ -282,10 +87,13 @@ def run(self, session: Session) -> None: @nox.session(name="dependency:licenses", python=False) def dependency_licenses(session: Session) -> None: """returns the packages and their licenses""" - toml = Path("pyproject.toml") - dependencies = _dependencies(toml.read_text()) - package_infos = _licenses() - print(_packages_to_markdown(dependencies=dependencies, packages=package_infos)) + working_directory = Path() + poetry_dep = PoetryToml.load_from_toml(working_directory=working_directory) + dependencies = PoetryDependencies( + groups=poetry_dep.groups, working_directory=working_directory + ).direct_dependencies + package_infos = licenses() + print(packages_to_markdown(dependencies=dependencies, packages=package_infos)) @nox.session(name="dependency:audit", python=False) diff --git a/exasol/toolbox/nox/_format.py b/exasol/toolbox/nox/_format.py index 31eeb29c5..cdba9c2fa 100644 --- a/exasol/toolbox/nox/_format.py +++ b/exasol/toolbox/nox/_format.py @@ -1,6 +1,6 @@ from __future__ import annotations -from typing import Iterable +from collections.abc import Iterable import nox from nox import Session @@ -10,7 +10,12 @@ _version, python_files, ) -from noxconfig import PROJECT_CONFIG +from noxconfig import ( + PROJECT_CONFIG, + Config, +) + +_PYUPGRADE_ARGS = ("--py39-plus",) def _code_format(session: Session, mode: Mode, files: Iterable[str]) -> None: @@ -21,12 +26,11 @@ def command(*args: str) -> Iterable[str]: session.run(*command("black"), *files) -def _pyupgrade(session: Session, files: Iterable[str]) -> None: +def _pyupgrade(session: Session, config: Config, files: Iterable[str]) -> None: + pyupgrade_args = getattr(config, "pyupgrade_args", _PYUPGRADE_ARGS) session.run( - "poetry", - "run", "pyupgrade", - "--py38-plus", + *pyupgrade_args, "--exit-zero-even-if-changed", *files, ) @@ -37,7 +41,7 @@ def fix(session: Session) -> None: """Runs all automated fixes on the code base""" py_files = [f"{file}" for file in python_files(PROJECT_CONFIG.root)] _version(session, Mode.Fix) - _pyupgrade(session, py_files) + _pyupgrade(session, config=PROJECT_CONFIG, files=py_files) _code_format(session, Mode.Fix, py_files) diff --git a/exasol/toolbox/nox/_lint.py b/exasol/toolbox/nox/_lint.py index 82004749d..2972a9c2e 100644 --- a/exasol/toolbox/nox/_lint.py +++ b/exasol/toolbox/nox/_lint.py @@ -2,12 +2,8 @@ import argparse import sys +from collections.abc import Iterable from pathlib import Path -from typing import ( - Dict, - Iterable, - List, -) import nox import rich.console @@ -20,10 +16,6 @@ def _pylint(session: Session, files: Iterable[str]) -> None: session.run( - "poetry", - "run", - "python", - "-m", "pylint", "--output-format", "colorized,json:.lint.json,text:.lint.txt", @@ -33,8 +25,6 @@ def _pylint(session: Session, files: Iterable[str]) -> None: def _type_check(session: Session, files: Iterable[str]) -> None: session.run( - "poetry", - "run", "mypy", "--explicit-package-bases", "--namespace-packages", @@ -49,8 +39,6 @@ def _type_check(session: Session, files: Iterable[str]) -> None: def _security_lint(session: Session, files: Iterable[str]) -> None: session.run( - "poetry", - "run", "bandit", "--severity-level", "low", @@ -63,8 +51,6 @@ def _security_lint(session: Session, files: Iterable[str]) -> None: *files, ) session.run( - "poetry", - "run", "bandit", "--severity-level", "low", diff --git a/exasol/toolbox/nox/tasks.py b/exasol/toolbox/nox/tasks.py index 65e690f3c..aeda05694 100644 --- a/exasol/toolbox/nox/tasks.py +++ b/exasol/toolbox/nox/tasks.py @@ -21,7 +21,6 @@ from exasol.toolbox.nox._format import ( _code_format, - _pyupgrade, fix, ) diff --git a/exasol/toolbox/tools/security.py b/exasol/toolbox/tools/security.py index 4eb96c908..8b5e9edcd 100644 --- a/exasol/toolbox/tools/security.py +++ b/exasol/toolbox/tools/security.py @@ -18,6 +18,7 @@ from functools import partial from inspect import cleandoc from pathlib import Path +from typing import Optional import typer @@ -269,7 +270,9 @@ def as_markdown_listing(elements: Iterable[str]): ) -def create_security_issue(issue: Issue, project: str | None = None) -> tuple[str, str]: +def create_security_issue( + issue: Issue, project: Optional[str] = None +) -> tuple[str, str]: # fmt: off command = [ "gh", "issue", "create", diff --git a/exasol/toolbox/util/dependencies/__init__.py b/exasol/toolbox/util/dependencies/__init__.py new file mode 100644 index 000000000..e69de29bb diff --git a/exasol/toolbox/util/dependencies/licenses.py b/exasol/toolbox/util/dependencies/licenses.py new file mode 100644 index 000000000..25a3ac17c --- /dev/null +++ b/exasol/toolbox/util/dependencies/licenses.py @@ -0,0 +1,178 @@ +from __future__ import annotations + +import subprocess +import tempfile +from inspect import cleandoc +from json import loads +from typing import Optional + +from pydantic import field_validator + +from exasol.toolbox.util.dependencies.shared_models import Package + +LICENSE_MAPPING_TO_ABBREVIATION = { + "BSD License": "BSD", + "MIT License": "MIT", + "The Unlicensed (Unlicensed)": "Unlicensed", + "Mozilla Public License 2.0 (MPL 2.0)": "MPLv2", + "GNU General Public License (GPL)": "GPL", + "GNU Lesser General Public License v2 (LGPLv2)": "LGPLv2", + "GNU General Public License v2 (GPLv2)": "GPLv2", + "GNU General Public License v2 or later (GPLv2+)": "GPLv2+", + "GNU General Public License v3 (GPLv3)": "GPLv3", + "Apache Software License": "Apache", +} + +LICENSE_MAPPING_TO_URL = { + "GPLv1": "https://www.gnu.org/licenses/old-licenses/gpl-1.0.html", + "GPLv2": "https://www.gnu.org/licenses/old-licenses/gpl-2.0.html", + "LGPLv2": "https://www.gnu.org/licenses/old-licenses/lgpl-2.0.html", + "GPLv3": "https://www.gnu.org/licenses/gpl-3.0.html", + "LGPLv3": "https://www.gnu.org/licenses/lgpl-3.0.html", + "Apache": "https://www.apache.org/licenses/LICENSE-2.0", + "MIT": "https://mit-license.org/", + "BSD": "https://opensource.org/license/bsd-3-clause", +} + + +class PackageLicense(Package): + package_link: Optional[str] + license: str + + @field_validator("package_link", mode="before") + def map_unknown_to_none(cls, v) -> Optional[str]: + if v == "UNKNOWN": + return None + return v + + @field_validator("license", mode="before") + def map_to_normalized_values(cls, v) -> Optional[str]: + return _normalize(v) + + @property + def license_link(self) -> Optional[str]: + return LICENSE_MAPPING_TO_URL.get(self.license, None) + + +def _normalize(_license: str) -> str: + def is_multi_license(l: str) -> bool: + return ";" in l + + def select_most_restrictive(licenses: list[str]) -> str: + lic = "Unknown" + _mapping = { + "Unknown": -1, + "Unlicensed": 0, + "BSD": 1, + "MIT": 2, + "MPLv2": 3, + "LGPLv2": 4, + "GPLv2": 5, + "GPLv3": 6, + } + for l in licenses: + if l in _mapping: + if _mapping[l] > _mapping[lic]: + lic = l + else: + return "
".join(licenses) + return lic + + if is_multi_license(_license): + items = [] + for item in _license.split(";"): + item = str(item).strip() + items.append(LICENSE_MAPPING_TO_ABBREVIATION.get(item, item)) + return select_most_restrictive(items) + + return LICENSE_MAPPING_TO_ABBREVIATION.get(_license, _license) + + +def _packages_from_json(json: str) -> list[PackageLicense]: + packages = loads(json) + return [ + PackageLicense( + name=package["Name"], + package_link=package["URL"], + version=package["Version"], + license=package["License"], + ) + for package in packages + ] + + +def licenses() -> list[PackageLicense]: + with tempfile.NamedTemporaryFile() as file: + subprocess.run( + [ + "pip-licenses", + "--format=json", + "--output-file=" + file.name, + "--with-system", + "--with-urls", + ], + capture_output=True, + check=True, + ) + return _packages_from_json(file.read().decode()) + + +def packages_to_markdown( + dependencies: dict[str, list], packages: list[PackageLicense] +) -> str: + def heading(): + return "# Dependencies\n" + + def dependency( + group: str, + group_packages: list[Package], + packages: list[PackageLicense], + ) -> str: + def _header(_group: str): + _group = "".join([word.capitalize() for word in _group.strip().split()]) + text = f"## {_group} Dependencies\n" + text += "|Package|Version|License|\n" + text += "|---|---|---|\n" + return text + + def _rows( + _group_packages: list[Package], _packages: list[PackageLicense] + ) -> str: + text = "" + for package in _group_packages: + consistent = filter( + lambda elem: elem.normalized_name == package.normalized_name, + _packages, + ) + for content in consistent: + if content.package_link: + text += f"|[{content.name}]({content.package_link})" + else: + text += f"|{content.name}" + text += f"|{content.version}" + if content.license_link: + text += f"|[{content.license}]({content.license_link})|\n" + else: + text += f"|{content.license}|\n" + text += "\n" + return text + + _template = cleandoc( + """ + {header}{rows} + """ + ) + return _template.format( + header=_header(group), rows=_rows(group_packages, packages) + ) + + template = cleandoc( + """ + {heading}{rows} + """ + ) + + rows = "" + for group in dependencies: + rows += dependency(group, dependencies[group], packages) + return template.format(heading=heading(), rows=rows) diff --git a/exasol/toolbox/util/dependencies/poetry_dependencies.py b/exasol/toolbox/util/dependencies/poetry_dependencies.py new file mode 100644 index 000000000..de6bb5acc --- /dev/null +++ b/exasol/toolbox/util/dependencies/poetry_dependencies.py @@ -0,0 +1,134 @@ +from __future__ import annotations + +import re +import subprocess +from pathlib import Path +from typing import Optional + +import tomlkit +from pydantic import ( + BaseModel, + model_validator, +) +from tomlkit import TOMLDocument + +from exasol.toolbox.util.dependencies.shared_models import Package + + +class PoetryGroup(BaseModel): + name: str + toml_section: Optional[str] + + class Config: + frozen = True + + +TRANSITIVE_GROUP = PoetryGroup(name="transitive", toml_section=None) + + +class PoetryToml(BaseModel): + content: TOMLDocument + + class Config: + frozen = True + arbitrary_types_allowed = True + + @classmethod + def load_from_toml(cls, working_directory: Path) -> PoetryToml: + file_path = working_directory / "pyproject.toml" + if not file_path.exists(): + raise ValueError(f"File not found: {file_path}") + + try: + text = file_path.read_text() + content = tomlkit.loads(text) + return cls(content=content) + except Exception as e: + raise ValueError(f"Error reading file: {str(e)}") + + def get_section_dict(self, section: str) -> Optional[dict]: + current = self.content.copy() + for section in section.split("."): + if section not in current: + return None + current = current[section] # type: ignore + return current + + @property + def groups(self) -> tuple[PoetryGroup, ...]: + groups = [] + + main_key = "project.dependencies" + if self.get_section_dict(main_key): + groups.append(PoetryGroup(name="main", toml_section=main_key)) + + main_dynamic_key = "tool.poetry.dependencies" + if self.get_section_dict(main_dynamic_key): + groups.append(PoetryGroup(name="main", toml_section=main_dynamic_key)) + + group_key = "tool.poetry.group" + if group_dict := self.get_section_dict(group_key): + for group, content in group_dict.items(): + if "dependencies" in content: + groups.append( + PoetryGroup( + name=group, + toml_section=f"{group_key}.{group}.dependencies", + ) + ) + return tuple(groups) + + +class PoetryDependencies(BaseModel): + groups: tuple[PoetryGroup, ...] + working_directory: Path + + @staticmethod + def _extract_from_line(line: str) -> Package: + pattern = r"\s+(\d+(?:\.\d+)*)\s+" + match = re.split(pattern, line) + return Package(name=match[0], version=match[1]) # + + def _extract_from_poetry_show(self, output_text: str) -> list[Package]: + return [self._extract_from_line(line) for line in output_text.splitlines()] + + @property + def direct_dependencies(self) -> dict[str, list[Package]]: + dependencies = {} + for group in self.groups: + command = ("poetry", "show", "--top-level", f"--only={group.name}") + output = subprocess.run( + command, + capture_output=True, + text=True, + cwd=self.working_directory, + check=True, + ) + result = self._extract_from_poetry_show(output_text=output.stdout) + dependencies[group.name] = result + return dependencies + + @property + def all_dependencies(self) -> dict[str, list[Package]]: + command = ("poetry", "show") + output = subprocess.run( + command, + capture_output=True, + text=True, + cwd=self.working_directory, + check=True, + ) + + direct_dependencies = self.direct_dependencies.copy() + transitive_dependencies = [] + names_direct_dependencies = { + dep.name + for group_list in direct_dependencies.values() + for dep in group_list + } + for line in output.stdout.splitlines(): + dep = self._extract_from_line(line=line) + if dep.name not in names_direct_dependencies: + transitive_dependencies.append(dep) + + return direct_dependencies | {TRANSITIVE_GROUP.name: transitive_dependencies} diff --git a/exasol/toolbox/util/dependencies/shared_models.py b/exasol/toolbox/util/dependencies/shared_models.py new file mode 100644 index 000000000..32ad25eb6 --- /dev/null +++ b/exasol/toolbox/util/dependencies/shared_models.py @@ -0,0 +1,24 @@ +from __future__ import annotations + +from packaging.version import Version +from pydantic import ( + BaseModel, + field_validator, +) + + +class Package(BaseModel): + name: str + version: Version + + class Config: + frozen = True + arbitrary_types_allowed = True + + @field_validator("version", mode="before") + def convert_version(cls, v: str) -> Version: + return Version(v) + + @property + def normalized_name(self) -> str: + return self.name.lower().replace("_", "-") diff --git a/noxconfig.py b/noxconfig.py index 739dbfb94..a5883f27f 100644 --- a/noxconfig.py +++ b/noxconfig.py @@ -53,6 +53,10 @@ class Config: python_versions = ["3.9", "3.10", "3.11", "3.12", "3.13"] exasol_versions = ["7.1.9"] plugins = [UpdateTemplates] + # need --keep-runtime-typing, as pydantic with python3.9 does not accept str | None + # format, and it is not resolved with from __future__ import annotations. pyupgrade + # will keep switching Optional[str] to str | None leading to issues. + pyupgrade_args = ("--py39-plus", "--keep-runtime-typing") PROJECT_CONFIG = Config() diff --git a/poetry.lock b/poetry.lock index a537bc104..2c6059565 100644 --- a/poetry.lock +++ b/poetry.lock @@ -18,7 +18,7 @@ version = "0.7.0" description = "Reusable constraint types to use with typing.Annotated" optional = false python-versions = ">=3.8" -groups = ["dev"] +groups = ["main", "dev"] files = [ {file = "annotated_types-0.7.0-py3-none-any.whl", hash = "sha256:1f02e8b43a8fbbc3f3e0d4f0f4bfc8131bcb4eebe8849b8e5c773f3a1c582a53"}, {file = "annotated_types-0.7.0.tar.gz", hash = "sha256:aff07c09a53a08bc8cfccb9c85b05f1aa9a2a6f23728d790723543408344ce89"}, @@ -1582,14 +1582,14 @@ defusedxml = ">=0.7.1,<0.8.0" [[package]] name = "pydantic" -version = "2.11.4" +version = "2.11.5" description = "Data validation using Python type hints" optional = false python-versions = ">=3.9" -groups = ["dev"] +groups = ["main", "dev"] files = [ - {file = "pydantic-2.11.4-py3-none-any.whl", hash = "sha256:d9615eaa9ac5a063471da949c8fc16376a84afb5024688b3ff885693506764eb"}, - {file = "pydantic-2.11.4.tar.gz", hash = "sha256:32738d19d63a226a52eed76645a98ee07c1f410ee41d93b4afbfa85ed8111c2d"}, + {file = "pydantic-2.11.5-py3-none-any.whl", hash = "sha256:f9c26ba06f9747749ca1e5c94d6a85cb84254577553c8785576fd38fa64dc0f7"}, + {file = "pydantic-2.11.5.tar.gz", hash = "sha256:7f853db3d0ce78ce8bbb148c401c2cdd6431b3473c0cdff2755c7690952a7b7a"}, ] [package.dependencies] @@ -1608,7 +1608,7 @@ version = "2.33.2" description = "Core functionality for Pydantic validation and serialization" optional = false python-versions = ">=3.9" -groups = ["dev"] +groups = ["main", "dev"] files = [ {file = "pydantic_core-2.33.2-cp310-cp310-macosx_10_12_x86_64.whl", hash = "sha256:2b3d326aaef0c0399d9afffeb6367d5e26ddc24d351dbc9c636840ac355dc5d8"}, {file = "pydantic_core-2.33.2-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:0e5b2671f05ba48b94cb90ce55d8bdcaaedb8ba00cc5359f6810fc918713983d"}, @@ -2406,7 +2406,7 @@ version = "0.4.0" description = "Runtime typing introspection tools" optional = false python-versions = ">=3.9" -groups = ["dev"] +groups = ["main", "dev"] files = [ {file = "typing_inspection-0.4.0-py3-none-any.whl", hash = "sha256:50e72559fcd2a6367a19f7a7e610e6afcb9fac940c650290eed893d61386832f"}, {file = "typing_inspection-0.4.0.tar.gz", hash = "sha256:9765c87de36671694a67904bf2c96e395be9c6439bb6c87b5142569dcdd65122"}, @@ -2502,4 +2502,4 @@ type = ["pytest-mypy"] [metadata] lock-version = "2.1" python-versions = ">=3.9,<4.0" -content-hash = "2fe3020811687baeb4344aeab5ac68bdb3731ffbe7eb8e8c5080072213b67a81" +content-hash = "5ad4971398eda95955839d4330f6073953ca6cb34b741801c483c163f54e08a0" diff --git a/project-template/{{cookiecutter.repo_name}}/noxconfig.py b/project-template/{{cookiecutter.repo_name}}/noxconfig.py index db37d3cac..4be08f301 100644 --- a/project-template/{{cookiecutter.repo_name}}/noxconfig.py +++ b/project-template/{{cookiecutter.repo_name}}/noxconfig.py @@ -16,7 +16,7 @@ class Config: / "version.py" ) path_filters: Iterable[str] = () - + pyupgrade_args = ("--py{{cookiecutter.python_version_min | replace('.', '')}}-plus",) plugins = [] diff --git a/pyproject.toml b/pyproject.toml index d86ed4556..5607d3594 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -64,6 +64,7 @@ bandit = {extras = ["toml"], version = "^1.7.9"} jinja2 = "^3.1.6" pip-licenses = "^5.0.0" pip-audit = "^2.7.3" +pydantic = "^2.11.5" [tool.poetry.group.dev.dependencies] autoimport = "^1.4.0" @@ -101,6 +102,8 @@ fail-under = 7.5 max-line-length = 88 max-module-lines = 800 +[tool.mypy] +plugins = ['pydantic.mypy'] [[tool.mypy.overrides]] module = [ diff --git a/test/unit/dependencies_test.py b/test/unit/dependencies_test.py index 49e83df63..c39b6f562 100644 --- a/test/unit/dependencies_test.py +++ b/test/unit/dependencies_test.py @@ -1,165 +1,6 @@ import json -import pytest - -from exasol.toolbox.nox._dependencies import ( - Audit, - Package, - _dependencies, - _normalize, - _packages_from_json, - _packages_to_markdown, -) - - -@pytest.mark.parametrize( - "toml,expected", - [ - ( - """ -[tool.poetry.dependencies] -pytest = ">=7.2.2,<9" -python = "^3.9" - """, - {"project": ["pytest", "python"]}, - ), - ( - """ -[tool.poetry.dependencies] -pytest = ">=7.2.2,<9" -python = "^3.9" - -[tool.poetry.dev.dependencies] -pip-licenses = "^5.0.0" - -[tool.poetry.group.dev.dependencies] -autoimport = "^1.4.0" - """, - {"project": ["pytest", "python"], "dev": ["pip-licenses", "autoimport"]}, - ), - ], -) -def test_dependencies(toml, expected): - actual = _dependencies(toml) - assert actual == expected - - -@pytest.mark.parametrize( - "licenses,expected", - [ - ("The Unlicensed (Unlicensed); BSD License", "BSD"), - ("BSD License; MIT License", "MIT"), - ("MIT License; Mozilla Public License 2.0 (MPL 2.0)", "MPLv2"), - ( - "Mozilla Public License 2.0 (MPL 2.0); GNU Lesser General Public License v2 (LGPLv2)", - "LGPLv2", - ), - ( - "GNU Lesser General Public License v2 (LGPLv2); GNU General Public License v2 (GPLv2)", - "GPLv2", - ), - ( - "GNU General Public License v2 (GPLv2); GNU General Public License v3 (GPLv3)", - "GPLv3", - ), - ], -) -def test_normalize(licenses, expected): - actual = _normalize(licenses) - assert actual == expected - - -@pytest.mark.parametrize( - "json,expected", - [ - ( - """ -[ - { - "License": "license1", - "Name": "name1", - "URL": "link1", - "Version": "version1" - }, - { - "License": "license2", - "Name": "name2", - "URL": "UNKNOWN", - "Version": "version2" - } -] - """, - [ - Package( - name="name1", - version="version1", - package_link="link1", - license="license1", - license_link="", - ), - Package( - name="name2", - version="version2", - package_link="", - license="license2", - license_link="", - ), - ], - ) - ], -) -def test_packages_from_json(json, expected): - actual = _packages_from_json(json) - assert actual == expected - - -@pytest.mark.parametrize( - "dependencies,packages,expected", - [ - ( - {"project": ["package1", "package3"], "dev": ["package2"]}, - [ - Package( - name="package1", - package_link="package_link1", - version="version1", - license="license1", - license_link="license_link1", - ), - Package( - name="package2", - package_link="package_link2", - version="version2", - license="license2", - license_link="license_link2", - ), - Package( - name="package3", - package_link="package_link3", - version="version3", - license="license3", - license_link="", - ), - ], - """# Dependencies -## Project Dependencies -|Package|version|Licence| -|---|---|---| -|[package1](package_link1)|version1|[license1](license_link1)| -|[package3](package_link3)|version3|license3| - -## Dev Dependencies -|Package|version|Licence| -|---|---|---| -|[package2](package_link2)|version2|[license2](license_link2)| - -""", - ) - ], -) -def test_packages_to_markdown(dependencies, packages, expected): - actual = _packages_to_markdown(dependencies, packages) - assert actual == expected +from exasol.toolbox.nox._dependencies import Audit class TestFilterJsonForVulnerabilities: diff --git a/test/unit/util/dependencies/licenses_test.py b/test/unit/util/dependencies/licenses_test.py new file mode 100644 index 000000000..5cf2ec183 --- /dev/null +++ b/test/unit/util/dependencies/licenses_test.py @@ -0,0 +1,160 @@ +import pytest + +from exasol.toolbox.util.dependencies.licenses import ( + LICENSE_MAPPING_TO_URL, + PackageLicense, + _normalize, + _packages_from_json, + packages_to_markdown, +) +from exasol.toolbox.util.dependencies.poetry_dependencies import ( + PoetryGroup, +) +from exasol.toolbox.util.dependencies.shared_models import Package + +MAIN_GROUP = PoetryGroup(name="main", toml_section="project.dependencies") +DEV_GROUP = PoetryGroup(name="dev", toml_section="tool.poetry.group.dev.dependencies") + + +class TestPackageLicense: + @staticmethod + def test_package_link_map_unknown_to_none(): + result = PackageLicense( + name="dummy", version="0.1.0", package_link="UNKNOWN", license="dummy" + ) + assert result.package_link is None + + @staticmethod + @pytest.mark.parametrize( + "license,expected", + [ + ("GPLv1", LICENSE_MAPPING_TO_URL["GPLv1"]), + ("DUMMY", None), + ], + ) + def test_license_link(license, expected): + result = PackageLicense( + name="dummy", version="0.1.0", package_link="dummy", license=license + ) + assert result.license_link == expected + + +@pytest.mark.parametrize( + "licenses,expected", + [ + ("The Unlicensed (Unlicensed); BSD License", "BSD"), + ("BSD License; MIT License", "MIT"), + ("MIT License; Mozilla Public License 2.0 (MPL 2.0)", "MPLv2"), + ( + "Mozilla Public License 2.0 (MPL 2.0); GNU Lesser General Public License v2 (LGPLv2)", + "LGPLv2", + ), + ( + "GNU Lesser General Public License v2 (LGPLv2); GNU General Public License v2 (GPLv2)", + "GPLv2", + ), + ( + "GNU General Public License v2 (GPLv2); GNU General Public License v3 (GPLv3)", + "GPLv3", + ), + ], +) +def test_normalize(licenses, expected): + actual = _normalize(licenses) + assert actual == expected + + +@pytest.mark.parametrize( + "json,expected", + [ + ( + """ + [ + { + "License": "license1", + "Name": "name1", + "URL": "link1", + "Version": "0.1.0" + }, + { + "License": "license2", + "Name": "name2", + "URL": "UNKNOWN", + "Version": "0.2.0" + } + ] + """, + [ + PackageLicense( + name="name1", + version="0.1.0", + package_link="link1", + license="license1", + ), + PackageLicense( + name="name2", + version="0.2.0", + package_link=None, + license="license2", + ), + ], + ) + ], +) +def test_packages_from_json(json, expected): + actual = _packages_from_json(json) + assert actual == expected + + +@pytest.mark.parametrize( + "dependencies,packages", + [ + ( + { + MAIN_GROUP.name: [ + Package(name="package1", version="0.1.0"), + Package(name="package3", version="0.1.0"), + ], + DEV_GROUP.name: [Package(name="package2", version="0.2.0")], + }, + [ + PackageLicense( + name="package1", + package_link="package_link1", + version="0.1.0", + license="GPLv1", + ), + PackageLicense( + name="package2", + package_link="package_link2", + version="0.2.0", + license="GPLv2", + ), + PackageLicense( + name="package3", + package_link="UNKNOWN", + version="0.3.0", + license="license3", + ), + ], + ) + ], +) +def test_packages_to_markdown(dependencies, packages): + actual = packages_to_markdown(dependencies, packages) + assert ( + actual + == """# Dependencies +## Main Dependencies +|Package|Version|License| +|---|---|---| +|[package1](package_link1)|0.1.0|[GPLv1](https://www.gnu.org/licenses/old-licenses/gpl-1.0.html)| +|package3|0.3.0|license3| + +## Dev Dependencies +|Package|Version|License| +|---|---|---| +|[package2](package_link2)|0.2.0|[GPLv2](https://www.gnu.org/licenses/old-licenses/gpl-2.0.html)| + +""" + ) diff --git a/test/unit/util/dependencies/poetry_dependencies_test.py b/test/unit/util/dependencies/poetry_dependencies_test.py new file mode 100644 index 000000000..0edf86092 --- /dev/null +++ b/test/unit/util/dependencies/poetry_dependencies_test.py @@ -0,0 +1,122 @@ +import subprocess + +import pytest + +from exasol.toolbox.util.dependencies.poetry_dependencies import ( + PoetryDependencies, + PoetryGroup, + PoetryToml, +) +from exasol.toolbox.util.dependencies.shared_models import Package + +MAIN_GROUP = PoetryGroup(name="main", toml_section="project.dependencies") +DEV_GROUP = PoetryGroup(name="dev", toml_section="tool.poetry.group.dev.dependencies") +ANALYSIS_GROUP = PoetryGroup( + name="analysis", toml_section="tool.poetry.group.analysis.dependencies" +) + +PYLINT = Package(name="pylint", version="3.3.7") +ISORT = Package(name="isort", version="6.0.1") +BLACK = Package(name="black", version="25.1.0") + +DIRECT_DEPENDENCIES = { + MAIN_GROUP.name: [PYLINT], + DEV_GROUP.name: [ISORT], + ANALYSIS_GROUP.name: [BLACK], +} + + +@pytest.fixture(scope="module") +def cwd(tmp_path_factory): + return tmp_path_factory.mktemp("test") + + +@pytest.fixture(scope="module") +def project_name(): + return "project" + + +@pytest.fixture(scope="module") +def project_path(cwd, project_name): + return cwd / project_name + + +@pytest.fixture(scope="module") +def create_poetry_project(cwd, project_name, project_path): + subprocess.run(["poetry", "new", project_name], cwd=cwd) + subprocess.run( + ["poetry", "add", f"{PYLINT.name}=={PYLINT.version}"], cwd=project_path + ) + subprocess.run( + ["poetry", "add", "--group", "dev", f"{ISORT.name}=={ISORT.version}"], + cwd=project_path, + ) + subprocess.run( + ["poetry", "add", "--group", "analysis", f"{BLACK.name}=={BLACK.version}"], + cwd=project_path, + ) + + +@pytest.fixture(scope="module") +def pyproject_toml(project_path, create_poetry_project): + return PoetryToml.load_from_toml(working_directory=project_path) + + +class TestPoetryToml: + @staticmethod + def test_get_section_dict_exists(pyproject_toml): + result = pyproject_toml.get_section_dict("project") + assert result is not None + + @staticmethod + def test_get_section_dict_does_not_exist(pyproject_toml): + result = pyproject_toml.get_section_dict("test") + assert result is None + + @staticmethod + def test_groups(pyproject_toml): + assert pyproject_toml.groups == (MAIN_GROUP, DEV_GROUP, ANALYSIS_GROUP) + + +class TestPoetryDependencies: + @staticmethod + @pytest.mark.parametrize( + "line,expected", + [ + ( + "coverage 7.8.0 Code coverage measurement for Python", + Package(name="coverage", version="7.8.0"), + ), + ( + "furo 2024.8.6 A clean customisable Sphinx documentation theme.", + Package(name="furo", version="2024.8.6"), + ), + ( + "import-linter 2.3 Enforces rules for the imports within and between Python packages.", + Package(name="import-linter", version="2.3"), + ), + ], + ) + def test_extract_from_line(line, expected): + result = PoetryDependencies._extract_from_line(line=line) + assert result == expected + + @staticmethod + def test_direct_dependencies(create_poetry_project, project_path): + poetry_dep = PoetryDependencies( + groups=(MAIN_GROUP, DEV_GROUP, ANALYSIS_GROUP), + working_directory=project_path, + ) + assert poetry_dep.direct_dependencies == DIRECT_DEPENDENCIES + + @staticmethod + def test_all_dependencies(create_poetry_project, project_path): + poetry_dep = PoetryDependencies( + groups=(MAIN_GROUP, DEV_GROUP, ANALYSIS_GROUP), + working_directory=project_path, + ) + result = poetry_dep.all_dependencies + + transitive = result.pop("transitive") + assert len(transitive) > 0 + assert result == DIRECT_DEPENDENCIES diff --git a/test/unit/util/dependencies/shared_models_test.py b/test/unit/util/dependencies/shared_models_test.py new file mode 100644 index 000000000..72e0cfa4d --- /dev/null +++ b/test/unit/util/dependencies/shared_models_test.py @@ -0,0 +1,19 @@ +import pytest + +from exasol.toolbox.util.dependencies.shared_models import Package + + +class TestPackage: + @staticmethod + @pytest.mark.parametrize( + "name,expected", + [ + ("numpy", "numpy"), + ("sphinxcontrib-applehelp", "sphinxcontrib-applehelp"), + ("Imaginary_package", "imaginary-package"), + ("Imaginary_package_2", "imaginary-package-2"), + ], + ) + def test_normalized_name(name, expected): + dep = Package(name=name, version="0.1.0") + assert dep.normalized_name == expected