From ea5044dda26f3ef18d46f139c4d9eb97baec5ba8 Mon Sep 17 00:00:00 2001 From: Jono Yang Date: Mon, 17 Jul 2023 13:28:38 -0700 Subject: [PATCH 01/69] Create script to update repo skeleton #80 Signed-off-by: Jono Yang --- etc/scripts/update_skeleton.py | 104 +++++++++++++++++++++++++++++++++ 1 file changed, 104 insertions(+) create mode 100644 etc/scripts/update_skeleton.py diff --git a/etc/scripts/update_skeleton.py b/etc/scripts/update_skeleton.py new file mode 100644 index 0000000..635898b --- /dev/null +++ b/etc/scripts/update_skeleton.py @@ -0,0 +1,104 @@ +#!/usr/bin/env python +# -*- coding: utf-8 -*- +# +# Copyright (c) nexB Inc. and others. All rights reserved. +# ScanCode is a trademark of nexB Inc. +# SPDX-License-Identifier: Apache-2.0 +# See http://www.apache.org/licenses/LICENSE-2.0 for the license text. +# See https://github.com/nexB/skeleton for support or download. +# See https://aboutcode.org for more information about nexB OSS projects. +# + +from pathlib import Path +import os +import subprocess + +import click + + +NEXB_PUBLIC_REPO_NAMES=[ + "aboutcode-toolkit", + "ahocode", + "bitcode", + "clearcode-toolkit", + "commoncode", + "container-inspector", + "debian-inspector", + "deltacode", + "elf-inspector", + "extractcode", + "fetchcode", + "gemfileparser2", + "gh-issue-sandbox", + "go-inspector", + "heritedcode", + "license-expression", + "license_copyright_pipeline", + "nuget-inspector", + "pip-requirements-parser", + "plugincode", + "purldb", + "pygmars", + "python-inspector", + "sanexml", + "saneyaml", + "scancode-analyzer", + "scancode-toolkit-contrib", + "scancode-toolkit-reference-scans", + "thirdparty-toolkit", + "tracecode-toolkit", + "tracecode-toolkit-strace", + "turbo-spdx", + "typecode", + "univers", +] + + +@click.command() +@click.help_option("-h", "--help") +def update_skeleton_files(repo_names=NEXB_PUBLIC_REPO_NAMES): + """ + Update project files of nexB projects that use the skeleton + + This script will: + - Clone the repo + - Add the skeleton repo as a new origin + - Create a new branch named "update-skeleton-files" + - Merge in the new skeleton files into the "update-skeleton-files" branch + + The user will need to save merge commit messages that pop up when running + this script in addition to resolving the merge conflicts on repos that have + them. + """ + + # Create working directory + work_dir_path = Path("/tmp/update_skeleton/") + if not os.path.exists(work_dir_path): + os.makedirs(work_dir_path, exist_ok=True) + + for repo_name in repo_names: + # Move to work directory + os.chdir(work_dir_path) + + # Clone repo + repo_git = f"git@github.com:nexB/{repo_name}.git" + subprocess.run(["git", "clone", repo_git]) + + # Go into cloned repo + os.chdir(work_dir_path / repo_name) + + # Add skeleton as an origin + subprocess.run(["git", "remote", "add", "skeleton", "git@github.com:nexB/skeleton.git"]) + + # Fetch skeleton files + subprocess.run(["git", "fetch", "skeleton"]) + + # Create and checkout new branch + subprocess.run(["git", "checkout", "-b", "update-skeleton-files"]) + + # Merge skeleton files into the repo + subprocess.run(["git", "merge", "skeleton/main", "--allow-unrelated-histories"]) + + +if __name__ == "__main__": + update_skeleton_files() From a92905297acf39ecd820bfb133f8670c39b40c97 Mon Sep 17 00:00:00 2001 From: Ayan Sinha Mahapatra Date: Fri, 17 Jan 2025 20:07:25 +0100 Subject: [PATCH 02/69] Drop deprecated macos-12 runner --- azure-pipelines.yml | 8 -------- 1 file changed, 8 deletions(-) diff --git a/azure-pipelines.yml b/azure-pipelines.yml index c2a3b52..39601e6 100644 --- a/azure-pipelines.yml +++ b/azure-pipelines.yml @@ -23,14 +23,6 @@ jobs: test_suites: all: venv/bin/pytest -n 2 -vvs - - template: etc/ci/azure-posix.yml - parameters: - job_name: macos12_cpython - image_name: macOS-12 - python_versions: ['3.8', '3.9', '3.10', '3.11', '3.12'] - test_suites: - all: venv/bin/pytest -n 2 -vvs - - template: etc/ci/azure-posix.yml parameters: job_name: macos13_cpython From 4af4fce3cc57d001c6c26f77d477cd44cef2ffef Mon Sep 17 00:00:00 2001 From: Ayan Sinha Mahapatra Date: Sat, 15 Feb 2025 00:09:49 +0530 Subject: [PATCH 03/69] Update CI/Actions runners Signed-off-by: Ayan Sinha Mahapatra --- .github/workflows/docs-ci.yml | 8 ++++---- .github/workflows/pypi-release.yml | 18 +++++++++--------- azure-pipelines.yml | 22 +++++++++++----------- 3 files changed, 24 insertions(+), 24 deletions(-) diff --git a/.github/workflows/docs-ci.yml b/.github/workflows/docs-ci.yml index 8c2abfe..621de4b 100644 --- a/.github/workflows/docs-ci.yml +++ b/.github/workflows/docs-ci.yml @@ -4,19 +4,19 @@ on: [push, pull_request] jobs: build: - runs-on: ubuntu-22.04 + runs-on: ubuntu-24.04 strategy: max-parallel: 4 matrix: - python-version: [3.9] + python-version: [3.12] steps: - name: Checkout code - uses: actions/checkout@v3 + uses: actions/checkout@v4 - name: Set up Python ${{ matrix.python-version }} - uses: actions/setup-python@v4 + uses: actions/setup-python@v5 with: python-version: ${{ matrix.python-version }} diff --git a/.github/workflows/pypi-release.yml b/.github/workflows/pypi-release.yml index d2206c8..a66c9c8 100644 --- a/.github/workflows/pypi-release.yml +++ b/.github/workflows/pypi-release.yml @@ -21,14 +21,14 @@ on: jobs: build-pypi-distribs: name: Build and publish library to PyPI - runs-on: ubuntu-22.04 + runs-on: ubuntu-24.04 steps: - uses: actions/checkout@v4 - name: Set up Python - uses: actions/setup-python@v4 + uses: actions/setup-python@v5 with: - python-version: 3.9 + python-version: 3.12 - name: Install pypa/build run: python -m pip install build --user @@ -37,7 +37,7 @@ jobs: run: python -m build --sdist --wheel --outdir dist/ - name: Upload built archives - uses: actions/upload-artifact@v3 + uses: actions/upload-artifact@v4 with: name: pypi_archives path: dist/* @@ -47,17 +47,17 @@ jobs: name: Create GH release needs: - build-pypi-distribs - runs-on: ubuntu-22.04 + runs-on: ubuntu-24.04 steps: - name: Download built archives - uses: actions/download-artifact@v3 + uses: actions/download-artifact@v4 with: name: pypi_archives path: dist - name: Create GH release - uses: softprops/action-gh-release@v1 + uses: softprops/action-gh-release@v2 with: draft: true files: dist/* @@ -67,11 +67,11 @@ jobs: name: Create PyPI release needs: - create-gh-release - runs-on: ubuntu-22.04 + runs-on: ubuntu-24.04 steps: - name: Download built archives - uses: actions/download-artifact@v3 + uses: actions/download-artifact@v4 with: name: pypi_archives path: dist diff --git a/azure-pipelines.yml b/azure-pipelines.yml index 39601e6..a220f2b 100644 --- a/azure-pipelines.yml +++ b/azure-pipelines.yml @@ -9,17 +9,17 @@ jobs: - template: etc/ci/azure-posix.yml parameters: - job_name: ubuntu20_cpython - image_name: ubuntu-20.04 - python_versions: ['3.8', '3.9', '3.10', '3.11', '3.12'] + job_name: ubuntu22_cpython + image_name: ubuntu-22.04 + python_versions: ['3.9', '3.10', '3.11', '3.12', '3.13'] test_suites: all: venv/bin/pytest -n 2 -vvs - template: etc/ci/azure-posix.yml parameters: - job_name: ubuntu22_cpython - image_name: ubuntu-22.04 - python_versions: ['3.8', '3.9', '3.10', '3.11', '3.12'] + job_name: ubuntu24_cpython + image_name: ubuntu-24.04 + python_versions: ['3.9', '3.10', '3.11', '3.12', '3.13'] test_suites: all: venv/bin/pytest -n 2 -vvs @@ -27,7 +27,7 @@ jobs: parameters: job_name: macos13_cpython image_name: macOS-13 - python_versions: ['3.8', '3.9', '3.10', '3.11', '3.12'] + python_versions: ['3.9', '3.10', '3.11', '3.12', '3.13'] test_suites: all: venv/bin/pytest -n 2 -vvs @@ -35,7 +35,7 @@ jobs: parameters: job_name: macos14_cpython_arm64 image_name: macOS-14 - python_versions: ['3.8', '3.9', '3.10', '3.11', '3.12'] + python_versions: ['3.9', '3.10', '3.11', '3.12', '3.13'] test_suites: all: venv/bin/pytest -n 2 -vvs @@ -43,7 +43,7 @@ jobs: parameters: job_name: macos14_cpython image_name: macOS-14-large - python_versions: ['3.8', '3.8', '3.9', '3.10', '3.12'] + python_versions: ['3.9', '3.10', '3.11', '3.12', '3.13'] test_suites: all: venv/bin/pytest -n 2 -vvs @@ -51,7 +51,7 @@ jobs: parameters: job_name: win2019_cpython image_name: windows-2019 - python_versions: ['3.8', '3.9', '3.10', '3.11', '3.12'] + python_versions: ['3.9', '3.10', '3.11', '3.12', '3.13'] test_suites: all: venv\Scripts\pytest -n 2 -vvs @@ -59,6 +59,6 @@ jobs: parameters: job_name: win2022_cpython image_name: windows-2022 - python_versions: ['3.8', '3.9', '3.10', '3.11', '3.12'] + python_versions: ['3.9', '3.10', '3.11', '3.12', '3.13'] test_suites: all: venv\Scripts\pytest -n 2 -vvs From 320ec21daa249ceae0c07787f9e52134b3ad06ab Mon Sep 17 00:00:00 2001 From: Jono Yang Date: Thu, 27 Mar 2025 14:54:31 -0700 Subject: [PATCH 04/69] Replace black and isort with ruff * Use ruff config and Make commands from scancode.io Signed-off-by: Jono Yang --- Makefile | 27 ++++++++++++--------------- pyproject.toml | 37 +++++++++++++++++++++++++++++++++++++ setup.cfg | 3 +-- 3 files changed, 50 insertions(+), 17 deletions(-) diff --git a/Makefile b/Makefile index 94451b3..1738b20 100644 --- a/Makefile +++ b/Makefile @@ -17,27 +17,24 @@ dev: @echo "-> Configure the development envt." ./configure --dev -isort: - @echo "-> Apply isort changes to ensure proper imports ordering" - ${VENV}/bin/isort --sl -l 100 src tests setup.py - -black: - @echo "-> Apply black code formatter" - ${VENV}/bin/black -l 100 src tests setup.py - doc8: @echo "-> Run doc8 validation" @${ACTIVATE} doc8 --max-line-length 100 --ignore-path docs/_build/ --quiet docs/ -valid: isort black +valid: + @echo "-> Run Ruff format" + @${ACTIVATE} ruff format + @echo "-> Run Ruff linter" + @${ACTIVATE} ruff check --fix check: - @echo "-> Run pycodestyle (PEP8) validation" - @${ACTIVATE} pycodestyle --max-line-length=100 --exclude=.eggs,venv,lib,thirdparty,docs,migrations,settings.py,.cache . - @echo "-> Run isort imports ordering validation" - @${ACTIVATE} isort --sl --check-only -l 100 setup.py src tests . - @echo "-> Run black validation" - @${ACTIVATE} black --check --check -l 100 src tests setup.py + @echo "-> Run Ruff linter validation (pycodestyle, bandit, isort, and more)" + @${ACTIVATE} ruff check + @echo "-> Run Ruff format validation" + @${ACTIVATE} ruff format --check + @$(MAKE) doc8 + @echo "-> Run ABOUT files validation" + @${ACTIVATE} about check etc/ clean: @echo "-> Clean the Python env" diff --git a/pyproject.toml b/pyproject.toml index cde7907..01e60fc 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -50,3 +50,40 @@ addopts = [ "--strict-markers", "--doctest-modules" ] + +[tool.ruff] +line-length = 88 +extend-exclude = [] +target-version = "py310" + +[tool.ruff.lint] +# Rules: https://docs.astral.sh/ruff/rules/ +select = [ + "E", # pycodestyle + "W", # pycodestyle warnings + "D", # pydocstyle + "F", # Pyflakes + "UP", # pyupgrade + "S", # flake8-bandit + "I", # isort + "C9", # McCabe complexity +] +ignore = ["D1", "D203", "D205", "D212", "D400", "D415"] + +[tool.ruff.lint.isort] +force-single-line = true +sections = { django = ["django"] } +section-order = [ + "future", + "standard-library", + "django", + "third-party", + "first-party", + "local-folder", +] + +[tool.ruff.lint.mccabe] +max-complexity = 10 + +[tool.ruff.lint.per-file-ignores] +# Place paths of files to be ignored by ruff here diff --git a/setup.cfg b/setup.cfg index ef7d369..aaec643 100644 --- a/setup.cfg +++ b/setup.cfg @@ -54,8 +54,7 @@ testing = aboutcode-toolkit >= 7.0.2 pycodestyle >= 2.8.0 twine - black - isort + ruff docs = Sphinx>=5.0.2 From d4e29c36c21ab81797604911cdeaea83d80e8088 Mon Sep 17 00:00:00 2001 From: Philippe Ombredanne Date: Sat, 29 Mar 2025 00:46:06 +0100 Subject: [PATCH 05/69] Use org standard 100 line length Signed-off-by: Philippe Ombredanne --- pyproject.toml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pyproject.toml b/pyproject.toml index 01e60fc..cea91bd 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -52,7 +52,7 @@ addopts = [ ] [tool.ruff] -line-length = 88 +line-length = 100 extend-exclude = [] target-version = "py310" From 6c028f7219ae876ea62074ae435e574525e205d6 Mon Sep 17 00:00:00 2001 From: Philippe Ombredanne Date: Sat, 29 Mar 2025 08:40:28 +0100 Subject: [PATCH 06/69] Lint all common code directories Signed-off-by: Philippe Ombredanne --- pyproject.toml | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/pyproject.toml b/pyproject.toml index cea91bd..9e62736 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -55,6 +55,14 @@ addopts = [ line-length = 100 extend-exclude = [] target-version = "py310" +include = [ + "pyproject.toml", + "src/**/*.py", + "etc/**/*.py", + "test/**/*.py", + "doc/**/*", + "*.py" +] [tool.ruff.lint] # Rules: https://docs.astral.sh/ruff/rules/ From 233f3edabfbab390029fb9f1842bf43766b04583 Mon Sep 17 00:00:00 2001 From: Philippe Ombredanne Date: Sat, 29 Mar 2025 09:07:47 +0100 Subject: [PATCH 07/69] Remove unused targets Signed-off-by: Philippe Ombredanne --- Makefile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Makefile b/Makefile index 1738b20..930e801 100644 --- a/Makefile +++ b/Makefile @@ -48,4 +48,4 @@ docs: rm -rf docs/_build/ @${ACTIVATE} sphinx-build docs/ docs/_build/ -.PHONY: conf dev check valid black isort clean test docs +.PHONY: conf dev check valid clean test docs From 55545bf7a1a8f119a560c7f548ce5a460f39f37d Mon Sep 17 00:00:00 2001 From: Philippe Ombredanne Date: Sat, 29 Mar 2025 11:03:05 +0100 Subject: [PATCH 08/69] Improve import sorting Signed-off-by: Philippe Ombredanne --- etc/scripts/check_thirdparty.py | 1 - etc/scripts/fetch_thirdparty.py | 2 +- etc/scripts/test_utils_pip_compatibility_tags.py | 3 +-- etc/scripts/utils_dejacode.py | 1 - etc/scripts/utils_pip_compatibility_tags.py | 14 ++++++-------- etc/scripts/utils_thirdparty.py | 3 +-- pyproject.toml | 7 ++++++- 7 files changed, 15 insertions(+), 16 deletions(-) diff --git a/etc/scripts/check_thirdparty.py b/etc/scripts/check_thirdparty.py index 2daded9..62dbb14 100644 --- a/etc/scripts/check_thirdparty.py +++ b/etc/scripts/check_thirdparty.py @@ -12,7 +12,6 @@ import utils_thirdparty - @click.command() @click.option( "-d", diff --git a/etc/scripts/fetch_thirdparty.py b/etc/scripts/fetch_thirdparty.py index 3f9ff52..30d376c 100644 --- a/etc/scripts/fetch_thirdparty.py +++ b/etc/scripts/fetch_thirdparty.py @@ -16,8 +16,8 @@ import click -import utils_thirdparty import utils_requirements +import utils_thirdparty TRACE = False TRACE_DEEP = False diff --git a/etc/scripts/test_utils_pip_compatibility_tags.py b/etc/scripts/test_utils_pip_compatibility_tags.py index 98187c5..a33b8b3 100644 --- a/etc/scripts/test_utils_pip_compatibility_tags.py +++ b/etc/scripts/test_utils_pip_compatibility_tags.py @@ -25,14 +25,13 @@ WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. """ -from unittest.mock import patch import sysconfig +from unittest.mock import patch import pytest import utils_pip_compatibility_tags - @pytest.mark.parametrize( "version_info, expected", [ diff --git a/etc/scripts/utils_dejacode.py b/etc/scripts/utils_dejacode.py index 652252d..c71543f 100644 --- a/etc/scripts/utils_dejacode.py +++ b/etc/scripts/utils_dejacode.py @@ -14,7 +14,6 @@ import requests import saneyaml - from packvers import version as packaging_version """ diff --git a/etc/scripts/utils_pip_compatibility_tags.py b/etc/scripts/utils_pip_compatibility_tags.py index af42a0c..de0ac95 100644 --- a/etc/scripts/utils_pip_compatibility_tags.py +++ b/etc/scripts/utils_pip_compatibility_tags.py @@ -27,14 +27,12 @@ import re -from packvers.tags import ( - compatible_tags, - cpython_tags, - generic_tags, - interpreter_name, - interpreter_version, - mac_platforms, -) +from packvers.tags import compatible_tags +from packvers.tags import cpython_tags +from packvers.tags import generic_tags +from packvers.tags import interpreter_name +from packvers.tags import interpreter_version +from packvers.tags import mac_platforms _osx_arch_pat = re.compile(r"(.+)_(\d+)_(\d+)_(.+)") diff --git a/etc/scripts/utils_thirdparty.py b/etc/scripts/utils_thirdparty.py index 46dc728..b0295ec 100644 --- a/etc/scripts/utils_thirdparty.py +++ b/etc/scripts/utils_thirdparty.py @@ -25,14 +25,13 @@ import packageurl import requests import saneyaml +import utils_pip_compatibility_tags from commoncode import fileutils from commoncode.hash import multi_checksums from commoncode.text import python_safe_name from packvers import tags as packaging_tags from packvers import version as packaging_version -import utils_pip_compatibility_tags - """ Utilities to manage Python thirparty libraries source, binaries and metadata in local directories and remote repositories. diff --git a/pyproject.toml b/pyproject.toml index 9e62736..ba55770 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -76,10 +76,15 @@ select = [ "I", # isort "C9", # McCabe complexity ] -ignore = ["D1", "D203", "D205", "D212", "D400", "D415"] +ignore = ["D1", "D200", "D203", "D205", "D212", "D400", "D415"] [tool.ruff.lint.isort] force-single-line = true +lines-after-imports = 1 +default-section = "first-party" +known-first-party = ["src", "tests", "etc/scripts/**/*.py"] +known-third-party = ["click", "pytest"] + sections = { django = ["django"] } section-order = [ "future", From 0b63e5073b6b1cdc0960abe35060ad0fdb67b665 Mon Sep 17 00:00:00 2001 From: Philippe Ombredanne Date: Sat, 29 Mar 2025 21:35:16 +0100 Subject: [PATCH 09/69] Apply small code updates Signed-off-by: Philippe Ombredanne --- etc/scripts/utils_requirements.py | 20 ++++++++----- etc/scripts/utils_thirdparty.py | 48 +++++++++++++++---------------- 2 files changed, 37 insertions(+), 31 deletions(-) diff --git a/etc/scripts/utils_requirements.py b/etc/scripts/utils_requirements.py index 1c50239..a9ac223 100644 --- a/etc/scripts/utils_requirements.py +++ b/etc/scripts/utils_requirements.py @@ -57,21 +57,25 @@ def get_required_name_version(requirement, with_unpinned=False): >>> assert get_required_name_version("fooA==1.2.3.DEV1") == ("fooa", "1.2.3.dev1") >>> assert get_required_name_version("foo==1.2.3", with_unpinned=False) == ("foo", "1.2.3") >>> assert get_required_name_version("foo", with_unpinned=True) == ("foo", "") - >>> assert get_required_name_version("foo>=1.2", with_unpinned=True) == ("foo", ""), get_required_name_version("foo>=1.2") + >>> expected = ("foo", ""), get_required_name_version("foo>=1.2") + >>> assert get_required_name_version("foo>=1.2", with_unpinned=True) == expected >>> try: ... assert not get_required_name_version("foo", with_unpinned=False) ... except Exception as e: ... assert "Requirement version must be pinned" in str(e) """ requirement = requirement and "".join(requirement.lower().split()) - assert requirement, f"specifier is required is empty:{requirement!r}" + if not requirement: + raise ValueError(f"specifier is required is empty:{requirement!r}") name, operator, version = split_req(requirement) - assert name, f"Name is required: {requirement}" + if not name: + raise ValueError(f"Name is required: {requirement}") is_pinned = operator == "==" if with_unpinned: version = "" else: - assert is_pinned and version, f"Requirement version must be pinned: {requirement}" + if not is_pinned and version: + raise ValueError(f"Requirement version must be pinned: {requirement}") return name, version @@ -120,7 +124,7 @@ def get_installed_reqs(site_packages_dir): # setuptools, pip args = ["pip", "freeze", "--exclude-editable", "--all", "--path", site_packages_dir] - return subprocess.check_output(args, encoding="utf-8") + return subprocess.check_output(args, encoding="utf-8") # noqa: S603 comparators = ( @@ -150,9 +154,11 @@ def split_req(req): >>> assert split_req("foo >= 1.2.3 ") == ("foo", ">=", "1.2.3"), split_req("foo >= 1.2.3 ") >>> assert split_req("foo>=1.2") == ("foo", ">=", "1.2"), split_req("foo>=1.2") """ - assert req + if not req: + raise ValueError("req is required") # do not allow multiple constraints and tags - assert not any(c in req for c in ",;") + if not any(c in req for c in ",;"): + raise Exception(f"complex requirements with : or ; not supported: {req}") req = "".join(req.split()) if not any(c in req for c in comparators): return req, "", "" diff --git a/etc/scripts/utils_thirdparty.py b/etc/scripts/utils_thirdparty.py index b0295ec..6d5ffdc 100644 --- a/etc/scripts/utils_thirdparty.py +++ b/etc/scripts/utils_thirdparty.py @@ -1,5 +1,4 @@ #!/usr/bin/env python -# -*- coding: utf-8 -*- # # Copyright (c) nexB Inc. and others. All rights reserved. # ScanCode is a trademark of nexB Inc. @@ -559,7 +558,8 @@ def download(self, dest_dir=THIRDPARTY_DIR): Download this distribution into `dest_dir` directory. Return the fetched filename. """ - assert self.filename + if not self.filename: + raise ValueError(f"self.filename has no value but is required: {self.filename!r}") if TRACE_DEEP: print( f"Fetching distribution of {self.name}=={self.version}:", @@ -829,10 +829,9 @@ def fetch_license_files(self, dest_dir=THIRDPARTY_DIR, use_cached_index=False): urls = LinksRepository.from_url( use_cached_index=use_cached_index).links errors = [] - extra_lic_names = [l.get("file") - for l in self.extra_data.get("licenses", {})] + extra_lic_names = [lic.get("file") for lic in self.extra_data.get("licenses", {})] extra_lic_names += [self.extra_data.get("license_file")] - extra_lic_names = [ln for ln in extra_lic_names if ln] + extra_lic_names = [eln for eln in extra_lic_names if eln] lic_names = [f"{key}.LICENSE" for key in self.get_license_keys()] for filename in lic_names + extra_lic_names: floc = os.path.join(dest_dir, filename) @@ -853,7 +852,7 @@ def fetch_license_files(self, dest_dir=THIRDPARTY_DIR, use_cached_index=False): if TRACE: print(f"Fetched license from remote: {lic_url}") - except: + except Exception: try: # try licensedb second lic_url = f"{LICENSEDB_API_URL}/{filename}" @@ -866,8 +865,9 @@ def fetch_license_files(self, dest_dir=THIRDPARTY_DIR, use_cached_index=False): if TRACE: print(f"Fetched license from licensedb: {lic_url}") - except: - msg = f'No text for license {filename} in expression "{self.license_expression}" from {self}' + except Exception: + msg = f"No text for license {filename} in expression " + f"{self.license_expression!r} from {self}" print(msg) errors.append(msg) @@ -1009,7 +1009,7 @@ def get_license_link_for_filename(filename, urls): exception if no link is found or if there are more than one link for that file name. """ - path_or_url = [l for l in urls if l.endswith(f"/{filename}")] + path_or_url = [url for url in urls if url.endswith(f"/{filename}")] if not path_or_url: raise Exception(f"Missing link to file: {filename}") if not len(path_or_url) == 1: @@ -1140,7 +1140,6 @@ def to_filename(self): @attr.attributes class Wheel(Distribution): - """ Represents a wheel file. @@ -1301,7 +1300,7 @@ def is_pure(self): def is_pure_wheel(filename): try: return Wheel.from_filename(filename).is_pure() - except: + except Exception: return False @@ -1489,8 +1488,7 @@ def dists_from_paths_or_urls(cls, paths_or_urls): ) except InvalidDistributionFilename: if TRACE_DEEP: - print( - f" Skipping invalid distribution from: {path_or_url}") + print(f" Skipping invalid distribution from: {path_or_url}") continue return dists @@ -1500,8 +1498,7 @@ def get_distributions(self): """ if self.sdist: yield self.sdist - for wheel in self.wheels: - yield wheel + yield from self.wheels def get_url_for_filename(self, filename): """ @@ -1632,7 +1629,8 @@ class PypiSimpleRepository: type=dict, default=attr.Factory(lambda: defaultdict(dict)), metadata=dict( - help="Mapping of {name: {version: PypiPackage, version: PypiPackage, etc} available in this repo" + help="Mapping of {name: {version: PypiPackage, version: PypiPackage, etc} " + "available in this repo" ), ) @@ -1647,7 +1645,8 @@ class PypiSimpleRepository: type=bool, default=False, metadata=dict( - help="If True, use any existing on-disk cached PyPI index files. Otherwise, fetch and cache." + help="If True, use any existing on-disk cached PyPI index files. " + "Otherwise, fetch and cache." ), ) @@ -1656,7 +1655,8 @@ def _get_package_versions_map(self, name): Return a mapping of all available PypiPackage version for this package name. The mapping may be empty. It is ordered by version from oldest to newest """ - assert name + if not name: + raise ValueError(f"name is required: {name!r}") normalized_name = NameVer.normalize_name(name) versions = self.packages[normalized_name] if not versions and normalized_name not in self.fetched_package_normalized_names: @@ -1713,7 +1713,7 @@ def fetch_links(self, normalized_name): ) links = collect_urls(text) # TODO: keep sha256 - links = [l.partition("#sha256=") for l in links] + links = [link.partition("#sha256=") for link in links] links = [url for url, _, _sha256 in links] return links @@ -1936,7 +1936,7 @@ def get_remote_file_content( # several redirects and that we can ignore content there. A HEAD request may # not get us this last header print(f" DOWNLOADING: {url}") - with requests.get(url, allow_redirects=True, stream=True, headers=headers) as response: + with requests.get(url, allow_redirects=True, stream=True, headers=headers) as response: # noqa: S113 status = response.status_code if status != requests.codes.ok: # NOQA if status == 429 and _delay < 20: @@ -2161,7 +2161,7 @@ def call(args, verbose=TRACE): """ if TRACE_DEEP: print("Calling:", " ".join(args)) - with subprocess.Popen( + with subprocess.Popen( # noqa: S603 args, stdout=subprocess.PIPE, stderr=subprocess.PIPE, encoding="utf-8" ) as process: @@ -2227,7 +2227,7 @@ def download_wheels_with_pip( cli_args.extend(["--requirement", req_file]) if TRACE: - print(f"Downloading wheels using command:", " ".join(cli_args)) + print("Downloading wheels using command:", " ".join(cli_args)) existing = set(os.listdir(dest_dir)) error = False @@ -2260,7 +2260,7 @@ def download_wheels_with_pip( def check_about(dest_dir=THIRDPARTY_DIR): try: - subprocess.check_output(f"venv/bin/about check {dest_dir}".split()) + subprocess.check_output(f"venv/bin/about check {dest_dir}".split()) # noqa: S603 except subprocess.CalledProcessError as cpe: print() print("Invalid ABOUT files:") @@ -2312,5 +2312,5 @@ def get_license_expression(declared_licenses): return get_only_expression_from_extracted_license(declared_licenses) except ImportError: # Scancode is not installed, clean and join all the licenses - lics = [python_safe_name(l).lower() for l in declared_licenses] + lics = [python_safe_name(lic).lower() for lic in declared_licenses] return " AND ".join(lics).lower() From 092f545f5b87442ae22884cb4d5381883343a1c2 Mon Sep 17 00:00:00 2001 From: Philippe Ombredanne Date: Sat, 29 Mar 2025 21:42:03 +0100 Subject: [PATCH 10/69] Format code Signed-off-by: Philippe Ombredanne --- etc/scripts/check_thirdparty.py | 3 +- etc/scripts/fetch_thirdparty.py | 26 ++++----- etc/scripts/gen_pypi_simple.py | 4 +- etc/scripts/utils_dejacode.py | 15 +++--- etc/scripts/utils_requirements.py | 9 ++-- etc/scripts/utils_thirdparty.py | 90 +++++++++++-------------------- 6 files changed, 50 insertions(+), 97 deletions(-) diff --git a/etc/scripts/check_thirdparty.py b/etc/scripts/check_thirdparty.py index 62dbb14..1aa4e28 100644 --- a/etc/scripts/check_thirdparty.py +++ b/etc/scripts/check_thirdparty.py @@ -16,8 +16,7 @@ @click.option( "-d", "--dest", - type=click.Path(exists=True, readable=True, - path_type=str, file_okay=False), + type=click.Path(exists=True, readable=True, path_type=str, file_okay=False), required=True, help="Path to the thirdparty directory to check.", ) diff --git a/etc/scripts/fetch_thirdparty.py b/etc/scripts/fetch_thirdparty.py index 30d376c..c224683 100644 --- a/etc/scripts/fetch_thirdparty.py +++ b/etc/scripts/fetch_thirdparty.py @@ -55,8 +55,7 @@ "-d", "--dest", "dest_dir", - type=click.Path(exists=True, readable=True, - path_type=str, file_okay=False), + type=click.Path(exists=True, readable=True, path_type=str, file_okay=False), metavar="DIR", default=utils_thirdparty.THIRDPARTY_DIR, show_default=True, @@ -121,7 +120,7 @@ show_default=False, multiple=True, help="Package name(s) that come only in sdist format (no wheels). " - "The command will not fail and exit if no wheel exists for these names", + "The command will not fail and exit if no wheel exists for these names", ) @click.option( "--wheel-only", @@ -132,7 +131,7 @@ show_default=False, multiple=True, help="Package name(s) that come only in wheel format (no sdist). " - "The command will not fail and exit if no sdist exists for these names", + "The command will not fail and exit if no sdist exists for these names", ) @click.option( "--no-dist", @@ -143,7 +142,7 @@ show_default=False, multiple=True, help="Package name(s) that do not come either in wheel or sdist format. " - "The command will not fail and exit if no distribution exists for these names", + "The command will not fail and exit if no distribution exists for these names", ) @click.help_option("-h", "--help") def fetch_thirdparty( @@ -225,8 +224,7 @@ def fetch_thirdparty( environments = None if wheels: evts = itertools.product(python_versions, operating_systems) - environments = [utils_thirdparty.Environment.from_pyver_and_os( - pyv, os) for pyv, os in evts] + environments = [utils_thirdparty.Environment.from_pyver_and_os(pyv, os) for pyv, os in evts] # Collect PyPI repos repos = [] @@ -250,7 +248,6 @@ def fetch_thirdparty( print(f"Processing: {name} @ {version}") if wheels: for environment in environments: - if TRACE: print(f" ==> Fetching wheel for envt: {environment}") @@ -262,14 +259,11 @@ def fetch_thirdparty( repos=repos, ) if not fetched: - wheels_or_sdist_not_found[f"{name}=={version}"].append( - environment) + wheels_or_sdist_not_found[f"{name}=={version}"].append(environment) if TRACE: print(f" NOT FOUND") - if (sdists or - (f"{name}=={version}" in wheels_or_sdist_not_found and name in sdist_only) - ): + if sdists or (f"{name}=={version}" in wheels_or_sdist_not_found and name in sdist_only): if TRACE: print(f" ==> Fetching sdist: {name}=={version}") @@ -292,8 +286,7 @@ def fetch_thirdparty( sdist_missing = sdists and "sdist" in dists and not name in wheel_only if sdist_missing: mia.append(f"SDist missing: {nv} {dists}") - wheels_missing = wheels and any( - d for d in dists if d != "sdist") and not name in sdist_only + wheels_missing = wheels and any(d for d in dists if d != "sdist") and not name in sdist_only if wheels_missing: mia.append(f"Wheels missing: {nv} {dists}") @@ -303,8 +296,7 @@ def fetch_thirdparty( raise Exception(mia) print(f"==> FETCHING OR CREATING ABOUT AND LICENSE FILES") - utils_thirdparty.fetch_abouts_and_licenses( - dest_dir=dest_dir, use_cached_index=use_cached_index) + utils_thirdparty.fetch_abouts_and_licenses(dest_dir=dest_dir, use_cached_index=use_cached_index) utils_thirdparty.clean_about_files(dest_dir=dest_dir) # check for problems diff --git a/etc/scripts/gen_pypi_simple.py b/etc/scripts/gen_pypi_simple.py index 214d90d..cfe68e6 100644 --- a/etc/scripts/gen_pypi_simple.py +++ b/etc/scripts/gen_pypi_simple.py @@ -69,7 +69,6 @@ def get_package_name_from_filename(filename): raise InvalidDistributionFilename(filename) elif filename.endswith(wheel_ext): - wheel_info = get_wheel_from_filename(filename) if not wheel_info: @@ -200,11 +199,10 @@ def build_pypi_index(directory, base_url="https://thirdparty.aboutcode.org/pypi" simple_html_index = [ "", "PyPI Simple Index", - '' '', + '', ] for pkg_file in directory.iterdir(): - pkg_filename = pkg_file.name if ( diff --git a/etc/scripts/utils_dejacode.py b/etc/scripts/utils_dejacode.py index c71543f..cd39cda 100644 --- a/etc/scripts/utils_dejacode.py +++ b/etc/scripts/utils_dejacode.py @@ -32,8 +32,7 @@ def can_do_api_calls(): if not DEJACODE_API_KEY and DEJACODE_API_URL: - print( - "DejaCode DEJACODE_API_KEY and DEJACODE_API_URL not configured. Doing nothing") + print("DejaCode DEJACODE_API_KEY and DEJACODE_API_URL not configured. Doing nothing") return False else: return True @@ -68,8 +67,7 @@ def get_package_data(distribution): return results[0] elif len_results > 1: - print( - f"More than 1 entry exists, review at: {DEJACODE_API_URL_PACKAGES}") + print(f"More than 1 entry exists, review at: {DEJACODE_API_URL_PACKAGES}") else: print("Could not find package:", distribution.download_url) @@ -150,12 +148,11 @@ def find_latest_dejacode_package(distribution): # there was no exact match, find the latest version # TODO: consider the closest version rather than the latest # or the version that has the best data - with_versions = [(packaging_version.parse(p["version"]), p) - for p in packages] + with_versions = [(packaging_version.parse(p["version"]), p) for p in packages] with_versions = sorted(with_versions) latest_version, latest_package_version = sorted(with_versions)[-1] print( - f"Found DejaCode latest version: {latest_version} " f"for dist: {distribution.package_url}", + f"Found DejaCode latest version: {latest_version} for dist: {distribution.package_url}", ) return latest_package_version @@ -181,7 +178,7 @@ def create_dejacode_package(distribution): } fields_to_carry_over = [ - "download_url" "type", + "download_urltype", "namespace", "name", "version", @@ -209,5 +206,5 @@ def create_dejacode_package(distribution): if response.status_code != 201: raise Exception(f"Error, cannot create package for: {distribution}") - print(f'New Package created at: {new_package_data["absolute_url"]}') + print(f"New Package created at: {new_package_data['absolute_url']}") return new_package_data diff --git a/etc/scripts/utils_requirements.py b/etc/scripts/utils_requirements.py index a9ac223..167bc9f 100644 --- a/etc/scripts/utils_requirements.py +++ b/etc/scripts/utils_requirements.py @@ -106,8 +106,7 @@ def lock_dev_requirements( all_req_nvs = get_required_name_versions(all_req_lines) dev_only_req_nvs = {n: v for n, v in all_req_nvs if n not in main_names} - new_reqs = "\n".join( - f"{n}=={v}" for n, v in sorted(dev_only_req_nvs.items())) + new_reqs = "\n".join(f"{n}=={v}" for n, v in sorted(dev_only_req_nvs.items())) with open(dev_requirements_file, "w") as fo: fo.write(new_reqs) @@ -118,12 +117,10 @@ def get_installed_reqs(site_packages_dir): as a text. """ if not os.path.exists(site_packages_dir): - raise Exception( - f"site_packages directory: {site_packages_dir!r} does not exists") + raise Exception(f"site_packages directory: {site_packages_dir!r} does not exists") # Also include these packages in the output with --all: wheel, distribute, # setuptools, pip - args = ["pip", "freeze", "--exclude-editable", - "--all", "--path", site_packages_dir] + args = ["pip", "freeze", "--exclude-editable", "--all", "--path", site_packages_dir] return subprocess.check_output(args, encoding="utf-8") # noqa: S603 diff --git a/etc/scripts/utils_thirdparty.py b/etc/scripts/utils_thirdparty.py index 6d5ffdc..4ea1bab 100644 --- a/etc/scripts/utils_thirdparty.py +++ b/etc/scripts/utils_thirdparty.py @@ -243,11 +243,9 @@ def download_wheel(name, version, environment, dest_dir=THIRDPARTY_DIR, repos=tu package = repo.get_package_version(name=name, version=version) if not package: if TRACE_DEEP: - print( - f" download_wheel: No package in {repo.index_url} for {name}=={version}") + print(f" download_wheel: No package in {repo.index_url} for {name}=={version}") continue - supported_wheels = list( - package.get_supported_wheels(environment=environment)) + supported_wheels = list(package.get_supported_wheels(environment=environment)) if not supported_wheels: if TRACE_DEEP: print( @@ -291,8 +289,7 @@ def download_sdist(name, version, dest_dir=THIRDPARTY_DIR, repos=tuple()): if not package: if TRACE_DEEP: - print( - f" download_sdist: No package in {repo.index_url} for {name}=={version}") + print(f" download_sdist: No package in {repo.index_url} for {name}=={version}") continue sdist = package.sdist if not sdist: @@ -301,8 +298,7 @@ def download_sdist(name, version, dest_dir=THIRDPARTY_DIR, repos=tuple()): continue if TRACE_DEEP: - print( - f" download_sdist: Getting sdist from index (or cache): {sdist.download_url}") + print(f" download_sdist: Getting sdist from index (or cache): {sdist.download_url}") fetched_sdist_filename = package.sdist.download(dest_dir=dest_dir) if fetched_sdist_filename: @@ -357,7 +353,6 @@ def sorted(cls, namevers): @attr.attributes class Distribution(NameVer): - # field names that can be updated from another Distribution or mapping updatable_fields = [ "license_expression", @@ -535,8 +530,7 @@ def get_best_download_url(self, repos=tuple()): repos = DEFAULT_PYPI_REPOS for repo in repos: - package = repo.get_package_version( - name=self.name, version=self.version) + package = repo.get_package_version(name=self.name, version=self.version) if not package: if TRACE: print( @@ -776,8 +770,7 @@ def load_remote_about_data(self): if notice_text: about_data["notice_text"] = notice_text except RemoteNotFetchedException: - print( - f"Failed to fetch NOTICE file: {self.notice_download_url}") + print(f"Failed to fetch NOTICE file: {self.notice_download_url}") return self.load_about_data(about_data) def get_checksums(self, dest_dir=THIRDPARTY_DIR): @@ -826,8 +819,7 @@ def fetch_license_files(self, dest_dir=THIRDPARTY_DIR, use_cached_index=False): Fetch license files if missing in `dest_dir`. Return True if license files were fetched. """ - urls = LinksRepository.from_url( - use_cached_index=use_cached_index).links + urls = LinksRepository.from_url(use_cached_index=use_cached_index).links errors = [] extra_lic_names = [lic.get("file") for lic in self.extra_data.get("licenses", {})] extra_lic_names += [self.extra_data.get("license_file")] @@ -840,8 +832,7 @@ def fetch_license_files(self, dest_dir=THIRDPARTY_DIR, use_cached_index=False): try: # try remotely first - lic_url = get_license_link_for_filename( - filename=filename, urls=urls) + lic_url = get_license_link_for_filename(filename=filename, urls=urls) fetch_and_save( path_or_url=lic_url, @@ -919,8 +910,7 @@ def load_pkginfo_data(self, dest_dir=THIRDPARTY_DIR): c for c in classifiers if c.startswith("License") ] license_expression = get_license_expression(declared_license) - other_classifiers = [ - c for c in classifiers if not c.startswith("License")] + other_classifiers = [c for c in classifiers if not c.startswith("License")] holder = raw_data["Author"] holder_contact = raw_data["Author-email"] @@ -962,8 +952,7 @@ def update(self, data, overwrite=False, keep_extra=True): package_url = data.get("package_url") if package_url: purl_from_data = packageurl.PackageURL.from_string(package_url) - purl_from_self = packageurl.PackageURL.from_string( - self.package_url) + purl_from_self = packageurl.PackageURL.from_string(self.package_url) if purl_from_data != purl_from_self: print( f"Invalid dist update attempt, no same same purl with dist: " @@ -1013,8 +1002,7 @@ def get_license_link_for_filename(filename, urls): if not path_or_url: raise Exception(f"Missing link to file: {filename}") if not len(path_or_url) == 1: - raise Exception( - f"Multiple links to file: {filename}: \n" + "\n".join(path_or_url)) + raise Exception(f"Multiple links to file: {filename}: \n" + "\n".join(path_or_url)) return path_or_url[0] @@ -1102,7 +1090,6 @@ def get_sdist_name_ver_ext(filename): @attr.attributes class Sdist(Distribution): - extension = attr.ib( repr=False, type=str, @@ -1407,8 +1394,7 @@ def packages_from_dir(cls, directory): """ base = os.path.abspath(directory) - paths = [os.path.join(base, f) - for f in os.listdir(base) if f.endswith(EXTENSIONS)] + paths = [os.path.join(base, f) for f in os.listdir(base) if f.endswith(EXTENSIONS)] if TRACE_ULTRA_DEEP: print("packages_from_dir: paths:", paths) @@ -1469,8 +1455,7 @@ def dists_from_paths_or_urls(cls, paths_or_urls): dists = [] if TRACE_ULTRA_DEEP: print(" ###paths_or_urls:", paths_or_urls) - installable = [f for f in paths_or_urls if f.endswith( - EXTENSIONS_INSTALLABLE)] + installable = [f for f in paths_or_urls if f.endswith(EXTENSIONS_INSTALLABLE)] for path_or_url in installable: try: dist = Distribution.from_path_or_url(path_or_url) @@ -1536,8 +1521,7 @@ class Environment: implementation = attr.ib( type=str, default="cp", - metadata=dict( - help="Python implementation supported by this environment."), + metadata=dict(help="Python implementation supported by this environment."), repr=False, ) @@ -1551,8 +1535,7 @@ class Environment: platforms = attr.ib( type=list, default=attr.Factory(list), - metadata=dict( - help="List of platform tags supported by this environment."), + metadata=dict(help="List of platform tags supported by this environment."), repr=False, ) @@ -1637,8 +1620,7 @@ class PypiSimpleRepository: fetched_package_normalized_names = attr.ib( type=set, default=attr.Factory(set), - metadata=dict( - help="A set of already fetched package normalized names."), + metadata=dict(help="A set of already fetched package normalized names."), ) use_cached_index = attr.ib( @@ -1671,12 +1653,10 @@ def _get_package_versions_map(self, name): self.packages[normalized_name] = versions except RemoteNotFetchedException as e: if TRACE: - print( - f"failed to fetch package name: {name} from: {self.index_url}:\n{e}") + print(f"failed to fetch package name: {name} from: {self.index_url}:\n{e}") if not versions and TRACE: - print( - f"WARNING: package {name} not found in repo: {self.index_url}") + print(f"WARNING: package {name} not found in repo: {self.index_url}") return versions @@ -1861,8 +1841,7 @@ def get(self, path_or_url, as_text=True, force=False): if force or not os.path.exists(cached): if TRACE_DEEP: print(f" FILE CACHE MISS: {path_or_url}") - content = get_file_content( - path_or_url=path_or_url, as_text=as_text) + content = get_file_content(path_or_url=path_or_url, as_text=as_text) wmode = "w" if as_text else "wb" with open(cached, wmode) as fo: fo.write(content) @@ -1884,8 +1863,7 @@ def get_file_content(path_or_url, as_text=True): if path_or_url.startswith("https://"): if TRACE_DEEP: print(f"Fetching: {path_or_url}") - _headers, content = get_remote_file_content( - url=path_or_url, as_text=as_text) + _headers, content = get_remote_file_content(url=path_or_url, as_text=as_text) return content elif path_or_url.startswith("file://") or ( @@ -1936,7 +1914,7 @@ def get_remote_file_content( # several redirects and that we can ignore content there. A HEAD request may # not get us this last header print(f" DOWNLOADING: {url}") - with requests.get(url, allow_redirects=True, stream=True, headers=headers) as response: # noqa: S113 + with requests.get(url, allow_redirects=True, stream=True, headers=headers) as response: # noqa: S113 status = response.status_code if status != requests.codes.ok: # NOQA if status == 429 and _delay < 20: @@ -1951,8 +1929,7 @@ def get_remote_file_content( ) else: - raise RemoteNotFetchedException( - f"Failed HTTP request from {url} with {status}") + raise RemoteNotFetchedException(f"Failed HTTP request from {url} with {status}") if headers_only: return response.headers, None @@ -2043,8 +2020,7 @@ def get_other_dists(_package, _dist): # if has key data we may look to improve later, but we can move on if local_dist.has_key_metadata(): local_dist.save_about_and_notice_files(dest_dir=dest_dir) - local_dist.fetch_license_files( - dest_dir=dest_dir, use_cached_index=use_cached_index) + local_dist.fetch_license_files(dest_dir=dest_dir, use_cached_index=use_cached_index) continue # lets try to get from another dist of the same local package @@ -2056,8 +2032,7 @@ def get_other_dists(_package, _dist): # if has key data we may look to improve later, but we can move on if local_dist.has_key_metadata(): local_dist.save_about_and_notice_files(dest_dir=dest_dir) - local_dist.fetch_license_files( - dest_dir=dest_dir, use_cached_index=use_cached_index) + local_dist.fetch_license_files(dest_dir=dest_dir, use_cached_index=use_cached_index) continue # try to get another version of the same package that is not our version @@ -2068,8 +2043,7 @@ def get_other_dists(_package, _dist): ] other_local_version = other_local_packages and other_local_packages[-1] if other_local_version: - latest_local_dists = list( - other_local_version.get_distributions()) + latest_local_dists = list(other_local_version.get_distributions()) for latest_local_dist in latest_local_dists: latest_local_dist.load_about_data(dest_dir=dest_dir) if not latest_local_dist.has_key_metadata(): @@ -2095,8 +2069,7 @@ def get_other_dists(_package, _dist): # if has key data we may look to improve later, but we can move on if local_dist.has_key_metadata(): local_dist.save_about_and_notice_files(dest_dir=dest_dir) - local_dist.fetch_license_files( - dest_dir=dest_dir, use_cached_index=use_cached_index) + local_dist.fetch_license_files(dest_dir=dest_dir, use_cached_index=use_cached_index) continue # try to get a latest version of the same package that is not our version @@ -2137,8 +2110,7 @@ def get_other_dists(_package, _dist): # if local_dist.has_key_metadata() or not local_dist.has_key_metadata(): local_dist.save_about_and_notice_files(dest_dir) - lic_errs = local_dist.fetch_license_files( - dest_dir, use_cached_index=use_cached_index) + lic_errs = local_dist.fetch_license_files(dest_dir, use_cached_index=use_cached_index) if not local_dist.has_key_metadata(): print(f"Unable to add essential ABOUT data for: {local_dist}") @@ -2161,10 +2133,9 @@ def call(args, verbose=TRACE): """ if TRACE_DEEP: print("Calling:", " ".join(args)) - with subprocess.Popen( # noqa: S603 + with subprocess.Popen( # noqa: S603 args, stdout=subprocess.PIPE, stderr=subprocess.PIPE, encoding="utf-8" ) as process: - stdouts = [] while True: line = process.stdout.readline() @@ -2260,7 +2231,7 @@ def download_wheels_with_pip( def check_about(dest_dir=THIRDPARTY_DIR): try: - subprocess.check_output(f"venv/bin/about check {dest_dir}".split()) # noqa: S603 + subprocess.check_output(f"venv/bin/about check {dest_dir}".split()) # noqa: S603 except subprocess.CalledProcessError as cpe: print() print("Invalid ABOUT files:") @@ -2286,8 +2257,7 @@ def find_problems( for dist in package.get_distributions(): dist.load_about_data(dest_dir=dest_dir) - abpth = os.path.abspath(os.path.join( - dest_dir, dist.about_filename)) + abpth = os.path.abspath(os.path.join(dest_dir, dist.about_filename)) if not dist.has_key_metadata(): print(f" Missing key ABOUT data in file://{abpth}") if "classifiers" in dist.extra_data: From d05665ad44a50b71f66b974ad24c81f7443e8180 Mon Sep 17 00:00:00 2001 From: Philippe Ombredanne Date: Sat, 29 Mar 2025 22:02:19 +0100 Subject: [PATCH 11/69] Apply cosmetic refactorings Signed-off-by: Philippe Ombredanne --- docs/source/conf.py | 3 ++- etc/scripts/check_thirdparty.py | 4 +--- etc/scripts/fetch_thirdparty.py | 17 ++++++++--------- etc/scripts/gen_pypi_simple.py | 15 +++++++-------- etc/scripts/gen_requirements.py | 4 ++-- etc/scripts/gen_requirements_dev.py | 4 ++-- .../test_utils_pip_compatibility_tags.py | 9 +++++---- etc/scripts/utils_dejacode.py | 9 +++++---- etc/scripts/utils_pip_compatibility_tags.py | 8 +++++--- etc/scripts/utils_requirements.py | 3 +-- etc/scripts/utils_thirdparty.py | 3 ++- 11 files changed, 40 insertions(+), 39 deletions(-) diff --git a/docs/source/conf.py b/docs/source/conf.py index 8c88fa2..8aad829 100644 --- a/docs/source/conf.py +++ b/docs/source/conf.py @@ -94,7 +94,8 @@ html_show_sphinx = True # Define CSS and HTML abbreviations used in .rst files. These are examples. -# .. role:: is used to refer to styles defined in _static/theme_overrides.css and is used like this: :red:`text` +# .. role:: is used to refer to styles defined in _static/theme_overrides.css +# and is used like this: :red:`text` rst_prolog = """ .. |psf| replace:: Python Software Foundation diff --git a/etc/scripts/check_thirdparty.py b/etc/scripts/check_thirdparty.py index 1aa4e28..bb8347a 100644 --- a/etc/scripts/check_thirdparty.py +++ b/etc/scripts/check_thirdparty.py @@ -1,5 +1,4 @@ #!/usr/bin/env python -# -*- coding: utf-8 -*- # # Copyright (c) nexB Inc. and others. All rights reserved. # ScanCode is a trademark of nexB Inc. @@ -41,8 +40,7 @@ def check_thirdparty_dir( """ Check a thirdparty directory for problems and print these on screen. """ - # check for problems - print(f"==> CHECK FOR PROBLEMS") + print("==> CHECK FOR PROBLEMS") utils_thirdparty.find_problems( dest_dir=dest, report_missing_sources=sdists, diff --git a/etc/scripts/fetch_thirdparty.py b/etc/scripts/fetch_thirdparty.py index c224683..76a19a6 100644 --- a/etc/scripts/fetch_thirdparty.py +++ b/etc/scripts/fetch_thirdparty.py @@ -1,5 +1,4 @@ #!/usr/bin/env python -# -*- coding: utf-8 -*- # # Copyright (c) nexB Inc. and others. All rights reserved. # ScanCode is a trademark of nexB Inc. @@ -10,7 +9,6 @@ # import itertools -import os import sys from collections import defaultdict @@ -109,7 +107,8 @@ @click.option( "--use-cached-index", is_flag=True, - help="Use on disk cached PyPI indexes list of packages and versions and do not refetch if present.", + help="Use on disk cached PyPI indexes list of packages and versions and " + "do not refetch if present.", ) @click.option( "--sdist-only", @@ -261,7 +260,7 @@ def fetch_thirdparty( if not fetched: wheels_or_sdist_not_found[f"{name}=={version}"].append(environment) if TRACE: - print(f" NOT FOUND") + print(" NOT FOUND") if sdists or (f"{name}=={version}" in wheels_or_sdist_not_found and name in sdist_only): if TRACE: @@ -276,17 +275,17 @@ def fetch_thirdparty( if not fetched: wheels_or_sdist_not_found[f"{name}=={version}"].append("sdist") if TRACE: - print(f" NOT FOUND") + print(" NOT FOUND") mia = [] for nv, dists in wheels_or_sdist_not_found.items(): name, _, version = nv.partition("==") if name in no_dist: continue - sdist_missing = sdists and "sdist" in dists and not name in wheel_only + sdist_missing = sdists and "sdist" in dists and name not in wheel_only if sdist_missing: mia.append(f"SDist missing: {nv} {dists}") - wheels_missing = wheels and any(d for d in dists if d != "sdist") and not name in sdist_only + wheels_missing = wheels and any(d for d in dists if d != "sdist") and name not in sdist_only if wheels_missing: mia.append(f"Wheels missing: {nv} {dists}") @@ -295,12 +294,12 @@ def fetch_thirdparty( print(m) raise Exception(mia) - print(f"==> FETCHING OR CREATING ABOUT AND LICENSE FILES") + print("==> FETCHING OR CREATING ABOUT AND LICENSE FILES") utils_thirdparty.fetch_abouts_and_licenses(dest_dir=dest_dir, use_cached_index=use_cached_index) utils_thirdparty.clean_about_files(dest_dir=dest_dir) # check for problems - print(f"==> CHECK FOR PROBLEMS") + print("==> CHECK FOR PROBLEMS") utils_thirdparty.find_problems( dest_dir=dest_dir, report_missing_sources=sdists, diff --git a/etc/scripts/gen_pypi_simple.py b/etc/scripts/gen_pypi_simple.py index cfe68e6..89d0626 100644 --- a/etc/scripts/gen_pypi_simple.py +++ b/etc/scripts/gen_pypi_simple.py @@ -1,5 +1,4 @@ #!/usr/bin/env python -# -*- coding: utf-8 -*- # SPDX-License-Identifier: BSD-2-Clause-Views AND MIT # Copyright (c) 2010 David Wolever . All rights reserved. @@ -132,7 +131,7 @@ def build_links_package_index(packages_by_package_name, base_url): Return an HTML document as string which is a links index of all packages """ document = [] - header = f""" + header = """ Links for all packages @@ -177,13 +176,13 @@ def simple_index_entry(self, base_url): def build_pypi_index(directory, base_url="https://thirdparty.aboutcode.org/pypi"): """ - Using a ``directory`` directory of wheels and sdists, create the a PyPI - simple directory index at ``directory``/simple/ populated with the proper - PyPI simple index directory structure crafted using symlinks. + Create the a PyPI simple directory index using a ``directory`` directory of wheels and sdists in + the direvctory at ``directory``/simple/ populated with the proper PyPI simple index directory + structure crafted using symlinks. - WARNING: The ``directory``/simple/ directory is removed if it exists. - NOTE: in addition to the a PyPI simple index.html there is also a links.html - index file generated which is suitable to use with pip's --find-links + WARNING: The ``directory``/simple/ directory is removed if it exists. NOTE: in addition to the a + PyPI simple index.html there is also a links.html index file generated which is suitable to use + with pip's --find-links """ directory = Path(directory) diff --git a/etc/scripts/gen_requirements.py b/etc/scripts/gen_requirements.py index 2b65ae8..1b87944 100644 --- a/etc/scripts/gen_requirements.py +++ b/etc/scripts/gen_requirements.py @@ -1,5 +1,4 @@ #!/usr/bin/env python -# -*- coding: utf-8 -*- # # Copyright (c) nexB Inc. and others. All rights reserved. # ScanCode is a trademark of nexB Inc. @@ -34,7 +33,8 @@ def gen_requirements(): type=pathlib.Path, required=True, metavar="DIR", - help="Path to the 'site-packages' directory where wheels are installed such as lib/python3.6/site-packages", + help="Path to the 'site-packages' directory where wheels are installed " + "such as lib/python3.12/site-packages", ) parser.add_argument( "-r", diff --git a/etc/scripts/gen_requirements_dev.py b/etc/scripts/gen_requirements_dev.py index 5db1c48..8548205 100644 --- a/etc/scripts/gen_requirements_dev.py +++ b/etc/scripts/gen_requirements_dev.py @@ -1,5 +1,4 @@ #!/usr/bin/env python -# -*- coding: utf-8 -*- # # Copyright (c) nexB Inc. and others. All rights reserved. # ScanCode is a trademark of nexB Inc. @@ -36,7 +35,8 @@ def gen_dev_requirements(): type=pathlib.Path, required=True, metavar="DIR", - help='Path to the "site-packages" directory where wheels are installed such as lib/python3.6/site-packages', + help="Path to the 'site-packages' directory where wheels are installed " + "such as lib/python3.12/site-packages", ) parser.add_argument( "-d", diff --git a/etc/scripts/test_utils_pip_compatibility_tags.py b/etc/scripts/test_utils_pip_compatibility_tags.py index a33b8b3..de4b706 100644 --- a/etc/scripts/test_utils_pip_compatibility_tags.py +++ b/etc/scripts/test_utils_pip_compatibility_tags.py @@ -1,4 +1,5 @@ -"""Generate and work with PEP 425 Compatibility Tags. +""" +Generate and work with PEP 425 Compatibility Tags. copied from pip-20.3.1 pip/tests/unit/test_utils_compatibility_tags.py download_url: https://raw.githubusercontent.com/pypa/pip/20.3.1/tests/unit/test_utils_compatibility_tags.py @@ -50,7 +51,7 @@ def test_version_info_to_nodot(version_info, expected): assert actual == expected -class Testcompatibility_tags(object): +class Testcompatibility_tags: def mock_get_config_var(self, **kwd): """ Patch sysconfig.get_config_var for arbitrary keys. @@ -81,7 +82,7 @@ def test_no_hyphen_tag(self): assert "-" not in tag.platform -class TestManylinux2010Tags(object): +class TestManylinux2010Tags: @pytest.mark.parametrize( "manylinux2010,manylinux1", [ @@ -104,7 +105,7 @@ def test_manylinux2010_implies_manylinux1(self, manylinux2010, manylinux1): assert arches[:2] == [manylinux2010, manylinux1] -class TestManylinux2014Tags(object): +class TestManylinux2014Tags: @pytest.mark.parametrize( "manylinuxA,manylinuxB", [ diff --git a/etc/scripts/utils_dejacode.py b/etc/scripts/utils_dejacode.py index cd39cda..b6bff51 100644 --- a/etc/scripts/utils_dejacode.py +++ b/etc/scripts/utils_dejacode.py @@ -1,5 +1,4 @@ #!/usr/bin/env python -# -*- coding: utf-8 -*- # # Copyright (c) nexB Inc. and others. All rights reserved. # ScanCode is a trademark of nexB Inc. @@ -25,7 +24,7 @@ DEJACODE_API_URL_PACKAGES = f"{DEJACODE_API_URL}packages/" DEJACODE_API_HEADERS = { - "Authorization": "Token {}".format(DEJACODE_API_KEY), + "Authorization": f"Token {DEJACODE_API_KEY}", "Accept": "application/json; indent=4", } @@ -50,6 +49,7 @@ def fetch_dejacode_packages(params): DEJACODE_API_URL_PACKAGES, params=params, headers=DEJACODE_API_HEADERS, + timeout=10, ) return response.json()["results"] @@ -93,7 +93,7 @@ def update_with_dejacode_about_data(distribution): if package_data: package_api_url = package_data["api_url"] about_url = f"{package_api_url}about" - response = requests.get(about_url, headers=DEJACODE_API_HEADERS) + response = requests.get(about_url, headers=DEJACODE_API_HEADERS, timeout=10) # note that this is YAML-formatted about_text = response.json()["about_data"] about_data = saneyaml.load(about_text) @@ -113,7 +113,7 @@ def fetch_and_save_about_files(distribution, dest_dir="thirdparty"): if package_data: package_api_url = package_data["api_url"] about_url = f"{package_api_url}about_files" - response = requests.get(about_url, headers=DEJACODE_API_HEADERS) + response = requests.get(about_url, headers=DEJACODE_API_HEADERS, timeout=10) about_zip = response.content with io.BytesIO(about_zip) as zf: with zipfile.ZipFile(zf) as zi: @@ -201,6 +201,7 @@ def create_dejacode_package(distribution): DEJACODE_API_URL_PACKAGES, data=new_package_payload, headers=DEJACODE_API_HEADERS, + timeout=10, ) new_package_data = response.json() if response.status_code != 201: diff --git a/etc/scripts/utils_pip_compatibility_tags.py b/etc/scripts/utils_pip_compatibility_tags.py index de0ac95..dd954bc 100644 --- a/etc/scripts/utils_pip_compatibility_tags.py +++ b/etc/scripts/utils_pip_compatibility_tags.py @@ -1,4 +1,5 @@ -"""Generate and work with PEP 425 Compatibility Tags. +""" +Generate and work with PEP 425 Compatibility Tags. copied from pip-20.3.1 pip/_internal/utils/compatibility_tags.py download_url: https://github.com/pypa/pip/blob/20.3.1/src/pip/_internal/utils/compatibility_tags.py @@ -130,7 +131,7 @@ def _get_custom_interpreter(implementation=None, version=None): implementation = interpreter_name() if version is None: version = interpreter_version() - return "{}{}".format(implementation, version) + return f"{implementation}{version}" def get_supported( @@ -140,7 +141,8 @@ def get_supported( abis=None, # type: Optional[List[str]] ): # type: (...) -> List[Tag] - """Return a list of supported tags for each version specified in + """ + Return a list of supported tags for each version specified in `versions`. :param version: a string version, of the form "33" or "32", diff --git a/etc/scripts/utils_requirements.py b/etc/scripts/utils_requirements.py index 167bc9f..b9b2c0e 100644 --- a/etc/scripts/utils_requirements.py +++ b/etc/scripts/utils_requirements.py @@ -1,5 +1,4 @@ #!/usr/bin/env python -# -*- coding: utf-8 -*- # # Copyright (c) nexB Inc. and others. All rights reserved. # ScanCode is a trademark of nexB Inc. @@ -40,7 +39,7 @@ def get_required_name_versions(requirement_lines, with_unpinned=False): req_line = req_line.strip() if not req_line or req_line.startswith("#"): continue - if req_line.startswith("-") or (not with_unpinned and not "==" in req_line): + if req_line.startswith("-") or (not with_unpinned and "==" not in req_line): print(f"Requirement line is not supported: ignored: {req_line}") continue yield get_required_name_version(requirement=req_line, with_unpinned=with_unpinned) diff --git a/etc/scripts/utils_thirdparty.py b/etc/scripts/utils_thirdparty.py index 4ea1bab..aafc1d6 100644 --- a/etc/scripts/utils_thirdparty.py +++ b/etc/scripts/utils_thirdparty.py @@ -91,7 +91,8 @@ - parse requirement file - create a TODO queue of requirements to process -- done: create an empty map of processed binary requirements as {package name: (list of versions/tags} +- done: create an empty map of processed binary requirements as + {package name: (list of versions/tags} - while we have package reqs in TODO queue, process one requirement: From 63bcbf507e8a25f22853d56605c107e47c3673cc Mon Sep 17 00:00:00 2001 From: Philippe Ombredanne Date: Sat, 29 Mar 2025 22:05:23 +0100 Subject: [PATCH 12/69] Reformat test code Signed-off-by: Philippe Ombredanne --- .gitignore | 1 + pyproject.toml | 19 +++++++++++-------- tests/test_skeleton_codestyle.py | 25 ++++++++++++++++--------- 3 files changed, 28 insertions(+), 17 deletions(-) diff --git a/.gitignore b/.gitignore index 2d48196..8a93c94 100644 --- a/.gitignore +++ b/.gitignore @@ -72,3 +72,4 @@ tcl # Ignore Jupyter Notebook related temp files .ipynb_checkpoints/ +/.ruff_cache/ diff --git a/pyproject.toml b/pyproject.toml index ba55770..a872ab3 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -67,16 +67,17 @@ include = [ [tool.ruff.lint] # Rules: https://docs.astral.sh/ruff/rules/ select = [ - "E", # pycodestyle - "W", # pycodestyle warnings - "D", # pydocstyle - "F", # Pyflakes - "UP", # pyupgrade - "S", # flake8-bandit +# "E", # pycodestyle +# "W", # pycodestyle warnings +# "D", # pydocstyle +# "F", # Pyflakes +# "UP", # pyupgrade +# "S", # flake8-bandit "I", # isort - "C9", # McCabe complexity +# "C9", # McCabe complexity ] -ignore = ["D1", "D200", "D203", "D205", "D212", "D400", "D415"] +ignore = ["D1", "D200", "D202", "D203", "D205", "D212", "D400", "D415"] + [tool.ruff.lint.isort] force-single-line = true @@ -100,3 +101,5 @@ max-complexity = 10 [tool.ruff.lint.per-file-ignores] # Place paths of files to be ignored by ruff here +"tests/*" = ["S101"] +"test_*.py" = ["S101"] diff --git a/tests/test_skeleton_codestyle.py b/tests/test_skeleton_codestyle.py index b4ce8c1..8cd85c9 100644 --- a/tests/test_skeleton_codestyle.py +++ b/tests/test_skeleton_codestyle.py @@ -7,30 +7,37 @@ # See https://aboutcode.org for more information about nexB OSS projects. # +import configparser import subprocess import unittest -import configparser - class BaseTests(unittest.TestCase): def test_skeleton_codestyle(self): - """ - This test shouldn't run in proliferated repositories. - """ + # This test shouldn't run in proliferated repositories. + + # TODO: update with switch to pyproject.toml setup_cfg = configparser.ConfigParser() setup_cfg.read("setup.cfg") if setup_cfg["metadata"]["name"] != "skeleton": return - args = "venv/bin/black --check -l 100 setup.py etc tests" + commands = [ + ["venv/bin/ruff", "--check"], + ["venv/bin/ruff", "format", "--check"], + ] + command = None try: - subprocess.check_output(args.split()) + for command in commands: + subprocess.check_output(command) # noqa: S603 except subprocess.CalledProcessError as e: print("===========================================================") print(e.output) print("===========================================================") raise Exception( - "Black style check failed; please format the code using:\n" - " python -m black -l 100 setup.py etc tests", + f"Code style and linting command check failed: {' '.join(command)!r}.\n" + "You can check and format the code using:\n" + " make valid\n", + "OR:\n ruff format\n", + " ruff check --fix\n", e.output, ) from e From 9d1393a85303bf8cf92c9a25aa5cc50bdfd080d1 Mon Sep 17 00:00:00 2001 From: Philippe Ombredanne Date: Sat, 29 Mar 2025 22:08:25 +0100 Subject: [PATCH 13/69] Format code Signed-off-by: Philippe Ombredanne --- etc/scripts/check_thirdparty.py | 1 + etc/scripts/test_utils_pip_compatibility_tags.py | 1 + tests/test_skeleton_codestyle.py | 1 + 3 files changed, 3 insertions(+) diff --git a/etc/scripts/check_thirdparty.py b/etc/scripts/check_thirdparty.py index bb8347a..65ae595 100644 --- a/etc/scripts/check_thirdparty.py +++ b/etc/scripts/check_thirdparty.py @@ -11,6 +11,7 @@ import utils_thirdparty + @click.command() @click.option( "-d", diff --git a/etc/scripts/test_utils_pip_compatibility_tags.py b/etc/scripts/test_utils_pip_compatibility_tags.py index de4b706..0e9c360 100644 --- a/etc/scripts/test_utils_pip_compatibility_tags.py +++ b/etc/scripts/test_utils_pip_compatibility_tags.py @@ -33,6 +33,7 @@ import utils_pip_compatibility_tags + @pytest.mark.parametrize( "version_info, expected", [ diff --git a/tests/test_skeleton_codestyle.py b/tests/test_skeleton_codestyle.py index 8cd85c9..7135ac0 100644 --- a/tests/test_skeleton_codestyle.py +++ b/tests/test_skeleton_codestyle.py @@ -11,6 +11,7 @@ import subprocess import unittest + class BaseTests(unittest.TestCase): def test_skeleton_codestyle(self): # This test shouldn't run in proliferated repositories. From f10b783b6b6fe33032a7862352ed532294efdf14 Mon Sep 17 00:00:00 2001 From: Philippe Ombredanne Date: Sat, 29 Mar 2025 22:10:45 +0100 Subject: [PATCH 14/69] Refine ruff configuration Signed-off-by: Philippe Ombredanne --- pyproject.toml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/pyproject.toml b/pyproject.toml index a872ab3..0f8bd58 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -72,11 +72,11 @@ select = [ # "D", # pydocstyle # "F", # Pyflakes # "UP", # pyupgrade -# "S", # flake8-bandit + "S", # flake8-bandit "I", # isort # "C9", # McCabe complexity ] -ignore = ["D1", "D200", "D202", "D203", "D205", "D212", "D400", "D415"] +ignore = ["D1", "D200", "D202", "D203", "D205", "D212", "D400", "D415", "I001"] [tool.ruff.lint.isort] From 1d6c8f3bb8755aa7c9d2804240c01b0161417328 Mon Sep 17 00:00:00 2001 From: Philippe Ombredanne Date: Sat, 29 Mar 2025 22:54:01 +0100 Subject: [PATCH 15/69] Format doc Signed-off-by: Philippe Ombredanne --- AUTHORS.rst | 2 +- README.rst | 16 ++++++++++------ 2 files changed, 11 insertions(+), 7 deletions(-) diff --git a/AUTHORS.rst b/AUTHORS.rst index 51a19cc..16e2046 100644 --- a/AUTHORS.rst +++ b/AUTHORS.rst @@ -1,3 +1,3 @@ The following organizations or individuals have contributed to this repo: -- +- diff --git a/README.rst b/README.rst index 6cbd839..3d6cb4e 100644 --- a/README.rst +++ b/README.rst @@ -1,5 +1,6 @@ A Simple Python Project Skeleton ================================ + This repo attempts to standardize the structure of the Python-based project's repositories using modern Python packaging and configuration techniques. Using this `blog post`_ as inspiration, this repository serves as the base for @@ -47,16 +48,19 @@ Release Notes - 2022-03-04: - Synchronize configure and configure.bat scripts for sanity - Update CI operating system support with latest Azure OS images - - Streamline utility scripts in etc/scripts/ to create, fetch and manage third-party dependencies - There are now fewer scripts. See etc/scripts/README.rst for details + - Streamline utility scripts in etc/scripts/ to create, fetch and manage third-party + dependencies. There are now fewer scripts. See etc/scripts/README.rst for details - 2021-09-03: - - ``configure`` now requires pinned dependencies via the use of ``requirements.txt`` and ``requirements-dev.txt`` + - ``configure`` now requires pinned dependencies via the use of ``requirements.txt`` + and ``requirements-dev.txt`` - ``configure`` can now accept multiple options at once - Add utility scripts from scancode-toolkit/etc/release/ for use in generating project files - Rename virtual environment directory from ``tmp`` to ``venv`` - - Update README.rst with instructions for generating ``requirements.txt`` and ``requirements-dev.txt``, - as well as collecting dependencies as wheels and generating ABOUT files for them. + - Update README.rst with instructions for generating ``requirements.txt`` + and ``requirements-dev.txt``, as well as collecting dependencies as wheels and generating + ABOUT files for them. - 2021-05-11: - - Adopt new configure scripts from ScanCode TK that allows correct configuration of which Python version is used. + - Adopt new configure scripts from ScanCode TK that allows correct configuration of which + Python version is used. From 0213c1ea9a15ab94a854b8d7af27a1a036e393f4 Mon Sep 17 00:00:00 2001 From: Philippe Ombredanne Date: Sat, 29 Mar 2025 22:54:35 +0100 Subject: [PATCH 16/69] Run doc8 on all rst files Signed-off-by: Philippe Ombredanne --- Makefile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Makefile b/Makefile index 930e801..debc404 100644 --- a/Makefile +++ b/Makefile @@ -19,7 +19,7 @@ dev: doc8: @echo "-> Run doc8 validation" - @${ACTIVATE} doc8 --max-line-length 100 --ignore-path docs/_build/ --quiet docs/ + @${ACTIVATE} doc8 --max-line-length 100 --ignore-path docs/_build/ --quiet docs/ *.rst valid: @echo "-> Run Ruff format" From c112f2a9c20d58e986424f5f32bd259814fc8e3f Mon Sep 17 00:00:00 2001 From: Philippe Ombredanne Date: Sat, 29 Mar 2025 22:55:20 +0100 Subject: [PATCH 17/69] Enable doc style checks Signed-off-by: Philippe Ombredanne --- pyproject.toml | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/pyproject.toml b/pyproject.toml index 0f8bd58..51761ff 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -61,7 +61,8 @@ include = [ "etc/**/*.py", "test/**/*.py", "doc/**/*", - "*.py" + "*.py", + "." ] [tool.ruff.lint] @@ -69,10 +70,10 @@ include = [ select = [ # "E", # pycodestyle # "W", # pycodestyle warnings -# "D", # pydocstyle + "D", # pydocstyle # "F", # Pyflakes # "UP", # pyupgrade - "S", # flake8-bandit +# "S", # flake8-bandit "I", # isort # "C9", # McCabe complexity ] From 944b6c5371bea2ce0763fd26888de6436116d185 Mon Sep 17 00:00:00 2001 From: Philippe Ombredanne Date: Sat, 29 Mar 2025 00:34:26 +0100 Subject: [PATCH 18/69] Add support for new OS versions Signed-off-by: Philippe Ombredanne --- README.rst | 50 +++++++++++++++++++++++++++++++++++++++++---- azure-pipelines.yml | 36 ++++++++++++++++++++++++++++++-- 2 files changed, 80 insertions(+), 6 deletions(-) diff --git a/README.rst b/README.rst index 6cbd839..f848b4b 100644 --- a/README.rst +++ b/README.rst @@ -1,9 +1,11 @@ A Simple Python Project Skeleton ================================ -This repo attempts to standardize the structure of the Python-based project's -repositories using modern Python packaging and configuration techniques. -Using this `blog post`_ as inspiration, this repository serves as the base for -all new Python projects and is mergeable in existing repositories as well. + +This repo attempts to standardize the structure of the Python-based project's repositories using +modern Python packaging and configuration techniques that can then be applied to many repos. + +Using this `blog post`_ as inspiration, this repository serves as the base for all new Python +projects and is mergeable in existing repositories as well. .. _blog post: https://blog.jaraco.com/a-project-skeleton-for-python-projects/ @@ -13,6 +15,7 @@ Usage A brand new project ------------------- + .. code-block:: bash git init my-new-repo @@ -26,6 +29,7 @@ From here, you can make the appropriate changes to the files for your specific p Update an existing project --------------------------- + .. code-block:: bash cd my-existing-project @@ -41,17 +45,54 @@ More usage instructions can be found in ``docs/skeleton-usage.rst``. Release Notes ============= +- 2025-03-29: + + - Add support for beta macOS-15 + - Add support for beta windows-2025 + +- 2025-02-14: + + - Drop support for Python 3.8, add support in CI for Python 3.13, use Python 3.12 as default + version. + +- 2025-01-17: + + - Drop support for macOS-12, add support for macOS-14 + - Add support in CI for ubuntu-24.04 + - Add support in CI for Python 3.12 + +- 2024-08-20: + + - Update references of ownership from nexB to aboutcode-org + +- 2024-07-01: + + - Drop support for Python 3.8 + - Drop support for macOS-11, add support for macOS-14 + +- 2024-02-19: + + - Replace support in CI of default ubuntu-20.04 by ubuntu-22.04 + +- 2023-10-18: + + - Add dark mode support in documentation + - 2023-07-18: + - Add macOS-13 job in azure-pipelines.yml - 2022-03-04: + - Synchronize configure and configure.bat scripts for sanity - Update CI operating system support with latest Azure OS images - Streamline utility scripts in etc/scripts/ to create, fetch and manage third-party dependencies There are now fewer scripts. See etc/scripts/README.rst for details - 2021-09-03: + - ``configure`` now requires pinned dependencies via the use of ``requirements.txt`` and ``requirements-dev.txt`` + - ``configure`` can now accept multiple options at once - Add utility scripts from scancode-toolkit/etc/release/ for use in generating project files - Rename virtual environment directory from ``tmp`` to ``venv`` @@ -59,4 +100,5 @@ Release Notes as well as collecting dependencies as wheels and generating ABOUT files for them. - 2021-05-11: + - Adopt new configure scripts from ScanCode TK that allows correct configuration of which Python version is used. diff --git a/azure-pipelines.yml b/azure-pipelines.yml index a220f2b..80ae45b 100644 --- a/azure-pipelines.yml +++ b/azure-pipelines.yml @@ -26,11 +26,27 @@ jobs: - template: etc/ci/azure-posix.yml parameters: job_name: macos13_cpython + image_name: macOS-13-xlarge + python_versions: ['3.9', '3.10', '3.11', '3.12', '3.13'] + test_suites: + all: venv/bin/pytest -n 2 -vvs + + - template: etc/ci/azure-posix.yml + parameters: + job_name: macos13_cpython_arm64 image_name: macOS-13 python_versions: ['3.9', '3.10', '3.11', '3.12', '3.13'] test_suites: all: venv/bin/pytest -n 2 -vvs + - template: etc/ci/azure-posix.yml + parameters: + job_name: macos14_cpython + image_name: macOS-14-large + python_versions: ['3.9', '3.10', '3.11', '3.12', '3.13'] + test_suites: + all: venv/bin/pytest -n 2 -vvs + - template: etc/ci/azure-posix.yml parameters: job_name: macos14_cpython_arm64 @@ -41,8 +57,16 @@ jobs: - template: etc/ci/azure-posix.yml parameters: - job_name: macos14_cpython - image_name: macOS-14-large + job_name: macos15_cpython + image_name: macOS-15 + python_versions: ['3.9', '3.10', '3.11', '3.12', '3.13'] + test_suites: + all: venv/bin/pytest -n 2 -vvs + + - template: etc/ci/azure-posix.yml + parameters: + job_name: macos15_cpython_arm64 + image_name: macOS-15-large python_versions: ['3.9', '3.10', '3.11', '3.12', '3.13'] test_suites: all: venv/bin/pytest -n 2 -vvs @@ -62,3 +86,11 @@ jobs: python_versions: ['3.9', '3.10', '3.11', '3.12', '3.13'] test_suites: all: venv\Scripts\pytest -n 2 -vvs + + - template: etc/ci/azure-win.yml + parameters: + job_name: win2025_cpython + image_name: windows-2025 + python_versions: ['3.9', '3.10', '3.11', '3.12', '3.13'] + test_suites: + all: venv\Scripts\pytest -n 2 -vvs From 136af3912336616fbd2431a96230961517a2c356 Mon Sep 17 00:00:00 2001 From: Philippe Ombredanne Date: Sun, 30 Mar 2025 12:45:32 +0200 Subject: [PATCH 19/69] Update scripts aboutcode references Signed-off-by: Philippe Ombredanne --- etc/scripts/update_skeleton.py | 15 +++++++-------- 1 file changed, 7 insertions(+), 8 deletions(-) diff --git a/etc/scripts/update_skeleton.py b/etc/scripts/update_skeleton.py index 635898b..5705fc4 100644 --- a/etc/scripts/update_skeleton.py +++ b/etc/scripts/update_skeleton.py @@ -1,11 +1,10 @@ #!/usr/bin/env python -# -*- coding: utf-8 -*- # -# Copyright (c) nexB Inc. and others. All rights reserved. +# Copyright (c) nexB Inc. AboutCode, and others. All rights reserved. # ScanCode is a trademark of nexB Inc. # SPDX-License-Identifier: Apache-2.0 # See http://www.apache.org/licenses/LICENSE-2.0 for the license text. -# See https://github.com/nexB/skeleton for support or download. +# See https://github.com/aboutcode-org/skeleton for support or download. # See https://aboutcode.org for more information about nexB OSS projects. # @@ -16,7 +15,7 @@ import click -NEXB_PUBLIC_REPO_NAMES=[ +ABOUTCODE_PUBLIC_REPO_NAMES=[ "aboutcode-toolkit", "ahocode", "bitcode", @@ -56,9 +55,9 @@ @click.command() @click.help_option("-h", "--help") -def update_skeleton_files(repo_names=NEXB_PUBLIC_REPO_NAMES): +def update_skeleton_files(repo_names=ABOUTCODE_PUBLIC_REPO_NAMES): """ - Update project files of nexB projects that use the skeleton + Update project files of AboutCode projects that use the skeleton This script will: - Clone the repo @@ -81,14 +80,14 @@ def update_skeleton_files(repo_names=NEXB_PUBLIC_REPO_NAMES): os.chdir(work_dir_path) # Clone repo - repo_git = f"git@github.com:nexB/{repo_name}.git" + repo_git = f"git@github.com:aboutcode-org/{repo_name}.git" subprocess.run(["git", "clone", repo_git]) # Go into cloned repo os.chdir(work_dir_path / repo_name) # Add skeleton as an origin - subprocess.run(["git", "remote", "add", "skeleton", "git@github.com:nexB/skeleton.git"]) + subprocess.run(["git", "remote", "add", "skeleton", "git@github.com:aboutcode-org/skeleton.git"]) # Fetch skeleton files subprocess.run(["git", "fetch", "skeleton"]) From da8eff0383611df60311b8bac599657450eaeb52 Mon Sep 17 00:00:00 2001 From: Philippe Ombredanne Date: Sun, 30 Mar 2025 14:40:36 +0200 Subject: [PATCH 20/69] Do not format more test data Signed-off-by: Philippe Ombredanne --- pyproject.toml | 15 +++++++++++++++ 1 file changed, 15 insertions(+) diff --git a/pyproject.toml b/pyproject.toml index 51761ff..7d807eb 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -60,10 +60,25 @@ include = [ "src/**/*.py", "etc/**/*.py", "test/**/*.py", + "tests/**/*.py", "doc/**/*", + "docs/**/*", "*.py", "." ] +# ignore test data and testfiles: they should never be linted nor formatted +exclude = [ +# main style + "**/tests/data/**/*", +# scancode-toolkit + "**/tests/*/data/**/*", +# dejacode, purldb + "**/tests/testfiles/**/*", +# vulnerablecode, fetchcode + "**/tests/*/test_data/**/*", + "**/tests/test_data/**/*", +] + [tool.ruff.lint] # Rules: https://docs.astral.sh/ruff/rules/ From 4f9e936d452acc3822df8d3f932cbd7071b31d72 Mon Sep 17 00:00:00 2001 From: Philippe Ombredanne Date: Sun, 30 Mar 2025 14:58:36 +0200 Subject: [PATCH 21/69] Do not treat rst as Python Signed-off-by: Philippe Ombredanne --- pyproject.toml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/pyproject.toml b/pyproject.toml index 7d807eb..5e16b56 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -61,8 +61,8 @@ include = [ "etc/**/*.py", "test/**/*.py", "tests/**/*.py", - "doc/**/*", - "docs/**/*", + "doc/**/*.py", + "docs/**/*.py", "*.py", "." ] From a2809fb28c60b54aec0c367285acacdea1cb03a8 Mon Sep 17 00:00:00 2001 From: Philippe Ombredanne Date: Sun, 30 Mar 2025 16:41:57 +0200 Subject: [PATCH 22/69] Combine testing and docs extra for simplicity Signed-off-by: Philippe Ombredanne --- configure | 2 -- configure.bat | 4 ---- setup.cfg | 3 --- 3 files changed, 9 deletions(-) diff --git a/configure b/configure index 22d9288..83fd203 100755 --- a/configure +++ b/configure @@ -30,7 +30,6 @@ CLI_ARGS=$1 # Requirement arguments passed to pip and used by default or with --dev. REQUIREMENTS="--editable . --constraint requirements.txt" DEV_REQUIREMENTS="--editable .[testing] --constraint requirements.txt --constraint requirements-dev.txt" -DOCS_REQUIREMENTS="--editable .[docs] --constraint requirements.txt" # where we create a virtualenv VIRTUALENV_DIR=venv @@ -185,7 +184,6 @@ while getopts :-: optchar; do help ) cli_help;; clean ) find_python && clean;; dev ) CFG_REQUIREMENTS="$DEV_REQUIREMENTS";; - docs ) CFG_REQUIREMENTS="$DOCS_REQUIREMENTS";; esac;; esac done diff --git a/configure.bat b/configure.bat index 5b9a9d6..18b3703 100644 --- a/configure.bat +++ b/configure.bat @@ -28,7 +28,6 @@ @rem # Requirement arguments passed to pip and used by default or with --dev. set "REQUIREMENTS=--editable . --constraint requirements.txt" set "DEV_REQUIREMENTS=--editable .[testing] --constraint requirements.txt --constraint requirements-dev.txt" -set "DOCS_REQUIREMENTS=--editable .[docs] --constraint requirements.txt" @rem # where we create a virtualenv set "VIRTUALENV_DIR=venv" @@ -76,9 +75,6 @@ if not "%1" == "" ( if "%1" EQU "--dev" ( set "CFG_REQUIREMENTS=%DEV_REQUIREMENTS%" ) - if "%1" EQU "--docs" ( - set "CFG_REQUIREMENTS=%DOCS_REQUIREMENTS%" - ) shift goto again ) diff --git a/setup.cfg b/setup.cfg index aaec643..ad8e0d8 100644 --- a/setup.cfg +++ b/setup.cfg @@ -55,8 +55,6 @@ testing = pycodestyle >= 2.8.0 twine ruff - -docs = Sphinx>=5.0.2 sphinx-rtd-theme>=1.0.0 sphinx-reredirects >= 0.1.2 @@ -64,4 +62,3 @@ docs = sphinx-autobuild sphinx-rtd-dark-mode>=1.3.0 sphinx-copybutton - From 43b96c28baaa1621d24b6f5791c6d915d2edc5f3 Mon Sep 17 00:00:00 2001 From: Philippe Ombredanne Date: Sun, 30 Mar 2025 17:18:19 +0200 Subject: [PATCH 23/69] Refine checking of docs with doc8 Signed-off-by: Philippe Ombredanne --- Makefile | 2 +- pyproject.toml | 7 +++++++ 2 files changed, 8 insertions(+), 1 deletion(-) diff --git a/Makefile b/Makefile index debc404..d21a2f9 100644 --- a/Makefile +++ b/Makefile @@ -19,7 +19,7 @@ dev: doc8: @echo "-> Run doc8 validation" - @${ACTIVATE} doc8 --max-line-length 100 --ignore-path docs/_build/ --quiet docs/ *.rst + @${ACTIVATE} doc8 docs/ *.rst valid: @echo "-> Run Ruff format" diff --git a/pyproject.toml b/pyproject.toml index 5e16b56..bfb1d35 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -119,3 +119,10 @@ max-complexity = 10 # Place paths of files to be ignored by ruff here "tests/*" = ["S101"] "test_*.py" = ["S101"] + + +[tool.doc8] + +ignore-path = ["docs/build", "doc/build", "docs/_build", "doc/_build"] +max-line-length=100 +verbose=0 From b7194c80c9425087f1d05e430bd9d6a14fb9c3a0 Mon Sep 17 00:00:00 2001 From: Philippe Ombredanne Date: Sun, 30 Mar 2025 18:41:00 +0200 Subject: [PATCH 24/69] Refine doc handling * remove CI scripts and use Makefile targets instead * ensure doc8 runs quiet * add new docs-check make target to run documentation and links checks * update oudated doc for docs contribution Signed-off-by: Philippe Ombredanne --- .github/workflows/docs-ci.yml | 12 +++++------- Makefile | 10 +++++++--- docs/scripts/doc8_style_check.sh | 5 ----- docs/scripts/sphinx_build_link_check.sh | 5 ----- docs/source/conf.py | 2 +- docs/source/contribute/contrib_doc.rst | 8 ++++---- pyproject.toml | 2 -- 7 files changed, 17 insertions(+), 27 deletions(-) delete mode 100755 docs/scripts/doc8_style_check.sh delete mode 100644 docs/scripts/sphinx_build_link_check.sh diff --git a/.github/workflows/docs-ci.yml b/.github/workflows/docs-ci.yml index 621de4b..10ba5fa 100644 --- a/.github/workflows/docs-ci.yml +++ b/.github/workflows/docs-ci.yml @@ -21,14 +21,12 @@ jobs: python-version: ${{ matrix.python-version }} - name: Install Dependencies - run: pip install -e .[docs] + run: ./configure --dev - - name: Check Sphinx Documentation build minimally - working-directory: ./docs - run: sphinx-build -E -W source build + - name: Check documentation and HTML for errors and dead links + run: make docs-check - - name: Check for documentation style errors - working-directory: ./docs - run: ./scripts/doc8_style_check.sh + - name: Check documentation for style errors + run: make doc8 diff --git a/Makefile b/Makefile index d21a2f9..413399e 100644 --- a/Makefile +++ b/Makefile @@ -19,7 +19,7 @@ dev: doc8: @echo "-> Run doc8 validation" - @${ACTIVATE} doc8 docs/ *.rst + @${ACTIVATE} doc8 --quiet docs/ *.rst valid: @echo "-> Run Ruff format" @@ -46,6 +46,10 @@ test: docs: rm -rf docs/_build/ - @${ACTIVATE} sphinx-build docs/ docs/_build/ + @${ACTIVATE} sphinx-build docs/source docs/_build/ -.PHONY: conf dev check valid clean test docs +docs-check: + @${ACTIVATE} sphinx-build -E -W -b html docs/source docs/_build/ + @${ACTIVATE} sphinx-build -E -W -b linkcheck docs/source docs/_build/ + +.PHONY: conf dev check valid clean test docs docs-check diff --git a/docs/scripts/doc8_style_check.sh b/docs/scripts/doc8_style_check.sh deleted file mode 100755 index 9416323..0000000 --- a/docs/scripts/doc8_style_check.sh +++ /dev/null @@ -1,5 +0,0 @@ -#!/bin/bash -# halt script on error -set -e -# Check for Style Code Violations -doc8 --max-line-length 100 source --ignore D000 --quiet \ No newline at end of file diff --git a/docs/scripts/sphinx_build_link_check.sh b/docs/scripts/sphinx_build_link_check.sh deleted file mode 100644 index c542686..0000000 --- a/docs/scripts/sphinx_build_link_check.sh +++ /dev/null @@ -1,5 +0,0 @@ -#!/bin/bash -# halt script on error -set -e -# Build locally, and then check links -sphinx-build -E -W -b linkcheck source build \ No newline at end of file diff --git a/docs/source/conf.py b/docs/source/conf.py index 8aad829..056ca6e 100644 --- a/docs/source/conf.py +++ b/docs/source/conf.py @@ -18,7 +18,7 @@ # -- Project information ----------------------------------------------------- project = "nexb-skeleton" -copyright = "nexB Inc. and others." +copyright = "nexB Inc., AboutCode and others." author = "AboutCode.org authors and contributors" diff --git a/docs/source/contribute/contrib_doc.rst b/docs/source/contribute/contrib_doc.rst index 5640db2..041b358 100644 --- a/docs/source/contribute/contrib_doc.rst +++ b/docs/source/contribute/contrib_doc.rst @@ -147,7 +147,7 @@ What is Checked? ^^^^^^^^^^^^^^^^ PyCQA is an Organization for code quality tools (and plugins) for the Python programming language. -Doc8 is a sub-project of the same Organization. Refer this `README `_ for more details. +Doc8 is a sub-project of the same Organization. Refer this `README `_ for more details. What is checked: @@ -169,11 +169,11 @@ What is checked: Interspinx ---------- -ScanCode toolkit documentation uses `Intersphinx `_ +ScanCode toolkit documentation uses `Intersphinx `_ to link to other Sphinx Documentations, to maintain links to other Aboutcode Projects. To link sections in the same documentation, standart reST labels are used. Refer -`Cross-Referencing `_ for more information. +`Cross-Referencing `_ for more information. For example:: @@ -230,7 +230,7 @@ Style Conventions for the Documentaion 1. Headings - (`Refer `_) + (`Refer `_) Normally, there are no heading levels assigned to certain characters as the structure is determined from the succession of headings. However, this convention is used in Python’s Style Guide for documenting which you may follow: diff --git a/pyproject.toml b/pyproject.toml index bfb1d35..c9e6772 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -122,7 +122,5 @@ max-complexity = 10 [tool.doc8] - ignore-path = ["docs/build", "doc/build", "docs/_build", "doc/_build"] max-line-length=100 -verbose=0 From a5bcdbdd71d1542a0e9ec9b190a2e3d573c53744 Mon Sep 17 00:00:00 2001 From: Philippe Ombredanne Date: Sun, 30 Mar 2025 18:49:01 +0200 Subject: [PATCH 25/69] Add twine check to release publication Signed-off-by: Philippe Ombredanne --- .github/workflows/pypi-release.yml | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/.github/workflows/pypi-release.yml b/.github/workflows/pypi-release.yml index a66c9c8..cf0579a 100644 --- a/.github/workflows/pypi-release.yml +++ b/.github/workflows/pypi-release.yml @@ -30,12 +30,15 @@ jobs: with: python-version: 3.12 - - name: Install pypa/build - run: python -m pip install build --user + - name: Install pypa/build and twine + run: python -m pip install --user build twine - name: Build a binary wheel and a source tarball run: python -m build --sdist --wheel --outdir dist/ + - name: Validate wheel and sdis for Pypi + run: python -m twine check dist/* + - name: Upload built archives uses: actions/upload-artifact@v4 with: From a6c25fb2a2fa35311d26621b9db400ca52bd376e Mon Sep 17 00:00:00 2001 From: Philippe Ombredanne Date: Sun, 30 Mar 2025 19:16:31 +0200 Subject: [PATCH 26/69] Refine doc contribution docs Signed-off-by: Philippe Ombredanne --- docs/source/contribute/contrib_doc.rst | 119 ++++++++----------------- 1 file changed, 38 insertions(+), 81 deletions(-) diff --git a/docs/source/contribute/contrib_doc.rst b/docs/source/contribute/contrib_doc.rst index 041b358..dee9296 100644 --- a/docs/source/contribute/contrib_doc.rst +++ b/docs/source/contribute/contrib_doc.rst @@ -8,109 +8,59 @@ Contributing to the Documentation Setup Local Build ----------------- -To get started, create or identify a working directory on your local machine. +To get started, check out and configure the repository for development:: -Open that directory and execute the following command in a terminal session:: + git clone https://github.com/aboutcode-org/.git - git clone https://github.com/aboutcode-org/skeleton.git + cd your-repo + ./configure --dev -That will create an ``/skeleton`` directory in your working directory. -Now you can install the dependencies in a virtualenv:: - - cd skeleton - ./configure --docs +(Or use "make dev") .. note:: - In case of windows, run ``configure --docs`` instead of this. - -Now, this will install the following prerequisites: - -- Sphinx -- sphinx_rtd_theme (the format theme used by ReadTheDocs) -- docs8 (style linter) + In case of windows, run ``configure --dev``. -These requirements are already present in setup.cfg and `./configure --docs` installs them. +This will install and configure all requirements foer development including for docs development. -Now you can build the HTML documents locally:: +Now you can build the HTML documentation locally:: source venv/bin/activate - cd docs - make html - -Assuming that your Sphinx installation was successful, Sphinx should build a local instance of the -documentation .html files:: - - open build/html/index.html - -.. note:: - - In case this command did not work, for example on Ubuntu 18.04 you may get a message like “Couldn’t - get a file descriptor referring to the console”, try: - - :: - - see build/html/index.html + make docs -You now have a local build of the AboutCode documents. +This will build a local instance of the ``docs/_build`` directory:: -.. _contrib_doc_share_improvements: + open docs/_build/index.html -Share Document Improvements ---------------------------- - -Ensure that you have the latest files:: - - git pull - git status -Before commiting changes run Continious Integration Scripts locally to run tests. Refer -:ref:`doc_ci` for instructions on the same. +To validate the documentation style and content, use:: -Follow standard git procedures to upload your new and modified files. The following commands are -examples:: - - git status - git add source/index.rst - git add source/how-to-scan.rst - git status - git commit -m "New how-to document that explains how to scan" - git status - git push - git status - -The Scancode-Toolkit webhook with ReadTheDocs should rebuild the documentation after your -Pull Request is Merged. + source venv/bin/activate + make doc8 + make docs-check -Refer the `Pro Git Book `_ available online for Git tutorials -covering more complex topics on Branching, Merging, Rebasing etc. .. _doc_ci: Continuous Integration ---------------------- -The documentations are checked on every new commit through Travis-CI, so that common errors are -avoided and documentation standards are enforced. Travis-CI presently checks for these 3 aspects -of the documentation : +The documentations are checked on every new commit, so that common errors are avoided and +documentation standards are enforced. We checks for these aspects of the documentation: 1. Successful Builds (By using ``sphinx-build``) -2. No Broken Links (By Using ``link-check``) -3. Linting Errors (By Using ``Doc8``) +2. No Broken Links (By Using ``linkcheck``) +3. Linting Errors (By Using ``doc8``) -So run these scripts at your local system before creating a Pull Request:: +You myst run these scripts locally before creating a pull request:: - cd docs - ./scripts/sphinx_build_link_check.sh - ./scripts/doc8_style_check.sh + make doc8 + make check-docs -If you don't have permission to run the scripts, run:: - - chmod u+x ./scripts/doc8_style_check.sh .. _doc_style_docs8: -Style Checks Using ``Doc8`` +Style Checks Using ``doc8`` --------------------------- How To Run Style Tests @@ -118,8 +68,7 @@ How To Run Style Tests In the project root, run the following commands:: - $ cd docs - $ ./scripts/doc8_style_check.sh + make doc8 A sample output is:: @@ -143,11 +92,13 @@ A sample output is:: Now fix the errors and run again till there isn't any style error in the documentation. + What is Checked? ^^^^^^^^^^^^^^^^ PyCQA is an Organization for code quality tools (and plugins) for the Python programming language. -Doc8 is a sub-project of the same Organization. Refer this `README `_ for more details. +Doc8 is a sub-project of the same Organization. Refer this +`README `_ for more details. What is checked: @@ -164,16 +115,19 @@ What is checked: - no carriage returns (use UNIX newlines) - D004 - no newline at end of file - D005 + .. _doc_interspinx: Interspinx ---------- -ScanCode toolkit documentation uses `Intersphinx `_ +AboutCode documentation uses +`Intersphinx `_ to link to other Sphinx Documentations, to maintain links to other Aboutcode Projects. To link sections in the same documentation, standart reST labels are used. Refer -`Cross-Referencing `_ for more information. +`Cross-Referencing `_ +for more information. For example:: @@ -223,6 +177,7 @@ Intersphinx, and you link to that label, it will create a link to the local labe For more information, refer this tutorial named `Using Intersphinx `_. + .. _doc_style_conv: Style Conventions for the Documentaion @@ -303,12 +258,14 @@ Style Conventions for the Documentaion ``rst_snippets/warning_snippets/`` and then included to eliminate redundancy, as these are frequently used in multiple files. + Converting from Markdown ------------------------ -If you want to convert a ``.md`` file to a ``.rst`` file, this `tool `_ -does it pretty well. You'd still have to clean up and check for errors as this contains a lot of -bugs. But this is definitely better than converting everything by yourself. +If you want to convert a ``.md`` file to a ``.rst`` file, this +`tool `_ does it pretty well. +You will still have to clean up and check for errors as this contains a lot of bugs. But this is +definitely better than converting everything by yourself. This will be helpful in converting GitHub wiki's (Markdown Files) to reStructuredtext files for Sphinx/ReadTheDocs hosting. From 68daae1e7e475a89568e353f64f29af13754ce9e Mon Sep 17 00:00:00 2001 From: Jono Yang Date: Thu, 27 Mar 2025 14:54:31 -0700 Subject: [PATCH 27/69] Replace black and isort with ruff * Use ruff config and Make commands from scancode.io Signed-off-by: Jono Yang --- Makefile | 27 ++++++++++++--------------- pyproject.toml | 37 +++++++++++++++++++++++++++++++++++++ setup.cfg | 3 +-- 3 files changed, 50 insertions(+), 17 deletions(-) diff --git a/Makefile b/Makefile index 94451b3..1738b20 100644 --- a/Makefile +++ b/Makefile @@ -17,27 +17,24 @@ dev: @echo "-> Configure the development envt." ./configure --dev -isort: - @echo "-> Apply isort changes to ensure proper imports ordering" - ${VENV}/bin/isort --sl -l 100 src tests setup.py - -black: - @echo "-> Apply black code formatter" - ${VENV}/bin/black -l 100 src tests setup.py - doc8: @echo "-> Run doc8 validation" @${ACTIVATE} doc8 --max-line-length 100 --ignore-path docs/_build/ --quiet docs/ -valid: isort black +valid: + @echo "-> Run Ruff format" + @${ACTIVATE} ruff format + @echo "-> Run Ruff linter" + @${ACTIVATE} ruff check --fix check: - @echo "-> Run pycodestyle (PEP8) validation" - @${ACTIVATE} pycodestyle --max-line-length=100 --exclude=.eggs,venv,lib,thirdparty,docs,migrations,settings.py,.cache . - @echo "-> Run isort imports ordering validation" - @${ACTIVATE} isort --sl --check-only -l 100 setup.py src tests . - @echo "-> Run black validation" - @${ACTIVATE} black --check --check -l 100 src tests setup.py + @echo "-> Run Ruff linter validation (pycodestyle, bandit, isort, and more)" + @${ACTIVATE} ruff check + @echo "-> Run Ruff format validation" + @${ACTIVATE} ruff format --check + @$(MAKE) doc8 + @echo "-> Run ABOUT files validation" + @${ACTIVATE} about check etc/ clean: @echo "-> Clean the Python env" diff --git a/pyproject.toml b/pyproject.toml index cde7907..01e60fc 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -50,3 +50,40 @@ addopts = [ "--strict-markers", "--doctest-modules" ] + +[tool.ruff] +line-length = 88 +extend-exclude = [] +target-version = "py310" + +[tool.ruff.lint] +# Rules: https://docs.astral.sh/ruff/rules/ +select = [ + "E", # pycodestyle + "W", # pycodestyle warnings + "D", # pydocstyle + "F", # Pyflakes + "UP", # pyupgrade + "S", # flake8-bandit + "I", # isort + "C9", # McCabe complexity +] +ignore = ["D1", "D203", "D205", "D212", "D400", "D415"] + +[tool.ruff.lint.isort] +force-single-line = true +sections = { django = ["django"] } +section-order = [ + "future", + "standard-library", + "django", + "third-party", + "first-party", + "local-folder", +] + +[tool.ruff.lint.mccabe] +max-complexity = 10 + +[tool.ruff.lint.per-file-ignores] +# Place paths of files to be ignored by ruff here diff --git a/setup.cfg b/setup.cfg index ef7d369..aaec643 100644 --- a/setup.cfg +++ b/setup.cfg @@ -54,8 +54,7 @@ testing = aboutcode-toolkit >= 7.0.2 pycodestyle >= 2.8.0 twine - black - isort + ruff docs = Sphinx>=5.0.2 From 6a8c9ae144a1985b59fb69a0b2c55e32831714b8 Mon Sep 17 00:00:00 2001 From: Philippe Ombredanne Date: Sat, 29 Mar 2025 00:46:06 +0100 Subject: [PATCH 28/69] Use org standard 100 line length Signed-off-by: Philippe Ombredanne --- pyproject.toml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pyproject.toml b/pyproject.toml index 01e60fc..cea91bd 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -52,7 +52,7 @@ addopts = [ ] [tool.ruff] -line-length = 88 +line-length = 100 extend-exclude = [] target-version = "py310" From 2fd31d54afa47418c764de0f1a30d67c7059ed7b Mon Sep 17 00:00:00 2001 From: Philippe Ombredanne Date: Sat, 29 Mar 2025 08:40:28 +0100 Subject: [PATCH 29/69] Lint all common code directories Signed-off-by: Philippe Ombredanne --- pyproject.toml | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/pyproject.toml b/pyproject.toml index cea91bd..9e62736 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -55,6 +55,14 @@ addopts = [ line-length = 100 extend-exclude = [] target-version = "py310" +include = [ + "pyproject.toml", + "src/**/*.py", + "etc/**/*.py", + "test/**/*.py", + "doc/**/*", + "*.py" +] [tool.ruff.lint] # Rules: https://docs.astral.sh/ruff/rules/ From eb5fc82ab1cab8a4742a2b9028d1436956960e81 Mon Sep 17 00:00:00 2001 From: Philippe Ombredanne Date: Sat, 29 Mar 2025 09:07:47 +0100 Subject: [PATCH 30/69] Remove unused targets Signed-off-by: Philippe Ombredanne --- Makefile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Makefile b/Makefile index 1738b20..930e801 100644 --- a/Makefile +++ b/Makefile @@ -48,4 +48,4 @@ docs: rm -rf docs/_build/ @${ACTIVATE} sphinx-build docs/ docs/_build/ -.PHONY: conf dev check valid black isort clean test docs +.PHONY: conf dev check valid clean test docs From 529d51621c9e2af8e1ec2503044b8752c71c3ba7 Mon Sep 17 00:00:00 2001 From: Philippe Ombredanne Date: Sat, 29 Mar 2025 11:03:05 +0100 Subject: [PATCH 31/69] Improve import sorting Signed-off-by: Philippe Ombredanne --- etc/scripts/check_thirdparty.py | 1 - etc/scripts/fetch_thirdparty.py | 2 +- etc/scripts/test_utils_pip_compatibility_tags.py | 3 +-- etc/scripts/utils_dejacode.py | 1 - etc/scripts/utils_pip_compatibility_tags.py | 14 ++++++-------- etc/scripts/utils_thirdparty.py | 3 +-- pyproject.toml | 7 ++++++- 7 files changed, 15 insertions(+), 16 deletions(-) diff --git a/etc/scripts/check_thirdparty.py b/etc/scripts/check_thirdparty.py index 2daded9..62dbb14 100644 --- a/etc/scripts/check_thirdparty.py +++ b/etc/scripts/check_thirdparty.py @@ -12,7 +12,6 @@ import utils_thirdparty - @click.command() @click.option( "-d", diff --git a/etc/scripts/fetch_thirdparty.py b/etc/scripts/fetch_thirdparty.py index 3f9ff52..30d376c 100644 --- a/etc/scripts/fetch_thirdparty.py +++ b/etc/scripts/fetch_thirdparty.py @@ -16,8 +16,8 @@ import click -import utils_thirdparty import utils_requirements +import utils_thirdparty TRACE = False TRACE_DEEP = False diff --git a/etc/scripts/test_utils_pip_compatibility_tags.py b/etc/scripts/test_utils_pip_compatibility_tags.py index 98187c5..a33b8b3 100644 --- a/etc/scripts/test_utils_pip_compatibility_tags.py +++ b/etc/scripts/test_utils_pip_compatibility_tags.py @@ -25,14 +25,13 @@ WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. """ -from unittest.mock import patch import sysconfig +from unittest.mock import patch import pytest import utils_pip_compatibility_tags - @pytest.mark.parametrize( "version_info, expected", [ diff --git a/etc/scripts/utils_dejacode.py b/etc/scripts/utils_dejacode.py index 652252d..c71543f 100644 --- a/etc/scripts/utils_dejacode.py +++ b/etc/scripts/utils_dejacode.py @@ -14,7 +14,6 @@ import requests import saneyaml - from packvers import version as packaging_version """ diff --git a/etc/scripts/utils_pip_compatibility_tags.py b/etc/scripts/utils_pip_compatibility_tags.py index af42a0c..de0ac95 100644 --- a/etc/scripts/utils_pip_compatibility_tags.py +++ b/etc/scripts/utils_pip_compatibility_tags.py @@ -27,14 +27,12 @@ import re -from packvers.tags import ( - compatible_tags, - cpython_tags, - generic_tags, - interpreter_name, - interpreter_version, - mac_platforms, -) +from packvers.tags import compatible_tags +from packvers.tags import cpython_tags +from packvers.tags import generic_tags +from packvers.tags import interpreter_name +from packvers.tags import interpreter_version +from packvers.tags import mac_platforms _osx_arch_pat = re.compile(r"(.+)_(\d+)_(\d+)_(.+)") diff --git a/etc/scripts/utils_thirdparty.py b/etc/scripts/utils_thirdparty.py index 46dc728..b0295ec 100644 --- a/etc/scripts/utils_thirdparty.py +++ b/etc/scripts/utils_thirdparty.py @@ -25,14 +25,13 @@ import packageurl import requests import saneyaml +import utils_pip_compatibility_tags from commoncode import fileutils from commoncode.hash import multi_checksums from commoncode.text import python_safe_name from packvers import tags as packaging_tags from packvers import version as packaging_version -import utils_pip_compatibility_tags - """ Utilities to manage Python thirparty libraries source, binaries and metadata in local directories and remote repositories. diff --git a/pyproject.toml b/pyproject.toml index 9e62736..ba55770 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -76,10 +76,15 @@ select = [ "I", # isort "C9", # McCabe complexity ] -ignore = ["D1", "D203", "D205", "D212", "D400", "D415"] +ignore = ["D1", "D200", "D203", "D205", "D212", "D400", "D415"] [tool.ruff.lint.isort] force-single-line = true +lines-after-imports = 1 +default-section = "first-party" +known-first-party = ["src", "tests", "etc/scripts/**/*.py"] +known-third-party = ["click", "pytest"] + sections = { django = ["django"] } section-order = [ "future", From aae1a2847c0e493b0e8bea542da30dbdfb2be68e Mon Sep 17 00:00:00 2001 From: Philippe Ombredanne Date: Sat, 29 Mar 2025 21:35:16 +0100 Subject: [PATCH 32/69] Apply small code updates Signed-off-by: Philippe Ombredanne --- etc/scripts/utils_requirements.py | 20 ++++++++----- etc/scripts/utils_thirdparty.py | 48 +++++++++++++++---------------- 2 files changed, 37 insertions(+), 31 deletions(-) diff --git a/etc/scripts/utils_requirements.py b/etc/scripts/utils_requirements.py index 1c50239..a9ac223 100644 --- a/etc/scripts/utils_requirements.py +++ b/etc/scripts/utils_requirements.py @@ -57,21 +57,25 @@ def get_required_name_version(requirement, with_unpinned=False): >>> assert get_required_name_version("fooA==1.2.3.DEV1") == ("fooa", "1.2.3.dev1") >>> assert get_required_name_version("foo==1.2.3", with_unpinned=False) == ("foo", "1.2.3") >>> assert get_required_name_version("foo", with_unpinned=True) == ("foo", "") - >>> assert get_required_name_version("foo>=1.2", with_unpinned=True) == ("foo", ""), get_required_name_version("foo>=1.2") + >>> expected = ("foo", ""), get_required_name_version("foo>=1.2") + >>> assert get_required_name_version("foo>=1.2", with_unpinned=True) == expected >>> try: ... assert not get_required_name_version("foo", with_unpinned=False) ... except Exception as e: ... assert "Requirement version must be pinned" in str(e) """ requirement = requirement and "".join(requirement.lower().split()) - assert requirement, f"specifier is required is empty:{requirement!r}" + if not requirement: + raise ValueError(f"specifier is required is empty:{requirement!r}") name, operator, version = split_req(requirement) - assert name, f"Name is required: {requirement}" + if not name: + raise ValueError(f"Name is required: {requirement}") is_pinned = operator == "==" if with_unpinned: version = "" else: - assert is_pinned and version, f"Requirement version must be pinned: {requirement}" + if not is_pinned and version: + raise ValueError(f"Requirement version must be pinned: {requirement}") return name, version @@ -120,7 +124,7 @@ def get_installed_reqs(site_packages_dir): # setuptools, pip args = ["pip", "freeze", "--exclude-editable", "--all", "--path", site_packages_dir] - return subprocess.check_output(args, encoding="utf-8") + return subprocess.check_output(args, encoding="utf-8") # noqa: S603 comparators = ( @@ -150,9 +154,11 @@ def split_req(req): >>> assert split_req("foo >= 1.2.3 ") == ("foo", ">=", "1.2.3"), split_req("foo >= 1.2.3 ") >>> assert split_req("foo>=1.2") == ("foo", ">=", "1.2"), split_req("foo>=1.2") """ - assert req + if not req: + raise ValueError("req is required") # do not allow multiple constraints and tags - assert not any(c in req for c in ",;") + if not any(c in req for c in ",;"): + raise Exception(f"complex requirements with : or ; not supported: {req}") req = "".join(req.split()) if not any(c in req for c in comparators): return req, "", "" diff --git a/etc/scripts/utils_thirdparty.py b/etc/scripts/utils_thirdparty.py index b0295ec..6d5ffdc 100644 --- a/etc/scripts/utils_thirdparty.py +++ b/etc/scripts/utils_thirdparty.py @@ -1,5 +1,4 @@ #!/usr/bin/env python -# -*- coding: utf-8 -*- # # Copyright (c) nexB Inc. and others. All rights reserved. # ScanCode is a trademark of nexB Inc. @@ -559,7 +558,8 @@ def download(self, dest_dir=THIRDPARTY_DIR): Download this distribution into `dest_dir` directory. Return the fetched filename. """ - assert self.filename + if not self.filename: + raise ValueError(f"self.filename has no value but is required: {self.filename!r}") if TRACE_DEEP: print( f"Fetching distribution of {self.name}=={self.version}:", @@ -829,10 +829,9 @@ def fetch_license_files(self, dest_dir=THIRDPARTY_DIR, use_cached_index=False): urls = LinksRepository.from_url( use_cached_index=use_cached_index).links errors = [] - extra_lic_names = [l.get("file") - for l in self.extra_data.get("licenses", {})] + extra_lic_names = [lic.get("file") for lic in self.extra_data.get("licenses", {})] extra_lic_names += [self.extra_data.get("license_file")] - extra_lic_names = [ln for ln in extra_lic_names if ln] + extra_lic_names = [eln for eln in extra_lic_names if eln] lic_names = [f"{key}.LICENSE" for key in self.get_license_keys()] for filename in lic_names + extra_lic_names: floc = os.path.join(dest_dir, filename) @@ -853,7 +852,7 @@ def fetch_license_files(self, dest_dir=THIRDPARTY_DIR, use_cached_index=False): if TRACE: print(f"Fetched license from remote: {lic_url}") - except: + except Exception: try: # try licensedb second lic_url = f"{LICENSEDB_API_URL}/{filename}" @@ -866,8 +865,9 @@ def fetch_license_files(self, dest_dir=THIRDPARTY_DIR, use_cached_index=False): if TRACE: print(f"Fetched license from licensedb: {lic_url}") - except: - msg = f'No text for license {filename} in expression "{self.license_expression}" from {self}' + except Exception: + msg = f"No text for license {filename} in expression " + f"{self.license_expression!r} from {self}" print(msg) errors.append(msg) @@ -1009,7 +1009,7 @@ def get_license_link_for_filename(filename, urls): exception if no link is found or if there are more than one link for that file name. """ - path_or_url = [l for l in urls if l.endswith(f"/{filename}")] + path_or_url = [url for url in urls if url.endswith(f"/{filename}")] if not path_or_url: raise Exception(f"Missing link to file: {filename}") if not len(path_or_url) == 1: @@ -1140,7 +1140,6 @@ def to_filename(self): @attr.attributes class Wheel(Distribution): - """ Represents a wheel file. @@ -1301,7 +1300,7 @@ def is_pure(self): def is_pure_wheel(filename): try: return Wheel.from_filename(filename).is_pure() - except: + except Exception: return False @@ -1489,8 +1488,7 @@ def dists_from_paths_or_urls(cls, paths_or_urls): ) except InvalidDistributionFilename: if TRACE_DEEP: - print( - f" Skipping invalid distribution from: {path_or_url}") + print(f" Skipping invalid distribution from: {path_or_url}") continue return dists @@ -1500,8 +1498,7 @@ def get_distributions(self): """ if self.sdist: yield self.sdist - for wheel in self.wheels: - yield wheel + yield from self.wheels def get_url_for_filename(self, filename): """ @@ -1632,7 +1629,8 @@ class PypiSimpleRepository: type=dict, default=attr.Factory(lambda: defaultdict(dict)), metadata=dict( - help="Mapping of {name: {version: PypiPackage, version: PypiPackage, etc} available in this repo" + help="Mapping of {name: {version: PypiPackage, version: PypiPackage, etc} " + "available in this repo" ), ) @@ -1647,7 +1645,8 @@ class PypiSimpleRepository: type=bool, default=False, metadata=dict( - help="If True, use any existing on-disk cached PyPI index files. Otherwise, fetch and cache." + help="If True, use any existing on-disk cached PyPI index files. " + "Otherwise, fetch and cache." ), ) @@ -1656,7 +1655,8 @@ def _get_package_versions_map(self, name): Return a mapping of all available PypiPackage version for this package name. The mapping may be empty. It is ordered by version from oldest to newest """ - assert name + if not name: + raise ValueError(f"name is required: {name!r}") normalized_name = NameVer.normalize_name(name) versions = self.packages[normalized_name] if not versions and normalized_name not in self.fetched_package_normalized_names: @@ -1713,7 +1713,7 @@ def fetch_links(self, normalized_name): ) links = collect_urls(text) # TODO: keep sha256 - links = [l.partition("#sha256=") for l in links] + links = [link.partition("#sha256=") for link in links] links = [url for url, _, _sha256 in links] return links @@ -1936,7 +1936,7 @@ def get_remote_file_content( # several redirects and that we can ignore content there. A HEAD request may # not get us this last header print(f" DOWNLOADING: {url}") - with requests.get(url, allow_redirects=True, stream=True, headers=headers) as response: + with requests.get(url, allow_redirects=True, stream=True, headers=headers) as response: # noqa: S113 status = response.status_code if status != requests.codes.ok: # NOQA if status == 429 and _delay < 20: @@ -2161,7 +2161,7 @@ def call(args, verbose=TRACE): """ if TRACE_DEEP: print("Calling:", " ".join(args)) - with subprocess.Popen( + with subprocess.Popen( # noqa: S603 args, stdout=subprocess.PIPE, stderr=subprocess.PIPE, encoding="utf-8" ) as process: @@ -2227,7 +2227,7 @@ def download_wheels_with_pip( cli_args.extend(["--requirement", req_file]) if TRACE: - print(f"Downloading wheels using command:", " ".join(cli_args)) + print("Downloading wheels using command:", " ".join(cli_args)) existing = set(os.listdir(dest_dir)) error = False @@ -2260,7 +2260,7 @@ def download_wheels_with_pip( def check_about(dest_dir=THIRDPARTY_DIR): try: - subprocess.check_output(f"venv/bin/about check {dest_dir}".split()) + subprocess.check_output(f"venv/bin/about check {dest_dir}".split()) # noqa: S603 except subprocess.CalledProcessError as cpe: print() print("Invalid ABOUT files:") @@ -2312,5 +2312,5 @@ def get_license_expression(declared_licenses): return get_only_expression_from_extracted_license(declared_licenses) except ImportError: # Scancode is not installed, clean and join all the licenses - lics = [python_safe_name(l).lower() for l in declared_licenses] + lics = [python_safe_name(lic).lower() for lic in declared_licenses] return " AND ".join(lics).lower() From 037f9fc1b03736eeac9e0eefac3e35acc916d193 Mon Sep 17 00:00:00 2001 From: Philippe Ombredanne Date: Sat, 29 Mar 2025 21:42:03 +0100 Subject: [PATCH 33/69] Format code Signed-off-by: Philippe Ombredanne --- etc/scripts/check_thirdparty.py | 3 +- etc/scripts/fetch_thirdparty.py | 26 ++++----- etc/scripts/gen_pypi_simple.py | 4 +- etc/scripts/utils_dejacode.py | 15 +++--- etc/scripts/utils_requirements.py | 9 ++-- etc/scripts/utils_thirdparty.py | 90 +++++++++++-------------------- 6 files changed, 50 insertions(+), 97 deletions(-) diff --git a/etc/scripts/check_thirdparty.py b/etc/scripts/check_thirdparty.py index 62dbb14..1aa4e28 100644 --- a/etc/scripts/check_thirdparty.py +++ b/etc/scripts/check_thirdparty.py @@ -16,8 +16,7 @@ @click.option( "-d", "--dest", - type=click.Path(exists=True, readable=True, - path_type=str, file_okay=False), + type=click.Path(exists=True, readable=True, path_type=str, file_okay=False), required=True, help="Path to the thirdparty directory to check.", ) diff --git a/etc/scripts/fetch_thirdparty.py b/etc/scripts/fetch_thirdparty.py index 30d376c..c224683 100644 --- a/etc/scripts/fetch_thirdparty.py +++ b/etc/scripts/fetch_thirdparty.py @@ -55,8 +55,7 @@ "-d", "--dest", "dest_dir", - type=click.Path(exists=True, readable=True, - path_type=str, file_okay=False), + type=click.Path(exists=True, readable=True, path_type=str, file_okay=False), metavar="DIR", default=utils_thirdparty.THIRDPARTY_DIR, show_default=True, @@ -121,7 +120,7 @@ show_default=False, multiple=True, help="Package name(s) that come only in sdist format (no wheels). " - "The command will not fail and exit if no wheel exists for these names", + "The command will not fail and exit if no wheel exists for these names", ) @click.option( "--wheel-only", @@ -132,7 +131,7 @@ show_default=False, multiple=True, help="Package name(s) that come only in wheel format (no sdist). " - "The command will not fail and exit if no sdist exists for these names", + "The command will not fail and exit if no sdist exists for these names", ) @click.option( "--no-dist", @@ -143,7 +142,7 @@ show_default=False, multiple=True, help="Package name(s) that do not come either in wheel or sdist format. " - "The command will not fail and exit if no distribution exists for these names", + "The command will not fail and exit if no distribution exists for these names", ) @click.help_option("-h", "--help") def fetch_thirdparty( @@ -225,8 +224,7 @@ def fetch_thirdparty( environments = None if wheels: evts = itertools.product(python_versions, operating_systems) - environments = [utils_thirdparty.Environment.from_pyver_and_os( - pyv, os) for pyv, os in evts] + environments = [utils_thirdparty.Environment.from_pyver_and_os(pyv, os) for pyv, os in evts] # Collect PyPI repos repos = [] @@ -250,7 +248,6 @@ def fetch_thirdparty( print(f"Processing: {name} @ {version}") if wheels: for environment in environments: - if TRACE: print(f" ==> Fetching wheel for envt: {environment}") @@ -262,14 +259,11 @@ def fetch_thirdparty( repos=repos, ) if not fetched: - wheels_or_sdist_not_found[f"{name}=={version}"].append( - environment) + wheels_or_sdist_not_found[f"{name}=={version}"].append(environment) if TRACE: print(f" NOT FOUND") - if (sdists or - (f"{name}=={version}" in wheels_or_sdist_not_found and name in sdist_only) - ): + if sdists or (f"{name}=={version}" in wheels_or_sdist_not_found and name in sdist_only): if TRACE: print(f" ==> Fetching sdist: {name}=={version}") @@ -292,8 +286,7 @@ def fetch_thirdparty( sdist_missing = sdists and "sdist" in dists and not name in wheel_only if sdist_missing: mia.append(f"SDist missing: {nv} {dists}") - wheels_missing = wheels and any( - d for d in dists if d != "sdist") and not name in sdist_only + wheels_missing = wheels and any(d for d in dists if d != "sdist") and not name in sdist_only if wheels_missing: mia.append(f"Wheels missing: {nv} {dists}") @@ -303,8 +296,7 @@ def fetch_thirdparty( raise Exception(mia) print(f"==> FETCHING OR CREATING ABOUT AND LICENSE FILES") - utils_thirdparty.fetch_abouts_and_licenses( - dest_dir=dest_dir, use_cached_index=use_cached_index) + utils_thirdparty.fetch_abouts_and_licenses(dest_dir=dest_dir, use_cached_index=use_cached_index) utils_thirdparty.clean_about_files(dest_dir=dest_dir) # check for problems diff --git a/etc/scripts/gen_pypi_simple.py b/etc/scripts/gen_pypi_simple.py index 214d90d..cfe68e6 100644 --- a/etc/scripts/gen_pypi_simple.py +++ b/etc/scripts/gen_pypi_simple.py @@ -69,7 +69,6 @@ def get_package_name_from_filename(filename): raise InvalidDistributionFilename(filename) elif filename.endswith(wheel_ext): - wheel_info = get_wheel_from_filename(filename) if not wheel_info: @@ -200,11 +199,10 @@ def build_pypi_index(directory, base_url="https://thirdparty.aboutcode.org/pypi" simple_html_index = [ "", "PyPI Simple Index", - '' '', + '', ] for pkg_file in directory.iterdir(): - pkg_filename = pkg_file.name if ( diff --git a/etc/scripts/utils_dejacode.py b/etc/scripts/utils_dejacode.py index c71543f..cd39cda 100644 --- a/etc/scripts/utils_dejacode.py +++ b/etc/scripts/utils_dejacode.py @@ -32,8 +32,7 @@ def can_do_api_calls(): if not DEJACODE_API_KEY and DEJACODE_API_URL: - print( - "DejaCode DEJACODE_API_KEY and DEJACODE_API_URL not configured. Doing nothing") + print("DejaCode DEJACODE_API_KEY and DEJACODE_API_URL not configured. Doing nothing") return False else: return True @@ -68,8 +67,7 @@ def get_package_data(distribution): return results[0] elif len_results > 1: - print( - f"More than 1 entry exists, review at: {DEJACODE_API_URL_PACKAGES}") + print(f"More than 1 entry exists, review at: {DEJACODE_API_URL_PACKAGES}") else: print("Could not find package:", distribution.download_url) @@ -150,12 +148,11 @@ def find_latest_dejacode_package(distribution): # there was no exact match, find the latest version # TODO: consider the closest version rather than the latest # or the version that has the best data - with_versions = [(packaging_version.parse(p["version"]), p) - for p in packages] + with_versions = [(packaging_version.parse(p["version"]), p) for p in packages] with_versions = sorted(with_versions) latest_version, latest_package_version = sorted(with_versions)[-1] print( - f"Found DejaCode latest version: {latest_version} " f"for dist: {distribution.package_url}", + f"Found DejaCode latest version: {latest_version} for dist: {distribution.package_url}", ) return latest_package_version @@ -181,7 +178,7 @@ def create_dejacode_package(distribution): } fields_to_carry_over = [ - "download_url" "type", + "download_urltype", "namespace", "name", "version", @@ -209,5 +206,5 @@ def create_dejacode_package(distribution): if response.status_code != 201: raise Exception(f"Error, cannot create package for: {distribution}") - print(f'New Package created at: {new_package_data["absolute_url"]}') + print(f"New Package created at: {new_package_data['absolute_url']}") return new_package_data diff --git a/etc/scripts/utils_requirements.py b/etc/scripts/utils_requirements.py index a9ac223..167bc9f 100644 --- a/etc/scripts/utils_requirements.py +++ b/etc/scripts/utils_requirements.py @@ -106,8 +106,7 @@ def lock_dev_requirements( all_req_nvs = get_required_name_versions(all_req_lines) dev_only_req_nvs = {n: v for n, v in all_req_nvs if n not in main_names} - new_reqs = "\n".join( - f"{n}=={v}" for n, v in sorted(dev_only_req_nvs.items())) + new_reqs = "\n".join(f"{n}=={v}" for n, v in sorted(dev_only_req_nvs.items())) with open(dev_requirements_file, "w") as fo: fo.write(new_reqs) @@ -118,12 +117,10 @@ def get_installed_reqs(site_packages_dir): as a text. """ if not os.path.exists(site_packages_dir): - raise Exception( - f"site_packages directory: {site_packages_dir!r} does not exists") + raise Exception(f"site_packages directory: {site_packages_dir!r} does not exists") # Also include these packages in the output with --all: wheel, distribute, # setuptools, pip - args = ["pip", "freeze", "--exclude-editable", - "--all", "--path", site_packages_dir] + args = ["pip", "freeze", "--exclude-editable", "--all", "--path", site_packages_dir] return subprocess.check_output(args, encoding="utf-8") # noqa: S603 diff --git a/etc/scripts/utils_thirdparty.py b/etc/scripts/utils_thirdparty.py index 6d5ffdc..4ea1bab 100644 --- a/etc/scripts/utils_thirdparty.py +++ b/etc/scripts/utils_thirdparty.py @@ -243,11 +243,9 @@ def download_wheel(name, version, environment, dest_dir=THIRDPARTY_DIR, repos=tu package = repo.get_package_version(name=name, version=version) if not package: if TRACE_DEEP: - print( - f" download_wheel: No package in {repo.index_url} for {name}=={version}") + print(f" download_wheel: No package in {repo.index_url} for {name}=={version}") continue - supported_wheels = list( - package.get_supported_wheels(environment=environment)) + supported_wheels = list(package.get_supported_wheels(environment=environment)) if not supported_wheels: if TRACE_DEEP: print( @@ -291,8 +289,7 @@ def download_sdist(name, version, dest_dir=THIRDPARTY_DIR, repos=tuple()): if not package: if TRACE_DEEP: - print( - f" download_sdist: No package in {repo.index_url} for {name}=={version}") + print(f" download_sdist: No package in {repo.index_url} for {name}=={version}") continue sdist = package.sdist if not sdist: @@ -301,8 +298,7 @@ def download_sdist(name, version, dest_dir=THIRDPARTY_DIR, repos=tuple()): continue if TRACE_DEEP: - print( - f" download_sdist: Getting sdist from index (or cache): {sdist.download_url}") + print(f" download_sdist: Getting sdist from index (or cache): {sdist.download_url}") fetched_sdist_filename = package.sdist.download(dest_dir=dest_dir) if fetched_sdist_filename: @@ -357,7 +353,6 @@ def sorted(cls, namevers): @attr.attributes class Distribution(NameVer): - # field names that can be updated from another Distribution or mapping updatable_fields = [ "license_expression", @@ -535,8 +530,7 @@ def get_best_download_url(self, repos=tuple()): repos = DEFAULT_PYPI_REPOS for repo in repos: - package = repo.get_package_version( - name=self.name, version=self.version) + package = repo.get_package_version(name=self.name, version=self.version) if not package: if TRACE: print( @@ -776,8 +770,7 @@ def load_remote_about_data(self): if notice_text: about_data["notice_text"] = notice_text except RemoteNotFetchedException: - print( - f"Failed to fetch NOTICE file: {self.notice_download_url}") + print(f"Failed to fetch NOTICE file: {self.notice_download_url}") return self.load_about_data(about_data) def get_checksums(self, dest_dir=THIRDPARTY_DIR): @@ -826,8 +819,7 @@ def fetch_license_files(self, dest_dir=THIRDPARTY_DIR, use_cached_index=False): Fetch license files if missing in `dest_dir`. Return True if license files were fetched. """ - urls = LinksRepository.from_url( - use_cached_index=use_cached_index).links + urls = LinksRepository.from_url(use_cached_index=use_cached_index).links errors = [] extra_lic_names = [lic.get("file") for lic in self.extra_data.get("licenses", {})] extra_lic_names += [self.extra_data.get("license_file")] @@ -840,8 +832,7 @@ def fetch_license_files(self, dest_dir=THIRDPARTY_DIR, use_cached_index=False): try: # try remotely first - lic_url = get_license_link_for_filename( - filename=filename, urls=urls) + lic_url = get_license_link_for_filename(filename=filename, urls=urls) fetch_and_save( path_or_url=lic_url, @@ -919,8 +910,7 @@ def load_pkginfo_data(self, dest_dir=THIRDPARTY_DIR): c for c in classifiers if c.startswith("License") ] license_expression = get_license_expression(declared_license) - other_classifiers = [ - c for c in classifiers if not c.startswith("License")] + other_classifiers = [c for c in classifiers if not c.startswith("License")] holder = raw_data["Author"] holder_contact = raw_data["Author-email"] @@ -962,8 +952,7 @@ def update(self, data, overwrite=False, keep_extra=True): package_url = data.get("package_url") if package_url: purl_from_data = packageurl.PackageURL.from_string(package_url) - purl_from_self = packageurl.PackageURL.from_string( - self.package_url) + purl_from_self = packageurl.PackageURL.from_string(self.package_url) if purl_from_data != purl_from_self: print( f"Invalid dist update attempt, no same same purl with dist: " @@ -1013,8 +1002,7 @@ def get_license_link_for_filename(filename, urls): if not path_or_url: raise Exception(f"Missing link to file: {filename}") if not len(path_or_url) == 1: - raise Exception( - f"Multiple links to file: {filename}: \n" + "\n".join(path_or_url)) + raise Exception(f"Multiple links to file: {filename}: \n" + "\n".join(path_or_url)) return path_or_url[0] @@ -1102,7 +1090,6 @@ def get_sdist_name_ver_ext(filename): @attr.attributes class Sdist(Distribution): - extension = attr.ib( repr=False, type=str, @@ -1407,8 +1394,7 @@ def packages_from_dir(cls, directory): """ base = os.path.abspath(directory) - paths = [os.path.join(base, f) - for f in os.listdir(base) if f.endswith(EXTENSIONS)] + paths = [os.path.join(base, f) for f in os.listdir(base) if f.endswith(EXTENSIONS)] if TRACE_ULTRA_DEEP: print("packages_from_dir: paths:", paths) @@ -1469,8 +1455,7 @@ def dists_from_paths_or_urls(cls, paths_or_urls): dists = [] if TRACE_ULTRA_DEEP: print(" ###paths_or_urls:", paths_or_urls) - installable = [f for f in paths_or_urls if f.endswith( - EXTENSIONS_INSTALLABLE)] + installable = [f for f in paths_or_urls if f.endswith(EXTENSIONS_INSTALLABLE)] for path_or_url in installable: try: dist = Distribution.from_path_or_url(path_or_url) @@ -1536,8 +1521,7 @@ class Environment: implementation = attr.ib( type=str, default="cp", - metadata=dict( - help="Python implementation supported by this environment."), + metadata=dict(help="Python implementation supported by this environment."), repr=False, ) @@ -1551,8 +1535,7 @@ class Environment: platforms = attr.ib( type=list, default=attr.Factory(list), - metadata=dict( - help="List of platform tags supported by this environment."), + metadata=dict(help="List of platform tags supported by this environment."), repr=False, ) @@ -1637,8 +1620,7 @@ class PypiSimpleRepository: fetched_package_normalized_names = attr.ib( type=set, default=attr.Factory(set), - metadata=dict( - help="A set of already fetched package normalized names."), + metadata=dict(help="A set of already fetched package normalized names."), ) use_cached_index = attr.ib( @@ -1671,12 +1653,10 @@ def _get_package_versions_map(self, name): self.packages[normalized_name] = versions except RemoteNotFetchedException as e: if TRACE: - print( - f"failed to fetch package name: {name} from: {self.index_url}:\n{e}") + print(f"failed to fetch package name: {name} from: {self.index_url}:\n{e}") if not versions and TRACE: - print( - f"WARNING: package {name} not found in repo: {self.index_url}") + print(f"WARNING: package {name} not found in repo: {self.index_url}") return versions @@ -1861,8 +1841,7 @@ def get(self, path_or_url, as_text=True, force=False): if force or not os.path.exists(cached): if TRACE_DEEP: print(f" FILE CACHE MISS: {path_or_url}") - content = get_file_content( - path_or_url=path_or_url, as_text=as_text) + content = get_file_content(path_or_url=path_or_url, as_text=as_text) wmode = "w" if as_text else "wb" with open(cached, wmode) as fo: fo.write(content) @@ -1884,8 +1863,7 @@ def get_file_content(path_or_url, as_text=True): if path_or_url.startswith("https://"): if TRACE_DEEP: print(f"Fetching: {path_or_url}") - _headers, content = get_remote_file_content( - url=path_or_url, as_text=as_text) + _headers, content = get_remote_file_content(url=path_or_url, as_text=as_text) return content elif path_or_url.startswith("file://") or ( @@ -1936,7 +1914,7 @@ def get_remote_file_content( # several redirects and that we can ignore content there. A HEAD request may # not get us this last header print(f" DOWNLOADING: {url}") - with requests.get(url, allow_redirects=True, stream=True, headers=headers) as response: # noqa: S113 + with requests.get(url, allow_redirects=True, stream=True, headers=headers) as response: # noqa: S113 status = response.status_code if status != requests.codes.ok: # NOQA if status == 429 and _delay < 20: @@ -1951,8 +1929,7 @@ def get_remote_file_content( ) else: - raise RemoteNotFetchedException( - f"Failed HTTP request from {url} with {status}") + raise RemoteNotFetchedException(f"Failed HTTP request from {url} with {status}") if headers_only: return response.headers, None @@ -2043,8 +2020,7 @@ def get_other_dists(_package, _dist): # if has key data we may look to improve later, but we can move on if local_dist.has_key_metadata(): local_dist.save_about_and_notice_files(dest_dir=dest_dir) - local_dist.fetch_license_files( - dest_dir=dest_dir, use_cached_index=use_cached_index) + local_dist.fetch_license_files(dest_dir=dest_dir, use_cached_index=use_cached_index) continue # lets try to get from another dist of the same local package @@ -2056,8 +2032,7 @@ def get_other_dists(_package, _dist): # if has key data we may look to improve later, but we can move on if local_dist.has_key_metadata(): local_dist.save_about_and_notice_files(dest_dir=dest_dir) - local_dist.fetch_license_files( - dest_dir=dest_dir, use_cached_index=use_cached_index) + local_dist.fetch_license_files(dest_dir=dest_dir, use_cached_index=use_cached_index) continue # try to get another version of the same package that is not our version @@ -2068,8 +2043,7 @@ def get_other_dists(_package, _dist): ] other_local_version = other_local_packages and other_local_packages[-1] if other_local_version: - latest_local_dists = list( - other_local_version.get_distributions()) + latest_local_dists = list(other_local_version.get_distributions()) for latest_local_dist in latest_local_dists: latest_local_dist.load_about_data(dest_dir=dest_dir) if not latest_local_dist.has_key_metadata(): @@ -2095,8 +2069,7 @@ def get_other_dists(_package, _dist): # if has key data we may look to improve later, but we can move on if local_dist.has_key_metadata(): local_dist.save_about_and_notice_files(dest_dir=dest_dir) - local_dist.fetch_license_files( - dest_dir=dest_dir, use_cached_index=use_cached_index) + local_dist.fetch_license_files(dest_dir=dest_dir, use_cached_index=use_cached_index) continue # try to get a latest version of the same package that is not our version @@ -2137,8 +2110,7 @@ def get_other_dists(_package, _dist): # if local_dist.has_key_metadata() or not local_dist.has_key_metadata(): local_dist.save_about_and_notice_files(dest_dir) - lic_errs = local_dist.fetch_license_files( - dest_dir, use_cached_index=use_cached_index) + lic_errs = local_dist.fetch_license_files(dest_dir, use_cached_index=use_cached_index) if not local_dist.has_key_metadata(): print(f"Unable to add essential ABOUT data for: {local_dist}") @@ -2161,10 +2133,9 @@ def call(args, verbose=TRACE): """ if TRACE_DEEP: print("Calling:", " ".join(args)) - with subprocess.Popen( # noqa: S603 + with subprocess.Popen( # noqa: S603 args, stdout=subprocess.PIPE, stderr=subprocess.PIPE, encoding="utf-8" ) as process: - stdouts = [] while True: line = process.stdout.readline() @@ -2260,7 +2231,7 @@ def download_wheels_with_pip( def check_about(dest_dir=THIRDPARTY_DIR): try: - subprocess.check_output(f"venv/bin/about check {dest_dir}".split()) # noqa: S603 + subprocess.check_output(f"venv/bin/about check {dest_dir}".split()) # noqa: S603 except subprocess.CalledProcessError as cpe: print() print("Invalid ABOUT files:") @@ -2286,8 +2257,7 @@ def find_problems( for dist in package.get_distributions(): dist.load_about_data(dest_dir=dest_dir) - abpth = os.path.abspath(os.path.join( - dest_dir, dist.about_filename)) + abpth = os.path.abspath(os.path.join(dest_dir, dist.about_filename)) if not dist.has_key_metadata(): print(f" Missing key ABOUT data in file://{abpth}") if "classifiers" in dist.extra_data: From 1189dda52570ed018f52eacd38c4990e8be8ff1a Mon Sep 17 00:00:00 2001 From: Philippe Ombredanne Date: Sat, 29 Mar 2025 22:02:19 +0100 Subject: [PATCH 34/69] Apply cosmetic refactorings Signed-off-by: Philippe Ombredanne --- docs/source/conf.py | 3 ++- etc/scripts/check_thirdparty.py | 4 +--- etc/scripts/fetch_thirdparty.py | 17 ++++++++--------- etc/scripts/gen_pypi_simple.py | 15 +++++++-------- etc/scripts/gen_requirements.py | 4 ++-- etc/scripts/gen_requirements_dev.py | 4 ++-- .../test_utils_pip_compatibility_tags.py | 9 +++++---- etc/scripts/utils_dejacode.py | 9 +++++---- etc/scripts/utils_pip_compatibility_tags.py | 8 +++++--- etc/scripts/utils_requirements.py | 3 +-- etc/scripts/utils_thirdparty.py | 3 ++- 11 files changed, 40 insertions(+), 39 deletions(-) diff --git a/docs/source/conf.py b/docs/source/conf.py index 8c88fa2..8aad829 100644 --- a/docs/source/conf.py +++ b/docs/source/conf.py @@ -94,7 +94,8 @@ html_show_sphinx = True # Define CSS and HTML abbreviations used in .rst files. These are examples. -# .. role:: is used to refer to styles defined in _static/theme_overrides.css and is used like this: :red:`text` +# .. role:: is used to refer to styles defined in _static/theme_overrides.css +# and is used like this: :red:`text` rst_prolog = """ .. |psf| replace:: Python Software Foundation diff --git a/etc/scripts/check_thirdparty.py b/etc/scripts/check_thirdparty.py index 1aa4e28..bb8347a 100644 --- a/etc/scripts/check_thirdparty.py +++ b/etc/scripts/check_thirdparty.py @@ -1,5 +1,4 @@ #!/usr/bin/env python -# -*- coding: utf-8 -*- # # Copyright (c) nexB Inc. and others. All rights reserved. # ScanCode is a trademark of nexB Inc. @@ -41,8 +40,7 @@ def check_thirdparty_dir( """ Check a thirdparty directory for problems and print these on screen. """ - # check for problems - print(f"==> CHECK FOR PROBLEMS") + print("==> CHECK FOR PROBLEMS") utils_thirdparty.find_problems( dest_dir=dest, report_missing_sources=sdists, diff --git a/etc/scripts/fetch_thirdparty.py b/etc/scripts/fetch_thirdparty.py index c224683..76a19a6 100644 --- a/etc/scripts/fetch_thirdparty.py +++ b/etc/scripts/fetch_thirdparty.py @@ -1,5 +1,4 @@ #!/usr/bin/env python -# -*- coding: utf-8 -*- # # Copyright (c) nexB Inc. and others. All rights reserved. # ScanCode is a trademark of nexB Inc. @@ -10,7 +9,6 @@ # import itertools -import os import sys from collections import defaultdict @@ -109,7 +107,8 @@ @click.option( "--use-cached-index", is_flag=True, - help="Use on disk cached PyPI indexes list of packages and versions and do not refetch if present.", + help="Use on disk cached PyPI indexes list of packages and versions and " + "do not refetch if present.", ) @click.option( "--sdist-only", @@ -261,7 +260,7 @@ def fetch_thirdparty( if not fetched: wheels_or_sdist_not_found[f"{name}=={version}"].append(environment) if TRACE: - print(f" NOT FOUND") + print(" NOT FOUND") if sdists or (f"{name}=={version}" in wheels_or_sdist_not_found and name in sdist_only): if TRACE: @@ -276,17 +275,17 @@ def fetch_thirdparty( if not fetched: wheels_or_sdist_not_found[f"{name}=={version}"].append("sdist") if TRACE: - print(f" NOT FOUND") + print(" NOT FOUND") mia = [] for nv, dists in wheels_or_sdist_not_found.items(): name, _, version = nv.partition("==") if name in no_dist: continue - sdist_missing = sdists and "sdist" in dists and not name in wheel_only + sdist_missing = sdists and "sdist" in dists and name not in wheel_only if sdist_missing: mia.append(f"SDist missing: {nv} {dists}") - wheels_missing = wheels and any(d for d in dists if d != "sdist") and not name in sdist_only + wheels_missing = wheels and any(d for d in dists if d != "sdist") and name not in sdist_only if wheels_missing: mia.append(f"Wheels missing: {nv} {dists}") @@ -295,12 +294,12 @@ def fetch_thirdparty( print(m) raise Exception(mia) - print(f"==> FETCHING OR CREATING ABOUT AND LICENSE FILES") + print("==> FETCHING OR CREATING ABOUT AND LICENSE FILES") utils_thirdparty.fetch_abouts_and_licenses(dest_dir=dest_dir, use_cached_index=use_cached_index) utils_thirdparty.clean_about_files(dest_dir=dest_dir) # check for problems - print(f"==> CHECK FOR PROBLEMS") + print("==> CHECK FOR PROBLEMS") utils_thirdparty.find_problems( dest_dir=dest_dir, report_missing_sources=sdists, diff --git a/etc/scripts/gen_pypi_simple.py b/etc/scripts/gen_pypi_simple.py index cfe68e6..89d0626 100644 --- a/etc/scripts/gen_pypi_simple.py +++ b/etc/scripts/gen_pypi_simple.py @@ -1,5 +1,4 @@ #!/usr/bin/env python -# -*- coding: utf-8 -*- # SPDX-License-Identifier: BSD-2-Clause-Views AND MIT # Copyright (c) 2010 David Wolever . All rights reserved. @@ -132,7 +131,7 @@ def build_links_package_index(packages_by_package_name, base_url): Return an HTML document as string which is a links index of all packages """ document = [] - header = f""" + header = """ Links for all packages @@ -177,13 +176,13 @@ def simple_index_entry(self, base_url): def build_pypi_index(directory, base_url="https://thirdparty.aboutcode.org/pypi"): """ - Using a ``directory`` directory of wheels and sdists, create the a PyPI - simple directory index at ``directory``/simple/ populated with the proper - PyPI simple index directory structure crafted using symlinks. + Create the a PyPI simple directory index using a ``directory`` directory of wheels and sdists in + the direvctory at ``directory``/simple/ populated with the proper PyPI simple index directory + structure crafted using symlinks. - WARNING: The ``directory``/simple/ directory is removed if it exists. - NOTE: in addition to the a PyPI simple index.html there is also a links.html - index file generated which is suitable to use with pip's --find-links + WARNING: The ``directory``/simple/ directory is removed if it exists. NOTE: in addition to the a + PyPI simple index.html there is also a links.html index file generated which is suitable to use + with pip's --find-links """ directory = Path(directory) diff --git a/etc/scripts/gen_requirements.py b/etc/scripts/gen_requirements.py index 2b65ae8..1b87944 100644 --- a/etc/scripts/gen_requirements.py +++ b/etc/scripts/gen_requirements.py @@ -1,5 +1,4 @@ #!/usr/bin/env python -# -*- coding: utf-8 -*- # # Copyright (c) nexB Inc. and others. All rights reserved. # ScanCode is a trademark of nexB Inc. @@ -34,7 +33,8 @@ def gen_requirements(): type=pathlib.Path, required=True, metavar="DIR", - help="Path to the 'site-packages' directory where wheels are installed such as lib/python3.6/site-packages", + help="Path to the 'site-packages' directory where wheels are installed " + "such as lib/python3.12/site-packages", ) parser.add_argument( "-r", diff --git a/etc/scripts/gen_requirements_dev.py b/etc/scripts/gen_requirements_dev.py index 5db1c48..8548205 100644 --- a/etc/scripts/gen_requirements_dev.py +++ b/etc/scripts/gen_requirements_dev.py @@ -1,5 +1,4 @@ #!/usr/bin/env python -# -*- coding: utf-8 -*- # # Copyright (c) nexB Inc. and others. All rights reserved. # ScanCode is a trademark of nexB Inc. @@ -36,7 +35,8 @@ def gen_dev_requirements(): type=pathlib.Path, required=True, metavar="DIR", - help='Path to the "site-packages" directory where wheels are installed such as lib/python3.6/site-packages', + help="Path to the 'site-packages' directory where wheels are installed " + "such as lib/python3.12/site-packages", ) parser.add_argument( "-d", diff --git a/etc/scripts/test_utils_pip_compatibility_tags.py b/etc/scripts/test_utils_pip_compatibility_tags.py index a33b8b3..de4b706 100644 --- a/etc/scripts/test_utils_pip_compatibility_tags.py +++ b/etc/scripts/test_utils_pip_compatibility_tags.py @@ -1,4 +1,5 @@ -"""Generate and work with PEP 425 Compatibility Tags. +""" +Generate and work with PEP 425 Compatibility Tags. copied from pip-20.3.1 pip/tests/unit/test_utils_compatibility_tags.py download_url: https://raw.githubusercontent.com/pypa/pip/20.3.1/tests/unit/test_utils_compatibility_tags.py @@ -50,7 +51,7 @@ def test_version_info_to_nodot(version_info, expected): assert actual == expected -class Testcompatibility_tags(object): +class Testcompatibility_tags: def mock_get_config_var(self, **kwd): """ Patch sysconfig.get_config_var for arbitrary keys. @@ -81,7 +82,7 @@ def test_no_hyphen_tag(self): assert "-" not in tag.platform -class TestManylinux2010Tags(object): +class TestManylinux2010Tags: @pytest.mark.parametrize( "manylinux2010,manylinux1", [ @@ -104,7 +105,7 @@ def test_manylinux2010_implies_manylinux1(self, manylinux2010, manylinux1): assert arches[:2] == [manylinux2010, manylinux1] -class TestManylinux2014Tags(object): +class TestManylinux2014Tags: @pytest.mark.parametrize( "manylinuxA,manylinuxB", [ diff --git a/etc/scripts/utils_dejacode.py b/etc/scripts/utils_dejacode.py index cd39cda..b6bff51 100644 --- a/etc/scripts/utils_dejacode.py +++ b/etc/scripts/utils_dejacode.py @@ -1,5 +1,4 @@ #!/usr/bin/env python -# -*- coding: utf-8 -*- # # Copyright (c) nexB Inc. and others. All rights reserved. # ScanCode is a trademark of nexB Inc. @@ -25,7 +24,7 @@ DEJACODE_API_URL_PACKAGES = f"{DEJACODE_API_URL}packages/" DEJACODE_API_HEADERS = { - "Authorization": "Token {}".format(DEJACODE_API_KEY), + "Authorization": f"Token {DEJACODE_API_KEY}", "Accept": "application/json; indent=4", } @@ -50,6 +49,7 @@ def fetch_dejacode_packages(params): DEJACODE_API_URL_PACKAGES, params=params, headers=DEJACODE_API_HEADERS, + timeout=10, ) return response.json()["results"] @@ -93,7 +93,7 @@ def update_with_dejacode_about_data(distribution): if package_data: package_api_url = package_data["api_url"] about_url = f"{package_api_url}about" - response = requests.get(about_url, headers=DEJACODE_API_HEADERS) + response = requests.get(about_url, headers=DEJACODE_API_HEADERS, timeout=10) # note that this is YAML-formatted about_text = response.json()["about_data"] about_data = saneyaml.load(about_text) @@ -113,7 +113,7 @@ def fetch_and_save_about_files(distribution, dest_dir="thirdparty"): if package_data: package_api_url = package_data["api_url"] about_url = f"{package_api_url}about_files" - response = requests.get(about_url, headers=DEJACODE_API_HEADERS) + response = requests.get(about_url, headers=DEJACODE_API_HEADERS, timeout=10) about_zip = response.content with io.BytesIO(about_zip) as zf: with zipfile.ZipFile(zf) as zi: @@ -201,6 +201,7 @@ def create_dejacode_package(distribution): DEJACODE_API_URL_PACKAGES, data=new_package_payload, headers=DEJACODE_API_HEADERS, + timeout=10, ) new_package_data = response.json() if response.status_code != 201: diff --git a/etc/scripts/utils_pip_compatibility_tags.py b/etc/scripts/utils_pip_compatibility_tags.py index de0ac95..dd954bc 100644 --- a/etc/scripts/utils_pip_compatibility_tags.py +++ b/etc/scripts/utils_pip_compatibility_tags.py @@ -1,4 +1,5 @@ -"""Generate and work with PEP 425 Compatibility Tags. +""" +Generate and work with PEP 425 Compatibility Tags. copied from pip-20.3.1 pip/_internal/utils/compatibility_tags.py download_url: https://github.com/pypa/pip/blob/20.3.1/src/pip/_internal/utils/compatibility_tags.py @@ -130,7 +131,7 @@ def _get_custom_interpreter(implementation=None, version=None): implementation = interpreter_name() if version is None: version = interpreter_version() - return "{}{}".format(implementation, version) + return f"{implementation}{version}" def get_supported( @@ -140,7 +141,8 @@ def get_supported( abis=None, # type: Optional[List[str]] ): # type: (...) -> List[Tag] - """Return a list of supported tags for each version specified in + """ + Return a list of supported tags for each version specified in `versions`. :param version: a string version, of the form "33" or "32", diff --git a/etc/scripts/utils_requirements.py b/etc/scripts/utils_requirements.py index 167bc9f..b9b2c0e 100644 --- a/etc/scripts/utils_requirements.py +++ b/etc/scripts/utils_requirements.py @@ -1,5 +1,4 @@ #!/usr/bin/env python -# -*- coding: utf-8 -*- # # Copyright (c) nexB Inc. and others. All rights reserved. # ScanCode is a trademark of nexB Inc. @@ -40,7 +39,7 @@ def get_required_name_versions(requirement_lines, with_unpinned=False): req_line = req_line.strip() if not req_line or req_line.startswith("#"): continue - if req_line.startswith("-") or (not with_unpinned and not "==" in req_line): + if req_line.startswith("-") or (not with_unpinned and "==" not in req_line): print(f"Requirement line is not supported: ignored: {req_line}") continue yield get_required_name_version(requirement=req_line, with_unpinned=with_unpinned) diff --git a/etc/scripts/utils_thirdparty.py b/etc/scripts/utils_thirdparty.py index 4ea1bab..aafc1d6 100644 --- a/etc/scripts/utils_thirdparty.py +++ b/etc/scripts/utils_thirdparty.py @@ -91,7 +91,8 @@ - parse requirement file - create a TODO queue of requirements to process -- done: create an empty map of processed binary requirements as {package name: (list of versions/tags} +- done: create an empty map of processed binary requirements as + {package name: (list of versions/tags} - while we have package reqs in TODO queue, process one requirement: From 84257fbe200e5780cb13536a5c8eb56a88539e6a Mon Sep 17 00:00:00 2001 From: Philippe Ombredanne Date: Sat, 29 Mar 2025 22:05:23 +0100 Subject: [PATCH 35/69] Reformat test code Signed-off-by: Philippe Ombredanne --- .gitignore | 1 + pyproject.toml | 19 +++++++++++-------- tests/test_skeleton_codestyle.py | 25 ++++++++++++++++--------- 3 files changed, 28 insertions(+), 17 deletions(-) diff --git a/.gitignore b/.gitignore index 2d48196..8a93c94 100644 --- a/.gitignore +++ b/.gitignore @@ -72,3 +72,4 @@ tcl # Ignore Jupyter Notebook related temp files .ipynb_checkpoints/ +/.ruff_cache/ diff --git a/pyproject.toml b/pyproject.toml index ba55770..a872ab3 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -67,16 +67,17 @@ include = [ [tool.ruff.lint] # Rules: https://docs.astral.sh/ruff/rules/ select = [ - "E", # pycodestyle - "W", # pycodestyle warnings - "D", # pydocstyle - "F", # Pyflakes - "UP", # pyupgrade - "S", # flake8-bandit +# "E", # pycodestyle +# "W", # pycodestyle warnings +# "D", # pydocstyle +# "F", # Pyflakes +# "UP", # pyupgrade +# "S", # flake8-bandit "I", # isort - "C9", # McCabe complexity +# "C9", # McCabe complexity ] -ignore = ["D1", "D200", "D203", "D205", "D212", "D400", "D415"] +ignore = ["D1", "D200", "D202", "D203", "D205", "D212", "D400", "D415"] + [tool.ruff.lint.isort] force-single-line = true @@ -100,3 +101,5 @@ max-complexity = 10 [tool.ruff.lint.per-file-ignores] # Place paths of files to be ignored by ruff here +"tests/*" = ["S101"] +"test_*.py" = ["S101"] diff --git a/tests/test_skeleton_codestyle.py b/tests/test_skeleton_codestyle.py index b4ce8c1..8cd85c9 100644 --- a/tests/test_skeleton_codestyle.py +++ b/tests/test_skeleton_codestyle.py @@ -7,30 +7,37 @@ # See https://aboutcode.org for more information about nexB OSS projects. # +import configparser import subprocess import unittest -import configparser - class BaseTests(unittest.TestCase): def test_skeleton_codestyle(self): - """ - This test shouldn't run in proliferated repositories. - """ + # This test shouldn't run in proliferated repositories. + + # TODO: update with switch to pyproject.toml setup_cfg = configparser.ConfigParser() setup_cfg.read("setup.cfg") if setup_cfg["metadata"]["name"] != "skeleton": return - args = "venv/bin/black --check -l 100 setup.py etc tests" + commands = [ + ["venv/bin/ruff", "--check"], + ["venv/bin/ruff", "format", "--check"], + ] + command = None try: - subprocess.check_output(args.split()) + for command in commands: + subprocess.check_output(command) # noqa: S603 except subprocess.CalledProcessError as e: print("===========================================================") print(e.output) print("===========================================================") raise Exception( - "Black style check failed; please format the code using:\n" - " python -m black -l 100 setup.py etc tests", + f"Code style and linting command check failed: {' '.join(command)!r}.\n" + "You can check and format the code using:\n" + " make valid\n", + "OR:\n ruff format\n", + " ruff check --fix\n", e.output, ) from e From 00684f733a0873bd837af85471814907ba93f456 Mon Sep 17 00:00:00 2001 From: Philippe Ombredanne Date: Sat, 29 Mar 2025 22:08:25 +0100 Subject: [PATCH 36/69] Format code Signed-off-by: Philippe Ombredanne --- etc/scripts/check_thirdparty.py | 1 + etc/scripts/test_utils_pip_compatibility_tags.py | 1 + tests/test_skeleton_codestyle.py | 1 + 3 files changed, 3 insertions(+) diff --git a/etc/scripts/check_thirdparty.py b/etc/scripts/check_thirdparty.py index bb8347a..65ae595 100644 --- a/etc/scripts/check_thirdparty.py +++ b/etc/scripts/check_thirdparty.py @@ -11,6 +11,7 @@ import utils_thirdparty + @click.command() @click.option( "-d", diff --git a/etc/scripts/test_utils_pip_compatibility_tags.py b/etc/scripts/test_utils_pip_compatibility_tags.py index de4b706..0e9c360 100644 --- a/etc/scripts/test_utils_pip_compatibility_tags.py +++ b/etc/scripts/test_utils_pip_compatibility_tags.py @@ -33,6 +33,7 @@ import utils_pip_compatibility_tags + @pytest.mark.parametrize( "version_info, expected", [ diff --git a/tests/test_skeleton_codestyle.py b/tests/test_skeleton_codestyle.py index 8cd85c9..7135ac0 100644 --- a/tests/test_skeleton_codestyle.py +++ b/tests/test_skeleton_codestyle.py @@ -11,6 +11,7 @@ import subprocess import unittest + class BaseTests(unittest.TestCase): def test_skeleton_codestyle(self): # This test shouldn't run in proliferated repositories. From 7c4278df4e8acf04888a188d115b4c687060f1e5 Mon Sep 17 00:00:00 2001 From: Philippe Ombredanne Date: Sat, 29 Mar 2025 22:10:45 +0100 Subject: [PATCH 37/69] Refine ruff configuration Signed-off-by: Philippe Ombredanne --- pyproject.toml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/pyproject.toml b/pyproject.toml index a872ab3..0f8bd58 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -72,11 +72,11 @@ select = [ # "D", # pydocstyle # "F", # Pyflakes # "UP", # pyupgrade -# "S", # flake8-bandit + "S", # flake8-bandit "I", # isort # "C9", # McCabe complexity ] -ignore = ["D1", "D200", "D202", "D203", "D205", "D212", "D400", "D415"] +ignore = ["D1", "D200", "D202", "D203", "D205", "D212", "D400", "D415", "I001"] [tool.ruff.lint.isort] From 47cb840db13d3e7328dba8d8e62197cda82e48ec Mon Sep 17 00:00:00 2001 From: Philippe Ombredanne Date: Sat, 29 Mar 2025 22:54:01 +0100 Subject: [PATCH 38/69] Format doc Signed-off-by: Philippe Ombredanne --- AUTHORS.rst | 2 +- README.rst | 29 ++++++++++++++--------------- 2 files changed, 15 insertions(+), 16 deletions(-) diff --git a/AUTHORS.rst b/AUTHORS.rst index 51a19cc..16e2046 100644 --- a/AUTHORS.rst +++ b/AUTHORS.rst @@ -1,3 +1,3 @@ The following organizations or individuals have contributed to this repo: -- +- diff --git a/README.rst b/README.rst index f848b4b..01d0210 100644 --- a/README.rst +++ b/README.rst @@ -1,11 +1,10 @@ A Simple Python Project Skeleton ================================ -This repo attempts to standardize the structure of the Python-based project's repositories using -modern Python packaging and configuration techniques that can then be applied to many repos. - -Using this `blog post`_ as inspiration, this repository serves as the base for all new Python -projects and is mergeable in existing repositories as well. +This repo attempts to standardize the structure of the Python-based project's +repositories using modern Python packaging and configuration techniques. +Using this `blog post`_ as inspiration, this repository serves as the base for +all new Python projects and is mergeable in existing repositories as well. .. _blog post: https://blog.jaraco.com/a-project-skeleton-for-python-projects/ @@ -69,7 +68,7 @@ Release Notes - Drop support for Python 3.8 - Drop support for macOS-11, add support for macOS-14 - + - 2024-02-19: - Replace support in CI of default ubuntu-20.04 by ubuntu-22.04 @@ -86,19 +85,19 @@ Release Notes - Synchronize configure and configure.bat scripts for sanity - Update CI operating system support with latest Azure OS images - - Streamline utility scripts in etc/scripts/ to create, fetch and manage third-party dependencies - There are now fewer scripts. See etc/scripts/README.rst for details + - Streamline utility scripts in etc/scripts/ to create, fetch and manage third-party + dependencies. There are now fewer scripts. See etc/scripts/README.rst for details - 2021-09-03: - - - ``configure`` now requires pinned dependencies via the use of ``requirements.txt`` and ``requirements-dev.txt`` - + - ``configure`` now requires pinned dependencies via the use of ``requirements.txt`` + and ``requirements-dev.txt`` - ``configure`` can now accept multiple options at once - Add utility scripts from scancode-toolkit/etc/release/ for use in generating project files - Rename virtual environment directory from ``tmp`` to ``venv`` - - Update README.rst with instructions for generating ``requirements.txt`` and ``requirements-dev.txt``, - as well as collecting dependencies as wheels and generating ABOUT files for them. + - Update README.rst with instructions for generating ``requirements.txt`` + and ``requirements-dev.txt``, as well as collecting dependencies as wheels and generating + ABOUT files for them. - 2021-05-11: - - - Adopt new configure scripts from ScanCode TK that allows correct configuration of which Python version is used. + - Adopt new configure scripts from ScanCode TK that allows correct configuration of which + Python version is used. From 7b29b5914a443069a7ff967eb4ef096034333248 Mon Sep 17 00:00:00 2001 From: Philippe Ombredanne Date: Sat, 29 Mar 2025 22:54:35 +0100 Subject: [PATCH 39/69] Run doc8 on all rst files Signed-off-by: Philippe Ombredanne --- Makefile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Makefile b/Makefile index 930e801..debc404 100644 --- a/Makefile +++ b/Makefile @@ -19,7 +19,7 @@ dev: doc8: @echo "-> Run doc8 validation" - @${ACTIVATE} doc8 --max-line-length 100 --ignore-path docs/_build/ --quiet docs/ + @${ACTIVATE} doc8 --max-line-length 100 --ignore-path docs/_build/ --quiet docs/ *.rst valid: @echo "-> Run Ruff format" From 86c7ca45d3132e5f6873658ab1743b6e27cfeb58 Mon Sep 17 00:00:00 2001 From: Philippe Ombredanne Date: Sat, 29 Mar 2025 22:55:20 +0100 Subject: [PATCH 40/69] Enable doc style checks Signed-off-by: Philippe Ombredanne --- pyproject.toml | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/pyproject.toml b/pyproject.toml index 0f8bd58..51761ff 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -61,7 +61,8 @@ include = [ "etc/**/*.py", "test/**/*.py", "doc/**/*", - "*.py" + "*.py", + "." ] [tool.ruff.lint] @@ -69,10 +70,10 @@ include = [ select = [ # "E", # pycodestyle # "W", # pycodestyle warnings -# "D", # pydocstyle + "D", # pydocstyle # "F", # Pyflakes # "UP", # pyupgrade - "S", # flake8-bandit +# "S", # flake8-bandit "I", # isort # "C9", # McCabe complexity ] From 71583c5ecdfe5dd352b7f2bb9d26deaad971e151 Mon Sep 17 00:00:00 2001 From: Philippe Ombredanne Date: Sun, 30 Mar 2025 14:40:36 +0200 Subject: [PATCH 41/69] Do not format more test data Signed-off-by: Philippe Ombredanne --- pyproject.toml | 15 +++++++++++++++ 1 file changed, 15 insertions(+) diff --git a/pyproject.toml b/pyproject.toml index 51761ff..7d807eb 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -60,10 +60,25 @@ include = [ "src/**/*.py", "etc/**/*.py", "test/**/*.py", + "tests/**/*.py", "doc/**/*", + "docs/**/*", "*.py", "." ] +# ignore test data and testfiles: they should never be linted nor formatted +exclude = [ +# main style + "**/tests/data/**/*", +# scancode-toolkit + "**/tests/*/data/**/*", +# dejacode, purldb + "**/tests/testfiles/**/*", +# vulnerablecode, fetchcode + "**/tests/*/test_data/**/*", + "**/tests/test_data/**/*", +] + [tool.ruff.lint] # Rules: https://docs.astral.sh/ruff/rules/ From 0f1a40382bdcadf82512395787faab50927256f6 Mon Sep 17 00:00:00 2001 From: Philippe Ombredanne Date: Sun, 30 Mar 2025 14:58:36 +0200 Subject: [PATCH 42/69] Do not treat rst as Python Signed-off-by: Philippe Ombredanne --- pyproject.toml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/pyproject.toml b/pyproject.toml index 7d807eb..5e16b56 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -61,8 +61,8 @@ include = [ "etc/**/*.py", "test/**/*.py", "tests/**/*.py", - "doc/**/*", - "docs/**/*", + "doc/**/*.py", + "docs/**/*.py", "*.py", "." ] From 6bea6577864bf432438dabaed9e46a721aae2961 Mon Sep 17 00:00:00 2001 From: Philippe Ombredanne Date: Sun, 30 Mar 2025 16:41:57 +0200 Subject: [PATCH 43/69] Combine testing and docs extra for simplicity Signed-off-by: Philippe Ombredanne --- configure | 2 -- configure.bat | 4 ---- setup.cfg | 3 --- 3 files changed, 9 deletions(-) diff --git a/configure b/configure index 22d9288..83fd203 100755 --- a/configure +++ b/configure @@ -30,7 +30,6 @@ CLI_ARGS=$1 # Requirement arguments passed to pip and used by default or with --dev. REQUIREMENTS="--editable . --constraint requirements.txt" DEV_REQUIREMENTS="--editable .[testing] --constraint requirements.txt --constraint requirements-dev.txt" -DOCS_REQUIREMENTS="--editable .[docs] --constraint requirements.txt" # where we create a virtualenv VIRTUALENV_DIR=venv @@ -185,7 +184,6 @@ while getopts :-: optchar; do help ) cli_help;; clean ) find_python && clean;; dev ) CFG_REQUIREMENTS="$DEV_REQUIREMENTS";; - docs ) CFG_REQUIREMENTS="$DOCS_REQUIREMENTS";; esac;; esac done diff --git a/configure.bat b/configure.bat index 5b9a9d6..18b3703 100644 --- a/configure.bat +++ b/configure.bat @@ -28,7 +28,6 @@ @rem # Requirement arguments passed to pip and used by default or with --dev. set "REQUIREMENTS=--editable . --constraint requirements.txt" set "DEV_REQUIREMENTS=--editable .[testing] --constraint requirements.txt --constraint requirements-dev.txt" -set "DOCS_REQUIREMENTS=--editable .[docs] --constraint requirements.txt" @rem # where we create a virtualenv set "VIRTUALENV_DIR=venv" @@ -76,9 +75,6 @@ if not "%1" == "" ( if "%1" EQU "--dev" ( set "CFG_REQUIREMENTS=%DEV_REQUIREMENTS%" ) - if "%1" EQU "--docs" ( - set "CFG_REQUIREMENTS=%DOCS_REQUIREMENTS%" - ) shift goto again ) diff --git a/setup.cfg b/setup.cfg index aaec643..ad8e0d8 100644 --- a/setup.cfg +++ b/setup.cfg @@ -55,8 +55,6 @@ testing = pycodestyle >= 2.8.0 twine ruff - -docs = Sphinx>=5.0.2 sphinx-rtd-theme>=1.0.0 sphinx-reredirects >= 0.1.2 @@ -64,4 +62,3 @@ docs = sphinx-autobuild sphinx-rtd-dark-mode>=1.3.0 sphinx-copybutton - From c615589b54bfac74b6f17b2233b2afe60bf1d0f6 Mon Sep 17 00:00:00 2001 From: Philippe Ombredanne Date: Sun, 30 Mar 2025 17:18:19 +0200 Subject: [PATCH 44/69] Refine checking of docs with doc8 Signed-off-by: Philippe Ombredanne --- Makefile | 2 +- pyproject.toml | 7 +++++++ 2 files changed, 8 insertions(+), 1 deletion(-) diff --git a/Makefile b/Makefile index debc404..d21a2f9 100644 --- a/Makefile +++ b/Makefile @@ -19,7 +19,7 @@ dev: doc8: @echo "-> Run doc8 validation" - @${ACTIVATE} doc8 --max-line-length 100 --ignore-path docs/_build/ --quiet docs/ *.rst + @${ACTIVATE} doc8 docs/ *.rst valid: @echo "-> Run Ruff format" diff --git a/pyproject.toml b/pyproject.toml index 5e16b56..bfb1d35 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -119,3 +119,10 @@ max-complexity = 10 # Place paths of files to be ignored by ruff here "tests/*" = ["S101"] "test_*.py" = ["S101"] + + +[tool.doc8] + +ignore-path = ["docs/build", "doc/build", "docs/_build", "doc/_build"] +max-line-length=100 +verbose=0 From 04e0a89a3bbf5359f27b6e3ef9a5026638e63de8 Mon Sep 17 00:00:00 2001 From: Philippe Ombredanne Date: Sun, 30 Mar 2025 18:41:00 +0200 Subject: [PATCH 45/69] Refine doc handling * remove CI scripts and use Makefile targets instead * ensure doc8 runs quiet * add new docs-check make target to run documentation and links checks * update oudated doc for docs contribution Signed-off-by: Philippe Ombredanne --- .github/workflows/docs-ci.yml | 12 +++++------- Makefile | 10 +++++++--- docs/scripts/doc8_style_check.sh | 5 ----- docs/scripts/sphinx_build_link_check.sh | 5 ----- docs/source/conf.py | 2 +- docs/source/contribute/contrib_doc.rst | 8 ++++---- pyproject.toml | 2 -- 7 files changed, 17 insertions(+), 27 deletions(-) delete mode 100755 docs/scripts/doc8_style_check.sh delete mode 100644 docs/scripts/sphinx_build_link_check.sh diff --git a/.github/workflows/docs-ci.yml b/.github/workflows/docs-ci.yml index 621de4b..10ba5fa 100644 --- a/.github/workflows/docs-ci.yml +++ b/.github/workflows/docs-ci.yml @@ -21,14 +21,12 @@ jobs: python-version: ${{ matrix.python-version }} - name: Install Dependencies - run: pip install -e .[docs] + run: ./configure --dev - - name: Check Sphinx Documentation build minimally - working-directory: ./docs - run: sphinx-build -E -W source build + - name: Check documentation and HTML for errors and dead links + run: make docs-check - - name: Check for documentation style errors - working-directory: ./docs - run: ./scripts/doc8_style_check.sh + - name: Check documentation for style errors + run: make doc8 diff --git a/Makefile b/Makefile index d21a2f9..413399e 100644 --- a/Makefile +++ b/Makefile @@ -19,7 +19,7 @@ dev: doc8: @echo "-> Run doc8 validation" - @${ACTIVATE} doc8 docs/ *.rst + @${ACTIVATE} doc8 --quiet docs/ *.rst valid: @echo "-> Run Ruff format" @@ -46,6 +46,10 @@ test: docs: rm -rf docs/_build/ - @${ACTIVATE} sphinx-build docs/ docs/_build/ + @${ACTIVATE} sphinx-build docs/source docs/_build/ -.PHONY: conf dev check valid clean test docs +docs-check: + @${ACTIVATE} sphinx-build -E -W -b html docs/source docs/_build/ + @${ACTIVATE} sphinx-build -E -W -b linkcheck docs/source docs/_build/ + +.PHONY: conf dev check valid clean test docs docs-check diff --git a/docs/scripts/doc8_style_check.sh b/docs/scripts/doc8_style_check.sh deleted file mode 100755 index 9416323..0000000 --- a/docs/scripts/doc8_style_check.sh +++ /dev/null @@ -1,5 +0,0 @@ -#!/bin/bash -# halt script on error -set -e -# Check for Style Code Violations -doc8 --max-line-length 100 source --ignore D000 --quiet \ No newline at end of file diff --git a/docs/scripts/sphinx_build_link_check.sh b/docs/scripts/sphinx_build_link_check.sh deleted file mode 100644 index c542686..0000000 --- a/docs/scripts/sphinx_build_link_check.sh +++ /dev/null @@ -1,5 +0,0 @@ -#!/bin/bash -# halt script on error -set -e -# Build locally, and then check links -sphinx-build -E -W -b linkcheck source build \ No newline at end of file diff --git a/docs/source/conf.py b/docs/source/conf.py index 8aad829..056ca6e 100644 --- a/docs/source/conf.py +++ b/docs/source/conf.py @@ -18,7 +18,7 @@ # -- Project information ----------------------------------------------------- project = "nexb-skeleton" -copyright = "nexB Inc. and others." +copyright = "nexB Inc., AboutCode and others." author = "AboutCode.org authors and contributors" diff --git a/docs/source/contribute/contrib_doc.rst b/docs/source/contribute/contrib_doc.rst index 5640db2..041b358 100644 --- a/docs/source/contribute/contrib_doc.rst +++ b/docs/source/contribute/contrib_doc.rst @@ -147,7 +147,7 @@ What is Checked? ^^^^^^^^^^^^^^^^ PyCQA is an Organization for code quality tools (and plugins) for the Python programming language. -Doc8 is a sub-project of the same Organization. Refer this `README `_ for more details. +Doc8 is a sub-project of the same Organization. Refer this `README `_ for more details. What is checked: @@ -169,11 +169,11 @@ What is checked: Interspinx ---------- -ScanCode toolkit documentation uses `Intersphinx `_ +ScanCode toolkit documentation uses `Intersphinx `_ to link to other Sphinx Documentations, to maintain links to other Aboutcode Projects. To link sections in the same documentation, standart reST labels are used. Refer -`Cross-Referencing `_ for more information. +`Cross-Referencing `_ for more information. For example:: @@ -230,7 +230,7 @@ Style Conventions for the Documentaion 1. Headings - (`Refer `_) + (`Refer `_) Normally, there are no heading levels assigned to certain characters as the structure is determined from the succession of headings. However, this convention is used in Python’s Style Guide for documenting which you may follow: diff --git a/pyproject.toml b/pyproject.toml index bfb1d35..c9e6772 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -122,7 +122,5 @@ max-complexity = 10 [tool.doc8] - ignore-path = ["docs/build", "doc/build", "docs/_build", "doc/_build"] max-line-length=100 -verbose=0 From 8897cc63eb9ef9b06a1fdc77ebfe21289c69961b Mon Sep 17 00:00:00 2001 From: Philippe Ombredanne Date: Sun, 30 Mar 2025 18:49:01 +0200 Subject: [PATCH 46/69] Add twine check to release publication Signed-off-by: Philippe Ombredanne --- .github/workflows/pypi-release.yml | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/.github/workflows/pypi-release.yml b/.github/workflows/pypi-release.yml index a66c9c8..cf0579a 100644 --- a/.github/workflows/pypi-release.yml +++ b/.github/workflows/pypi-release.yml @@ -30,12 +30,15 @@ jobs: with: python-version: 3.12 - - name: Install pypa/build - run: python -m pip install build --user + - name: Install pypa/build and twine + run: python -m pip install --user build twine - name: Build a binary wheel and a source tarball run: python -m build --sdist --wheel --outdir dist/ + - name: Validate wheel and sdis for Pypi + run: python -m twine check dist/* + - name: Upload built archives uses: actions/upload-artifact@v4 with: From 3d42985990860188c5fbf9f64f3fd4d14c590a65 Mon Sep 17 00:00:00 2001 From: Philippe Ombredanne Date: Sun, 30 Mar 2025 19:16:31 +0200 Subject: [PATCH 47/69] Refine doc contribution docs Signed-off-by: Philippe Ombredanne --- docs/source/contribute/contrib_doc.rst | 119 ++++++++----------------- 1 file changed, 38 insertions(+), 81 deletions(-) diff --git a/docs/source/contribute/contrib_doc.rst b/docs/source/contribute/contrib_doc.rst index 041b358..dee9296 100644 --- a/docs/source/contribute/contrib_doc.rst +++ b/docs/source/contribute/contrib_doc.rst @@ -8,109 +8,59 @@ Contributing to the Documentation Setup Local Build ----------------- -To get started, create or identify a working directory on your local machine. +To get started, check out and configure the repository for development:: -Open that directory and execute the following command in a terminal session:: + git clone https://github.com/aboutcode-org/.git - git clone https://github.com/aboutcode-org/skeleton.git + cd your-repo + ./configure --dev -That will create an ``/skeleton`` directory in your working directory. -Now you can install the dependencies in a virtualenv:: - - cd skeleton - ./configure --docs +(Or use "make dev") .. note:: - In case of windows, run ``configure --docs`` instead of this. - -Now, this will install the following prerequisites: - -- Sphinx -- sphinx_rtd_theme (the format theme used by ReadTheDocs) -- docs8 (style linter) + In case of windows, run ``configure --dev``. -These requirements are already present in setup.cfg and `./configure --docs` installs them. +This will install and configure all requirements foer development including for docs development. -Now you can build the HTML documents locally:: +Now you can build the HTML documentation locally:: source venv/bin/activate - cd docs - make html - -Assuming that your Sphinx installation was successful, Sphinx should build a local instance of the -documentation .html files:: - - open build/html/index.html - -.. note:: - - In case this command did not work, for example on Ubuntu 18.04 you may get a message like “Couldn’t - get a file descriptor referring to the console”, try: - - :: - - see build/html/index.html + make docs -You now have a local build of the AboutCode documents. +This will build a local instance of the ``docs/_build`` directory:: -.. _contrib_doc_share_improvements: + open docs/_build/index.html -Share Document Improvements ---------------------------- - -Ensure that you have the latest files:: - - git pull - git status -Before commiting changes run Continious Integration Scripts locally to run tests. Refer -:ref:`doc_ci` for instructions on the same. +To validate the documentation style and content, use:: -Follow standard git procedures to upload your new and modified files. The following commands are -examples:: - - git status - git add source/index.rst - git add source/how-to-scan.rst - git status - git commit -m "New how-to document that explains how to scan" - git status - git push - git status - -The Scancode-Toolkit webhook with ReadTheDocs should rebuild the documentation after your -Pull Request is Merged. + source venv/bin/activate + make doc8 + make docs-check -Refer the `Pro Git Book `_ available online for Git tutorials -covering more complex topics on Branching, Merging, Rebasing etc. .. _doc_ci: Continuous Integration ---------------------- -The documentations are checked on every new commit through Travis-CI, so that common errors are -avoided and documentation standards are enforced. Travis-CI presently checks for these 3 aspects -of the documentation : +The documentations are checked on every new commit, so that common errors are avoided and +documentation standards are enforced. We checks for these aspects of the documentation: 1. Successful Builds (By using ``sphinx-build``) -2. No Broken Links (By Using ``link-check``) -3. Linting Errors (By Using ``Doc8``) +2. No Broken Links (By Using ``linkcheck``) +3. Linting Errors (By Using ``doc8``) -So run these scripts at your local system before creating a Pull Request:: +You myst run these scripts locally before creating a pull request:: - cd docs - ./scripts/sphinx_build_link_check.sh - ./scripts/doc8_style_check.sh + make doc8 + make check-docs -If you don't have permission to run the scripts, run:: - - chmod u+x ./scripts/doc8_style_check.sh .. _doc_style_docs8: -Style Checks Using ``Doc8`` +Style Checks Using ``doc8`` --------------------------- How To Run Style Tests @@ -118,8 +68,7 @@ How To Run Style Tests In the project root, run the following commands:: - $ cd docs - $ ./scripts/doc8_style_check.sh + make doc8 A sample output is:: @@ -143,11 +92,13 @@ A sample output is:: Now fix the errors and run again till there isn't any style error in the documentation. + What is Checked? ^^^^^^^^^^^^^^^^ PyCQA is an Organization for code quality tools (and plugins) for the Python programming language. -Doc8 is a sub-project of the same Organization. Refer this `README `_ for more details. +Doc8 is a sub-project of the same Organization. Refer this +`README `_ for more details. What is checked: @@ -164,16 +115,19 @@ What is checked: - no carriage returns (use UNIX newlines) - D004 - no newline at end of file - D005 + .. _doc_interspinx: Interspinx ---------- -ScanCode toolkit documentation uses `Intersphinx `_ +AboutCode documentation uses +`Intersphinx `_ to link to other Sphinx Documentations, to maintain links to other Aboutcode Projects. To link sections in the same documentation, standart reST labels are used. Refer -`Cross-Referencing `_ for more information. +`Cross-Referencing `_ +for more information. For example:: @@ -223,6 +177,7 @@ Intersphinx, and you link to that label, it will create a link to the local labe For more information, refer this tutorial named `Using Intersphinx `_. + .. _doc_style_conv: Style Conventions for the Documentaion @@ -303,12 +258,14 @@ Style Conventions for the Documentaion ``rst_snippets/warning_snippets/`` and then included to eliminate redundancy, as these are frequently used in multiple files. + Converting from Markdown ------------------------ -If you want to convert a ``.md`` file to a ``.rst`` file, this `tool `_ -does it pretty well. You'd still have to clean up and check for errors as this contains a lot of -bugs. But this is definitely better than converting everything by yourself. +If you want to convert a ``.md`` file to a ``.rst`` file, this +`tool `_ does it pretty well. +You will still have to clean up and check for errors as this contains a lot of bugs. But this is +definitely better than converting everything by yourself. This will be helpful in converting GitHub wiki's (Markdown Files) to reStructuredtext files for Sphinx/ReadTheDocs hosting. From f428366859a143add93160bd0b1ae685be89fcbb Mon Sep 17 00:00:00 2001 From: Jono Yang Date: Mon, 31 Mar 2025 13:35:36 -0700 Subject: [PATCH 48/69] Update codestyle command * Remove trailing whitespace Signed-off-by: Jono Yang --- docs/source/contribute/contrib_doc.rst | 4 ++-- tests/test_skeleton_codestyle.py | 2 +- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/docs/source/contribute/contrib_doc.rst b/docs/source/contribute/contrib_doc.rst index dee9296..2a719a5 100644 --- a/docs/source/contribute/contrib_doc.rst +++ b/docs/source/contribute/contrib_doc.rst @@ -97,7 +97,7 @@ What is Checked? ^^^^^^^^^^^^^^^^ PyCQA is an Organization for code quality tools (and plugins) for the Python programming language. -Doc8 is a sub-project of the same Organization. Refer this +Doc8 is a sub-project of the same Organization. Refer this `README `_ for more details. What is checked: @@ -263,7 +263,7 @@ Converting from Markdown ------------------------ If you want to convert a ``.md`` file to a ``.rst`` file, this -`tool `_ does it pretty well. +`tool `_ does it pretty well. You will still have to clean up and check for errors as this contains a lot of bugs. But this is definitely better than converting everything by yourself. diff --git a/tests/test_skeleton_codestyle.py b/tests/test_skeleton_codestyle.py index 7135ac0..6060c08 100644 --- a/tests/test_skeleton_codestyle.py +++ b/tests/test_skeleton_codestyle.py @@ -23,7 +23,7 @@ def test_skeleton_codestyle(self): return commands = [ - ["venv/bin/ruff", "--check"], + ["venv/bin/ruff", "check"], ["venv/bin/ruff", "format", "--check"], ] command = None From f0d0e21d5e6f98645b02ff9a8fee6ee3def1be75 Mon Sep 17 00:00:00 2001 From: Jono Yang Date: Mon, 31 Mar 2025 13:44:30 -0700 Subject: [PATCH 49/69] Update README.rst Signed-off-by: Jono Yang --- README.rst | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/README.rst b/README.rst index 01d0210..11a4dfb 100644 --- a/README.rst +++ b/README.rst @@ -44,6 +44,10 @@ More usage instructions can be found in ``docs/skeleton-usage.rst``. Release Notes ============= +- 2025-03-31: + + - Use ruff as the main code formatting tool, add ruff rules to pyproject.toml + - 2025-03-29: - Add support for beta macOS-15 From f3a8aa6cee5f645a668750ab7e6bf0cdc774e041 Mon Sep 17 00:00:00 2001 From: Jono Yang Date: Mon, 31 Mar 2025 14:31:37 -0700 Subject: [PATCH 50/69] Update BUILDDIR envvar in docs/Makefile Signed-off-by: Jono Yang --- docs/Makefile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/Makefile b/docs/Makefile index 788b039..94f686b 100644 --- a/docs/Makefile +++ b/docs/Makefile @@ -7,7 +7,7 @@ SPHINXOPTS ?= SPHINXBUILD ?= sphinx-build SPHINXAUTOBUILD = sphinx-autobuild SOURCEDIR = source -BUILDDIR = build +BUILDDIR = _build # Put it first so that "make" without argument is like "make help". help: From 5b0f4d6b4079719caa9ed97efb2ba776bc2bbac1 Mon Sep 17 00:00:00 2001 From: Philippe Ombredanne Date: Tue, 1 Apr 2025 14:42:52 +0200 Subject: [PATCH 51/69] Fix doc line length Signed-off-by: Philippe Ombredanne --- docs/source/contribute/contrib_doc.rst | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/docs/source/contribute/contrib_doc.rst b/docs/source/contribute/contrib_doc.rst index dee9296..2a719a5 100644 --- a/docs/source/contribute/contrib_doc.rst +++ b/docs/source/contribute/contrib_doc.rst @@ -97,7 +97,7 @@ What is Checked? ^^^^^^^^^^^^^^^^ PyCQA is an Organization for code quality tools (and plugins) for the Python programming language. -Doc8 is a sub-project of the same Organization. Refer this +Doc8 is a sub-project of the same Organization. Refer this `README `_ for more details. What is checked: @@ -263,7 +263,7 @@ Converting from Markdown ------------------------ If you want to convert a ``.md`` file to a ``.rst`` file, this -`tool `_ does it pretty well. +`tool `_ does it pretty well. You will still have to clean up and check for errors as this contains a lot of bugs. But this is definitely better than converting everything by yourself. From e776fef5ad595378752d39425109a4cbd2cb5175 Mon Sep 17 00:00:00 2001 From: Philippe Ombredanne Date: Tue, 1 Apr 2025 14:49:40 +0200 Subject: [PATCH 52/69] Format code Signed-off-by: Philippe Ombredanne --- etc/scripts/update_skeleton.py | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/etc/scripts/update_skeleton.py b/etc/scripts/update_skeleton.py index 5705fc4..374c06f 100644 --- a/etc/scripts/update_skeleton.py +++ b/etc/scripts/update_skeleton.py @@ -15,7 +15,7 @@ import click -ABOUTCODE_PUBLIC_REPO_NAMES=[ +ABOUTCODE_PUBLIC_REPO_NAMES = [ "aboutcode-toolkit", "ahocode", "bitcode", @@ -87,7 +87,9 @@ def update_skeleton_files(repo_names=ABOUTCODE_PUBLIC_REPO_NAMES): os.chdir(work_dir_path / repo_name) # Add skeleton as an origin - subprocess.run(["git", "remote", "add", "skeleton", "git@github.com:aboutcode-org/skeleton.git"]) + subprocess.run( + ["git", "remote", "add", "skeleton", "git@github.com:aboutcode-org/skeleton.git"] + ) # Fetch skeleton files subprocess.run(["git", "fetch", "skeleton"]) From 2a43f4cdc8105473b279eea873db00addaa14551 Mon Sep 17 00:00:00 2001 From: Philippe Ombredanne Date: Tue, 1 Apr 2025 14:59:05 +0200 Subject: [PATCH 53/69] Correct supported runner on Azure See for details: https://learn.microsoft.com/en-us/azure/devops/pipelines/agents/hosted?view=azure-devops&tabs=yaml macOS ARM images do not seem to be supported there Signed-off-by: Philippe Ombredanne --- azure-pipelines.yml | 24 ------------------------ 1 file changed, 24 deletions(-) diff --git a/azure-pipelines.yml b/azure-pipelines.yml index 80ae45b..fb03c09 100644 --- a/azure-pipelines.yml +++ b/azure-pipelines.yml @@ -26,14 +26,6 @@ jobs: - template: etc/ci/azure-posix.yml parameters: job_name: macos13_cpython - image_name: macOS-13-xlarge - python_versions: ['3.9', '3.10', '3.11', '3.12', '3.13'] - test_suites: - all: venv/bin/pytest -n 2 -vvs - - - template: etc/ci/azure-posix.yml - parameters: - job_name: macos13_cpython_arm64 image_name: macOS-13 python_versions: ['3.9', '3.10', '3.11', '3.12', '3.13'] test_suites: @@ -42,14 +34,6 @@ jobs: - template: etc/ci/azure-posix.yml parameters: job_name: macos14_cpython - image_name: macOS-14-large - python_versions: ['3.9', '3.10', '3.11', '3.12', '3.13'] - test_suites: - all: venv/bin/pytest -n 2 -vvs - - - template: etc/ci/azure-posix.yml - parameters: - job_name: macos14_cpython_arm64 image_name: macOS-14 python_versions: ['3.9', '3.10', '3.11', '3.12', '3.13'] test_suites: @@ -63,14 +47,6 @@ jobs: test_suites: all: venv/bin/pytest -n 2 -vvs - - template: etc/ci/azure-posix.yml - parameters: - job_name: macos15_cpython_arm64 - image_name: macOS-15-large - python_versions: ['3.9', '3.10', '3.11', '3.12', '3.13'] - test_suites: - all: venv/bin/pytest -n 2 -vvs - - template: etc/ci/azure-win.yml parameters: job_name: win2019_cpython From 4a15550b7bcea5ec949a5049fe1a501d3bb888ff Mon Sep 17 00:00:00 2001 From: Philippe Ombredanne Date: Tue, 1 Apr 2025 19:34:14 +0200 Subject: [PATCH 54/69] Add code checks to CI Remove running "make check" as a test Signed-off-by: Philippe Ombredanne --- azure-pipelines.yml | 8 ++++++ tests/test_skeleton_codestyle.py | 44 -------------------------------- 2 files changed, 8 insertions(+), 44 deletions(-) delete mode 100644 tests/test_skeleton_codestyle.py diff --git a/azure-pipelines.yml b/azure-pipelines.yml index fb03c09..ad18b28 100644 --- a/azure-pipelines.yml +++ b/azure-pipelines.yml @@ -7,6 +7,14 @@ jobs: + - template: etc/ci/azure-posix.yml + parameters: + job_name: run_code_checks + image_name: ubuntu-24.04 + python_versions: ['3.12'] + test_suites: + all: make check + - template: etc/ci/azure-posix.yml parameters: job_name: ubuntu22_cpython diff --git a/tests/test_skeleton_codestyle.py b/tests/test_skeleton_codestyle.py deleted file mode 100644 index 6060c08..0000000 --- a/tests/test_skeleton_codestyle.py +++ /dev/null @@ -1,44 +0,0 @@ -# -# Copyright (c) nexB Inc. and others. All rights reserved. -# ScanCode is a trademark of nexB Inc. -# SPDX-License-Identifier: Apache-2.0 -# See http://www.apache.org/licenses/LICENSE-2.0 for the license text. -# See https://github.com/aboutcode-org/skeleton for support or download. -# See https://aboutcode.org for more information about nexB OSS projects. -# - -import configparser -import subprocess -import unittest - - -class BaseTests(unittest.TestCase): - def test_skeleton_codestyle(self): - # This test shouldn't run in proliferated repositories. - - # TODO: update with switch to pyproject.toml - setup_cfg = configparser.ConfigParser() - setup_cfg.read("setup.cfg") - if setup_cfg["metadata"]["name"] != "skeleton": - return - - commands = [ - ["venv/bin/ruff", "check"], - ["venv/bin/ruff", "format", "--check"], - ] - command = None - try: - for command in commands: - subprocess.check_output(command) # noqa: S603 - except subprocess.CalledProcessError as e: - print("===========================================================") - print(e.output) - print("===========================================================") - raise Exception( - f"Code style and linting command check failed: {' '.join(command)!r}.\n" - "You can check and format the code using:\n" - " make valid\n", - "OR:\n ruff format\n", - " ruff check --fix\n", - e.output, - ) from e From b2d7512735ca257088d2cac4b55590fc5d7b20b4 Mon Sep 17 00:00:00 2001 From: Philippe Ombredanne Date: Tue, 1 Apr 2025 20:15:18 +0200 Subject: [PATCH 55/69] Revert support for Python 3.13 This is not yet supported everywhere Signed-off-by: Philippe Ombredanne --- azure-pipelines.yml | 16 ++++++++-------- 1 file changed, 8 insertions(+), 8 deletions(-) diff --git a/azure-pipelines.yml b/azure-pipelines.yml index ad18b28..7a2d4d9 100644 --- a/azure-pipelines.yml +++ b/azure-pipelines.yml @@ -19,7 +19,7 @@ jobs: parameters: job_name: ubuntu22_cpython image_name: ubuntu-22.04 - python_versions: ['3.9', '3.10', '3.11', '3.12', '3.13'] + python_versions: ['3.9', '3.10', '3.11', '3.12'] test_suites: all: venv/bin/pytest -n 2 -vvs @@ -27,7 +27,7 @@ jobs: parameters: job_name: ubuntu24_cpython image_name: ubuntu-24.04 - python_versions: ['3.9', '3.10', '3.11', '3.12', '3.13'] + python_versions: ['3.9', '3.10', '3.11', '3.12'] test_suites: all: venv/bin/pytest -n 2 -vvs @@ -35,7 +35,7 @@ jobs: parameters: job_name: macos13_cpython image_name: macOS-13 - python_versions: ['3.9', '3.10', '3.11', '3.12', '3.13'] + python_versions: ['3.9', '3.10', '3.11', '3.12'] test_suites: all: venv/bin/pytest -n 2 -vvs @@ -43,7 +43,7 @@ jobs: parameters: job_name: macos14_cpython image_name: macOS-14 - python_versions: ['3.9', '3.10', '3.11', '3.12', '3.13'] + python_versions: ['3.9', '3.10', '3.11', '3.12'] test_suites: all: venv/bin/pytest -n 2 -vvs @@ -51,7 +51,7 @@ jobs: parameters: job_name: macos15_cpython image_name: macOS-15 - python_versions: ['3.9', '3.10', '3.11', '3.12', '3.13'] + python_versions: ['3.9', '3.10', '3.11', '3.12'] test_suites: all: venv/bin/pytest -n 2 -vvs @@ -59,7 +59,7 @@ jobs: parameters: job_name: win2019_cpython image_name: windows-2019 - python_versions: ['3.9', '3.10', '3.11', '3.12', '3.13'] + python_versions: ['3.9', '3.10', '3.11', '3.12'] test_suites: all: venv\Scripts\pytest -n 2 -vvs @@ -67,7 +67,7 @@ jobs: parameters: job_name: win2022_cpython image_name: windows-2022 - python_versions: ['3.9', '3.10', '3.11', '3.12', '3.13'] + python_versions: ['3.9', '3.10', '3.11', '3.12'] test_suites: all: venv\Scripts\pytest -n 2 -vvs @@ -75,6 +75,6 @@ jobs: parameters: job_name: win2025_cpython image_name: windows-2025 - python_versions: ['3.9', '3.10', '3.11', '3.12', '3.13'] + python_versions: ['3.9', '3.10', '3.11', '3.12'] test_suites: all: venv\Scripts\pytest -n 2 -vvs From 2e3464b79811bcf505d93692da2418e2444150ed Mon Sep 17 00:00:00 2001 From: Philippe Ombredanne Date: Wed, 2 Apr 2025 16:52:32 +0200 Subject: [PATCH 56/69] Ignore local .env file Signed-off-by: Philippe Ombredanne --- .gitignore | 1 + 1 file changed, 1 insertion(+) diff --git a/.gitignore b/.gitignore index 8a93c94..4818bb3 100644 --- a/.gitignore +++ b/.gitignore @@ -73,3 +73,4 @@ tcl # Ignore Jupyter Notebook related temp files .ipynb_checkpoints/ /.ruff_cache/ +.env \ No newline at end of file From d4af79f0da82ab0d16dcf60b363a6a5290cd9403 Mon Sep 17 00:00:00 2001 From: Philippe Ombredanne Date: Wed, 2 Apr 2025 16:54:29 +0200 Subject: [PATCH 57/69] Add correct extras for documentation Signed-off-by: Philippe Ombredanne --- .readthedocs.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.readthedocs.yml b/.readthedocs.yml index 8ab2368..7e399c8 100644 --- a/.readthedocs.yml +++ b/.readthedocs.yml @@ -26,4 +26,4 @@ python: - method: pip path: . extra_requirements: - - docs + - testing From 49bfd37c7273f2118d35585c67e53f0cf7642f43 Mon Sep 17 00:00:00 2001 From: Philippe Ombredanne Date: Wed, 2 Apr 2025 16:58:37 +0200 Subject: [PATCH 58/69] Improve MANIFEST Signed-off-by: Philippe Ombredanne --- MANIFEST.in | 14 ++++++++++++-- 1 file changed, 12 insertions(+), 2 deletions(-) diff --git a/MANIFEST.in b/MANIFEST.in index ef3721e..0f19707 100644 --- a/MANIFEST.in +++ b/MANIFEST.in @@ -1,4 +1,6 @@ graft src +graft docs +graft etc include *.LICENSE include NOTICE @@ -6,10 +8,18 @@ include *.ABOUT include *.toml include *.yml include *.rst +include *.png include setup.* include configure* include requirements* -include .git* +include .dockerignore +include .gitignore +include .readthedocs.yml +include manage.py +include Dockerfile* +include Makefile +include MANIFEST.in -global-exclude *.py[co] __pycache__ *.*~ +include .VERSION +global-exclude *.py[co] __pycache__ *.*~ From 5bc987a16cb3ae0f6de101a9c9e277df431f4317 Mon Sep 17 00:00:00 2001 From: Philippe Ombredanne Date: Wed, 2 Apr 2025 17:12:58 +0200 Subject: [PATCH 59/69] Improve cleaning on POSIX Signed-off-by: Philippe Ombredanne --- configure | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/configure b/configure index 83fd203..3dd9a0a 100755 --- a/configure +++ b/configure @@ -35,7 +35,7 @@ DEV_REQUIREMENTS="--editable .[testing] --constraint requirements.txt --constrai VIRTUALENV_DIR=venv # Cleanable files and directories to delete with the --clean option -CLEANABLE="build dist venv .cache .eggs" +CLEANABLE="build dist venv .cache .eggs *.egg-info docs/_build/ pip-selfcheck.json" # extra arguments passed to pip PIP_EXTRA_ARGS=" " @@ -167,6 +167,7 @@ clean() { for cln in $CLEANABLE; do rm -rf "${CFG_ROOT_DIR:?}/${cln:?}"; done + find . -type f -name '*.py[co]' -delete -o -type d -name __pycache__ -delete set +e exit } From 887779a9bd36650ffc6751f0069b492e80dd2f08 Mon Sep 17 00:00:00 2001 From: Philippe Ombredanne Date: Wed, 2 Apr 2025 17:19:44 +0200 Subject: [PATCH 60/69] Rename dev extra to "dev" Instead of testing ... and update references accordingly Signed-off-by: Philippe Ombredanne --- .readthedocs.yml | 2 +- Makefile | 7 ++++++- configure | 2 +- configure.bat | 2 +- setup.cfg | 2 +- 5 files changed, 10 insertions(+), 5 deletions(-) diff --git a/.readthedocs.yml b/.readthedocs.yml index 7e399c8..683f3a8 100644 --- a/.readthedocs.yml +++ b/.readthedocs.yml @@ -26,4 +26,4 @@ python: - method: pip path: . extra_requirements: - - testing + - dev diff --git a/Makefile b/Makefile index 413399e..3041547 100644 --- a/Makefile +++ b/Makefile @@ -13,8 +13,13 @@ PYTHON_EXE?=python3 VENV=venv ACTIVATE?=. ${VENV}/bin/activate; + +conf: + @echo "-> Install dependencies" + ./configure + dev: - @echo "-> Configure the development envt." + @echo "-> Configure and install development dependencies" ./configure --dev doc8: diff --git a/configure b/configure index 3dd9a0a..5ef0e06 100755 --- a/configure +++ b/configure @@ -29,7 +29,7 @@ CLI_ARGS=$1 # Requirement arguments passed to pip and used by default or with --dev. REQUIREMENTS="--editable . --constraint requirements.txt" -DEV_REQUIREMENTS="--editable .[testing] --constraint requirements.txt --constraint requirements-dev.txt" +DEV_REQUIREMENTS="--editable .[dev] --constraint requirements.txt --constraint requirements-dev.txt" # where we create a virtualenv VIRTUALENV_DIR=venv diff --git a/configure.bat b/configure.bat index 18b3703..3e9881f 100644 --- a/configure.bat +++ b/configure.bat @@ -27,7 +27,7 @@ @rem # Requirement arguments passed to pip and used by default or with --dev. set "REQUIREMENTS=--editable . --constraint requirements.txt" -set "DEV_REQUIREMENTS=--editable .[testing] --constraint requirements.txt --constraint requirements-dev.txt" +set "DEV_REQUIREMENTS=--editable .[dev] --constraint requirements.txt --constraint requirements-dev.txt" @rem # where we create a virtualenv set "VIRTUALENV_DIR=venv" diff --git a/setup.cfg b/setup.cfg index ad8e0d8..99ba260 100644 --- a/setup.cfg +++ b/setup.cfg @@ -48,7 +48,7 @@ where = src [options.extras_require] -testing = +dev = pytest >= 6, != 7.0.0 pytest-xdist >= 2 aboutcode-toolkit >= 7.0.2 From 209231f0d27de0b0cfcc51eeb0fbaf9393d3df1c Mon Sep 17 00:00:00 2001 From: Philippe Ombredanne Date: Wed, 2 Apr 2025 17:50:44 +0200 Subject: [PATCH 61/69] Add more excludes from tests Signed-off-by: Philippe Ombredanne --- pyproject.toml | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/pyproject.toml b/pyproject.toml index c9e6772..bcca1a8 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -14,7 +14,6 @@ norecursedirs = [ "dist", "build", "_build", - "dist", "etc", "local", "ci", @@ -34,7 +33,9 @@ norecursedirs = [ "thirdparty", "tmp", "venv", + ".venv", "tests/data", + "*/tests/test_data", ".eggs", "src/*/data", "tests/*/data" From 47bce2da33db6b3ce3bb16831ddca89a65494e23 Mon Sep 17 00:00:00 2001 From: Philippe Ombredanne Date: Wed, 2 Apr 2025 17:54:23 +0200 Subject: [PATCH 62/69] Do not lint django migrations Signed-off-by: Philippe Ombredanne --- pyproject.toml | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/pyproject.toml b/pyproject.toml index bcca1a8..d79574e 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -66,6 +66,7 @@ include = [ "docs/**/*.py", "*.py", "." + ] # ignore test data and testfiles: they should never be linted nor formatted exclude = [ @@ -78,9 +79,10 @@ exclude = [ # vulnerablecode, fetchcode "**/tests/*/test_data/**/*", "**/tests/test_data/**/*", +# django migrations + "**/migrations/**/*" ] - [tool.ruff.lint] # Rules: https://docs.astral.sh/ruff/rules/ select = [ From 5025cfb59f0555bf4b40cd75e75ce41188e19e11 Mon Sep 17 00:00:00 2001 From: Philippe Ombredanne Date: Wed, 2 Apr 2025 17:58:07 +0200 Subject: [PATCH 63/69] Add README.rst to list of "license files" Signed-off-by: Philippe Ombredanne --- setup.cfg | 1 + 1 file changed, 1 insertion(+) diff --git a/setup.cfg b/setup.cfg index 99ba260..e5b56da 100644 --- a/setup.cfg +++ b/setup.cfg @@ -28,6 +28,7 @@ license_files = AUTHORS.rst CHANGELOG.rst CODE_OF_CONDUCT.rst + README.rst [options] package_dir = From 548a72eac69e4400e4b01f22941d38fe1cb4648d Mon Sep 17 00:00:00 2001 From: Philippe Ombredanne Date: Wed, 2 Apr 2025 18:59:08 +0200 Subject: [PATCH 64/69] Use Python 3.9 as lowest suupported version Signed-off-by: Philippe Ombredanne --- setup.cfg | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/setup.cfg b/setup.cfg index e5b56da..a9c5dbc 100644 --- a/setup.cfg +++ b/setup.cfg @@ -31,6 +31,8 @@ license_files = README.rst [options] +python_requires = >=3.9 + package_dir = =src packages = find: @@ -39,7 +41,6 @@ zip_safe = false setup_requires = setuptools_scm[toml] >= 4 -python_requires = >=3.8 install_requires = From 3d256b4ac7976b46c23424e86bb62a38f0e4a095 Mon Sep 17 00:00:00 2001 From: Philippe Ombredanne Date: Wed, 2 Apr 2025 19:01:22 +0200 Subject: [PATCH 65/69] Drop pycodestyle Not used anymore Signed-off-by: Philippe Ombredanne --- setup.cfg | 1 - 1 file changed, 1 deletion(-) diff --git a/setup.cfg b/setup.cfg index a9c5dbc..6d0b648 100644 --- a/setup.cfg +++ b/setup.cfg @@ -54,7 +54,6 @@ dev = pytest >= 6, != 7.0.0 pytest-xdist >= 2 aboutcode-toolkit >= 7.0.2 - pycodestyle >= 2.8.0 twine ruff Sphinx>=5.0.2 From 645052974bf7e6f45c1e55a24a2acaa0cee24523 Mon Sep 17 00:00:00 2001 From: Philippe Ombredanne Date: Wed, 2 Apr 2025 19:26:17 +0200 Subject: [PATCH 66/69] Bump pytest minimal version Signed-off-by: Philippe Ombredanne --- setup.cfg | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/setup.cfg b/setup.cfg index 6d0b648..69f850c 100644 --- a/setup.cfg +++ b/setup.cfg @@ -51,7 +51,7 @@ where = src [options.extras_require] dev = - pytest >= 6, != 7.0.0 + pytest >= 7.0.1 pytest-xdist >= 2 aboutcode-toolkit >= 7.0.2 twine From af87cfab2d06fb034a90412a87e0d4e660e214ee Mon Sep 17 00:00:00 2001 From: Ayan Sinha Mahapatra Date: Wed, 25 Jun 2025 00:40:47 +0530 Subject: [PATCH 67/69] Update CI runners and scripts Signed-off-by: Ayan Sinha Mahapatra --- .github/workflows/docs-ci.yml | 2 +- .github/workflows/pypi-release.yml | 4 +- azure-pipelines.yml | 24 ++++-------- configure | 2 +- configure.bat | 4 +- etc/ci/azure-container-deb.yml | 2 +- etc/ci/azure-container-rpm.yml | 2 +- etc/scripts/utils_thirdparty.py | 61 +++++++++++++++--------------- 8 files changed, 46 insertions(+), 55 deletions(-) diff --git a/.github/workflows/docs-ci.yml b/.github/workflows/docs-ci.yml index 10ba5fa..8d8aa55 100644 --- a/.github/workflows/docs-ci.yml +++ b/.github/workflows/docs-ci.yml @@ -9,7 +9,7 @@ jobs: strategy: max-parallel: 4 matrix: - python-version: [3.12] + python-version: [3.13] steps: - name: Checkout code diff --git a/.github/workflows/pypi-release.yml b/.github/workflows/pypi-release.yml index cf0579a..7f81361 100644 --- a/.github/workflows/pypi-release.yml +++ b/.github/workflows/pypi-release.yml @@ -31,10 +31,10 @@ jobs: python-version: 3.12 - name: Install pypa/build and twine - run: python -m pip install --user build twine + run: python -m pip install --user --upgrade build twine pkginfo - name: Build a binary wheel and a source tarball - run: python -m build --sdist --wheel --outdir dist/ + run: python -m build --sdist --outdir dist/ - name: Validate wheel and sdis for Pypi run: python -m twine check dist/* diff --git a/azure-pipelines.yml b/azure-pipelines.yml index 7a2d4d9..4d347b7 100644 --- a/azure-pipelines.yml +++ b/azure-pipelines.yml @@ -11,7 +11,7 @@ jobs: parameters: job_name: run_code_checks image_name: ubuntu-24.04 - python_versions: ['3.12'] + python_versions: ['3.13'] test_suites: all: make check @@ -19,7 +19,7 @@ jobs: parameters: job_name: ubuntu22_cpython image_name: ubuntu-22.04 - python_versions: ['3.9', '3.10', '3.11', '3.12'] + python_versions: ['3.9', '3.10', '3.11', '3.12', '3.13'] test_suites: all: venv/bin/pytest -n 2 -vvs @@ -27,7 +27,7 @@ jobs: parameters: job_name: ubuntu24_cpython image_name: ubuntu-24.04 - python_versions: ['3.9', '3.10', '3.11', '3.12'] + python_versions: ['3.9', '3.10', '3.11', '3.12', '3.13'] test_suites: all: venv/bin/pytest -n 2 -vvs @@ -35,7 +35,7 @@ jobs: parameters: job_name: macos13_cpython image_name: macOS-13 - python_versions: ['3.9', '3.10', '3.11', '3.12'] + python_versions: ['3.9', '3.10', '3.11', '3.12', '3.13'] test_suites: all: venv/bin/pytest -n 2 -vvs @@ -43,7 +43,7 @@ jobs: parameters: job_name: macos14_cpython image_name: macOS-14 - python_versions: ['3.9', '3.10', '3.11', '3.12'] + python_versions: ['3.9', '3.10', '3.11', '3.12', '3.13'] test_suites: all: venv/bin/pytest -n 2 -vvs @@ -51,23 +51,15 @@ jobs: parameters: job_name: macos15_cpython image_name: macOS-15 - python_versions: ['3.9', '3.10', '3.11', '3.12'] + python_versions: ['3.9', '3.10', '3.11', '3.12', '3.13'] test_suites: all: venv/bin/pytest -n 2 -vvs - - template: etc/ci/azure-win.yml - parameters: - job_name: win2019_cpython - image_name: windows-2019 - python_versions: ['3.9', '3.10', '3.11', '3.12'] - test_suites: - all: venv\Scripts\pytest -n 2 -vvs - - template: etc/ci/azure-win.yml parameters: job_name: win2022_cpython image_name: windows-2022 - python_versions: ['3.9', '3.10', '3.11', '3.12'] + python_versions: ['3.9', '3.10', '3.11', '3.12', '3.13'] test_suites: all: venv\Scripts\pytest -n 2 -vvs @@ -75,6 +67,6 @@ jobs: parameters: job_name: win2025_cpython image_name: windows-2025 - python_versions: ['3.9', '3.10', '3.11', '3.12'] + python_versions: ['3.9', '3.10', '3.11', '3.12', '3.13'] test_suites: all: venv\Scripts\pytest -n 2 -vvs diff --git a/configure b/configure index 5ef0e06..6d317d4 100755 --- a/configure +++ b/configure @@ -110,7 +110,7 @@ create_virtualenv() { fi $PYTHON_EXECUTABLE "$VIRTUALENV_PYZ" \ - --wheel embed --pip embed --setuptools embed \ + --pip embed --setuptools embed \ --seeder pip \ --never-download \ --no-periodic-update \ diff --git a/configure.bat b/configure.bat index 3e9881f..15ab701 100644 --- a/configure.bat +++ b/configure.bat @@ -110,7 +110,7 @@ if not exist "%CFG_BIN_DIR%\python.exe" ( if exist "%CFG_ROOT_DIR%\etc\thirdparty\virtualenv.pyz" ( %PYTHON_EXECUTABLE% "%CFG_ROOT_DIR%\etc\thirdparty\virtualenv.pyz" ^ - --wheel embed --pip embed --setuptools embed ^ + --pip embed --setuptools embed ^ --seeder pip ^ --never-download ^ --no-periodic-update ^ @@ -126,7 +126,7 @@ if not exist "%CFG_BIN_DIR%\python.exe" ( ) ) %PYTHON_EXECUTABLE% "%CFG_ROOT_DIR%\%VIRTUALENV_DIR%\virtualenv.pyz" ^ - --wheel embed --pip embed --setuptools embed ^ + --pip embed --setuptools embed ^ --seeder pip ^ --never-download ^ --no-periodic-update ^ diff --git a/etc/ci/azure-container-deb.yml b/etc/ci/azure-container-deb.yml index 85b611d..d80e8df 100644 --- a/etc/ci/azure-container-deb.yml +++ b/etc/ci/azure-container-deb.yml @@ -21,7 +21,7 @@ jobs: - job: ${{ parameters.job_name }} pool: - vmImage: 'ubuntu-16.04' + vmImage: 'ubuntu-22.04' container: image: ${{ parameters.container }} diff --git a/etc/ci/azure-container-rpm.yml b/etc/ci/azure-container-rpm.yml index 1e6657d..a64138c 100644 --- a/etc/ci/azure-container-rpm.yml +++ b/etc/ci/azure-container-rpm.yml @@ -1,6 +1,6 @@ parameters: job_name: '' - image_name: 'ubuntu-16.04' + image_name: 'ubuntu-22.04' container: '' python_path: '' python_version: '' diff --git a/etc/scripts/utils_thirdparty.py b/etc/scripts/utils_thirdparty.py index aafc1d6..6f812f0 100644 --- a/etc/scripts/utils_thirdparty.py +++ b/etc/scripts/utils_thirdparty.py @@ -1,4 +1,5 @@ #!/usr/bin/env python +# -*- coding: utf-8 -*- # # Copyright (c) nexB Inc. and others. All rights reserved. # ScanCode is a trademark of nexB Inc. @@ -24,13 +25,14 @@ import packageurl import requests import saneyaml -import utils_pip_compatibility_tags from commoncode import fileutils from commoncode.hash import multi_checksums from commoncode.text import python_safe_name from packvers import tags as packaging_tags from packvers import version as packaging_version +import utils_pip_compatibility_tags + """ Utilities to manage Python thirparty libraries source, binaries and metadata in local directories and remote repositories. @@ -91,8 +93,7 @@ - parse requirement file - create a TODO queue of requirements to process -- done: create an empty map of processed binary requirements as - {package name: (list of versions/tags} +- done: create an empty map of processed binary requirements as {package name: (list of versions/tags} - while we have package reqs in TODO queue, process one requirement: @@ -114,13 +115,14 @@ TRACE_ULTRA_DEEP = False # Supported environments -PYTHON_VERSIONS = "37", "38", "39", "310" +PYTHON_VERSIONS = "39", "310", "311", "312", "313" PYTHON_DOT_VERSIONS_BY_VER = { - "37": "3.7", - "38": "3.8", "39": "3.9", "310": "3.10", + "311": "3.11", + "312": "3.12", + "313": "3.13", } @@ -132,10 +134,11 @@ def get_python_dot_version(version): ABIS_BY_PYTHON_VERSION = { - "37": ["cp37", "cp37m", "abi3"], - "38": ["cp38", "cp38m", "abi3"], "39": ["cp39", "cp39m", "abi3"], "310": ["cp310", "cp310m", "abi3"], + "311": ["cp311", "cp311m", "abi3"], + "312": ["cp312", "cp312m", "abi3"], + "313": ["cp313", "cp313m", "abi3"], } PLATFORMS_BY_OS = { @@ -553,8 +556,7 @@ def download(self, dest_dir=THIRDPARTY_DIR): Download this distribution into `dest_dir` directory. Return the fetched filename. """ - if not self.filename: - raise ValueError(f"self.filename has no value but is required: {self.filename!r}") + assert self.filename if TRACE_DEEP: print( f"Fetching distribution of {self.name}=={self.version}:", @@ -822,9 +824,9 @@ def fetch_license_files(self, dest_dir=THIRDPARTY_DIR, use_cached_index=False): """ urls = LinksRepository.from_url(use_cached_index=use_cached_index).links errors = [] - extra_lic_names = [lic.get("file") for lic in self.extra_data.get("licenses", {})] + extra_lic_names = [l.get("file") for l in self.extra_data.get("licenses", {})] extra_lic_names += [self.extra_data.get("license_file")] - extra_lic_names = [eln for eln in extra_lic_names if eln] + extra_lic_names = [ln for ln in extra_lic_names if ln] lic_names = [f"{key}.LICENSE" for key in self.get_license_keys()] for filename in lic_names + extra_lic_names: floc = os.path.join(dest_dir, filename) @@ -844,7 +846,7 @@ def fetch_license_files(self, dest_dir=THIRDPARTY_DIR, use_cached_index=False): if TRACE: print(f"Fetched license from remote: {lic_url}") - except Exception: + except: try: # try licensedb second lic_url = f"{LICENSEDB_API_URL}/{filename}" @@ -857,9 +859,8 @@ def fetch_license_files(self, dest_dir=THIRDPARTY_DIR, use_cached_index=False): if TRACE: print(f"Fetched license from licensedb: {lic_url}") - except Exception: - msg = f"No text for license {filename} in expression " - f"{self.license_expression!r} from {self}" + except: + msg = f'No text for license {filename} in expression "{self.license_expression}" from {self}' print(msg) errors.append(msg) @@ -999,7 +1000,7 @@ def get_license_link_for_filename(filename, urls): exception if no link is found or if there are more than one link for that file name. """ - path_or_url = [url for url in urls if url.endswith(f"/{filename}")] + path_or_url = [l for l in urls if l.endswith(f"/{filename}")] if not path_or_url: raise Exception(f"Missing link to file: {filename}") if not len(path_or_url) == 1: @@ -1288,7 +1289,7 @@ def is_pure(self): def is_pure_wheel(filename): try: return Wheel.from_filename(filename).is_pure() - except Exception: + except: return False @@ -1484,7 +1485,8 @@ def get_distributions(self): """ if self.sdist: yield self.sdist - yield from self.wheels + for wheel in self.wheels: + yield wheel def get_url_for_filename(self, filename): """ @@ -1613,8 +1615,7 @@ class PypiSimpleRepository: type=dict, default=attr.Factory(lambda: defaultdict(dict)), metadata=dict( - help="Mapping of {name: {version: PypiPackage, version: PypiPackage, etc} " - "available in this repo" + help="Mapping of {name: {version: PypiPackage, version: PypiPackage, etc} available in this repo" ), ) @@ -1628,8 +1629,7 @@ class PypiSimpleRepository: type=bool, default=False, metadata=dict( - help="If True, use any existing on-disk cached PyPI index files. " - "Otherwise, fetch and cache." + help="If True, use any existing on-disk cached PyPI index files. Otherwise, fetch and cache." ), ) @@ -1638,8 +1638,7 @@ def _get_package_versions_map(self, name): Return a mapping of all available PypiPackage version for this package name. The mapping may be empty. It is ordered by version from oldest to newest """ - if not name: - raise ValueError(f"name is required: {name!r}") + assert name normalized_name = NameVer.normalize_name(name) versions = self.packages[normalized_name] if not versions and normalized_name not in self.fetched_package_normalized_names: @@ -1694,7 +1693,7 @@ def fetch_links(self, normalized_name): ) links = collect_urls(text) # TODO: keep sha256 - links = [link.partition("#sha256=") for link in links] + links = [l.partition("#sha256=") for l in links] links = [url for url, _, _sha256 in links] return links @@ -1915,7 +1914,7 @@ def get_remote_file_content( # several redirects and that we can ignore content there. A HEAD request may # not get us this last header print(f" DOWNLOADING: {url}") - with requests.get(url, allow_redirects=True, stream=True, headers=headers) as response: # noqa: S113 + with requests.get(url, allow_redirects=True, stream=True, headers=headers) as response: status = response.status_code if status != requests.codes.ok: # NOQA if status == 429 and _delay < 20: @@ -2134,7 +2133,7 @@ def call(args, verbose=TRACE): """ if TRACE_DEEP: print("Calling:", " ".join(args)) - with subprocess.Popen( # noqa: S603 + with subprocess.Popen( args, stdout=subprocess.PIPE, stderr=subprocess.PIPE, encoding="utf-8" ) as process: stdouts = [] @@ -2199,7 +2198,7 @@ def download_wheels_with_pip( cli_args.extend(["--requirement", req_file]) if TRACE: - print("Downloading wheels using command:", " ".join(cli_args)) + print(f"Downloading wheels using command:", " ".join(cli_args)) existing = set(os.listdir(dest_dir)) error = False @@ -2232,7 +2231,7 @@ def download_wheels_with_pip( def check_about(dest_dir=THIRDPARTY_DIR): try: - subprocess.check_output(f"venv/bin/about check {dest_dir}".split()) # noqa: S603 + subprocess.check_output(f"venv/bin/about check {dest_dir}".split()) except subprocess.CalledProcessError as cpe: print() print("Invalid ABOUT files:") @@ -2283,5 +2282,5 @@ def get_license_expression(declared_licenses): return get_only_expression_from_extracted_license(declared_licenses) except ImportError: # Scancode is not installed, clean and join all the licenses - lics = [python_safe_name(lic).lower() for lic in declared_licenses] + lics = [python_safe_name(l).lower() for l in declared_licenses] return " AND ".join(lics).lower() From d7de327f03afe8cac7556bbdecdf222c304fb943 Mon Sep 17 00:00:00 2001 From: Ayan Sinha Mahapatra Date: Wed, 25 Jun 2025 00:59:19 +0530 Subject: [PATCH 68/69] Add licenses from scancode licenseDB and bump version Signed-off-by: Ayan Sinha Mahapatra --- CHANGELOG.rst | 9 + setup.cfg | 2 +- .../data/license_key_index.json.ABOUT | 3 +- .../data/scancode-licensedb-index.json | 1035 ++++++++++++++++- 4 files changed, 1041 insertions(+), 8 deletions(-) diff --git a/CHANGELOG.rst b/CHANGELOG.rst index c246018..4ed279c 100644 --- a/CHANGELOG.rst +++ b/CHANGELOG.rst @@ -1,6 +1,15 @@ Changelog ========= +v30.4.2 - 2025-06-25 +-------------------- + +This is a minor release without API changes: + +- Use latest skeleton +- Update license list to latest ScanCode + + v30.4.1 - 2025-01-10 -------------------- diff --git a/setup.cfg b/setup.cfg index e685ba3..7c5680d 100644 --- a/setup.cfg +++ b/setup.cfg @@ -1,6 +1,6 @@ [metadata] name = license-expression -version = 30.4.1 +version = 30.4.2 license = Apache-2.0 # description must be on ONE line https://github.com/pypa/setuptools/issues/1390 diff --git a/src/license_expression/data/license_key_index.json.ABOUT b/src/license_expression/data/license_key_index.json.ABOUT index adc86b6..d373358 100644 --- a/src/license_expression/data/license_key_index.json.ABOUT +++ b/src/license_expression/data/license_key_index.json.ABOUT @@ -1,7 +1,8 @@ about_resource: scancode-licensedb-index.json -download_url: https://raw.githubusercontent.com/aboutcode-org/scancode-licensedb/1e9ff1927b89bae4ca1356de77aa29cc18916025/docs/index.json +download_url: https://raw.githubusercontent.com/aboutcode-org/scancode-licensedb/5ea6b7355db6801cfd3d437cdc36a5dbd0f05dc7/docs/index.json spdx_license_list_version: 3.26 name: scancode-licensedb-index.json license_expression: cc-by-4.0 copyright: Copyright (c) nexB Inc. and others. homepage_url: https://scancode-licensedb.aboutcode.org/ +note: Last updated on June 20, 2025 diff --git a/src/license_expression/data/scancode-licensedb-index.json b/src/license_expression/data/scancode-licensedb-index.json index e6df30e..825c737 100644 --- a/src/license_expression/data/scancode-licensedb-index.json +++ b/src/license_expression/data/scancode-licensedb-index.json @@ -219,6 +219,18 @@ "html": "activestate-komodo-edit.html", "license": "activestate-komodo-edit.LICENSE" }, + { + "license_key": "activision-eula", + "category": "Proprietary Free", + "spdx_license_key": "LicenseRef-scancode-activision-eula", + "other_spdx_license_keys": [], + "is_exception": false, + "is_deprecated": false, + "json": "activision-eula.json", + "yaml": "activision-eula.yml", + "html": "activision-eula.html", + "license": "activision-eula.LICENSE" + }, { "license_key": "actuate-birt-ihub-ftype-sla", "category": "Proprietary Free", @@ -665,6 +677,18 @@ "html": "afpl-9.0.html", "license": "afpl-9.0.LICENSE" }, + { + "license_key": "ag-grid-enterprise", + "category": "Commercial", + "spdx_license_key": "LicenseRef-scancode-ag-grid-enterprise", + "other_spdx_license_keys": [], + "is_exception": false, + "is_deprecated": false, + "json": "ag-grid-enterprise.json", + "yaml": "ag-grid-enterprise.yml", + "html": "ag-grid-enterprise.html", + "license": "ag-grid-enterprise.LICENSE" + }, { "license_key": "agentxpp", "category": "Commercial", @@ -1087,6 +1111,18 @@ "html": "ampas.html", "license": "ampas.LICENSE" }, + { + "license_key": "amplication-ee-2022", + "category": "Proprietary Free", + "spdx_license_key": "LicenseRef-scancode-amplication-ee-2022", + "other_spdx_license_keys": [], + "is_exception": false, + "is_deprecated": false, + "json": "amplication-ee-2022.json", + "yaml": "amplication-ee-2022.yml", + "html": "amplication-ee-2022.html", + "license": "amplication-ee-2022.LICENSE" + }, { "license_key": "ams-fonts", "category": "Permissive", @@ -1099,6 +1135,18 @@ "html": "ams-fonts.html", "license": "ams-fonts.LICENSE" }, + { + "license_key": "anaconda-tos-2024-03-30", + "category": "Commercial", + "spdx_license_key": "LicenseRef-scancode-anaconda-tos-2024-03-30", + "other_spdx_license_keys": [], + "is_exception": false, + "is_deprecated": false, + "json": "anaconda-tos-2024-03-30.json", + "yaml": "anaconda-tos-2024-03-30.yml", + "html": "anaconda-tos-2024-03-30.html", + "license": "anaconda-tos-2024-03-30.LICENSE" + }, { "license_key": "android-sdk-2009", "category": "Proprietary Free", @@ -1536,6 +1584,18 @@ "html": "appsflyer-framework.html", "license": "appsflyer-framework.LICENSE" }, + { + "license_key": "apromore-exception-2.0", + "category": "Copyleft", + "spdx_license_key": "LicenseRef-scancode-apromore-exception-2.0", + "other_spdx_license_keys": [], + "is_exception": true, + "is_deprecated": false, + "json": "apromore-exception-2.0.json", + "yaml": "apromore-exception-2.0.yml", + "html": "apromore-exception-2.0.html", + "license": "apromore-exception-2.0.LICENSE" + }, { "license_key": "apsl-1.0", "category": "Copyleft Limited", @@ -2160,6 +2220,18 @@ "html": "avsystem-5-clause.html", "license": "avsystem-5-clause.LICENSE" }, + { + "license_key": "aws-ip-2021", + "category": "Proprietary Free", + "spdx_license_key": "LicenseRef-scancode-aws-ip-2021", + "other_spdx_license_keys": [], + "is_exception": false, + "is_deprecated": false, + "json": "aws-ip-2021.json", + "yaml": "aws-ip-2021.yml", + "html": "aws-ip-2021.html", + "license": "aws-ip-2021.LICENSE" + }, { "license_key": "bacula-exception", "category": "Copyleft Limited", @@ -2234,6 +2306,30 @@ "html": "barr-tex.html", "license": "barr-tex.LICENSE" }, + { + "license_key": "baserow-ee-2019", + "category": "Commercial", + "spdx_license_key": "LicenseRef-scancode-baserow-ee-2019", + "other_spdx_license_keys": [], + "is_exception": false, + "is_deprecated": false, + "json": "baserow-ee-2019.json", + "yaml": "baserow-ee-2019.yml", + "html": "baserow-ee-2019.html", + "license": "baserow-ee-2019.LICENSE" + }, + { + "license_key": "baserow-pe-2019", + "category": "Commercial", + "spdx_license_key": "LicenseRef-scancode-baserow-pe-2019", + "other_spdx_license_keys": [], + "is_exception": false, + "is_deprecated": false, + "json": "baserow-pe-2019.json", + "yaml": "baserow-pe-2019.yml", + "html": "baserow-pe-2019.html", + "license": "baserow-pe-2019.LICENSE" + }, { "license_key": "bash-exception-gpl", "category": "Copyleft", @@ -2282,6 +2378,18 @@ "html": "beal-screamer.html", "license": "beal-screamer.LICENSE" }, + { + "license_key": "beegfs-eula-2024", + "category": "Source-available", + "spdx_license_key": "LicenseRef-scancode-beegfs-eula-2024", + "other_spdx_license_keys": [], + "is_exception": false, + "is_deprecated": false, + "json": "beegfs-eula-2024.json", + "yaml": "beegfs-eula-2024.yml", + "html": "beegfs-eula-2024.html", + "license": "beegfs-eula-2024.LICENSE" + }, { "license_key": "beerware", "category": "Permissive", @@ -2612,6 +2720,30 @@ "html": "bohl-0.2.html", "license": "bohl-0.2.LICENSE" }, + { + "license_key": "bola10", + "category": "Permissive", + "spdx_license_key": "LicenseRef-scancode-bola10", + "other_spdx_license_keys": [], + "is_exception": false, + "is_deprecated": false, + "json": "bola10.json", + "yaml": "bola10.yml", + "html": "bola10.html", + "license": "bola10.LICENSE" + }, + { + "license_key": "bola11", + "category": "Permissive", + "spdx_license_key": "LicenseRef-scancode-bola11", + "other_spdx_license_keys": [], + "is_exception": false, + "is_deprecated": false, + "json": "bola11.json", + "yaml": "bola11.yml", + "html": "bola11.html", + "license": "bola11.LICENSE" + }, { "license_key": "boost-1.0", "category": "Permissive", @@ -3342,6 +3474,18 @@ "html": "bsd-dpt.html", "license": "bsd-dpt.LICENSE" }, + { + "license_key": "bsd-endorsement-allowed", + "category": "Permissive", + "spdx_license_key": "LicenseRef-scancode-bsd-endorsement-allowed", + "other_spdx_license_keys": [], + "is_exception": false, + "is_deprecated": false, + "json": "bsd-endorsement-allowed.json", + "yaml": "bsd-endorsement-allowed.yml", + "html": "bsd-endorsement-allowed.html", + "license": "bsd-endorsement-allowed.LICENSE" + }, { "license_key": "bsd-export", "category": "Permissive", @@ -3899,6 +4043,18 @@ "html": "c-uda-1.0.html", "license": "c-uda-1.0.LICENSE" }, + { + "license_key": "ca-ossl-1.0", + "category": "Permissive", + "spdx_license_key": "LicenseRef-scancode-ca-ossl-1.0", + "other_spdx_license_keys": [], + "is_exception": false, + "is_deprecated": false, + "json": "ca-ossl-1.0.json", + "yaml": "ca-ossl-1.0.yml", + "html": "ca-ossl-1.0.html", + "license": "ca-ossl-1.0.LICENSE" + }, { "license_key": "ca-tosl-1.1", "category": "Copyleft Limited", @@ -4079,6 +4235,18 @@ "html": "capec-tou.html", "license": "capec-tou.LICENSE" }, + { + "license_key": "caramel-license-1.0", + "category": "Permissive", + "spdx_license_key": "LicenseRef-scancode-caramel-1.0", + "other_spdx_license_keys": [], + "is_exception": false, + "is_deprecated": false, + "json": "caramel-license-1.0.json", + "yaml": "caramel-license-1.0.yml", + "html": "caramel-license-1.0.html", + "license": "caramel-license-1.0.LICENSE" + }, { "license_key": "careware", "category": "Permissive", @@ -4975,6 +5143,18 @@ "html": "cc0-1.0.html", "license": "cc0-1.0.LICENSE" }, + { + "license_key": "ccg-research-academic", + "category": "Proprietary Free", + "spdx_license_key": "LicenseRef-scancode-ccg-research-academic", + "other_spdx_license_keys": [], + "is_exception": false, + "is_deprecated": false, + "json": "ccg-research-academic.json", + "yaml": "ccg-research-academic.yml", + "html": "ccg-research-academic.html", + "license": "ccg-research-academic.LICENSE" + }, { "license_key": "cclrc", "category": "Free Restricted", @@ -5515,6 +5695,18 @@ "html": "clear-bsd-1-clause.html", "license": "clear-bsd-1-clause.LICENSE" }, + { + "license_key": "clearthought-2.0", + "category": "Proprietary Free", + "spdx_license_key": "LicenseRef-scancode-clearthought-2.0", + "other_spdx_license_keys": [], + "is_exception": false, + "is_deprecated": false, + "json": "clearthought-2.0.json", + "yaml": "clearthought-2.0.yml", + "html": "clearthought-2.0.html", + "license": "clearthought-2.0.LICENSE" + }, { "license_key": "click-license", "category": "Permissive", @@ -5776,16 +5968,16 @@ "license": "cockroachdb-use-grant-for-bsl-1.1.LICENSE" }, { - "license_key": "code-credit-license-1.0-0", + "license_key": "code-credit-license-1.0.0", "category": "Permissive", "spdx_license_key": "LicenseRef-scancode-code-credit-license-1.0.0", "other_spdx_license_keys": [], "is_exception": false, "is_deprecated": false, - "json": "code-credit-license-1.0-0.json", - "yaml": "code-credit-license-1.0-0.yml", - "html": "code-credit-license-1.0-0.html", - "license": "code-credit-license-1.0-0.LICENSE" + "json": "code-credit-license-1.0.0.json", + "yaml": "code-credit-license-1.0.0.yml", + "html": "code-credit-license-1.0.0.html", + "license": "code-credit-license-1.0.0.LICENSE" }, { "license_key": "code-credit-license-1.0.1", @@ -6673,6 +6865,30 @@ "html": "dante-treglia.html", "license": "dante-treglia.LICENSE" }, + { + "license_key": "databricks-db", + "category": "Proprietary Free", + "spdx_license_key": "LicenseRef-scancode-databricks-db", + "other_spdx_license_keys": [], + "is_exception": false, + "is_deprecated": false, + "json": "databricks-db.json", + "yaml": "databricks-db.yml", + "html": "databricks-db.html", + "license": "databricks-db.LICENSE" + }, + { + "license_key": "databricks-dbx-2021", + "category": "Proprietary Free", + "spdx_license_key": "LicenseRef-scancode-databricks-dbx-2021", + "other_spdx_license_keys": [], + "is_exception": false, + "is_deprecated": false, + "json": "databricks-dbx-2021.json", + "yaml": "databricks-dbx-2021.yml", + "html": "databricks-dbx-2021.html", + "license": "databricks-dbx-2021.LICENSE" + }, { "license_key": "datamekanix-license", "category": "Permissive", @@ -6733,6 +6949,18 @@ "html": "dbcl-1.0.html", "license": "dbcl-1.0.LICENSE" }, + { + "license_key": "dbisl-1.0", + "category": "Copyleft Limited", + "spdx_license_key": "LicenseRef-scancode-dbisl-1.0", + "other_spdx_license_keys": [], + "is_exception": false, + "is_deprecated": false, + "json": "dbisl-1.0.json", + "yaml": "dbisl-1.0.yml", + "html": "dbisl-1.0.html", + "license": "dbisl-1.0.LICENSE" + }, { "license_key": "dbmx-foss-exception-1.0.9", "category": "Copyleft Limited", @@ -6757,6 +6985,18 @@ "html": "dbmx-linking-exception-1.0.html", "license": "dbmx-linking-exception-1.0.LICENSE" }, + { + "license_key": "dco-1.0", + "category": "CLA", + "spdx_license_key": "LicenseRef-scancode-dco-1.0", + "other_spdx_license_keys": [], + "is_exception": false, + "is_deprecated": false, + "json": "dco-1.0.json", + "yaml": "dco-1.0.yml", + "html": "dco-1.0.html", + "license": "dco-1.0.LICENSE" + }, { "license_key": "dco-1.1", "category": "CLA", @@ -6781,6 +7021,18 @@ "html": "dec-3-clause.html", "license": "dec-3-clause.LICENSE" }, + { + "license_key": "deepseek-la-1.0", + "category": "Source-available", + "spdx_license_key": "LicenseRef-scancode-deepseek-la-1.0", + "other_spdx_license_keys": [], + "is_exception": false, + "is_deprecated": false, + "json": "deepseek-la-1.0.json", + "yaml": "deepseek-la-1.0.yml", + "html": "deepseek-la-1.0.html", + "license": "deepseek-la-1.0.LICENSE" + }, { "license_key": "defensive-patent-1.1", "category": "Copyleft", @@ -7339,6 +7591,18 @@ "html": "duende-sla-2022.html", "license": "duende-sla-2022.LICENSE" }, + { + "license_key": "dumb", + "category": "Permissive", + "spdx_license_key": "LicenseRef-scancode-dumb", + "other_spdx_license_keys": [], + "is_exception": false, + "is_deprecated": false, + "json": "dumb.json", + "yaml": "dumb.yml", + "html": "dumb.html", + "license": "dumb.LICENSE" + }, { "license_key": "dune-exception", "category": "Copyleft Limited", @@ -7567,6 +7831,18 @@ "html": "eclipse-sua-2017.html", "license": "eclipse-sua-2017.LICENSE" }, + { + "license_key": "eclipse-tck-1.1", + "category": "Proprietary Free", + "spdx_license_key": "LicenseRef-scancode-eclipse-tck-1.1", + "other_spdx_license_keys": [], + "is_exception": false, + "is_deprecated": false, + "json": "eclipse-tck-1.1.json", + "yaml": "eclipse-tck-1.1.yml", + "html": "eclipse-tck-1.1.html", + "license": "eclipse-tck-1.1.LICENSE" + }, { "license_key": "ecma-documentation", "category": "Free Restricted", @@ -7723,6 +7999,18 @@ "html": "efsl-1.0.html", "license": "efsl-1.0.LICENSE" }, + { + "license_key": "efsl-2.0", + "category": "Proprietary Free", + "spdx_license_key": "LicenseRef-scancode-efsl-2.0", + "other_spdx_license_keys": [], + "is_exception": false, + "is_deprecated": false, + "json": "efsl-2.0.json", + "yaml": "efsl-2.0.yml", + "html": "efsl-2.0.html", + "license": "efsl-2.0.LICENSE" + }, { "license_key": "egenix-1.0.0", "category": "Permissive", @@ -7833,6 +8121,20 @@ "html": "elib-gpl.html", "license": "elib-gpl.LICENSE" }, + { + "license_key": "elixir-trademark-policy", + "category": "Proprietary Free", + "spdx_license_key": "LicenseRef-scancode-elixir-trademark-policy", + "other_spdx_license_keys": [ + "LicenseRef-elixir-trademark-policy" + ], + "is_exception": false, + "is_deprecated": false, + "json": "elixir-trademark-policy.json", + "yaml": "elixir-trademark-policy.yml", + "html": "elixir-trademark-policy.html", + "license": "elixir-trademark-policy.LICENSE" + }, { "license_key": "ellis-lab", "category": "Permissive", @@ -8037,6 +8339,42 @@ "html": "epo-osl-2005.1.html", "license": "epo-osl-2005.1.LICENSE" }, + { + "license_key": "epson-avasys-pl-2008", + "category": "Proprietary Free", + "spdx_license_key": "LicenseRef-scancode-epson-avasys-pl-2008", + "other_spdx_license_keys": [], + "is_exception": false, + "is_deprecated": false, + "json": "epson-avasys-pl-2008.json", + "yaml": "epson-avasys-pl-2008.yml", + "html": "epson-avasys-pl-2008.html", + "license": "epson-avasys-pl-2008.LICENSE" + }, + { + "license_key": "epson-linux-sla-2023", + "category": "Proprietary Free", + "spdx_license_key": "LicenseRef-scancode-epson-linux-sla-2023", + "other_spdx_license_keys": [], + "is_exception": false, + "is_deprecated": false, + "json": "epson-linux-sla-2023.json", + "yaml": "epson-linux-sla-2023.yml", + "html": "epson-linux-sla-2023.html", + "license": "epson-linux-sla-2023.LICENSE" + }, + { + "license_key": "eqvsl-1.0", + "category": "Proprietary Free", + "spdx_license_key": "LicenseRef-scancode-eqvsl-1.0", + "other_spdx_license_keys": [], + "is_exception": false, + "is_deprecated": false, + "json": "eqvsl-1.0.json", + "yaml": "eqvsl-1.0.yml", + "html": "eqvsl-1.0.html", + "license": "eqvsl-1.0.LICENSE" + }, { "license_key": "eric-glass", "category": "Permissive", @@ -9206,6 +9544,42 @@ "html": "gdcl.html", "license": "gdcl.LICENSE" }, + { + "license_key": "geant4-sl-1.0", + "category": "Permissive", + "spdx_license_key": "LicenseRef-scancode-geant4-sl-1.0", + "other_spdx_license_keys": [], + "is_exception": false, + "is_deprecated": false, + "json": "geant4-sl-1.0.json", + "yaml": "geant4-sl-1.0.yml", + "html": "geant4-sl-1.0.html", + "license": "geant4-sl-1.0.LICENSE" + }, + { + "license_key": "gemma-tou-2024-04-01", + "category": "Proprietary Free", + "spdx_license_key": "LicenseRef-scancode-gemma-tou-2024-04-01", + "other_spdx_license_keys": [], + "is_exception": false, + "is_deprecated": false, + "json": "gemma-tou-2024-04-01.json", + "yaml": "gemma-tou-2024-04-01.yml", + "html": "gemma-tou-2024-04-01.html", + "license": "gemma-tou-2024-04-01.LICENSE" + }, + { + "license_key": "generaluser-gs-2.0", + "category": "Permissive", + "spdx_license_key": "LicenseRef-scancode-generaluser-gs-2.0", + "other_spdx_license_keys": [], + "is_exception": false, + "is_deprecated": false, + "json": "generaluser-gs-2.0.json", + "yaml": "generaluser-gs-2.0.yml", + "html": "generaluser-gs-2.0.html", + "license": "generaluser-gs-2.0.LICENSE" + }, { "license_key": "generic-amiwm", "category": "Proprietary Free", @@ -9340,6 +9714,18 @@ "html": "geoff-kuenning-1993.html", "license": "geoff-kuenning-1993.LICENSE" }, + { + "license_key": "geogebra-ncla-2022", + "category": "Proprietary Free", + "spdx_license_key": "LicenseRef-scancode-geogebra-ncla-2022", + "other_spdx_license_keys": [], + "is_exception": false, + "is_deprecated": false, + "json": "geogebra-ncla-2022.json", + "yaml": "geogebra-ncla-2022.yml", + "html": "geogebra-ncla-2022.html", + "license": "geogebra-ncla-2022.LICENSE" + }, { "license_key": "geoserver-exception-2.0-plus", "category": "Copyleft Limited", @@ -10214,7 +10600,8 @@ "spdx_license_key": "GPL-1.0-or-later", "other_spdx_license_keys": [ "GPL-1.0+", - "LicenseRef-GPL" + "LicenseRef-GPL", + "GPL" ], "is_exception": false, "is_deprecated": false, @@ -11102,6 +11489,18 @@ "html": "gregory-pietsch.html", "license": "gregory-pietsch.LICENSE" }, + { + "license_key": "gretelai-sal-1.0", + "category": "Source-available", + "spdx_license_key": "LicenseRef-scancode-gretelai-sal-1.0", + "other_spdx_license_keys": [], + "is_exception": false, + "is_deprecated": false, + "json": "gretelai-sal-1.0.json", + "yaml": "gretelai-sal-1.0.yml", + "html": "gretelai-sal-1.0.html", + "license": "gretelai-sal-1.0.LICENSE" + }, { "license_key": "gsoap-1.3a", "category": "Copyleft Limited", @@ -12631,6 +13030,30 @@ "html": "inno-setup.html", "license": "inno-setup.LICENSE" }, + { + "license_key": "inria-compcert", + "category": "Proprietary Free", + "spdx_license_key": "LicenseRef-scancode-inria-compcert", + "other_spdx_license_keys": [], + "is_exception": false, + "is_deprecated": false, + "json": "inria-compcert.json", + "yaml": "inria-compcert.yml", + "html": "inria-compcert.html", + "license": "inria-compcert.LICENSE" + }, + { + "license_key": "inria-icesl", + "category": "Proprietary Free", + "spdx_license_key": "LicenseRef-scancode-inria-icesl", + "other_spdx_license_keys": [], + "is_exception": false, + "is_deprecated": false, + "json": "inria-icesl.json", + "yaml": "inria-icesl.yml", + "html": "inria-icesl.html", + "license": "inria-icesl.LICENSE" + }, { "license_key": "inria-linking-exception", "category": "Copyleft Limited", @@ -12645,6 +13068,18 @@ "html": "inria-linking-exception.html", "license": "inria-linking-exception.LICENSE" }, + { + "license_key": "inria-zelus", + "category": "Proprietary Free", + "spdx_license_key": "LicenseRef-scancode-inria-zelus", + "other_spdx_license_keys": [], + "is_exception": false, + "is_deprecated": false, + "json": "inria-zelus.json", + "yaml": "inria-zelus.yml", + "html": "inria-zelus.html", + "license": "inria-zelus.LICENSE" + }, { "license_key": "installsite", "category": "Free Restricted", @@ -13411,6 +13846,18 @@ "html": "jmagnetic.html", "license": "jmagnetic.LICENSE" }, + { + "license_key": "joplin-server-personal-v1", + "category": "Proprietary Free", + "spdx_license_key": "LicenseRef-scancode-joplin-server-personal-v1", + "other_spdx_license_keys": [], + "is_exception": false, + "is_deprecated": false, + "json": "joplin-server-personal-v1.json", + "yaml": "joplin-server-personal-v1.yml", + "html": "joplin-server-personal-v1.html", + "license": "joplin-server-personal-v1.LICENSE" + }, { "license_key": "josl-1.0", "category": "Copyleft Limited", @@ -13727,6 +14174,18 @@ "html": "kde-accepted-lgpl.html", "license": "kde-accepted-lgpl.LICENSE" }, + { + "license_key": "keep-ee-2024", + "category": "Commercial", + "spdx_license_key": "LicenseRef-scancode-keep-ee-2024", + "other_spdx_license_keys": [], + "is_exception": false, + "is_deprecated": false, + "json": "keep-ee-2024.json", + "yaml": "keep-ee-2024.yml", + "html": "keep-ee-2024.html", + "license": "keep-ee-2024.LICENSE" + }, { "license_key": "keith-rule", "category": "Permissive", @@ -14129,6 +14588,18 @@ "html": "leap-motion-sdk-2019.html", "license": "leap-motion-sdk-2019.LICENSE" }, + { + "license_key": "lens-tos-2023", + "category": "Proprietary Free", + "spdx_license_key": "LicenseRef-scancode-lens-tos-2023", + "other_spdx_license_keys": [], + "is_exception": false, + "is_deprecated": false, + "json": "lens-tos-2023.json", + "yaml": "lens-tos-2023.yml", + "html": "lens-tos-2023.html", + "license": "lens-tos-2023.LICENSE" + }, { "license_key": "leptonica", "category": "Permissive", @@ -15008,6 +15479,42 @@ "html": "llama-3.1-license-2024.html", "license": "llama-3.1-license-2024.LICENSE" }, + { + "license_key": "llama-3.2-license-2024", + "category": "Proprietary Free", + "spdx_license_key": "LicenseRef-scancode-llama-3.2-license-2024", + "other_spdx_license_keys": [], + "is_exception": false, + "is_deprecated": false, + "json": "llama-3.2-license-2024.json", + "yaml": "llama-3.2-license-2024.yml", + "html": "llama-3.2-license-2024.html", + "license": "llama-3.2-license-2024.LICENSE" + }, + { + "license_key": "llama-3.3-license-2024", + "category": "Proprietary Free", + "spdx_license_key": "LicenseRef-scancode-llama-3.3-license-2024", + "other_spdx_license_keys": [], + "is_exception": false, + "is_deprecated": false, + "json": "llama-3.3-license-2024.json", + "yaml": "llama-3.3-license-2024.yml", + "html": "llama-3.3-license-2024.html", + "license": "llama-3.3-license-2024.LICENSE" + }, + { + "license_key": "llama-4-license-2025", + "category": "Proprietary Free", + "spdx_license_key": "LicenseRef-scancode-llama-4-license-2025", + "other_spdx_license_keys": [], + "is_exception": false, + "is_deprecated": false, + "json": "llama-4-license-2025.json", + "yaml": "llama-4-license-2025.yml", + "html": "llama-4-license-2025.html", + "license": "llama-4-license-2025.LICENSE" + }, { "license_key": "llama-license-2023", "category": "Proprietary Free", @@ -15492,6 +15999,18 @@ "html": "mapbox-tos-2021.html", "license": "mapbox-tos-2021.LICENSE" }, + { + "license_key": "mapbox-tos-2024", + "category": "Commercial", + "spdx_license_key": "LicenseRef-scancode-mapbox-tos-2024", + "other_spdx_license_keys": [], + "is_exception": false, + "is_deprecated": false, + "json": "mapbox-tos-2024.json", + "yaml": "mapbox-tos-2024.yml", + "html": "mapbox-tos-2024.html", + "license": "mapbox-tos-2024.LICENSE" + }, { "license_key": "markus-kuhn-license", "category": "Permissive", @@ -15556,6 +16075,18 @@ "html": "marvell-firmware-2019.html", "license": "marvell-firmware-2019.LICENSE" }, + { + "license_key": "matplotlib-1.3.0", + "category": "Permissive", + "spdx_license_key": "LicenseRef-scancode-matplotlib-1.3.0", + "other_spdx_license_keys": [], + "is_exception": false, + "is_deprecated": false, + "json": "matplotlib-1.3.0.json", + "yaml": "matplotlib-1.3.0.yml", + "html": "matplotlib-1.3.0.html", + "license": "matplotlib-1.3.0.LICENSE" + }, { "license_key": "matt-gallagher-attribution", "category": "Permissive", @@ -15568,6 +16099,18 @@ "html": "matt-gallagher-attribution.html", "license": "matt-gallagher-attribution.LICENSE" }, + { + "license_key": "mattermost-sal-2024", + "category": "Source-available", + "spdx_license_key": "LicenseRef-scancode-mattermost-sal-2024", + "other_spdx_license_keys": [], + "is_exception": false, + "is_deprecated": false, + "json": "mattermost-sal-2024.json", + "yaml": "mattermost-sal-2024.yml", + "html": "mattermost-sal-2024.html", + "license": "mattermost-sal-2024.LICENSE" + }, { "license_key": "matthew-kwan", "category": "Permissive", @@ -15676,6 +16219,18 @@ "html": "mcrae-pl-4-r53.html", "license": "mcrae-pl-4-r53.LICENSE" }, + { + "license_key": "mdl-2021", + "category": "Proprietary Free", + "spdx_license_key": "LicenseRef-scancode-mdl-2021", + "other_spdx_license_keys": [], + "is_exception": false, + "is_deprecated": false, + "json": "mdl-2021.json", + "yaml": "mdl-2021.yml", + "html": "mdl-2021.html", + "license": "mdl-2021.LICENSE" + }, { "license_key": "mediainfo-lib", "category": "Permissive", @@ -15796,6 +16351,18 @@ "html": "metrolink-1.0.html", "license": "metrolink-1.0.LICENSE" }, + { + "license_key": "mgb-1.0", + "category": "Permissive", + "spdx_license_key": "LicenseRef-scancode-mgb-1.0", + "other_spdx_license_keys": [], + "is_exception": false, + "is_deprecated": false, + "json": "mgb-1.0.json", + "yaml": "mgb-1.0.yml", + "html": "mgb-1.0.html", + "license": "mgb-1.0.LICENSE" + }, { "license_key": "mgopen-font-license", "category": "Permissive", @@ -16398,6 +16965,18 @@ "html": "motosoto-0.9.1.html", "license": "motosoto-0.9.1.LICENSE" }, + { + "license_key": "mov-ai-1.0", + "category": "Source-available", + "spdx_license_key": "LicenseRef-scancode-mov-ai-1.0", + "other_spdx_license_keys": [], + "is_exception": false, + "is_deprecated": false, + "json": "mov-ai-1.0.json", + "yaml": "mov-ai-1.0.yml", + "html": "mov-ai-1.0.html", + "license": "mov-ai-1.0.LICENSE" + }, { "license_key": "moxa-linux-firmware", "category": "Proprietary Free", @@ -16682,6 +17261,42 @@ "html": "ms-azure-data-studio.html", "license": "ms-azure-data-studio.LICENSE" }, + { + "license_key": "ms-azure-rtos-2020-05", + "category": "Proprietary Free", + "spdx_license_key": "LicenseRef-scancode-ms-azure-rtos-2020-05", + "other_spdx_license_keys": [], + "is_exception": false, + "is_deprecated": false, + "json": "ms-azure-rtos-2020-05.json", + "yaml": "ms-azure-rtos-2020-05.yml", + "html": "ms-azure-rtos-2020-05.html", + "license": "ms-azure-rtos-2020-05.LICENSE" + }, + { + "license_key": "ms-azure-rtos-2020-07", + "category": "Proprietary Free", + "spdx_license_key": "LicenseRef-scancode-ms-azure-rtos-2020-07", + "other_spdx_license_keys": [], + "is_exception": false, + "is_deprecated": false, + "json": "ms-azure-rtos-2020-07.json", + "yaml": "ms-azure-rtos-2020-07.yml", + "html": "ms-azure-rtos-2020-07.html", + "license": "ms-azure-rtos-2020-07.LICENSE" + }, + { + "license_key": "ms-azure-rtos-2023-05", + "category": "Proprietary Free", + "spdx_license_key": "LicenseRef-scancode-ms-azure-rtos-2023-05", + "other_spdx_license_keys": [], + "is_exception": false, + "is_deprecated": false, + "json": "ms-azure-rtos-2023-05.json", + "yaml": "ms-azure-rtos-2023-05.yml", + "html": "ms-azure-rtos-2023-05.html", + "license": "ms-azure-rtos-2023-05.LICENSE" + }, { "license_key": "ms-azure-spatialanchors-2.9.0", "category": "Proprietary Free", @@ -18362,6 +18977,18 @@ "html": "new-relic.html", "license": "new-relic.LICENSE" }, + { + "license_key": "new-relic-1.0", + "category": "Commercial", + "spdx_license_key": "LicenseRef-scancode-new-relic-1.0", + "other_spdx_license_keys": [], + "is_exception": false, + "is_deprecated": false, + "json": "new-relic-1.0.json", + "yaml": "new-relic-1.0.yml", + "html": "new-relic-1.0.html", + "license": "new-relic-1.0.LICENSE" + }, { "license_key": "newlib-historical", "category": "Permissive", @@ -18700,6 +19327,18 @@ "html": "nortel-dasa.html", "license": "nortel-dasa.LICENSE" }, + { + "license_key": "northwoods-evaluation-2024", + "category": "Proprietary Free", + "spdx_license_key": "LicenseRef-scancode-northwoods-evaluation-2024", + "other_spdx_license_keys": [], + "is_exception": false, + "is_deprecated": false, + "json": "northwoods-evaluation-2024.json", + "yaml": "northwoods-evaluation-2024.yml", + "html": "northwoods-evaluation-2024.html", + "license": "northwoods-evaluation-2024.LICENSE" + }, { "license_key": "northwoods-sla-2021", "category": "Commercial", @@ -18712,6 +19351,18 @@ "html": "northwoods-sla-2021.html", "license": "northwoods-sla-2021.LICENSE" }, + { + "license_key": "northwoods-sla-2024", + "category": "Commercial", + "spdx_license_key": "LicenseRef-scancode-northwoods-sla-2024", + "other_spdx_license_keys": [], + "is_exception": false, + "is_deprecated": false, + "json": "northwoods-sla-2024.json", + "yaml": "northwoods-sla-2024.yml", + "html": "northwoods-sla-2024.html", + "license": "northwoods-sla-2024.LICENSE" + }, { "license_key": "nosl-1.0", "category": "Copyleft Limited", @@ -19028,6 +19679,18 @@ "html": "nvidia-isaac-eula-2019.1.html", "license": "nvidia-isaac-eula-2019.1.LICENSE" }, + { + "license_key": "nvidia-nccl-sla-2016", + "category": "Proprietary Free", + "spdx_license_key": "LicenseRef-scancode-nvidia-nccl-sla-2016", + "other_spdx_license_keys": [], + "is_exception": false, + "is_deprecated": false, + "json": "nvidia-nccl-sla-2016.json", + "yaml": "nvidia-nccl-sla-2016.yml", + "html": "nvidia-nccl-sla-2016.html", + "license": "nvidia-nccl-sla-2016.LICENSE" + }, { "license_key": "nvidia-ngx-eula-2019", "category": "Proprietary Free", @@ -19040,6 +19703,18 @@ "html": "nvidia-ngx-eula-2019.html", "license": "nvidia-ngx-eula-2019.LICENSE" }, + { + "license_key": "nvidia-sdk-12.8", + "category": "Proprietary Free", + "spdx_license_key": "LicenseRef-scancode-nvidia-sdk-12.8", + "other_spdx_license_keys": [], + "is_exception": false, + "is_deprecated": false, + "json": "nvidia-sdk-12.8.json", + "yaml": "nvidia-sdk-12.8.yml", + "html": "nvidia-sdk-12.8.html", + "license": "nvidia-sdk-12.8.LICENSE" + }, { "license_key": "nvidia-sdk-eula-v0.11", "category": "Proprietary Free", @@ -19248,6 +19923,18 @@ "html": "object-form-exception-to-mit.html", "license": "object-form-exception-to-mit.LICENSE" }, + { + "license_key": "obsidian-tos-2025", + "category": "Proprietary Free", + "spdx_license_key": "LicenseRef-scancode-obsidian-tos-2025", + "other_spdx_license_keys": [], + "is_exception": false, + "is_deprecated": false, + "json": "obsidian-tos-2025.json", + "yaml": "obsidian-tos-2025.yml", + "html": "obsidian-tos-2025.html", + "license": "obsidian-tos-2025.LICENSE" + }, { "license_key": "ocaml-lgpl-linking-exception", "category": "Copyleft Limited", @@ -19260,6 +19947,18 @@ "html": "ocaml-lgpl-linking-exception.html", "license": "ocaml-lgpl-linking-exception.LICENSE" }, + { + "license_key": "ocamlpro-nc-v1", + "category": "Proprietary Free", + "spdx_license_key": "LicenseRef-scancode-ocamlpro-nc-v1", + "other_spdx_license_keys": [], + "is_exception": false, + "is_deprecated": false, + "json": "ocamlpro-nc-v1.json", + "yaml": "ocamlpro-nc-v1.yml", + "html": "ocamlpro-nc-v1.html", + "license": "ocamlpro-nc-v1.LICENSE" + }, { "license_key": "ocb-non-military-2013", "category": "Proprietary Free", @@ -19356,6 +20055,18 @@ "html": "ocsl-1.0.html", "license": "ocsl-1.0.LICENSE" }, + { + "license_key": "octl-0.21", + "category": "Source-available", + "spdx_license_key": "LicenseRef-scancode-octl-0.21", + "other_spdx_license_keys": [], + "is_exception": false, + "is_deprecated": false, + "json": "octl-0.21.json", + "yaml": "octl-0.21.yml", + "html": "octl-0.21.html", + "license": "octl-0.21.LICENSE" + }, { "license_key": "oculus-sdk", "category": "Copyleft Limited", @@ -19586,6 +20297,30 @@ "html": "ofrak-community-1.0.html", "license": "ofrak-community-1.0.LICENSE" }, + { + "license_key": "ofrak-community-1.1", + "category": "Proprietary Free", + "spdx_license_key": "LicenseRef-scancode-ofrak-community-1.1", + "other_spdx_license_keys": [], + "is_exception": false, + "is_deprecated": false, + "json": "ofrak-community-1.1.json", + "yaml": "ofrak-community-1.1.yml", + "html": "ofrak-community-1.1.html", + "license": "ofrak-community-1.1.LICENSE" + }, + { + "license_key": "ofrak-pro-1.0", + "category": "Commercial", + "spdx_license_key": "LicenseRef-scancode-ofrak-pro-1.0", + "other_spdx_license_keys": [], + "is_exception": false, + "is_deprecated": false, + "json": "ofrak-pro-1.0.json", + "yaml": "ofrak-pro-1.0.yml", + "html": "ofrak-pro-1.0.html", + "license": "ofrak-pro-1.0.LICENSE" + }, { "license_key": "ogc", "category": "Permissive", @@ -19802,6 +20537,18 @@ "html": "on2-patent.html", "license": "on2-patent.LICENSE" }, + { + "license_key": "onezoom-np-sal-v1", + "category": "Source-available", + "spdx_license_key": "LicenseRef-scancode-onezoom-np-sal-v1", + "other_spdx_license_keys": [], + "is_exception": false, + "is_deprecated": false, + "json": "onezoom-np-sal-v1.json", + "yaml": "onezoom-np-sal-v1.yml", + "html": "onezoom-np-sal-v1.html", + "license": "onezoom-np-sal-v1.LICENSE" + }, { "license_key": "ooura-2001", "category": "Proprietary Free", @@ -19862,6 +20609,18 @@ "html": "openai-tou-20230314.html", "license": "openai-tou-20230314.LICENSE" }, + { + "license_key": "openai-tou-20241211", + "category": "Commercial", + "spdx_license_key": "LicenseRef-scancode-openai-tou-20241211", + "other_spdx_license_keys": [], + "is_exception": false, + "is_deprecated": false, + "json": "openai-tou-20241211.json", + "yaml": "openai-tou-20241211.yml", + "html": "openai-tou-20241211.html", + "license": "openai-tou-20241211.LICENSE" + }, { "license_key": "openbd-exception-3.0", "category": "Copyleft Limited", @@ -20856,6 +21615,18 @@ "html": "oracle-sql-developer.html", "license": "oracle-sql-developer.LICENSE" }, + { + "license_key": "oracle-vb-puel-12", + "category": "Proprietary Free", + "spdx_license_key": "LicenseRef-scancode-oracle-vb-puel-12", + "other_spdx_license_keys": [], + "is_exception": false, + "is_deprecated": false, + "json": "oracle-vb-puel-12.json", + "yaml": "oracle-vb-puel-12.yml", + "html": "oracle-vb-puel-12.html", + "license": "oracle-vb-puel-12.LICENSE" + }, { "license_key": "oracle-web-sites-tou", "category": "Proprietary Free", @@ -21756,6 +22527,18 @@ "html": "pine.html", "license": "pine.LICENSE" }, + { + "license_key": "pipedream-sal-1.0", + "category": "Source-available", + "spdx_license_key": "LicenseRef-scancode-pipedream-sal-1.0", + "other_spdx_license_keys": [], + "is_exception": false, + "is_deprecated": false, + "json": "pipedream-sal-1.0.json", + "yaml": "pipedream-sal-1.0.yml", + "html": "pipedream-sal-1.0.html", + "license": "pipedream-sal-1.0.LICENSE" + }, { "license_key": "pivotal-tou", "category": "Commercial", @@ -21974,6 +22757,18 @@ "html": "postgresql.html", "license": "postgresql.LICENSE" }, + { + "license_key": "postman-tos-2024", + "category": "Proprietary Free", + "spdx_license_key": "LicenseRef-scancode-postman-tos-2024", + "other_spdx_license_keys": [], + "is_exception": false, + "is_deprecated": false, + "json": "postman-tos-2024.json", + "yaml": "postman-tos-2024.yml", + "html": "postman-tos-2024.html", + "license": "postman-tos-2024.LICENSE" + }, { "license_key": "powervr-tools-software-eula", "category": "Proprietary Free", @@ -22668,6 +23463,30 @@ "html": "rackspace.html", "license": "rackspace.LICENSE" }, + { + "license_key": "radiance-sl-v1.0", + "category": "Permissive", + "spdx_license_key": "LicenseRef-scancode-radiance-sl-v1.0", + "other_spdx_license_keys": [], + "is_exception": false, + "is_deprecated": false, + "json": "radiance-sl-v1.0.json", + "yaml": "radiance-sl-v1.0.yml", + "html": "radiance-sl-v1.0.html", + "license": "radiance-sl-v1.0.LICENSE" + }, + { + "license_key": "radiance-sl-v2.0", + "category": "Copyleft Limited", + "spdx_license_key": "LicenseRef-scancode-radiance-sl-v2.0", + "other_spdx_license_keys": [], + "is_exception": false, + "is_deprecated": false, + "json": "radiance-sl-v2.0.json", + "yaml": "radiance-sl-v2.0.yml", + "html": "radiance-sl-v2.0.html", + "license": "radiance-sl-v2.0.LICENSE" + }, { "license_key": "radvd", "category": "Permissive", @@ -23094,6 +23913,18 @@ "html": "rogue-wave.html", "license": "rogue-wave.LICENSE" }, + { + "license_key": "root-cert-3.0", + "category": "Proprietary Free", + "spdx_license_key": "LicenseRef-scancode-root-cert-3.0", + "other_spdx_license_keys": [], + "is_exception": false, + "is_deprecated": false, + "json": "root-cert-3.0.json", + "yaml": "root-cert-3.0.yml", + "html": "root-cert-3.0.html", + "license": "root-cert-3.0.LICENSE" + }, { "license_key": "rpl-1.1", "category": "Copyleft Limited", @@ -23384,6 +24215,18 @@ "html": "salesforcesans-font.html", "license": "salesforcesans-font.LICENSE" }, + { + "license_key": "samba-dc-1.0", + "category": "CLA", + "spdx_license_key": "LicenseRef-scancode-samba-dc-1.0", + "other_spdx_license_keys": [], + "is_exception": false, + "is_deprecated": false, + "json": "samba-dc-1.0.json", + "yaml": "samba-dc-1.0.yml", + "html": "samba-dc-1.0.html", + "license": "samba-dc-1.0.LICENSE" + }, { "license_key": "san-francisco-font", "category": "Proprietary Free", @@ -23688,6 +24531,18 @@ "html": "scsl-3.0.html", "license": "scsl-3.0.LICENSE" }, + { + "license_key": "scylladb-sla-1.0", + "category": "Source-available", + "spdx_license_key": "LicenseRef-scancode-scylladb-sla-1.0", + "other_spdx_license_keys": [], + "is_exception": false, + "is_deprecated": false, + "json": "scylladb-sla-1.0.json", + "yaml": "scylladb-sla-1.0.yml", + "html": "scylladb-sla-1.0.html", + "license": "scylladb-sla-1.0.LICENSE" + }, { "license_key": "secret-labs-2011", "category": "Permissive", @@ -23724,6 +24579,18 @@ "html": "selinux-nsa-declaration-1.0.html", "license": "selinux-nsa-declaration-1.0.LICENSE" }, + { + "license_key": "selv1", + "category": "Commercial", + "spdx_license_key": "LicenseRef-scancode-selv1", + "other_spdx_license_keys": [], + "is_exception": false, + "is_deprecated": false, + "json": "selv1.json", + "yaml": "selv1.yml", + "html": "selv1.html", + "license": "selv1.LICENSE" + }, { "license_key": "semgrep-registry", "category": "Source-available", @@ -23736,6 +24603,18 @@ "html": "semgrep-registry.html", "license": "semgrep-registry.LICENSE" }, + { + "license_key": "semgrep-rules-1.0", + "category": "Proprietary Free", + "spdx_license_key": "LicenseRef-scancode-semgrep-rules-1.0", + "other_spdx_license_keys": [], + "is_exception": false, + "is_deprecated": false, + "json": "semgrep-rules-1.0.json", + "yaml": "semgrep-rules-1.0.yml", + "html": "semgrep-rules-1.0.html", + "license": "semgrep-rules-1.0.LICENSE" + }, { "license_key": "sencha-app-floss-exception", "category": "Copyleft", @@ -23928,6 +24807,18 @@ "html": "sglib.html", "license": "sglib.LICENSE" }, + { + "license_key": "sgmlug", + "category": "Permissive", + "spdx_license_key": "LicenseRef-scancode-sgmlug", + "other_spdx_license_keys": [], + "is_exception": false, + "is_deprecated": false, + "json": "sgmlug.json", + "yaml": "sgmlug.yml", + "html": "sgmlug.html", + "license": "sgmlug.LICENSE" + }, { "license_key": "sgp4", "category": "Permissive", @@ -24414,6 +25305,18 @@ "html": "soml-1.0.html", "license": "soml-1.0.LICENSE" }, + { + "license_key": "sonar-sal-1.0", + "category": "Source-available", + "spdx_license_key": "LicenseRef-scancode-sonar-sal-1.0", + "other_spdx_license_keys": [], + "is_exception": false, + "is_deprecated": false, + "json": "sonar-sal-1.0.json", + "yaml": "sonar-sal-1.0.yml", + "html": "sonar-sal-1.0.html", + "license": "sonar-sal-1.0.LICENSE" + }, { "license_key": "soundex", "category": "Permissive", @@ -24824,6 +25727,18 @@ "html": "subcommander-exception-2.0-plus.html", "license": "subcommander-exception-2.0-plus.LICENSE" }, + { + "license_key": "sudo", + "category": "Permissive", + "spdx_license_key": "LicenseRef-scancode-sudo", + "other_spdx_license_keys": [], + "is_exception": false, + "is_deprecated": false, + "json": "sudo.json", + "yaml": "sudo.yml", + "html": "sudo.html", + "license": "sudo.LICENSE" + }, { "license_key": "sugarcrm-1.1.3", "category": "Copyleft", @@ -26384,6 +27299,18 @@ "html": "treeview-distributor.html", "license": "treeview-distributor.LICENSE" }, + { + "license_key": "trendmicro-cl-1.0", + "category": "Proprietary Free", + "spdx_license_key": "LicenseRef-scancode-trendmicro-cl-1.0", + "other_spdx_license_keys": [], + "is_exception": false, + "is_deprecated": false, + "json": "trendmicro-cl-1.0.json", + "yaml": "trendmicro-cl-1.0.yml", + "html": "trendmicro-cl-1.0.html", + "license": "trendmicro-cl-1.0.LICENSE" + }, { "license_key": "triptracker", "category": "Proprietary Free", @@ -26727,6 +27654,18 @@ "html": "umich-merit.html", "license": "umich-merit.LICENSE" }, + { + "license_key": "un-cefact-2016", + "category": "Permissive", + "spdx_license_key": "LicenseRef-scancode-un-cefact-2016", + "other_spdx_license_keys": [], + "is_exception": false, + "is_deprecated": false, + "json": "un-cefact-2016.json", + "yaml": "un-cefact-2016.yml", + "html": "un-cefact-2016.html", + "license": "un-cefact-2016.LICENSE" + }, { "license_key": "unbuntu-font-1.0", "category": "Free Restricted", @@ -26823,6 +27762,18 @@ "html": "unicode-tou.html", "license": "unicode-tou.LICENSE" }, + { + "license_key": "unicode-ucd", + "category": "Permissive", + "spdx_license_key": "LicenseRef-scancode-unicode-ucd", + "other_spdx_license_keys": [], + "is_exception": false, + "is_deprecated": false, + "json": "unicode-ucd.json", + "yaml": "unicode-ucd.yml", + "html": "unicode-ucd.html", + "license": "unicode-ucd.LICENSE" + }, { "license_key": "unicode-v3", "category": "Permissive", @@ -27384,6 +28335,18 @@ "html": "volatility-vsl-v1.0.html", "license": "volatility-vsl-v1.0.LICENSE" }, + { + "license_key": "volla-1.0", + "category": "Proprietary Free", + "spdx_license_key": "LicenseRef-scancode-volla-1.0", + "other_spdx_license_keys": [], + "is_exception": false, + "is_deprecated": false, + "json": "volla-1.0.json", + "yaml": "volla-1.0.yml", + "html": "volla-1.0.html", + "license": "volla-1.0.LICENSE" + }, { "license_key": "vostrom", "category": "Copyleft", @@ -27516,6 +28479,18 @@ "html": "w3c-community-cla.html", "license": "w3c-community-cla.LICENSE" }, + { + "license_key": "w3c-community-final-spec", + "category": "Permissive", + "spdx_license_key": "LicenseRef-scancode-w3c-community-final-spec", + "other_spdx_license_keys": [], + "is_exception": false, + "is_deprecated": false, + "json": "w3c-community-final-spec.json", + "yaml": "w3c-community-final-spec.yml", + "html": "w3c-community-final-spec.html", + "license": "w3c-community-final-spec.LICENSE" + }, { "license_key": "w3c-docs-19990405", "category": "Free Restricted", @@ -27624,6 +28599,18 @@ "html": "w3m.html", "license": "w3m.LICENSE" }, + { + "license_key": "wadalab", + "category": "Permissive", + "spdx_license_key": "LicenseRef-scancode-wadalab", + "other_spdx_license_keys": [], + "is_exception": false, + "is_deprecated": false, + "json": "wadalab.json", + "yaml": "wadalab.yml", + "html": "wadalab.html", + "license": "wadalab.LICENSE" + }, { "license_key": "warranty-disclaimer", "category": "Unstated License", @@ -28020,6 +29007,18 @@ "html": "wxwindows-exception-3.1.html", "license": "wxwindows-exception-3.1.LICENSE" }, + { + "license_key": "wxwindows-free-doc-3", + "category": "Copyleft Limited", + "spdx_license_key": "LicenseRef-scancode-wxwindows-free-doc-3", + "other_spdx_license_keys": [], + "is_exception": false, + "is_deprecated": false, + "json": "wxwindows-free-doc-3.json", + "yaml": "wxwindows-free-doc-3.yml", + "html": "wxwindows-free-doc-3.html", + "license": "wxwindows-free-doc-3.LICENSE" + }, { "license_key": "wxwindows-r-3.0", "category": "Copyleft Limited", @@ -28734,6 +29733,30 @@ "html": "zed.html", "license": "zed.LICENSE" }, + { + "license_key": "zeebe-community-1.0", + "category": "Source-available", + "spdx_license_key": "LicenseRef-scancode-zeebe-community-1.0", + "other_spdx_license_keys": [], + "is_exception": false, + "is_deprecated": false, + "json": "zeebe-community-1.0.json", + "yaml": "zeebe-community-1.0.yml", + "html": "zeebe-community-1.0.html", + "license": "zeebe-community-1.0.LICENSE" + }, + { + "license_key": "zeebe-community-1.1", + "category": "Source-available", + "spdx_license_key": "LicenseRef-scancode-zeebe-community-1.1", + "other_spdx_license_keys": [], + "is_exception": false, + "is_deprecated": false, + "json": "zeebe-community-1.1.json", + "yaml": "zeebe-community-1.1.yml", + "html": "zeebe-community-1.1.html", + "license": "zeebe-community-1.1.LICENSE" + }, { "license_key": "zeeff", "category": "Permissive", From ccdfcd21247db2c146c71edb2c61753461406b4b Mon Sep 17 00:00:00 2001 From: Ayan Sinha Mahapatra Date: Wed, 25 Jun 2025 01:17:44 +0530 Subject: [PATCH 69/69] Fix test failures Signed-off-by: Ayan Sinha Mahapatra --- Makefile | 2 - README.rst | 93 +- docs/source/conf.py | 6 +- docs/source/index.rst | 1 + src/license_expression/__init__.py | 420 ++-- src/license_expression/_pyahocorasick.py | 114 +- tests/test__pyahocorasick.py | 323 ++- tests/test_license_expression.py | 2368 +++++++++++----------- 8 files changed, 1650 insertions(+), 1677 deletions(-) diff --git a/Makefile b/Makefile index 3041547..3cbba76 100644 --- a/Makefile +++ b/Makefile @@ -38,8 +38,6 @@ check: @echo "-> Run Ruff format validation" @${ACTIVATE} ruff format --check @$(MAKE) doc8 - @echo "-> Run ABOUT files validation" - @${ACTIVATE} about check etc/ clean: @echo "-> Clean the Python env" diff --git a/README.rst b/README.rst index 2f7eb58..4174607 100644 --- a/README.rst +++ b/README.rst @@ -34,9 +34,8 @@ license expression engine in several projects and products such as: - AboutCode-toolkit https://github.com/aboutcode-org/aboutcode-toolkit - AlekSIS (School Information System) https://edugit.org/AlekSIS/official/AlekSIS-Core -- Barista https://github.com/Optum/barista - Conda forge tools https://github.com/conda-forge/conda-smithy -- DejaCode https://dejacode.com +- DejaCode https://enterprise.dejacode.com - DeltaCode https://github.com/nexB/deltacode - FenixscanX https://github.com/SmartsYoung/FenixscanX - FetchCode https://github.com/aboutcode-org/fetchcode @@ -49,7 +48,7 @@ license expression engine in several projects and products such as: - SecObserve https://github.com/MaibornWolff/SecObserve See also for details: -- https://spdx.github.io/spdx-spec/appendix-IV-SPDX-license-expressions/ +- https://spdx.github.io/spdx-spec/v2.3/SPDX-license-expressions ``license-expression`` is also packaged for most Linux distributions. See below. @@ -65,17 +64,6 @@ libraries in other languages (but not as powerful of course!): - Ada https://github.com/Fabien-Chouteau/spdx_ada - Java https://github.com/spdx/tools and https://github.com/aschet/spdx-license-expression-tools -Build and tests status -====================== - -+--------------------------+------------------------+----------------------------------+ -|**Linux & macOS (Travis)**| **Windows (AppVeyor)** |**Linux, Windows & macOS (Azure)**| -+==========================+========================+==================================+ -| | | | -| |travis-badge-icon| | |appveyor-badge-icon| | |azure-badge-icon| | -| | | | -+--------------------------+------------------------+----------------------------------+ - Source code and download ======================== @@ -125,36 +113,36 @@ validate, compare, simplify and normalize license expressions. Create an SPDX Licensing and parse expressions:: - >>> from license_expression import get_spdx_licensing - >>> licensing = get_spdx_licensing() - >>> expression = ' GPL-2.0 or LGPL-2.1 and mit ' - >>> parsed = licensing.parse(expression) - >>> print(parsed.pretty()) - OR( - LicenseSymbol('GPL-2.0-only'), - AND( - LicenseSymbol('LGPL-2.1-only'), - LicenseSymbol('MIT') - ) - ) - - >>> str(parsed) - 'GPL-2.0-only OR (LGPL-2.1-only AND MIT)' - - >>> licensing.parse('unknwon with foo', validate=True, strict=True) - license_expression.ExpressionParseError: A plain license symbol cannot be used - as an exception in a "WITH symbol" statement. for token: "foo" at position: 13 - - >>> licensing.parse('unknwon with foo', validate=True) - license_expression.ExpressionError: Unknown license key(s): unknwon, foo - - >>> licensing.validate('foo and MIT and GPL-2.0+') - ExpressionInfo( - original_expression='foo and MIT and GPL-2.0+', - normalized_expression=None, - errors=['Unknown license key(s): foo'], - invalid_symbols=['foo'] - ) + >>> from license_expression import get_spdx_licensing + >>> licensing = get_spdx_licensing() + >>> expression = ' GPL-2.0 or LGPL-2.1 and mit ' + >>> parsed = licensing.parse(expression) + >>> print(parsed.pretty()) + OR( + LicenseSymbol('GPL-2.0-only'), + AND( + LicenseSymbol('LGPL-2.1-only'), + LicenseSymbol('MIT') + ) + ) + + >>> str(parsed) + 'GPL-2.0-only OR (LGPL-2.1-only AND MIT)' + + >>> licensing.parse('unknwon with foo', validate=True, strict=True) + license_expression.ExpressionParseError: A plain license symbol cannot be used + as an exception in a "WITH symbol" statement. for token: "foo" at position: 13 + + >>> licensing.parse('unknwon with foo', validate=True) + license_expression.ExpressionError: Unknown license key(s): unknwon, foo + + >>> licensing.validate('foo and MIT and GPL-2.0+') + ExpressionInfo( + original_expression='foo and MIT and GPL-2.0+', + normalized_expression=None, + errors=['Unknown license key(s): foo'], + invalid_symbols=['foo'] + ) Create a simple Licensing and parse expressions:: @@ -243,20 +231,3 @@ Development - On Windows run ``configure.bat --dev`` and then ``Scripts\bin\activate`` instead. - To run the tests, run ``pytest -vvs`` - - -.. |travis-badge-icon| image:: https://api.travis-ci.org/nexB/license-expression.png?branch=master - :target: https://travis-ci.org/nexB/license-expression - :alt: Travis tests status - :align: middle - -.. |appveyor-badge-icon| image:: https://ci.appveyor.com/api/projects/status/github/nexB/license-expression?svg=true - :target: https://ci.appveyor.com/project/nexB/license-expression - :alt: Appveyor tests status - :align: middle - -.. |azure-badge-icon| image:: https://dev.azure.com/nexB/license-expression/_apis/build/status/nexB.license-expression?branchName=master - :target: https://dev.azure.com/nexB/license-expression/_build/latest?definitionId=2&branchName=master - :alt: Azure pipelines tests status - :align: middle - diff --git a/docs/source/conf.py b/docs/source/conf.py index 1a10373..ae14b3a 100644 --- a/docs/source/conf.py +++ b/docs/source/conf.py @@ -12,7 +12,7 @@ import pathlib import sys -srcdir = pathlib.Path(__file__).resolve().parents[2].joinpath('src') +srcdir = pathlib.Path(__file__).resolve().parents[2].joinpath("src") sys.path.insert(0, srcdir.as_posix()) # -- Project information ----------------------------------------------------- @@ -40,7 +40,7 @@ # FIXME: including AND, NOT and OR will result in endless recursion autodoc_default_options = { - 'exclude-members': 'AND, NOT, OR', + "exclude-members": "AND, NOT, OR", } @@ -61,7 +61,7 @@ } # Setting for sphinxcontrib.apidoc to automatically create API documentation. -apidoc_module_dir = srcdir.joinpath('license_expression').as_posix() +apidoc_module_dir = srcdir.joinpath("license_expression").as_posix() # Reference to other Sphinx documentations intersphinx_mapping = { diff --git a/docs/source/index.rst b/docs/source/index.rst index 6468e02..cff5912 100644 --- a/docs/source/index.rst +++ b/docs/source/index.rst @@ -7,6 +7,7 @@ Welcome to license-expressions's documentation! readme_link API + contribute/contrib_doc Indices and tables ================== diff --git a/src/license_expression/__init__.py b/src/license_expression/__init__.py index a4814fe..dc1ab31 100644 --- a/src/license_expression/__init__.py +++ b/src/license_expression/__init__.py @@ -6,10 +6,10 @@ # See https://aboutcode.org for more information about nexB OSS projects. # """ -This module defines a mini language to parse, validate, deduplicate, simplify, +Define a mini language to parse, validate, deduplicate, simplify, normalize and compare license expressions using a boolean logic engine. -This supports SPDX and ScanCode license expressions and also accepts other +This module supports SPDX and ScanCode license expressions and also accepts other license naming conventions and license identifiers aliases to recognize and normalize licenses. @@ -57,38 +57,33 @@ from license_expression._pyahocorasick import Token curr_dir = dirname(abspath(__file__)) -data_dir = join(curr_dir, 'data') +data_dir = join(curr_dir, "data") vendored_scancode_licensedb_index_location = join( data_dir, - 'scancode-licensedb-index.json', + "scancode-licensedb-index.json", ) # append new error codes to PARSE_ERRORS by monkey patching PARSE_EXPRESSION_NOT_UNICODE = 100 if PARSE_EXPRESSION_NOT_UNICODE not in PARSE_ERRORS: - PARSE_ERRORS[PARSE_EXPRESSION_NOT_UNICODE] = ( - 'Expression string must be a string.' - ) + PARSE_ERRORS[PARSE_EXPRESSION_NOT_UNICODE] = "Expression string must be a string." PARSE_INVALID_EXCEPTION = 101 if PARSE_INVALID_EXCEPTION not in PARSE_ERRORS: PARSE_ERRORS[PARSE_INVALID_EXCEPTION] = ( - 'A license exception symbol can only be used as an exception ' + "A license exception symbol can only be used as an exception " 'in a "WITH exception" statement.' ) PARSE_INVALID_SYMBOL_AS_EXCEPTION = 102 if PARSE_INVALID_SYMBOL_AS_EXCEPTION not in PARSE_ERRORS: PARSE_ERRORS[PARSE_INVALID_SYMBOL_AS_EXCEPTION] = ( - 'A plain license symbol cannot be used as an exception ' - 'in a "WITH symbol" statement.' + 'A plain license symbol cannot be used as an exception in a "WITH symbol" statement.' ) PARSE_INVALID_SYMBOL = 103 if PARSE_INVALID_SYMBOL not in PARSE_ERRORS: - PARSE_ERRORS[PARSE_INVALID_SYMBOL] = ( - 'A proper license symbol is needed.' - ) + PARSE_ERRORS[PARSE_INVALID_SYMBOL] = "A proper license symbol is needed." class ExpressionError(Exception): @@ -100,7 +95,7 @@ class ExpressionParseError(ParseError, ExpressionError): # Used for tokenizing -Keyword = namedtuple('Keyword', 'value type') +Keyword = namedtuple("Keyword", "value type") Keyword.__len__ = lambda self: len(self.value) # id for the "WITH" token which is not a proper boolean symbol but an expression @@ -109,19 +104,26 @@ class ExpressionParseError(ParseError, ExpressionError): # keyword types that include operators and parens -KW_LPAR = Keyword('(', TOKEN_LPAR) -KW_RPAR = Keyword(')', TOKEN_RPAR) -KW_AND = Keyword('and', TOKEN_AND) -KW_OR = Keyword('or', TOKEN_OR) -KW_WITH = Keyword('with', TOKEN_WITH) - -KEYWORDS = (KW_AND, KW_OR, KW_LPAR, KW_RPAR, KW_WITH,) +KW_LPAR = Keyword("(", TOKEN_LPAR) +KW_RPAR = Keyword(")", TOKEN_RPAR) +KW_AND = Keyword("and", TOKEN_AND) +KW_OR = Keyword("or", TOKEN_OR) +KW_WITH = Keyword("with", TOKEN_WITH) + +KEYWORDS = ( + KW_AND, + KW_OR, + KW_LPAR, + KW_RPAR, + KW_WITH, +) KEYWORDS_STRINGS = set(kw.value for kw in KEYWORDS) # mapping of lowercase operator strings to an operator object -OPERATORS = {'and': KW_AND, 'or': KW_OR, 'with': KW_WITH} +OPERATORS = {"and": KW_AND, "or": KW_OR, "with": KW_WITH} -_simple_tokenizer = re.compile(r''' +_simple_tokenizer = re.compile( + r""" (?P[^\s\(\)]+) | (?P\s+) @@ -129,8 +131,8 @@ class ExpressionParseError(ParseError, ExpressionError): (?P\() | (?P\)) - ''', - re.VERBOSE | re.MULTILINE | re.UNICODE + """, + re.VERBOSE | re.MULTILINE | re.UNICODE, ).finditer @@ -175,12 +177,12 @@ def __init__( def __repr__(self): return ( - 'ExpressionInfo(\n' - f' original_expression={self.original_expression!r},\n' - f' normalized_expression={self.normalized_expression!r},\n' - f' errors={self.errors!r},\n' - f' invalid_symbols={self.invalid_symbols!r}\n' - ')' + "ExpressionInfo(\n" + f" original_expression={self.original_expression!r},\n" + f" normalized_expression={self.normalized_expression!r},\n" + f" errors={self.errors!r},\n" + f" invalid_symbols={self.invalid_symbols!r}\n" + ")" ) @@ -286,19 +288,13 @@ def __init__(self, symbols=tuple(), quiet=True): print(e) if errors: - raise ValueError('\n'.join(warns + errors)) + raise ValueError("\n".join(warns + errors)) # mapping of known symbol key to symbol for reference - self.known_symbols = { - symbol.key: symbol - for symbol in symbols - } + self.known_symbols = {symbol.key: symbol for symbol in symbols} # mapping of known symbol lowercase key to symbol for reference - self.known_symbols_lowercase = { - symbol.key.lower(): symbol - for symbol in symbols - } + self.known_symbols_lowercase = {symbol.key.lower(): symbol for symbol in symbols} # Aho-Corasick automaton-based Advanced Tokenizer self.advanced_tokenizer = None @@ -333,9 +329,7 @@ def _parse_and_simplify(self, expression, **kwargs): return None if not isinstance(expression, LicenseExpression): - raise TypeError( - f'expression must be LicenseExpression object: {expression!r}' - ) + raise TypeError(f"expression must be LicenseExpression object: {expression!r}") return expression.simplify() @@ -398,7 +392,7 @@ def primary_license_key(self, expression, **kwargs): ``expression`` is either a string or a LicenseExpression object. Extra ``kwargs`` are passed down to the parse() function. - """ + """ prim = self.primary_license_symbol( expression=expression, decompose=True, @@ -476,17 +470,10 @@ def unknown_license_keys(self, expression, unique=True, **kwargs): def validate_license_keys(self, expression): unknown_keys = self.unknown_license_keys(expression, unique=True) if unknown_keys: - msg = 'Unknown license key(s): {}'.format(', '.join(unknown_keys)) + msg = "Unknown license key(s): {}".format(", ".join(unknown_keys)) raise ExpressionError(msg) - def parse( - self, - expression, - validate=False, - strict=False, - simple=False, - **kwargs - ): + def parse(self, expression, validate=False, strict=False, simple=False, **kwargs): """ Return a new license LicenseExpression object by parsing a license ``expression``. Check that the ``expression`` syntax is valid and @@ -536,25 +523,23 @@ def parse( expression = str(expression) except: ext = type(expression) - raise ExpressionError( - f'expression must be a string and not: {ext!r}' - ) + raise ExpressionError(f"expression must be a string and not: {ext!r}") if not isinstance(expression, str): ext = type(expression) - raise ExpressionError( - f'expression must be a string and not: {ext!r}' - ) + raise ExpressionError(f"expression must be a string and not: {ext!r}") if not expression or not expression.strip(): return try: # this will raise a ParseError on errors - tokens = list(self.tokenize( - expression=expression, - strict=strict, - simple=simple, - )) + tokens = list( + self.tokenize( + expression=expression, + strict=strict, + simple=simple, + ) + ) expression = super(Licensing, self).parse(tokens) except ParseError as e: @@ -566,8 +551,7 @@ def parse( ) from e if not isinstance(expression, LicenseExpression): - raise ExpressionError( - 'expression must be a LicenseExpression once parsed.') + raise ExpressionError("expression must be a LicenseExpression once parsed.") if validate: self.validate_license_keys(expression) @@ -654,12 +638,12 @@ def get_advanced_tokenizer(self): for key, symbol in self.known_symbols.items(): # always use the key even if there are no aliases. add_item(key, symbol) - aliases = getattr(symbol, 'aliases', []) + aliases = getattr(symbol, "aliases", []) for alias in aliases: # normalize spaces for each alias. The AdvancedTokenizer will # lowercase them if alias: - alias = ' '.join(alias.split()) + alias = " ".join(alias.split()) add_item(alias, symbol) tokenizer.make_automaton() @@ -695,19 +679,19 @@ def simple_tokenizer(self, expression): end = end - 1 match_getter = match.groupdict().get - space = match_getter('space') + space = match_getter("space") if space: yield Token(start, end, space, None) - lpar = match_getter('lpar') + lpar = match_getter("lpar") if lpar: yield Token(start, end, lpar, KW_LPAR) - rpar = match_getter('rpar') + rpar = match_getter("rpar") if rpar: yield Token(start, end, rpar, KW_RPAR) - sym_or_op = match_getter('symop') + sym_or_op = match_getter("symop") if sym_or_op: sym_or_op_lower = sym_or_op.lower() @@ -743,7 +727,13 @@ def dedup(self, expression): exp = self.parse(expression) expressions = [] for arg in exp.args: - if isinstance(arg, (self.AND, self.OR,)): + if isinstance( + arg, + ( + self.AND, + self.OR, + ), + ): # Run this recursive function if there is another AND/OR # expression and add the expression to the expressions list. expressions.append(self.dedup(arg)) @@ -752,7 +742,13 @@ def dedup(self, expression): if isinstance(exp, BaseSymbol): deduped = exp - elif isinstance(exp, (self.AND, self.OR,)): + elif isinstance( + exp, + ( + self.AND, + self.OR, + ), + ): relation = exp.__class__.__name__ deduped = combine_expressions( expressions, @@ -761,7 +757,7 @@ def dedup(self, expression): licensing=self, ) else: - raise ExpressionError(f'Unknown expression type: {expression!r}') + raise ExpressionError(f"Unknown expression type: {expression!r}") return deduped def validate(self, expression, strict=True, **kwargs): @@ -811,9 +807,7 @@ def validate(self, expression, strict=True, **kwargs): return expression_info -def get_scancode_licensing( - license_index_location=vendored_scancode_licensedb_index_location -): +def get_scancode_licensing(license_index_location=vendored_scancode_licensedb_index_location): """ Return a Licensing object using ScanCode license keys loaded from a ``license_index_location`` location of a license db JSON index files @@ -822,9 +816,7 @@ def get_scancode_licensing( return build_licensing(get_license_index(license_index_location)) -def get_spdx_licensing( - license_index_location=vendored_scancode_licensedb_index_location -): +def get_spdx_licensing(license_index_location=vendored_scancode_licensedb_index_location): """ Return a Licensing object using SPDX license keys loaded from a ``license_index_location`` location of a license db JSON index files @@ -833,9 +825,7 @@ def get_spdx_licensing( return build_spdx_licensing(get_license_index(license_index_location)) -def get_license_index( - license_index_location=vendored_scancode_licensedb_index_location -): +def get_license_index(license_index_location=vendored_scancode_licensedb_index_location): """ Return a list of mappings that contain license key information from ``license_index_location`` @@ -863,10 +853,11 @@ def build_licensing(license_index): """ lics = [ { - 'key': l.get('license_key', ''), - 'is_deprecated': l.get('is_deprecated', False), - 'is_exception': l.get('is_exception', False), - } for l in license_index + "key": l.get("license_key", ""), + "is_deprecated": l.get("is_deprecated", False), + "is_exception": l.get("is_exception", False), + } + for l in license_index ] return load_licensing_from_license_index(lics) @@ -879,11 +870,13 @@ def build_spdx_licensing(license_index): # Massage data such that SPDX license key is the primary license key lics = [ { - 'key': l.get('spdx_license_key', ''), - 'aliases': l.get('other_spdx_license_keys', []), - 'is_deprecated': l.get('is_deprecated', False), - 'is_exception': l.get('is_exception', False), - } for l in license_index if l.get('spdx_license_key') + "key": l.get("spdx_license_key", ""), + "aliases": l.get("other_spdx_license_keys", []), + "is_deprecated": l.get("is_deprecated", False), + "is_exception": l.get("is_exception", False), + } + for l in license_index + if l.get("spdx_license_key") ] return load_licensing_from_license_index(lics) @@ -909,7 +902,7 @@ def build_token_with_symbol(): trailing_spaces.append(unmatched.pop()) if unmatched: - string = ' '.join(t.string for t in unmatched if t.string.strip()) + string = " ".join(t.string for t in unmatched if t.string.strip()) start = unmatched[0].start end = unmatched[-1].end toksym = LicenseSymbol(string) @@ -988,7 +981,8 @@ def is_with_subexpression(tokens_tripple): expression. """ lic, wit, exc = tokens_tripple - return (isinstance(lic.value, LicenseSymbol) + return ( + isinstance(lic.value, LicenseSymbol) and wit.value == KW_WITH and isinstance(exc.value, LicenseSymbol) ) @@ -1042,15 +1036,14 @@ def replace_with_subexpression_by_license_symbol(tokens, strict=False): else: # this should not be possible by design - raise Exception( - f'Licensing.tokenize is internally confused...: {tval!r}') + raise Exception(f"Licensing.tokenize is internally confused...: {tval!r}") yield token continue if len_group != 3: # this should never happen - string = ' '.join([tok.string for tok in token_group]) + string = " ".join([tok.string for tok in token_group]) start = token_group[0].start raise ParseError( token_type=TOKEN_SYMBOL, @@ -1061,12 +1054,12 @@ def replace_with_subexpression_by_license_symbol(tokens, strict=False): # from now on we have a tripple of tokens: a WITH sub-expression such as # "A with B" seq of three tokens - lic_token, WITH , exc_token = token_group + lic_token, WITH, exc_token = token_group lic = lic_token.string exc = exc_token.string WITH = WITH.string.strip() - token_string = f'{lic} {WITH} {exc}' + token_string = f"{lic} {WITH} {exc}" # the left hand side license symbol lic_sym = lic_token.value @@ -1127,7 +1120,7 @@ class Renderable(object): An interface for renderable objects. """ - def render(self, template='{symbol.key}', *args, **kwargs): + def render(self, template="{symbol.key}", *args, **kwargs): """ Return a formatted string rendering for this expression using the ``template`` format string to render each license symbol. The variables @@ -1143,7 +1136,7 @@ def render(self, template='{symbol.key}', *args, **kwargs): """ return NotImplementedError - def render_as_readable(self, template='{symbol.key}', *args, **kwargs): + def render_as_readable(self, template="{symbol.key}", *args, **kwargs): """ Return a formatted string rendering for this expression using the ``template`` format string to render each symbol. Add extra parenthesis @@ -1151,19 +1144,9 @@ def render_as_readable(self, template='{symbol.key}', *args, **kwargs): readbility. See ``render()`` for other arguments. """ if isinstance(self, LicenseWithExceptionSymbol): - return self.render( - template=template, - wrap_with_in_parens=False, - *args, - **kwargs - ) + return self.render(template=template, wrap_with_in_parens=False, *args, **kwargs) - return self.render( - template=template, - wrap_with_in_parens=True, - *args, - **kwargs - ) + return self.render(template=template, wrap_with_in_parens=True, *args, **kwargs) class BaseSymbol(Renderable, boolean.Symbol): @@ -1191,7 +1174,7 @@ def __contains__(self, other): # validate license keys -is_valid_license_key = re.compile(r'^[-:\w\s\.\+]+$', re.UNICODE).match +is_valid_license_key = re.compile(r"^[-:\w\s\.\+]+$", re.UNICODE).match # TODO: we need to implement comparison by hand instead @@ -1202,48 +1185,54 @@ class LicenseSymbol(BaseSymbol): expression. """ - def __init__(self, key, aliases=tuple(), is_deprecated=False, is_exception=False, *args, **kwargs): + def __init__( + self, key, aliases=tuple(), is_deprecated=False, is_exception=False, *args, **kwargs + ): if not key: - raise ExpressionError(f'A license key cannot be empty: {key!r}') + raise ExpressionError(f"A license key cannot be empty: {key!r}") if not isinstance(key, str): if isinstance(key, bytes): try: key = str(key) except: - raise ExpressionError( - f'A license key must be a string: {key!r}') + raise ExpressionError(f"A license key must be a string: {key!r}") else: - raise ExpressionError( - f'A license key must be a string: {key!r}') + raise ExpressionError(f"A license key must be a string: {key!r}") key = key.strip() if not key: - raise ExpressionError(f'A license key cannot be blank: {key!r}') + raise ExpressionError(f"A license key cannot be blank: {key!r}") # note: key can contain spaces if not is_valid_license_key(key): raise ExpressionError( - 'Invalid license key: the valid characters are: letters and ' - 'numbers, underscore, dot, colon or hyphen signs and ' - f'spaces: {key!r}' + "Invalid license key: the valid characters are: letters and " + "numbers, underscore, dot, colon or hyphen signs and " + f"spaces: {key!r}" ) # normalize spaces - key = ' '.join(key.split()) + key = " ".join(key.split()) if key.lower() in KEYWORDS_STRINGS: raise ExpressionError( 'Invalid license key: a key cannot be a reserved keyword: "or",' - f' "and" or "with": {key!r}') + f' "and" or "with": {key!r}' + ) self.key = key - if aliases and not isinstance(aliases, (list, tuple,)): + if aliases and not isinstance( + aliases, + ( + list, + tuple, + ), + ): raise TypeError( - f'License aliases: {aliases!r} must be a sequence ' - f'and not: {type(aliases)}.' + f"License aliases: {aliases!r} must be a sequence and not: {type(aliases)}." ) self.aliases = aliases and tuple(aliases) or tuple() self.is_deprecated = is_deprecated @@ -1277,10 +1266,7 @@ def __ne__(self, other): if not (isinstance(other, self.__class__) or self.symbol_like(other)): return True - return ( - self.key != other.key - or self.is_exception != other.is_exception - ) + return self.key != other.key or self.is_exception != other.is_exception def __lt__(self, other): if isinstance( @@ -1293,7 +1279,7 @@ def __lt__(self, other): __nonzero__ = __bool__ = lambda s: True - def render(self, template='{symbol.key}', *args, **kwargs): + def render(self, template="{symbol.key}", *args, **kwargs): return template.format(symbol=self) def __str__(self): @@ -1305,9 +1291,9 @@ def __len__(self): def __repr__(self): cls = self.__class__.__name__ key = self.key - aliases = self.aliases and f'aliases={self.aliases!r}, ' or '' + aliases = self.aliases and f"aliases={self.aliases!r}, " or "" is_exception = self.is_exception - return f'{cls}({key!r}, {aliases}is_exception={is_exception!r})' + return f"{cls}({key!r}, {aliases}is_exception={is_exception!r})" def __copy__(self): return LicenseSymbol( @@ -1322,7 +1308,7 @@ def symbol_like(cls, symbol): Return True if ``symbol`` is a symbol-like object with its essential attributes. """ - return hasattr(symbol, 'key') and hasattr(symbol, 'is_exception') + return hasattr(symbol, "key") and hasattr(symbol, "is_exception") # TODO: we need to implement comparison by hand instead @@ -1335,29 +1321,25 @@ class LicenseSymbolLike(LicenseSymbol): def __init__(self, symbol_like, *args, **kwargs): if not self.symbol_like(symbol_like): - raise ExpressionError(f'Not a symbol-like object: {symbol_like!r}') + raise ExpressionError(f"Not a symbol-like object: {symbol_like!r}") self.wrapped = symbol_like - super(LicenseSymbolLike, self).__init__( - key=self.wrapped.key, - *args, - **kwargs - ) + super(LicenseSymbolLike, self).__init__(key=self.wrapped.key, *args, **kwargs) self.is_exception = self.wrapped.is_exception - self.aliases = getattr(self.wrapped, 'aliases', tuple()) + self.aliases = getattr(self.wrapped, "aliases", tuple()) # can we delegate rendering to a render method of the wrapped object? # we can if we have a .render() callable on the wrapped object. self._render = None - renderer = getattr(symbol_like, 'render', None) + renderer = getattr(symbol_like, "render", None) if callable(renderer): self._render = renderer def __copy__(self): return LicenseSymbolLike(symbol_like=self.wrapped) - def render(self, template='{symbol.key}', *args, **kwargs): + def render(self, template="{symbol.key}", *args, **kwargs): if self._render: return self._render(template, *args, **kwargs) @@ -1380,11 +1362,10 @@ def __ne__(self, other): return False if not (isinstance(other, self.__class__) or self.symbol_like(other)): return True - return (self.key != other.key or self.is_exception != other.is_exception) + return self.key != other.key or self.is_exception != other.is_exception def __lt__(self, other): - if isinstance( - other, (LicenseSymbol, LicenseWithExceptionSymbol, LicenseSymbolLike)): + if isinstance(other, (LicenseSymbol, LicenseWithExceptionSymbol, LicenseSymbolLike)): return str(self) < str(other) else: return NotImplemented @@ -1402,14 +1383,7 @@ class LicenseWithExceptionSymbol(BaseSymbol): resolution, validation and representation. """ - def __init__( - self, - license_symbol, - exception_symbol, - strict=False, - *args, - **kwargs - ): + def __init__(self, license_symbol, exception_symbol, strict=False, *args, **kwargs): """ Initialize a new LicenseWithExceptionSymbol from a ``license_symbol`` and a ``exception_symbol`` symbol-like objects. @@ -1420,26 +1394,24 @@ def __init__( """ if not LicenseSymbol.symbol_like(license_symbol): raise ExpressionError( - 'license_symbol must be a LicenseSymbol-like object: ' - f'{license_symbol!r}', + f"license_symbol must be a LicenseSymbol-like object: {license_symbol!r}", ) if strict and license_symbol.is_exception: raise ExpressionError( 'license_symbol cannot be an exception with the "is_exception" ' - f'attribute set to True:{license_symbol!r}', + f"attribute set to True:{license_symbol!r}", ) if not LicenseSymbol.symbol_like(exception_symbol): raise ExpressionError( - 'exception_symbol must be a LicenseSymbol-like object: ' - f'{exception_symbol!r}', + f"exception_symbol must be a LicenseSymbol-like object: {exception_symbol!r}", ) if strict and not exception_symbol.is_exception: raise ExpressionError( 'exception_symbol must be an exception with "is_exception" ' - f'set to True: {exception_symbol!r}', + f"set to True: {exception_symbol!r}", ) self.license_symbol = license_symbol @@ -1457,26 +1429,25 @@ def decompose(self): yield self.license_symbol yield self.exception_symbol - def render( - self, - template='{symbol.key}', - wrap_with_in_parens=False, - *args, - **kwargs - ): + def render(self, template="{symbol.key}", wrap_with_in_parens=False, *args, **kwargs): """ Return a formatted "WITH" expression. If ``wrap_with_in_parens``, wrap the expression in parens as in "(XXX WITH YYY)". """ lic = self.license_symbol.render(template, *args, **kwargs) exc = self.exception_symbol.render(template, *args, **kwargs) - rend = f'{lic} WITH {exc}' + rend = f"{lic} WITH {exc}" if wrap_with_in_parens: - rend = f'({rend})' + rend = f"({rend})" return rend def __hash__(self, *args, **kwargs): - return hash((self.license_symbol, self.exception_symbol,)) + return hash( + ( + self.license_symbol, + self.exception_symbol, + ) + ) def __eq__(self, other): if self is other: @@ -1497,18 +1468,13 @@ def __ne__(self, other): if not isinstance(other, self.__class__): return True - return ( - not ( - self.license_symbol == other.license_symbol - and self.exception_symbol == other.exception_symbol - ) + return not ( + self.license_symbol == other.license_symbol + and self.exception_symbol == other.exception_symbol ) def __lt__(self, other): - if isinstance( - other, - (LicenseSymbol, LicenseWithExceptionSymbol, LicenseSymbolLike) - ): + if isinstance(other, (LicenseSymbol, LicenseWithExceptionSymbol, LicenseSymbolLike)): return str(self) < str(other) else: return NotImplemented @@ -1516,23 +1482,23 @@ def __lt__(self, other): __nonzero__ = __bool__ = lambda s: True def __str__(self): - return f'{self.license_symbol.key} WITH {self.exception_symbol.key}' + return f"{self.license_symbol.key} WITH {self.exception_symbol.key}" def __repr__(self): cls = self.__class__.__name__ data = dict(cls=self.__class__.__name__) data.update(self.__dict__) return ( - f'{cls}(' - f'license_symbol={self.license_symbol!r}, ' - f'exception_symbol={self.exception_symbol!r})' + f"{cls}(" + f"license_symbol={self.license_symbol!r}, " + f"exception_symbol={self.exception_symbol!r})" ) class RenderableFunction(Renderable): # derived from the __str__ code in boolean.py - def render(self, template='{symbol.key}', *args, **kwargs): + def render(self, template="{symbol.key}", *args, **kwargs): """ Render an expression as a string, recursively applying the string ``template`` to every symbols and operators. @@ -1547,16 +1513,15 @@ def render(self, template='{symbol.key}', *args, **kwargs): else: # FIXME: CAN THIS EVER HAPPEN since we only have symbols OR and AND? print( - 'WARNING: symbol is not renderable: using plain string ' - f'representation: {sym!r}' + f"WARNING: symbol is not renderable: using plain string representation: {sym!r}" ) sym = str(sym) # NB: the operator str already has a leading and trailing space if self.isliteral: - rendered = f'{self.operator}{sym}' + rendered = f"{self.operator}{sym}" else: - rendered = f'{self.operator}({sym})' + rendered = f"{self.operator}({sym})" return rendered rendered_items = [] @@ -1569,15 +1534,15 @@ def render(self, template='{symbol.key}', *args, **kwargs): else: # FIXME: CAN THIS EVER HAPPEN since we only have symbols OR and AND? print( - 'WARNING: object in expression is not renderable: ' - f'falling back to plain string representation: {arg!r}.' + "WARNING: object in expression is not renderable: " + f"falling back to plain string representation: {arg!r}." ) rendered = str(arg) if arg.isliteral: rendered_items_append(rendered) else: - rendered_items_append(f'({rendered})') + rendered_items_append(f"({rendered})") return self.operator.join(rendered_items) @@ -1589,10 +1554,9 @@ class AND(RenderableFunction, boolean.AND): def __init__(self, *args): if len(args) < 2: - raise ExpressionError( - 'AND requires two or more licenses as in: MIT AND BSD') + raise ExpressionError("AND requires two or more licenses as in: MIT AND BSD") super(AND, self).__init__(*args) - self.operator = ' AND ' + self.operator = " AND " class OR(RenderableFunction, boolean.OR): @@ -1602,10 +1566,9 @@ class OR(RenderableFunction, boolean.OR): def __init__(self, *args): if len(args) < 2: - raise ExpressionError( - 'OR requires two or more licenses as in: MIT OR BSD') + raise ExpressionError("OR requires two or more licenses as in: MIT OR BSD") super(OR, self).__init__(*args) - self.operator = ' OR ' + self.operator = " OR " def ordered_unique(seq): @@ -1640,7 +1603,7 @@ def as_symbols(symbols): try: symbol = str(symbol) except: - raise TypeError(f'{symbol!r} is not a string.') + raise TypeError(f"{symbol!r} is not a string.") if isinstance(symbol, str): if symbol.strip(): @@ -1653,9 +1616,7 @@ def as_symbols(symbols): yield LicenseSymbolLike(symbol) else: - raise TypeError( - f'{symbol!r} is neither a string nor LicenseSymbol-like.' - ) + raise TypeError(f"{symbol!r} is neither a string nor LicenseSymbol-like.") def validate_symbols(symbols, validate_keys=False): @@ -1710,13 +1671,11 @@ def validate_symbols(symbols, validate_keys=False): seen_keys.add(keyl) # aliases is an optional attribute - aliases = getattr(symbol, 'aliases', []) + aliases = getattr(symbol, "aliases", []) initial_alias_len = len(aliases) # always normalize aliases for spaces and case - aliases = set([ - ' '.join(alias.lower().strip().split()) for alias in aliases - ]) + aliases = set([" ".join(alias.lower().strip().split()) for alias in aliases]) # KEEP UNIQUES, remove empties aliases = set(a for a in aliases if a) @@ -1752,46 +1711,41 @@ def validate_symbols(symbols, validate_keys=False): # build warning and error messages from invalid data errors = [] for ind in sorted(not_symbol_classes): - errors.append(f'Invalid item: not a LicenseSymbol object: {ind!r}.') + errors.append(f"Invalid item: not a LicenseSymbol object: {ind!r}.") for dupe in sorted(dupe_keys): - errors.append(f'Invalid duplicated license key: {dupe!r}.') + errors.append(f"Invalid duplicated license key: {dupe!r}.") for dalias, dkeys in sorted(dupe_aliases.items()): - dkeys = ', '.join(dkeys) + dkeys = ", ".join(dkeys) errors.append( - f'Invalid duplicated alias pointing to multiple keys: ' - f'{dalias} point to keys: {dkeys!r}.' + f"Invalid duplicated alias pointing to multiple keys: " + f"{dalias} point to keys: {dkeys!r}." ) for ikey, ialiases in sorted(invalid_alias_as_kw.items()): - ialiases = ', '.join(ialiases) + ialiases = ", ".join(ialiases) errors.append( - f'Invalid aliases: an alias cannot be an expression keyword. ' - f'key: {ikey!r}, aliases: {ialiases}.' + f"Invalid aliases: an alias cannot be an expression keyword. " + f"key: {ikey!r}, aliases: {ialiases}." ) for dupe in sorted(dupe_exceptions): - errors.append(f'Invalid duplicated license exception key: {dupe}.') + errors.append(f"Invalid duplicated license exception key: {dupe}.") for ikw in sorted(invalid_keys_as_kw): - errors.append( - f'Invalid key: a key cannot be an expression keyword: {ikw}.' - ) + errors.append(f"Invalid key: a key cannot be an expression keyword: {ikw}.") warnings = [] for dupe_alias in sorted(dupe_aliases): - errors.append( - f'Duplicated or empty aliases ignored for license key: ' - f'{dupe_alias!r}.' - ) + errors.append(f"Duplicated or empty aliases ignored for license key: {dupe_alias!r}.") return warnings, errors def combine_expressions( expressions, - relation='AND', + relation="AND", unique=True, licensing=Licensing(), ): @@ -1825,13 +1779,13 @@ def combine_expressions( return if not isinstance(expressions, (list, tuple)): - raise TypeError( - f'expressions should be a list or tuple and not: {type(expressions)}' - ) - - if not relation or relation.upper() not in ('AND', 'OR',): - raise TypeError(f'relation should be one of AND, OR and not: {relation}') + raise TypeError(f"expressions should be a list or tuple and not: {type(expressions)}") + if not relation or relation.upper() not in ( + "AND", + "OR", + ): + raise TypeError(f"relation should be one of AND, OR and not: {relation}") # only deal with LicenseExpression objects expressions = [licensing.parse(le, simple=True) for le in expressions] @@ -1844,5 +1798,5 @@ def combine_expressions( if len(expressions) == 1: return expressions[0] - relation = {'AND': licensing.AND, 'OR': licensing.OR}[relation] + relation = {"AND": licensing.AND, "OR": licensing.OR}[relation] return relation(*expressions) diff --git a/src/license_expression/_pyahocorasick.py b/src/license_expression/_pyahocorasick.py index 9810fe2..2a1f5bb 100644 --- a/src/license_expression/_pyahocorasick.py +++ b/src/license_expression/_pyahocorasick.py @@ -19,6 +19,7 @@ - improve returned results with the actual start,end and matched string. - support returning non-matched parts of a string """ + from collections import deque from collections import OrderedDict import logging @@ -36,9 +37,10 @@ def logger_debug(*args): if TRACE: def logger_debug(*args): - return logger.debug(' '.join(isinstance(a, str) and a or repr(a) for a in args)) + return logger.debug(" ".join(isinstance(a, str) and a or repr(a) for a in args)) import sys + logging.basicConfig(stream=sys.stdout) logger.setLevel(logging.DEBUG) @@ -50,7 +52,8 @@ class TrieNode(object): """ Node of the Trie/Aho-Corasick automaton. """ - __slots__ = ['token', 'output', 'fail', 'children'] + + __slots__ = ["token", "output", "fail", "children"] def __init__(self, token, output=nil): # token of a tokens string added to the Trie as a string @@ -70,9 +73,9 @@ def __init__(self, token, output=nil): def __repr__(self): if self.output is not nil: - return 'TrieNode(%r, %r)' % (self.token, self.output) + return "TrieNode(%r, %r)" % (self.token, self.output) else: - return 'TrieNode(%r)' % self.token + return "TrieNode(%r)" % self.token class Trie(object): @@ -85,7 +88,7 @@ def __init__(self): """ Initialize a new Trie. """ - self.root = TrieNode('') + self.root = TrieNode("") # set of any unique tokens in the trie, updated on each addition we keep # track of the set of tokens added to the trie to build the automaton @@ -106,8 +109,9 @@ def add(self, tokens_string, value=None): to the Trie. """ if self._converted: - raise Exception('This Trie has been converted to an Aho-Corasick ' - 'automaton and cannot be modified.') + raise Exception( + "This Trie has been converted to an Aho-Corasick automaton and cannot be modified." + ) if not tokens_string or not isinstance(tokens_string, str): return @@ -193,7 +197,12 @@ def walk(node, tokens): """ tokens = [t for t in tokens + [node.token] if t] if node.output is not nil: - items.append((node.output[0], node.output[1],)) + items.append( + ( + node.output[0], + node.output[1], + ) + ) for child in node.children.values(): if child is not node: @@ -296,37 +305,37 @@ def iter(self, tokens_string, include_unmatched=False, include_space=False): state = self.root if TRACE: - logger_debug('Trie.iter() with:', repr(tokens_string)) - logger_debug(' tokens:', tokens) + logger_debug("Trie.iter() with:", repr(tokens_string)) + logger_debug(" tokens:", tokens) end_pos = -1 for token_string in tokens: end_pos += len(token_string) if TRACE: logger_debug() - logger_debug('token_string', repr(token_string)) - logger_debug(' end_pos', end_pos) + logger_debug("token_string", repr(token_string)) + logger_debug(" end_pos", end_pos) if not include_space and not token_string.strip(): if TRACE: - logger_debug(' include_space skipped') + logger_debug(" include_space skipped") continue if token_string not in self._known_tokens: state = self.root if TRACE: - logger_debug(' unmatched') + logger_debug(" unmatched") if include_unmatched: n = len(token_string) start_pos = end_pos - n + 1 tok = Token( start=start_pos, end=end_pos, - string=tokens_string[start_pos: end_pos + 1], - value=None + string=tokens_string[start_pos : end_pos + 1], + value=None, ) if TRACE: - logger_debug(' unmatched tok:', tok) + logger_debug(" unmatched tok:", tok) yield tok continue @@ -343,31 +352,31 @@ def iter(self, tokens_string, include_unmatched=False, include_space=False): if match.output is not nil: matched_string, output_value = match.output if TRACE: - logger_debug(' type output', repr( - output_value), type(matched_string)) + logger_debug(" type output", repr(output_value), type(matched_string)) n = len(matched_string) start_pos = end_pos - n + 1 if TRACE: - logger_debug(' start_pos', start_pos) - yield Token(start_pos, end_pos, tokens_string[start_pos: end_pos + 1], output_value) + logger_debug(" start_pos", start_pos) + yield Token( + start_pos, end_pos, tokens_string[start_pos : end_pos + 1], output_value + ) yielded = True match = match.fail if not yielded and include_unmatched: if TRACE: - logger_debug(' unmatched but known token') + logger_debug(" unmatched but known token") n = len(token_string) start_pos = end_pos - n + 1 - tok = Token(start_pos, end_pos, - tokens_string[start_pos: end_pos + 1], None) + tok = Token(start_pos, end_pos, tokens_string[start_pos : end_pos + 1], None) if TRACE: - logger_debug(' unmatched tok 2:', tok) + logger_debug(" unmatched tok 2:", tok) yield tok logger_debug() def tokenize(self, string, include_unmatched=True, include_space=False): """ - tokenize a string for matched and unmatched sub-sequences and yield non- + Tokenize a string for matched and unmatched sub-sequences and yield non- overlapping Token objects performing a modified Aho-Corasick search procedure: @@ -408,11 +417,10 @@ def tokenize(self, string, include_unmatched=True, include_space=False): >>> tokens == expected True """ - tokens = self.iter(string, - include_unmatched=include_unmatched, include_space=include_space) + tokens = self.iter(string, include_unmatched=include_unmatched, include_space=include_space) tokens = list(tokens) if TRACE: - logger_debug('tokenize.tokens:', tokens) + logger_debug("tokenize.tokens:", tokens) if not include_space: tokens = [t for t in tokens if t.string.strip()] tokens = filter_overlapping(tokens) @@ -462,15 +470,15 @@ def filter_overlapping(tokens): curr_tok = tokens[i] next_tok = tokens[j] - logger_debug('curr_tok, i, next_tok, j:', curr_tok, i, next_tok, j) + logger_debug("curr_tok, i, next_tok, j:", curr_tok, i, next_tok, j) # disjoint tokens: break, there is nothing to do if next_tok.is_after(curr_tok): - logger_debug(' break to next', curr_tok) + logger_debug(" break to next", curr_tok) break # contained token: discard the contained token if next_tok in curr_tok: - logger_debug(' del next_tok contained:', next_tok) + logger_debug(" del next_tok contained:", next_tok) del tokens[j] continue @@ -478,11 +486,11 @@ def filter_overlapping(tokens): # tokens. In case of length tie: keep the left most if curr_tok.overlap(next_tok): if len(curr_tok) >= len(next_tok): - logger_debug(' del next_tok smaller overlap:', next_tok) + logger_debug(" del next_tok smaller overlap:", next_tok) del tokens[j] continue else: - logger_debug(' del curr_tok smaller overlap:', curr_tok) + logger_debug(" del curr_tok smaller overlap:", curr_tok) del tokens[i] break j += 1 @@ -504,16 +512,23 @@ class Token(object): - None if this is a space. """ - __slots__ = 'start', 'end', 'string', 'value', + __slots__ = ( + "start", + "end", + "string", + "value", + ) - def __init__(self, start, end, string='', value=None): + def __init__(self, start, end, string="", value=None): self.start = start self.end = end self.string = string self.value = value def __repr__(self): - return self.__class__.__name__ + '(%(start)r, %(end)r, %(string)r, %(value)r)' % self.as_dict() + return ( + self.__class__.__name__ + "(%(start)r, %(end)r, %(string)r, %(value)r)" % self.as_dict() + ) def as_dict(self): return OrderedDict([(s, getattr(self, s)) for s in self.__slots__]) @@ -523,10 +538,10 @@ def __len__(self): def __eq__(self, other): return isinstance(other, Token) and ( - self.start == other.start and - self.end == other.end and - self.string == other.string and - self.value == other.value + self.start == other.start + and self.end == other.end + and self.string == other.string + and self.value == other.value ) def __hash__(self): @@ -547,7 +562,13 @@ def sort(cls, tokens): >>> expected == Token.sort(tokens) True """ - def key(s): return (s.start, -len(s),) + + def key(s): + return ( + s.start, + -len(s), + ) + return sorted(tokens, key=key) def is_after(self, other): @@ -609,15 +630,16 @@ def overlap(self, other): # tokenize to separate text from parens -_tokenizer = re.compile(r''' +_tokenizer = re.compile( + r""" (?P[^\s\(\)]+) | (?P\s+) | (?P[\(\)]) - ''', - re.VERBOSE | re.MULTILINE | re.UNICODE - ) + """, + re.VERBOSE | re.MULTILINE | re.UNICODE, +) def get_tokens(tokens_string): diff --git a/tests/test__pyahocorasick.py b/tests/test__pyahocorasick.py index cccf43b..2cc4213 100644 --- a/tests/test__pyahocorasick.py +++ b/tests/test__pyahocorasick.py @@ -20,105 +20,104 @@ class TestTrie(unittest.TestCase): - def test_add_can_get(self): t = Trie() - t.add('python', 'value') - assert ('python', 'value') == t.get('python') + t.add("python", "value") + assert ("python", "value") == t.get("python") def test_add_existing_WordShouldReplaceAssociatedValue(self): t = Trie() - t.add('python', 'value') - assert ('python', 'value') == t.get('python') + t.add("python", "value") + assert ("python", "value") == t.get("python") - t.add('python', 'other') - assert ('python', 'other') == t.get('python') + t.add("python", "other") + assert ("python", "other") == t.get("python") def test_get_UnknowWordWithoutDefaultValueShouldRaiseException(self): t = Trie() with self.assertRaises(KeyError): - t.get('python') + t.get("python") def test_get_UnknowWordWithDefaultValueShouldReturnDefault(self): t = Trie() - self.assertEqual(t.get('python', 'default'), 'default') + self.assertEqual(t.get("python", "default"), "default") def test_exists_ShouldDetectAddedWords(self): t = Trie() - t.add('python', 'value') - t.add('ada', 'value') + t.add("python", "value") + t.add("ada", "value") - self.assertTrue(t.exists('python')) - self.assertTrue(t.exists('ada')) + self.assertTrue(t.exists("python")) + self.assertTrue(t.exists("ada")) def test_exists_ShouldReturnFailOnUnknownWord(self): t = Trie() - t.add('python', 'value') + t.add("python", "value") - self.assertFalse(t.exists('ada')) + self.assertFalse(t.exists("ada")) def test_is_prefix_ShouldDetecAllPrefixesIncludingWord(self): t = Trie() - t.add('python', 'value') - t.add('ada lovelace', 'value') + t.add("python", "value") + t.add("ada lovelace", "value") - self.assertFalse(t.is_prefix('a')) - self.assertFalse(t.is_prefix('ad')) - self.assertTrue(t.is_prefix('ada')) + self.assertFalse(t.is_prefix("a")) + self.assertFalse(t.is_prefix("ad")) + self.assertTrue(t.is_prefix("ada")) - self.assertFalse(t.is_prefix('p')) - self.assertFalse(t.is_prefix('py')) - self.assertFalse(t.is_prefix('pyt')) - self.assertFalse(t.is_prefix('pyth')) - self.assertFalse(t.is_prefix('pytho')) - self.assertTrue(t.is_prefix('python')) + self.assertFalse(t.is_prefix("p")) + self.assertFalse(t.is_prefix("py")) + self.assertFalse(t.is_prefix("pyt")) + self.assertFalse(t.is_prefix("pyth")) + self.assertFalse(t.is_prefix("pytho")) + self.assertTrue(t.is_prefix("python")) - self.assertFalse(t.is_prefix('lovelace')) + self.assertFalse(t.is_prefix("lovelace")) def test_items_ShouldReturnAllItemsAlreadyAddedToTheTrie(self): t = Trie() - t.add('python', 1) - t.add('ada', 2) - t.add('perl', 3) - t.add('pascal', 4) - t.add('php', 5) - t.add('php that', 6) + t.add("python", 1) + t.add("ada", 2) + t.add("perl", 3) + t.add("pascal", 4) + t.add("php", 5) + t.add("php that", 6) result = list(t.items()) - self.assertIn(('python', 1), result) - self.assertIn(('ada', 2), result) - self.assertIn(('perl', 3), result) - self.assertIn(('pascal', 4), result) - self.assertIn(('php', 5), result) - self.assertIn(('php that', 6), result) + self.assertIn(("python", 1), result) + self.assertIn(("ada", 2), result) + self.assertIn(("perl", 3), result) + self.assertIn(("pascal", 4), result) + self.assertIn(("php", 5), result) + self.assertIn(("php that", 6), result) def test_keys_ShouldReturnAllKeysAlreadyAddedToTheTrie(self): t = Trie() - t.add('python', 1) - t.add('ada', 2) - t.add('perl', 3) - t.add('pascal', 4) - t.add('php', 5) - t.add('php that', 6) + t.add("python", 1) + t.add("ada", 2) + t.add("perl", 3) + t.add("pascal", 4) + t.add("php", 5) + t.add("php that", 6) result = list(t.keys()) - self.assertIn('python', result) - self.assertIn('ada', result) - self.assertIn('perl', result) - self.assertIn('pascal', result) - self.assertIn('php', result) - self.assertIn('php that', result) + self.assertIn("python", result) + self.assertIn("ada", result) + self.assertIn("perl", result) + self.assertIn("pascal", result) + self.assertIn("php", result) + self.assertIn("php that", result) def test_values_ShouldReturnAllValuesAlreadyAddedToTheTrie(self): t = Trie() - t.add('python', 1) - t.add('ada', 2) - t.add('perl', 3) - t.add('pascal', 4) - t.add('php', 5) + t.add("python", 1) + t.add("ada", 2) + t.add("perl", 3) + t.add("pascal", 4) + t.add("php", 5) result = list(t.values()) self.assertIn(1, result) @@ -128,199 +127,191 @@ def test_values_ShouldReturnAllValuesAlreadyAddedToTheTrie(self): self.assertIn(5, result) def test_iter_should_not_return_non_matches_by_default(self): - def get_test_automaton(): - words = 'he her hers his she hi him man himan'.split() + words = "he her hers his she hi him man himan".split() t = Trie() for w in words: t.add(w, w) t.make_automaton() return t - test_string = 'he she himan' + test_string = "he she himan" t = get_test_automaton() result = list(t.iter(test_string)) - assert 'he she himan'.split() == [r.value for r in result] + assert "he she himan".split() == [r.value for r in result] def test_iter_should_can_return_non_matches_optionally(self): - def get_test_automaton(): - words = 'he her hers his she hi him man himan'.split() + words = "he her hers his she hi him man himan".split() t = Trie() for w in words: t.add(w, w) t.make_automaton() return t - test_string = ' he she junk himan other stuffs ' + test_string = " he she junk himan other stuffs " # 111111111122222222223333333 # 0123456789012345678901234567890123456 t = get_test_automaton() - result = list( - t.iter(test_string, include_unmatched=True, include_space=True)) + result = list(t.iter(test_string, include_unmatched=True, include_space=True)) expected = [ - Token(0, 1, u' ', None), - Token(2, 3, u'he', u'he'), - Token(4, 4, u' ', None), - Token(5, 7, u'she', u'she'), - Token(8, 8, u' ', None), - Token(9, 12, u'junk', None), - Token(13, 14, u' ', None), - Token(15, 19, u'himan', u'himan'), - Token(20, 21, u' ', None), - Token(22, 26, u'other', None), - Token(27, 27, u' ', None), - Token(28, 33, u'stuffs', None), - Token(34, 36, u' ', None), + Token(0, 1, " ", None), + Token(2, 3, "he", "he"), + Token(4, 4, " ", None), + Token(5, 7, "she", "she"), + Token(8, 8, " ", None), + Token(9, 12, "junk", None), + Token(13, 14, " ", None), + Token(15, 19, "himan", "himan"), + Token(20, 21, " ", None), + Token(22, 26, "other", None), + Token(27, 27, " ", None), + Token(28, 33, "stuffs", None), + Token(34, 36, " ", None), ] assert expected == result def test_iter_vs_tokenize(self): - def get_test_automaton(): - words = '( AND ) OR'.split() + words = "( AND ) OR".split() t = Trie() for w in words: t.add(w, w) t.make_automaton() return t - test_string = '((l-a + AND l-b) OR (l -c+))' + test_string = "((l-a + AND l-b) OR (l -c+))" t = get_test_automaton() - result = list( - t.iter(test_string, include_unmatched=True, include_space=True)) + result = list(t.iter(test_string, include_unmatched=True, include_space=True)) expected = [ - Token(0, 0, u'(', u'('), - Token(1, 1, u'(', u'('), - Token(2, 4, u'l-a', None), - Token(5, 5, u' ', None), - Token(6, 6, u'+', None), - Token(7, 7, u' ', None), - Token(8, 10, u'AND', u'AND'), - Token(11, 11, u' ', None), - Token(12, 14, u'l-b', None), - Token(15, 15, u')', u')'), - Token(16, 16, u' ', None), - Token(17, 18, u'OR', u'OR'), - Token(19, 19, u' ', None), - Token(20, 20, u'(', u'('), - Token(21, 21, u'l', None), - Token(22, 22, u' ', None), - Token(23, 25, u'-c+', None), - Token(26, 26, u')', u')'), - Token(27, 27, u')', u')') + Token(0, 0, "(", "("), + Token(1, 1, "(", "("), + Token(2, 4, "l-a", None), + Token(5, 5, " ", None), + Token(6, 6, "+", None), + Token(7, 7, " ", None), + Token(8, 10, "AND", "AND"), + Token(11, 11, " ", None), + Token(12, 14, "l-b", None), + Token(15, 15, ")", ")"), + Token(16, 16, " ", None), + Token(17, 18, "OR", "OR"), + Token(19, 19, " ", None), + Token(20, 20, "(", "("), + Token(21, 21, "l", None), + Token(22, 22, " ", None), + Token(23, 25, "-c+", None), + Token(26, 26, ")", ")"), + Token(27, 27, ")", ")"), ] assert expected == result - result = list(t.tokenize( - test_string, include_unmatched=True, include_space=True)) + result = list(t.tokenize(test_string, include_unmatched=True, include_space=True)) assert expected == result def test_tokenize_with_unmatched_and_space(self): - def get_test_automaton(): - words = '( AND ) OR'.split() + words = "( AND ) OR".split() t = Trie() for w in words: t.add(w, w) t.make_automaton() return t - test_string = '((l-a + AND l-b) OR an (l -c+))' + test_string = "((l-a + AND l-b) OR an (l -c+))" # 111111111122222222223 # 0123456789012345678901234567890 t = get_test_automaton() - result = list(t.tokenize( - test_string, include_unmatched=True, include_space=True)) + result = list(t.tokenize(test_string, include_unmatched=True, include_space=True)) expected = [ - Token(0, 0, u'(', u'('), - Token(1, 1, u'(', u'('), - Token(2, 4, u'l-a', None), - Token(5, 5, u' ', None), - Token(6, 6, u'+', None), - Token(7, 7, u' ', None), - Token(8, 10, u'AND', u'AND'), - Token(11, 11, u' ', None), - Token(12, 14, u'l-b', None), - Token(15, 15, u')', u')'), - Token(16, 16, u' ', None), - Token(17, 18, u'OR', u'OR'), - Token(19, 19, u' ', None), - Token(20, 21, u'an', None), - Token(22, 22, u' ', None), - Token(23, 23, u'(', u'('), - Token(24, 24, u'l', None), - Token(25, 25, u' ', None), - Token(26, 28, u'-c+', None), - Token(29, 29, u')', u')'), - Token(30, 30, u')', u')') + Token(0, 0, "(", "("), + Token(1, 1, "(", "("), + Token(2, 4, "l-a", None), + Token(5, 5, " ", None), + Token(6, 6, "+", None), + Token(7, 7, " ", None), + Token(8, 10, "AND", "AND"), + Token(11, 11, " ", None), + Token(12, 14, "l-b", None), + Token(15, 15, ")", ")"), + Token(16, 16, " ", None), + Token(17, 18, "OR", "OR"), + Token(19, 19, " ", None), + Token(20, 21, "an", None), + Token(22, 22, " ", None), + Token(23, 23, "(", "("), + Token(24, 24, "l", None), + Token(25, 25, " ", None), + Token(26, 28, "-c+", None), + Token(29, 29, ")", ")"), + Token(30, 30, ")", ")"), ] assert expected == result - assert test_string == ''.join(t.string for t in result) + assert test_string == "".join(t.string for t in result) def test_iter_with_unmatched_simple(self): t = Trie() - t.add('And', 'And') + t.add("And", "And") t.make_automaton() - test_string = 'AND an a And' + test_string = "AND an a And" result = list(t.iter(test_string)) - assert ['And', 'And'] == [r.value for r in result] + assert ["And", "And"] == [r.value for r in result] def test_iter_with_unmatched_simple2(self): t = Trie() - t.add('AND', 'AND') + t.add("AND", "AND") t.make_automaton() - test_string = 'AND an a and' + test_string = "AND an a and" result = list(t.iter(test_string)) - assert ['AND', 'AND'] == [r.value for r in result] + assert ["AND", "AND"] == [r.value for r in result] def test_iter_with_unmatched_simple3(self): t = Trie() - t.add('AND', 'AND') + t.add("AND", "AND") t.make_automaton() - test_string = 'AND an a andersom' + test_string = "AND an a andersom" result = list(t.iter(test_string)) - assert ['AND'] == [r.value for r in result] + assert ["AND"] == [r.value for r in result] def test_iter_simple(self): t = Trie() - t.add('AND', 'AND') - t.add('OR', 'OR') - t.add('WITH', 'WITH') - t.add('(', '(') - t.add(')', ')') - t.add('GPL-2.0', 'GPL-2.0') - t.add('mit', 'MIT') - t.add('Classpath', 'Classpath') + t.add("AND", "AND") + t.add("OR", "OR") + t.add("WITH", "WITH") + t.add("(", "(") + t.add(")", ")") + t.add("GPL-2.0", "GPL-2.0") + t.add("mit", "MIT") + t.add("Classpath", "Classpath") t.make_automaton() - test_string = '(GPL-2.0 with Classpath) or (gpl-2.0) and (classpath or gpl-2.0 OR mit) ' + test_string = "(GPL-2.0 with Classpath) or (gpl-2.0) and (classpath or gpl-2.0 OR mit) " # 111111111122222222223333333333444444444455555555556666666666777 # 0123456789012345678901234567890123456789012345678901234567890123456789012 result = list(t.iter(test_string)) expected = [ - Token(0, 0, u'(', u'('), - Token(1, 7, u'GPL-2.0', u'GPL-2.0'), - Token(9, 12, u'with', u'WITH'), - Token(14, 22, u'Classpath', u'Classpath'), - Token(23, 23, u')', u')'), - Token(25, 26, u'or', u'OR'), - Token(28, 28, u'(', u'('), - Token(29, 35, u'gpl-2.0', u'GPL-2.0'), - Token(36, 36, u')', u')'), - Token(38, 40, u'and', u'AND'), - Token(42, 42, u'(', u'('), - Token(43, 51, u'classpath', u'Classpath'), - Token(53, 54, u'or', u'OR'), - Token(57, 63, u'gpl-2.0', u'GPL-2.0'), - Token(65, 66, u'OR', u'OR'), - Token(68, 70, u'mit', u'MIT'), - Token(71, 71, u')', u')') + Token(0, 0, "(", "("), + Token(1, 7, "GPL-2.0", "GPL-2.0"), + Token(9, 12, "with", "WITH"), + Token(14, 22, "Classpath", "Classpath"), + Token(23, 23, ")", ")"), + Token(25, 26, "or", "OR"), + Token(28, 28, "(", "("), + Token(29, 35, "gpl-2.0", "GPL-2.0"), + Token(36, 36, ")", ")"), + Token(38, 40, "and", "AND"), + Token(42, 42, "(", "("), + Token(43, 51, "classpath", "Classpath"), + Token(53, 54, "or", "OR"), + Token(57, 63, "gpl-2.0", "GPL-2.0"), + Token(65, 66, "OR", "OR"), + Token(68, 70, "mit", "MIT"), + Token(71, 71, ")", ")"), ] assert expected == result diff --git a/tests/test_license_expression.py b/tests/test_license_expression.py index 1476a14..193fafd 100644 --- a/tests/test_license_expression.py +++ b/tests/test_license_expression.py @@ -61,29 +61,28 @@ def _parse_error_as_dict(pe): class LicenseSymbolTest(TestCase): - def test_LicenseSymbol(self): - sym1 = LicenseSymbol('MIT', ['MIT license']) + sym1 = LicenseSymbol("MIT", ["MIT license"]) assert sym1 == sym1 - assert 'MIT' == sym1.key - assert ('MIT license',) == sym1.aliases + assert "MIT" == sym1.key + assert ("MIT license",) == sym1.aliases - sym2 = LicenseSymbol('mit', ['MIT license']) - assert 'mit' == sym2.key - assert ('MIT license',) == sym2.aliases + sym2 = LicenseSymbol("mit", ["MIT license"]) + assert "mit" == sym2.key + assert ("MIT license",) == sym2.aliases assert not sym2.is_exception assert sym1 != sym2 assert sym1 is not sym2 - sym3 = LicenseSymbol('mit', ['MIT license'], is_exception=True) - assert 'mit' == sym3.key - assert ('MIT license',) == sym3.aliases + sym3 = LicenseSymbol("mit", ["MIT license"], is_exception=True) + assert "mit" == sym3.key + assert ("MIT license",) == sym3.aliases assert sym3.is_exception assert sym2 != sym3 - sym4 = LicenseSymbol('mit', ['MIT license']) - assert 'mit' == sym4.key - assert ('MIT license',) == sym4.aliases + sym4 = LicenseSymbol("mit", ["MIT license"]) + assert "mit" == sym4.key + assert ("MIT license",) == sym4.aliases # symbol equality is based ONLY on the key assert sym2 == sym4 assert sym1 != sym4 @@ -99,13 +98,15 @@ def test_LicenseSymbol(self): def test_python_operators_simple(self): licensing = Licensing() - sym1 = LicenseSymbol('MIT') - sym2 = LicenseSymbol('BSD-2') + sym1 = LicenseSymbol("MIT") + sym2 = LicenseSymbol("BSD-2") assert sym1 & sym2 == licensing.AND(sym1, sym2) assert sym1 | sym2 == licensing.OR(sym1, sym2) - sym3 = LicenseWithExceptionSymbol(LicenseSymbol("GPL-3.0-or-later"), LicenseSymbol("GCC-exception-3.1")) + sym3 = LicenseWithExceptionSymbol( + LicenseSymbol("GPL-3.0-or-later"), LicenseSymbol("GCC-exception-3.1") + ) # Make sure LicenseWithExceptionSymbol operation work on left and right side assert sym3 & sym1 == licensing.AND(sym3, sym1) @@ -114,7 +115,6 @@ def test_python_operators_simple(self): assert sym1 | sym3 == licensing.OR(sym3, sym1) def test_boolean_expression_operators(self): - # Make sure LicenseWithExceptionSymbol boolean expression are set assert LicenseWithExceptionSymbol.Symbol is not None assert LicenseWithExceptionSymbol.TRUE is not None @@ -132,9 +132,7 @@ def test_boolean_expression_operators(self): assert LicenseWithExceptionSymbol.NOT == LicenseSymbol.NOT - class LicensingTest(TestCase): - def test_Licensing_create(self): Licensing() Licensing(None) @@ -142,93 +140,92 @@ def test_Licensing_create(self): class LicensingTokenizeWithoutSymbolsTest(TestCase): - def test_tokenize_plain1(self): licensing = Licensing() expected = [ - (TOKEN_LPAR, '(', 1), - (LicenseSymbol(key='mit'), 'mit', 3), - (TOKEN_RPAR, ')', 7), - (TOKEN_AND, 'and', 9), - (LicenseSymbol(key='gpl'), 'gpl', 13) + (TOKEN_LPAR, "(", 1), + (LicenseSymbol(key="mit"), "mit", 3), + (TOKEN_RPAR, ")", 7), + (TOKEN_AND, "and", 9), + (LicenseSymbol(key="gpl"), "gpl", 13), ] - assert list(licensing.tokenize(' ( mit ) and gpl')) == expected + assert list(licensing.tokenize(" ( mit ) and gpl")) == expected def test_tokenize_plain2(self): licensing = Licensing() expected = [ - (TOKEN_LPAR, '(', 0), - (LicenseSymbol(key='mit'), 'mit', 1), - (TOKEN_AND, 'and', 5), - (LicenseSymbol(key='gpl'), 'gpl', 9), - (TOKEN_RPAR, ')', 12) + (TOKEN_LPAR, "(", 0), + (LicenseSymbol(key="mit"), "mit", 1), + (TOKEN_AND, "and", 5), + (LicenseSymbol(key="gpl"), "gpl", 9), + (TOKEN_RPAR, ")", 12), ] - assert list(licensing.tokenize('(mit and gpl)')) == expected + assert list(licensing.tokenize("(mit and gpl)")) == expected def test_tokenize_plain3(self): licensing = Licensing() expected = [ - (LicenseSymbol(key='mit'), 'mit', 0), - (TOKEN_AND, 'AND', 4), - (LicenseSymbol(key='gpl'), 'gpl', 8), - (TOKEN_OR, 'or', 12), - (LicenseSymbol(key='gpl'), 'gpl', 15) + (LicenseSymbol(key="mit"), "mit", 0), + (TOKEN_AND, "AND", 4), + (LicenseSymbol(key="gpl"), "gpl", 8), + (TOKEN_OR, "or", 12), + (LicenseSymbol(key="gpl"), "gpl", 15), ] - assert list(licensing.tokenize('mit AND gpl or gpl')) == expected + assert list(licensing.tokenize("mit AND gpl or gpl")) == expected def test_tokenize_plain4(self): licensing = Licensing() expected = [ - (TOKEN_LPAR, '(', 0), - (TOKEN_LPAR, '(', 1), - (LicenseSymbol(key=u'l-a+'), 'l-a+', 2), - (TOKEN_AND, 'AND', 7), - (LicenseSymbol(key=u'l-b'), 'l-b', 11), - (TOKEN_RPAR, ')', 14), - (TOKEN_OR, 'OR', 16), - (TOKEN_LPAR, '(', 19), - (LicenseSymbol(key='l-c+'), 'l-c+', 20), - (TOKEN_RPAR, ')', 24), - (TOKEN_RPAR, ')', 25) + (TOKEN_LPAR, "(", 0), + (TOKEN_LPAR, "(", 1), + (LicenseSymbol(key="l-a+"), "l-a+", 2), + (TOKEN_AND, "AND", 7), + (LicenseSymbol(key="l-b"), "l-b", 11), + (TOKEN_RPAR, ")", 14), + (TOKEN_OR, "OR", 16), + (TOKEN_LPAR, "(", 19), + (LicenseSymbol(key="l-c+"), "l-c+", 20), + (TOKEN_RPAR, ")", 24), + (TOKEN_RPAR, ")", 25), ] - assert list(licensing.tokenize( - '((l-a+ AND l-b) OR (l-c+))')) == expected + assert list(licensing.tokenize("((l-a+ AND l-b) OR (l-c+))")) == expected def test_tokenize_plain5(self): licensing = Licensing() expected = [ - (TOKEN_LPAR, '(', 0), - (TOKEN_LPAR, '(', 1), - (LicenseSymbol(key='l-a+'), 'l-a+', 2), - (TOKEN_AND, 'AND', 7), - (LicenseSymbol(key='l-b'), 'l-b', 11), - (TOKEN_RPAR, ')', 14), - (TOKEN_OR, 'OR', 16), - (TOKEN_LPAR, '(', 19), - (LicenseSymbol(key='l-c+'), 'l-c+', 20), - (TOKEN_RPAR, ')', 24), - (TOKEN_RPAR, ')', 25), - (TOKEN_AND, 'and', 27), - (LicenseWithExceptionSymbol( - license_symbol=LicenseSymbol(key='gpl'), - exception_symbol=LicenseSymbol(key='classpath')), - 'gpl with classpath', 31 - ) + (TOKEN_LPAR, "(", 0), + (TOKEN_LPAR, "(", 1), + (LicenseSymbol(key="l-a+"), "l-a+", 2), + (TOKEN_AND, "AND", 7), + (LicenseSymbol(key="l-b"), "l-b", 11), + (TOKEN_RPAR, ")", 14), + (TOKEN_OR, "OR", 16), + (TOKEN_LPAR, "(", 19), + (LicenseSymbol(key="l-c+"), "l-c+", 20), + (TOKEN_RPAR, ")", 24), + (TOKEN_RPAR, ")", 25), + (TOKEN_AND, "and", 27), + ( + LicenseWithExceptionSymbol( + license_symbol=LicenseSymbol(key="gpl"), + exception_symbol=LicenseSymbol(key="classpath"), + ), + "gpl with classpath", + 31, + ), ] - tokens = licensing.tokenize( - '((l-a+ AND l-b) OR (l-c+)) and gpl with classpath' - ) + tokens = licensing.tokenize("((l-a+ AND l-b) OR (l-c+)) and gpl with classpath") assert list(tokens) == expected class LicensingTokenizeWithSymbolsTest(TestCase): - def get_symbols_and_licensing(self): - gpl_20 = LicenseSymbol('GPL-2.0', ['The GNU GPL 20']) - gpl_20_plus = LicenseSymbol('gpl-2.0+', - ['The GNU GPL 20 or later', 'GPL-2.0 or later', 'GPL v2.0 or later']) - lgpl_21 = LicenseSymbol('LGPL-2.1', ['LGPL v2.1']) - mit = LicenseSymbol('MIT', ['MIT license']) + gpl_20 = LicenseSymbol("GPL-2.0", ["The GNU GPL 20"]) + gpl_20_plus = LicenseSymbol( + "gpl-2.0+", ["The GNU GPL 20 or later", "GPL-2.0 or later", "GPL v2.0 or later"] + ) + lgpl_21 = LicenseSymbol("LGPL-2.1", ["LGPL v2.1"]) + mit = LicenseSymbol("MIT", ["MIT license"]) symbols = [gpl_20, gpl_20_plus, lgpl_21, mit] licensing = Licensing(symbols) return gpl_20, gpl_20_plus, lgpl_21, mit, licensing @@ -236,45 +233,43 @@ def get_symbols_and_licensing(self): def test_tokenize_1_with_symbols(self): gpl_20, _gpl_20_plus, lgpl_21, mit, licensing = self.get_symbols_and_licensing() - result = licensing.tokenize( - 'The GNU GPL 20 or LGPL v2.1 AND MIT license ') + result = licensing.tokenize("The GNU GPL 20 or LGPL v2.1 AND MIT license ") # 111111111122222222223333333333444 # 0123456789012345678901234567890123456789012 expected = [ - (gpl_20, 'The GNU GPL 20', 0), - (TOKEN_OR, 'or', 15), - (lgpl_21, 'LGPL v2.1', 18), - (TOKEN_AND, 'AND', 28), - (mit, 'MIT license', 32) + (gpl_20, "The GNU GPL 20", 0), + (TOKEN_OR, "or", 15), + (lgpl_21, "LGPL v2.1", 18), + (TOKEN_AND, "AND", 28), + (mit, "MIT license", 32), ] assert list(result) == expected def test_tokenize_1_no_symbols(self): licensing = Licensing() - result = licensing.tokenize( - 'The GNU GPL 20 or LGPL v2.1 AND MIT license') + result = licensing.tokenize("The GNU GPL 20 or LGPL v2.1 AND MIT license") expected = [ - (LicenseSymbol(u'The GNU GPL 20'), 'The GNU GPL 20', 0), - (TOKEN_OR, 'or', 15), - (LicenseSymbol(u'LGPL v2.1'), 'LGPL v2.1', 18), - (TOKEN_AND, 'AND', 28), - (LicenseSymbol(u'MIT license'), 'MIT license', 32) + (LicenseSymbol("The GNU GPL 20"), "The GNU GPL 20", 0), + (TOKEN_OR, "or", 15), + (LicenseSymbol("LGPL v2.1"), "LGPL v2.1", 18), + (TOKEN_AND, "AND", 28), + (LicenseSymbol("MIT license"), "MIT license", 32), ] assert list(result) == expected def test_tokenize_with_trailing_unknown(self): gpl_20, _gpl_20_plus, lgpl_21, _mit, licensing = self.get_symbols_and_licensing() - result = licensing.tokenize('The GNU GPL 20 or LGPL-2.1 and mit2') + result = licensing.tokenize("The GNU GPL 20 or LGPL-2.1 and mit2") expected = [ - (gpl_20, 'The GNU GPL 20', 0), - (TOKEN_OR, 'or', 15), - (lgpl_21, 'LGPL-2.1', 18), - (TOKEN_AND, 'and', 27), - (LicenseSymbol(key='mit2'), 'mit2', 31), + (gpl_20, "The GNU GPL 20", 0), + (TOKEN_OR, "or", 15), + (lgpl_21, "LGPL-2.1", 18), + (TOKEN_AND, "and", 27), + (LicenseSymbol(key="mit2"), "mit2", 31), ] assert list(result) == expected @@ -282,162 +277,153 @@ def test_tokenize_3(self): gpl_20, gpl_20_plus, lgpl_21, mit, licensing = self.get_symbols_and_licensing() result = licensing.tokenize( - 'The GNU GPL 20 or later or (LGPL-2.1 and mit) or The GNU GPL 20 or mit') + "The GNU GPL 20 or later or (LGPL-2.1 and mit) or The GNU GPL 20 or mit" + ) expected = [ - (gpl_20_plus, 'The GNU GPL 20 or later', 0), - (TOKEN_OR, 'or', 24), - (TOKEN_LPAR, '(', 27), - (lgpl_21, 'LGPL-2.1', 28), - (TOKEN_AND, 'and', 37), - (mit, 'mit', 41), - (TOKEN_RPAR, ')', 44), - (TOKEN_OR, 'or', 46), - (gpl_20, 'The GNU GPL 20', 49), - (2, 'or', 64), - (mit, 'mit', 67) + (gpl_20_plus, "The GNU GPL 20 or later", 0), + (TOKEN_OR, "or", 24), + (TOKEN_LPAR, "(", 27), + (lgpl_21, "LGPL-2.1", 28), + (TOKEN_AND, "and", 37), + (mit, "mit", 41), + (TOKEN_RPAR, ")", 44), + (TOKEN_OR, "or", 46), + (gpl_20, "The GNU GPL 20", 49), + (2, "or", 64), + (mit, "mit", 67), ] assert list(result) == expected def test_tokenize_unknown_as_trailing_single_attached_character(self): - symbols = [LicenseSymbol('MIT', ['MIT license'])] + symbols = [LicenseSymbol("MIT", ["MIT license"])] l = Licensing(symbols) - result = list(l.tokenize('mit2')) + result = list(l.tokenize("mit2")) expected = [ - (LicenseSymbol(u'mit2'), 'mit2', 0), + (LicenseSymbol("mit2"), "mit2", 0), ] assert result == expected def test_tokenize_with_unknown_symbol_containing_known_symbol_leading(self): - l = Licensing(['gpl-2.0']) - result = list(l.tokenize('gpl-2.0 AND gpl-2.0-plus', strict=False)) + l = Licensing(["gpl-2.0"]) + result = list(l.tokenize("gpl-2.0 AND gpl-2.0-plus", strict=False)) result = [s for s, _, _ in result] expected = [ - LicenseSymbol(key='gpl-2.0'), + LicenseSymbol(key="gpl-2.0"), TOKEN_AND, - LicenseSymbol(key='gpl-2.0-plus'), + LicenseSymbol(key="gpl-2.0-plus"), ] assert result == expected def test_tokenize_with_unknown_symbol_containing_known_symbol_contained(self): - l = Licensing(['gpl-2.0']) - result = list(l.tokenize( - 'gpl-2.0 WITH exception-gpl-2.0-plus', strict=False)) + l = Licensing(["gpl-2.0"]) + result = list(l.tokenize("gpl-2.0 WITH exception-gpl-2.0-plus", strict=False)) result = [s for s, _, _ in result] expected = [ LicenseWithExceptionSymbol( - LicenseSymbol(u'gpl-2.0'), - LicenseSymbol(u'exception-gpl-2.0-plus') + LicenseSymbol("gpl-2.0"), LicenseSymbol("exception-gpl-2.0-plus") ) ] assert result == expected def test_tokenize_with_unknown_symbol_containing_known_symbol_trailing(self): - l = Licensing(['gpl-2.0']) - result = list(l.tokenize( - 'gpl-2.0 AND exception-gpl-2.0', strict=False)) + l = Licensing(["gpl-2.0"]) + result = list(l.tokenize("gpl-2.0 AND exception-gpl-2.0", strict=False)) result = [s for s, _, _ in result] - expected = [ - LicenseSymbol(u'gpl-2.0'), - TOKEN_AND, - LicenseSymbol(u'exception-gpl-2.0') - ] + expected = [LicenseSymbol("gpl-2.0"), TOKEN_AND, LicenseSymbol("exception-gpl-2.0")] assert result == expected class LicensingParseTest(TestCase): - def test_parse_does_not_raise_error_for_empty_expression(self): licensing = Licensing() - assert None == licensing.parse('') + assert None == licensing.parse("") def test_parse(self): - expression = ' ( (( gpl and bsd ) or lgpl) and gpl-exception) ' - expected = '((gpl AND bsd) OR lgpl) AND gpl-exception' + expression = " ( (( gpl and bsd ) or lgpl) and gpl-exception) " + expected = "((gpl AND bsd) OR lgpl) AND gpl-exception" licensing = Licensing() self.assertEqual(expected, str(licensing.parse(expression))) def test_parse_raise_ParseError(self): - expression = ' ( (( gpl and bsd ) or lgpl) and gpl-exception)) ' + expression = " ( (( gpl and bsd ) or lgpl) and gpl-exception)) " licensing = Licensing() try: licensing.parse(expression) - self.fail('ParseError should be raised') + self.fail("ParseError should be raised") except ParseError as pe: expected = { - 'error_code': PARSE_UNBALANCED_CLOSING_PARENS, - 'position': 48, - 'token_string': ')', - 'token_type': TOKEN_RPAR + "error_code": PARSE_UNBALANCED_CLOSING_PARENS, + "position": 48, + "token_string": ")", + "token_type": TOKEN_RPAR, } assert _parse_error_as_dict(pe) == expected def test_parse_raise_ExpressionError_when_validating(self): - expression = 'gpl and bsd or lgpl with exception' + expression = "gpl and bsd or lgpl with exception" licensing = Licensing() try: licensing.parse(expression, validate=True) - self.fail('Exception not raised') + self.fail("Exception not raised") except ExpressionError as ee: - assert 'Unknown license key(s): gpl, bsd, lgpl, exception' == str( - ee) + assert "Unknown license key(s): gpl, bsd, lgpl, exception" == str(ee) def test_parse_raise_ParseError_when_validating_strict(self): - expression = 'gpl and bsd or lgpl with exception' + expression = "gpl and bsd or lgpl with exception" licensing = Licensing() try: licensing.parse(expression, validate=True, strict=True) - self.fail('Exception not raised') + self.fail("Exception not raised") except ParseError as pe: expected = { - 'error_code': PARSE_INVALID_SYMBOL_AS_EXCEPTION, - 'position': 25, - 'token_string': 'exception', - 'token_type': TOKEN_SYMBOL + "error_code": PARSE_INVALID_SYMBOL_AS_EXCEPTION, + "position": 25, + "token_string": "exception", + "token_type": TOKEN_SYMBOL, } assert _parse_error_as_dict(pe) == expected def test_parse_raise_ParseError_when_strict_no_validate(self): - expression = 'gpl and bsd or lgpl with exception' + expression = "gpl and bsd or lgpl with exception" licensing = Licensing() try: licensing.parse(expression, validate=False, strict=True) - self.fail('Exception not raised') + self.fail("Exception not raised") except ParseError as pe: expected = { - 'error_code': PARSE_INVALID_SYMBOL_AS_EXCEPTION, - 'position': 25, - 'token_string': 'exception', - 'token_type': TOKEN_SYMBOL + "error_code": PARSE_INVALID_SYMBOL_AS_EXCEPTION, + "position": 25, + "token_string": "exception", + "token_type": TOKEN_SYMBOL, } assert _parse_error_as_dict(pe) == expected def test_parse_raise_ExpressionError_when_validating_strict_with_unknown(self): - expression = 'gpl and bsd or lgpl with exception' - licensing = Licensing( - symbols=[LicenseSymbol('exception', is_exception=True)]) + expression = "gpl and bsd or lgpl with exception" + licensing = Licensing(symbols=[LicenseSymbol("exception", is_exception=True)]) try: licensing.parse(expression, validate=True, strict=True) except ExpressionError as ee: - assert 'Unknown license key(s): gpl, bsd, lgpl' == str(ee) + assert "Unknown license key(s): gpl, bsd, lgpl" == str(ee) def test_parse_in_strict_mode_for_solo_symbol(self): - expression = 'lgpl' + expression = "lgpl" licensing = Licensing() licensing.parse(expression, strict=True) def test_parse_invalid_expression_raise_exception(self): licensing = Licensing() - expr = 'wrong' + expr = "wrong" licensing.parse(expr) def test_parse_not_invalid_expression_rais_not_exception(self): licensing = Licensing() - expr = 'l-a AND none' + expr = "l-a AND none" licensing.parse(expr) def test_parse_invalid_expression_raise_exception3(self): licensing = Licensing() - expr = '(l-a + AND l-b' + expr = "(l-a + AND l-b" try: licensing.parse(expr) self.fail("Exception not raised when validating '%s'" % expr) @@ -446,7 +432,7 @@ def test_parse_invalid_expression_raise_exception3(self): def test_parse_invalid_expression_raise_exception4(self): licensing = Licensing() - expr = '(l-a + AND l-b))' + expr = "(l-a + AND l-b))" try: licensing.parse(expr) self.fail("Exception not raised when validating '%s'" % expr) @@ -455,46 +441,45 @@ def test_parse_invalid_expression_raise_exception4(self): def test_parse_invalid_expression_raise_exception5(self): licensing = Licensing() - expr = 'l-a AND' + expr = "l-a AND" try: licensing.parse(expr) self.fail("Exception not raised when validating '%s'" % expr) except ExpressionError as ee: - assert 'AND requires two or more licenses as in: MIT AND BSD' == str( - ee) + assert "AND requires two or more licenses as in: MIT AND BSD" == str(ee) def test_parse_invalid_expression_raise_exception6(self): licensing = Licensing() - expr = 'OR l-a' + expr = "OR l-a" try: licensing.parse(expr) self.fail("Exception not raised when validating '%s'" % expr) - self.fail('Exception not raised') + self.fail("Exception not raised") except ParseError as pe: expected = { - 'error_code': PARSE_INVALID_OPERATOR_SEQUENCE, - 'position': 0, - 'token_string': 'OR', - 'token_type': TOKEN_OR + "error_code": PARSE_INVALID_OPERATOR_SEQUENCE, + "position": 0, + "token_string": "OR", + "token_type": TOKEN_OR, } assert _parse_error_as_dict(pe) == expected def test_parse_not_invalid_expression_raise_no_exception2(self): licensing = Licensing() - expr = '+l-a' + expr = "+l-a" licensing.parse(expr) def test_parse_can_parse(self): licensing = Licensing() - expr = ' GPL-2.0 or LGPL2.1 and mit ' + expr = " GPL-2.0 or LGPL2.1 and mit " parsed = licensing.parse(expr) - gpl2 = LicenseSymbol('GPL-2.0') - lgpl = LicenseSymbol('LGPL2.1') - mit = LicenseSymbol('mit') + gpl2 = LicenseSymbol("GPL-2.0") + lgpl = LicenseSymbol("LGPL2.1") + mit = LicenseSymbol("mit") expected = [gpl2, lgpl, mit] self.assertEqual(expected, licensing.license_symbols(parsed)) self.assertEqual(expected, licensing.license_symbols(expr)) - self.assertEqual('GPL-2.0 OR (LGPL2.1 AND mit)', str(parsed)) + self.assertEqual("GPL-2.0 OR (LGPL2.1 AND mit)", str(parsed)) expected = licensing.OR(gpl2, licensing.AND(lgpl, mit)) assert parsed == expected @@ -502,56 +487,56 @@ def test_parse_can_parse(self): def test_parse_errors_catch_invalid_nesting(self): licensing = Licensing() try: - licensing.parse('mit (and LGPL 2.1)') - self.fail('Exception not raised') + licensing.parse("mit (and LGPL 2.1)") + self.fail("Exception not raised") except ParseError as pe: expected = { - 'error_code': PARSE_INVALID_NESTING, - 'position': 4, - 'token_string': '(', - 'token_type': TOKEN_LPAR + "error_code": PARSE_INVALID_NESTING, + "position": 4, + "token_string": "(", + "token_type": TOKEN_LPAR, } assert _parse_error_as_dict(pe) == expected def test_parse_errors_catch_invalid_expression_with_bare_and(self): licensing = Licensing() try: - licensing.parse('and') - self.fail('Exception not raised') + licensing.parse("and") + self.fail("Exception not raised") except ParseError as pe: expected = { - 'error_code': PARSE_INVALID_OPERATOR_SEQUENCE, - 'position': 0, - 'token_string': 'and', - 'token_type': TOKEN_AND + "error_code": PARSE_INVALID_OPERATOR_SEQUENCE, + "position": 0, + "token_string": "and", + "token_type": TOKEN_AND, } assert _parse_error_as_dict(pe) == expected def test_parse_errors_catch_invalid_expression_with_or_and_no_other(self): licensing = Licensing() try: - licensing.parse('or that') - self.fail('Exception not raised') + licensing.parse("or that") + self.fail("Exception not raised") except ParseError as pe: expected = { - 'error_code': PARSE_INVALID_OPERATOR_SEQUENCE, - 'position': 0, - 'token_string': 'or', - 'token_type': TOKEN_OR + "error_code": PARSE_INVALID_OPERATOR_SEQUENCE, + "position": 0, + "token_string": "or", + "token_type": TOKEN_OR, } assert _parse_error_as_dict(pe) == expected def test_parse_errors_catch_invalid_expression_with_empty_parens(self): licensing = Licensing() try: - licensing.parse('with ( )this') - self.fail('Exception not raised') + licensing.parse("with ( )this") + self.fail("Exception not raised") except ParseError as pe: expected = { - 'error_code': PARSE_INVALID_EXPRESSION, - 'position': 0, - 'token_string': 'with', - 'token_type': TOKEN_WITH + "error_code": PARSE_INVALID_EXPRESSION, + "position": 0, + "token_string": "with", + "token_type": TOKEN_WITH, } assert _parse_error_as_dict(pe) == expected @@ -564,43 +549,41 @@ def test_parse_errors_catch_invalid_non_unicode_byte_strings_on_python3(self): if py2: extra_bytes = bytes(chr(0) + chr(12) + chr(255)) try: - licensing.parse('mit (and LGPL 2.1)'.encode( - 'utf-8') + extra_bytes) - self.fail('Exception not raised') + licensing.parse("mit (and LGPL 2.1)".encode("utf-8") + extra_bytes) + self.fail("Exception not raised") except ExpressionError as ee: - assert str(ee).startswith('expression must be a string and') + assert str(ee).startswith("expression must be a string and") if py3: - extra_bytes = bytes(chr(0) + chr(12) + chr(255), encoding='utf-8') + extra_bytes = bytes(chr(0) + chr(12) + chr(255), encoding="utf-8") try: - licensing.parse('mit (and LGPL 2.1)'.encode( - 'utf-8') + extra_bytes) - self.fail('Exception not raised') + licensing.parse("mit (and LGPL 2.1)".encode("utf-8") + extra_bytes) + self.fail("Exception not raised") except ExpressionError as ee: - assert str(ee).startswith('Invalid license key') + assert str(ee).startswith("Invalid license key") def test_parse_errors_does_not_raise_error_on_plain_non_unicode_raw_string(self): # plain non-unicode string does not raise error licensing = Licensing() - x = licensing.parse(r'mit and (LGPL-2.1)') + x = licensing.parse(r"mit and (LGPL-2.1)") self.assertTrue(isinstance(x, LicenseExpression)) def test_parse_simplify_and_contain_and_equal(self): licensing = Licensing() - expr = licensing.parse(' GPL-2.0 or LGPL2.1 and mit ') + expr = licensing.parse(" GPL-2.0 or LGPL2.1 and mit ") - expr2 = licensing.parse(' (mit and LGPL2.1) or GPL-2.0 ') + expr2 = licensing.parse(" (mit and LGPL2.1) or GPL-2.0 ") self.assertEqual(expr2.simplify(), expr.simplify()) self.assertEqual(expr2, expr) - expr3 = licensing.parse('mit and LGPL2.1') + expr3 = licensing.parse("mit and LGPL2.1") self.assertTrue(expr3 in expr2) def test_parse_simplify_no_sort(self): licensing = Licensing() - expr = licensing.parse('gpl-2.0 OR apache-2.0') - expr2 = licensing.parse('apache-2.0 OR gpl-2.0') + expr = licensing.parse("gpl-2.0 OR apache-2.0") + expr2 = licensing.parse("apache-2.0 OR gpl-2.0") self.assertEqual(expr, expr2) self.assertEqual(expr.simplify(), expr2.simplify()) @@ -611,212 +594,202 @@ def test_license_expression_is_equivalent(self): lic = Licensing() is_equiv = lic.is_equivalent - self.assertTrue(is_equiv(lic.parse('mit AND gpl'), - lic.parse('mit AND gpl'))) - self.assertTrue(is_equiv(lic.parse('mit AND gpl'), - lic.parse('gpl AND mit'))) - self.assertTrue(is_equiv(lic.parse('mit AND gpl and apache'), - lic.parse('apache and gpl AND mit'))) - self.assertTrue(is_equiv( - lic.parse('mit AND (gpl AND apache)'), lic.parse('(mit AND gpl) AND apache'))) + self.assertTrue(is_equiv(lic.parse("mit AND gpl"), lic.parse("mit AND gpl"))) + self.assertTrue(is_equiv(lic.parse("mit AND gpl"), lic.parse("gpl AND mit"))) + self.assertTrue( + is_equiv(lic.parse("mit AND gpl and apache"), lic.parse("apache and gpl AND mit")) + ) + self.assertTrue( + is_equiv(lic.parse("mit AND (gpl AND apache)"), lic.parse("(mit AND gpl) AND apache")) + ) # same but without parsing: - self.assertTrue(is_equiv('mit AND gpl', 'mit AND gpl')) - self.assertTrue(is_equiv('mit AND gpl', 'gpl AND mit')) - self.assertTrue(is_equiv('mit AND gpl and apache', - 'apache and gpl AND mit')) - self.assertTrue(is_equiv('mit AND (gpl AND apache)', - '(mit AND gpl) AND apache')) + self.assertTrue(is_equiv("mit AND gpl", "mit AND gpl")) + self.assertTrue(is_equiv("mit AND gpl", "gpl AND mit")) + self.assertTrue(is_equiv("mit AND gpl and apache", "apache and gpl AND mit")) + self.assertTrue(is_equiv("mit AND (gpl AND apache)", "(mit AND gpl) AND apache")) # Real-case example of generated expression vs. stored expression: - ex1 = '''Commercial + ex1 = """Commercial AND apache-1.1 AND apache-2.0 AND aslr AND bsd-new AND cpl-1.0 AND epl-1.0 AND ibm-icu AND ijg AND jdom AND lgpl-2.1 AND mit-open-group AND mpl-1.1 AND sax-pd AND unicode AND w3c AND - w3c-documentation''' + w3c-documentation""" - ex2 = ''' + ex2 = """ apache-1.1 AND apache-2.0 AND aslr AND bsd-new AND cpl-1.0 AND epl-1.0 AND lgpl-2.1 AND ibm-icu AND ijg AND jdom AND mit-open-group AND mpl-1.1 AND Commercial AND sax-pd AND unicode - AND w3c-documentation AND w3c''' + AND w3c-documentation AND w3c""" self.assertTrue(is_equiv(lic.parse(ex1), lic.parse(ex2))) - self.assertFalse( - is_equiv(lic.parse('mit AND gpl'), lic.parse('mit OR gpl'))) - self.assertFalse( - is_equiv(lic.parse('mit AND gpl'), lic.parse('gpl OR mit'))) + self.assertFalse(is_equiv(lic.parse("mit AND gpl"), lic.parse("mit OR gpl"))) + self.assertFalse(is_equiv(lic.parse("mit AND gpl"), lic.parse("gpl OR mit"))) def test_license_expression_license_keys(self): licensing = Licensing() - assert ['mit', 'gpl'] == licensing.license_keys( - licensing.parse(' ( mit ) and gpl')) - assert ['mit', 'gpl'] == licensing.license_keys( - licensing.parse('(mit and gpl)')) + assert ["mit", "gpl"] == licensing.license_keys(licensing.parse(" ( mit ) and gpl")) + assert ["mit", "gpl"] == licensing.license_keys(licensing.parse("(mit and gpl)")) # these two are surprising for now: this is because the expression is a # logical expression so the order may be different on more complex expressions - assert ['mit', 'gpl'] == licensing.license_keys( - licensing.parse('mit AND gpl or gpl')) - assert ['l-a+', 'l-b', '+l-c'] == licensing.license_keys( - licensing.parse('((l-a+ AND l-b) OR (+l-c))')) + assert ["mit", "gpl"] == licensing.license_keys(licensing.parse("mit AND gpl or gpl")) + assert ["l-a+", "l-b", "+l-c"] == licensing.license_keys( + licensing.parse("((l-a+ AND l-b) OR (+l-c))") + ) # same without parsing - assert ['mit', 'gpl'] == licensing.license_keys('mit AND gpl or gpl') - assert ['l-a+', 'l-b', - 'l-c+'] == licensing.license_keys('((l-a+ AND l-b) OR (l-c+))') + assert ["mit", "gpl"] == licensing.license_keys("mit AND gpl or gpl") + assert ["l-a+", "l-b", "l-c+"] == licensing.license_keys("((l-a+ AND l-b) OR (l-c+))") def test_end_to_end(self): # these were formerly doctest ported to actual real code tests here l = Licensing() - expr = l.parse(' GPL-2.0 or LGPL-2.1 and mit ') - expected = 'GPL-2.0 OR (LGPL-2.1 AND mit)' + expr = l.parse(" GPL-2.0 or LGPL-2.1 and mit ") + expected = "GPL-2.0 OR (LGPL-2.1 AND mit)" assert str(expr) == expected expected = [ - LicenseSymbol('GPL-2.0'), - LicenseSymbol('LGPL-2.1'), - LicenseSymbol('mit'), + LicenseSymbol("GPL-2.0"), + LicenseSymbol("LGPL-2.1"), + LicenseSymbol("mit"), ] assert l.license_symbols(expr) == expected def test_pretty(self): l = Licensing() - expr = l.parse(' GPL-2.0 or LGPL2.1 and mit ') + expr = l.parse(" GPL-2.0 or LGPL2.1 and mit ") - expected = '''OR( + expected = """OR( LicenseSymbol('GPL-2.0'), AND( LicenseSymbol('LGPL2.1'), LicenseSymbol('mit') ) -)''' +)""" assert expr.pretty() == expected def test_simplify_and_contains(self): l = Licensing() - expr = l.parse(' GPL-2.0 or LGPL2.1 and mit ') - expr2 = l.parse(' GPL-2.0 or (mit and LGPL2.1) ') + expr = l.parse(" GPL-2.0 or LGPL2.1 and mit ") + expr2 = l.parse(" GPL-2.0 or (mit and LGPL2.1) ") assert expr2.simplify() == expr.simplify() - expr3 = l.parse('mit and LGPL2.1') + expr3 = l.parse("mit and LGPL2.1") assert expr3 in expr2 def test_dedup_expressions_can_be_simplified_1(self): l = Licensing() - exp = 'mit OR mit AND apache-2.0 AND bsd-new OR mit' + exp = "mit OR mit AND apache-2.0 AND bsd-new OR mit" result = l.dedup(exp) - expected = l.parse('mit OR (mit AND apache-2.0 AND bsd-new)') + expected = l.parse("mit OR (mit AND apache-2.0 AND bsd-new)") assert result == expected def test_dedup_expressions_can_be_simplified_2(self): l = Licensing() - exp = 'mit AND (mit OR bsd-new) AND mit OR mit' + exp = "mit AND (mit OR bsd-new) AND mit OR mit" result = l.dedup(exp) - expected = l.parse('(mit AND (mit OR bsd-new)) OR mit') + expected = l.parse("(mit AND (mit OR bsd-new)) OR mit") assert result == expected def test_dedup_expressions_multiple_occurrences(self): l = Licensing() - exp = ' GPL-2.0 or (mit and LGPL-2.1) or bsd Or GPL-2.0 or (mit and LGPL-2.1)' + exp = " GPL-2.0 or (mit and LGPL-2.1) or bsd Or GPL-2.0 or (mit and LGPL-2.1)" result = l.dedup(exp) - expected = l.parse('GPL-2.0 OR (mit AND LGPL-2.1) OR bsd') + expected = l.parse("GPL-2.0 OR (mit AND LGPL-2.1) OR bsd") assert result == expected def test_dedup_expressions_cannot_be_simplified(self): l = Licensing() - exp = l.parse('mit AND (mit OR bsd-new)') + exp = l.parse("mit AND (mit OR bsd-new)") result = l.dedup(exp) - expected = l.parse('mit AND (mit OR bsd-new)') + expected = l.parse("mit AND (mit OR bsd-new)") assert result == expected def test_dedup_expressions_single_license(self): l = Licensing() - exp = l.parse('mit') + exp = l.parse("mit") result = l.dedup(exp) - expected = l.parse('mit') + expected = l.parse("mit") assert result == expected def test_dedup_expressions_WITH(self): l = Licensing() - exp = l.parse('gpl-2.0 with autoconf-exception-2.0') + exp = l.parse("gpl-2.0 with autoconf-exception-2.0") result = l.dedup(exp) - expected = l.parse('gpl-2.0 with autoconf-exception-2.0') + expected = l.parse("gpl-2.0 with autoconf-exception-2.0") assert result == expected def test_dedup_expressions_WITH_OR(self): l = Licensing() - exp = l.parse('gpl-2.0 with autoconf-exception-2.0 OR gpl-2.0') + exp = l.parse("gpl-2.0 with autoconf-exception-2.0 OR gpl-2.0") result = l.dedup(exp) - expected = l.parse('gpl-2.0 with autoconf-exception-2.0 OR gpl-2.0') + expected = l.parse("gpl-2.0 with autoconf-exception-2.0 OR gpl-2.0") assert result == expected def test_dedup_expressions_WITH_AND(self): l = Licensing() - exp = l.parse( - 'gpl-2.0 AND gpl-2.0 with autoconf-exception-2.0 AND gpl-2.0') + exp = l.parse("gpl-2.0 AND gpl-2.0 with autoconf-exception-2.0 AND gpl-2.0") result = l.dedup(exp) - expected = l.parse('gpl-2.0 AND gpl-2.0 with autoconf-exception-2.0') + expected = l.parse("gpl-2.0 AND gpl-2.0 with autoconf-exception-2.0") assert result == expected def test_dedup_licensexpressions_can_be_simplified_3(self): l = Licensing() - exp = l.parse('mit AND mit') + exp = l.parse("mit AND mit") result = l.dedup(exp) - expected = l.parse('mit') + expected = l.parse("mit") assert result == expected def test_dedup_licensexpressions_works_with_subexpressions(self): l = Licensing() - exp = l.parse( - '(mit OR gpl-2.0) AND mit AND bsd-new AND (mit OR gpl-2.0)') + exp = l.parse("(mit OR gpl-2.0) AND mit AND bsd-new AND (mit OR gpl-2.0)") result = l.dedup(exp) - expected = l.parse('(mit OR gpl-2.0) AND mit AND bsd-new') + expected = l.parse("(mit OR gpl-2.0) AND mit AND bsd-new") assert result == expected def test_simplify_and_equivalent_and_contains(self): l = Licensing() - expr2 = l.parse( - ' GPL-2.0 or (mit and LGPL-2.1) or bsd Or GPL-2.0 or (mit and LGPL-2.1)') + expr2 = l.parse(" GPL-2.0 or (mit and LGPL-2.1) or bsd Or GPL-2.0 or (mit and LGPL-2.1)") # note thats simplification does SORT the symbols such that they can # eventually be compared sequence-wise. This sorting is based on license key - expected = 'GPL-2.0 OR bsd OR (LGPL-2.1 AND mit)' + expected = "GPL-2.0 OR bsd OR (LGPL-2.1 AND mit)" assert str(expr2.simplify()) == expected # Two expressions can be compared for equivalence: - expr1 = l.parse(' GPL-2.0 or (LGPL-2.1 and mit) ') - assert 'GPL-2.0 OR (LGPL-2.1 AND mit)' == str(expr1) - expr2 = l.parse(' (mit and LGPL-2.1) or GPL-2.0 ') - assert '(mit AND LGPL-2.1) OR GPL-2.0' == str(expr2) + expr1 = l.parse(" GPL-2.0 or (LGPL-2.1 and mit) ") + assert "GPL-2.0 OR (LGPL-2.1 AND mit)" == str(expr1) + expr2 = l.parse(" (mit and LGPL-2.1) or GPL-2.0 ") + assert "(mit AND LGPL-2.1) OR GPL-2.0" == str(expr2) assert l.is_equivalent(expr1, expr2) - assert 'GPL-2.0 OR (LGPL-2.1 AND mit)' == str(expr1.simplify()) - assert 'GPL-2.0 OR (LGPL-2.1 AND mit)' == str(expr2.simplify()) + assert "GPL-2.0 OR (LGPL-2.1 AND mit)" == str(expr1.simplify()) + assert "GPL-2.0 OR (LGPL-2.1 AND mit)" == str(expr2.simplify()) assert expr1.simplify() == expr2.simplify() - expr3 = l.parse(' GPL-2.0 or mit or LGPL-2.1') + expr3 = l.parse(" GPL-2.0 or mit or LGPL-2.1") assert not l.is_equivalent(expr2, expr3) - expr4 = l.parse('mit and LGPL-2.1') + expr4 = l.parse("mit and LGPL-2.1") assert expr4.simplify() in expr2.simplify() assert l.contains(expr2, expr4) def test_contains_works_with_plain_symbol(self): l = Licensing() - assert not l.contains('mit', 'mit and LGPL-2.1') - assert l.contains('mit and LGPL-2.1', 'mit') - assert l.contains('mit', 'mit') - assert not l.contains(l.parse('mit'), l.parse('mit and LGPL-2.1')) - assert l.contains(l.parse('mit and LGPL-2.1'), l.parse('mit')) - - assert l.contains('mit with GPL', 'GPL') - assert l.contains('mit with GPL', 'mit') - assert l.contains('mit with GPL', 'mit with GPL') - assert not l.contains('mit with GPL', 'GPL with mit') - assert not l.contains('mit with GPL', 'GPL and mit') - assert not l.contains('GPL', 'mit with GPL') - assert l.contains('mit with GPL and GPL and BSD', 'GPL and BSD') + assert not l.contains("mit", "mit and LGPL-2.1") + assert l.contains("mit and LGPL-2.1", "mit") + assert l.contains("mit", "mit") + assert not l.contains(l.parse("mit"), l.parse("mit and LGPL-2.1")) + assert l.contains(l.parse("mit and LGPL-2.1"), l.parse("mit")) + + assert l.contains("mit with GPL", "GPL") + assert l.contains("mit with GPL", "mit") + assert l.contains("mit with GPL", "mit with GPL") + assert not l.contains("mit with GPL", "GPL with mit") + assert not l.contains("mit with GPL", "GPL and mit") + assert not l.contains("GPL", "mit with GPL") + assert l.contains("mit with GPL and GPL and BSD", "GPL and BSD") def test_create_from_python(self): # Expressions can be built from Python expressions, using bitwise operators @@ -824,273 +797,280 @@ def test_create_from_python(self): # well specified that using text expression and parse licensing = Licensing() - expr1 = (licensing.LicenseSymbol('GPL-2.0') - | (licensing.LicenseSymbol('mit') - & licensing.LicenseSymbol('LGPL-2.1'))) - expr2 = licensing.parse(' GPL-2.0 or (mit and LGPL-2.1) ') + expr1 = licensing.LicenseSymbol("GPL-2.0") | ( + licensing.LicenseSymbol("mit") & licensing.LicenseSymbol("LGPL-2.1") + ) + expr2 = licensing.parse(" GPL-2.0 or (mit and LGPL-2.1) ") - assert 'GPL-2.0 OR (LGPL-2.1 AND mit)' == str(expr1.simplify()) - assert 'GPL-2.0 OR (LGPL-2.1 AND mit)' == str(expr2.simplify()) + assert "GPL-2.0 OR (LGPL-2.1 AND mit)" == str(expr1.simplify()) + assert "GPL-2.0 OR (LGPL-2.1 AND mit)" == str(expr2.simplify()) assert licensing.is_equivalent(expr1, expr2) a = licensing.OR( - LicenseSymbol(key='gpl-2.0'), - licensing.AND(LicenseSymbol(key='mit'), - LicenseSymbol(key='lgpl-2.1') - ) + LicenseSymbol(key="gpl-2.0"), + licensing.AND(LicenseSymbol(key="mit"), LicenseSymbol(key="lgpl-2.1")), ) b = licensing.OR( - LicenseSymbol(key='gpl-2.0'), - licensing.AND(LicenseSymbol(key='mit'), - LicenseSymbol(key='lgpl-2.1') - ) + LicenseSymbol(key="gpl-2.0"), + licensing.AND(LicenseSymbol(key="mit"), LicenseSymbol(key="lgpl-2.1")), ) assert a == b def test_parse_with_repeated_or_later_does_not_raise_parse_error(self): l = Licensing() - expr = 'LGPL2.1+ + and mit' + expr = "LGPL2.1+ + and mit" parsed = l.parse(expr) - assert 'LGPL2.1+ + AND mit' == str(parsed) + assert "LGPL2.1+ + AND mit" == str(parsed) def test_render_complex(self): licensing = Licensing() - expression = ''' + expression = """ EPL-1.0 AND Apache-1.1 AND Apache-2.0 AND BSD-Modified AND CPL-1.0 AND ICU-Composite-License AND JPEG-License AND JDOM-License AND LGPL-2.0 AND MIT-Open-Group AND MPL-1.1 AND SAX-PD AND Unicode-Inc-License-Agreement - AND W3C-Software-Notice and License AND W3C-Documentation-License''' + AND W3C-Software-Notice and License AND W3C-Documentation-License""" result = licensing.parse(expression) - expected = ('EPL-1.0 AND Apache-1.1 AND Apache-2.0 AND BSD-Modified ' - 'AND CPL-1.0 AND ICU-Composite-License AND JPEG-License ' - 'AND JDOM-License AND LGPL-2.0 AND MIT-Open-Group AND MPL-1.1 ' - 'AND SAX-PD AND Unicode-Inc-License-Agreement ' - 'AND W3C-Software-Notice AND License AND W3C-Documentation-License') - - assert result.render('{symbol.key}') == expected - expectedkey = ('EPL-1.0 AND Apache-1.1 AND Apache-2.0 AND BSD-Modified AND ' - 'CPL-1.0 AND ICU-Composite-License AND JPEG-License AND JDOM-License AND ' - 'LGPL-2.0 AND MIT-Open-Group AND MPL-1.1 AND SAX-PD AND ' - 'Unicode-Inc-License-Agreement AND W3C-Software-Notice AND License AND' - ' W3C-Documentation-License') - assert expectedkey == result.render('{symbol.key}') + expected = ( + "EPL-1.0 AND Apache-1.1 AND Apache-2.0 AND BSD-Modified " + "AND CPL-1.0 AND ICU-Composite-License AND JPEG-License " + "AND JDOM-License AND LGPL-2.0 AND MIT-Open-Group AND MPL-1.1 " + "AND SAX-PD AND Unicode-Inc-License-Agreement " + "AND W3C-Software-Notice AND License AND W3C-Documentation-License" + ) + + assert result.render("{symbol.key}") == expected + expectedkey = ( + "EPL-1.0 AND Apache-1.1 AND Apache-2.0 AND BSD-Modified AND " + "CPL-1.0 AND ICU-Composite-License AND JPEG-License AND JDOM-License AND " + "LGPL-2.0 AND MIT-Open-Group AND MPL-1.1 AND SAX-PD AND " + "Unicode-Inc-License-Agreement AND W3C-Software-Notice AND License AND" + " W3C-Documentation-License" + ) + assert expectedkey == result.render("{symbol.key}") def test_render_with(self): licensing = Licensing() - expression = 'GPL-2.0 with Classpath-2.0 OR BSD-new' + expression = "GPL-2.0 with Classpath-2.0 OR BSD-new" result = licensing.parse(expression) - expected = 'GPL-2.0 WITH Classpath-2.0 OR BSD-new' - assert result.render('{symbol.key}') == expected + expected = "GPL-2.0 WITH Classpath-2.0 OR BSD-new" + assert result.render("{symbol.key}") == expected expected_html = ( 'GPL-2.0 WITH ' 'Classpath-2.0 ' - 'OR BSD-new') - assert expected_html == result.render( - '{symbol.key}') + 'OR BSD-new' + ) + assert expected_html == result.render('{symbol.key}') - expected = 'GPL-2.0 WITH Classpath-2.0 OR BSD-new' - assert result.render('{symbol.key}') == expected + expected = "GPL-2.0 WITH Classpath-2.0 OR BSD-new" + assert result.render("{symbol.key}") == expected def test_parse_complex(self): licensing = Licensing() - expression = ' GPL-2.0 or later with classpath-Exception and mit or LPL-2.1 and mit or later ' + expression = ( + " GPL-2.0 or later with classpath-Exception and mit or LPL-2.1 and mit or later " + ) result = licensing.parse(expression) # this may look weird, but we did not provide symbols hence in "or later", # "later" is treated as if it were a license - expected = 'GPL-2.0 OR (later WITH classpath-Exception AND mit) OR (LPL-2.1 AND mit) OR later' - assert result.render('{symbol.key}') == expected + expected = ( + "GPL-2.0 OR (later WITH classpath-Exception AND mit) OR (LPL-2.1 AND mit) OR later" + ) + assert result.render("{symbol.key}") == expected def test_parse_complex2(self): licensing = Licensing() expr = licensing.parse(" GPL-2.0 or LGPL-2.1 and mit ") - expected = [ - LicenseSymbol('GPL-2.0'), - LicenseSymbol('LGPL-2.1'), - LicenseSymbol('mit') - ] + expected = [LicenseSymbol("GPL-2.0"), LicenseSymbol("LGPL-2.1"), LicenseSymbol("mit")] assert sorted(licensing.license_symbols(expr)) == expected - expected = 'GPL-2.0 OR (LGPL-2.1 AND mit)' - assert expr.render('{symbol.key}') == expected + expected = "GPL-2.0 OR (LGPL-2.1 AND mit)" + assert expr.render("{symbol.key}") == expected def test_Licensing_can_tokenize_valid_expressions_with_symbols_that_contain_and_with_or(self): licensing = Licensing() - expression = 'orgpl or withbsd with orclasspath and andmit or anlgpl and ormit or withme' + expression = "orgpl or withbsd with orclasspath and andmit or anlgpl and ormit or withme" result = list(licensing.tokenize(expression)) expected = [ - (LicenseSymbol(key='orgpl'), 'orgpl', 0), - (2, 'or', 6), - (LicenseWithExceptionSymbol( - license_symbol=LicenseSymbol(key='withbsd'), - exception_symbol=LicenseSymbol(key='orclasspath')), - 'withbsd with orclasspath', 9), - (1, 'and', 34), - (LicenseSymbol(key='andmit'), 'andmit', 38), - (2, 'or', 45), - (LicenseSymbol(key='anlgpl'), 'anlgpl', 48), - (1, 'and', 55), - (LicenseSymbol(key='ormit'), 'ormit', 59), - (2, 'or', 65), - (LicenseSymbol(key='withme'), 'withme', 68) + (LicenseSymbol(key="orgpl"), "orgpl", 0), + (2, "or", 6), + ( + LicenseWithExceptionSymbol( + license_symbol=LicenseSymbol(key="withbsd"), + exception_symbol=LicenseSymbol(key="orclasspath"), + ), + "withbsd with orclasspath", + 9, + ), + (1, "and", 34), + (LicenseSymbol(key="andmit"), "andmit", 38), + (2, "or", 45), + (LicenseSymbol(key="anlgpl"), "anlgpl", 48), + (1, "and", 55), + (LicenseSymbol(key="ormit"), "ormit", 59), + (2, "or", 65), + (LicenseSymbol(key="withme"), "withme", 68), ] assert result == expected - def test_Licensing_can_simple_tokenize_valid_expressions_with_symbols_that_contain_and_with_or(self): + def test_Licensing_can_simple_tokenize_valid_expressions_with_symbols_that_contain_and_with_or( + self, + ): licensing = Licensing() - expression = 'orgpl or withbsd with orclasspath and andmit or andlgpl and ormit or withme' + expression = "orgpl or withbsd with orclasspath and andmit or andlgpl and ormit or withme" result = [r.string for r in licensing.simple_tokenizer(expression)] expected = [ - 'orgpl', - ' ', - 'or', - ' ', - 'withbsd', - ' ', - 'with', - ' ', - 'orclasspath', - ' ', - 'and', - ' ', - 'andmit', - ' ', - 'or', - ' ', - 'andlgpl', - ' ', - 'and', - ' ', - 'ormit', - ' ', - 'or', - ' ', - 'withme' + "orgpl", + " ", + "or", + " ", + "withbsd", + " ", + "with", + " ", + "orclasspath", + " ", + "and", + " ", + "andmit", + " ", + "or", + " ", + "andlgpl", + " ", + "and", + " ", + "ormit", + " ", + "or", + " ", + "withme", ] assert result == expected def test_Licensing_can_parse_valid_expressions_with_symbols_that_contain_and_with_or(self): licensing = Licensing() - expression = 'orgpl or withbsd with orclasspath and andmit or anlgpl and ormit or withme' + expression = "orgpl or withbsd with orclasspath and andmit or anlgpl and ormit or withme" result = licensing.parse(expression) - expected = 'orgpl OR (withbsd WITH orclasspath AND andmit) OR (anlgpl AND ormit) OR withme' - assert result.render('{symbol.key}') == expected + expected = "orgpl OR (withbsd WITH orclasspath AND andmit) OR (anlgpl AND ormit) OR withme" + assert result.render("{symbol.key}") == expected def test_Licensing_can_parse_valid_expressions_with_symbols_that_contain_spaces(self): licensing = Licensing() - expression = ' GPL-2.0 or (mit and LGPL 2.1) or bsd Or GPL-2.0 or (mit and LGPL 2.1)' + expression = " GPL-2.0 or (mit and LGPL 2.1) or bsd Or GPL-2.0 or (mit and LGPL 2.1)" parsed = licensing.parse(expression) - expected = 'GPL-2.0 OR (mit AND LGPL 2.1) OR bsd OR GPL-2.0 OR (mit AND LGPL 2.1)' + expected = "GPL-2.0 OR (mit AND LGPL 2.1) OR bsd OR GPL-2.0 OR (mit AND LGPL 2.1)" assert str(parsed) == expected def test_parse_invalid_expression_with_trailing_or(self): licensing = Licensing() - expr = 'mit or' + expr = "mit or" try: licensing.parse(expr) self.fail("Exception not raised when validating '%s'" % expr) except ExpressionError as ee: - assert 'OR requires two or more licenses as in: MIT OR BSD' == str( - ee) + assert "OR requires two or more licenses as in: MIT OR BSD" == str(ee) - def test_parse_invalid_expression_with_trailing_or_and_valid_start_does_not_raise_exception(self): + def test_parse_invalid_expression_with_trailing_or_and_valid_start_does_not_raise_exception( + self, + ): licensing = Licensing() - expression = ' mit or mit or ' + expression = " mit or mit or " parsed = licensing.parse(expression) # ExpressionError: OR requires two or more licenses as in: MIT OR BSD - expected = 'mit OR mit' + expected = "mit OR mit" assert str(parsed) == expected def test_parse_invalid_expression_with_repeated_trailing_or_raise_exception(self): licensing = Licensing() - expression = 'mit or mit or or' + expression = "mit or mit or or" try: licensing.parse(expression, simple=False) - self.fail('Exception not raised') + self.fail("Exception not raised") except ParseError as pe: expected = { - 'error_code': PARSE_INVALID_OPERATOR_SEQUENCE, - 'position': 14, - 'token_string': 'or', - 'token_type': TOKEN_OR + "error_code": PARSE_INVALID_OPERATOR_SEQUENCE, + "position": 14, + "token_string": "or", + "token_type": TOKEN_OR, } assert _parse_error_as_dict(pe) == expected def test_parse_invalid_expression_drops_single_trailing_or(self): licensing = Licensing() - expression = 'mit or mit or' + expression = "mit or mit or" e = licensing.parse(expression, simple=False) - assert str(e) == 'mit OR mit' + assert str(e) == "mit OR mit" def test_parse_invalid_expression_drops_single_trailing_or2(self): licensing = Licensing() - expression = 'mit or mit or' + expression = "mit or mit or" e = licensing.parse(expression, simple=True) - assert str(e) == 'mit OR mit' + assert str(e) == "mit OR mit" def test_parse_invalid_expression_with_single_trailing_and_raise_exception(self): licensing = Licensing() - expression = 'mit or mit and' + expression = "mit or mit and" try: licensing.parse(expression, simple=False) - self.fail('Exception not raised') + self.fail("Exception not raised") except ExpressionError as ee: - assert 'AND requires two or more licenses as in: MIT AND BSD' == str( - ee) + assert "AND requires two or more licenses as in: MIT AND BSD" == str(ee) def test_parse_invalid_expression_with_single_leading_or_raise_exception(self): licensing = Licensing() - expression = 'or mit or mit' + expression = "or mit or mit" try: licensing.parse(expression, simple=False) - self.fail('Exception not raised') + self.fail("Exception not raised") except ParseError as pe: expected = { - 'error_code': PARSE_INVALID_OPERATOR_SEQUENCE, - 'position': 0, - 'token_string': 'or', - 'token_type': TOKEN_OR + "error_code": PARSE_INVALID_OPERATOR_SEQUENCE, + "position": 0, + "token_string": "or", + "token_type": TOKEN_OR, } assert _parse_error_as_dict(pe) == expected def test_Licensing_can_parse_expressions_with_symbols_that_contain_a_colon(self): licensing = Licensing() - expression = 'DocumentRef-James-1.0:LicenseRef-Eric-2.0' + expression = "DocumentRef-James-1.0:LicenseRef-Eric-2.0" result = licensing.parse(expression) - expected = 'DocumentRef-James-1.0:LicenseRef-Eric-2.0' - assert result.render('{symbol.key}') == expected + expected = "DocumentRef-James-1.0:LicenseRef-Eric-2.0" + assert result.render("{symbol.key}") == expected class LicensingParseWithSymbolsSimpleTest(TestCase): - def test_Licensing_with_overlapping_symbols_with_keywords_does_not_raise_Exception(self): - Licensing([ - 'GPL-2.0 or LATER', - 'classpath Exception', - 'something with else+', - 'mit', - 'LGPL 2.1', - 'mit or later' - ]) + Licensing( + [ + "GPL-2.0 or LATER", + "classpath Exception", + "something with else+", + "mit", + "LGPL 2.1", + "mit or later", + ] + ) def get_syms_and_licensing(self): - a = LicenseSymbol('l-a') - ap = LicenseSymbol('L-a+', ['l-a +']) - b = LicenseSymbol('l-b') - c = LicenseSymbol('l-c') + a = LicenseSymbol("l-a") + ap = LicenseSymbol("L-a+", ["l-a +"]) + b = LicenseSymbol("l-b") + c = LicenseSymbol("l-c") symbols = [a, ap, b, c] return a, ap, b, c, Licensing(symbols) def test_parse_license_expression1(self): a, _ap, _b, _c, licensing = self.get_syms_and_licensing() - express_string = 'l-a' + express_string = "l-a" result = licensing.parse(express_string) assert express_string == str(result) expected = a @@ -1099,136 +1079,136 @@ def test_parse_license_expression1(self): def test_parse_license_expression_with_alias(self): _a, ap, _b, _c, licensing = self.get_syms_and_licensing() - express_string = 'l-a +' + express_string = "l-a +" result = licensing.parse(express_string) - assert 'L-a+' == str(result) + assert "L-a+" == str(result) expected = ap assert result == expected assert [] == licensing.unknown_license_keys(result) def test_parse_license_expression3(self): _a, ap, _b, _c, licensing = self.get_syms_and_licensing() - express_string = 'l-a+' + express_string = "l-a+" result = licensing.parse(express_string) - assert 'L-a+' == str(result) + assert "L-a+" == str(result) expected = ap assert result == expected assert [] == licensing.unknown_license_keys(result) def test_parse_license_expression4(self): _a, _ap, _b, _c, licensing = self.get_syms_and_licensing() - express_string = '(l-a)' + express_string = "(l-a)" result = licensing.parse(express_string) - assert 'l-a' == str(result) - expected = LicenseSymbol(key='l-a', aliases=()) + assert "l-a" == str(result) + expected = LicenseSymbol(key="l-a", aliases=()) assert result == expected assert [] == licensing.unknown_license_keys(result) def test_parse_license_expression5(self): _a, ap, b, c, licensing = self.get_syms_and_licensing() - express_string = '((l-a+ AND l-b) OR (l-c))' + express_string = "((l-a+ AND l-b) OR (l-c))" result = licensing.parse(express_string) - assert '(L-a+ AND l-b) OR l-c' == str(result) + assert "(L-a+ AND l-b) OR l-c" == str(result) expected = licensing.OR(licensing.AND(ap, b), c) assert result == expected assert [] == licensing.unknown_license_keys(result) def test_parse_license_expression6(self): a, _ap, b, _c, licensing = self.get_syms_and_licensing() - express_string = 'l-a and l-b' + express_string = "l-a and l-b" result = licensing.parse(express_string) - assert 'l-a AND l-b' == str(result) + assert "l-a AND l-b" == str(result) expected = licensing.AND(a, b) assert result == expected assert [] == licensing.unknown_license_keys(result) def test_parse_license_expression7(self): a, _ap, b, _c, licensing = self.get_syms_and_licensing() - express_string = 'l-a or l-b' + express_string = "l-a or l-b" result = licensing.parse(express_string) - assert 'l-a OR l-b' == str(result) + assert "l-a OR l-b" == str(result) expected = licensing.OR(a, b) assert result == expected assert [] == licensing.unknown_license_keys(result) def test_parse_license_expression8(self): a, _ap, b, c, licensing = self.get_syms_and_licensing() - express_string = 'l-a and l-b OR l-c' + express_string = "l-a and l-b OR l-c" result = licensing.parse(express_string) - assert '(l-a AND l-b) OR l-c' == str(result) + assert "(l-a AND l-b) OR l-c" == str(result) expected = licensing.OR(licensing.AND(a, b), c) assert result == expected assert [] == licensing.unknown_license_keys(result) def test_parse_license_expression8_twice(self): _a, _ap, _b, _c, licensing = self.get_syms_and_licensing() - express_string = 'l-a and l-b OR l-c' + express_string = "l-a and l-b OR l-c" result = licensing.parse(express_string) - assert '(l-a AND l-b) OR l-c' == str(result) + assert "(l-a AND l-b) OR l-c" == str(result) # there was some issues with reusing a Licensing result = licensing.parse(express_string) - assert '(l-a AND l-b) OR l-c' == str(result) + assert "(l-a AND l-b) OR l-c" == str(result) def test_parse_license_expression_with_trailing_space_plus(self): symbols = [ - LicenseSymbol('l-a'), - LicenseSymbol('L-a+', ['l-a +']), - LicenseSymbol('l-b'), - LicenseSymbol('l-c'), + LicenseSymbol("l-a"), + LicenseSymbol("L-a+", ["l-a +"]), + LicenseSymbol("l-b"), + LicenseSymbol("l-c"), ] licensing = Licensing(symbols) - expresssion_str = 'l-a' + expresssion_str = "l-a" result = licensing.parse(expresssion_str) assert str(result) == expresssion_str assert licensing.unknown_license_keys(result) == [] # plus sign is not attached to the symbol, but an alias - expresssion_str = 'l-a +' + expresssion_str = "l-a +" result = licensing.parse(expresssion_str) - assert str(result).lower() == 'l-a+' + assert str(result).lower() == "l-a+" assert licensing.unknown_license_keys(result) == [] - expresssion_str = '(l-a)' + expresssion_str = "(l-a)" result = licensing.parse(expresssion_str) - assert str(result).lower() == 'l-a' + assert str(result).lower() == "l-a" assert licensing.unknown_license_keys(result) == [] - expresssion_str = '((l-a+ AND l-b) OR (l-c))' + expresssion_str = "((l-a+ AND l-b) OR (l-c))" result = licensing.parse(expresssion_str) - assert str(result) == '(L-a+ AND l-b) OR l-c' + assert str(result) == "(L-a+ AND l-b) OR l-c" assert licensing.unknown_license_keys(result) == [] - expresssion_str = 'l-a and l-b' + expresssion_str = "l-a and l-b" result = licensing.parse(expresssion_str) - assert str(result) == 'l-a AND l-b' + assert str(result) == "l-a AND l-b" assert licensing.unknown_license_keys(result) == [] - expresssion_str = 'l-a or l-b' + expresssion_str = "l-a or l-b" result = licensing.parse(expresssion_str) - assert str(result) == 'l-a OR l-b' + assert str(result) == "l-a OR l-b" assert licensing.unknown_license_keys(result) == [] - expresssion_str = 'l-a and l-b OR l-c' + expresssion_str = "l-a and l-b OR l-c" result = licensing.parse(expresssion_str) - assert str(result) == '(l-a AND l-b) OR l-c' + assert str(result) == "(l-a AND l-b) OR l-c" assert licensing.unknown_license_keys(result) == [] def test_parse_of_side_by_side_symbols_raise_exception(self): - gpl2 = LicenseSymbol('gpl') + gpl2 = LicenseSymbol("gpl") l = Licensing([gpl2]) try: - l.parse('gpl mit') - self.fail('ParseError not raised') + l.parse("gpl mit") + self.fail("ParseError not raised") except ParseError: pass def test_validate_symbols(self): symbols = [ - LicenseSymbol('l-a', is_exception=True), - LicenseSymbol('l-a'), - LicenseSymbol('l-b'), - LicenseSymbol('l-c'), + LicenseSymbol("l-a", is_exception=True), + LicenseSymbol("l-a"), + LicenseSymbol("l-b"), + LicenseSymbol("l-c"), ] warnings, errors = validate_symbols(symbols) @@ -1242,84 +1222,91 @@ def test_validate_symbols(self): class LicensingParseWithSymbolsTest(TestCase): - def test_parse_raise_ParseError_when_validating_strict_with_non_exception_symbols(self): - licensing = Licensing(['gpl', 'bsd', 'lgpl', 'exception']) + licensing = Licensing(["gpl", "bsd", "lgpl", "exception"]) - expression = 'gpl and bsd or lgpl with exception' + expression = "gpl and bsd or lgpl with exception" try: licensing.parse(expression, validate=True, strict=True) - self.fail('Exception not raised') + self.fail("Exception not raised") except ParseError as pe: expected = { - 'error_code': PARSE_INVALID_SYMBOL_AS_EXCEPTION, - 'position': 25, - 'token_string': 'exception', - 'token_type': TOKEN_SYMBOL} + "error_code": PARSE_INVALID_SYMBOL_AS_EXCEPTION, + "position": 25, + "token_string": "exception", + "token_type": TOKEN_SYMBOL, + } assert _parse_error_as_dict(pe) == expected - def test_parse_raise_ParseError_when_validating_strict_with_exception_symbols_in_incorrect_spot(self): - licensing = Licensing([LicenseSymbol('gpl', is_exception=False), - LicenseSymbol('exception', is_exception=True)]) - licensing.parse('gpl with exception', validate=True, strict=True) + def test_parse_raise_ParseError_when_validating_strict_with_exception_symbols_in_incorrect_spot( + self, + ): + licensing = Licensing( + [ + LicenseSymbol("gpl", is_exception=False), + LicenseSymbol("exception", is_exception=True), + ] + ) + licensing.parse("gpl with exception", validate=True, strict=True) try: - licensing.parse('exception with gpl', validate=True, strict=True) - self.fail('Exception not raised') + licensing.parse("exception with gpl", validate=True, strict=True) + self.fail("Exception not raised") except ParseError as pe: expected = { - 'error_code': PARSE_INVALID_EXCEPTION, - 'position': 0, - 'token_string': 'exception', - 'token_type': TOKEN_SYMBOL} + "error_code": PARSE_INVALID_EXCEPTION, + "position": 0, + "token_string": "exception", + "token_type": TOKEN_SYMBOL, + } assert _parse_error_as_dict(pe) == expected try: - licensing.parse('gpl with gpl', validate=True, strict=True) - self.fail('Exception not raised') + licensing.parse("gpl with gpl", validate=True, strict=True) + self.fail("Exception not raised") except ParseError as pe: expected = { - 'error_code': PARSE_INVALID_SYMBOL_AS_EXCEPTION, - 'position': 9, - 'token_string': 'gpl', - 'token_type': TOKEN_SYMBOL} + "error_code": PARSE_INVALID_SYMBOL_AS_EXCEPTION, + "position": 9, + "token_string": "gpl", + "token_type": TOKEN_SYMBOL, + } assert _parse_error_as_dict(pe) == expected def test_with_unknown_symbol_string_contained_in_known_symbol_does_not_crash_with(self): - l = Licensing(['lgpl-3.0-plus']) - license_expression = 'lgpl-3.0-plus WITH openssl-exception-lgpl-3.0-plus' + l = Licensing(["lgpl-3.0-plus"]) + license_expression = "lgpl-3.0-plus WITH openssl-exception-lgpl-3.0-plus" l.parse(license_expression) def test_with_unknown_symbol_string_contained_in_known_symbol_does_not_crash_and(self): - l = Licensing(['lgpl-3.0-plus']) - license_expression = 'lgpl-3.0-plus AND openssl-exception-lgpl-3.0-plus' + l = Licensing(["lgpl-3.0-plus"]) + license_expression = "lgpl-3.0-plus AND openssl-exception-lgpl-3.0-plus" l.parse(license_expression) def test_with_unknown_symbol_string_contained_in_known_symbol_does_not_crash_or(self): - l = Licensing(['lgpl-3.0-plus']) - license_expression = 'lgpl-3.0-plus OR openssl-exception-lgpl-3.0-plus' + l = Licensing(["lgpl-3.0-plus"]) + license_expression = "lgpl-3.0-plus OR openssl-exception-lgpl-3.0-plus" l.parse(license_expression) def test_with_known_symbol_string_contained_in_known_symbol_does_not_crash_or(self): - l = Licensing(['lgpl-3.0-plus', 'openssl-exception-lgpl-3.0-plus']) - license_expression = 'lgpl-3.0-plus OR openssl-exception-lgpl-3.0-plus' + l = Licensing(["lgpl-3.0-plus", "openssl-exception-lgpl-3.0-plus"]) + license_expression = "lgpl-3.0-plus OR openssl-exception-lgpl-3.0-plus" l.parse(license_expression) def test_with_known_symbol_string_contained_in_known_symbol_does_not_crash_with(self): - l = Licensing(['lgpl-3.0-plus', 'openssl-exception-lgpl-3.0-plus']) - license_expression = 'lgpl-3.0-plus WITH openssl-exception-lgpl-3.0-plus' + l = Licensing(["lgpl-3.0-plus", "openssl-exception-lgpl-3.0-plus"]) + license_expression = "lgpl-3.0-plus WITH openssl-exception-lgpl-3.0-plus" l.parse(license_expression) class LicensingSymbolsReplacement(TestCase): - def get_symbols_and_licensing(self): - gpl2 = LicenseSymbol( - 'gpl-2.0', ['The GNU GPL 20', 'GPL-2.0', 'GPL v2.0']) + gpl2 = LicenseSymbol("gpl-2.0", ["The GNU GPL 20", "GPL-2.0", "GPL v2.0"]) gpl2plus = LicenseSymbol( - 'gpl-2.0+', ['The GNU GPL 20 or later', 'GPL-2.0 or later', 'GPL v2.0 or later']) - lgpl = LicenseSymbol('LGPL-2.1', ['LGPL v2.1']) - mit = LicenseSymbol('MIT', ['MIT license']) - mitand2 = LicenseSymbol('mitand2', ['mitand2', 'mitand2 license']) + "gpl-2.0+", ["The GNU GPL 20 or later", "GPL-2.0 or later", "GPL v2.0 or later"] + ) + lgpl = LicenseSymbol("LGPL-2.1", ["LGPL v2.1"]) + mit = LicenseSymbol("MIT", ["MIT license"]) + mitand2 = LicenseSymbol("mitand2", ["mitand2", "mitand2 license"]) symbols = [gpl2, gpl2plus, lgpl, mit, mitand2] licensing = Licensing(symbols) return gpl2, gpl2plus, lgpl, mit, mitand2, licensing @@ -1328,208 +1315,217 @@ def test_simple_substitution(self): gpl2, gpl2plus, _lgpl, _mit, _mitand2, licensing = self.get_symbols_and_licensing() subs = {gpl2plus: gpl2} - expr = licensing.parse('gpl-2.0 or gpl-2.0+') + expr = licensing.parse("gpl-2.0 or gpl-2.0+") result = expr.subs(subs) - assert 'gpl-2.0 OR gpl-2.0' == result.render() + assert "gpl-2.0 OR gpl-2.0" == result.render() def test_advanced_substitution(self): _gpl2, _gpl2plus, lgpl, _mit, _mitand2, licensing = self.get_symbols_and_licensing() - source = licensing.parse('gpl-2.0+ and mit') + source = licensing.parse("gpl-2.0+ and mit") target = lgpl subs = {source: target} - expr = licensing.parse('gpl-2.0 or gpl-2.0+ and mit') + expr = licensing.parse("gpl-2.0 or gpl-2.0+ and mit") result = expr.subs(subs) - assert 'gpl-2.0 OR LGPL-2.1' == result.render() + assert "gpl-2.0 OR LGPL-2.1" == result.render() def test_multiple_substitutions(self): gpl2, gpl2plus, lgpl, mit, _mitand2, licensing = self.get_symbols_and_licensing() - source1 = licensing.parse('gpl-2.0+ and mit') + source1 = licensing.parse("gpl-2.0+ and mit") target1 = lgpl - source2 = licensing.parse('mitand2') + source2 = licensing.parse("mitand2") target2 = mit source3 = gpl2 target3 = gpl2plus - subs = dict([ - (source1, target1), - (source2, target2), - (source3, target3), - ]) + subs = dict( + [ + (source1, target1), + (source2, target2), + (source3, target3), + ] + ) - expr = licensing.parse('gpl-2.0 or gpl-2.0+ and mit') + expr = licensing.parse("gpl-2.0 or gpl-2.0+ and mit") # step 1: yields 'gpl-2.0 or lgpl' # step 2: yields 'gpl-2.0+ or LGPL-2.1' result = expr.subs(subs) - assert 'gpl-2.0+ OR LGPL-2.1' == result.render() + assert "gpl-2.0+ OR LGPL-2.1" == result.render() def test_multiple_substitutions_complex(self): gpl2, gpl2plus, lgpl, mit, _mitand2, licensing = self.get_symbols_and_licensing() - source1 = licensing.parse('gpl-2.0+ and mit') + source1 = licensing.parse("gpl-2.0+ and mit") target1 = lgpl - source2 = licensing.parse('mitand2') + source2 = licensing.parse("mitand2") target2 = mit source3 = gpl2 target3 = gpl2plus - subs = dict([ - (source1, target1), - (source2, target2), - (source3, target3), - ]) + subs = dict( + [ + (source1, target1), + (source2, target2), + (source3, target3), + ] + ) - expr = licensing.parse( - '(gpl-2.0 or gpl-2.0+ and mit) and (gpl-2.0 or gpl-2.0+ and mit)') + expr = licensing.parse("(gpl-2.0 or gpl-2.0+ and mit) and (gpl-2.0 or gpl-2.0+ and mit)") # step 1: yields 'gpl-2.0 or lgpl' # step 2: yields 'gpl-2.0+ or LGPL-2.1' result = expr.subs(subs) - assert '(gpl-2.0+ OR LGPL-2.1) AND (gpl-2.0+ OR LGPL-2.1)' == result.render() + assert "(gpl-2.0+ OR LGPL-2.1) AND (gpl-2.0+ OR LGPL-2.1)" == result.render() - expr = licensing.parse( - '(gpl-2.0 or mit and gpl-2.0+) and (gpl-2.0 or gpl-2.0+ and mit)') + expr = licensing.parse("(gpl-2.0 or mit and gpl-2.0+) and (gpl-2.0 or gpl-2.0+ and mit)") # step 1: yields 'gpl-2.0 or lgpl' # step 2: yields 'gpl-2.0+ or LGPL-2.1' result = expr.subs(subs) - assert '(gpl-2.0+ OR LGPL-2.1) AND (gpl-2.0+ OR LGPL-2.1)' == result.render() + assert "(gpl-2.0+ OR LGPL-2.1) AND (gpl-2.0+ OR LGPL-2.1)" == result.render() class LicensingParseWithSymbolsAdvancedTest(TestCase): - def get_symbols_and_licensing(self): - gpl2 = LicenseSymbol( - 'gpl-2.0', ['The GNU GPL 20', 'GPL-2.0', 'GPL v2.0']) + gpl2 = LicenseSymbol("gpl-2.0", ["The GNU GPL 20", "GPL-2.0", "GPL v2.0"]) gpl2plus = LicenseSymbol( - 'gpl-2.0+', ['The GNU GPL 20 or later', 'GPL-2.0 or later', 'GPL v2.0 or later']) - lgpl = LicenseSymbol('LGPL-2.1', ['LGPL v2.1']) - mit = LicenseSymbol('MIT', ['MIT license']) - mitand2 = LicenseSymbol('mitand2', ['mitand2', 'mitand2 license']) + "gpl-2.0+", ["The GNU GPL 20 or later", "GPL-2.0 or later", "GPL v2.0 or later"] + ) + lgpl = LicenseSymbol("LGPL-2.1", ["LGPL v2.1"]) + mit = LicenseSymbol("MIT", ["MIT license"]) + mitand2 = LicenseSymbol("mitand2", ["mitand2", "mitand2 license"]) symbols = [gpl2, gpl2plus, lgpl, mit, mitand2] licensing = Licensing(symbols) return gpl2, gpl2plus, lgpl, mit, mitand2, licensing def test_parse_trailing_char_does_not_raise_exception_without_validate(self): _gpl2, _gpl2plus, _lgpl, _mit, _mitand2, licensing = self.get_symbols_and_licensing() - e = licensing.parse( - 'The GNU GPL 20 or LGPL-2.1 and mit2', validate=False) - assert 'gpl-2.0 OR (LGPL-2.1 AND mit2)' == str(e) + e = licensing.parse("The GNU GPL 20 or LGPL-2.1 and mit2", validate=False) + assert "gpl-2.0 OR (LGPL-2.1 AND mit2)" == str(e) def test_parse_trailing_char_raise_exception_with_validate(self): _gpl2, _gpl2plus, _lgpl, _mit, _mitand2, licensing = self.get_symbols_and_licensing() try: - licensing.parse( - 'The GNU GPL 20 or LGPL-2.1 and mit2', validate=True) - self.fail('Exception not raised') + licensing.parse("The GNU GPL 20 or LGPL-2.1 and mit2", validate=True) + self.fail("Exception not raised") except ExpressionError as ee: - assert 'Unknown license key(s): mit2' == str(ee) + assert "Unknown license key(s): mit2" == str(ee) def test_parse_expression_with_trailing_unknown_should_raise_exception(self): gpl2, gpl2plus, lgpl, mit, _mitand2, licensing = self.get_symbols_and_licensing() - unknown = LicenseSymbol(key='123') + unknown = LicenseSymbol(key="123") - tokens = list(licensing.tokenize( - 'The GNU GPL 20 or later or (LGPL-2.1 and mit) or The GNU GPL 20 or mit 123')) + tokens = list( + licensing.tokenize( + "The GNU GPL 20 or later or (LGPL-2.1 and mit) or The GNU GPL 20 or mit 123" + ) + ) expected = [ - (gpl2plus, 'The GNU GPL 20 or later', 0), - (TOKEN_OR, 'or', 24), - (TOKEN_LPAR, '(', 27), - (lgpl, 'LGPL-2.1', 28), - (TOKEN_AND, 'and', 37), - (mit, 'mit', 41), - (TOKEN_RPAR, ')', 44), - (TOKEN_OR, 'or', 46), - (gpl2, 'The GNU GPL 20', 49), - (TOKEN_OR, 'or', 64), - (mit, 'mit', 67), - (unknown, '123', 71) + (gpl2plus, "The GNU GPL 20 or later", 0), + (TOKEN_OR, "or", 24), + (TOKEN_LPAR, "(", 27), + (lgpl, "LGPL-2.1", 28), + (TOKEN_AND, "and", 37), + (mit, "mit", 41), + (TOKEN_RPAR, ")", 44), + (TOKEN_OR, "or", 46), + (gpl2, "The GNU GPL 20", 49), + (TOKEN_OR, "or", 64), + (mit, "mit", 67), + (unknown, "123", 71), ] assert tokens == expected try: licensing.parse( - 'The GNU GPL 20 or later or (LGPL-2.1 and mit) or The GNU GPL 20 or mit 123') - self.fail('Exception not raised') + "The GNU GPL 20 or later or (LGPL-2.1 and mit) or The GNU GPL 20 or mit 123" + ) + self.fail("Exception not raised") except ParseError as pe: - expected = {'error_code': PARSE_INVALID_SYMBOL_SEQUENCE, 'position': 71, - 'token_string': '123', 'token_type': unknown} + expected = { + "error_code": PARSE_INVALID_SYMBOL_SEQUENCE, + "position": 71, + "token_string": "123", + "token_type": unknown, + } assert _parse_error_as_dict(pe) == expected def test_parse_expression_with_trailing_unknown_should_raise_exception2(self): _gpl2, _gpl2_plus, _lgpl, _mit, _mitand2, licensing = self.get_symbols_and_licensing() - unknown = LicenseSymbol(key='123') + unknown = LicenseSymbol(key="123") try: - licensing.parse('The GNU GPL 20 or mit 123') + licensing.parse("The GNU GPL 20 or mit 123") # 01234567890123456789012345 - self.fail('Exception not raised') + self.fail("Exception not raised") except ParseError as pe: - expected = {'error_code': PARSE_INVALID_SYMBOL_SEQUENCE, 'position': 22, - 'token_string': '123', 'token_type': unknown} + expected = { + "error_code": PARSE_INVALID_SYMBOL_SEQUENCE, + "position": 22, + "token_string": "123", + "token_type": unknown, + } assert _parse_error_as_dict(pe) == expected def test_parse_expression_with_WITH(self): gpl2, _gpl2plus, lgpl, mit, mitand2, _ = self.get_symbols_and_licensing() - mitexp = LicenseSymbol('mitexp', ('mit exp',), is_exception=True) - gpl_20_or_later = LicenseSymbol( - 'GPL-2.0+', ['The GNU GPL 20 or later']) + mitexp = LicenseSymbol("mitexp", ("mit exp",), is_exception=True) + gpl_20_or_later = LicenseSymbol("GPL-2.0+", ["The GNU GPL 20 or later"]) symbols = [gpl2, lgpl, mit, mitand2, mitexp, gpl_20_or_later] licensing = Licensing(symbols) - expr = 'The GNU GPL 20 or later or (LGPL-2.1 and mit) or The GNU GPL 20 or mit with mit exp' + expr = "The GNU GPL 20 or later or (LGPL-2.1 and mit) or The GNU GPL 20 or mit with mit exp" tokens = list(licensing.tokenize(expr)) expected = [ - (gpl_20_or_later, 'The GNU GPL 20 or later', 0), - (TOKEN_OR, 'or', 24), - (TOKEN_LPAR, '(', 27), - (lgpl, 'LGPL-2.1', 28), - (TOKEN_AND, 'and', 37), - (mit, 'mit', 41), - (TOKEN_RPAR, ')', 44), - (TOKEN_OR, 'or', 46), - (gpl2, 'The GNU GPL 20', 49), - (TOKEN_OR, 'or', 64), - (LicenseWithExceptionSymbol(mit, mitexp), 'mit with mit exp', 67) + (gpl_20_or_later, "The GNU GPL 20 or later", 0), + (TOKEN_OR, "or", 24), + (TOKEN_LPAR, "(", 27), + (lgpl, "LGPL-2.1", 28), + (TOKEN_AND, "and", 37), + (mit, "mit", 41), + (TOKEN_RPAR, ")", 44), + (TOKEN_OR, "or", 46), + (gpl2, "The GNU GPL 20", 49), + (TOKEN_OR, "or", 64), + (LicenseWithExceptionSymbol(mit, mitexp), "mit with mit exp", 67), ] assert tokens == expected parsed = licensing.parse(expr) - expected = 'GPL-2.0+ OR (LGPL-2.1 AND MIT) OR gpl-2.0 OR MIT WITH mitexp' + expected = "GPL-2.0+ OR (LGPL-2.1 AND MIT) OR gpl-2.0 OR MIT WITH mitexp" assert str(parsed) == expected - expected = 'GPL-2.0+ OR (LGPL-2.1 AND MIT) OR gpl-2.0 OR MIT WITH mitexp' + expected = "GPL-2.0+ OR (LGPL-2.1 AND MIT) OR gpl-2.0 OR MIT WITH mitexp" assert parsed.render() == expected def test_parse_expression_with_WITH_and_unknown_symbol(self): gpl2, _gpl2plus, lgpl, mit, mitand2, _ = self.get_symbols_and_licensing() - mitexp = LicenseSymbol('mitexp', ('mit exp',), is_exception=True) - gpl_20_or_later = LicenseSymbol( - 'GPL-2.0+', ['The GNU GPL 20 or later']) + mitexp = LicenseSymbol("mitexp", ("mit exp",), is_exception=True) + gpl_20_or_later = LicenseSymbol("GPL-2.0+", ["The GNU GPL 20 or later"]) symbols = [gpl2, lgpl, mit, mitand2, mitexp, gpl_20_or_later] licensing = Licensing(symbols) - expr = 'The GNU GPL 20 or later or (LGPL-2.1 and mit) or The GNU GPL 20 or mit with 123' + expr = "The GNU GPL 20 or later or (LGPL-2.1 and mit) or The GNU GPL 20 or mit with 123" parsed = licensing.parse(expr) - assert ['123'] == licensing.unknown_license_keys(parsed) - assert ['123'] == licensing.unknown_license_keys(expr) + assert ["123"] == licensing.unknown_license_keys(parsed) + assert ["123"] == licensing.unknown_license_keys(expr) def test_unknown_keys(self): _gpl2, _gpl2plus, _lgpl, _mit, _mitand2, licensing = self.get_symbols_and_licensing() - expr = 'The GNU GPL 20 or LGPL-2.1 and mit' + expr = "The GNU GPL 20 or LGPL-2.1 and mit" parsed = licensing.parse(expr) - expected = 'gpl-2.0 OR (LGPL-2.1 AND MIT)' + expected = "gpl-2.0 OR (LGPL-2.1 AND MIT)" assert str(parsed) == expected - assert 'gpl-2.0 OR (LGPL-2.1 AND MIT)' == parsed.render('{symbol.key}') + assert "gpl-2.0 OR (LGPL-2.1 AND MIT)" == parsed.render("{symbol.key}") assert [] == licensing.unknown_license_keys(parsed) assert [] == licensing.unknown_license_keys(expr) def test_unknown_keys_with_trailing_char(self): gpl2, _gpl2plus, lgpl, _mit, mitand2, licensing = self.get_symbols_and_licensing() - expr = 'The GNU GPL 20 or LGPL-2.1 and mitand2' + expr = "The GNU GPL 20 or LGPL-2.1 and mitand2" parsed = licensing.parse(expr) expected = [gpl2, lgpl, mitand2] assert licensing.license_symbols(parsed) == expected @@ -1540,596 +1536,628 @@ def test_unknown_keys_with_trailing_char(self): def test_unknown_keys_with_trailing_char_2_with_validate(self): _gpl2, _gpl2plus, _lgpl, _mit, _mitand2, licensing = self.get_symbols_and_licensing() - expr = 'The GNU GPL 20 or LGPL-2.1 and mitand3' + expr = "The GNU GPL 20 or LGPL-2.1 and mitand3" try: licensing.parse(expr, validate=True) - self.fail('Exception should be raised') + self.fail("Exception should be raised") except ExpressionError as ee: - assert 'Unknown license key(s): mitand3' == str(ee) + assert "Unknown license key(s): mitand3" == str(ee) def test_unknown_keys_with_trailing_char_2_without_validate(self): _gpl2, _gpl2plus, _lgpl, _mit, _mitand2, licensing = self.get_symbols_and_licensing() - expr = 'The GNU GPL 20 or LGPL-2.1 and mitand3' + expr = "The GNU GPL 20 or LGPL-2.1 and mitand3" parsed = licensing.parse(expr, validate=False) - assert 'gpl-2.0 OR (LGPL-2.1 AND mitand3)' == str(parsed) + assert "gpl-2.0 OR (LGPL-2.1 AND mitand3)" == str(parsed) def test_parse_with_overlapping_key_without_symbols(self): - expression = 'mit or mit AND zlib or mit or mit with verylonglicense' + expression = "mit or mit AND zlib or mit or mit with verylonglicense" # 1111111111222222222233333333334444444444555555555566666 # 0123456789012345678901234567890123456789012345678901234 licensing = Licensing() results = str(licensing.parse(expression)) - expected = 'mit OR (mit AND zlib) OR mit OR mit WITH verylonglicense' + expected = "mit OR (mit AND zlib) OR mit OR mit WITH verylonglicense" assert results == expected - def test_advanced_tokenizer_tokenize_with_overlapping_key_with_symbols_and_trailing_unknown(self): - expression = 'mit or mit AND zlib or mit or mit with verylonglicense' + def test_advanced_tokenizer_tokenize_with_overlapping_key_with_symbols_and_trailing_unknown( + self, + ): + expression = "mit or mit AND zlib or mit or mit with verylonglicense" # 111111111122222222223333333333444444444455555 # 0123456789012345678901234567890123456789012345678901234 symbols = [ - LicenseSymbol('MIT', ['MIT license']), - LicenseSymbol('LGPL-2.1', ['LGPL v2.1']), - LicenseSymbol('zlib', ['zlib']), - LicenseSymbol('d-zlib', ['D zlib']), - LicenseSymbol('mito', ['mit o']), - LicenseSymbol('hmit', ['h verylonglicense']), + LicenseSymbol("MIT", ["MIT license"]), + LicenseSymbol("LGPL-2.1", ["LGPL v2.1"]), + LicenseSymbol("zlib", ["zlib"]), + LicenseSymbol("d-zlib", ["D zlib"]), + LicenseSymbol("mito", ["mit o"]), + LicenseSymbol("hmit", ["h verylonglicense"]), ] licensing = Licensing(symbols) results = list(licensing.get_advanced_tokenizer().tokenize(expression)) expected = [ - Token(0, 2, 'mit', LicenseSymbol( - u'MIT', aliases=(u'MIT license',))), - Token(4, 5, 'or', Keyword(value=u'or', type=2)), - Token(7, 9, 'mit', LicenseSymbol( - u'MIT', aliases=(u'MIT license',))), - Token(11, 13, 'AND', Keyword(value=u'and', type=1)), - Token(15, 18, 'zlib', LicenseSymbol(u'zlib', aliases=(u'zlib',))), - Token(20, 21, 'or', Keyword(value=u'or', type=2)), - Token(23, 25, 'mit', LicenseSymbol( - u'MIT', aliases=(u'MIT license',))), - Token(27, 28, 'or', Keyword(value=u'or', type=2)), - Token(30, 32, 'mit', LicenseSymbol( - u'MIT', aliases=(u'MIT license',))), - Token(34, 37, 'with', Keyword(value=u'with', type=10)), - Token(39, 53, 'verylonglicense', None), + Token(0, 2, "mit", LicenseSymbol("MIT", aliases=("MIT license",))), + Token(4, 5, "or", Keyword(value="or", type=2)), + Token(7, 9, "mit", LicenseSymbol("MIT", aliases=("MIT license",))), + Token(11, 13, "AND", Keyword(value="and", type=1)), + Token(15, 18, "zlib", LicenseSymbol("zlib", aliases=("zlib",))), + Token(20, 21, "or", Keyword(value="or", type=2)), + Token(23, 25, "mit", LicenseSymbol("MIT", aliases=("MIT license",))), + Token(27, 28, "or", Keyword(value="or", type=2)), + Token(30, 32, "mit", LicenseSymbol("MIT", aliases=("MIT license",))), + Token(34, 37, "with", Keyword(value="with", type=10)), + Token(39, 53, "verylonglicense", None), ] assert results == expected def test_advanced_tokenizer_iter_with_overlapping_key_with_symbols_and_trailing_unknown(self): - expression = 'mit or mit AND zlib or mit or mit with verylonglicense' + expression = "mit or mit AND zlib or mit or mit with verylonglicense" # 111111111122222222223333333333444444444455555 # 0123456789012345678901234567890123456789012345678901234 symbols = [ - LicenseSymbol('MIT', ['MIT license']), - LicenseSymbol('LGPL-2.1', ['LGPL v2.1']), - LicenseSymbol('zlib', ['zlib']), - LicenseSymbol('d-zlib', ['D zlib']), - LicenseSymbol('mito', ['mit o']), - LicenseSymbol('hmit', ['h verylonglicense']), + LicenseSymbol("MIT", ["MIT license"]), + LicenseSymbol("LGPL-2.1", ["LGPL v2.1"]), + LicenseSymbol("zlib", ["zlib"]), + LicenseSymbol("d-zlib", ["D zlib"]), + LicenseSymbol("mito", ["mit o"]), + LicenseSymbol("hmit", ["h verylonglicense"]), ] licensing = Licensing(symbols) - results = list(licensing.get_advanced_tokenizer().iter( - expression, include_unmatched=True)) + results = list(licensing.get_advanced_tokenizer().iter(expression, include_unmatched=True)) expected = [ - Token(0, 2, 'mit', LicenseSymbol( - u'MIT', aliases=(u'MIT license',))), - Token(4, 5, 'or', Keyword(value=u'or', type=2)), - Token(7, 9, 'mit', LicenseSymbol( - u'MIT', aliases=(u'MIT license',))), - Token(11, 13, 'AND', Keyword(value=u'and', type=1)), - Token(15, 18, 'zlib', LicenseSymbol(u'zlib', aliases=(u'zlib',))), - Token(20, 21, 'or', Keyword(value=u'or', type=2)), - Token(23, 25, 'mit', LicenseSymbol( - u'MIT', aliases=(u'MIT license',))), - Token(27, 28, 'or', Keyword(value=u'or', type=2)), - Token(30, 32, 'mit', LicenseSymbol( - u'MIT', aliases=(u'MIT license',))), - Token(34, 37, 'with', Keyword(value=u'with', type=10)), - Token(39, 53, 'verylonglicense', None), + Token(0, 2, "mit", LicenseSymbol("MIT", aliases=("MIT license",))), + Token(4, 5, "or", Keyword(value="or", type=2)), + Token(7, 9, "mit", LicenseSymbol("MIT", aliases=("MIT license",))), + Token(11, 13, "AND", Keyword(value="and", type=1)), + Token(15, 18, "zlib", LicenseSymbol("zlib", aliases=("zlib",))), + Token(20, 21, "or", Keyword(value="or", type=2)), + Token(23, 25, "mit", LicenseSymbol("MIT", aliases=("MIT license",))), + Token(27, 28, "or", Keyword(value="or", type=2)), + Token(30, 32, "mit", LicenseSymbol("MIT", aliases=("MIT license",))), + Token(34, 37, "with", Keyword(value="with", type=10)), + Token(39, 53, "verylonglicense", None), ] assert results == expected def test_advanced_tokenizer_iter_with_overlapping_key_with_symbols_and_trailing_unknown2(self): - expression = 'mit with verylonglicense' + expression = "mit with verylonglicense" symbols = [ - LicenseSymbol('MIT', ['MIT license']), - LicenseSymbol('hmit', ['h verylonglicense']), + LicenseSymbol("MIT", ["MIT license"]), + LicenseSymbol("hmit", ["h verylonglicense"]), ] licensing = Licensing(symbols) - results = list(licensing.get_advanced_tokenizer().iter( - expression, include_unmatched=True)) + results = list(licensing.get_advanced_tokenizer().iter(expression, include_unmatched=True)) expected = [ - Token(0, 2, 'mit', LicenseSymbol( - u'MIT', aliases=(u'MIT license',))), - Token(4, 7, 'with', Keyword(value=u'with', type=10)), - Token(9, 23, 'verylonglicense', None), + Token(0, 2, "mit", LicenseSymbol("MIT", aliases=("MIT license",))), + Token(4, 7, "with", Keyword(value="with", type=10)), + Token(9, 23, "verylonglicense", None), ] assert results == expected def test_tokenize_with_overlapping_key_with_symbols_and_trailing_unknown(self): - expression = 'mit or mit AND zlib or mit or mit with verylonglicense' + expression = "mit or mit AND zlib or mit or mit with verylonglicense" # 1111111111222222222233333333334444444444555555555566666 # 0123456789012345678901234567890123456789012345678901234 symbols = [ - LicenseSymbol('MIT', ['MIT license']), - LicenseSymbol('LGPL-2.1', ['LGPL v2.1']), - LicenseSymbol('zlib', ['zlib']), - LicenseSymbol('d-zlib', ['D zlib']), - LicenseSymbol('mito', ['mit o']), - LicenseSymbol('hmit', ['h verylonglicense']), + LicenseSymbol("MIT", ["MIT license"]), + LicenseSymbol("LGPL-2.1", ["LGPL v2.1"]), + LicenseSymbol("zlib", ["zlib"]), + LicenseSymbol("d-zlib", ["D zlib"]), + LicenseSymbol("mito", ["mit o"]), + LicenseSymbol("hmit", ["h verylonglicense"]), ] licensing = Licensing(symbols) results = list(licensing.tokenize(expression)) expected = [ - (LicenseSymbol(u'MIT', aliases=(u'MIT license',)), 'mit', 0), - (2, 'or', 4), - (LicenseSymbol(u'MIT', aliases=(u'MIT license',)), 'mit', 7), - (1, 'AND', 11), - (LicenseSymbol(u'zlib', aliases=(u'zlib',)), 'zlib', 15), - (2, 'or', 20), - (LicenseSymbol(u'MIT', aliases=(u'MIT license',)), 'mit', 23), - (2, 'or', 27), - (LicenseWithExceptionSymbol( - license_symbol=LicenseSymbol( - u'MIT', aliases=(u'MIT license',)), - exception_symbol=LicenseSymbol(u'verylonglicense')), 'mit with verylonglicense', - 30) + (LicenseSymbol("MIT", aliases=("MIT license",)), "mit", 0), + (2, "or", 4), + (LicenseSymbol("MIT", aliases=("MIT license",)), "mit", 7), + (1, "AND", 11), + (LicenseSymbol("zlib", aliases=("zlib",)), "zlib", 15), + (2, "or", 20), + (LicenseSymbol("MIT", aliases=("MIT license",)), "mit", 23), + (2, "or", 27), + ( + LicenseWithExceptionSymbol( + license_symbol=LicenseSymbol("MIT", aliases=("MIT license",)), + exception_symbol=LicenseSymbol("verylonglicense"), + ), + "mit with verylonglicense", + 30, + ), ] assert results == expected results = str(licensing.parse(expression)) - expected = 'MIT OR (MIT AND zlib) OR MIT OR MIT WITH verylonglicense' + expected = "MIT OR (MIT AND zlib) OR MIT OR MIT WITH verylonglicense" assert results == expected class LicensingSymbolsTest(TestCase): - def test_get_license_symbols(self): - symbols = [ - LicenseSymbol('GPL-2.0'), - LicenseSymbol('mit'), - LicenseSymbol('LGPL 2.1') - ] + symbols = [LicenseSymbol("GPL-2.0"), LicenseSymbol("mit"), LicenseSymbol("LGPL 2.1")] l = Licensing(symbols) - assert symbols == l.license_symbols( - l.parse(' GPL-2.0 and mit or LGPL 2.1 and mit ')) + assert symbols == l.license_symbols(l.parse(" GPL-2.0 and mit or LGPL 2.1 and mit ")) def test_get_license_symbols2(self): symbols = [ - LicenseSymbol('GPL-2.0'), - LicenseSymbol('LATER'), - LicenseSymbol('mit'), - LicenseSymbol('LGPL 2.1+'), - LicenseSymbol('Foo exception', is_exception=True), + LicenseSymbol("GPL-2.0"), + LicenseSymbol("LATER"), + LicenseSymbol("mit"), + LicenseSymbol("LGPL 2.1+"), + LicenseSymbol("Foo exception", is_exception=True), ] l = Licensing(symbols) - expr = ' GPL-2.0 or LATER and mit or LGPL 2.1+ and mit with Foo exception ' + expr = " GPL-2.0 or LATER and mit or LGPL 2.1+ and mit with Foo exception " expected = [ - LicenseSymbol('GPL-2.0'), - LicenseSymbol('LATER'), - LicenseSymbol('mit'), - LicenseSymbol('LGPL 2.1+'), - LicenseSymbol('mit'), - LicenseSymbol('Foo exception', is_exception=True), + LicenseSymbol("GPL-2.0"), + LicenseSymbol("LATER"), + LicenseSymbol("mit"), + LicenseSymbol("LGPL 2.1+"), + LicenseSymbol("mit"), + LicenseSymbol("Foo exception", is_exception=True), ] assert l.license_symbols(l.parse(expr), unique=False) == expected def test_get_license_symbols3(self): symbols = [ - LicenseSymbol('mit'), - LicenseSymbol('LGPL 2.1+'), - LicenseSymbol('Foo exception', is_exception=True), - LicenseSymbol('GPL-2.0'), - LicenseSymbol('LATER'), + LicenseSymbol("mit"), + LicenseSymbol("LGPL 2.1+"), + LicenseSymbol("Foo exception", is_exception=True), + LicenseSymbol("GPL-2.0"), + LicenseSymbol("LATER"), ] l = Licensing(symbols) - expr = 'mit or LGPL 2.1+ and mit with Foo exception or GPL-2.0 or LATER ' + expr = "mit or LGPL 2.1+ and mit with Foo exception or GPL-2.0 or LATER " assert symbols == l.license_symbols(l.parse(expr)) def test_get_license_symbols4(self): symbols = [ - LicenseSymbol('GPL-2.0'), - LicenseSymbol('LATER'), - LicenseSymbol('big exception', is_exception=True), - LicenseSymbol('mit'), - LicenseSymbol('LGPL 2.1+'), - LicenseSymbol('Foo exception', is_exception=True), + LicenseSymbol("GPL-2.0"), + LicenseSymbol("LATER"), + LicenseSymbol("big exception", is_exception=True), + LicenseSymbol("mit"), + LicenseSymbol("LGPL 2.1+"), + LicenseSymbol("Foo exception", is_exception=True), ] l = Licensing(symbols) - expr = (' GPL-2.0 or LATER with big exception and mit or ' - 'LGPL 2.1+ and mit or later with Foo exception ') + expr = ( + " GPL-2.0 or LATER with big exception and mit or " + "LGPL 2.1+ and mit or later with Foo exception " + ) expected = [ - LicenseSymbol('GPL-2.0'), - LicenseSymbol('LATER'), - LicenseSymbol('big exception', is_exception=True), - LicenseSymbol('mit'), - LicenseSymbol('LGPL 2.1+'), - LicenseSymbol('mit'), - LicenseSymbol('LATER'), - LicenseSymbol('Foo exception', is_exception=True), + LicenseSymbol("GPL-2.0"), + LicenseSymbol("LATER"), + LicenseSymbol("big exception", is_exception=True), + LicenseSymbol("mit"), + LicenseSymbol("LGPL 2.1+"), + LicenseSymbol("mit"), + LicenseSymbol("LATER"), + LicenseSymbol("Foo exception", is_exception=True), ] assert l.license_symbols(l.parse(expr), unique=False) == expected def test_license_symbols(self): - licensing = Licensing([ - 'GPL-2.0 or LATER', - 'classpath Exception', - 'something with else+', - 'mit', - 'LGPL 2.1', - 'mit or later' - ]) - - expr = (' GPL-2.0 or LATER with classpath Exception and mit and ' - 'mit with SOMETHING with ELSE+ or LGPL 2.1 and ' - 'GPL-2.0 or LATER with classpath Exception and ' - 'mit or later or LGPL 2.1 or mit or GPL-2.0 or LATER ' - 'with SOMETHING with ELSE+ and lgpl 2.1') - - gpl2plus = LicenseSymbol(key='GPL-2.0 or LATER') - cpex = LicenseSymbol(key='classpath Exception') - someplus = LicenseSymbol(key='something with else+') - mitplus = LicenseSymbol(key='mit or later') - mit = LicenseSymbol(key='mit') - lgpl = LicenseSymbol(key='LGPL 2.1') - gpl_with_cp = LicenseWithExceptionSymbol( - license_symbol=gpl2plus, exception_symbol=cpex) - mit_with_some = LicenseWithExceptionSymbol( - license_symbol=mit, exception_symbol=someplus) + licensing = Licensing( + [ + "GPL-2.0 or LATER", + "classpath Exception", + "something with else+", + "mit", + "LGPL 2.1", + "mit or later", + ] + ) + + expr = ( + " GPL-2.0 or LATER with classpath Exception and mit and " + "mit with SOMETHING with ELSE+ or LGPL 2.1 and " + "GPL-2.0 or LATER with classpath Exception and " + "mit or later or LGPL 2.1 or mit or GPL-2.0 or LATER " + "with SOMETHING with ELSE+ and lgpl 2.1" + ) + + gpl2plus = LicenseSymbol(key="GPL-2.0 or LATER") + cpex = LicenseSymbol(key="classpath Exception") + someplus = LicenseSymbol(key="something with else+") + mitplus = LicenseSymbol(key="mit or later") + mit = LicenseSymbol(key="mit") + lgpl = LicenseSymbol(key="LGPL 2.1") + gpl_with_cp = LicenseWithExceptionSymbol(license_symbol=gpl2plus, exception_symbol=cpex) + mit_with_some = LicenseWithExceptionSymbol(license_symbol=mit, exception_symbol=someplus) gpl2_with_someplus = LicenseWithExceptionSymbol( - license_symbol=gpl2plus, exception_symbol=someplus) + license_symbol=gpl2plus, exception_symbol=someplus + ) parsed = licensing.parse(expr) - expected = [gpl_with_cp, mit, mit_with_some, lgpl, - gpl_with_cp, mitplus, lgpl, mit, gpl2_with_someplus, lgpl] + expected = [ + gpl_with_cp, + mit, + mit_with_some, + lgpl, + gpl_with_cp, + mitplus, + lgpl, + mit, + gpl2_with_someplus, + lgpl, + ] - assert licensing.license_symbols( - parsed, unique=False, decompose=False) == expected + assert licensing.license_symbols(parsed, unique=False, decompose=False) == expected - expected = [gpl_with_cp, mit, mit_with_some, - lgpl, mitplus, gpl2_with_someplus] - assert licensing.license_symbols( - parsed, unique=True, decompose=False) == expected + expected = [gpl_with_cp, mit, mit_with_some, lgpl, mitplus, gpl2_with_someplus] + assert licensing.license_symbols(parsed, unique=True, decompose=False) == expected - expected = [gpl2plus, cpex, mit, mit, someplus, lgpl, - gpl2plus, cpex, mitplus, lgpl, mit, gpl2plus, someplus, lgpl] - assert licensing.license_symbols( - parsed, unique=False, decompose=True) == expected + expected = [ + gpl2plus, + cpex, + mit, + mit, + someplus, + lgpl, + gpl2plus, + cpex, + mitplus, + lgpl, + mit, + gpl2plus, + someplus, + lgpl, + ] + assert licensing.license_symbols(parsed, unique=False, decompose=True) == expected expected = [gpl2plus, cpex, mit, someplus, lgpl, mitplus] - assert licensing.license_symbols( - parsed, unique=True, decompose=True) == expected + assert licensing.license_symbols(parsed, unique=True, decompose=True) == expected def test_primary_license_symbol_and_primary_license_key(self): - licensing = Licensing([ - 'GPL-2.0 or LATER', - 'classpath Exception', - 'mit', - 'LGPL 2.1', - 'mit or later' - ]) - - expr = ' GPL-2.0 or LATER with classpath Exception and mit or LGPL 2.1 and mit or later ' - gpl = LicenseSymbol('GPL-2.0 or LATER') - cpex = LicenseSymbol('classpath Exception') + licensing = Licensing( + ["GPL-2.0 or LATER", "classpath Exception", "mit", "LGPL 2.1", "mit or later"] + ) + + expr = " GPL-2.0 or LATER with classpath Exception and mit or LGPL 2.1 and mit or later " + gpl = LicenseSymbol("GPL-2.0 or LATER") + cpex = LicenseSymbol("classpath Exception") expected = LicenseWithExceptionSymbol(gpl, cpex) parsed = licensing.parse(expr) - assert licensing.primary_license_symbol( - parsed, decompose=False) == expected + assert licensing.primary_license_symbol(parsed, decompose=False) == expected assert gpl == licensing.primary_license_symbol(parsed, decompose=True) - assert 'GPL-2.0 or LATER' == licensing.primary_license_key(parsed) - - expr = ' GPL-2.0 or later with classpath Exception and mit or LGPL 2.1 and mit or later ' - expected = 'GPL-2.0 or LATER WITH classpath Exception' - result = licensing.primary_license_symbol( - parsed, - decompose=False - ).render('{symbol.key}') + assert "GPL-2.0 or LATER" == licensing.primary_license_key(parsed) + + expr = " GPL-2.0 or later with classpath Exception and mit or LGPL 2.1 and mit or later " + expected = "GPL-2.0 or LATER WITH classpath Exception" + result = licensing.primary_license_symbol(parsed, decompose=False).render("{symbol.key}") assert result == expected def test_render_plain(self): l = Licensing() - result = l.parse('gpl-2.0 WITH exception-gpl-2.0-plus or MIT').render() - expected = 'gpl-2.0 WITH exception-gpl-2.0-plus OR MIT' + result = l.parse("gpl-2.0 WITH exception-gpl-2.0-plus or MIT").render() + expected = "gpl-2.0 WITH exception-gpl-2.0-plus OR MIT" assert result == expected def test_render_as_readable_does_not_wrap_in_parens_single_with(self): l = Licensing() - result = l.parse( - 'gpl-2.0 WITH exception-gpl-2.0-plus').render_as_readable() - expected = 'gpl-2.0 WITH exception-gpl-2.0-plus' + result = l.parse("gpl-2.0 WITH exception-gpl-2.0-plus").render_as_readable() + expected = "gpl-2.0 WITH exception-gpl-2.0-plus" assert result == expected def test_render_as_readable_wraps_in_parens_with_and_other_subexpressions(self): l = Licensing() - result = l.parse( - 'mit AND gpl-2.0 WITH exception-gpl-2.0-plus').render_as_readable() - expected = 'mit AND (gpl-2.0 WITH exception-gpl-2.0-plus)' + result = l.parse("mit AND gpl-2.0 WITH exception-gpl-2.0-plus").render_as_readable() + expected = "mit AND (gpl-2.0 WITH exception-gpl-2.0-plus)" assert result == expected def test_render_as_readable_does_not_wrap_in_parens_if_no_with(self): l = Licensing() - result1 = l.parse('gpl-2.0 and exception OR that').render_as_readable() - result2 = l.parse('gpl-2.0 and exception OR that').render() + result1 = l.parse("gpl-2.0 and exception OR that").render_as_readable() + result2 = l.parse("gpl-2.0 and exception OR that").render() assert result1 == result2 class SplitAndTokenizeTest(TestCase): - def test_simple_tokenizer(self): - expr = (' GPL-2.0 or later with classpath Exception and mit and ' - 'mit with SOMETHING with ELSE+ or LGPL 2.1 and ' - 'GPL-2.0 or LATER with (Classpath Exception and ' - 'mit or later) or LGPL 2.1 or mit or GPL-2.0 or LATER ' - 'with SOMETHING with ELSE+ and lgpl 2.1') + expr = ( + " GPL-2.0 or later with classpath Exception and mit and " + "mit with SOMETHING with ELSE+ or LGPL 2.1 and " + "GPL-2.0 or LATER with (Classpath Exception and " + "mit or later) or LGPL 2.1 or mit or GPL-2.0 or LATER " + "with SOMETHING with ELSE+ and lgpl 2.1" + ) licensing = Licensing() results = list(licensing.simple_tokenizer(expr)) expected = [ - Token(0, 0, ' ', None), - Token(1, 7, 'GPL-2.0', LicenseSymbol(key='GPL-2.0')), - Token(8, 8, ' ', None), - Token(9, 10, 'or', Keyword(value='or', type=TOKEN_OR)), - Token(11, 11, ' ', None), - Token(12, 16, 'later', LicenseSymbol(key='later')), - Token(17, 17, ' ', None), - Token(18, 21, 'with', Keyword(value='with', type=TOKEN_WITH)), - Token(22, 22, ' ', None), - Token(23, 31, 'classpath', LicenseSymbol(key='classpath')), - Token(32, 32, ' ', None), - Token(33, 41, 'Exception', LicenseSymbol(key='Exception')), - Token(42, 42, ' ', None), - Token(43, 45, 'and', Keyword(value='and', type=TOKEN_AND)), - Token(46, 46, ' ', None), - Token(47, 49, 'mit', LicenseSymbol(key='mit')), - Token(50, 50, ' ', None), - Token(51, 53, 'and', Keyword(value='and', type=TOKEN_AND)), - Token(54, 54, ' ', None), - Token(55, 57, 'mit', LicenseSymbol(key='mit')), - Token(58, 58, ' ', None), - Token(59, 62, 'with', Keyword(value='with', type=TOKEN_WITH)), - Token(63, 63, ' ', None), - Token(64, 72, 'SOMETHING', LicenseSymbol(key='SOMETHING')), - Token(73, 73, ' ', None), - Token(74, 77, 'with', Keyword(value='with', type=TOKEN_WITH)), - Token(78, 78, ' ', None), - Token(79, 83, 'ELSE+', LicenseSymbol(key='ELSE+')), - Token(84, 84, ' ', None), - Token(85, 86, 'or', Keyword(value='or', type=TOKEN_OR)), - Token(87, 87, ' ', None), - Token(88, 91, 'LGPL', LicenseSymbol(key='LGPL')), - Token(92, 92, ' ', None), - Token(93, 95, '2.1', LicenseSymbol(key='2.1')), - Token(96, 96, ' ', None), - Token(97, 99, 'and', Keyword(value='and', type=TOKEN_AND)), - Token(100, 100, ' ', None), - Token(101, 107, 'GPL-2.0', LicenseSymbol(key='GPL-2.0')), - Token(108, 108, ' ', None), - Token(109, 110, 'or', Keyword(value='or', type=TOKEN_OR)), - Token(111, 111, ' ', None), - Token(112, 116, 'LATER', LicenseSymbol(key='LATER')), - Token(117, 117, ' ', None), - Token(118, 121, 'with', Keyword(value='with', type=TOKEN_WITH)), - Token(122, 122, ' ', None), - Token(123, 123, '(', Keyword(value='(', type=TOKEN_LPAR)), - Token(124, 132, 'Classpath', LicenseSymbol(key='Classpath')), - Token(133, 133, ' ', None), - Token(134, 142, 'Exception', LicenseSymbol(key='Exception')), - Token(143, 143, ' ', None), - Token(144, 146, 'and', Keyword(value='and', type=TOKEN_AND)), - Token(147, 147, ' ', None), - Token(148, 150, 'mit', LicenseSymbol(key='mit')), - Token(151, 151, ' ', None), - Token(152, 153, 'or', Keyword(value='or', type=TOKEN_OR)), - Token(154, 154, ' ', None), - Token(155, 159, 'later', LicenseSymbol(key='later')), - Token(160, 160, ')', Keyword(value=')', type=TOKEN_RPAR)), - Token(161, 161, ' ', None), - Token(162, 163, 'or', Keyword(value='or', type=TOKEN_OR)), - Token(164, 164, ' ', None), - Token(165, 168, 'LGPL', LicenseSymbol(key='LGPL')), - Token(169, 169, ' ', None), - Token(170, 172, '2.1', LicenseSymbol(key='2.1')), - Token(173, 173, ' ', None), - Token(174, 175, 'or', Keyword(value='or', type=TOKEN_OR)), - Token(176, 176, ' ', None), - Token(177, 179, 'mit', LicenseSymbol(key='mit')), - Token(180, 180, ' ', None), - Token(181, 182, 'or', Keyword(value='or', type=TOKEN_OR)), - Token(183, 183, ' ', None), - Token(184, 190, 'GPL-2.0', LicenseSymbol(key='GPL-2.0')), - Token(191, 191, ' ', None), - Token(192, 193, 'or', Keyword(value='or', type=TOKEN_OR)), - Token(194, 194, ' ', None), - Token(195, 199, 'LATER', LicenseSymbol(key='LATER')), - Token(200, 200, ' ', None), - Token(201, 204, 'with', Keyword(value='with', type=TOKEN_WITH)), - Token(205, 205, ' ', None), - Token(206, 214, 'SOMETHING', LicenseSymbol(key='SOMETHING')), - Token(215, 215, ' ', None), - Token(216, 219, 'with', Keyword(value='with', type=TOKEN_WITH)), - Token(220, 220, ' ', None), - Token(221, 225, 'ELSE+', LicenseSymbol(key='ELSE+')), - Token(226, 226, ' ', None), - Token(227, 229, 'and', Keyword(value='and', type=TOKEN_AND)), - Token(230, 230, ' ', None), - Token(231, 234, 'lgpl', LicenseSymbol(key='lgpl')), - Token(235, 235, ' ', None), - Token(236, 238, '2.1', LicenseSymbol(key='2.1',)) + Token(0, 0, " ", None), + Token(1, 7, "GPL-2.0", LicenseSymbol(key="GPL-2.0")), + Token(8, 8, " ", None), + Token(9, 10, "or", Keyword(value="or", type=TOKEN_OR)), + Token(11, 11, " ", None), + Token(12, 16, "later", LicenseSymbol(key="later")), + Token(17, 17, " ", None), + Token(18, 21, "with", Keyword(value="with", type=TOKEN_WITH)), + Token(22, 22, " ", None), + Token(23, 31, "classpath", LicenseSymbol(key="classpath")), + Token(32, 32, " ", None), + Token(33, 41, "Exception", LicenseSymbol(key="Exception")), + Token(42, 42, " ", None), + Token(43, 45, "and", Keyword(value="and", type=TOKEN_AND)), + Token(46, 46, " ", None), + Token(47, 49, "mit", LicenseSymbol(key="mit")), + Token(50, 50, " ", None), + Token(51, 53, "and", Keyword(value="and", type=TOKEN_AND)), + Token(54, 54, " ", None), + Token(55, 57, "mit", LicenseSymbol(key="mit")), + Token(58, 58, " ", None), + Token(59, 62, "with", Keyword(value="with", type=TOKEN_WITH)), + Token(63, 63, " ", None), + Token(64, 72, "SOMETHING", LicenseSymbol(key="SOMETHING")), + Token(73, 73, " ", None), + Token(74, 77, "with", Keyword(value="with", type=TOKEN_WITH)), + Token(78, 78, " ", None), + Token(79, 83, "ELSE+", LicenseSymbol(key="ELSE+")), + Token(84, 84, " ", None), + Token(85, 86, "or", Keyword(value="or", type=TOKEN_OR)), + Token(87, 87, " ", None), + Token(88, 91, "LGPL", LicenseSymbol(key="LGPL")), + Token(92, 92, " ", None), + Token(93, 95, "2.1", LicenseSymbol(key="2.1")), + Token(96, 96, " ", None), + Token(97, 99, "and", Keyword(value="and", type=TOKEN_AND)), + Token(100, 100, " ", None), + Token(101, 107, "GPL-2.0", LicenseSymbol(key="GPL-2.0")), + Token(108, 108, " ", None), + Token(109, 110, "or", Keyword(value="or", type=TOKEN_OR)), + Token(111, 111, " ", None), + Token(112, 116, "LATER", LicenseSymbol(key="LATER")), + Token(117, 117, " ", None), + Token(118, 121, "with", Keyword(value="with", type=TOKEN_WITH)), + Token(122, 122, " ", None), + Token(123, 123, "(", Keyword(value="(", type=TOKEN_LPAR)), + Token(124, 132, "Classpath", LicenseSymbol(key="Classpath")), + Token(133, 133, " ", None), + Token(134, 142, "Exception", LicenseSymbol(key="Exception")), + Token(143, 143, " ", None), + Token(144, 146, "and", Keyword(value="and", type=TOKEN_AND)), + Token(147, 147, " ", None), + Token(148, 150, "mit", LicenseSymbol(key="mit")), + Token(151, 151, " ", None), + Token(152, 153, "or", Keyword(value="or", type=TOKEN_OR)), + Token(154, 154, " ", None), + Token(155, 159, "later", LicenseSymbol(key="later")), + Token(160, 160, ")", Keyword(value=")", type=TOKEN_RPAR)), + Token(161, 161, " ", None), + Token(162, 163, "or", Keyword(value="or", type=TOKEN_OR)), + Token(164, 164, " ", None), + Token(165, 168, "LGPL", LicenseSymbol(key="LGPL")), + Token(169, 169, " ", None), + Token(170, 172, "2.1", LicenseSymbol(key="2.1")), + Token(173, 173, " ", None), + Token(174, 175, "or", Keyword(value="or", type=TOKEN_OR)), + Token(176, 176, " ", None), + Token(177, 179, "mit", LicenseSymbol(key="mit")), + Token(180, 180, " ", None), + Token(181, 182, "or", Keyword(value="or", type=TOKEN_OR)), + Token(183, 183, " ", None), + Token(184, 190, "GPL-2.0", LicenseSymbol(key="GPL-2.0")), + Token(191, 191, " ", None), + Token(192, 193, "or", Keyword(value="or", type=TOKEN_OR)), + Token(194, 194, " ", None), + Token(195, 199, "LATER", LicenseSymbol(key="LATER")), + Token(200, 200, " ", None), + Token(201, 204, "with", Keyword(value="with", type=TOKEN_WITH)), + Token(205, 205, " ", None), + Token(206, 214, "SOMETHING", LicenseSymbol(key="SOMETHING")), + Token(215, 215, " ", None), + Token(216, 219, "with", Keyword(value="with", type=TOKEN_WITH)), + Token(220, 220, " ", None), + Token(221, 225, "ELSE+", LicenseSymbol(key="ELSE+")), + Token(226, 226, " ", None), + Token(227, 229, "and", Keyword(value="and", type=TOKEN_AND)), + Token(230, 230, " ", None), + Token(231, 234, "lgpl", LicenseSymbol(key="lgpl")), + Token(235, 235, " ", None), + Token( + 236, + 238, + "2.1", + LicenseSymbol( + key="2.1", + ), + ), ] assert results == expected def test_tokenize_can_handle_expressions_with_symbols_that_contain_a_colon(self): licensing = Licensing() - expression = 'DocumentRef-James-1.0:LicenseRef-Eric-2.0' + expression = "DocumentRef-James-1.0:LicenseRef-Eric-2.0" result = list(licensing.tokenize(expression)) expected = [ - (LicenseSymbol(u'DocumentRef-James-1.0:LicenseRef-Eric-2.0', is_exception=False), - u'DocumentRef-James-1.0:LicenseRef-Eric-2.0', 0) + ( + LicenseSymbol("DocumentRef-James-1.0:LicenseRef-Eric-2.0", is_exception=False), + "DocumentRef-James-1.0:LicenseRef-Eric-2.0", + 0, + ) ] assert result == expected def test_tokenize_simple_can_handle_expressions_with_symbols_that_contain_a_colon(self): licensing = Licensing() - expression = 'DocumentRef-James-1.0:LicenseRef-Eric-2.0' + expression = "DocumentRef-James-1.0:LicenseRef-Eric-2.0" result = list(licensing.tokenize(expression, simple=True)) expected = [ - (LicenseSymbol(u'DocumentRef-James-1.0:LicenseRef-Eric-2.0', is_exception=False), - u'DocumentRef-James-1.0:LicenseRef-Eric-2.0', 0) + ( + LicenseSymbol("DocumentRef-James-1.0:LicenseRef-Eric-2.0", is_exception=False), + "DocumentRef-James-1.0:LicenseRef-Eric-2.0", + 0, + ) ] assert result == expected def test_tokenize_can_handle_expressions_with_tabs_and_new_lines(self): licensing = Licensing() - expression = 'this\t \tis \n\n an expression' + expression = "this\t \tis \n\n an expression" result = list(licensing.tokenize(expression, simple=False)) expected = [ - (LicenseSymbol(u'this is an expression', is_exception=False), - u'this is an expression', 0) + (LicenseSymbol("this is an expression", is_exception=False), "this is an expression", 0) ] assert result == expected def test_tokenize_simple_can_handle_expressions_with_tabs_and_new_lines(self): licensing = Licensing() - expression = 'this\t \tis \n\n an expression' + expression = "this\t \tis \n\n an expression" result = list(licensing.tokenize(expression, simple=True)) expected = [ - (LicenseSymbol(u'this', is_exception=False), u'this', 0), - (LicenseSymbol(u'is', is_exception=False), u'is', 7), - (LicenseSymbol(u'an', is_exception=False), u'an', 13), - (LicenseSymbol(u'expression', is_exception=False), u'expression', 16) + (LicenseSymbol("this", is_exception=False), "this", 0), + (LicenseSymbol("is", is_exception=False), "is", 7), + (LicenseSymbol("an", is_exception=False), "an", 13), + (LicenseSymbol("expression", is_exception=False), "expression", 16), ] assert result == expected def test_tokenize_step_by_step_does_not_munge_trailing_symbols(self): - gpl2 = LicenseSymbol(key='GPL-2.0') - gpl2plus = LicenseSymbol(key='GPL-2.0 or LATER') - cpex = LicenseSymbol(key='classpath Exception', is_exception=True) + gpl2 = LicenseSymbol(key="GPL-2.0") + gpl2plus = LicenseSymbol(key="GPL-2.0 or LATER") + cpex = LicenseSymbol(key="classpath Exception", is_exception=True) - mitthing = LicenseSymbol(key='mithing') - mitthing_with_else = LicenseSymbol( - key='mitthing with else+', is_exception=False) + mitthing = LicenseSymbol(key="mithing") + mitthing_with_else = LicenseSymbol(key="mitthing with else+", is_exception=False) - mit = LicenseSymbol(key='mit') - mitplus = LicenseSymbol(key='mit or later') + mit = LicenseSymbol(key="mit") + mitplus = LicenseSymbol(key="mit or later") - elsish = LicenseSymbol(key='else') - elsishplus = LicenseSymbol(key='else+') + elsish = LicenseSymbol(key="else") + elsishplus = LicenseSymbol(key="else+") - lgpl = LicenseSymbol(key='LGPL 2.1') + lgpl = LicenseSymbol(key="LGPL 2.1") - licensing = Licensing([ - gpl2, - gpl2plus, - cpex, - mitthing, - mitthing_with_else, - mit, - mitplus, - elsish, - elsishplus, - lgpl, - ]) + licensing = Licensing( + [ + gpl2, + gpl2plus, + cpex, + mitthing, + mitthing_with_else, + mit, + mitplus, + elsish, + elsishplus, + lgpl, + ] + ) - expr = (' GPL-2.0 or later with classpath Exception and mit and ' - 'mit with mitthing with ELSE+ or LGPL 2.1 and ' - 'GPL-2.0 or LATER with Classpath Exception and ' - 'mit or later or LGPL 2.1 or mit or GPL-2.0 or LATER ' - 'with mitthing with ELSE+ and lgpl 2.1 or gpl-2.0') + expr = ( + " GPL-2.0 or later with classpath Exception and mit and " + "mit with mitthing with ELSE+ or LGPL 2.1 and " + "GPL-2.0 or LATER with Classpath Exception and " + "mit or later or LGPL 2.1 or mit or GPL-2.0 or LATER " + "with mitthing with ELSE+ and lgpl 2.1 or gpl-2.0" + ) # fist tokenize tokenizer = licensing.get_advanced_tokenizer() result = list(tokenizer.tokenize(expr)) expected = [ - Token(1, 16, 'GPL-2.0 or later', - LicenseSymbol(u'GPL-2.0 or LATER')), - Token(18, 21, 'with', Keyword(value=u'with', type=10)), - Token(23, 41, 'classpath Exception', LicenseSymbol( - u'classpath Exception', is_exception=True)), - Token(43, 45, 'and', Keyword(value=u'and', type=1)), - Token(47, 49, 'mit', LicenseSymbol(u'mit')), - Token(51, 53, 'and', Keyword(value=u'and', type=1)), - Token(55, 57, 'mit', LicenseSymbol(u'mit')), - Token(59, 62, 'with', Keyword(value=u'with', type=10)), - Token(64, 82, 'mitthing with ELSE+', - LicenseSymbol(u'mitthing with else+')), - Token(84, 85, 'or', Keyword(value=u'or', type=2)), - Token(87, 94, 'LGPL 2.1', LicenseSymbol(u'LGPL 2.1')), - Token(96, 98, 'and', Keyword(value=u'and', type=1)), - Token(100, 115, 'GPL-2.0 or LATER', - LicenseSymbol(u'GPL-2.0 or LATER')), - Token(117, 120, 'with', Keyword(value=u'with', type=10)), - Token(122, 140, 'Classpath Exception', LicenseSymbol( - u'classpath Exception', is_exception=True)), - Token(142, 144, 'and', Keyword(value=u'and', type=1)), - Token(146, 157, 'mit or later', LicenseSymbol(u'mit or later')), - Token(159, 160, 'or', Keyword(value=u'or', type=2)), - Token(162, 169, 'LGPL 2.1', LicenseSymbol(u'LGPL 2.1')), - Token(171, 172, 'or', Keyword(value=u'or', type=2)), - Token(174, 176, 'mit', LicenseSymbol(u'mit')), - Token(178, 179, 'or', Keyword(value=u'or', type=2)), - Token(181, 196, 'GPL-2.0 or LATER', - LicenseSymbol(u'GPL-2.0 or LATER')), - Token(198, 201, 'with', Keyword(value=u'with', type=10)), - Token(203, 221, 'mitthing with ELSE+', - LicenseSymbol(u'mitthing with else+')), - Token(223, 225, 'and', Keyword(value=u'and', type=1)), - Token(227, 234, 'lgpl 2.1', LicenseSymbol(u'LGPL 2.1')), - Token(236, 237, 'or', Keyword(value=u'or', type=2)), - Token(239, 245, 'gpl-2.0', LicenseSymbol(u'GPL-2.0')) + Token(1, 16, "GPL-2.0 or later", LicenseSymbol("GPL-2.0 or LATER")), + Token(18, 21, "with", Keyword(value="with", type=10)), + Token( + 23, + 41, + "classpath Exception", + LicenseSymbol("classpath Exception", is_exception=True), + ), + Token(43, 45, "and", Keyword(value="and", type=1)), + Token(47, 49, "mit", LicenseSymbol("mit")), + Token(51, 53, "and", Keyword(value="and", type=1)), + Token(55, 57, "mit", LicenseSymbol("mit")), + Token(59, 62, "with", Keyword(value="with", type=10)), + Token(64, 82, "mitthing with ELSE+", LicenseSymbol("mitthing with else+")), + Token(84, 85, "or", Keyword(value="or", type=2)), + Token(87, 94, "LGPL 2.1", LicenseSymbol("LGPL 2.1")), + Token(96, 98, "and", Keyword(value="and", type=1)), + Token(100, 115, "GPL-2.0 or LATER", LicenseSymbol("GPL-2.0 or LATER")), + Token(117, 120, "with", Keyword(value="with", type=10)), + Token( + 122, + 140, + "Classpath Exception", + LicenseSymbol("classpath Exception", is_exception=True), + ), + Token(142, 144, "and", Keyword(value="and", type=1)), + Token(146, 157, "mit or later", LicenseSymbol("mit or later")), + Token(159, 160, "or", Keyword(value="or", type=2)), + Token(162, 169, "LGPL 2.1", LicenseSymbol("LGPL 2.1")), + Token(171, 172, "or", Keyword(value="or", type=2)), + Token(174, 176, "mit", LicenseSymbol("mit")), + Token(178, 179, "or", Keyword(value="or", type=2)), + Token(181, 196, "GPL-2.0 or LATER", LicenseSymbol("GPL-2.0 or LATER")), + Token(198, 201, "with", Keyword(value="with", type=10)), + Token(203, 221, "mitthing with ELSE+", LicenseSymbol("mitthing with else+")), + Token(223, 225, "and", Keyword(value="and", type=1)), + Token(227, 234, "lgpl 2.1", LicenseSymbol("LGPL 2.1")), + Token(236, 237, "or", Keyword(value="or", type=2)), + Token(239, 245, "gpl-2.0", LicenseSymbol("GPL-2.0")), ] assert result == expected expected_groups = [ - (Token(1, 16, 'GPL-2.0 or later', LicenseSymbol(u'GPL-2.0 or LATER')), - Token(18, 21, 'with', Keyword(value=u'with', type=10)), - Token(23, 41, 'classpath Exception', LicenseSymbol(u'classpath Exception', is_exception=True))), - - (Token(43, 45, 'and', Keyword(value=u'and', type=1)),), - (Token(47, 49, 'mit', LicenseSymbol(u'mit')),), - (Token(51, 53, 'and', Keyword(value=u'and', type=1)),), - - (Token(55, 57, 'mit', LicenseSymbol(u'mit')), - Token(59, 62, 'with', Keyword(value=u'with', type=10)), - Token(64, 82, 'mitthing with ELSE+', LicenseSymbol(u'mitthing with else+'))), - - (Token(84, 85, 'or', Keyword(value=u'or', type=2)),), - (Token(87, 94, 'LGPL 2.1', LicenseSymbol(u'LGPL 2.1')),), - (Token(96, 98, 'and', Keyword(value=u'and', type=1)),), - - (Token(100, 115, 'GPL-2.0 or LATER', LicenseSymbol(u'GPL-2.0 or LATER')), - Token(117, 120, 'with', Keyword(value=u'with', type=10)), - Token(122, 140, 'Classpath Exception', LicenseSymbol(u'classpath Exception', is_exception=True))), - - (Token(142, 144, 'and', Keyword(value=u'and', type=1)),), - (Token(146, 157, 'mit or later', LicenseSymbol(u'mit or later')),), - (Token(159, 160, 'or', Keyword(value=u'or', type=2)),), - (Token(162, 169, 'LGPL 2.1', LicenseSymbol(u'LGPL 2.1')),), - (Token(171, 172, 'or', Keyword(value=u'or', type=2)),), - (Token(174, 176, 'mit', LicenseSymbol(u'mit')),), - (Token(178, 179, 'or', Keyword(value=u'or', type=2)),), - - (Token(181, 196, 'GPL-2.0 or LATER', LicenseSymbol(u'GPL-2.0 or LATER')), - Token(198, 201, 'with', Keyword(value=u'with', type=10)), - Token(203, 221, 'mitthing with ELSE+', LicenseSymbol(u'mitthing with else+'))), - - (Token(223, 225, 'and', Keyword(value=u'and', type=1)),), - (Token(227, 234, 'lgpl 2.1', LicenseSymbol(u'LGPL 2.1')),), - (Token(236, 237, 'or', Keyword(value=u'or', type=2)),), - (Token(239, 245, 'gpl-2.0', LicenseSymbol(u'GPL-2.0')),) + ( + Token(1, 16, "GPL-2.0 or later", LicenseSymbol("GPL-2.0 or LATER")), + Token(18, 21, "with", Keyword(value="with", type=10)), + Token( + 23, + 41, + "classpath Exception", + LicenseSymbol("classpath Exception", is_exception=True), + ), + ), + (Token(43, 45, "and", Keyword(value="and", type=1)),), + (Token(47, 49, "mit", LicenseSymbol("mit")),), + (Token(51, 53, "and", Keyword(value="and", type=1)),), + ( + Token(55, 57, "mit", LicenseSymbol("mit")), + Token(59, 62, "with", Keyword(value="with", type=10)), + Token(64, 82, "mitthing with ELSE+", LicenseSymbol("mitthing with else+")), + ), + (Token(84, 85, "or", Keyword(value="or", type=2)),), + (Token(87, 94, "LGPL 2.1", LicenseSymbol("LGPL 2.1")),), + (Token(96, 98, "and", Keyword(value="and", type=1)),), + ( + Token(100, 115, "GPL-2.0 or LATER", LicenseSymbol("GPL-2.0 or LATER")), + Token(117, 120, "with", Keyword(value="with", type=10)), + Token( + 122, + 140, + "Classpath Exception", + LicenseSymbol("classpath Exception", is_exception=True), + ), + ), + (Token(142, 144, "and", Keyword(value="and", type=1)),), + (Token(146, 157, "mit or later", LicenseSymbol("mit or later")),), + (Token(159, 160, "or", Keyword(value="or", type=2)),), + (Token(162, 169, "LGPL 2.1", LicenseSymbol("LGPL 2.1")),), + (Token(171, 172, "or", Keyword(value="or", type=2)),), + (Token(174, 176, "mit", LicenseSymbol("mit")),), + (Token(178, 179, "or", Keyword(value="or", type=2)),), + ( + Token(181, 196, "GPL-2.0 or LATER", LicenseSymbol("GPL-2.0 or LATER")), + Token(198, 201, "with", Keyword(value="with", type=10)), + Token(203, 221, "mitthing with ELSE+", LicenseSymbol("mitthing with else+")), + ), + (Token(223, 225, "and", Keyword(value="and", type=1)),), + (Token(227, 234, "lgpl 2.1", LicenseSymbol("LGPL 2.1")),), + (Token(236, 237, "or", Keyword(value="or", type=2)),), + (Token(239, 245, "gpl-2.0", LicenseSymbol("GPL-2.0")),), ] result_groups = list(build_token_groups_for_with_subexpression(result)) assert expected_groups == result_groups @@ -2137,61 +2165,63 @@ def test_tokenize_step_by_step_does_not_munge_trailing_symbols(self): # finally retest it all with tokenize gpl2plus_with_cpex = LicenseWithExceptionSymbol( - license_symbol=gpl2plus, exception_symbol=cpex) + license_symbol=gpl2plus, exception_symbol=cpex + ) gpl2plus_with_someplus = LicenseWithExceptionSymbol( - license_symbol=gpl2plus, exception_symbol=mitthing_with_else) + license_symbol=gpl2plus, exception_symbol=mitthing_with_else + ) mit_with_mitthing_with_else = LicenseWithExceptionSymbol( - license_symbol=mit, exception_symbol=mitthing_with_else) + license_symbol=mit, exception_symbol=mitthing_with_else + ) expected = [ - (gpl2plus_with_cpex, 'GPL-2.0 or later with classpath Exception', 1), - (TOKEN_AND, 'and', 43), - (mit, 'mit', 47), - (TOKEN_AND, 'and', 51), - (mit_with_mitthing_with_else, 'mit with mitthing with ELSE+', 55), - (TOKEN_OR, 'or', 84), - (lgpl, 'LGPL 2.1', 87), - (TOKEN_AND, 'and', 96), - (gpl2plus_with_cpex, 'GPL-2.0 or LATER with Classpath Exception', 100), - (TOKEN_AND, 'and', 142), - (mitplus, 'mit or later', 146), - (TOKEN_OR, 'or', 159), - (lgpl, 'LGPL 2.1', 162), - (TOKEN_OR, 'or', 171), - (mit, 'mit', 174), - (TOKEN_OR, 'or', 178), - (gpl2plus_with_someplus, 'GPL-2.0 or LATER with mitthing with ELSE+', 181), - (TOKEN_AND, 'and', 223), - (lgpl, 'lgpl 2.1', 227), - (TOKEN_OR, 'or', 236), - (gpl2, 'gpl-2.0', 239), + (gpl2plus_with_cpex, "GPL-2.0 or later with classpath Exception", 1), + (TOKEN_AND, "and", 43), + (mit, "mit", 47), + (TOKEN_AND, "and", 51), + (mit_with_mitthing_with_else, "mit with mitthing with ELSE+", 55), + (TOKEN_OR, "or", 84), + (lgpl, "LGPL 2.1", 87), + (TOKEN_AND, "and", 96), + (gpl2plus_with_cpex, "GPL-2.0 or LATER with Classpath Exception", 100), + (TOKEN_AND, "and", 142), + (mitplus, "mit or later", 146), + (TOKEN_OR, "or", 159), + (lgpl, "LGPL 2.1", 162), + (TOKEN_OR, "or", 171), + (mit, "mit", 174), + (TOKEN_OR, "or", 178), + (gpl2plus_with_someplus, "GPL-2.0 or LATER with mitthing with ELSE+", 181), + (TOKEN_AND, "and", 223), + (lgpl, "lgpl 2.1", 227), + (TOKEN_OR, "or", 236), + (gpl2, "gpl-2.0", 239), ] assert list(licensing.tokenize(expr)) == expected class LicensingExpression(TestCase): - def test_is_equivalent_with_same_Licensing(self): licensing = Licensing() - parsed1 = licensing.parse('gpl-2.0 AND zlib') - parsed2 = licensing.parse('gpl-2.0 AND zlib AND zlib') + parsed1 = licensing.parse("gpl-2.0 AND zlib") + parsed2 = licensing.parse("gpl-2.0 AND zlib AND zlib") assert licensing.is_equivalent(parsed1, parsed2) assert Licensing().is_equivalent(parsed1, parsed2) def test_is_equivalent_with_same_Licensing2(self): licensing = Licensing() - parsed1 = licensing.parse('(gpl-2.0 AND zlib) or lgpl') - parsed2 = licensing.parse('lgpl or (gpl-2.0 AND zlib)') + parsed1 = licensing.parse("(gpl-2.0 AND zlib) or lgpl") + parsed2 = licensing.parse("lgpl or (gpl-2.0 AND zlib)") assert licensing.is_equivalent(parsed1, parsed2) assert Licensing().is_equivalent(parsed1, parsed2) def test_is_equivalent_with_different_Licensing_and_compound_expression(self): licensing1 = Licensing() licensing2 = Licensing() - parsed1 = licensing1.parse('gpl-2.0 AND zlib') - parsed2 = licensing2.parse('gpl-2.0 AND zlib AND zlib') + parsed1 = licensing1.parse("gpl-2.0 AND zlib") + parsed2 = licensing2.parse("gpl-2.0 AND zlib AND zlib") assert Licensing().is_equivalent(parsed1, parsed2) assert licensing1.is_equivalent(parsed1, parsed2) assert licensing2.is_equivalent(parsed1, parsed2) @@ -2199,8 +2229,8 @@ def test_is_equivalent_with_different_Licensing_and_compound_expression(self): def test_is_equivalent_with_different_Licensing_and_compound_expression2(self): licensing1 = Licensing() licensing2 = Licensing() - parsed1 = licensing1.parse('gpl-2.0 AND zlib') - parsed2 = licensing2.parse('zlib and gpl-2.0') + parsed1 = licensing1.parse("gpl-2.0 AND zlib") + parsed2 = licensing2.parse("zlib and gpl-2.0") assert Licensing().is_equivalent(parsed1, parsed2) assert licensing1.is_equivalent(parsed1, parsed2) assert licensing2.is_equivalent(parsed1, parsed2) @@ -2208,46 +2238,53 @@ def test_is_equivalent_with_different_Licensing_and_compound_expression2(self): def test_is_equivalent_with_different_Licensing_and_simple_expression(self): licensing1 = Licensing() licensing2 = Licensing() - parsed1 = licensing1.parse('gpl-2.0') - parsed2 = licensing2.parse('gpl-2.0') + parsed1 = licensing1.parse("gpl-2.0") + parsed2 = licensing2.parse("gpl-2.0") assert Licensing().is_equivalent(parsed1, parsed2) assert licensing1.is_equivalent(parsed1, parsed2) assert licensing2.is_equivalent(parsed1, parsed2) def test_is_equivalent_with_symbols_and_complex_expression(self): licensing_no_sym = Licensing() - licensing1 = Licensing([ - 'GPL-2.0 or LATER', - 'classpath Exception', - 'agpl+', - 'mit', - 'LGPL 2.1', - ]) - licensing2 = Licensing([ - 'GPL-2.0 or LATER', - 'classpath Exception', - 'agpl+', - 'mit', - 'LGPL 2.1', - ]) + licensing1 = Licensing( + [ + "GPL-2.0 or LATER", + "classpath Exception", + "agpl+", + "mit", + "LGPL 2.1", + ] + ) + licensing2 = Licensing( + [ + "GPL-2.0 or LATER", + "classpath Exception", + "agpl+", + "mit", + "LGPL 2.1", + ] + ) parsed1 = licensing1.parse( - ' ((LGPL 2.1 or mit) and GPL-2.0 or LATER with classpath Exception) and agpl+') + " ((LGPL 2.1 or mit) and GPL-2.0 or LATER with classpath Exception) and agpl+" + ) parsed2 = licensing2.parse( - ' agpl+ and (GPL-2.0 or LATER with classpath Exception and (mit or LGPL 2.1))') + " agpl+ and (GPL-2.0 or LATER with classpath Exception and (mit or LGPL 2.1))" + ) assert licensing1.is_equivalent(parsed1, parsed2) assert licensing2.is_equivalent(parsed1, parsed2) assert licensing_no_sym.is_equivalent(parsed1, parsed2) parsed3 = licensing1.parse( - ' ((LGPL 2.1 or mit) OR GPL-2.0 or LATER with classpath Exception) and agpl+') + " ((LGPL 2.1 or mit) OR GPL-2.0 or LATER with classpath Exception) and agpl+" + ) assert not licensing1.is_equivalent(parsed1, parsed3) assert not licensing2.is_equivalent(parsed1, parsed3) assert not licensing_no_sym.is_equivalent(parsed1, parsed3) def test_all_symbol_classes_can_compare_and_sort(self): - l1 = LicenseSymbol('a') - l2 = LicenseSymbol('b') + l1 = LicenseSymbol("a") + l2 = LicenseSymbol("b") lx = LicenseWithExceptionSymbol(l1, l2) lx2 = LicenseWithExceptionSymbol(l1, l2) assert not (lx < lx2) @@ -2260,12 +2297,11 @@ def test_all_symbol_classes_can_compare_and_sort(self): assert l2 != l1 class SymLike(object): - def __init__(self, key, is_exception=False): self.key = key self.is_exception = is_exception - l3 = LicenseSymbolLike(SymLike('b')) + l3 = LicenseSymbolLike(SymLike("b")) lx3 = LicenseWithExceptionSymbol(l1, l3) assert not (lx < lx3) assert not (lx3 < lx) @@ -2276,35 +2312,34 @@ def __init__(self, key, is_exception=False): assert l2 == l3 assert hash(l2) == hash(l3) - l4 = LicenseSymbolLike(SymLike('c')) + l4 = LicenseSymbolLike(SymLike("c")) expected = [l1, lx, lx2, lx3, l3, l2, l4] assert sorted([l4, l3, l2, l1, lx, lx2, lx3]) == expected class MockLicensesTest(TestCase): - def test_licensing_can_use_mocklicense_tuple(self): - MockLicense = namedtuple('MockLicense', 'key aliases is_exception') + MockLicense = namedtuple("MockLicense", "key aliases is_exception") licenses = [ - MockLicense('gpl-2.0', ['GPL-2.0'], False), - MockLicense('classpath-2.0', ['Classpath-Exception-2.0'], True), - MockLicense('gpl-2.0-plus', - ['GPL-2.0-or-later', 'GPL-2.0 or-later'], False), - MockLicense('lgpl-2.1-plus', ['LGPL-2.1-or-later'], False), + MockLicense("gpl-2.0", ["GPL-2.0"], False), + MockLicense("classpath-2.0", ["Classpath-Exception-2.0"], True), + MockLicense("gpl-2.0-plus", ["GPL-2.0-or-later", "GPL-2.0 or-later"], False), + MockLicense("lgpl-2.1-plus", ["LGPL-2.1-or-later"], False), ] licensing = Licensing(licenses) - ex1 = '(GPL-2.0-or-later with Classpath-Exception-2.0 or GPL-2.0 or-later) and LGPL-2.1-or-later' + ex1 = "(GPL-2.0-or-later with Classpath-Exception-2.0 or GPL-2.0 or-later) and LGPL-2.1-or-later" expression1 = licensing.parse(ex1, validate=False, strict=False) - assert ['gpl-2.0-plus', 'classpath-2.0', - 'lgpl-2.1-plus'] == licensing.license_keys(expression1) + assert ["gpl-2.0-plus", "classpath-2.0", "lgpl-2.1-plus"] == licensing.license_keys( + expression1 + ) - ex2 = 'LGPL-2.1-or-later and (GPL-2.0-or-later oR GPL-2.0-or-later with Classpath-Exception-2.0)' + ex2 = "LGPL-2.1-or-later and (GPL-2.0-or-later oR GPL-2.0-or-later with Classpath-Exception-2.0)" expression2 = licensing.parse(ex2, validate=True, strict=False) - ex3 = 'LGPL-2.1-or-later and (GPL-2.0-or-later oR GPL-2.0-or-later)' + ex3 = "LGPL-2.1-or-later and (GPL-2.0-or-later oR GPL-2.0-or-later)" expression3 = licensing.parse(ex3, validate=True, strict=False) self.assertTrue(licensing.is_equivalent(expression1, expression2)) @@ -2313,44 +2348,51 @@ def test_licensing_can_use_mocklicense_tuple(self): self.assertFalse(licensing.is_equivalent(expression2, expression3)) def test_and_and_or_is_invalid(self): - expression = 'gpl-2.0 with classpath and and or gpl-2.0-plus' + expression = "gpl-2.0 with classpath and and or gpl-2.0-plus" licensing = Licensing() try: licensing.parse(expression) - self.fail('Exception not raised') + self.fail("Exception not raised") except ParseError as pe: expected = { - 'error_code': PARSE_INVALID_OPERATOR_SEQUENCE, - 'position': 27, - 'token_string': 'and', - 'token_type': TOKEN_AND} + "error_code": PARSE_INVALID_OPERATOR_SEQUENCE, + "position": 27, + "token_string": "and", + "token_type": TOKEN_AND, + } assert _parse_error_as_dict(pe) == expected def test_or_or_is_invalid(self): - expression = 'gpl-2.0 with classpath or or or or gpl-2.0-plus' + expression = "gpl-2.0 with classpath or or or or gpl-2.0-plus" licensing = Licensing() try: licensing.parse(expression) except ParseError as pe: expected = { - 'error_code': PARSE_INVALID_OPERATOR_SEQUENCE, - 'position': 26, - 'token_string': 'or', - 'token_type': TOKEN_OR} + "error_code": PARSE_INVALID_OPERATOR_SEQUENCE, + "position": 26, + "token_string": "or", + "token_type": TOKEN_OR, + } assert _parse_error_as_dict(pe) == expected def test_tokenize_or_or(self): - expression = 'gpl-2.0 with classpath or or or gpl-2.0-plus' + expression = "gpl-2.0 with classpath or or or gpl-2.0-plus" licensing = Licensing() results = list(licensing.tokenize(expression)) expected = [ - (LicenseWithExceptionSymbol( - license_symbol=LicenseSymbol(u'gpl-2.0'), - exception_symbol=LicenseSymbol(u'classpath')), 'gpl-2.0 with classpath', 0), - (2, 'or', 23), - (2, 'or', 26), - (2, 'or', 29), - (LicenseSymbol(u'gpl-2.0-plus'), 'gpl-2.0-plus', 32) + ( + LicenseWithExceptionSymbol( + license_symbol=LicenseSymbol("gpl-2.0"), + exception_symbol=LicenseSymbol("classpath"), + ), + "gpl-2.0 with classpath", + 0, + ), + (2, "or", 23), + (2, "or", 26), + (2, "or", 29), + (LicenseSymbol("gpl-2.0-plus"), "gpl-2.0-plus", 32), ] assert results == expected @@ -2359,78 +2401,80 @@ def test_tokenize_or_or(self): class LicensingValidateTest(TestCase): licensing = Licensing( [ - LicenseSymbol(key='GPL-2.0-or-later', is_exception=False), - LicenseSymbol(key='MIT', is_exception=False), - LicenseSymbol(key='Apache-2.0', is_exception=False), - LicenseSymbol(key='WxWindows-exception-3.1', is_exception=True), + LicenseSymbol(key="GPL-2.0-or-later", is_exception=False), + LicenseSymbol(key="MIT", is_exception=False), + LicenseSymbol(key="Apache-2.0", is_exception=False), + LicenseSymbol(key="WxWindows-exception-3.1", is_exception=True), ] ) def test_validate_simple(self): - result = self.licensing.validate('GPL-2.0-or-later AND MIT') - assert result.original_expression == 'GPL-2.0-or-later AND MIT' - assert result.normalized_expression == 'GPL-2.0-or-later AND MIT' + result = self.licensing.validate("GPL-2.0-or-later AND MIT") + assert result.original_expression == "GPL-2.0-or-later AND MIT" + assert result.normalized_expression == "GPL-2.0-or-later AND MIT" assert result.errors == [] assert result.invalid_symbols == [] def test_validation_invalid_license_key(self): - result = self.licensing.validate('cool-license') - assert result.original_expression == 'cool-license' + result = self.licensing.validate("cool-license") + assert result.original_expression == "cool-license" assert not result.normalized_expression - assert result.errors == ['Unknown license key(s): cool-license'] - assert result.invalid_symbols == ['cool-license'] + assert result.errors == ["Unknown license key(s): cool-license"] + assert result.invalid_symbols == ["cool-license"] def test_validate_exception(self): - result = self.licensing.validate( - 'GPL-2.0-or-later WITH WxWindows-exception-3.1') - assert result.original_expression == 'GPL-2.0-or-later WITH WxWindows-exception-3.1' - assert result.normalized_expression == 'GPL-2.0-or-later WITH WxWindows-exception-3.1' + result = self.licensing.validate("GPL-2.0-or-later WITH WxWindows-exception-3.1") + assert result.original_expression == "GPL-2.0-or-later WITH WxWindows-exception-3.1" + assert result.normalized_expression == "GPL-2.0-or-later WITH WxWindows-exception-3.1" assert result.errors == [] assert result.invalid_symbols == [] def test_validation_exception_with_choice(self): - result = self.licensing.validate( - 'GPL-2.0-or-later WITH WxWindows-exception-3.1 OR MIT') - assert result.original_expression == 'GPL-2.0-or-later WITH WxWindows-exception-3.1 OR MIT' - assert result.normalized_expression == 'GPL-2.0-or-later WITH WxWindows-exception-3.1 OR MIT' + result = self.licensing.validate("GPL-2.0-or-later WITH WxWindows-exception-3.1 OR MIT") + assert result.original_expression == "GPL-2.0-or-later WITH WxWindows-exception-3.1 OR MIT" + assert ( + result.normalized_expression == "GPL-2.0-or-later WITH WxWindows-exception-3.1 OR MIT" + ) assert result.errors == [] assert result.invalid_symbols == [] def test_validation_exception_as_regular_key(self): - result = self.licensing.validate( - 'GPL-2.0-or-later AND WxWindows-exception-3.1') - assert result.original_expression == 'GPL-2.0-or-later AND WxWindows-exception-3.1' + result = self.licensing.validate("GPL-2.0-or-later AND WxWindows-exception-3.1") + assert result.original_expression == "GPL-2.0-or-later AND WxWindows-exception-3.1" assert not result.normalized_expression assert result.errors == [ - 'A license exception symbol can only be used as an exception in a "WITH exception" statement. for token: "WxWindows-exception-3.1" at position: 21'] - assert result.invalid_symbols == ['WxWindows-exception-3.1'] + 'A license exception symbol can only be used as an exception in a "WITH exception" statement. for token: "WxWindows-exception-3.1" at position: 21' + ] + assert result.invalid_symbols == ["WxWindows-exception-3.1"] def test_validation_bad_syntax(self): - result = self.licensing.validate('Apache-2.0 + MIT') - assert result.original_expression == 'Apache-2.0 + MIT' + result = self.licensing.validate("Apache-2.0 + MIT") + assert result.original_expression == "Apache-2.0 + MIT" assert not result.normalized_expression assert result.errors == [ - 'Invalid symbols sequence such as (A B) for token: "+" at position: 11'] - assert result.invalid_symbols == ['+'] + 'Invalid symbols sequence such as (A B) for token: "+" at position: 11' + ] + assert result.invalid_symbols == ["+"] def test_validation_invalid_license_exception(self): - result = self.licensing.validate('Apache-2.0 WITH MIT') - assert result.original_expression == 'Apache-2.0 WITH MIT' + result = self.licensing.validate("Apache-2.0 WITH MIT") + assert result.original_expression == "Apache-2.0 WITH MIT" assert not result.normalized_expression assert result.errors == [ - "A plain license symbol cannot be used as an exception in a \"WITH symbol\" statement. for token: \"MIT\" at position: 16"] - assert result.invalid_symbols == ['MIT'] + 'A plain license symbol cannot be used as an exception in a "WITH symbol" statement. for token: "MIT" at position: 16' + ] + assert result.invalid_symbols == ["MIT"] def test_validation_invalid_license_exception_strict_false(self): - result = self.licensing.validate('Apache-2.0 WITH MIT', strict=False) - assert result.original_expression == 'Apache-2.0 WITH MIT' - assert result.normalized_expression == 'Apache-2.0 WITH MIT' + result = self.licensing.validate("Apache-2.0 WITH MIT", strict=False) + assert result.original_expression == "Apache-2.0 WITH MIT" + assert result.normalized_expression == "Apache-2.0 WITH MIT" assert result.errors == [] assert result.invalid_symbols == [] class UtilTest(TestCase): - test_data_dir = join(dirname(__file__), 'data') + test_data_dir = join(dirname(__file__), "data") def test_build_licensing(self): test_license_index_location = join(self.test_data_dir, "test_license_key_index.json") @@ -2461,8 +2505,7 @@ def test_build_spdx_licensing(self): assert known_symbols_lowercase == {sym.lower() for sym in expected_symbols} def test_get_license_key_info(self): - test_license_index_location = join( - self.test_data_dir, 'test_license_key_index.json') + test_license_index_location = join(self.test_data_dir, "test_license_key_index.json") with open(test_license_index_location) as f: expected = json.load(f) result = get_license_index(test_license_index_location) @@ -2472,10 +2515,7 @@ def test_get_license_key_info_vendored(self): curr_dir = dirname(abspath(__file__)) parent_dir = pathlib.Path(curr_dir).parent vendored_license_key_index_location = parent_dir.joinpath( - 'src', - 'license_expression', - 'data', - 'scancode-licensedb-index.json' + "src", "license_expression", "data", "scancode-licensedb-index.json" ) with open(vendored_license_key_index_location) as f: expected = json.load(f) @@ -2484,19 +2524,15 @@ def test_get_license_key_info_vendored(self): class CombineExpressionTest(TestCase): - def test_combine_expressions_with_empty_input(self): assert combine_expressions(None) == None assert combine_expressions([]) == None def test_combine_expressions_with_regular(self): - assert str(combine_expressions( - ['mit', 'apache-2.0'])) == 'mit AND apache-2.0' + assert str(combine_expressions(["mit", "apache-2.0"])) == "mit AND apache-2.0" def test_combine_expressions_with_duplicated_elements(self): - assert str(combine_expressions( - ['mit', 'apache-2.0', 'mit'])) == 'mit AND apache-2.0' + assert str(combine_expressions(["mit", "apache-2.0", "mit"])) == "mit AND apache-2.0" def test_combine_expressions_with_or_relationship(self): - assert str(combine_expressions( - ['mit', 'apache-2.0'], 'OR')) == 'mit OR apache-2.0' + assert str(combine_expressions(["mit", "apache-2.0"], "OR")) == "mit OR apache-2.0"