From d4b658f5bda06663d700a4a10e1e4c37b278f62e Mon Sep 17 00:00:00 2001 From: Ayan Sinha Mahapatra Date: Tue, 21 Oct 2025 19:29:50 +0530 Subject: [PATCH 01/11] Do not set flag_value as None in click Option From click 8.3.0 flag_value is set as a sentinel value UNSET instead of None, and since we maintain a PluggableCommanLineOption subclass, we need to maintain the same default values for an Option to avoid issues. Reference: https://github.com/pallets/click/releases/tag/8.3.0 Reference: https://github.com/aboutcode-org/scancode-toolkit/issues/4572 Signed-off-by: Ayan Sinha Mahapatra --- src/commoncode/cliutils.py | 15 ++++++++++++++- 1 file changed, 14 insertions(+), 1 deletion(-) diff --git a/src/commoncode/cliutils.py b/src/commoncode/cliutils.py index 026e26b..a9273a5 100644 --- a/src/commoncode/cliutils.py +++ b/src/commoncode/cliutils.py @@ -22,6 +22,19 @@ # Tracing flags TRACE = False +try: + # Introduced in click 8.3.0 to have a sentinel value + # (https://peps.python.org/pep-0661/) for flag values + # and default values instead of None to differentiate + # between explicitly setting a `None` value and + # not setting and value. + # See https://github.com/pallets/click/pull/3030 and + # https://github.com/pallets/click/releases/tag/8.3.0 + from click.core import UNSET +except ImportError: + # to maintain compatibility with click < 8.3.0 + UNSET = None + def logger_debug(*args): pass @@ -429,7 +442,7 @@ def __init__( confirmation_prompt=False, hide_input=False, is_flag=None, - flag_value=None, + flag_value=UNSET, multiple=False, count=False, allow_from_autoenv=True, From 2cd886ba3a38897f9ef7fd4e514d87d6f1fe674e Mon Sep 17 00:00:00 2001 From: Ayan Sinha Mahapatra Date: Tue, 21 Oct 2025 19:49:27 +0530 Subject: [PATCH 02/11] Drop python3.9 support and add python3.14 Signed-off-by: Ayan Sinha Mahapatra --- azure-pipelines.yml | 4 ++-- requirements.txt | 3 +-- setup.cfg | 5 ++--- src/commoncode/cliutils.py | 15 ++------------- 4 files changed, 7 insertions(+), 20 deletions(-) diff --git a/azure-pipelines.yml b/azure-pipelines.yml index cd391c1..6c013b4 100644 --- a/azure-pipelines.yml +++ b/azure-pipelines.yml @@ -65,10 +65,10 @@ jobs: parameters: job_name: ubuntu24_test_all_supported_click_versions image_name: ubuntu-24.04 - python_versions: ["3.9", "3.10", "3.11", "3.12", "3.13"] + python_versions: ["3.10", "3.11", "3.12", "3.13", "3.14"] test_suites: click_versions: | - for clk_ver in 8.2.0 8.2.1 8.1.8 8.1.7 8.1.6 8.1.5 8.1.4 8.1.3 8.1.2 8.1.1 8.1.0 8.0.4 8.0.2 8.0.3 8.0.1 7.1.2 7.1.1 7.1 6.7; + for clk_ver in 8.3.0 8.2.0 8.2.1 8.1.8 8.1.7 8.1.6 8.1.5 8.1.4 8.1.3 8.1.2 8.1.1 8.1.0 8.0.4 8.0.2 8.0.3 8.0.1 7.1.2 7.1.1 7.1 6.7; do venv/bin/pip install click==$clk_ver; venv/bin/pytest -vvs tests/test_cliutils_progressbar.py; diff --git a/requirements.txt b/requirements.txt index ec24059..0dbf9fe 100644 --- a/requirements.txt +++ b/requirements.txt @@ -3,8 +3,7 @@ beautifulsoup4==4.13.4 certifi==2025.4.26 chardet==5.2.0 charset-normalizer==3.4.2 -click==8.2.1;python_version>='3.10' -click==8.1.8;python_version<'3.10' +click==8.3.0 idna==3.10 pip==25.1.1 PyYAML==6.0.2 diff --git a/setup.cfg b/setup.cfg index 667e65e..75df2cc 100644 --- a/setup.cfg +++ b/setup.cfg @@ -31,7 +31,7 @@ license_files = commoncode.ABOUT [options] -python_requires = >=3.9 +python_requires = >=3.10 package_dir = =src packages = find: @@ -44,8 +44,7 @@ install_requires = attrs >= 18.1,!=20.1.0;python_version<'3.11' attrs >= 22.1.0;python_version>='3.11' Beautifulsoup4[chardet] >= 4.13.0 - click >= 6.7, !=7.0;python_version<'3.10' - click >= 8.2.0;python_version>='3.10' + click >= 8.2.0 requests[use_chardet_on_py3] >= 2.7.0 saneyaml >= 0.5.2 text_unidecode >= 1.0 diff --git a/src/commoncode/cliutils.py b/src/commoncode/cliutils.py index a9273a5..8290db4 100644 --- a/src/commoncode/cliutils.py +++ b/src/commoncode/cliutils.py @@ -177,20 +177,9 @@ def format_options(self, ctx, formatter): formatter.write_dl(sorted_records) -class CompatProgressBar(ProgressBar): - # TODO Remove when dropping support for Python 3.9 or Click 8.1. - @property - def is_hidden(self) -> bool: - return self.hidden - - @is_hidden.setter - def is_hidden(self, value: bool) -> None: - self.hidden = value - - # overriden and copied from Click to work around Click woes for # https://github.com/aboutcode-org/scancode-toolkit/issues/2583 -class DebuggedProgressBar(CompatProgressBar): +class DebuggedProgressBar(ProgressBar): # overriden and copied from Click to work around Click woes for # https://github.com/aboutcode-org/scancode-toolkit/issues/2583 def make_step(self, n_steps): @@ -224,7 +213,7 @@ def render_progress(self): return super(EnhancedProgressBar, self).render_progress() -class ProgressLogger(CompatProgressBar): +class ProgressLogger(ProgressBar): """ A subclass of Click ProgressBar providing a verbose line-by-line progress reporting. From a4f89a158e7ea8731758734919edf1d1030d2762 Mon Sep 17 00:00:00 2001 From: Ayan Sinha Mahapatra Date: Tue, 21 Oct 2025 19:52:50 +0530 Subject: [PATCH 03/11] Remove --wheel option from virtualenv configure This removes a warning seen in initial virtualenv configuration. Signed-off-by: Ayan Sinha Mahapatra --- configure | 2 +- configure.bat | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/configure b/configure index 5ef0e06..6d317d4 100755 --- a/configure +++ b/configure @@ -110,7 +110,7 @@ create_virtualenv() { fi $PYTHON_EXECUTABLE "$VIRTUALENV_PYZ" \ - --wheel embed --pip embed --setuptools embed \ + --pip embed --setuptools embed \ --seeder pip \ --never-download \ --no-periodic-update \ diff --git a/configure.bat b/configure.bat index 3e9881f..58ed98e 100644 --- a/configure.bat +++ b/configure.bat @@ -110,7 +110,7 @@ if not exist "%CFG_BIN_DIR%\python.exe" ( if exist "%CFG_ROOT_DIR%\etc\thirdparty\virtualenv.pyz" ( %PYTHON_EXECUTABLE% "%CFG_ROOT_DIR%\etc\thirdparty\virtualenv.pyz" ^ - --wheel embed --pip embed --setuptools embed ^ + --pip embed --setuptools embed ^ --seeder pip ^ --never-download ^ --no-periodic-update ^ @@ -126,7 +126,7 @@ if not exist "%CFG_BIN_DIR%\python.exe" ( ) ) %PYTHON_EXECUTABLE% "%CFG_ROOT_DIR%\%VIRTUALENV_DIR%\virtualenv.pyz" ^ - --wheel embed --pip embed --setuptools embed ^ + embed --pip embed --setuptools embed ^ --seeder pip ^ --never-download ^ --no-periodic-update ^ From 2e593d0fbc9d74e70bef5c159aae5948fb591ca0 Mon Sep 17 00:00:00 2001 From: Ayan Sinha Mahapatra Date: Tue, 21 Oct 2025 19:56:52 +0530 Subject: [PATCH 04/11] Replace is_hidden with hidden in ProgressBar Reference: https://github.com/aboutcode-org/scancode-toolkit/issues/4369 Signed-off-by: Ayan Sinha Mahapatra --- src/commoncode/cliutils.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/src/commoncode/cliutils.py b/src/commoncode/cliutils.py index 8290db4..0a5eccb 100644 --- a/src/commoncode/cliutils.py +++ b/src/commoncode/cliutils.py @@ -190,7 +190,7 @@ def make_step(self, n_steps): # overriden and copied from Click to work around Click woes for # https://github.com/aboutcode-org/scancode-toolkit/issues/2583 def generator(self): - if self.is_hidden: + if self.hidden: yield from self.iter else: for rv in self.iter: @@ -209,7 +209,7 @@ class EnhancedProgressBar(DebuggedProgressBar): """ def render_progress(self): - if not self.is_hidden: + if not self.hidden: return super(EnhancedProgressBar, self).render_progress() @@ -230,7 +230,7 @@ class ProgressLogger(ProgressBar): def __init__(self, *args, **kwargs): super(ProgressLogger, self).__init__(*args, **kwargs) - self.is_hidden = False + self.hidden = False def render_progress(self): line = self.format_progress_line() From 1af7a71958632d69fc4ffb34545c93cc81321728 Mon Sep 17 00:00:00 2001 From: Ayan Sinha Mahapatra Date: Tue, 21 Oct 2025 20:03:24 +0530 Subject: [PATCH 05/11] Update azure pipelines supported python versions Signed-off-by: Ayan Sinha Mahapatra --- azure-pipelines.yml | 16 ++++++++-------- 1 file changed, 8 insertions(+), 8 deletions(-) diff --git a/azure-pipelines.yml b/azure-pipelines.yml index 6c013b4..1946a60 100644 --- a/azure-pipelines.yml +++ b/azure-pipelines.yml @@ -13,7 +13,7 @@ jobs: parameters: job_name: ubuntu22_cpython image_name: ubuntu-22.04 - python_versions: ["3.9", "3.10", "3.11", "3.12", "3.13"] + python_versions: ["3.10", "3.11", "3.12", "3.13", "3.14"] test_suites: all: venv/bin/pytest -n 2 -vvs @@ -21,7 +21,7 @@ jobs: parameters: job_name: ubuntu24_cpython image_name: ubuntu-24.04 - python_versions: ["3.9", "3.10", "3.11", "3.12", "3.13"] + python_versions: ["3.10", "3.11", "3.12", "3.13", "3.14"] test_suites: all: venv/bin/pytest -n 2 -vvs @@ -29,7 +29,7 @@ jobs: parameters: job_name: macos13_cpython image_name: macOS-13 - python_versions: ["3.9", "3.10", "3.11", "3.12", "3.13"] + python_versions: ["3.10", "3.11", "3.12", "3.13", "3.14"] test_suites: all: venv/bin/pytest -n 2 -vvs @@ -37,7 +37,7 @@ jobs: parameters: job_name: macos14_cpython image_name: macOS-14 - python_versions: ['3.9', '3.10', '3.11', '3.12', '3.13'] + python_versions: ["3.10", "3.11", "3.12", "3.13", "3.14"] test_suites: all: venv/bin/pytest -n 2 -vvs @@ -45,7 +45,7 @@ jobs: parameters: job_name: win2025_cpython image_name: windows-2025 - python_versions: ["3.9", "3.10", "3.11", "3.12", "3.13"] + python_versions: ["3.10", "3.11", "3.12", "3.13", "3.14"] test_suites: all: venv\Scripts\pytest -n 2 -vvs @@ -53,7 +53,7 @@ jobs: parameters: job_name: win2022_cpython image_name: windows-2022 - python_versions: ["3.9", "3.10", "3.11", "3.12", "3.13"] + python_versions: ["3.10", "3.11", "3.12", "3.13", "3.14"] test_suites: all: venv\Scripts\pytest -n 2 -vvs @@ -82,7 +82,7 @@ jobs: parameters: job_name: ubuntu24_cpython_latest_from_pip image_name: ubuntu-24.04 - python_versions: ["3.9", "3.10", "3.11", "3.12", "3.13"] + python_versions: ["3.10", "3.11", "3.12", "3.13", "3.14"] test_suites: all: | venv/bin/pip install --upgrade-strategy eager --force-reinstall --upgrade -e . @@ -93,7 +93,7 @@ jobs: parameters: job_name: win2022_cpython_latest_from_pip image_name: windows-2022 - python_versions: ["3.9", "3.10", "3.11", "3.12", "3.13"] + python_versions: ["3.10", "3.11", "3.12", "3.13", "3.14"] test_suites: all: | venv\Scripts\pip install --upgrade-strategy eager --force-reinstall --upgrade -e . From b8372311550088bcbf1d4dbcde7aec28436cd567 Mon Sep 17 00:00:00 2001 From: Ayan Sinha Mahapatra Date: Wed, 22 Oct 2025 17:10:59 +0530 Subject: [PATCH 06/11] Bump requirements to support python3.14 Signed-off-by: Ayan Sinha Mahapatra --- etc/scripts/utils_requirements.py | 2 +- requirements-dev.txt | 6 +++--- requirements.txt | 16 ++++++++-------- 3 files changed, 12 insertions(+), 12 deletions(-) diff --git a/etc/scripts/utils_requirements.py b/etc/scripts/utils_requirements.py index b9b2c0e..424bed2 100644 --- a/etc/scripts/utils_requirements.py +++ b/etc/scripts/utils_requirements.py @@ -153,7 +153,7 @@ def split_req(req): if not req: raise ValueError("req is required") # do not allow multiple constraints and tags - if not any(c in req for c in ",;"): + if any(c in req for c in ",;"): raise Exception(f"complex requirements with : or ; not supported: {req}") req = "".join(req.split()) if not any(c in req for c in comparators): diff --git a/requirements-dev.txt b/requirements-dev.txt index 0dc3378..02135b3 100644 --- a/requirements-dev.txt +++ b/requirements-dev.txt @@ -2,7 +2,7 @@ aboutcode-toolkit==11.1.1 black==23.1.0 bleach==6.0.0 boolean.py==4.0 -cffi==1.15.1 +cffi==2.0.0 cryptography==39.0.1 docutils==0.19 et-xmlfile==1.1.0 @@ -27,11 +27,11 @@ packaging==23.0 pathspec==0.11.0 pkginfo==1.9.6 platformdirs==3.0.0 -pluggy==1.0.0 +pluggy==1.6.0 pycodestyle==2.13.0 pycparser==2.21 pygments==2.14.0 -pytest==7.2.1 +pytest==8.4.2 pytest-xdist==3.2.0 readme-renderer==37.3 requests-toolbelt==0.10.1 diff --git a/requirements.txt b/requirements.txt index 0dbf9fe..665ff89 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1,17 +1,17 @@ -attrs==25.3.0 -beautifulsoup4==4.13.4 -certifi==2025.4.26 +attrs==25.4.0 +beautifulsoup4==4.14.2 +certifi==2025.10.5 chardet==5.2.0 -charset-normalizer==3.4.2 +charset-normalizer==3.4.4 click==8.3.0 idna==3.10 pip==25.1.1 PyYAML==6.0.2 requests==2.32.2 saneyaml==0.6.1 -setuptools==80.3.1 -soupsieve==2.7 +setuptools==80.9.0 +soupsieve==2.8 text-unidecode==1.3 -typing_extensions==4.13.2 -urllib3==1.26.20 +typing_extensions==4.15.0 +urllib3==2.5.0 wheel==0.38.4 \ No newline at end of file From 72b737bff26db96ba17b324bca31f6ad59c71839 Mon Sep 17 00:00:00 2001 From: Ayan Sinha Mahapatra Date: Wed, 22 Oct 2025 17:11:20 +0530 Subject: [PATCH 07/11] Remove special file to fix py3.14 tests Remove special file '6-FIFOTYPE' from types.tar to fix test failure as in py3.14 tarfiles extraction fails with tarfile.SpecialFileError for special files. Reference: https://github.com/aboutcode-org/commoncode/issues/88 Signed-off-by: Ayan Sinha Mahapatra --- tests/data/filetype/types.tar | Bin 109056 -> 112640 bytes tests/test_filetype.py | 1 - 2 files changed, 1 deletion(-) diff --git a/tests/data/filetype/types.tar b/tests/data/filetype/types.tar index 458dc4476efa3cb3b3507135af450adc925a1bfa..24fc04c44da43ac26ebb53a04feca6c8d1a04342 100644 GIT binary patch delta 846 zcmb7?Pfo%>6vn5dshGF`G?Hi+a{_sf&Yw#d&tOSobOQ!3LCT^r#5HYqZcJEs062jc zaHB_ny(civYeYb8QfD)1lli^xd*2tmInldQI%%Z7)k^McskLMPfLh|_ZGL5kp#-f^ zASjOkqV}4MOY=CW-t_u@mviRd`t9CT`_hlA_1*7}^ttk4#v}`xvaf{Lq|>xCT)2)z zU>pkI=Kk`r67$A%%%2Xe36Vz`S<7Gg({SWEP4tQjO)8oh8GKvB#n?QL4B5OR>Wx;S(llqQHGn_B}XdAu|n zxMiV`am>aRjJB4~s~zunSyhTEoE?!f0^=ATTpAVE|Hw=B8lUz|h2Wa`hbAM@K!$(-P{)czX9W1*7U_n#MubdU7>3)n z57O)=hGyhjx9Jm;-(;p^E#f8&h6V=a<|byQCI*HK28M=4<|YgZJ|Iu9Lp?E3&=H%9 zH#431$H>IMFq!E@3T}sWbiIafemMbq#<)tQ!Iwd zu$h<`Gbosnmxj2^blp7N{2_jb!W}0^fYz8BFesRjXAMWNZg7BOP%tn*#Ne`IdbI Date: Wed, 22 Oct 2025 17:15:33 +0530 Subject: [PATCH 08/11] Fix test failures Signed-off-by: Ayan Sinha Mahapatra --- configure.bat | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/configure.bat b/configure.bat index 58ed98e..15ab701 100644 --- a/configure.bat +++ b/configure.bat @@ -126,7 +126,7 @@ if not exist "%CFG_BIN_DIR%\python.exe" ( ) ) %PYTHON_EXECUTABLE% "%CFG_ROOT_DIR%\%VIRTUALENV_DIR%\virtualenv.pyz" ^ - embed --pip embed --setuptools embed ^ + --pip embed --setuptools embed ^ --seeder pip ^ --never-download ^ --no-periodic-update ^ From 810f63f64c2f58a038f1e46509c48df4b131137a Mon Sep 17 00:00:00 2001 From: Ayan Sinha Mahapatra Date: Wed, 22 Oct 2025 17:28:20 +0530 Subject: [PATCH 09/11] Revert to CompatProgressBar to support old click versions Signed-off-by: Ayan Sinha Mahapatra --- setup.cfg | 2 +- src/commoncode/cliutils.py | 15 +++++++++++++-- 2 files changed, 14 insertions(+), 3 deletions(-) diff --git a/setup.cfg b/setup.cfg index 75df2cc..812f168 100644 --- a/setup.cfg +++ b/setup.cfg @@ -44,7 +44,7 @@ install_requires = attrs >= 18.1,!=20.1.0;python_version<'3.11' attrs >= 22.1.0;python_version>='3.11' Beautifulsoup4[chardet] >= 4.13.0 - click >= 8.2.0 + click >= 8.3.0 requests[use_chardet_on_py3] >= 2.7.0 saneyaml >= 0.5.2 text_unidecode >= 1.0 diff --git a/src/commoncode/cliutils.py b/src/commoncode/cliutils.py index 0a5eccb..5b0a01e 100644 --- a/src/commoncode/cliutils.py +++ b/src/commoncode/cliutils.py @@ -177,9 +177,20 @@ def format_options(self, ctx, formatter): formatter.write_dl(sorted_records) +class CompatProgressBar(ProgressBar): + # TODO Remove when dropping support for Click 8.1. + @property + def is_hidden(self) -> bool: + return self.hidden + + @is_hidden.setter + def is_hidden(self, value: bool) -> None: + self.hidden = value + + # overriden and copied from Click to work around Click woes for # https://github.com/aboutcode-org/scancode-toolkit/issues/2583 -class DebuggedProgressBar(ProgressBar): +class DebuggedProgressBar(CompatProgressBar): # overriden and copied from Click to work around Click woes for # https://github.com/aboutcode-org/scancode-toolkit/issues/2583 def make_step(self, n_steps): @@ -213,7 +224,7 @@ def render_progress(self): return super(EnhancedProgressBar, self).render_progress() -class ProgressLogger(ProgressBar): +class ProgressLogger(CompatProgressBar): """ A subclass of Click ProgressBar providing a verbose line-by-line progress reporting. From 495ef21837f1784f4cca9a995a085dc0dcf1ea26 Mon Sep 17 00:00:00 2001 From: Ayan Sinha Mahapatra Date: Wed, 22 Oct 2025 18:10:45 +0530 Subject: [PATCH 10/11] Bump CHANGELOG for v32.4.0 Signed-off-by: Ayan Sinha Mahapatra --- CHANGELOG.rst | 14 +++++++++++++- 1 file changed, 13 insertions(+), 1 deletion(-) diff --git a/CHANGELOG.rst b/CHANGELOG.rst index 54a4973..4550c80 100644 --- a/CHANGELOG.rst +++ b/CHANGELOG.rst @@ -1,7 +1,19 @@ Release notes ============= -Version 32.3.0 - (2025-03-06) +Version 32.4.0 - (2025-10-22) +----------------------------- + +- Fix ``click`` 8.3.0 compatibililty issues. + https://github.com/aboutcode-org/commoncode/pull/92 + +- Drop python 3.9 support and add python 3.14 + https://github.com/aboutcode-org/commoncode/pull/92 + +- Handle paths with non-utf-8 bytes + https://github.com/aboutcode-org/commoncode/pull/91 + +Version 32.3.0 - (2025-06-11) ----------------------------- - Fix ``click`` compatibililty issues. From b404b0b8558bb388dd4c074e7c70e691cb03c814 Mon Sep 17 00:00:00 2001 From: Ayan Sinha Mahapatra Date: Wed, 22 Oct 2025 20:43:57 +0530 Subject: [PATCH 11/11] Update pypi publishing to use Trusted Publishing Signed-off-by: Ayan Sinha Mahapatra --- .github/workflows/pypi-release.yml | 3 +++ 1 file changed, 3 insertions(+) diff --git a/.github/workflows/pypi-release.yml b/.github/workflows/pypi-release.yml index cf0579a..294039d 100644 --- a/.github/workflows/pypi-release.yml +++ b/.github/workflows/pypi-release.yml @@ -71,6 +71,9 @@ jobs: needs: - create-gh-release runs-on: ubuntu-24.04 + environment: pypi-publish + permissions: + id-token: write steps: - name: Download built archives