From e742493b02aef8010fc70d9aba9bfd1a68302828 Mon Sep 17 00:00:00 2001 From: Andrey Rakhmatullin Date: Sat, 26 Jul 2025 20:11:24 +0500 Subject: [PATCH 1/7] Add Python 3.14 RC1 to CI. --- .github/workflows/tests-macos.yml | 2 +- .github/workflows/tests-ubuntu.yml | 2 +- .github/workflows/tests-windows.yml | 2 +- tox.ini | 2 +- 4 files changed, 4 insertions(+), 4 deletions(-) diff --git a/.github/workflows/tests-macos.yml b/.github/workflows/tests-macos.yml index 73788ff..84d3a45 100644 --- a/.github/workflows/tests-macos.yml +++ b/.github/workflows/tests-macos.yml @@ -15,7 +15,7 @@ jobs: strategy: fail-fast: false matrix: - python-version: ["3.9", "3.10", "3.11", "3.12", "3.13"] + python-version: ["3.9", "3.10", "3.11", "3.12", "3.13", "3.14rc1"] steps: - uses: actions/checkout@v4 diff --git a/.github/workflows/tests-ubuntu.yml b/.github/workflows/tests-ubuntu.yml index 566a9cc..666bcde 100644 --- a/.github/workflows/tests-ubuntu.yml +++ b/.github/workflows/tests-ubuntu.yml @@ -15,7 +15,7 @@ jobs: strategy: fail-fast: false matrix: - python-version: ["3.9", "3.10", "3.11", "3.12", "3.13", "pypy3.10", "pypy3.11"] + python-version: ["3.9", "3.10", "3.11", "3.12", "3.13", "3.14rc1", "pypy3.10", "pypy3.11"] steps: - uses: actions/checkout@v4 diff --git a/.github/workflows/tests-windows.yml b/.github/workflows/tests-windows.yml index cf8f0cc..3c8a114 100644 --- a/.github/workflows/tests-windows.yml +++ b/.github/workflows/tests-windows.yml @@ -15,7 +15,7 @@ jobs: strategy: fail-fast: false matrix: - python-version: ["3.9", "3.10", "3.11", "3.12", "3.13"] + python-version: ["3.9", "3.10", "3.11", "3.12", "3.13", "3.14rc1"] steps: - uses: actions/checkout@v4 diff --git a/tox.ini b/tox.ini index 0e31606..d1dcbfe 100644 --- a/tox.ini +++ b/tox.ini @@ -4,7 +4,7 @@ # and then run "tox" from this directory. [tox] -envlist = py39, py310, py311, py312, py313, pypy3.10, docs, pylint, typing, pre-commit, twinecheck +envlist = py39, py310, py311, py312, py313, py314, pypy3.10, pypy3.11, docs, pylint, typing, pre-commit, twinecheck [testenv] deps = From c006579713286fe36664e56c36bf5e3e6017de21 Mon Sep 17 00:00:00 2001 From: Andrey Rakhmatullin Date: Sat, 26 Jul 2025 20:13:14 +0500 Subject: [PATCH 2/7] Fix version number spelling. --- .github/workflows/tests-macos.yml | 2 +- .github/workflows/tests-ubuntu.yml | 2 +- .github/workflows/tests-windows.yml | 2 +- 3 files changed, 3 insertions(+), 3 deletions(-) diff --git a/.github/workflows/tests-macos.yml b/.github/workflows/tests-macos.yml index 84d3a45..30fe097 100644 --- a/.github/workflows/tests-macos.yml +++ b/.github/workflows/tests-macos.yml @@ -15,7 +15,7 @@ jobs: strategy: fail-fast: false matrix: - python-version: ["3.9", "3.10", "3.11", "3.12", "3.13", "3.14rc1"] + python-version: ["3.9", "3.10", "3.11", "3.12", "3.13", "3.14.0-rc.1"] steps: - uses: actions/checkout@v4 diff --git a/.github/workflows/tests-ubuntu.yml b/.github/workflows/tests-ubuntu.yml index 666bcde..2358254 100644 --- a/.github/workflows/tests-ubuntu.yml +++ b/.github/workflows/tests-ubuntu.yml @@ -15,7 +15,7 @@ jobs: strategy: fail-fast: false matrix: - python-version: ["3.9", "3.10", "3.11", "3.12", "3.13", "3.14rc1", "pypy3.10", "pypy3.11"] + python-version: ["3.9", "3.10", "3.11", "3.12", "3.13", "3.14.0-rc.1", "pypy3.10", "pypy3.11"] steps: - uses: actions/checkout@v4 diff --git a/.github/workflows/tests-windows.yml b/.github/workflows/tests-windows.yml index 3c8a114..9087e72 100644 --- a/.github/workflows/tests-windows.yml +++ b/.github/workflows/tests-windows.yml @@ -15,7 +15,7 @@ jobs: strategy: fail-fast: false matrix: - python-version: ["3.9", "3.10", "3.11", "3.12", "3.13", "3.14rc1"] + python-version: ["3.9", "3.10", "3.11", "3.12", "3.13", "3.14.0-rc.1"] steps: - uses: actions/checkout@v4 From 042b2f703e96118a2218c49fd77a151879a848fb Mon Sep 17 00:00:00 2001 From: Andrey Rakhmatullin Date: Sat, 26 Jul 2025 20:18:26 +0500 Subject: [PATCH 3/7] Bump ruff. --- .pre-commit-config.yaml | 4 ++-- docs/conf.py | 2 +- pyproject.toml | 11 +++++++++++ tests/test_encoding.py | 5 ++--- 4 files changed, 16 insertions(+), 6 deletions(-) diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index 3c92c4d..a8eebd9 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -1,7 +1,7 @@ repos: - repo: https://github.com/astral-sh/ruff-pre-commit - rev: v0.11.2 + rev: v0.12.5 hooks: - - id: ruff + - id: ruff-check args: [ --fix ] - id: ruff-format diff --git a/docs/conf.py b/docs/conf.py index 9502c72..a722875 100644 --- a/docs/conf.py +++ b/docs/conf.py @@ -46,7 +46,7 @@ # General information about the project. project = "w3lib" -copyright = "2014, w3lib developers" +project_copyright = "2014, w3lib developers" # The version info for the project you're documenting, acts as replacement for # |version| and |release|, also used in various other places throughout the diff --git a/pyproject.toml b/pyproject.toml index 58f05a6..ed91c17 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -116,10 +116,16 @@ disable = [ [tool.ruff.lint] extend-select = [ + # flake8-builtins + "A", + # flake8-async + "ASYNC", # flake8-bugbear "B", # flake8-comprehensions "C4", + # flake8-commas + "COM", # pydocstyle "D", # flake8-future-annotations @@ -176,6 +182,8 @@ extend-select = [ ignore = [ # Within an `except` clause, raise exceptions with `raise ... from` "B904", + # Trailing comma missing + "COM812", # Missing docstring in public module "D100", # Missing docstring in public class @@ -232,5 +240,8 @@ ignore = [ "S101", ] +[tool.ruff.lint.isort] +split-on-trailing-comma = false + [tool.ruff.lint.pydocstyle] convention = "pep257" diff --git a/tests/test_encoding.py b/tests/test_encoding.py index 4f00615..3502ed7 100644 --- a/tests/test_encoding.py +++ b/tests/test_encoding.py @@ -1,6 +1,8 @@ from __future__ import annotations import codecs +import random +from io import BytesIO from typing import Any from w3lib.encoding import ( @@ -266,9 +268,6 @@ def test_utf16_32(self): self._assert_encoding("utf-32", "hi".encode("utf-32-be"), "utf-32-be", "hi") def test_python_crash(self): - import random - from io import BytesIO - random.seed(42) buf = BytesIO() for _ in range(150000): From 9b4813126dc9ee17b73823858eb47a201ddd18fc Mon Sep 17 00:00:00 2001 From: Andrey Rakhmatullin Date: Sat, 26 Jul 2025 20:20:05 +0500 Subject: [PATCH 4/7] Add more linters. --- .pre-commit-config.yaml | 14 ++++++++++++++ LICENSE | 6 +++--- 2 files changed, 17 insertions(+), 3 deletions(-) diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index a8eebd9..678dcf3 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -5,3 +5,17 @@ repos: - id: ruff-check args: [ --fix ] - id: ruff-format +- repo: https://github.com/adamchainz/blacken-docs + rev: 1.19.1 + hooks: + - id: blacken-docs + additional_dependencies: + - black==25.1.0 +- repo: https://github.com/pre-commit/pre-commit-hooks + rev: v5.0.0 + hooks: + - id: trailing-whitespace +- repo: https://github.com/sphinx-contrib/sphinx-lint + rev: v1.0.0 + hooks: + - id: sphinx-lint diff --git a/LICENSE b/LICENSE index b789a76..213dfe2 100644 --- a/LICENSE +++ b/LICENSE @@ -4,10 +4,10 @@ All rights reserved. Redistribution and use in source and binary forms, with or without modification, are permitted provided that the following conditions are met: - 1. Redistributions of source code must retain the above copyright notice, + 1. Redistributions of source code must retain the above copyright notice, this list of conditions and the following disclaimer. - - 2. Redistributions in binary form must reproduce the above copyright + + 2. Redistributions in binary form must reproduce the above copyright notice, this list of conditions and the following disclaimer in the documentation and/or other materials provided with the distribution. From 2a4a460cf3cc73555463dcfb3170e58d082affc0 Mon Sep 17 00:00:00 2001 From: Andrey Rakhmatullin Date: Sat, 26 Jul 2025 20:22:07 +0500 Subject: [PATCH 5/7] Bump pylint and mypy. --- tox.ini | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/tox.ini b/tox.ini index d1dcbfe..9a2de58 100644 --- a/tox.ini +++ b/tox.ini @@ -21,14 +21,14 @@ commands = basepython = python3 deps = pytest - mypy==1.15.0 + mypy==1.17.0 commands = mypy --strict {posargs: w3lib tests} [testenv:pylint] deps = {[testenv]deps} - pylint==3.3.6 + pylint==3.3.7 commands = pylint docs tests w3lib From 3c921d6c4140f87f0b1183240a87441352cf98bd Mon Sep 17 00:00:00 2001 From: Andrey Rakhmatullin Date: Sat, 26 Jul 2025 20:27:58 +0500 Subject: [PATCH 6/7] Update classifiers and project URLs. --- pyproject.toml | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/pyproject.toml b/pyproject.toml index ed91c17..105017d 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -17,13 +17,14 @@ classifiers = [ "Operating System :: OS Independent", "Programming Language :: Python", "Programming Language :: Python :: 3", - "Programming Language :: Python :: 3.13", "Programming Language :: Python :: 3.9", - "Programming Language :: Python :: Implementation :: CPython", - "Programming Language :: Python :: Implementation :: PyPy", "Programming Language :: Python :: 3.10", "Programming Language :: Python :: 3.11", "Programming Language :: Python :: 3.12", + "Programming Language :: Python :: 3.13", + "Programming Language :: Python :: 3.14", + "Programming Language :: Python :: Implementation :: CPython", + "Programming Language :: Python :: Implementation :: PyPy", "Topic :: Internet :: WWW/HTTP", "Topic :: Software Development :: Libraries :: Python Modules", ] @@ -35,8 +36,7 @@ Homepage = "https://github.com/scrapy/w3lib" Documentation = "https://w3lib.readthedocs.io/en/latest/" Source = "https://github.com/scrapy/w3lib" Tracker = "https://github.com/scrapy/w3lib/issues" -Changelog = "https://github.com/scrapy/w3lib/commits/master/" -releasenotes = "https://w3lib.readthedocs.io/en/latest/#changelog" +"Release notes" = "https://w3lib.readthedocs.io/en/latest/#changelog" [tool.hatch.version] path = "w3lib/__init__.py" From 5a264f9d1690f961c3939be0046e01ebe3174dc7 Mon Sep 17 00:00:00 2001 From: Andrey Rakhmatullin Date: Sat, 26 Jul 2025 21:07:36 +0500 Subject: [PATCH 7/7] Add the trailing-whitespace pre-commit hook. --- .git-blame-ignore-revs | 2 +- .pre-commit-config.yaml | 1 + 2 files changed, 2 insertions(+), 1 deletion(-) diff --git a/.git-blame-ignore-revs b/.git-blame-ignore-revs index e86e521..d75e246 100644 --- a/.git-blame-ignore-revs +++ b/.git-blame-ignore-revs @@ -1,3 +1,3 @@ # This is a list of commits that git blame should ignore # running pre-commit on all the code -47be6dd35faf78f06b6183ea9dad6b5f78f5c756 \ No newline at end of file +47be6dd35faf78f06b6183ea9dad6b5f78f5c756 diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index 678dcf3..119b328 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -14,6 +14,7 @@ repos: - repo: https://github.com/pre-commit/pre-commit-hooks rev: v5.0.0 hooks: + - id: end-of-file-fixer - id: trailing-whitespace - repo: https://github.com/sphinx-contrib/sphinx-lint rev: v1.0.0