Merge pull request #225 from scrapy-plugins/modernize

Add Python 3.13, drop Python 3.8, update tool versions

Author: wRAR

.github/workflows/publish.yml

@@ -17,7 +17,7 @@ jobs:
     - name: Set up Python
       uses: actions/setup-python@v5
       with:
-        python-version: '3.12'
+        python-version: '3.13'
     - name: Install dependencies
       run: |
         python -m pip install --upgrade pip

.github/workflows/test.yml

@@ -17,31 +17,31 @@ jobs:
       fail-fast: false
       matrix:
         include:
-        - python-version: '3.8'
+        - python-version: '3.9'
           toxenv: pinned-scrapy-2x0
-        - python-version: '3.8'
+        - python-version: '3.9'
           toxenv: pinned-scrapy-2x1
-        - python-version: '3.8'
+        - python-version: '3.9'
           toxenv: pinned-scrapy-2x3
-        - python-version: '3.8'
+        - python-version: '3.9'
           toxenv: pinned-scrapy-2x4
-        - python-version: '3.8'
+        - python-version: '3.9'
           toxenv: pinned-scrapy-2x5
-        - python-version: '3.8'
-          toxenv: pinned-scrapy-2x6
         - python-version: '3.9'
+          toxenv: pinned-scrapy-2x6
         - python-version: '3.10'
         - python-version: '3.11'
         - python-version: '3.12'
+        - python-version: '3.13'
 
-        - python-version: '3.8'
+        - python-version: '3.9'
           toxenv: pinned-provider
-        - python-version: '3.12'
+        - python-version: '3.13'
           toxenv: provider
 
-        - python-version: '3.8'
+        - python-version: '3.9'
           toxenv: pinned-extra
-        - python-version: '3.12'
+        - python-version: '3.13'
           toxenv: extra
 
     steps:
@@ -67,7 +67,7 @@ jobs:
     strategy:
       fail-fast: false
       matrix:
-        python-version: ["3.12"]
+        python-version: ["3.12"]  # Keep in sync with .readthedocs.yml
         tox-job: ["mypy", "linters", "twine-check", "docs"]
 
     steps:

.pre-commit-config.yaml

@@ -4,19 +4,19 @@ repos:
     hooks:
     - id: isort
 -   repo: https://github.com/psf/black
-    rev: 24.2.0
+    rev: 24.10.0
     hooks:
     - id: black
 -   repo: https://github.com/pycqa/flake8
-    rev: 7.1.0
+    rev: 7.1.1
     hooks:
     - id: flake8
       additional_dependencies:
         - flake8-docstrings
         - flake8-print
 - repo: https://github.com/adamchainz/blacken-docs
-  rev: 1.16.0
+  rev: 1.19.0
   hooks:
     - id: blacken-docs
       additional_dependencies:
-        - black==24.2.0
+        - black==24.10.0

.readthedocs.yml

@@ -5,7 +5,7 @@ sphinx:
 build:
   os: ubuntu-22.04
   tools:
-    python: "3.11"  # Keep in sync with .github/workflows/test.yml
+    python: "3.12"  # Keep in sync with .github/workflows/test.yml
 python:
   install:
     - requirements: docs/requirements.txt

docs/setup.rst

@@ -18,7 +18,7 @@ You need at least:
 -   A :ref:`Zyte API <zyte-api>` subscription (there’s a :ref:`free trial
     <zapi-trial>`).
 
--   Python 3.8+
+-   Python 3.9+
 
 -   Scrapy 2.0.1+
 

docs/usage/scrapy-poet.rst

@@ -39,8 +39,6 @@ Dependency annotations
 
 ``ZyteApiProvider`` understands and makes use of some dependency annotations.
 
-.. note:: Dependency annotations require Python 3.9+.
-
 Item annotations
 ----------------
 

pyproject.toml

@@ -1,5 +1,5 @@
 [tool.black]
-target-version = ["py38", "py39", "py310", "py311"]
+target-version = ["py39", "py310", "py311", "py312", "py313"]
 
 [tool.isort]
 profile = "black"

setup.py

@@ -43,10 +43,10 @@ def get_version():
         "Natural Language :: English",
         "Operating System :: OS Independent",
         "Programming Language :: Python :: 3",
-        "Programming Language :: Python :: 3.8",
         "Programming Language :: Python :: 3.9",
         "Programming Language :: Python :: 3.10",
         "Programming Language :: Python :: 3.11",
         "Programming Language :: Python :: 3.12",
+        "Programming Language :: Python :: 3.13",
     ],
 )

tests/test_providers.py

@@ -1,5 +1,5 @@
-import sys
 from collections import defaultdict
+from typing import Annotated
 
 import pytest
 
@@ -259,13 +259,8 @@ async def test_provider_params_remove_unused_options(mockserver):
     )
 
 
-@pytest.mark.skipif(
-    sys.version_info < (3, 9), reason="No Annotated support in Python < 3.9"
-)
 @ensureDeferred
 async def test_provider_extractfrom(mockserver):
-    from typing import Annotated
-
     @attrs.define
     class AnnotatedProductPage(BasePage):
         product: Annotated[Product, ExtractFrom.httpResponseBody]
@@ -295,13 +290,8 @@ def parse_(self, response: DummyResponse, page: AnnotatedProductPage):  # type:
     )
 
 
-@pytest.mark.skipif(
-    sys.version_info < (3, 9), reason="No Annotated support in Python < 3.9"
-)
 @ensureDeferred
 async def test_provider_extractfrom_double(mockserver, caplog):
-    from typing import Annotated
-
     @attrs.define
     class AnnotatedProductPage(BasePage):
         product: Annotated[Product, ExtractFrom.httpResponseBody]
@@ -322,13 +312,8 @@ def parse_(self, response: DummyResponse, page: AnnotatedProductPage):  # type:
     assert "Multiple different extractFrom specified for product" in caplog.text
 
 
-@pytest.mark.skipif(
-    sys.version_info < (3, 9), reason="No Annotated support in Python < 3.9"
-)
 @ensureDeferred
 async def test_provider_extractfrom_override(mockserver):
-    from typing import Annotated
-
     @attrs.define
     class AnnotatedProductPage(BasePage):
         product: Annotated[Product, ExtractFrom.httpResponseBody]
@@ -359,13 +344,8 @@ def parse_(self, response: DummyResponse, page: AnnotatedProductPage):  # type:
     )
 
 
-@pytest.mark.skipif(
-    sys.version_info < (3, 9), reason="No Annotated support in Python < 3.9"
-)
 @ensureDeferred
 async def test_provider_geolocation(mockserver):
-    from typing import Annotated
-
     @attrs.define
     class GeoProductPage(BasePage):
         product: Product
@@ -385,9 +365,6 @@ def parse_(self, response: DummyResponse, page: GeoProductPage):  # type: ignore
     assert item["product"].name == "Product name (country DE)"
 
 
-@pytest.mark.skipif(
-    sys.version_info < (3, 9), reason="No Annotated support in Python < 3.9"
-)
 @ensureDeferred
 async def test_provider_geolocation_unannotated(mockserver, caplog):
     @attrs.define
@@ -414,9 +391,6 @@ def parse_(self, response: DummyResponse, page: GeoProductPage):  # type: ignore
 }
 
 
-@pytest.mark.skipif(
-    sys.version_info < (3, 9), reason="No Annotated support in Python < 3.9"
-)
 @pytest.mark.parametrize(
     "annotation",
     [
@@ -428,8 +402,6 @@ def parse_(self, response: DummyResponse, page: GeoProductPage):  # type: ignore
 )
 @ensureDeferred
 async def test_provider_custom_attrs(mockserver, annotation):
-    from typing import Annotated
-
     @attrs.define
     class CustomAttrsPage(BasePage):
         product: Product
@@ -468,13 +440,8 @@ def parse_(self, response: DummyResponse, page: CustomAttrsPage):  # type: ignor
     )
 
 
-@pytest.mark.skipif(
-    sys.version_info < (3, 9), reason="No Annotated support in Python < 3.9"
-)
 @ensureDeferred
 async def test_provider_custom_attrs_values(mockserver):
-    from typing import Annotated
-
     @attrs.define
     class CustomAttrsPage(BasePage):
         product: Product
@@ -1086,13 +1053,8 @@ def parse_(self, response: DummyResponse, screenshot: Screenshot):
     assert item["screenshot"].body == b"screenshot-body-contents"
 
 
-@pytest.mark.skipif(
-    sys.version_info < (3, 9), reason="No Annotated support in Python < 3.9"
-)
 @ensureDeferred
 async def test_provider_actions(mockserver, caplog):
-    from typing import Annotated
-
     @attrs.define
     class ActionProductPage(BasePage):
         product: Product

tox.ini

@@ -1,5 +1,5 @@
 [tox]
-envlist = py38,py39,py310,py311,mypy,linters,twine-check,docs
+envlist = py39,py310,py311,py312,py313,mypy,linters,twine-check,docs
 
 [testenv]
 deps =
@@ -37,43 +37,43 @@ deps =
 
 # Earliest supported Scrapy version.
 [testenv:pinned-scrapy-2x0]
-basepython=python3.8
+basepython=python3.9
 deps =
     {[pinned-pre-scrapy-2x5]deps}
     scrapy==2.0.1
 
 # Scrapy version introducing Response.ip_address.
 [testenv:pinned-scrapy-2x1]
-basepython=python3.8
+basepython=python3.9
 deps =
     {[pinned-pre-scrapy-2x5]deps}
     scrapy==2.1.0
 
 # Latest Scrapy version since 2.0.1 not requiring to install the reactor early.
 [testenv:pinned-scrapy-2x3]
-basepython=python3.8
+basepython=python3.9
 deps =
     {[pinned-pre-scrapy-2x5]deps}
     scrapy==2.3.0
 
 # First Scrapy version requiring to install the reactor early.
 [testenv:pinned-scrapy-2x4]
-basepython=python3.8
+basepython=python3.9
 deps =
     {[pinned-pre-scrapy-2x5]deps}
     scrapy==2.4.0
 
 # Scrapy version introducing Response.protocol.
 [testenv:pinned-scrapy-2x5]
-basepython=python3.8
+basepython=python3.9
 deps =
     {[pinned]deps}
     scrapy==2.5.0
 
 # First Scrapy version since 2.4.0 where installing the reactor earlier is not
 # necessary.
 [testenv:pinned-scrapy-2x6]
-basepython=python3.8
+basepython=python3.9
 deps =
     {[pinned]deps}
     scrapy==2.6.0
@@ -82,7 +82,7 @@ deps =
 extras = provider
 
 [testenv:pinned-provider]
-basepython=python3.8
+basepython=python3.9
 extras = provider
 deps =
     # scrapy-poet >= 0.4.0 depends on scrapy >= 2.6.0
@@ -93,14 +93,14 @@ deps =
     zyte-common-items==0.24.0
 
 [testenv:pinned-extra]
-basepython=python3.8
+basepython=python3.9
 deps =
     {[testenv:pinned-scrapy-2x0]deps}
     scrapy-crawlera==1.1.0
     scrapy-zyte-smartproxy==2.0.0
 
 [testenv:extra]
-basepython=python3.12
+basepython=python3.13
 deps =
     {[testenv]deps}
     scrapy-crawlera
@@ -109,8 +109,8 @@ deps =
 [testenv:mypy]
 extras = provider
 deps =
-    mypy==1.8.0
-    types-setuptools
+    mypy==1.11.2
+    pytest
 
 commands = mypy scrapy_zyte_api tests
 
@@ -120,9 +120,10 @@ commands = pre-commit run --all-files --show-diff-on-failure
 
 [testenv:twine-check]
 deps =
-    twine
+    twine==5.1.1
+    build==1.2.2
 commands =
-    python setup.py sdist
+    python -m build --sdist
     twine check dist/*
 
 [testenv:docs]