Scrapy 2.14 compatibility (#100)

* Bump minimum versions

* Update DiskQuota middlewares

* Update HubstorageDownloaderMiddleware

* Update HubstorageSpiderMiddleware, rename flag

* Update HubStorageStatsCollector

* Hardcode version in setup.py, add it to bumpversion.cfg

* Remove async

* Multiline function def

* CI: update pinned reqs env

* Update tests

* Remove py39, add py314

* Update argument type hint

* Replace typing.Optional with pipe syntax

* Update DiskQuota middlewares again

* Remove py314

* Remove unnecessary type annotations

* Update tests

* async for in spider middleware

Avoid warning:
[scrapy.core.spidermw] sh_scrapy.middlewares.HubstorageSpiderMiddleware.process_spider_output_async is not an async generator function, skipping this method.

* Annotate classmethods without quotes

Author: elacuesta

.bumpversion.cfg

@@ -5,3 +5,5 @@ tag = True
 tag_name = v{new_version}
 
 [bumpversion:file:sh_scrapy/__init__.py]
+
+[bumpversion:file:setup.py]

.github/workflows/publish.yml

@@ -13,7 +13,7 @@ jobs:
     - name: Set up Python
       uses: actions/setup-python@v4
       with:
-        python-version: 3.9
+        python-version: "3.10"
 
     - name: Publish to PyPI
       run: |

.github/workflows/tests.yml

@@ -10,18 +10,10 @@ jobs:
       fail-fast: false
       matrix:
         include:
-        - python-version: "3.8"
+        - python-version: "3.10"
           os: ubuntu-22.04
           env:
-            TOXENV: py-scrapy16
-        - python-version: "3.8"
-          os: ubuntu-24.04
-          env:
-            TOXENV: py
-        - python-version: "3.9"
-          os: ubuntu-24.04
-          env:
-            TOXENV: py
+            TOXENV: py-scrapy27
         - python-version: "3.10"
           os: ubuntu-24.04
           env:

setup.py

@@ -1,18 +1,16 @@
 from setuptools import setup, find_packages
 
-from sh_scrapy import __version__
-
 
 setup(
     name='scrapinghub-entrypoint-scrapy',
-    version=__version__,
+    version='0.17.7',
     license='BSD',
     description='Scrapy entrypoint for Scrapinghub job runner',
     long_description=open('README.md').read(),
     packages=find_packages(),
     install_requires=[
-        'Scrapy>=1.6',
-        'scrapinghub>=2.1.0',
+        'Scrapy>=2.7',
+        'scrapinghub>=2.4.0',
     ],
     entry_points={
         'console_scripts': [
@@ -21,7 +19,7 @@
             'shub-image-info = sh_scrapy.crawl:shub_image_info',
         ],
     },
-    python_requires='>=3.8',
+    python_requires='>=3.10',
     classifiers=[
         'Framework :: Scrapy',
         'Development Status :: 5 - Production/Stable',
@@ -30,8 +28,6 @@
         'Operating System :: OS Independent',
         'Programming Language :: Python',
         'Programming Language :: Python :: 3',
-        'Programming Language :: Python :: 3.8',
-        'Programming Language :: Python :: 3.9',
         'Programming Language :: Python :: 3.10',
         'Programming Language :: Python :: 3.11',
         'Programming Language :: Python :: 3.12',

sh_scrapy/__init__.py

@@ -1 +1,7 @@
 __version__ = "0.17.7"
+
+
+from scrapy import version_info as scrapy_version_info
+
+
+_SCRAPY_NO_SPIDER_ARG = scrapy_version_info >= (2, 14, 0)

sh_scrapy/diskquota.py

@@ -3,33 +3,69 @@
 The goal is to catch disk quota errors and stop spider gently.
 """
 
+from __future__ import annotations
+
+import asyncio
+
+from scrapy import Spider
+from scrapy.crawler import Crawler
 from scrapy.exceptions import NotConfigured
+from scrapy.http import Request, Response
+
+from sh_scrapy import _SCRAPY_NO_SPIDER_ARG
 
 
-class DiskQuota(object):
+class DiskQuota:
 
-    def __init__(self, crawler):
-        if not crawler.settings.getbool('DISK_QUOTA_STOP_ON_ERROR'):
+    def __init__(self, crawler: Crawler):
+        if not crawler.settings.getbool("DISK_QUOTA_STOP_ON_ERROR"):
             raise NotConfigured
         self.crawler = crawler
 
     @classmethod
-    def from_crawler(cls, crawler):
+    def from_crawler(cls, crawler: Crawler) -> DiskQuota:
         return cls(crawler)
 
-    def _is_disk_quota_error(self, error):
+    def _is_disk_quota_error(self, error: Exception) -> bool:
         return isinstance(error, (OSError, IOError)) and error.errno == 122
 
 
 class DiskQuotaDownloaderMiddleware(DiskQuota):
 
-    def process_exception(self, request, exception, spider):
-        if self._is_disk_quota_error(exception):
-            self.crawler.engine.close_spider(spider, 'diskusage_exceeded')
+    if _SCRAPY_NO_SPIDER_ARG:
+
+        async def process_exception(self, request: Request, exception: Exception) -> None:
+            if self._is_disk_quota_error(exception):
+                await self.crawler.engine.close_spider_async(reason="diskusage_exceeded")
+
+    else:
+
+        def process_exception(
+            self, request: Request, exception: Exception, spider: Spider
+        ) -> None:
+            if self._is_disk_quota_error(exception):
+                self.crawler.engine.close_spider(spider, "diskusage_exceeded")
 
 
 class DiskQuotaSpiderMiddleware(DiskQuota):
 
-    def process_spider_exception(self, response, exception, spider):
-        if self._is_disk_quota_error(exception):
-            self.crawler.engine.close_spider(spider, 'diskusage_exceeded')
+    def __init__(self, crawler: Crawler):
+        super().__init__(crawler)
+        self._tasks: set[asyncio.Task] = set()
+
+    if _SCRAPY_NO_SPIDER_ARG:
+
+        def process_spider_exception(self, response: Response, exception: Exception) -> None:
+            if self._is_disk_quota_error(exception):
+                coro = self.crawler.engine.close_spider_async(reason="diskusage_exceeded")
+                task = asyncio.create_task(coro)
+                self._tasks.add(task)
+                task.add_done_callback(self._tasks.discard)
+
+    else:
+
+        def process_spider_exception(
+            self, response: Response, exception: Exception, spider: Spider
+        ) -> None:
+            if self._is_disk_quota_error(exception):
+                self.crawler.engine.close_spider(spider, "diskusage_exceeded")

sh_scrapy/middlewares.py

@@ -1,63 +1,96 @@
 # -*- coding: utf-8 -*-
+from __future__ import annotations
+
 import itertools
+from typing import AsyncIterable, AsyncGenerator, Iterable
 from warnings import warn
 from weakref import WeakKeyDictionary
 
-from scrapy import Request
+from scrapy import Spider
+from scrapy.crawler import Crawler
+from scrapy.http import Request, Response
 
+from sh_scrapy import _SCRAPY_NO_SPIDER_ARG
 from sh_scrapy.writer import pipe_writer
 
+
 HS_REQUEST_ID_KEY = '_hsid'
 HS_PARENT_ID_KEY = '_hsparent'
 request_id_sequence = itertools.count()
 seen_requests = WeakKeyDictionary()
 
 
-class HubstorageSpiderMiddleware(object):
+class HubstorageSpiderMiddleware:
     """Hubstorage spider middleware.
-    
+
     What it does:
-    
+
     - Sets parent request ids to the requests coming out of the spider.
-    
+
     """
 
-    def __init__(self):
+    def __init__(self) -> None:
         self._seen_requests = seen_requests
 
-    def process_spider_output(self, response, result, spider):
+    if _SCRAPY_NO_SPIDER_ARG:
+
+        def process_spider_output(self, response: Response, result: Iterable) -> Iterable:
+            return self._process_spider_output(response, result)
+
+        async def process_spider_output_async(
+            self, response: Response, result: Iterable
+        ) -> AsyncGenerator:
+            async for x in self._process_spider_output_async(response, result):
+                yield x
+
+    else:
+
+        def process_spider_output(
+            self, response: Response, result: Iterable, spider: Spider
+        ) -> Iterable:
+            return self._process_spider_output(response, result)
+
+        async def process_spider_output_async(
+            self, response: Response, result: Iterable, spider: Spider
+        ) -> AsyncGenerator:
+            async for x in self._process_spider_output_async(response, result):
+                yield x
+
+    def _process_spider_output(self, response: Response, result: Iterable) -> Iterable:
         parent = self._seen_requests.pop(response.request, None)
         for x in result:
             if isinstance(x, Request):
                 self._process_request(x, parent)
             yield x
 
-    async def process_spider_output_async(self, response, result, spider):
+    async def _process_spider_output_async(
+        self, response: Response, result: AsyncIterable
+    ) -> AsyncGenerator:
         parent = self._seen_requests.pop(response.request, None)
         async for x in result:
             if isinstance(x, Request):
                 self._process_request(x, parent)
             yield x
 
-    def _process_request(self, request, parent):
+    def _process_request(self, request: Request, parent: int | None) -> None:
         request.meta[HS_PARENT_ID_KEY] = parent
         # Remove request id if it was for some reason set in the request coming from Spider.
         request.meta.pop(HS_REQUEST_ID_KEY, None)
 
 
 class HubstorageDownloaderMiddleware:
     """Hubstorage dowloader middleware.
-    
+
     What it does:
-    
+
     - Generates request ids for all downloaded requests.
     - Sets parent request ids for requests generated in downloader middlewares.
     - Stores all downloaded requests into Hubstorage.
-    
+
     """
 
     @classmethod
-    def from_crawler(cls, crawler):
+    def from_crawler(cls, crawler: Crawler) -> HubstorageDownloaderMiddleware:
         try:
             result = cls(crawler)
         except TypeError:
@@ -74,29 +107,45 @@ def from_crawler(cls, crawler):
             result._load_fingerprinter()
         return result
 
-    def __init__(self, crawler):
+    def __init__(self, crawler: Crawler):
         self._crawler = crawler
         self._seen_requests = seen_requests
         self.pipe_writer = pipe_writer
         self.request_id_sequence = request_id_sequence
         self._load_fingerprinter()
 
-    def _load_fingerprinter(self):
+    def _load_fingerprinter(self) -> None:
         if hasattr(self._crawler, "request_fingerprinter"):
             self._fingerprint = lambda request: self._crawler.request_fingerprinter.fingerprint(request).hex()
         else:
             from scrapy.utils.request import request_fingerprint
             self._fingerprint = request_fingerprint
 
-    def process_request(self, request, spider):
+    if _SCRAPY_NO_SPIDER_ARG:
+
+        def process_request(self, request: Request) -> None:
+            return self._process_request(request)
+
+        def process_response(self, request: Request, response: Response) -> Response:
+            return self._process_response(request, response)
+
+    else:
+
+        def process_request(self, request: Request, spider: Spider) -> None:
+            return self._process_request(request)
+
+        def process_response(self, request: Request, response: Response, spider: Spider) -> Response:
+            return self._process_response(request, response)
+
+    def _process_request(self, request: Request) -> None:
         # Check if request id is set, which usually happens for retries or redirects because
         # those requests are usually copied from the original one.
         request_id = request.meta.pop(HS_REQUEST_ID_KEY, None)
         if request_id is not None:
             # Set original request id or None as a parent request id.
             request.meta[HS_PARENT_ID_KEY] = request_id
 
-    def process_response(self, request, response, spider):
+    def _process_response(self, request: Request, response: Response) -> Response:
         # This class of response check is intended to fix the bug described here
         # https://github.com/scrapy-plugins/scrapy-zyte-api/issues/112
         if type(response).__name__ == "DummyResponse" and type(response).__module__.startswith("scrapy_poet"):

sh_scrapy/stats.py

@@ -1,32 +1,49 @@
-from twisted.internet import task
+from scrapy import Spider
+from scrapy.crawler import Crawler
 from scrapy.statscollectors import StatsCollector
+from twisted.internet import task
 
-from sh_scrapy import hsref
+from sh_scrapy import hsref, _SCRAPY_NO_SPIDER_ARG
 from sh_scrapy.writer import pipe_writer
 
 
 class HubStorageStatsCollector(StatsCollector):
 
     INTERVAL = 30
 
-    def __init__(self, crawler):
+    def __init__(self, crawler: Crawler):
         super(HubStorageStatsCollector, self).__init__(crawler)
         self.hsref = hsref.hsref
         self.pipe_writer = pipe_writer
 
-    def _upload_stats(self):
+    def _upload_stats(self) -> None:
         self.pipe_writer.write_stats(self._stats)
 
-    def open_spider(self, spider):
-        self._setup_looping_call(now=True)
-
-    def _setup_looping_call(self, _ignored=None, **kwargs):
+    def _setup_looping_call(self, _ignored=None, **kwargs) -> None:
         self._samplestask = task.LoopingCall(self._upload_stats)
         d = self._samplestask.start(self.INTERVAL, **kwargs)
         d.addErrback(self._setup_looping_call, now=False)
 
-    def close_spider(self, spider, reason):
-        super(HubStorageStatsCollector, self).close_spider(spider, reason)
+    def _close_spider(self, spider: Spider | None = None, reason: str | None = None) -> None:
+        super().close_spider(spider=spider, reason=reason)
         if self._samplestask.running:
             self._samplestask.stop()
         self._upload_stats()
+
+    if _SCRAPY_NO_SPIDER_ARG:
+
+        def open_spider(self) -> None:
+            self._setup_looping_call(now=True)
+
+        def close_spider(self, reason: str | None = None) -> None:
+            self._close_spider(reason=reason)
+
+    else:
+
+        def open_spider(self, spider: Spider | None = None) -> None:
+            self._setup_looping_call(now=True)
+
+        def close_spider(
+            self, spider: Spider | None = None, reason: str | None = None
+        ) -> None:
+            self._close_spider(spider=spider, reason=reason)