Support Scrapy 2.12+ (#317)

Author: J-CPelletier

.github/workflows/tests.yml

@@ -9,11 +9,11 @@ jobs:
       fail-fast: false
       matrix:
         include:
-        - python-version: '3.7'
-        - python-version: '3.8'
         - python-version: '3.9'
         - python-version: '3.10'
         - python-version: '3.11'
+        - python-version: '3.12'
+        - python-version: '3.13'
 
     steps:
     - uses: actions/checkout@v2

README.rst

@@ -75,25 +75,9 @@ Configuration
    also allows to save network traffic by not sending these duplicate
    arguments to Splash server multiple times.
 
-4. Set a custom ``DUPEFILTER_CLASS``::
+4. Set a custom ``REQUEST_FINGERPRINTER_CLASS``::
 
-      DUPEFILTER_CLASS = 'scrapy_splash.SplashAwareDupeFilter'
-
-5. If you use Scrapy HTTP cache then a custom cache storage backend
-   is required. scrapy-splash provides a subclass of
-   ``scrapy.contrib.httpcache.FilesystemCacheStorage``::
-
-      HTTPCACHE_STORAGE = 'scrapy_splash.SplashAwareFSCacheStorage'
-
-   If you use other cache storage then it is necesary to subclass it and
-   replace all ``scrapy.util.request.request_fingerprint`` calls with
-   ``scrapy_splash.splash_request_fingerprint``.
-
-.. note::
-
-    Steps (4) and (5) are necessary because Scrapy doesn't provide a way
-    to override request fingerprints calculation algorithm globally; this
-    could change in future.
+      REQUEST_FINGERPRINTER_CLASS = 'scrapy_splash.SplashRequestFingerprinter'
 
 
 There are also some additional options available.
@@ -111,6 +95,7 @@ Put them into your ``settings.py`` if you want to change the defaults:
   It specifies how concurrency & politeness are maintained for Splash requests,
   and specify the default value for ``slot_policy`` argument for
   ``SplashRequest``, which is described below.
+* ``SCRAPY_SPLASH_REQUEST_FINGERPRINTER_BASE_CLASS`` is ``scrapy.settings.default_settings.REQUEST_FINGERPRINTER_CLASS`` by default. This changes the base class the Fingerprinter uses to get a fingerprint.
 
 
 Usage

scrapy_splash/__init__.py

@@ -10,4 +10,4 @@
 from .dupefilter import SplashAwareDupeFilter, splash_request_fingerprint
 from .cache import SplashAwareFSCacheStorage
 from .response import SplashResponse, SplashTextResponse, SplashJsonResponse
-from .request import SplashRequest, SplashFormRequest
+from .request import SplashRequest, SplashFormRequest, SplashRequestFingerprinter

scrapy_splash/cache.py

@@ -7,13 +7,26 @@
 """
 from __future__ import absolute_import
 import os
+from warnings import warn
 
 from scrapy.extensions.httpcache import FilesystemCacheStorage
 
 from .dupefilter import splash_request_fingerprint
 
 
 class SplashAwareFSCacheStorage(FilesystemCacheStorage):
+    def __init__(self, settings):
+        warn(
+            (
+                "scrapy-splash.SplashAwareFSCacheStorage is deprecated. Set "
+                "the REQUEST_FINGERPRINTER_CLASS Scrapy setting to "
+                "\"scrapy_splash.SplashRequestFingerprinter\" instead."
+            ),
+            DeprecationWarning,
+            stacklevel=2,
+        )
+        super().__init__(settings)
+
     def _get_request_path(self, spider, request):
         key = splash_request_fingerprint(request)
         return os.path.join(self.cachedir, spider.name, key[0:2], key)

scrapy_splash/dupefilter.py

@@ -5,17 +5,100 @@
 """
 from __future__ import absolute_import
 from copy import deepcopy
+import hashlib
+from weakref import WeakKeyDictionary
+from warnings import warn
 
 from scrapy.dupefilters import RFPDupeFilter
 
+from scrapy.utils.python import to_bytes
 from scrapy.utils.url import canonicalize_url
-from scrapy.utils.request import request_fingerprint
 
 from .utils import dict_hash
 
 
+_deprecated_fingerprint_cache = WeakKeyDictionary()
+
+
+def _serialize_headers(
+    headers, request
+):
+    for header in headers:
+        if header in request.headers:
+            yield header
+            for value in request.headers.getlist(header):
+                yield value
+
+
+# From https://docs.scrapy.org/en/2.11/_modules/scrapy/utils/request.html
+# Needs to be added here since it was deletedin Scrapy 2.12
+def request_fingerprint(
+    request,
+    include_headers=None,
+    keep_fragments=False,
+):
+    """
+    Return the request fingerprint as an hexadecimal string.
+
+    The request fingerprint is a hash that uniquely identifies the resource the
+    request points to. For example, take the following two urls:
+
+    http://www.example.com/query?id=111&cat=222
+    http://www.example.com/query?cat=222&id=111
+
+    Even though those are two different URLs both point to the same resource
+    and are equivalent (i.e. they should return the same response).
+
+    Another example are cookies used to store session ids. Suppose the
+    following page is only accessible to authenticated users:
+
+    http://www.example.com/members/offers.html
+
+    Lots of sites use a cookie to store the session id, which adds a random
+    component to the HTTP Request and thus should be ignored when calculating
+    the fingerprint.
+
+    For this reason, request headers are ignored by default when calculating
+    the fingerprint. If you want to include specific headers use the
+    include_headers argument, which is a list of Request headers to include.
+
+    Also, servers usually ignore fragments in urls when handling requests,
+    so they are also ignored by default when calculating the fingerprint.
+    If you want to include them, set the keep_fragments argument to True
+    (for instance when handling requests with a headless browser).
+    """
+    processed_include_headers = None
+    if include_headers:
+        processed_include_headers = tuple(
+            to_bytes(h.lower()) for h in sorted(include_headers)
+        )
+    cache = _deprecated_fingerprint_cache.setdefault(request, {})
+    cache_key = (processed_include_headers, keep_fragments)
+    if cache_key not in cache:
+        fp = hashlib.sha1()
+        fp.update(to_bytes(request.method))
+        fp.update(
+            to_bytes(canonicalize_url(request.url, keep_fragments=keep_fragments))
+        )
+        fp.update(request.body or b"")
+        if processed_include_headers:
+            for part in _serialize_headers(processed_include_headers, request):
+                fp.update(part)
+        cache[cache_key] = fp.hexdigest()
+    return cache[cache_key]
+
+
 def splash_request_fingerprint(request, include_headers=None):
     """ Request fingerprint which takes 'splash' meta key into account """
+    warn(
+        (
+            "scrapy_splash.splash_request_fingerprint is deprecated. Set "
+            "the REQUEST_FINGERPRINTER_CLASS Scrapy setting to "
+            "\"scrapy_splash.SplashRequestFingerprinter\" instead."
+        ),
+        DeprecationWarning,
+        stacklevel=2,
+    )
 
     fp = request_fingerprint(request, include_headers=include_headers)
     if 'splash' not in request.meta:
@@ -35,5 +118,17 @@ class SplashAwareDupeFilter(RFPDupeFilter):
     DupeFilter that takes 'splash' meta key in account.
     It should be used with SplashMiddleware.
     """
+
+    def __init__(self):
+        warn(
+            (
+                "SplashAwareDupeFilter is deprecated. Set "
+                "the REQUEST_FINGERPRINTER_CLASS Scrapy setting to "
+                "\"scrapy_splash.SplashRequestFingerprinter\" instead."
+            ),
+            DeprecationWarning,
+            stacklevel=2,
+        )
+
     def request_fingerprint(self, request):
         return splash_request_fingerprint(request)

scrapy_splash/request.py

@@ -3,9 +3,20 @@
 import copy
 import scrapy
 from scrapy.http import FormRequest
+from scrapy.utils.url import canonicalize_url
 
 from scrapy_splash import SlotPolicy
-from scrapy_splash.utils import to_unicode
+from scrapy_splash.utils import to_unicode, dict_hash
+from scrapy.settings.default_settings import REQUEST_FINGERPRINTER_CLASS
+from scrapy.utils.misc import load_object
+
+try:
+    from scrapy.utils.misc import build_from_crawler
+except ImportError:  # Scrapy < 2.12
+    from scrapy.utils.misc import create_instance
+
+    def build_from_crawler(objcls, crawler, /, *args, **kwargs):
+        return create_instance(objcls, None, crawler, *args, **kwargs)
 
 # XXX: we can't implement SplashRequest without middleware support
 # because there is no way to set Splash URL based on settings
@@ -115,3 +126,35 @@ def __init__(self, url=None, callback=None, method=None, formdata=None,
         SplashRequest.__init__(
             self, url=url, callback=callback, method=method, body=body,
             **kwargs)
+
+
+class SplashRequestFingerprinter:
+    @classmethod
+    def from_crawler(cls, crawler):
+        return cls(crawler)
+
+    def __init__(self, crawler):
+        self._base_request_fingerprinter = build_from_crawler(
+                load_object(
+                    crawler.settings.get(
+                        "SCRAPY_SPLASH_REQUEST_FINGERPRINTER_BASE_CLASS",
+                        REQUEST_FINGERPRINTER_CLASS,
+                    )
+                ),
+                crawler,
+            )
+
+    def fingerprint(self, request):
+        """ Request fingerprint which takes 'splash' meta key into account """
+
+        fp = self._base_request_fingerprinter.fingerprint(request)
+        if 'splash' not in request.meta:
+            return fp
+
+        splash_options = copy.deepcopy(request.meta['splash'])
+        args = splash_options.setdefault('args', {})
+
+        if 'url' in args:
+            args['url'] = canonicalize_url(args['url'], keep_fragments=True)
+
+        return dict_hash(splash_options, fp).encode()

setup.py

@@ -30,5 +30,5 @@
         'Topic :: Software Development :: Libraries :: Application Frameworks',
         'Topic :: Software Development :: Libraries :: Python Modules',
     ],
-    requires=['scrapy', 'six'],
+    install_requires=['scrapy', 'six'],
 )

tests/conftest.py

@@ -26,8 +26,7 @@ def settings():
         SPIDER_MIDDLEWARES={
             'scrapy_splash.SplashDeduplicateArgsMiddleware': 100,
         },
-        DUPEFILTER_CLASS='scrapy_splash.SplashAwareDupeFilter',
-        HTTPCACHE_STORAGE='scrapy_splash.SplashAwareFSCacheStorage',
+        REQUEST_FINGERPRINTER_CLASS='scrapy_splash.SplashRequestFingerprinter',
     )
     return s
 

tests/test_fingerprints.py

@@ -4,13 +4,14 @@
 
 import pytest
 import scrapy
-from scrapy.utils.request import request_fingerprint
 
 from scrapy_splash import SplashRequest
-from scrapy_splash.dupefilter import splash_request_fingerprint
+from scrapy_splash.dupefilter import request_fingerprint, splash_request_fingerprint
 from scrapy_splash.utils import dict_hash
 
 from .test_middleware import _get_mw
+from .utils import make_crawler
+from scrapy_splash.request import SplashRequestFingerprinter
 
 
 def test_dict_hash():
@@ -79,6 +80,39 @@ def test_request_fingerprint_splash():
     assert_fingerprints_match(r2, r4)
 
 
+def assert_fingerprints_match_fingerprinter(fingerprinter, r1, r2):
+    assert fingerprinter.fingerprint(r1) == fingerprinter.fingerprint(r2)
+
+
+def assert_fingerprints_dont_match_fingerprinter(fingerprinter, r1, r2):
+    assert fingerprinter.fingerprint(r1) != fingerprinter.fingerprint(r2)
+
+
+class TestSpider(scrapy.Spider):
+    name = 'test_spider'
+
+
+def test_splash_request_fingerprinter():
+    crawler = make_crawler(TestSpider, {})
+    fingerprinter = SplashRequestFingerprinter(crawler)
+
+    r1 = scrapy.Request("http://example.com")
+    r2 = scrapy.Request("http://example.com", meta={"splash": {"args": {"html": 1}}})
+    r3 = scrapy.Request("http://example.com", meta={"splash": {"args": {"png": 1}}})
+    r4 = scrapy.Request("http://example.com", meta={"foo": "bar", "splash": {"args": {"html": 1}}})
+    r5 = scrapy.Request("http://example.com", meta={"splash": {"args": {"html": 1, "wait": 1.0}}})
+
+    assert request_fingerprint(r1) == request_fingerprint(r2)
+    assert_fingerprints_dont_match_fingerprinter(fingerprinter, r1, r2)
+    assert_fingerprints_dont_match_fingerprinter(fingerprinter, r1, r3)
+    assert_fingerprints_dont_match_fingerprinter(fingerprinter, r1, r4)
+    assert_fingerprints_dont_match_fingerprinter(fingerprinter, r1, r5)
+    assert_fingerprints_dont_match_fingerprinter(fingerprinter, r2, r3)
+
+    # only "splash" contents is taken into account
+    assert_fingerprints_match_fingerprinter(fingerprinter, r2, r4)
+
+
 @pytest.fixture()
 def splash_middleware():
     return _get_mw()

tests/test_middleware.py

@@ -7,7 +7,7 @@
 import scrapy
 from scrapy.core.engine import ExecutionEngine
 from scrapy.utils.test import get_crawler
-from scrapy.http import Response, TextResponse
+from scrapy.http import Response, TextResponse, JsonResponse
 from scrapy.downloadermiddlewares.httpcache import HttpCacheMiddleware
 
 import scrapy_splash
@@ -505,7 +505,7 @@ def _get_req():
     cached_resp = cache_mw.process_request(req, spider) or req
 
     # response should be from cache:
-    assert cached_resp.__class__ is TextResponse
+    assert cached_resp.__class__ is JsonResponse
     assert cached_resp.body == resp_body
     resp2_1 = cache_mw.process_response(req, cached_resp, spider)
     resp3_1 = mw.process_response(req, resp2_1, spider)