Scrapy 2.14 compatibility (#359)

* Scrapy 2.14 compatibility (download handler arguments)

* Ignore pylint warning

* Update default download handler invocation

* Bump minimum versions

* Paint it black

* Update greenlet version

* Ensure running event loop

* Update incorrect ubuntu worker

* Async syntax for download_request and close

* Also ignore flake8 warning

* Bump minimum versions again

* Tests: refactor process killing

* Update download handling

* Update close method

* Twisted tests on pinned versions env

* Remove py314 for now

* Test coroutine download_request

* Update tests/tests_asyncio/test_browser.py

Co-authored-by: Copilot <175728472+Copilot@users.noreply.github.com>

---------

Co-authored-by: Copilot <175728472+Copilot@users.noreply.github.com>

Author: elacuesta

.github/workflows/tests.yml

@@ -10,7 +10,7 @@ jobs:
       fail-fast: false
       matrix:
         os: [ubuntu-22.04]
-        python-version: ["3.9", "3.10", "3.11", "3.12", "3.13"]
+        python-version: ["3.10", "3.11", "3.12", "3.13"]
         include:
           - os: macos-14
             python-version: "3.12"
@@ -65,3 +65,38 @@ jobs:
         $ProgressPreference = 'SilentlyContinue'
         Invoke-WebRequest -Uri https://uploader.codecov.io/latest/windows/codecov.exe -Outfile codecov.exe
         .\codecov.exe
+
+  tests-pinned:
+    if: github.event_name == 'push' || github.event.pull_request.head.repo.full_name != github.repository
+    runs-on: ubuntu-22.04
+    timeout-minutes: 20
+
+    steps:
+    - uses: actions/checkout@v4
+
+    - name: Set up Python 3.10
+      uses: actions/setup-python@v5
+      with:
+        python-version: "3.10"
+
+    - name: Set up node
+      uses: actions/setup-node@v4
+      with:
+        node-version: 18
+
+    - name: Install tox
+      run: pip install tox
+
+    - name: Run asyncio tests with pinned versions
+      run: tox -e py-pinned
+
+    - name: Run twisted tests with pinned versions
+      run: tox -e py-pinned-twisted
+
+    - name: Upload coverage report
+      env:
+        CODECOV_TOKEN: ${{ secrets.CODECOV_TOKEN }}
+      run: |
+        curl -Os https://uploader.codecov.io/latest/linux/codecov
+        chmod +x codecov
+        ./codecov

README.md

@@ -22,9 +22,9 @@ to integrate `asyncio`-based projects such as `Playwright`.
 
 ### Minimum required versions
 
-* Python >= 3.9
-* Scrapy >= 2.0 (!= 2.4.0)
-* Playwright >= 1.15
+* Python >= 3.10
+* Scrapy >= 2.7
+* Playwright >= 1.40
 
 
 ## Installation

scrapy_playwright/handler.py

@@ -23,15 +23,15 @@
     Response as PlaywrightResponse,
     Route,
 )
-from scrapy import Spider, signals
+from scrapy import Spider, signals, version_info as scrapy_version_info
 from scrapy.core.downloader.handlers.http11 import HTTP11DownloadHandler
 from scrapy.crawler import Crawler
 from scrapy.exceptions import NotSupported, ScrapyDeprecationWarning
 from scrapy.http import Request, Response
 from scrapy.http.headers import Headers
 from scrapy.responsetypes import responsetypes
 from scrapy.settings import Settings
-from scrapy.utils.defer import deferred_from_coro
+from scrapy.utils.defer import deferred_from_coro, maybe_deferred_to_future
 from scrapy.utils.misc import load_object
 from scrapy.utils.reactor import verify_installed_reactor
 from twisted.internet.defer import Deferred, inlineCallbacks
@@ -52,6 +52,9 @@
 __all__ = ["ScrapyPlaywrightDownloadHandler"]
 
 
+_SCRAPY_ASYNC_API = scrapy_version_info >= (2, 14, 0)
+
+
 PlaywrightHandler = TypeVar("PlaywrightHandler", bound="ScrapyPlaywrightDownloadHandler")
 
 
@@ -138,7 +141,12 @@ class ScrapyPlaywrightDownloadHandler(HTTP11DownloadHandler):
     playwright: Optional[AsyncPlaywright] = None
 
     def __init__(self, crawler: Crawler) -> None:
-        super().__init__(settings=crawler.settings, crawler=crawler)
+        if _SCRAPY_ASYNC_API:
+            super().__init__(crawler=crawler)
+        else:
+            super().__init__(  # pylint: disable=unexpected-keyword-arg
+                settings=crawler.settings, crawler=crawler
+            )
         verify_installed_reactor("twisted.internet.asyncioreactor.AsyncioSelectorReactor")
         crawler.signals.connect(self._engine_started, signals.engine_started)
         self.stats = crawler.stats
@@ -348,13 +356,20 @@ def _set_max_concurrent_context_count(self):
                 "playwright/context_count/max_concurrent", len(self.context_wrappers)
             )
 
-    @inlineCallbacks
-    def close(self) -> Deferred:
-        logger.info("Closing download handler")
-        yield super().close()
-        yield self._deferred_from_coro(self._close())
-        if self.config.use_threaded_loop:
-            _ThreadedLoopAdapter.stop(id(self))
+    if _SCRAPY_ASYNC_API:
+
+        async def close(self) -> None:
+            logger.info("Closing download handler")
+            await super().close()
+            await self._close()
+
+    else:
+
+        @inlineCallbacks
+        def close(self) -> Deferred:  # pylint: disable=invalid-overridden-method
+            logger.info("Closing download handler")
+            yield super().close()
+            yield self._deferred_from_coro(self._close())
 
     async def _close(self) -> None:
         with suppress(TargetClosedError):
@@ -367,11 +382,30 @@ async def _close(self) -> None:
             await self.playwright_context_manager.__aexit__()
         if self.playwright:
             await self.playwright.stop()
+        if self.config.use_threaded_loop:
+            _ThreadedLoopAdapter.stop(id(self))
+
+    if _SCRAPY_ASYNC_API:
+
+        async def download_request(self, request: Request) -> Response:
+            if request.meta.get("playwright"):
+                return await maybe_deferred_to_future(
+                    self._deferred_from_coro(self._download_request(request, self._crawler.spider))
+                )
+            return await super().download_request(  # pylint: disable=no-value-for-parameter
+                request
+            )
 
-    def download_request(self, request: Request, spider: Spider) -> Deferred:
-        if request.meta.get("playwright"):
-            return self._deferred_from_coro(self._download_request(request, spider))
-        return super().download_request(request, spider)
+    else:
+
+        def download_request(  # type: ignore[misc] # pylint: disable=invalid-overridden-method,arguments-differ # noqa: E501
+            self, request: Request, spider: Spider
+        ) -> Deferred:
+            if request.meta.get("playwright"):
+                return self._deferred_from_coro(self._download_request(request, spider))
+            return super().download_request(  # pylint: disable=unexpected-keyword-arg
+                request=request, spider=spider
+            )
 
     async def _download_request(self, request: Request, spider: Spider) -> Response:
         counter = 0
@@ -564,8 +598,7 @@ async def _handle_response(response: PlaywrightResponse) -> None:
             response = await page.goto(url=request.url, **page_goto_kwargs)
         except PlaywrightError as err:
             if not (
-                self.config.browser_type_name in ("firefox", "webkit")
-                and "Download is starting" in err.message
+                "Download is starting" in err.message
                 or self.config.browser_type_name == "chromium"
                 and "net::ERR_ABORTED" in err.message
             ):

setup.py

@@ -22,7 +22,6 @@
         "Development Status :: 4 - Beta",
         "License :: OSI Approved :: BSD License",
         "Programming Language :: Python",
-        "Programming Language :: Python :: 3.9",
         "Programming Language :: Python :: 3.10",
         "Programming Language :: Python :: 3.11",
         "Programming Language :: Python :: 3.12",
@@ -33,9 +32,9 @@
         "Topic :: Software Development :: Libraries :: Application Frameworks",
         "Topic :: Software Development :: Libraries :: Python Modules",
     ],
-    python_requires=">=3.9",
+    python_requires=">=3.10",
     install_requires=[
-        "scrapy>=2.0,!=2.4.0",
-        "playwright>=1.15",
+        "scrapy>=2.7",
+        "playwright>=1.40",
     ],
 )

tests/__init__.py

@@ -10,6 +10,8 @@
 from scrapy.http.response.html import HtmlResponse
 from scrapy.utils.test import get_crawler
 
+from scrapy_playwright.handler import _SCRAPY_ASYNC_API
+
 
 logger = logging.getLogger("scrapy-playwright-tests")
 
@@ -54,7 +56,10 @@ async def make_handler(settings_dict: Optional[dict] = None):
     else:
         yield handler
     finally:
-        await handler._close()
+        if _SCRAPY_ASYNC_API:
+            await handler.close()
+        else:
+            await handler._close()
 
 
 def assert_correct_response(response: HtmlResponse, request: Request) -> None:

tests/conftest.py

@@ -8,12 +8,18 @@ def pytest_configure(config):
     # https://twistedmatrix.com/trac/ticket/9766
     # https://github.com/pytest-dev/pytest-twisted/issues/80
 
-    if config.getoption("reactor", "default") == "asyncio" and platform.system() == "Windows":
-        import asyncio
+    import asyncio
 
+    if config.getoption("reactor", "default") == "asyncio" and platform.system() == "Windows":
         selector_policy = asyncio.WindowsSelectorEventLoopPolicy()
         asyncio.set_event_loop_policy(selector_policy)
 
+    # Ensure there's a running event loop for the tests
+    try:
+        asyncio.get_running_loop()
+    except RuntimeError:
+        asyncio.set_event_loop(asyncio.new_event_loop())
+
 
 def pytest_sessionstart(session):  # pylint: disable=unused-argument
     """

tests/tests_asyncio/test_browser.py

@@ -1,10 +1,8 @@
 import asyncio
 import logging
-import os
 import platform
 import random
 import re
-import signal
 import subprocess
 import time
 import uuid
@@ -148,8 +146,14 @@ def inject_fixtures(self, caplog):
     @staticmethod
     def kill_chrome():
         for proc in psutil.process_iter(["pid", "name"]):
-            if proc.info["name"].lower() in ("chrome", "chromium"):
-                os.kill(proc.info["pid"], signal.SIGKILL)
+            started_time = proc.create_time()  # seconds since January 1, 1970 UTC
+            # only consider processes started in the last 10 seconds
+            if not started_time >= time.time() - 10:
+                continue
+            name = proc.info["name"].lower()
+            if "chrome" in name or "chromium" in name:
+                proc.kill()
+                print("Killed process:", proc)
 
     @allow_windows
     async def test_browser_closed_restart(self):

tests/tests_asyncio/test_playwright_requests.py

@@ -15,7 +15,7 @@
 )
 from scrapy import Spider, Request, FormRequest
 
-from scrapy_playwright.handler import DEFAULT_CONTEXT_NAME
+from scrapy_playwright.handler import DEFAULT_CONTEXT_NAME, _SCRAPY_ASYNC_API
 from scrapy_playwright.page import PageMethod
 
 from tests import allow_windows, make_handler, assert_correct_response
@@ -51,12 +51,22 @@ async def test_basic_response(self):
                 req = Request(server.urljoin("/index.html"), meta=meta)
                 resp = await handler._download_request(req, Spider("foo"))
 
+                if _SCRAPY_ASYNC_API:
+                    req2 = Request(server.urljoin("/gallery.html"), meta=meta)
+                    resp2 = await handler.download_request(req2)
+
             assert_correct_response(resp, req)
             assert resp.css("a::text").getall() == ["Lorem Ipsum", "Infinite Scroll"]
             assert isinstance(resp.meta["playwright_page"], PlaywrightPage)
             assert resp.meta["playwright_page"].url == resp.url
             await resp.meta["playwright_page"].close()
 
+            if _SCRAPY_ASYNC_API:
+                assert_correct_response(resp2, req2)
+                assert isinstance(resp2.meta["playwright_page"], PlaywrightPage)
+                assert resp2.meta["playwright_page"].url == resp2.url
+                await resp2.meta["playwright_page"].close()
+
     @allow_windows
     async def test_post_request(self):
         async with make_handler({"PLAYWRIGHT_BROWSER_TYPE": self.browser_type}) as handler:
@@ -499,7 +509,7 @@ async def test_download_file_delay_error(self):
     @allow_windows
     async def test_download_file_failure(self):
         if self.browser_type != "chromium":
-            pytest.skip()
+            pytest.skip("Test only on Chromium")
 
         async def cancel_download(download):
             await download.cancel()

tests/tests_twisted/test_mixed_requests.py

@@ -1,4 +1,5 @@
-from scrapy import Spider
+import pytest
+from scrapy import Spider, version_info as scrapy_version_info
 from scrapy.http import Request, Response
 from scrapy.utils.test import get_crawler
 from twisted.internet import defer
@@ -8,11 +9,15 @@
 from tests.mockserver import StaticMockServer
 
 
+@pytest.mark.skipif(scrapy_version_info >= (2, 14, 0), reason="Does not apply to Scrapy >= 2.14.0")
 class MixedRequestsTestCase(TestCase):
     """
-    This test case ensures the handler's 'download_request' method works as expected, and
+    This test case ensures the handler's 'download_request' method works as expected and
     non-playwright requests are processed correctly. The rest of the tests directly call
     '_download_request', which is a coroutine ('download_request' returns a Deferred).
+
+    Update: since Scrapy 2.14.0, the default download handler uses the async API, returning a
+    coroutine from the 'download_request' method, making this test case obsolete.
     """
 
     timeout_ms = 500
@@ -50,7 +55,7 @@ def _check_playwright_ok(response, request):
         def _check_playwright_error(failure, url):
             # different errors depending on the platform
             self.assertTrue(
-                f"Page.goto: net::ERR_CONNECTION_REFUSED at {url}" in str(failure.value)
+                f"net::ERR_CONNECTION_REFUSED at {url}" in str(failure.value)
                 or f"Page.goto: Timeout {self.timeout_ms}ms exceeded" in str(failure.value)
             )