Merge remote-tracking branch 'origin/main' into cleanup-deprecated-header-handling

Author: elacuesta

.bumpversion.cfg

@@ -1,5 +1,5 @@
 [bumpversion]
-current_version = 0.0.44
+current_version = 0.0.46
 commit = True
 tag = True
 

.github/workflows/tests.yml

@@ -10,7 +10,7 @@ jobs:
       fail-fast: false
       matrix:
         os: [ubuntu-22.04]
-        python-version: ["3.9", "3.10", "3.11", "3.12", "3.13"]
+        python-version: ["3.10", "3.11", "3.12", "3.13"]
         include:
           - os: macos-14
             python-version: "3.12"
@@ -39,6 +39,14 @@ jobs:
     - name: Run twisted tests
       run: tox -e py-twisted
 
+    - name: Run pinned dependency tests (Windows, asyncio tests)
+      if: runner.os == 'Windows'
+      run: tox -e py-pinned
+
+    - name: Run pinned dependency tests (Windows, Twisted tests)
+      if: runner.os == 'Windows'
+      run: tox -e py-pinned-twisted
+
     - name: Upload coverage report (Linux)
       if: runner.os == 'Linux'
       env:
@@ -65,3 +73,38 @@ jobs:
         $ProgressPreference = 'SilentlyContinue'
         Invoke-WebRequest -Uri https://uploader.codecov.io/latest/windows/codecov.exe -Outfile codecov.exe
         .\codecov.exe
+
+  tests-pinned:
+    if: github.event_name == 'push' || github.event.pull_request.head.repo.full_name != github.repository
+    runs-on: ubuntu-22.04
+    timeout-minutes: 20
+
+    steps:
+    - uses: actions/checkout@v4
+
+    - name: Set up Python 3.10
+      uses: actions/setup-python@v5
+      with:
+        python-version: "3.10"
+
+    - name: Set up node
+      uses: actions/setup-node@v4
+      with:
+        node-version: 18
+
+    - name: Install tox
+      run: pip install tox
+
+    - name: Run asyncio tests with pinned versions
+      run: tox -e py-pinned
+
+    - name: Run twisted tests with pinned versions
+      run: tox -e py-pinned-twisted
+
+    - name: Upload coverage report
+      env:
+        CODECOV_TOKEN: ${{ secrets.CODECOV_TOKEN }}
+      run: |
+        curl -Os https://uploader.codecov.io/latest/linux/codecov
+        chmod +x codecov
+        ./codecov

README.md

@@ -22,9 +22,9 @@ to integrate `asyncio`-based projects such as `Playwright`.
 
 ### Minimum required versions
 
-* Python >= 3.9
-* Scrapy >= 2.0 (!= 2.4.0)
-* Playwright >= 1.15
+* Python >= 3.10
+* Scrapy >= 2.7
+* Playwright >= 1.40
 
 
 ## Installation

docs/changelog.md

@@ -1,6 +1,16 @@
 # scrapy-playwright changelog
 
 
+### [v0.0.46](https://github.com/scrapy-plugins/scrapy-playwright/releases/tag/v0.0.46) (2026-01-21)
+
+* Threaded loop updates & fixes (#361)
+
+
+### [v0.0.45](https://github.com/scrapy-plugins/scrapy-playwright/releases/tag/v0.0.45) (2026-01-16)
+
+* Scrapy 2.14 compatibility (#356, #359)
+
+
 ### [v0.0.44](https://github.com/scrapy-plugins/scrapy-playwright/releases/tag/v0.0.44) (2025-08-13)
 
 * Fix crawl getting stuck on Windows with Scrapy>=2.13 (#351)

pylintrc

@@ -9,6 +9,7 @@ disable=
     too-few-public-methods,
     too-many-arguments,
     too-many-instance-attributes,
+    too-many-lines,
     # tests
     duplicate-code,
     import-outside-toplevel,

pyproject.toml

@@ -1,2 +1,5 @@
 [tool.black]
 line-length = 99
+
+[tool.pytest.ini_options]
+reruns = "2"

scrapy_playwright/__init__.py

@@ -1 +1 @@
-__version__ = "0.0.44"
+__version__ = "0.0.46"

scrapy_playwright/_utils.py

@@ -2,6 +2,7 @@
 import logging
 import platform
 import threading
+from dataclasses import dataclass
 from typing import Awaitable, Dict, Iterator, Optional, Tuple, Union
 
 import scrapy
@@ -104,6 +105,13 @@ async def _get_header_value(
         return None
 
 
+@dataclass
+class _QueueItem:
+    coro: Awaitable
+    promise: Deferred | asyncio.Future
+    loop: asyncio.AbstractEventLoop | None = None
+
+
 class _ThreadedLoopAdapter:
     """Utility class to start an asyncio event loop in a new thread and redirect coroutines.
     This allows to run Playwright in a different loop than the Scrapy crawler, allowing to
@@ -116,32 +124,60 @@ class _ThreadedLoopAdapter:
     _stop_events: Dict[int, asyncio.Event] = {}
 
     @classmethod
-    async def _handle_coro(cls, coro: Awaitable, dfd: Deferred) -> None:
+    async def _handle_coro_deferred(cls, queue_item: _QueueItem) -> None:
         from twisted.internet import reactor
 
+        dfd: Deferred = queue_item.promise
+
         try:
-            result = await coro
+            result = await queue_item.coro
         except Exception as exc:
             reactor.callFromThread(dfd.errback, failure.Failure(exc))
         else:
             reactor.callFromThread(dfd.callback, result)
 
+    @classmethod
+    async def _handle_coro_future(cls, queue_item: _QueueItem) -> None:
+        future: asyncio.Future = queue_item.promise
+        loop: asyncio.AbstractEventLoop = queue_item.loop  # type: ignore[assignment]
+        try:
+            result = await queue_item.coro
+        except Exception as exc:
+            loop.call_soon_threadsafe(future.set_exception, exc)
+        else:
+            loop.call_soon_threadsafe(future.set_result, result)
+
     @classmethod
     async def _process_queue(cls) -> None:
         while any(not ev.is_set() for ev in cls._stop_events.values()):
-            coro, dfd = await cls._coro_queue.get()
-            asyncio.create_task(cls._handle_coro(coro, dfd))
+            queue_item = await cls._coro_queue.get()
+            if isinstance(queue_item.promise, asyncio.Future):
+                asyncio.create_task(cls._handle_coro_future(queue_item))
+            elif isinstance(queue_item.promise, Deferred):
+                asyncio.create_task(cls._handle_coro_deferred(queue_item))
             cls._coro_queue.task_done()
 
     @classmethod
-    def _deferred_from_coro(cls, coro) -> Deferred:
+    def _deferred_from_coro(cls, coro: Awaitable) -> Deferred:
         dfd: Deferred = Deferred()
-        asyncio.run_coroutine_threadsafe(cls._coro_queue.put((coro, dfd)), cls._loop)
+        queue_item = _QueueItem(coro=coro, promise=dfd)
+        asyncio.run_coroutine_threadsafe(cls._coro_queue.put(queue_item), cls._loop)
         return dfd
 
     @classmethod
-    def start(cls, caller_id: int) -> None:
-        cls._stop_events[caller_id] = asyncio.Event()
+    def _future_from_coro(cls, coro: Awaitable) -> asyncio.Future:
+        target_loop = asyncio.get_running_loop()  # Scrapy thread loop
+        future: asyncio.Future = asyncio.Future()
+        queue_item = _QueueItem(coro=coro, promise=future, loop=target_loop)
+        asyncio.run_coroutine_threadsafe(cls._coro_queue.put(queue_item), cls._loop)
+        return future
+
+    @classmethod
+    def start(cls, download_handler_id: int) -> None:
+        """Start the event loop in a new thread if not already started.
+        Should be called from the Scrapy thread.
+        """
+        cls._stop_events[download_handler_id] = asyncio.Event()
         if not getattr(cls, "_loop", None):
             policy = asyncio.DefaultEventLoopPolicy()
             if platform.system() == "Windows":
@@ -155,9 +191,11 @@ def start(cls, caller_id: int) -> None:
             asyncio.run_coroutine_threadsafe(cls._process_queue(), cls._loop)
 
     @classmethod
-    def stop(cls, caller_id: int) -> None:
-        """Wait until all handlers are closed to stop the event loop and join the thread."""
-        cls._stop_events[caller_id].set()
+    def stop(cls, download_handler_id: int) -> None:
+        """Wait until all handlers are closed to stop the event loop and join the thread.
+        Should be called from the Scrapy thread.
+        """
+        cls._stop_events[download_handler_id].set()
         if all(ev.is_set() for ev in cls._stop_events.values()):
             asyncio.run_coroutine_threadsafe(cls._coro_queue.join(), cls._loop)
             cls._loop.call_soon_threadsafe(cls._loop.stop)

scrapy_playwright/handler.py

@@ -21,8 +21,8 @@
     Response as PlaywrightResponse,
     Route,
 )
-from scrapy import Spider, signals
-from scrapy.core.downloader.handlers.http import HTTPDownloadHandler
+from scrapy import Spider, signals, version_info as scrapy_version_info
+from scrapy.core.downloader.handlers.http11 import HTTP11DownloadHandler
 from scrapy.crawler import Crawler
 from scrapy.exceptions import NotSupported
 from scrapy.http import Request, Response
@@ -50,6 +50,9 @@
 __all__ = ["ScrapyPlaywrightDownloadHandler"]
 
 
+_SCRAPY_ASYNC_API = scrapy_version_info >= (2, 14, 0)
+
+
 PlaywrightHandler = TypeVar("PlaywrightHandler", bound="ScrapyPlaywrightDownloadHandler")
 
 
@@ -131,20 +134,29 @@ def from_settings(cls, settings: Settings) -> "Config":
         return cfg
 
 
-class ScrapyPlaywrightDownloadHandler(HTTPDownloadHandler):
+class ScrapyPlaywrightDownloadHandler(HTTP11DownloadHandler):
     playwright_context_manager: Optional[PlaywrightContextManager] = None
     playwright: Optional[AsyncPlaywright] = None
 
     def __init__(self, crawler: Crawler) -> None:
-        super().__init__(settings=crawler.settings, crawler=crawler)
         verify_installed_reactor("twisted.internet.asyncioreactor.AsyncioSelectorReactor")
-        crawler.signals.connect(self._engine_started, signals.engine_started)
+        if _SCRAPY_ASYNC_API:
+            super().__init__(crawler=crawler)
+        else:
+            super().__init__(  # pylint: disable=unexpected-keyword-arg
+                settings=crawler.settings, crawler=crawler
+            )
         self.stats = crawler.stats
         self.config = Config.from_settings(crawler.settings)
 
         if self.config.use_threaded_loop:
             _ThreadedLoopAdapter.start(id(self))
 
+        if _SCRAPY_ASYNC_API:
+            crawler.signals.connect(self._maybe_launch_in_thread, signals.engine_started)
+        else:
+            crawler.signals.connect(self._engine_started, signals.engine_started)
+
         self.browser_launch_lock = asyncio.Lock()
         self.context_launch_lock = asyncio.Lock()
         self.context_wrappers: Dict[str, BrowserContextWrapper] = {}
@@ -175,10 +187,17 @@ def _deferred_from_coro(self, coro: Awaitable) -> Deferred:
             return _ThreadedLoopAdapter._deferred_from_coro(coro)
         return deferred_from_coro(coro)
 
+    def _maybe_future_from_coro(self, coro: Awaitable) -> Awaitable | asyncio.Future:
+        if self.config.use_threaded_loop:
+            return _ThreadedLoopAdapter._future_from_coro(coro)
+        return coro
+
     def _engine_started(self) -> Deferred:
-        """Launch the browser. Use the engine_started signal as it supports returning deferreds."""
         return self._deferred_from_coro(self._launch())
 
+    async def _maybe_launch_in_thread(self) -> None:
+        await self._maybe_future_from_coro(self._launch())
+
     async def _launch(self) -> None:
         """Launch Playwright manager and configured startup context(s)."""
         logger.info("Starting download handler")
@@ -346,13 +365,24 @@ def _set_max_concurrent_context_count(self):
                 "playwright/context_count/max_concurrent", len(self.context_wrappers)
             )
 
-    @inlineCallbacks
-    def close(self) -> Deferred:
-        logger.info("Closing download handler")
-        yield super().close()
-        yield self._deferred_from_coro(self._close())
-        if self.config.use_threaded_loop:
-            _ThreadedLoopAdapter.stop(id(self))
+    if _SCRAPY_ASYNC_API:
+
+        async def close(self) -> None:
+            logger.info("Closing download handler")
+            await super().close()
+            await self._maybe_future_from_coro(self._close())
+            if self.config.use_threaded_loop:
+                _ThreadedLoopAdapter.stop(id(self))
+
+    else:
+
+        @inlineCallbacks
+        def close(self) -> Deferred:  # pylint: disable=invalid-overridden-method
+            logger.info("Closing download handler")
+            yield super().close()
+            yield self._deferred_from_coro(self._close())
+            if self.config.use_threaded_loop:
+                _ThreadedLoopAdapter.stop(id(self))
 
     async def _close(self) -> None:
         with suppress(TargetClosedError):
@@ -366,12 +396,29 @@ async def _close(self) -> None:
         if self.playwright:
             await self.playwright.stop()
 
-    def download_request(self, request: Request, spider: Spider) -> Deferred:
-        if request.meta.get("playwright"):
-            return self._deferred_from_coro(self._download_request(request, spider))
-        return super().download_request(request, spider)
+    if _SCRAPY_ASYNC_API:
+
+        async def download_request(self, request: Request) -> Response:
+            if request.meta.get("playwright"):
+                coro = self._download_request(request)
+                return await self._maybe_future_from_coro(coro)
+            return await super().download_request(  # pylint: disable=no-value-for-parameter
+                request
+            )
+
+    else:
+
+        def download_request(  # type: ignore[misc] # pylint: disable=invalid-overridden-method,arguments-differ # noqa: E501
+            self, request: Request, spider: Spider
+        ) -> Deferred:
+            if request.meta.get("playwright"):
+                return self._deferred_from_coro(self._download_request(request, spider))
+            return super().download_request(  # pylint: disable=unexpected-keyword-arg
+                request=request, spider=spider
+            )
 
-    async def _download_request(self, request: Request, spider: Spider) -> Response:
+    async def _download_request(self, request: Request, spider: Spider | None = None) -> Response:
+        spider = spider or self._crawler.spider
         counter = 0
         while True:
             try:
@@ -562,8 +609,7 @@ async def _handle_response(response: PlaywrightResponse) -> None:
             response = await page.goto(url=request.url, **page_goto_kwargs)
         except PlaywrightError as err:
             if not (
-                self.config.browser_type_name in ("firefox", "webkit")
-                and "Download is starting" in err.message
+                "Download is starting" in err.message
                 or self.config.browser_type_name == "chromium"
                 and "net::ERR_ABORTED" in err.message
             ):

setup.py

@@ -22,7 +22,6 @@
         "Development Status :: 4 - Beta",
         "License :: OSI Approved :: BSD License",
         "Programming Language :: Python",
-        "Programming Language :: Python :: 3.9",
         "Programming Language :: Python :: 3.10",
         "Programming Language :: Python :: 3.11",
         "Programming Language :: Python :: 3.12",
@@ -33,9 +32,9 @@
         "Topic :: Software Development :: Libraries :: Application Frameworks",
         "Topic :: Software Development :: Libraries :: Python Modules",
     ],
-    python_requires=">=3.9",
+    python_requires=">=3.10",
     install_requires=[
-        "scrapy>=2.0,!=2.4.0",
-        "playwright>=1.15",
+        "scrapy>=2.7",
+        "playwright>=1.40",
     ],
 )