Close thread loop adapter from the correct thread

Author: elacuesta

scrapy_playwright/_utils.py

@@ -140,8 +140,11 @@ def _deferred_from_coro(cls, coro) -> Deferred:
         return dfd
 
     @classmethod
-    def start(cls, caller_id: int) -> None:
-        cls._stop_events[caller_id] = asyncio.Event()
+    def start(cls, download_handler_id: int) -> None:
+        """Start the event loop in a new thread if not already started.
+        Should be called from the Scrapy thread.
+        """
+        cls._stop_events[download_handler_id] = asyncio.Event()
         if not getattr(cls, "_loop", None):
             policy = asyncio.DefaultEventLoopPolicy()
             if platform.system() == "Windows":
@@ -155,9 +158,11 @@ def start(cls, caller_id: int) -> None:
             asyncio.run_coroutine_threadsafe(cls._process_queue(), cls._loop)
 
     @classmethod
-    def stop(cls, caller_id: int) -> None:
-        """Wait until all handlers are closed to stop the event loop and join the thread."""
-        cls._stop_events[caller_id].set()
+    def stop(cls, download_handler_id: int) -> None:
+        """Wait until all handlers are closed to stop the event loop and join the thread.
+        Should be called from the Scrapy thread.
+        """
+        cls._stop_events[download_handler_id].set()
         if all(ev.is_set() for ev in cls._stop_events.values()):
             asyncio.run_coroutine_threadsafe(cls._coro_queue.join(), cls._loop)
             cls._loop.call_soon_threadsafe(cls._loop.stop)

scrapy_playwright/handler.py

@@ -363,6 +363,8 @@ async def close(self) -> None:
             logger.info("Closing download handler")
             await super().close()
             await self._close()
+            if self.config.use_threaded_loop:
+                _ThreadedLoopAdapter.stop(id(self))
 
     else:
 
@@ -371,6 +373,8 @@ def close(self) -> Deferred:  # pylint: disable=invalid-overridden-method
             logger.info("Closing download handler")
             yield super().close()
             yield self._deferred_from_coro(self._close())
+            if self.config.use_threaded_loop:
+                _ThreadedLoopAdapter.stop(id(self))
 
     async def _close(self) -> None:
         with suppress(TargetClosedError):
@@ -383,8 +387,6 @@ async def _close(self) -> None:
             await self.playwright_context_manager.__aexit__()
         if self.playwright:
             await self.playwright.stop()
-        if self.config.use_threaded_loop:
-            _ThreadedLoopAdapter.stop(id(self))
 
     if _SCRAPY_ASYNC_API: