Fix tests and bugs

Author: janbuchar

src/crawlee/crawlers/_playwright/_playwright_crawler.py

@@ -7,6 +7,7 @@
 from functools import partial
 from typing import TYPE_CHECKING, Any, Generic, Literal
 
+import playwright.async_api
 from more_itertools import partition
 from pydantic import ValidationError
 from typing_extensions import NotRequired, TypedDict, TypeVar
@@ -272,6 +273,7 @@ async def _navigate(
         Raises:
             ValueError: If the browser pool is not initialized.
             SessionError: If the URL cannot be loaded by the browser.
+            TimeoutError: If navigation does not succeed within the navigation timeout.
 
         Yields:
             The enhanced crawling context with the Playwright-specific features (page, response, enqueue_links,
@@ -303,9 +305,12 @@ async def _navigate(
                 # Set route_handler only for current request
                 await context.page.route(context.request.url, route_handler)
 
-            response = await asyncio.wait_for(
-                context.page.goto(context.request.url), timeout=self._navigation_timeout.total_seconds()
-            )
+            try:
+                response = await context.page.goto(
+                    context.request.url, timeout=self._navigation_timeout.total_seconds() * 1000
+                )
+            except playwright.async_api.TimeoutError as exc:
+                raise asyncio.TimeoutError from exc
 
             if response is None:
                 raise SessionError(f'Failed to load the URL: {context.request.url}')

src/crawlee/http_clients/_curl_impersonate.py

@@ -1,5 +1,6 @@
 from __future__ import annotations
 
+import asyncio
 from contextlib import asynccontextmanager
 from typing import TYPE_CHECKING, Any
 
@@ -10,6 +11,7 @@
 from curl_cffi.requests.cookies import CurlMorsel
 from curl_cffi.requests.exceptions import ProxyError as CurlProxyError
 from curl_cffi.requests.exceptions import RequestException as CurlRequestError
+from curl_cffi.requests.exceptions import Timeout
 from curl_cffi.requests.impersonate import DEFAULT_CHROME as CURL_DEFAULT_CHROME
 from typing_extensions import override
 
@@ -160,6 +162,8 @@ async def crawl(
                 cookies=session.cookies.jar if session else None,
                 timeout=timeout.total_seconds() if timeout else None,
             )
+        except Timeout as exc:
+            raise asyncio.TimeoutError from exc
         except CurlRequestError as exc:
             if self._is_proxy_error(exc):
                 raise ProxyError from exc
@@ -205,6 +209,8 @@ async def send_request(
                 cookies=session.cookies.jar if session else None,
                 timeout=timeout.total_seconds() if timeout else None,
             )
+        except Timeout as exc:
+            raise asyncio.TimeoutError from exc
         except CurlRequestError as exc:
             if self._is_proxy_error(exc):
                 raise ProxyError from exc
@@ -245,6 +251,8 @@ async def stream(
                 stream=True,
                 timeout=timeout.total_seconds() if timeout else None,
             )
+        except Timeout as exc:
+            raise asyncio.TimeoutError from exc
         except CurlRequestError as exc:
             if self._is_proxy_error(exc):
                 raise ProxyError from exc

src/crawlee/http_clients/_httpx.py

@@ -1,5 +1,6 @@
 from __future__ import annotations
 
+import asyncio
 from contextlib import asynccontextmanager
 from logging import getLogger
 from typing import TYPE_CHECKING, Any, cast
@@ -163,6 +164,8 @@ async def crawl(
 
         try:
             response = await client.send(http_request)
+        except httpx.TimeoutException as exc:
+            raise asyncio.TimeoutError from exc
         except httpx.TransportError as exc:
             if self._is_proxy_error(exc):
                 raise ProxyError from exc
@@ -203,6 +206,8 @@ async def send_request(
 
         try:
             response = await client.send(http_request)
+        except httpx.TimeoutException as exc:
+            raise asyncio.TimeoutError from exc
         except httpx.TransportError as exc:
             if self._is_proxy_error(exc):
                 raise ProxyError from exc
@@ -235,7 +240,10 @@ async def stream(
             timeout=httpx.Timeout(None, connect=timeout.total_seconds()) if timeout else None,
         )
 
-        response = await client.send(http_request, stream=True)
+        try:
+            response = await client.send(http_request, stream=True)
+        except httpx.TimeoutException as exc:
+            raise asyncio.TimeoutError from exc
 
         try:
             yield _HttpxResponse(response)

src/crawlee/http_clients/_impit.py

@@ -138,7 +138,7 @@ async def crawl(
                 timeout=timeout.total_seconds() if timeout else None,
             )
         except TimeoutException as exc:
-            raise TimeoutError from exc
+            raise asyncio.TimeoutError from exc
         except (TransportError, HTTPError) as exc:
             if self._is_proxy_error(exc):
                 raise ProxyError from exc
@@ -177,7 +177,7 @@ async def send_request(
                 timeout=timeout.total_seconds() if timeout else None,
             )
         except TimeoutException as exc:
-            raise TimeoutError from exc
+            raise asyncio.TimeoutError from exc
         except (TransportError, HTTPError) as exc:
             if self._is_proxy_error(exc):
                 raise ProxyError from exc
@@ -210,7 +210,7 @@ async def stream(
                 stream=True,
             )
         except TimeoutException as exc:
-            raise TimeoutError from exc
+            raise asyncio.TimeoutError from exc
 
         try:
             yield _ImpitResponse(response)

tests/unit/crawlers/_beautifulsoup/test_beautifulsoup_crawler.py

@@ -1,5 +1,7 @@
 from __future__ import annotations
 
+import asyncio
+from datetime import timedelta
 from typing import TYPE_CHECKING
 from unittest import mock
 
@@ -341,3 +343,42 @@ async def handler(context: BeautifulSoupCrawlingContext) -> None:
             await context.enqueue_links(rq_id=queue_id, rq_name=queue_name, rq_alias=queue_alias)
 
     await crawler.run([str(server_url / 'start_enqueue')])
+
+
+async def test_navigation_timeout_on_slow_request(server_url: URL, http_client: HttpClient) -> None:
+    """Test that navigation_timeout causes TimeoutError on slow HTTP requests."""
+    crawler = BeautifulSoupCrawler(
+        http_client=http_client,
+        navigation_timeout=timedelta(seconds=1),
+        max_request_retries=0,
+    )
+
+    failed_request_handler = mock.AsyncMock()
+    crawler.failed_request_handler(failed_request_handler)
+
+    request_handler = mock.AsyncMock()
+    crawler.router.default_handler(request_handler)
+
+    # Request endpoint that delays 5 seconds - should timeout at 1 second
+    await crawler.run([str(server_url.with_path('/slow').with_query(delay=5))])
+
+    assert failed_request_handler.call_count == 1
+    assert isinstance(failed_request_handler.call_args[0][1], asyncio.TimeoutError)
+
+
+async def test_slow_navigation_does_not_count_toward_handler_timeout(server_url: URL, http_client: HttpClient) -> None:
+    crawler = BeautifulSoupCrawler(
+        http_client=http_client,
+        request_handler_timeout=timedelta(seconds=0.5),
+        max_request_retries=0,
+    )
+
+    request_handler = mock.AsyncMock()
+    crawler.router.default_handler(request_handler)
+
+    # Navigation takes 1 second (exceeds handler timeout), but should still succeed
+    result = await crawler.run([str(server_url.with_path('/slow').with_query(delay=1))])
+
+    assert result.requests_failed == 0
+    assert result.requests_finished == 1
+    assert request_handler.call_count == 1

tests/unit/crawlers/_playwright/test_playwright_crawler.py

@@ -4,11 +4,13 @@
 
 from __future__ import annotations
 
+import asyncio
 import json
 import logging
+from datetime import timedelta
 from typing import TYPE_CHECKING, Any, Literal
 from unittest import mock
-from unittest.mock import Mock
+from unittest.mock import AsyncMock, Mock
 
 import pytest
 
@@ -925,3 +927,43 @@ async def handler(context: PlaywrightCrawlingContext) -> None:
             await context.enqueue_links(rq_id=queue_id, rq_name=queue_name, rq_alias=queue_alias)
 
     await crawler.run([str(server_url / 'start_enqueue')])
+
+
+async def test_navigation_timeout_on_slow_page_load(server_url: URL) -> None:
+    crawler = PlaywrightCrawler(
+        navigation_timeout=timedelta(seconds=1),
+        max_request_retries=0,
+    )
+
+    request_handler = AsyncMock()
+    crawler.router.default_handler(request_handler)
+
+    failed_request_handler = AsyncMock()
+    crawler.failed_request_handler(failed_request_handler)
+
+    result = await crawler.run([str((server_url / 'slow').with_query(delay=2))])
+
+    assert result.requests_failed == 1
+    assert result.requests_finished == 0
+
+    assert request_handler.call_count == 0
+
+    assert failed_request_handler.call_count == 1
+    assert isinstance(failed_request_handler.call_args[0][1], asyncio.TimeoutError)
+
+
+async def test_slow_navigation_does_not_count_toward_handler_timeout(server_url: URL) -> None:
+    crawler = PlaywrightCrawler(
+        request_handler_timeout=timedelta(seconds=0.5),
+        max_request_retries=0,
+    )
+
+    request_handler = AsyncMock()
+    crawler.router.default_handler(request_handler)
+
+    # Navigation takes 1 second (exceeds handler timeout), but should still succeed
+    result = await crawler.run([str((server_url / 'slow').with_query(delay=1))])
+
+    assert result.requests_failed == 0
+    assert result.requests_finished == 1
+    assert request_handler.call_count == 1

tests/unit/server.py

@@ -121,6 +121,7 @@ async def app(scope: dict[str, Any], receive: Receive, send: Send) -> None:
         'xml': hello_world_xml,
         'robots.txt': robots_txt,
         'get_compressed': get_compressed,
+        'slow': slow_response,
     }
     path = URL(scope['path']).parts[1]
     # Route requests to appropriate handlers
@@ -411,6 +412,15 @@ async def get_compressed(_scope: dict[str, Any], _receive: Receive, send: Send)
     await send({'type': 'http.response.body', 'body': gzip.compress(HELLO_WORLD * 1000)})
 
 
+async def slow_response(scope: dict[str, Any], _receive: Receive, send: Send) -> None:
+    """Handle requests with a configurable delay to test timeouts."""
+    query_params = get_query_params(scope.get('query_string', b''))
+    delay = float(query_params.get('delay', '5'))  # Default 5 second delay
+
+    await asyncio.sleep(delay)
+    await send_html_response(send, HELLO_WORLD)
+
+
 class TestServer(Server):
     """A test HTTP server implementation based on Uvicorn Server."""