Finalize

Author: Pijukatel

src/crawlee/crawlers/_basic/_basic_crawler.py

@@ -59,6 +59,7 @@
     RequestHandlerError,
     SessionError,
     UserDefinedErrorHandlerError,
+    UserHandlerTimeoutError,
 )
 from crawlee.events._types import Event, EventCrawlerStatusData
 from crawlee.http_clients import ImpitHttpClient
@@ -1222,10 +1223,11 @@ def _get_message_from_error(self, error: Exception) -> str:
 
         if (
             isinstance(error, asyncio.exceptions.TimeoutError)
+            and traceback_parts
             and self._request_handler_timeout_text in traceback_parts[-1]
-        ):
+        ) or isinstance(error, UserHandlerTimeoutError):
             used_traceback_parts = reduce_asyncio_timeout_error_to_relevant_traceback_parts(error)
-            used_traceback_parts.append(traceback_parts[-1])
+            used_traceback_parts.extend(traceback_parts[-1:])
 
         return ''.join(used_traceback_parts).strip('\n')
 

src/crawlee/crawlers/_basic/_logging_utils.py

@@ -2,6 +2,7 @@
 import re
 import traceback
 
+import crawlee.errors
 import crawlee.router
 
 
@@ -10,7 +11,7 @@ def _get_only_innermost_exception(error: BaseException) -> BaseException:
 
     If the innermost exception is UserHandlerTimeoutError, return whatever caused that if possible.
     """
-    if type(error) is crawlee.router.UserHandlerTimeoutError:
+    if type(error) is crawlee.errors.UserHandlerTimeoutError:
         if error.__cause__:
             return error.__cause__
         if error.__context__:
@@ -46,7 +47,7 @@ def _strip_pep657_highlighting(traceback_part: str) -> str:
 
 
 def reduce_asyncio_timeout_error_to_relevant_traceback_parts(
-    timeout_error: asyncio.exceptions.TimeoutError,
+    timeout_error: asyncio.exceptions.TimeoutError | crawlee.errors.UserHandlerTimeoutError,
 ) -> list[str]:
     innermost_error_traceback_parts = _get_traceback_parts_for_innermost_exception(timeout_error)
     return _get_filtered_traceback_parts_for_asyncio_timeout_error(innermost_error_traceback_parts)
@@ -63,7 +64,7 @@ def get_one_line_error_summary_if_possible(error: Exception) -> str:
     if isinstance(error, asyncio.exceptions.TimeoutError):
         relevant_part = reduce_asyncio_timeout_error_to_relevant_traceback_parts(error)
         most_relevant_part = (',' + relevant_part[-1]) if len(relevant_part) else ''
-    elif isinstance(error, crawlee.router.UserHandlerTimeoutError):
+    elif isinstance(error, crawlee.errors.UserHandlerTimeoutError):
         # Error is user defined handler. First two lines should be location of the `UserHandlerTimeoutError` in crawlee
         # code and third line the topmost user error
         traceback_parts = _get_traceback_parts_for_innermost_exception(error)

src/crawlee/errors.py

@@ -29,6 +29,10 @@ class UserDefinedErrorHandlerError(Exception):
     """Wraps an exception thrown from an user-defined error handler."""
 
 
+class UserHandlerTimeoutError(UserDefinedErrorHandlerError):
+    """Raised when a router fails due to user raised timeout. This is different from user-defined handler timing out."""
+
+
 @docs_group('Errors')
 class SessionError(Exception):
     """Errors of `SessionError` type will trigger a session rotation.

src/crawlee/router.py

@@ -9,6 +9,8 @@
 
 __all__ = ['Router']
 
+from crawlee.errors import UserHandlerTimeoutError
+
 TCrawlingContext = TypeVar('TCrawlingContext', bound=BasicCrawlingContext)
 RequestHandler = Callable[[TCrawlingContext], Awaitable[None]]
 
@@ -105,7 +107,3 @@ async def __call__(self, context: TCrawlingContext) -> None:
         except asyncio.TimeoutError as e:
             # Timeout in handler, but not timeout of handler.
             raise UserHandlerTimeoutError('Timeout raised by user defined handler') from e
-
-
-class UserHandlerTimeoutError(Exception):
-    """Raised when a router fails due to user timeout."""

tests/unit/crawlers/_basic/test_basic_crawler.py

@@ -1542,6 +1542,7 @@ async def test_reduced_logs_from_timed_out_request_handler(caplog: pytest.LogCap
     caplog.set_level(logging.INFO)
     crawler = BasicCrawler(
         configure_logging=False,
+        max_request_retries=1,
         request_handler_timeout=timedelta(seconds=1),
     )
 
@@ -1568,7 +1569,7 @@ async def handler(context: BasicCrawlingContext) -> None:
 
 
 async def test_reduced_logs_from_time_out_in_request_handler(caplog: pytest.LogCaptureFixture) -> None:
-    crawler = BasicCrawler(configure_logging=False)
+    crawler = BasicCrawler(configure_logging=False, max_request_retries=1)
 
     @crawler.router.default_handler
     async def default_handler(_: BasicCrawlingContext) -> None: