merge

Author: Mantisus

.github/workflows/run_code_checks.yaml

@@ -36,6 +36,7 @@ jobs:
       httpbin_url: ${{ secrets.APIFY_HTTPBIN_TOKEN && format('https://httpbin.apify.actor?token={0}', secrets.APIFY_HTTPBIN_TOKEN) || 'https://httpbin.org'}}
     with:
       python-versions: '["3.10", "3.11", "3.12", "3.13", "3.14"]'
+      os: '["ubuntu-latest", "windows-latest", "macos-latest"]'
 
   docs_check:
     name: Docs check

CHANGELOG.md

@@ -3,7 +3,16 @@
 All notable changes to this project will be documented in this file.
 
 <!-- git-cliff-unreleased-start -->
-## 1.1.2 - **not yet released**
+## 1.2.1 - **not yet released**
+
+### 🐛 Bug Fixes
+
+- Fix short error summary ([#1605](https://github.com/apify/crawlee-python/pull/1605)) ([b751208](https://github.com/apify/crawlee-python/commit/b751208d9a56e9d923e4559baeba35e2eede0450)) by [@Pijukatel](https://github.com/Pijukatel), closes [#1602](https://github.com/apify/crawlee-python/issues/1602)
+- Freeze core `Request` fields ([#1603](https://github.com/apify/crawlee-python/pull/1603)) ([ae6d86b](https://github.com/apify/crawlee-python/commit/ae6d86b8c82900116032596201d94cd7875aaadc)) by [@Mantisus](https://github.com/Mantisus)
+
+
+<!-- git-cliff-unreleased-end -->
+## [1.2.0](https://github.com/apify/crawlee-python/releases/tag/v1.2.0) (2025-12-08)
 
 ### 🚀 Features
 
@@ -14,9 +23,9 @@ All notable changes to this project will be documented in this file.
 
 - Only apply requestHandlerTimeout to request handler ([#1474](https://github.com/apify/crawlee-python/pull/1474)) ([0dfb6c2](https://github.com/apify/crawlee-python/commit/0dfb6c2a13b6650736245fa39b3fbff397644df7)) by [@janbuchar](https://github.com/janbuchar)
 - Handle the case when `error_handler` returns `Request` ([#1595](https://github.com/apify/crawlee-python/pull/1595)) ([8a961a2](https://github.com/apify/crawlee-python/commit/8a961a2b07d0d33a7302dbb13c17f3d90999d390)) by [@Mantisus](https://github.com/Mantisus)
+- Align `Request.state` transitions with `Request` lifecycle ([#1601](https://github.com/apify/crawlee-python/pull/1601)) ([383225f](https://github.com/apify/crawlee-python/commit/383225f9f055d95ffb1302b8cf96f42ec264f1fc)) by [@Mantisus](https://github.com/Mantisus)
 
 
-<!-- git-cliff-unreleased-end -->
 ## [1.1.1](https://github.com/apify/crawlee-python/releases/tag/v1.1.1) (2025-12-02)
 
 ### 🐛 Bug Fixes

pyproject.toml

@@ -4,7 +4,7 @@ build-backend = "hatchling.build"
 
 [project]
 name = "crawlee"
-version = "1.1.2"
+version = "1.2.1"
 description = "Crawlee for Python"
 authors = [{ name = "Apify Technologies s.r.o.", email = "[email protected]" }]
 license = { file = "LICENSE" }

src/crawlee/__init__.py

@@ -1,6 +1,6 @@
 from importlib import metadata
 
-from ._request import Request, RequestOptions
+from ._request import Request, RequestOptions, RequestState
 from ._service_locator import service_locator
 from ._types import ConcurrencySettings, EnqueueStrategy, HttpHeaders, RequestTransformAction, SkippedReason
 from ._utils.globs import Glob
@@ -14,6 +14,7 @@
     'HttpHeaders',
     'Request',
     'RequestOptions',
+    'RequestState',
     'RequestTransformAction',
     'SkippedReason',
     'service_locator',

src/crawlee/_request.py

@@ -34,14 +34,14 @@ class RequestState(IntEnum):
 class CrawleeRequestData(BaseModel):
     """Crawlee-specific configuration stored in the `user_data`."""
 
-    max_retries: Annotated[int | None, Field(alias='maxRetries')] = None
+    max_retries: Annotated[int | None, Field(alias='maxRetries', frozen=True)] = None
     """Maximum number of retries for this request. Allows to override the global `max_request_retries` option of
     `BasicCrawler`."""
 
     enqueue_strategy: Annotated[EnqueueStrategy | None, Field(alias='enqueueStrategy')] = None
     """The strategy that was used for enqueuing the request."""
 
-    state: RequestState | None = None
+    state: RequestState = RequestState.UNPROCESSED
     """Describes the request's current lifecycle state."""
 
     session_rotation_count: Annotated[int | None, Field(alias='sessionRotationCount')] = None
@@ -137,6 +137,8 @@ class RequestOptions(TypedDict):
     always_enqueue: NotRequired[bool]
     user_data: NotRequired[dict[str, JsonSerializable]]
     no_retry: NotRequired[bool]
+    enqueue_strategy: NotRequired[EnqueueStrategy]
+    max_retries: NotRequired[int | None]
 
 
 @docs_group('Storage data')
@@ -166,7 +168,7 @@ class Request(BaseModel):
 
     model_config = ConfigDict(validate_by_name=True, validate_by_alias=True)
 
-    unique_key: Annotated[str, Field(alias='uniqueKey')]
+    unique_key: Annotated[str, Field(alias='uniqueKey', frozen=True)]
     """A unique key identifying the request. Two requests with the same `unique_key` are considered as pointing
     to the same URL.
 
@@ -178,17 +180,18 @@ class Request(BaseModel):
     and specify which URLs shall be considered equal.
     """
 
-    url: Annotated[str, BeforeValidator(validate_http_url), Field()]
+    url: Annotated[str, BeforeValidator(validate_http_url), Field(frozen=True)]
     """The URL of the web page to crawl. Must be a valid HTTP or HTTPS URL, and may include query parameters
     and fragments."""
 
-    method: HttpMethod = 'GET'
+    method: Annotated[HttpMethod, Field(frozen=True)] = 'GET'
     """HTTP request method."""
 
     payload: Annotated[
         HttpPayload | None,
         BeforeValidator(lambda v: v.encode() if isinstance(v, str) else v),
         PlainSerializer(lambda v: v.decode() if isinstance(v, bytes) else v),
+        Field(frozen=True),
     ] = None
     """HTTP request payload."""
 
@@ -250,6 +253,8 @@ def from_url(
         keep_url_fragment: bool = False,
         use_extended_unique_key: bool = False,
         always_enqueue: bool = False,
+        enqueue_strategy: EnqueueStrategy | None = None,
+        max_retries: int | None = None,
         **kwargs: Any,
     ) -> Self:
         """Create a new `Request` instance from a URL.
@@ -277,6 +282,9 @@ def from_url(
                 `unique_key` computation. This is only relevant when `unique_key` is not provided.
             always_enqueue: If set to `True`, the request will be enqueued even if it is already present in the queue.
                 Using this is not allowed when a custom `unique_key` is also provided and will result in a `ValueError`.
+            enqueue_strategy: The strategy that will be used for enqueuing the request.
+            max_retries: Maximum number of retries for this request. Allows to override the global `max_request_retries`
+                option of `BasicCrawler`.
             **kwargs: Additional request properties.
         """
         if unique_key is not None and always_enqueue:
@@ -301,12 +309,27 @@ def from_url(
         if always_enqueue:
             unique_key = f'{crypto_random_object_id()}|{unique_key}'
 
+        user_data_dict = kwargs.pop('user_data', {}) or {}
+        crawlee_data_dict = user_data_dict.get('__crawlee', {})
+
+        if max_retries is not None:
+            crawlee_data_dict['maxRetries'] = max_retries
+
+        if enqueue_strategy is not None:
+            crawlee_data_dict['enqueueStrategy'] = enqueue_strategy
+
+        crawlee_data = CrawleeRequestData(**crawlee_data_dict)
+
+        if crawlee_data:
+            user_data_dict['__crawlee'] = crawlee_data
+
         request = cls(
             url=url,
             unique_key=unique_key,
             method=method,
             headers=headers,
             payload=payload,
+            user_data=user_data_dict,
             **kwargs,
         )
 
@@ -352,7 +375,7 @@ def crawl_depth(self, new_value: int) -> None:
         self.crawlee_data.crawl_depth = new_value
 
     @property
-    def state(self) -> RequestState | None:
+    def state(self) -> RequestState:
         """Crawlee-specific request handling state."""
         return self.crawlee_data.state
 
@@ -365,10 +388,6 @@ def max_retries(self) -> int | None:
         """Crawlee-specific limit on the number of retries of the request."""
         return self.crawlee_data.max_retries
 
-    @max_retries.setter
-    def max_retries(self, new_max_retries: int) -> None:
-        self.crawlee_data.max_retries = new_max_retries
-
     @property
     def session_rotation_count(self) -> int | None:
         """Crawlee-specific number of finished session rotations for the request."""

src/crawlee/_types.py

@@ -266,7 +266,6 @@ def __init__(
         *,
         key_value_store_getter: GetKeyValueStoreFunction,
         request: Request,
-        session: Session | None = None,
     ) -> None:
         self._key_value_store_getter = key_value_store_getter
         self.add_requests_calls = list[AddRequestsKwargs]()
@@ -275,16 +274,11 @@ def __init__(
 
         # Isolated copies for handler execution
         self._request = deepcopy(request)
-        self._session = deepcopy(session) if session else None
 
     @property
     def request(self) -> Request:
         return self._request
 
-    @property
-    def session(self) -> Session | None:
-        return self._session
-
     async def add_requests(
         self,
         requests: Sequence[str | Request],
@@ -337,27 +331,11 @@ async def get_key_value_store(
     def apply_request_changes(self, target: Request) -> None:
         """Apply tracked changes from handler copy to original request."""
         if self.request.user_data != target.user_data:
-            target.user_data.update(self.request.user_data)
+            target.user_data = self.request.user_data
 
         if self.request.headers != target.headers:
             target.headers = target.headers | self.request.headers
 
-    def apply_session_changes(self, target: Session | None = None) -> None:
-        """Apply tracked changes from handler copy to original session."""
-        simple_fields: set[str] = {'_usage_count', '_error_score'}
-
-        if self.session and target:
-            if self.session.user_data != target.user_data:
-                target.user_data.update(self.session.user_data)
-
-            if self.session.cookies != target.cookies:
-                target.cookies.set_cookies(self.session.cookies.get_cookies_as_dicts())
-            for field in simple_fields:
-                value = getattr(self.session, field)
-                original_value = getattr(target, field)
-                if value != original_value:
-                    object.__setattr__(target, field, value)
-
 
 @docs_group('Functions')
 class AddRequestsFunction(Protocol):

src/crawlee/crawlers/_abstract_http/_abstract_http_crawler.py

@@ -10,7 +10,7 @@
 from pydantic import ValidationError
 from typing_extensions import NotRequired, TypeVar
 
-from crawlee._request import Request, RequestOptions
+from crawlee._request import Request, RequestOptions, RequestState
 from crawlee._utils.docs import docs_group
 from crawlee._utils.time import SharedTimeout
 from crawlee._utils.urls import to_absolute_url_iterator
@@ -257,6 +257,7 @@ async def _make_http_request(self, context: BasicCrawlingContext) -> AsyncGenera
                 timeout=remaining_timeout,
             )
 
+        context.request.state = RequestState.AFTER_NAV
         yield HttpCrawlingContext.from_basic_crawling_context(context=context, http_response=result.http_response)
 
     async def _handle_status_code_response(

src/crawlee/crawlers/_adaptive_playwright/_adaptive_playwright_crawler.py

@@ -291,12 +291,13 @@ async def get_input_state(
 
         # New result is created and injected to newly created context. This is done to ensure isolation of sub crawlers.
         result = RequestHandlerRunResult(
-            key_value_store_getter=self.get_key_value_store, request=context.request, session=context.session
+            key_value_store_getter=self.get_key_value_store,
+            request=context.request,
         )
         context_linked_to_result = BasicCrawlingContext(
             request=result.request,
-            session=result.session,
-            proxy_info=deepcopy(context.proxy_info),
+            session=context.session,
+            proxy_info=context.proxy_info,
             send_request=context.send_request,
             add_requests=result.add_requests,
             push_data=result.push_data,

src/crawlee/crawlers/_basic/_basic_crawler.py

@@ -59,6 +59,7 @@
     RequestHandlerError,
     SessionError,
     UserDefinedErrorHandlerError,
+    UserHandlerTimeoutError,
 )
 from crawlee.events._types import Event, EventCrawlerStatusData
 from crawlee.http_clients import ImpitHttpClient
@@ -1135,7 +1136,7 @@ async def _handle_request_retries(
             request.retry_count += 1
             reduced_error = str(error).split('\n')[0]
             self.log.warning(
-                f'Retrying request to {context.request.url} due to: {reduced_error}'
+                f'Retrying request to {context.request.url} due to: {reduced_error}. '
                 f'{get_one_line_error_summary_if_possible(error)}'
             )
             await self._statistics.error_tracker.add(error=error, context=context)
@@ -1153,6 +1154,7 @@ async def _handle_request_retries(
 
             await request_manager.reclaim_request(request)
         else:
+            request.state = RequestState.ERROR
             await self._mark_request_as_handled(request)
             await self._handle_failed_request(context, error)
             self._statistics.record_request_processing_failure(request.unique_key)
@@ -1168,8 +1170,6 @@ async def _handle_request_error(self, context: TCrawlingContext | BasicCrawlingC
                 f'{self._internal_timeout.total_seconds()} seconds',
                 logger=self._logger,
             )
-
-            context.request.state = RequestState.DONE
         except UserDefinedErrorHandlerError:
             context.request.state = RequestState.ERROR
             raise
@@ -1202,8 +1202,8 @@ async def _handle_skipped_request(
         self, request: Request | str, reason: SkippedReason, *, need_mark: bool = False
     ) -> None:
         if need_mark and isinstance(request, Request):
-            await self._mark_request_as_handled(request)
             request.state = RequestState.SKIPPED
+            await self._mark_request_as_handled(request)
 
         url = request.url if isinstance(request, Request) else request
 
@@ -1223,10 +1223,11 @@ def _get_message_from_error(self, error: Exception) -> str:
 
         if (
             isinstance(error, asyncio.exceptions.TimeoutError)
+            and traceback_parts
             and self._request_handler_timeout_text in traceback_parts[-1]
-        ):
+        ) or isinstance(error, UserHandlerTimeoutError):
             used_traceback_parts = reduce_asyncio_timeout_error_to_relevant_traceback_parts(error)
-            used_traceback_parts.append(traceback_parts[-1])
+            used_traceback_parts.extend(traceback_parts[-1:])
 
         return ''.join(used_traceback_parts).strip('\n')
 
@@ -1324,7 +1325,6 @@ async def _commit_request_handler_result(
 
         await self._commit_key_value_store_changes(result, get_kvs=self.get_key_value_store)
 
-        result.apply_session_changes(target=context.session)
         result.apply_request_changes(target=context.request)
 
     @staticmethod
@@ -1392,13 +1392,11 @@ async def __run_task_function(self) -> None:
         else:
             session = await self._get_session()
         proxy_info = await self._get_proxy_info(request, session)
-        result = RequestHandlerRunResult(
-            key_value_store_getter=self.get_key_value_store, request=request, session=session
-        )
+        result = RequestHandlerRunResult(key_value_store_getter=self.get_key_value_store, request=request)
 
         context = BasicCrawlingContext(
             request=result.request,
-            session=result.session,
+            session=session,
             proxy_info=proxy_info,
             send_request=self._prepare_send_request_function(session, proxy_info),
             add_requests=result.add_requests,
@@ -1415,18 +1413,18 @@ async def __run_task_function(self) -> None:
             request.state = RequestState.REQUEST_HANDLER
 
             try:
-                with swaped_context(context, request, session):
+                with swaped_context(context, request):
                     self._check_request_collision(request, session)
                     await self._run_request_handler(context=context)
             except asyncio.TimeoutError as e:
                 raise RequestHandlerError(e, context) from e
 
             await self._commit_request_handler_result(context)
 
-            await self._mark_request_as_handled(request)
-
             request.state = RequestState.DONE
 
+            await self._mark_request_as_handled(request)
+
             if session and session.is_usable:
                 session.mark_good()
 
@@ -1493,6 +1491,7 @@ async def __run_task_function(self) -> None:
             raise
 
     async def _run_request_handler(self, context: BasicCrawlingContext) -> None:
+        context.request.state = RequestState.BEFORE_NAV
         await self._context_pipeline(
             context,
             lambda final_context: wait_for(