Use Pydantic to handle raw inputs

Fix typing issues

WIP

Author: Pijukatel

pyproject.toml

@@ -141,6 +141,9 @@ indent-style = "space"
 docstring-quotes = "double"
 inline-quotes = "single"
 
+[tool.ruff.lint.flake8-type-checking]
+runtime-evaluated-base-classes = ["pydantic.BaseModel"]
+
 [tool.ruff.lint.flake8-builtins]
 builtins-ignorelist = ["id"]
 

src/apify/_actor.py

@@ -2,10 +2,8 @@
 
 import asyncio
 import os
-import re
 import sys
 from datetime import timedelta
-from itertools import chain
 from typing import TYPE_CHECKING, Any, Callable, TypeVar, cast
 
 from lazy_object_proxy import Proxy
@@ -15,10 +13,8 @@
 from apify_client import ApifyClientAsync
 from apify_shared.consts import ActorEnvVars, ActorExitCodes, ApifyEnvVars
 from apify_shared.utils import ignore_docs, maybe_extract_enum_member_value
-from crawlee import Request, service_container
+from crawlee import service_container
 from crawlee.events._types import Event, EventPersistStateData
-from crawlee.http_clients import BaseHttpClient, HttpResponse, HttpxHttpClient
-from crawlee.storages import RequestList
 
 from apify._actor_inputs import _create_request_list
 from apify._configuration import Configuration
@@ -36,15 +32,16 @@
     import logging
     from types import TracebackType
 
+    from crawlee.http_clients import BaseHttpClient
     from crawlee.proxy_configuration import _NewUrlFunction
+    from crawlee.storages import RequestList
 
     from apify._models import Webhook
 
 
 MainReturnType = TypeVar('MainReturnType')
 
 
-
 class _ActorType:
     """The class of `Actor`. Only make a new instance if you're absolutely sure you need to."""
 
@@ -982,19 +979,20 @@ async def create_proxy_configuration(
 
     @staticmethod
     async def create_request_list(
-        *, actor_start_urls_input: list[dict[str,str]], http_client: BaseHttpClient | None= None
+        *, actor_start_urls_input: list[dict[str, Any]], http_client: BaseHttpClient | None = None
     ) -> RequestList:
-        """Creates request list from Actor input requestListSources. This accepts list of urls and requestsFromUrl.
+        """Creates request list from Actor input requestListSources. This accepts list of urls and requests_from_url.
 
         Example:
             actor_start_urls_input = [
                 # Gather urls from response body.
-                {'requestsFromUrl': 'https://crawlee.dev/file.txt', 'method': 'GET'},
+                {'requests_from_url': 'https://crawlee.dev/file.txt', 'method': 'GET'},
                 # Directly include this url.
                 {'url': 'https://crawlee.dev', 'method': 'GET'}
             ]
-            """
+        """
         return await _create_request_list(actor_start_urls_input=actor_start_urls_input, http_client=http_client)
 
+
 Actor = cast(_ActorType, Proxy(_ActorType))
 """The entry point of the SDK, through which all the Actor operations should be done."""

src/apify/_actor_inputs.py

@@ -1,26 +1,46 @@
+from __future__ import annotations
+
 import asyncio
-from itertools import chain
 import re
+from asyncio import Task
+from typing import Any
+
+from pydantic import BaseModel, Field
 
 from crawlee import Request
-from crawlee.http_clients import BaseHttpClient, HttpxHttpClient, HttpResponse
+from crawlee._types import HttpMethod  #  TODO: Make public in Crawlee?
+from crawlee.http_clients import BaseHttpClient, HttpxHttpClient
 from crawlee.storages import RequestList
 
 URL_NO_COMMAS_REGEX = re.compile(
     r'https?:\/\/(www\.)?([a-zA-Z0-9]|[a-zA-Z0-9][-a-zA-Z0-9@:%._+~#=]{0,254}[a-zA-Z0-9])\.[a-z]{2,63}(:\d{1,5})?(\/[-a-zA-Z0-9@:%_+.~#?&/=()]*)?'
 )
 
+class _RequestDetails(BaseModel):
+    method: HttpMethod
+    payload: str = ''
+    headers: dict[str, str] = Field(default_factory=dict)
+    user_data: dict[str, str]= Field(default_factory=dict, alias='user_data')
+
+class _RequestsFromUrlInput(_RequestDetails):
+    requests_from_url: str = Field(alias='requests_from_url')
+
+class _SimpleUrlInput(_RequestDetails):
+    url: str
+
+
 @staticmethod
 async def _create_request_list(
-    *, actor_start_urls_input: dict, http_client: BaseHttpClient | None = None
+    *, actor_start_urls_input: list[dict[str, Any]], http_client: BaseHttpClient | None = None
 ) -> RequestList:
     if not http_client:
         http_client = HttpxHttpClient()
     simple_url_requests_inputs = [
-        request_input for request_input in actor_start_urls_input if 'url' in request_input
-    ]
+        _SimpleUrlInput(**request_input) for request_input in actor_start_urls_input
+        if 'url' in request_input]
     remote_url_requests_inputs = [
-        request_input for request_input in actor_start_urls_input if 'requestsFromUrl' in request_input
+        _RequestsFromUrlInput(**request_input) for request_input in actor_start_urls_input
+        if 'requests_from_url' in request_input
     ]
 
     simple_url_requests = _create_requests_from_input(simple_url_requests_inputs)
@@ -30,44 +50,41 @@ async def _create_request_list(
 
 
 @staticmethod
-def _create_requests_from_input(simple_url_requests_inputs: list[dict[str, str]]) -> list[Request]:
+def _create_requests_from_input(simple_url_inputs: list[_SimpleUrlInput]) -> list[Request]:
     return [
         Request.from_url(
-            method=request_input.get('method'),
-            url=request_input.get('url'),
-            payload=request_input.get('payload', '').encode('utf-8'),
-            headers=request_input.get('headers', {}),
-            user_data=request_input.get('userData', {}),
+            method=request_input.method,
+            url=request_input.url,
+            payload=request_input.payload.encode('utf-8'),
+            headers=request_input.headers,
+            user_data=request_input.user_data,
         )
-        for request_input in simple_url_requests_inputs
+        for request_input in simple_url_inputs
     ]
 
 
 @staticmethod
 async def _create_requests_from_url(
-    remote_url_requests_inputs: list[dict[str, str]], http_client: BaseHttpClient
+    remote_url_requests_inputs: list[_RequestsFromUrlInput], http_client: BaseHttpClient
 ) -> list[Request]:
+    created_requests: list[Request] = []
+
+    def extract_requests_from_response(task: Task) -> list[Request]:
+        matches = re.finditer(URL_NO_COMMAS_REGEX, task.result().read().decode('utf-8'))
+        created_requests.extend([Request.from_url(match.group(0)) for match in matches])
+
     remote_url_requests = []
     for request_input in remote_url_requests_inputs:
-        remote_url_requests.append(
-            asyncio.create_task(
-                http_client.send_request(
-                    method=request_input['method'],
-                    url=request_input['requestsFromUrl'],
-                    headers=request_input.get('headers', {}),
-                    payload=request_input.get('payload', '').encode('utf-8'),
-                )
+        task = asyncio.create_task(
+            http_client.send_request(
+                method=request_input.method,
+                url=request_input.requests_from_url,
+                headers=request_input.headers,
+                payload=request_input.payload.encode('utf-8'),
             )
         )
-    await asyncio.gather(*remote_url_requests)
-    # TODO as callbacks
-    a = list(
-        extract_requests_from_response(finished_request.result()) for finished_request in remote_url_requests
-    )
-    return list(chain.from_iterable(a))
+        task.add_done_callback(extract_requests_from_response)
+        remote_url_requests.append(task)
 
-
-@staticmethod
-def extract_requests_from_response(response: HttpResponse) -> list[Request]:
-    matches = list(re.finditer(URL_NO_COMMAS_REGEX, response.read().decode('utf-8')))
-    return [Request.from_url(match.group(0)) for match in matches]
+    await asyncio.gather(*remote_url_requests)
+    return created_requests

src/apify/_platform_event_manager.py

@@ -1,7 +1,7 @@
 from __future__ import annotations
 
 import asyncio
-from datetime import datetime  # noqa: TCH003
+from datetime import datetime
 from typing import TYPE_CHECKING, Annotated, Any, Literal, Union
 
 import websockets.client

tests/unit/actor/test_actor_create_proxy_configuration.py

@@ -1,18 +1,12 @@
 from __future__ import annotations
 
-import typing
 from typing import TYPE_CHECKING
-from unittest import mock
-from unittest.mock import call
 
 import httpx
 import pytest
 
 from apify_client import ApifyClientAsync
 from apify_shared.consts import ApifyEnvVars
-from crawlee._request import UserData
-from crawlee._types import HttpHeaders, HttpMethod
-from crawlee.http_clients import HttpResponse, HttpxHttpClient
 
 from apify import Actor
 
@@ -147,128 +141,3 @@ async def test_proxy_configuration_with_actor_proxy_input(
     assert len(route.calls) == 2
 
     await Actor.exit()
-
-
-@pytest.mark.parametrize('request_method', typing.get_args(HttpMethod))
-@pytest.mark.parametrize(
-    'optional_input',
-    [
-        {},
-        {'payload': 'some payload', 'userData': {'some key': 'some value'}, 'headers': {'h1': 'v1', 'h2': 'v2'}},
-    ],
-    ids=['minimal', 'all_options'],
-)
-async def test_actor_create_request_list_request_types(
-    request_method: HttpMethod, optional_input: dict[str, str]
-) -> None:
-    """Test proper request list generation from both minimal and full inputs for all method types for simple input."""
-    minimal_request_dict_input = {'url': 'https://www.abc.com', 'method': request_method}
-    request_dict_input = {**minimal_request_dict_input, **optional_input}
-    example_start_urls_input = [
-        request_dict_input,
-    ]
-
-    generated_request_list = await Actor.create_request_list(actor_start_urls_input=example_start_urls_input)
-
-    assert not await generated_request_list.is_empty()
-    generated_request = await generated_request_list.fetch_next_request()
-    assert await generated_request_list.is_empty()
-
-    assert generated_request.method == request_dict_input['method']
-    assert generated_request.url == request_dict_input['url']
-    assert generated_request.payload == request_dict_input.get('payload', '').encode('utf-8')
-    expected_user_data = UserData()
-    if 'userData' in optional_input:
-        for key, value in optional_input['userData'].items():
-            expected_user_data[key] = value
-    assert generated_request.user_data == expected_user_data
-    expected_headers = HttpHeaders(root=optional_input.get('headers', {}))
-    assert generated_request.headers == expected_headers
-
-
-def _create_dummy_response(read_output: typing.Iterable[str]) -> HttpResponse:
-    """Create dummy_response that will iterate through read_output when called like dummy_response.read()"""
-
-    class DummyResponse(HttpResponse):
-        @property
-        def http_version(self) -> str:
-            return ''
-
-        @property
-        def status_code(self) -> int:
-            return 200
-
-        @property
-        def headers(self) -> HttpHeaders:
-            return HttpHeaders()
-
-        def read(self) -> bytes:
-            return next(read_output).encode('utf-8')
-
-    return DummyResponse()
-
-
-async def test_actor_create_request_list_from_url_correctly_send_requests() -> None:
-    """Test that injected HttpClient's method send_request is called with properly passed arguments."""
-
-    example_start_urls_input = [
-        {'requestsFromUrl': 'https://crawlee.dev/file.txt', 'method': 'GET'},
-        {'requestsFromUrl': 'https://www.crawlee.dev/file2', 'method': 'PUT'},
-        {
-            'requestsFromUrl': 'https://www.something.som',
-            'method': 'POST',
-            'headers': {'key': 'value'},
-            'payload': 'some_payload',
-            'userData': 'irrelevant',
-        },
-    ]
-    mocked_read_outputs = ('' for url in example_start_urls_input)
-    http_client = HttpxHttpClient()
-    with mock.patch.object(
-        http_client, 'send_request', return_value=_create_dummy_response(mocked_read_outputs)
-    ) as mocked_send_request:
-        await Actor.create_request_list(actor_start_urls_input=example_start_urls_input, http_client=http_client)
-
-    expected_calls = [
-        call(
-            method=example_input['method'],
-            url=example_input['requestsFromUrl'],
-            headers=example_input.get('headers', {}),
-            payload=example_input.get('payload', '').encode('utf-8'),
-        )
-        for example_input in example_start_urls_input
-    ]
-    mocked_send_request.assert_has_calls(expected_calls)
-
-
-async def test_actor_create_request_list_from_url() -> None:
-    """Test that create_request_list is correctly reading urls from remote url sources and also from simple input."""
-    expected_simple_url = 'https://www.someurl.com'
-    expected_remote_urls_1 = {'http://www.something.com', 'https://www.somethingelse.com', 'http://www.bla.net'}
-    expected_remote_urls_2 = {'http://www.ok.com', 'https://www.true-positive.com'}
-    expected_urls = expected_remote_urls_1 | expected_remote_urls_2 | {expected_simple_url}
-    response_bodies = iter(
-        (
-            'blablabla{} more blablabla{} , even more blablabla. {} '.format(*expected_remote_urls_1),
-            'some stuff{} more stuff{} www.falsepositive www.false_positive.com'.format(*expected_remote_urls_2),
-        )
-    )
-
-    example_start_urls_input = [
-        {'requestsFromUrl': 'https://crawlee.dev/file.txt', 'method': 'GET'},
-        {'url': expected_simple_url, 'method': 'GET'},
-        {'requestsFromUrl': 'https://www.crawlee.dev/file2', 'method': 'GET'},
-    ]
-
-    http_client = HttpxHttpClient()
-    with mock.patch.object(http_client, 'send_request', return_value=_create_dummy_response(response_bodies)):
-        generated_request_list = await Actor.create_request_list(
-            actor_start_urls_input=example_start_urls_input, http_client=http_client
-        )
-        generated_requests = []
-        while request := await generated_request_list.fetch_next_request():
-            print(request)
-            generated_requests.append(request)
-
-    # Check correctly created requests' urls in request list
-    assert {generated_request.url for generated_request in generated_requests} == expected_urls