WIP

Finalize tests.
Split to its own file.

Author: Pijukatel

src/apify/_actor.py

@@ -8,7 +8,6 @@
 from itertools import chain
 from typing import TYPE_CHECKING, Any, Callable, TypeVar, cast
 
-from crawlee.http_clients import HttpxHttpClient, HttpResponse, BaseHttpClient
 from lazy_object_proxy import Proxy
 from pydantic import AliasChoices
 from typing_extensions import Self
@@ -18,8 +17,10 @@
 from apify_shared.utils import ignore_docs, maybe_extract_enum_member_value
 from crawlee import Request, service_container
 from crawlee.events._types import Event, EventPersistStateData
+from crawlee.http_clients import BaseHttpClient, HttpResponse, HttpxHttpClient
 from crawlee.storages import RequestList
 
+from apify._actor_inputs import _create_request_list
 from apify._configuration import Configuration
 from apify._consts import EVENT_LISTENERS_TIMEOUT
 from apify._crypto import decrypt_input_secrets, load_private_key
@@ -42,9 +43,6 @@
 
 MainReturnType = TypeVar('MainReturnType')
 
-URL_NO_COMMAS_REGEX = re.compile(r"https?:\/\/(www\.)?([a-zA-Z0-9]|[a-zA-Z0-9][-a-zA-Z0-9@:%._+~#=]{0,254}[a-zA-Z0-9])\.[a-z]{2,63}(:\d{1,5})?(\/[-a-zA-Z0-9@:%_+.~#?&/=()]*)?")
-# JS version. TODO rewrite to Python regexp
-# /https?:\/\/(www\.)?([\p{L}0-9]|[\p{L}0-9][-\p{L}0-9@:%._+~#=]{0,254}[\p{L}0-9])\.[a-z]{2,63}(:\d{1,5})?(\/[-\p{L}0-9@:%_+.~#?&/=()]*)?/giu;
 
 
 class _ActorType:
@@ -983,44 +981,20 @@ async def create_proxy_configuration(
         return proxy_configuration
 
     @staticmethod
-    async def create_request_list(*, actor_start_urls_input: dict, http_client: BaseHttpClient = HttpxHttpClient()) -> RequestList:
-        simple_url_requests_inputs = [request_input for request_input in actor_start_urls_input if "url" in request_input]
-        remote_url_requests_inputs = [request_input for request_input in actor_start_urls_input if "requestsFromUrl" in request_input]
-
-        simple_url_requests = Actor._create_requests_from_input(simple_url_requests_inputs)
-        remote_url_requests = await Actor._create_requests_from_url(remote_url_requests_inputs, http_client=http_client)
-
-        return RequestList(requests=simple_url_requests + remote_url_requests)
-
-    @staticmethod
-    def _create_requests_from_input(simple_url_requests_inputs: list[dict[str,str]]) -> list[Request]:
-        return [
-                Request.from_url(
-                    method=request_input.get('method'),
-                    url=request_input.get('url'),
-                    payload=request_input.get('payload', '').encode('utf-8'),
-                    headers=request_input.get('headers', {}),
-                    user_data=request_input.get('userData', {}),
-                )
-                for request_input in simple_url_requests_inputs]
-
-    @staticmethod
-    async def _create_requests_from_url(remote_url_requests_inputs: list[dict[str,str]], http_client: BaseHttpClient ) -> list[Request]:
-        remote_url_requests = []
-        for input in remote_url_requests_inputs:
-            remote_url_requests.append(asyncio.create_task(http_client.send_request(
-                url=input["requestsFromUrl"],
-                headers=input.get("headers", {}),
-                payload=input.get("payload", "").encode('utf-8'),
-            )))
-        await asyncio.gather(*remote_url_requests)
-        # TODO as callbacks
-        return list(chain.from_iterable((Actor.extract_requests_from_response(finished_request.result()) for finished_request in remote_url_requests)))
-
-    @staticmethod
-    def extract_requests_from_response(response: HttpResponse) -> list[Request]:
-        matches = list(re.finditer(URL_NO_COMMAS_REGEX, response.read().decode('utf-8')))
-        return [Request.from_url(match.group(0)) for match in matches]
+    async def create_request_list(
+        *, actor_start_urls_input: list[dict[str,str]], http_client: BaseHttpClient | None= None
+    ) -> RequestList:
+        """Creates request list from Actor input requestListSources. This accepts list of urls and requestsFromUrl.
+
+        Example:
+            actor_start_urls_input = [
+                # Gather urls from response body.
+                {'requestsFromUrl': 'https://crawlee.dev/file.txt', 'method': 'GET'},
+                # Directly include this url.
+                {'url': 'https://crawlee.dev', 'method': 'GET'}
+            ]
+            """
+        return await _create_request_list(actor_start_urls_input=actor_start_urls_input, http_client=http_client)
 
 Actor = cast(_ActorType, Proxy(_ActorType))
 """The entry point of the SDK, through which all the Actor operations should be done."""

src/apify/_actor_inputs.py

@@ -0,0 +1,73 @@
+import asyncio
+from itertools import chain
+import re
+
+from crawlee import Request
+from crawlee.http_clients import BaseHttpClient, HttpxHttpClient, HttpResponse
+from crawlee.storages import RequestList
+
+URL_NO_COMMAS_REGEX = re.compile(
+    r'https?:\/\/(www\.)?([a-zA-Z0-9]|[a-zA-Z0-9][-a-zA-Z0-9@:%._+~#=]{0,254}[a-zA-Z0-9])\.[a-z]{2,63}(:\d{1,5})?(\/[-a-zA-Z0-9@:%_+.~#?&/=()]*)?'
+)
+
+@staticmethod
+async def _create_request_list(
+    *, actor_start_urls_input: dict, http_client: BaseHttpClient | None = None
+) -> RequestList:
+    if not http_client:
+        http_client = HttpxHttpClient()
+    simple_url_requests_inputs = [
+        request_input for request_input in actor_start_urls_input if 'url' in request_input
+    ]
+    remote_url_requests_inputs = [
+        request_input for request_input in actor_start_urls_input if 'requestsFromUrl' in request_input
+    ]
+
+    simple_url_requests = _create_requests_from_input(simple_url_requests_inputs)
+    remote_url_requests = await _create_requests_from_url(remote_url_requests_inputs, http_client=http_client)
+
+    return RequestList(requests=simple_url_requests + remote_url_requests)
+
+
+@staticmethod
+def _create_requests_from_input(simple_url_requests_inputs: list[dict[str, str]]) -> list[Request]:
+    return [
+        Request.from_url(
+            method=request_input.get('method'),
+            url=request_input.get('url'),
+            payload=request_input.get('payload', '').encode('utf-8'),
+            headers=request_input.get('headers', {}),
+            user_data=request_input.get('userData', {}),
+        )
+        for request_input in simple_url_requests_inputs
+    ]
+
+
+@staticmethod
+async def _create_requests_from_url(
+    remote_url_requests_inputs: list[dict[str, str]], http_client: BaseHttpClient
+) -> list[Request]:
+    remote_url_requests = []
+    for request_input in remote_url_requests_inputs:
+        remote_url_requests.append(
+            asyncio.create_task(
+                http_client.send_request(
+                    method=request_input['method'],
+                    url=request_input['requestsFromUrl'],
+                    headers=request_input.get('headers', {}),
+                    payload=request_input.get('payload', '').encode('utf-8'),
+                )
+            )
+        )
+    await asyncio.gather(*remote_url_requests)
+    # TODO as callbacks
+    a = list(
+        extract_requests_from_response(finished_request.result()) for finished_request in remote_url_requests
+    )
+    return list(chain.from_iterable(a))
+
+
+@staticmethod
+def extract_requests_from_response(response: HttpResponse) -> list[Request]:
+    matches = list(re.finditer(URL_NO_COMMAS_REGEX, response.read().decode('utf-8')))
+    return [Request.from_url(match.group(0)) for match in matches]

tests/unit/actor/test_actor_create_proxy_configuration.py

@@ -3,6 +3,7 @@
 import typing
 from typing import TYPE_CHECKING
 from unittest import mock
+from unittest.mock import call
 
 import httpx
 import pytest
@@ -11,7 +12,7 @@
 from apify_shared.consts import ApifyEnvVars
 from crawlee._request import UserData
 from crawlee._types import HttpHeaders, HttpMethod
-from crawlee.http_clients import HttpxHttpClient, HttpResponse
+from crawlee.http_clients import HttpResponse, HttpxHttpClient
 
 from apify import Actor
 
@@ -160,14 +161,14 @@ async def test_proxy_configuration_with_actor_proxy_input(
 async def test_actor_create_request_list_request_types(
     request_method: HttpMethod, optional_input: dict[str, str]
 ) -> None:
-    """Tests proper request list generation from both minimal and full inputs for all method types."""
+    """Test proper request list generation from both minimal and full inputs for all method types for simple input."""
     minimal_request_dict_input = {'url': 'https://www.abc.com', 'method': request_method}
     request_dict_input = {**minimal_request_dict_input, **optional_input}
     example_start_urls_input = [
         request_dict_input,
     ]
 
-    generated_request_list =await Actor.create_request_list(actor_start_urls_input=example_start_urls_input)
+    generated_request_list = await Actor.create_request_list(actor_start_urls_input=example_start_urls_input)
 
     assert not await generated_request_list.is_empty()
     generated_request = await generated_request_list.fetch_next_request()
@@ -185,42 +186,89 @@ async def test_actor_create_request_list_request_types(
     assert generated_request.headers == expected_headers
 
 
-async def test_actor_create_request_list_from_url():
-    expected_urls = {"http://www.something.com", "https://www.something_else.com", "http://www.bla.net"}
-    response_body = "blablabla{} more blablabla{} ,\n even more blablbablba.{}".format(*expected_urls)
-    mocked_http_client = HttpxHttpClient()
+def _create_dummy_response(read_output: typing.Iterable[str]) -> HttpResponse:
+    """Create dummy_response that will iterate through read_output when called like dummy_response.read()"""
+
     class DummyResponse(HttpResponse):
         @property
         def http_version(self) -> str:
-            """The HTTP version used in the response."""
-            return ""
+            return ''
 
         @property
         def status_code(self) -> int:
-            """The HTTP status code received from the server."""
             return 200
 
         @property
         def headers(self) -> HttpHeaders:
-            """The HTTP headers received in the response."""
             return HttpHeaders()
 
         def read(self) -> bytes:
-            return response_body.encode('utf-8')
+            return next(read_output).encode('utf-8')
 
+    return DummyResponse()
 
-    async def mocked_send_request(*args, **kwargs):
-        return DummyResponse()
-    with mock.patch.object(mocked_http_client, "send_request", mocked_send_request) as mocked_send_request2:
 
-        example_start_urls_input = [
-            {"requestsFromUrl": "https://crawlee.dev/file.txt", 'method': "GET"}
-            ]
+async def test_actor_create_request_list_from_url_correctly_send_requests() -> None:
+    """Test that injected HttpClient's method send_request is called with properly passed arguments."""
 
+    example_start_urls_input = [
+        {'requestsFromUrl': 'https://crawlee.dev/file.txt', 'method': 'GET'},
+        {'requestsFromUrl': 'https://www.crawlee.dev/file2', 'method': 'PUT'},
+        {
+            'requestsFromUrl': 'https://www.something.som',
+            'method': 'POST',
+            'headers': {'key': 'value'},
+            'payload': 'some_payload',
+            'userData': 'irrelevant',
+        },
+    ]
+    mocked_read_outputs = ('' for url in example_start_urls_input)
+    http_client = HttpxHttpClient()
+    with mock.patch.object(
+        http_client, 'send_request', return_value=_create_dummy_response(mocked_read_outputs)
+    ) as mocked_send_request:
+        await Actor.create_request_list(actor_start_urls_input=example_start_urls_input, http_client=http_client)
+
+    expected_calls = [
+        call(
+            method=example_input['method'],
+            url=example_input['requestsFromUrl'],
+            headers=example_input.get('headers', {}),
+            payload=example_input.get('payload', '').encode('utf-8'),
+        )
+        for example_input in example_start_urls_input
+    ]
+    mocked_send_request.assert_has_calls(expected_calls)
+
+
+async def test_actor_create_request_list_from_url() -> None:
+    """Test that create_request_list is correctly reading urls from remote url sources and also from simple input."""
+    expected_simple_url = 'https://www.someurl.com'
+    expected_remote_urls_1 = {'http://www.something.com', 'https://www.somethingelse.com', 'http://www.bla.net'}
+    expected_remote_urls_2 = {'http://www.ok.com', 'https://www.true-positive.com'}
+    expected_urls = expected_remote_urls_1 | expected_remote_urls_2 | {expected_simple_url}
+    response_bodies = iter(
+        (
+            'blablabla{} more blablabla{} , even more blablabla. {} '.format(*expected_remote_urls_1),
+            'some stuff{} more stuff{} www.falsepositive www.false_positive.com'.format(*expected_remote_urls_2),
+        )
+    )
 
-        generated_request_list =await Actor.create_request_list(actor_start_urls_input=example_start_urls_input, http_client=mocked_http_client)
+    example_start_urls_input = [
+        {'requestsFromUrl': 'https://crawlee.dev/file.txt', 'method': 'GET'},
+        {'url': expected_simple_url, 'method': 'GET'},
+        {'requestsFromUrl': 'https://www.crawlee.dev/file2', 'method': 'GET'},
+    ]
+
+    http_client = HttpxHttpClient()
+    with mock.patch.object(http_client, 'send_request', return_value=_create_dummy_response(response_bodies)):
+        generated_request_list = await Actor.create_request_list(
+            actor_start_urls_input=example_start_urls_input, http_client=http_client
+        )
         generated_requests = []
-        while request:= await generated_request_list.fetch_next_request():
+        while request := await generated_request_list.fetch_next_request():
+            print(request)
             generated_requests.append(request)
 
-        assert set(generated_request.url for generated_request in generated_requests) == expected_urls
+    # Check correctly created requests' urls in request list
+    assert {generated_request.url for generated_request in generated_requests} == expected_urls