WIP

Author: Pijukatel

src/apify/_actor.py

@@ -2,10 +2,13 @@
 
 import asyncio
 import os
+import re
 import sys
 from datetime import timedelta
+from itertools import chain
 from typing import TYPE_CHECKING, Any, Callable, TypeVar, cast
 
+from crawlee.http_clients import HttpxHttpClient, HttpResponse, BaseHttpClient
 from lazy_object_proxy import Proxy
 from pydantic import AliasChoices
 from typing_extensions import Self
@@ -39,6 +42,10 @@
 
 MainReturnType = TypeVar('MainReturnType')
 
+URL_NO_COMMAS_REGEX = re.compile(r"https?:\/\/(www\.)?([a-zA-Z0-9]|[a-zA-Z0-9][-a-zA-Z0-9@:%._+~#=]{0,254}[a-zA-Z0-9])\.[a-z]{2,63}(:\d{1,5})?(\/[-a-zA-Z0-9@:%_+.~#?&/=()]*)?")
+# JS version. TODO rewrite to Python regexp
+# /https?:\/\/(www\.)?([\p{L}0-9]|[\p{L}0-9][-\p{L}0-9@:%._+~#=]{0,254}[\p{L}0-9])\.[a-z]{2,63}(:\d{1,5})?(\/[-\p{L}0-9@:%_+.~#?&/=()]*)?/giu;
+
 
 class _ActorType:
     """The class of `Actor`. Only make a new instance if you're absolutely sure you need to."""
@@ -976,20 +983,44 @@ async def create_proxy_configuration(
         return proxy_configuration
 
     @staticmethod
-    def create_request_list(*, actor_start_urls_input: dict) -> RequestList:
-        return RequestList(
-            requests=[
+    async def create_request_list(*, actor_start_urls_input: dict, http_client: BaseHttpClient = HttpxHttpClient()) -> RequestList:
+        simple_url_requests_inputs = [request_input for request_input in actor_start_urls_input if "url" in request_input]
+        remote_url_requests_inputs = [request_input for request_input in actor_start_urls_input if "requestsFromUrl" in request_input]
+
+        simple_url_requests = Actor._create_requests_from_input(simple_url_requests_inputs)
+        remote_url_requests = await Actor._create_requests_from_url(remote_url_requests_inputs, http_client=http_client)
+
+        return RequestList(requests=simple_url_requests + remote_url_requests)
+
+    @staticmethod
+    def _create_requests_from_input(simple_url_requests_inputs: list[dict[str,str]]) -> list[Request]:
+        return [
                 Request.from_url(
                     method=request_input.get('method'),
                     url=request_input.get('url'),
                     payload=request_input.get('payload', '').encode('utf-8'),
                     headers=request_input.get('headers', {}),
                     user_data=request_input.get('userData', {}),
                 )
-                for request_input in actor_start_urls_input
-            ]
-        )
+                for request_input in simple_url_requests_inputs]
 
+    @staticmethod
+    async def _create_requests_from_url(remote_url_requests_inputs: list[dict[str,str]], http_client: BaseHttpClient ) -> list[Request]:
+        remote_url_requests = []
+        for input in remote_url_requests_inputs:
+            remote_url_requests.append(asyncio.create_task(http_client.send_request(
+                url=input["requestsFromUrl"],
+                headers=input.get("headers", {}),
+                payload=input.get("payload", "").encode('utf-8'),
+            )))
+        await asyncio.gather(*remote_url_requests)
+        # TODO as callbacks
+        return list(chain.from_iterable((Actor.extract_requests_from_response(finished_request.result()) for finished_request in remote_url_requests)))
+
+    @staticmethod
+    def extract_requests_from_response(response: HttpResponse) -> list[Request]:
+        matches = list(re.finditer(URL_NO_COMMAS_REGEX, response.read().decode('utf-8')))
+        return [Request.from_url(match.group(0)) for match in matches]
 
 Actor = cast(_ActorType, Proxy(_ActorType))
 """The entry point of the SDK, through which all the Actor operations should be done."""

tests/unit/actor/test_actor_create_proxy_configuration.py

@@ -2,6 +2,7 @@
 
 import typing
 from typing import TYPE_CHECKING
+from unittest import mock
 
 import httpx
 import pytest
@@ -10,6 +11,7 @@
 from apify_shared.consts import ApifyEnvVars
 from crawlee._request import UserData
 from crawlee._types import HttpHeaders, HttpMethod
+from crawlee.http_clients import HttpxHttpClient, HttpResponse
 
 from apify import Actor
 
@@ -165,7 +167,7 @@ async def test_actor_create_request_list_request_types(
         request_dict_input,
     ]
 
-    generated_request_list = Actor.create_request_list(actor_start_urls_input=example_start_urls_input)
+    generated_request_list =await Actor.create_request_list(actor_start_urls_input=example_start_urls_input)
 
     assert not await generated_request_list.is_empty()
     generated_request = await generated_request_list.fetch_next_request()
@@ -181,3 +183,44 @@ async def test_actor_create_request_list_request_types(
     assert generated_request.user_data == expected_user_data
     expected_headers = HttpHeaders(root=optional_input.get('headers', {}))
     assert generated_request.headers == expected_headers
+
+
+async def test_actor_create_request_list_from_url():
+    expected_urls = {"http://www.something.com", "https://www.something_else.com", "http://www.bla.net"}
+    response_body = "blablabla{} more blablabla{} ,\n even more blablbablba.{}".format(*expected_urls)
+    mocked_http_client = HttpxHttpClient()
+    class DummyResponse(HttpResponse):
+        @property
+        def http_version(self) -> str:
+            """The HTTP version used in the response."""
+            return ""
+
+        @property
+        def status_code(self) -> int:
+            """The HTTP status code received from the server."""
+            return 200
+
+        @property
+        def headers(self) -> HttpHeaders:
+            """The HTTP headers received in the response."""
+            return HttpHeaders()
+
+        def read(self) -> bytes:
+            return response_body.encode('utf-8')
+
+
+    async def mocked_send_request(*args, **kwargs):
+        return DummyResponse()
+    with mock.patch.object(mocked_http_client, "send_request", mocked_send_request) as mocked_send_request2:
+
+        example_start_urls_input = [
+            {"requestsFromUrl": "https://crawlee.dev/file.txt", 'method': "GET"}
+            ]
+
+
+        generated_request_list =await Actor.create_request_list(actor_start_urls_input=example_start_urls_input, http_client=mocked_http_client)
+        generated_requests = []
+        while request:= await generated_request_list.fetch_next_request():
+            generated_requests.append(request)
+
+        assert set(generated_request.url for generated_request in generated_requests) == expected_urls