Skip to content

Commit 594a8e5

Browse files
committed
Fix request list (HttpResponse.read is now async)
1 parent a3f8c6e commit 594a8e5

File tree

1 file changed

+9
-4
lines changed

1 file changed

+9
-4
lines changed

src/apify/request_loaders/_apify_request_list.py

Lines changed: 9 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -3,7 +3,6 @@
33
import asyncio
44
import re
55
from asyncio import Task
6-
from functools import partial
76
from typing import Annotated, Any
87

98
from pydantic import BaseModel, Field, TypeAdapter
@@ -121,13 +120,15 @@ async def _fetch_requests_from_url(
121120
"""
122121
created_requests: list[Request] = []
123122

124-
def create_requests_from_response(request_input: _RequestsFromUrlInput, task: Task) -> None:
123+
async def create_requests_from_response(request_input: _RequestsFromUrlInput, task: Task) -> None:
125124
"""Extract links from response body and use them to create `Request` objects.
126125
127126
Use the regular expression to find all matching links in the response body, then create `Request`
128127
objects from these links and the provided input attributes.
129128
"""
130-
matches = re.finditer(URL_NO_COMMAS_REGEX, task.result().read().decode('utf-8'))
129+
response = await (task.result()).read()
130+
matches = re.finditer(URL_NO_COMMAS_REGEX, response.decode('utf-8'))
131+
131132
created_requests.extend(
132133
[
133134
Request.from_url(
@@ -150,7 +151,11 @@ def create_requests_from_response(request_input: _RequestsFromUrlInput, task: Ta
150151
)
151152
)
152153

153-
get_response_task.add_done_callback(partial(create_requests_from_response, remote_url_requests_input))
154+
get_response_task.add_done_callback(
155+
lambda task, inp=remote_url_requests_input: asyncio.create_task( # type: ignore[misc]
156+
create_requests_from_response(inp, task)
157+
)
158+
)
154159
remote_url_requests.append(get_response_task)
155160

156161
await asyncio.gather(*remote_url_requests)

0 commit comments

Comments
 (0)