Skip to content

Commit a095194

Browse files
committed
Hotfix for batch_add_requests batch size limit
1 parent 741d222 commit a095194

File tree

1 file changed

+15
-3
lines changed

1 file changed

+15
-3
lines changed

src/apify/apify_storage_client/_request_queue_client.py

Lines changed: 15 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -2,6 +2,7 @@
22

33
from typing import TYPE_CHECKING
44

5+
from more_itertools import chunked
56
from typing_extensions import override
67

78
from crawlee import Request
@@ -157,8 +158,11 @@ async def batch_add_requests(
157158
*,
158159
forefront: bool = False,
159160
) -> BatchRequestsOperationResponse:
160-
return BatchRequestsOperationResponse.model_validate(
161-
await self._client.batch_add_requests(
161+
processed = []
162+
unprocessed = []
163+
164+
for chunk in chunked(requests, 25):
165+
response = await self._client.batch_add_requests(
162166
requests=[
163167
r.model_dump(
164168
by_alias=True,
@@ -170,10 +174,18 @@ async def batch_add_requests(
170174
'data',
171175
},
172176
)
173-
for r in requests
177+
for r in chunk
174178
],
175179
forefront=forefront,
176180
)
181+
processed.extend(response['processedRequests'])
182+
unprocessed.extend(response['unprocessedRequests'])
183+
184+
return BatchRequestsOperationResponse.model_validate(
185+
{
186+
'processedRequests': processed,
187+
'unprocessedRequests': unprocessed,
188+
}
177189
)
178190

179191
@override

0 commit comments

Comments
 (0)