File tree Expand file tree Collapse file tree 1 file changed +15
-3
lines changed
src/apify/apify_storage_client Expand file tree Collapse file tree 1 file changed +15
-3
lines changed Original file line number Diff line number Diff line change 2
2
3
3
from typing import TYPE_CHECKING
4
4
5
+ from more_itertools import chunked
5
6
from typing_extensions import override
6
7
7
8
from crawlee import Request
@@ -157,8 +158,11 @@ async def batch_add_requests(
157
158
* ,
158
159
forefront : bool = False ,
159
160
) -> BatchRequestsOperationResponse :
160
- return BatchRequestsOperationResponse .model_validate (
161
- await self ._client .batch_add_requests (
161
+ processed = []
162
+ unprocessed = []
163
+
164
+ for chunk in chunked (requests , 25 ):
165
+ response = await self ._client .batch_add_requests (
162
166
requests = [
163
167
r .model_dump (
164
168
by_alias = True ,
@@ -170,10 +174,18 @@ async def batch_add_requests(
170
174
'data' ,
171
175
},
172
176
)
173
- for r in requests
177
+ for r in chunk
174
178
],
175
179
forefront = forefront ,
176
180
)
181
+ processed .extend (response ['processedRequests' ])
182
+ unprocessed .extend (response ['unprocessedRequests' ])
183
+
184
+ return BatchRequestsOperationResponse .model_validate (
185
+ {
186
+ 'processedRequests' : processed ,
187
+ 'unprocessedRequests' : unprocessed ,
188
+ }
177
189
)
178
190
179
191
@override
You can’t perform that action at this time.
0 commit comments