Skip to content

Commit 3af326c

Browse files
authored
fix: Do not add a request that is already in progress to MemoryRequestQueueClient (#1384)
### Description - Do not add a request that is already in progress for `MemoryRequestQueueClient`. ### Issues - Closes: #1383
1 parent 0e82451 commit 3af326c

File tree

1 file changed

+12
-0
lines changed

1 file changed

+12
-0
lines changed

src/crawlee/storage_clients/_memory/_request_queue_client.py

Lines changed: 12 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -137,6 +137,7 @@ async def add_batch_of_requests(
137137

138138
was_already_present = existing_request is not None
139139
was_already_handled = was_already_present and existing_request and existing_request.handled_at is not None
140+
is_in_progress = request.unique_key in self._in_progress_requests
140141

141142
# If the request is already in the queue and handled, don't add it again.
142143
if was_already_handled:
@@ -149,6 +150,17 @@ async def add_batch_of_requests(
149150
)
150151
continue
151152

153+
# If the request is already in progress, don't add it again.
154+
if is_in_progress:
155+
processed_requests.append(
156+
ProcessedRequest(
157+
unique_key=request.unique_key,
158+
was_already_present=True,
159+
was_already_handled=False,
160+
)
161+
)
162+
continue
163+
152164
# If the request is already in the queue but not handled, update it.
153165
if was_already_present and existing_request:
154166
# Update the existing request with any new data and

0 commit comments

Comments
 (0)