File tree Expand file tree Collapse file tree 1 file changed +12
-0
lines changed
src/crawlee/storage_clients/_memory Expand file tree Collapse file tree 1 file changed +12
-0
lines changed Original file line number Diff line number Diff line change @@ -137,6 +137,7 @@ async def add_batch_of_requests(
137137
138138 was_already_present = existing_request is not None
139139 was_already_handled = was_already_present and existing_request and existing_request .handled_at is not None
140+ is_in_progress = request .unique_key in self ._in_progress_requests
140141
141142 # If the request is already in the queue and handled, don't add it again.
142143 if was_already_handled :
@@ -149,6 +150,17 @@ async def add_batch_of_requests(
149150 )
150151 continue
151152
153+ # If the request is already in progress, don't add it again.
154+ if is_in_progress :
155+ processed_requests .append (
156+ ProcessedRequest (
157+ unique_key = request .unique_key ,
158+ was_already_present = True ,
159+ was_already_handled = False ,
160+ )
161+ )
162+ continue
163+
152164 # If the request is already in the queue but not handled, update it.
153165 if was_already_present and existing_request :
154166 # Update the existing request with any new data and
You can’t perform that action at this time.
0 commit comments