diff --git a/src/crawlee/storage_clients/_memory/_request_queue_client.py b/src/crawlee/storage_clients/_memory/_request_queue_client.py index cf8a91897e..e169441870 100644 --- a/src/crawlee/storage_clients/_memory/_request_queue_client.py +++ b/src/crawlee/storage_clients/_memory/_request_queue_client.py @@ -137,6 +137,7 @@ async def add_batch_of_requests( was_already_present = existing_request is not None was_already_handled = was_already_present and existing_request and existing_request.handled_at is not None + is_in_progress = request.unique_key in self._in_progress_requests # If the request is already in the queue and handled, don't add it again. if was_already_handled: @@ -149,6 +150,17 @@ async def add_batch_of_requests( ) continue + # If the request is already in progress, don't add it again. + if is_in_progress: + processed_requests.append( + ProcessedRequest( + unique_key=request.unique_key, + was_already_present=True, + was_already_handled=False, + ) + ) + continue + # If the request is already in the queue but not handled, update it. if was_already_present and existing_request: # Update the existing request with any new data and