Skip to content
Merged
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
12 changes: 12 additions & 0 deletions src/crawlee/storage_clients/_memory/_request_queue_client.py
Original file line number Diff line number Diff line change
Expand Up @@ -137,6 +137,7 @@ async def add_batch_of_requests(

was_already_present = existing_request is not None
was_already_handled = was_already_present and existing_request and existing_request.handled_at is not None
is_in_progress = request.unique_key in self._in_progress_requests

# If the request is already in the queue and handled, don't add it again.
if was_already_handled:
Expand All @@ -149,6 +150,17 @@ async def add_batch_of_requests(
)
continue

# If the request is already in progress, don't add it again.
if is_in_progress:
processed_requests.append(
ProcessedRequest(
unique_key=request.unique_key,
was_already_present=True,
was_already_handled=False,
)
)
continue

# If the request is already in the queue but not handled, update it.
if was_already_present and existing_request:
# Update the existing request with any new data and
Expand Down
Loading