Skip to content

[PRMP-386] Allow bulk upload to have parallel executions #892

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
robg-test merged 4 commits into from
Nov 26, 2025
Merged

[PRMP-386] Allow bulk upload to have parallel executions #892

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
4 commits
Select commit Hold shift + click to select a range
0f928ae
[PRMP-386] changes how messages are grouped in the queue to now use n…
PedroSoaresNHS Nov 18, 2025
217c474
Merge remote-tracking branch 'origin/main' into PRMP-386
PedroSoaresNHS Nov 20, 2025
6889dcc
Merge remote-tracking branch 'origin/main' into PRMP-386
PedroSoaresNHS Nov 21, 2025
be0ce99
Merge branch 'main' into PRMP-386
robg-test Nov 26, 2025
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
5 changes: 1 addition & 4 deletions lambdas/services/bulk_upload_metadata_processor_service.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -2,7 +2,6 @@
import os
import shutil
import tempfile
import uuid
from collections import defaultdict
from datetime import datetime

Expand Down Expand Up @@ -217,16 +216,14 @@ def send_metadata_to_fifo_sqs(
self, staging_sqs_metadata_list: list[StagingSqsMetadata]
) -> None:
"""Send validated metadata entries to SQS FIFO queue."""
sqs_group_id = f"bulk_upload_{uuid.uuid4()}"

for staging_sqs_metadata in staging_sqs_metadata_list:
nhs_number = staging_sqs_metadata.nhs_number
logger.info(f"Sending metadata for patientId: {nhs_number}")
self.sqs_service.send_message_with_nhs_number_attr_fifo(
queue_url=self.metadata_queue_url,
message_body=staging_sqs_metadata.model_dump_json(by_alias=True),
nhs_number=nhs_number,
group_id=sqs_group_id,
group_id=f"bulk_upload_{nhs_number}",
)

def copy_metadata_to_dated_folder(self):
Expand Down
7 changes: 2 additions & 5 deletions lambdas/tests/unit/services/test_bulk_upload_metadata_processor_service.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -126,15 +126,13 @@ def test_process_metadata_send_metadata_to_sqs_queue(
mock_download_metadata_from_s3,
):
fake_csv_path = "fake/path/metadata.csv"
fake_uuid = "123412342"

mock_download_metadata_from_s3.return_value = fake_csv_path

mocker.patch.object(
test_service.s3_service, "copy_across_bucket", return_value=None
)
mocker.patch.object(test_service.s3_service, "delete_object", return_value=None)
mocker.patch("uuid.uuid4", return_value=fake_uuid)

fake_metadata = [
{"nhs_number": "1234567890", "some_data": "value1"},
Expand Down Expand Up @@ -365,19 +363,18 @@ def test_duplicates_csv_to_sqs_metadata(mocker, bulk_upload_service):


def test_send_metadata_to_sqs(set_env, mocker, mock_sqs_service, test_service):
mocker.patch("uuid.uuid4", return_value="123412342")
expected_calls = [
call(
queue_url=MOCK_LG_METADATA_SQS_QUEUE,
message_body=EXPECTED_SQS_MSG_FOR_PATIENT_1234567890,
nhs_number="1234567890",
group_id="bulk_upload_123412342",
group_id="bulk_upload_1234567890",
),
call(
queue_url=MOCK_LG_METADATA_SQS_QUEUE,
message_body=EXPECTED_SQS_MSG_FOR_PATIENT_123456789,
nhs_number="123456789",
group_id="bulk_upload_123412342",
group_id="bulk_upload_123456789",
),
]

Expand Down
Loading