[PRMP-990] Add S3 version id support

Author: adamwhitingnhs

.github/workflows/full-deploy-to-pre-prod.yml

@@ -27,7 +27,7 @@ jobs:
     steps:
       - name: Display client passed variables
         run: |
-          echo Build Branch: main
+          echo Build Branch: ${{ github.event.inputs.branch_or_tag }}
           echo Sandbox: pre-prod
           echo Environment: pre-prod
           echo Cypress Base Url: ${{ vars.CYPRESS_BASE_URL }}
@@ -37,27 +37,27 @@ jobs:
     uses: ./.github/workflows/base-lambdas-reusable-test.yml
     with:
       python_version: "3.11"
-      build_branch: main
+      build_branch: ${{ github.event.inputs.branch_or_tag }}
 
   react_testing_job:
     name: Run UI Unit Tests
     uses: ./.github/workflows/base-vitest-test.yml
     with:
-      build_branch: main
+      build_branch: ${{ github.event.inputs.branch_or_tag }}
 
   cypress_build_job:
     name: Build UI version for E2E Tests
     uses: ./.github/workflows/base-cypress-build.yml
     with:
-      build_branch: main
+      build_branch: ${{ github.event.inputs.branch_or_tag }}
 
   cypress_test_job:
     name: Run Cypress E2E Tests
     needs: [cypress_build_job]
     uses: ./.github/workflows/base-cypress-test-all-env.yml
     with:
       cypress_base_url: ${{ vars.CYPRESS_BASE_URL }}
-      build_branch: main
+      build_branch: ${{ github.event.inputs.branch_or_tag }}
 
   tag_and_release:
     needs: ["lambda_test_job", "react_testing_job", "cypress_test_job"]

lambdas/models/document_reference.py

@@ -137,7 +137,7 @@ class DocumentReference(BaseModel):
     raw_request: str | None = None
     s3_bucket_name: str = Field(exclude=True, default=None)
     s3_file_key: str = Field(default=None)
-    s3_version_id: Optional[str] = Field(default=None, exclude=True)
+    s3_version_id: Optional[str] = Field(default=None, alias="S3VersionID")
     s3_upload_key: str = Field(default=None, exclude=True)
     status: Literal["current", "superseded", "entered-in-error"] = Field(
         default="current"

lambdas/repositories/bulk_upload/bulk_upload_s3_repository.py

@@ -68,8 +68,8 @@ def check_virus_result(
             f"Verified that all documents for patient {staging_metadata.nhs_number} are clean."
         )
 
-    def copy_to_lg_bucket(self, source_file_key: str, dest_file_key: str):
-        self.s3_repository.copy_across_bucket(
+    def copy_to_lg_bucket(self, source_file_key: str, dest_file_key: str): 
+        result = self.s3_repository.copy_across_bucket(
             source_bucket=self.staging_bucket_name,
             source_file_key=source_file_key,
             dest_bucket=self.lg_bucket_name,
@@ -78,6 +78,8 @@ def copy_to_lg_bucket(self, source_file_key: str, dest_file_key: str):
         self.source_bucket_files_in_transaction.append(source_file_key)
         self.dest_bucket_files_in_transaction.append(dest_file_key)
 
+        return result
+
     def remove_ingested_file_from_source_bucket(self):
         for source_file_key in self.source_bucket_files_in_transaction:
             self.s3_repository.delete_object(

lambdas/services/bulk_upload_service.py

@@ -363,11 +363,13 @@ def create_lg_records_and_copy_files(
             source_file_key = self.file_path_cache[file_metadata.file_path]
             dest_file_key = document_reference.s3_file_key
 
-            self.bulk_upload_s3_repository.copy_to_lg_bucket(
+            copy_result = self.bulk_upload_s3_repository.copy_to_lg_bucket(
                 source_file_key=source_file_key, dest_file_key=dest_file_key
             )
             s3_bucket_name = self.bulk_upload_s3_repository.lg_bucket_name
 
+            document_reference.s3_version_id = copy_result.get("VersionId")
+
             document_reference.file_size = (
                 self.bulk_upload_s3_repository.s3_repository.get_file_size(
                     s3_bucket_name=s3_bucket_name, object_key=dest_file_key

lambdas/services/pdf_stitching_service.py

@@ -45,7 +45,7 @@ def __init__(self):
         self.document_service = DocumentService()
         self.sqs_service = SQSService()
         self.multipart_references: list[DocumentReference] = []
-        self.stitched_reference: DocumentReference = None
+        self.stitched_reference: DocumentReference | None = None
 
     def retrieve_multipart_references(
         self, nhs_number: str, doc_type: SupportedDocumentTypes
@@ -101,6 +101,7 @@ def process_message(self, stitching_message: PdfStitchingSqsMessage):
                 stitch_file_size=sys.getsizeof(stitching_data_stream),
             )
             self.upload_stitched_file(stitching_data_stream=stitching_data_stream)
+            self.update_stitched_reference_with_version_id()
             self.migrate_multipart_references()
             self.write_stitching_reference()
             self.publish_nrl_message(
@@ -129,6 +130,11 @@ def create_stitched_reference(
             deep=True,
         )
 
+    def update_stitched_reference_with_version_id(self):
+        self.stitched_reference.s3_version_id = self.s3_service.get_head_object(
+            self.target_bucket, self.stitched_reference.s3_file_key
+        ).get("VersionId")
+
     def process_stitching(self, s3_object_keys: list[str]) -> BytesIO:
         pdf_writer = PdfWriter()
 
@@ -370,6 +376,6 @@ def process_manual_trigger(self, ods_code: str, queue_url):
             except (InvalidMessageException, Exception) as e:
                 logger.error(f"Error sending batch to SQS: {str(e)}")
             # 1 batch is 10 messages
-            # we can send up to 300 messages a second
-            # a 0.1s delay means 10 batches, so 100 messages
+            # we can send up to 300 messages a second.
+            # a 0.1 s delay means 10 batches, so 100 messages
             time.sleep(0.1)

lambdas/services/upload_document_reference_service.py

@@ -245,6 +245,7 @@ def _finalize_and_supersede_with_transaction(self, new_document: DocumentReferen
                     "file_size",
                     "uploaded",
                     "uploading",
+                    "s3_version_id",
                 },
             )
 

lambdas/tests/e2e/helpers/data_helper.py

@@ -77,6 +77,7 @@ def create_metadata(self, document_details):
             "Uploaded": True,
             "Uploading": False,
             "Version": "1",
+            "S3VersionID": "some-version-id",
             "VirusScannerResult": "Clean",
         }
         self.dynamo_service.create_item(self.dynamo_table, dynamo_item)

lambdas/tests/unit/helpers/data/s3_responses.py

@@ -82,3 +82,27 @@
         ],
     },
 ]
+
+MOCK_COPY_OBJECT_RESPONSE = {
+    "CopyObjectResult": {
+        "ETag": '"d41d8cd98f00b204e9800998ecf8427e"',
+        "LastModified": "2024-06-01T12:34:56.000Z"
+    },
+    "Expiration": "expiry-date=\"2024-12-31T23:59:59Z\", rule-id=\"rule1\"",
+    "CopySourceVersionId": "source-version-id",
+    "VersionId": "new-version-id",
+    "ResponseMetadata": {
+        "RequestId": "EXAMPLE123456789",
+        "HostId": "EXAMPLEHostId",
+        "HTTPStatusCode": 200,
+        "HTTPHeaders": {
+            "x-amz-id-2": "EXAMPLEid2",
+            "x-amz-request-id": "EXAMPLErequestid",
+            "date": "Mon, 01 Jun 2024 12:34:56 GMT",
+            "etag": '"d41d8cd98f00b204e9800998ecf8427e"',
+            "content-length": "123",
+            "server": "AmazonS3"
+        },
+        "RetryAttempts": 0
+    }
+}

lambdas/tests/unit/services/test_bulk_upload_service.py

@@ -38,6 +38,7 @@
     PDS_PATIENT_DECEASED_INFORMAL,
     PDS_PATIENT_RESTRICTED,
 )
+from tests.unit.helpers.data.s3_responses import MOCK_COPY_OBJECT_RESPONSE
 from tests.unit.utils.test_unicode_utils import (
     NAME_WITH_ACCENT_NFC_FORM,
     NAME_WITH_ACCENT_NFD_FORM,
@@ -417,7 +418,7 @@ def test_handle_sqs_message_calls_report_upload_failure_when_lg_file_name_invali
         TEST_STAGING_METADATA_WITH_INVALID_FILENAME,
         UploadStatus.FAILED,
         str(mocked_error),
-        "Y12345"
+        "Y12345",
     )
     repo_under_test.sqs_repository.send_message_to_pdf_stitching_queue.assert_not_called()
 
@@ -712,8 +713,8 @@ def test_handle_sqs_message_rollback_transaction_when_validation_pass_but_file_t
 
     # simulate a client error occur when copying the 3rd file
     repo_under_test.bulk_upload_s3_repository.copy_to_lg_bucket.side_effect = [
-        None,
-        None,
+        MOCK_COPY_OBJECT_RESPONSE,
+        MOCK_COPY_OBJECT_RESPONSE,
         mock_client_error,
     ]
 
@@ -847,6 +848,9 @@ def test_create_lg_records_and_copy_files(set_env, mocker, mock_uuid, repo_under
     repo_under_test.convert_to_document_reference = mocker.MagicMock(
         return_value=test_document_reference
     )
+    repo_under_test.bulk_upload_s3_repository.copy_to_lg_bucket = mocker.MagicMock(
+        return_value=MOCK_COPY_OBJECT_RESPONSE
+    )
     TEST_STAGING_METADATA.retries = 0
     repo_under_test.resolve_source_file_path(TEST_STAGING_METADATA)
 
@@ -864,6 +868,10 @@ def test_create_lg_records_and_copy_files(set_env, mocker, mock_uuid, repo_under
             dest_file_key=expected_dest_file_key,
         )
         assert test_document_reference.uploaded.__eq__(True)
+        assert (
+            test_document_reference.s3_version_id
+            == MOCK_COPY_OBJECT_RESPONSE["VersionId"]
+        )
     assert repo_under_test.bulk_upload_s3_repository.copy_to_lg_bucket.call_count == 3
     repo_under_test.dynamo_repository.create_record_in_lg_dynamo_table.assert_any_call(
         test_document_reference

lambdas/tests/unit/services/test_pdf_stitching_service.py

@@ -71,6 +71,13 @@ def mock_upload_stitched_file(mocker, mock_service):
     return mocker.patch.object(mock_service, "upload_stitched_file")
 
 
+@pytest.fixture
+def mock_update_stitched_reference_with_version_id(mocker, mock_service):
+    return mocker.patch.object(
+        mock_service, "update_stitched_reference_with_version_id"
+    )
+
+
 @pytest.fixture
 def mock_migrate_multipart_references(mocker, mock_service):
     return mocker.patch.object(mock_service, "migrate_multipart_references")
@@ -164,6 +171,7 @@ def test_process_message(
     mock_sort_multipart_object_keys,
     mock_process_stitching,
     mock_upload_stitched_file,
+    mock_update_stitched_reference_with_version_id,
     mock_migrate_multipart_references,
     mock_write_stitching_reference,
     mock_publish_nrl_message,
@@ -193,6 +201,7 @@ def set_stitched_reference(document_reference, stitch_file_size, *args, **kwargs
     mock_sort_multipart_object_keys.assert_called_once_with()
     mock_process_stitching.assert_called_once_with(s3_object_keys=test_sorted_keys)
     mock_upload_stitched_file.assert_called_once_with(stitching_data_stream=test_stream)
+    mock_update_stitched_reference_with_version_id.assert_called_once()
     mock_migrate_multipart_references.assert_called_once()
     mock_write_stitching_reference.assert_called_once()
     mock_publish_nrl_message.assert_called_once()
@@ -671,6 +680,24 @@ def test_rollback_reference_migration_handles_exception(mock_service):
         mock_service.rollback_reference_migration()
 
 
+def test_update_stitched_reference_with_version_id(mock_service):
+    test_version_id = "test-version-id-12345"
+    mock_service.stitched_reference = TEST_1_OF_1_DOCUMENT_REFERENCE
+
+    mock_service.s3_service.get_head_object.return_value = {
+        "VersionId": test_version_id,
+        "ContentType": "application/pdf",
+        "ContentLength": 1234,
+    }
+
+    mock_service.update_stitched_reference_with_version_id()
+
+    mock_service.s3_service.get_head_object.assert_called_once_with(
+        MOCK_LG_BUCKET, TEST_1_OF_1_DOCUMENT_REFERENCE.s3_file_key
+    )
+    assert mock_service.stitched_reference.s3_version_id == test_version_id
+
+
 def test_process_manual_trigger_calls_process_message_for_each_nhs_number(
     mocker, mock_service
 ):