Merge branch 'VED-901-Test-Extended-Attribute' into VED-902-1-extended-attribute

Author: JamesW1-NHS

lambdas/filenameprocessor/src/constants.py

@@ -11,14 +11,18 @@
 )
 
 SOURCE_BUCKET_NAME = os.getenv("SOURCE_BUCKET_NAME")
+
+# We have used an internal temporary bucket here and an acutal dps bucket will replace this
 DPS_DESTINATION_BUCKET_NAME = os.getenv("ACK_BUCKET_NAME")
+EXPECTED_BUCKET_OWNER_ACCOUNT = os.getenv("ACCOUNT_ID")
 AUDIT_TABLE_NAME = os.getenv("AUDIT_TABLE_NAME")
 AUDIT_TABLE_TTL_DAYS = os.getenv("AUDIT_TABLE_TTL_DAYS")
 VALID_VERSIONS = ["V5"]
 
 VACCINE_TYPE_TO_DISEASES_HASH_KEY = "vacc_to_diseases"
 ODS_CODE_TO_SUPPLIER_SYSTEM_HASH_KEY = "ods_code_to_supplier"
-EXTENDED_ATTRIBUTES_PREFIXES = "Vaccination_Extended_Attributes"
+EXTENDED_ATTRIBUTES_FILE_PREFIX = "Vaccination_Extended_Attributes"
+DPS_DESTINATION_PREFIX = "dps_destination/"
 
 ERROR_TYPE_TO_STATUS_CODE_MAP = {
     VaccineTypePermissionsError: 403,

lambdas/filenameprocessor/src/file_name_processor.py

@@ -13,7 +13,7 @@
 
 from audit_table import upsert_audit_table
 from common.aws_s3_utils import (
-    copy_file_outside_bucket,
+    copy_file_to_external_bucket,
     delete_file,
     is_file_in_bucket,
     move_file,
@@ -23,8 +23,9 @@
 from common.models.errors import UnhandledAuditTableError
 from constants import (
     DPS_DESTINATION_BUCKET_NAME,
+    DPS_DESTINATION_PREFIX,
     ERROR_TYPE_TO_STATUS_CODE_MAP,
-    EXTENDED_ATTRIBUTES_PREFIXES,
+    EXTENDED_ATTRIBUTES_FILE_PREFIX,
     SOURCE_BUCKET_NAME,
     FileNotProcessedReason,
     FileStatus,
@@ -83,7 +84,7 @@ def handle_record(record) -> dict:
     s3_response = get_s3_client().get_object(Bucket=bucket_name, Key=file_key)
     created_at_formatted_string, expiry_timestamp = get_creation_and_expiry_times(s3_response)
 
-    if file_key.startswith(EXTENDED_ATTRIBUTES_PREFIXES):
+    if file_key.startswith(EXTENDED_ATTRIBUTES_FILE_PREFIX):
         return handle_extended_attributes_file(
             file_key,
             bucket_name,
@@ -113,7 +114,7 @@ def handle_unexpected_bucket_name(bucket_name: str, file_key: str) -> dict:
     """Handles scenario where Lambda was not invoked by the data-sources bucket. Should not occur due to terraform
     config and overarching design"""
     try:
-        if file_key.startswith(EXTENDED_ATTRIBUTES_PREFIXES):
+        if file_key.startswith(EXTENDED_ATTRIBUTES_FILE_PREFIX):
             extended_attribute_identifier = validate_extended_attributes_file_key(file_key)
             logger.error(
                 "Unable to process file %s due to unexpected bucket name %s",
@@ -162,7 +163,9 @@ def handle_unexpected_bucket_name(bucket_name: str, file_key: str) -> dict:
         }
 
 
-def handle_batch_file(file_key, bucket_name, message_id, created_at_formatted_string, expiry_timestamp) -> dict:
+def handle_batch_file(
+    file_key: str, bucket_name: str, message_id: str, created_at_formatted_string: str, expiry_timestamp: str
+) -> dict:
     """
     Processes a single record for batch file.
     Returns a dictionary containing information to be included in the logs.
@@ -244,7 +247,7 @@ def handle_batch_file(file_key, bucket_name, message_id, created_at_formatted_st
 
 
 def handle_extended_attributes_file(
-    file_key, bucket_name, message_id, created_at_formatted_string, expiry_timestamp
+    file_key: str, bucket_name: str, message_id: str, created_at_formatted_string: str, expiry_timestamp: str
 ) -> dict:
     """
     Processes a single record for extended attributes file.
@@ -261,19 +264,18 @@ def handle_extended_attributes_file(
 
     try:
         extended_attribute_identifier = validate_extended_attributes_file_key(file_key)
-        queue_name = extended_attribute_identifier
 
         upsert_audit_table(
             message_id,
             file_key,
             created_at_formatted_string,
             expiry_timestamp,
-            queue_name,
+            extended_attribute_identifier,
             FileStatus.PROCESSING,
         )
 
         dest_file_key = f"dps_destination/{file_key}"
-        copy_file_outside_bucket(bucket_name, file_key, DPS_DESTINATION_BUCKET_NAME, dest_file_key)
+        copy_file_to_external_bucket(bucket_name, file_key, DPS_DESTINATION_BUCKET_NAME, dest_file_key)
         is_file_in_bucket(DPS_DESTINATION_BUCKET_NAME, dest_file_key)
         delete_file(bucket_name, dest_file_key)
 
@@ -282,7 +284,7 @@ def handle_extended_attributes_file(
             file_key,
             created_at_formatted_string,
             expiry_timestamp,
-            queue_name,
+            extended_attribute_identifier,
             FileStatus.PROCESSED,
         )
 
@@ -291,7 +293,7 @@ def handle_extended_attributes_file(
             "message": "Extended Attributes file successfully processed",
             "file_key": file_key,
             "message_id": message_id,
-            "queue_name": queue_name,
+            "queue_name": extended_attribute_identifier,
         }
 
     except (  # pylint: disable=broad-exception-caught
@@ -307,8 +309,8 @@ def handle_extended_attributes_file(
         file_status = get_file_status_for_error(error)
 
         # NB if we got InvalidFileKeyError we won't have a valid queue name
-        if not queue_name:
-            queue_name = "unknown"
+        if not extended_attribute_identifier:
+            extended_attribute_identifier = "unknown"
 
         # Move file to archive
         move_file(bucket_name, file_key, f"archive/{file_key}")
@@ -318,7 +320,7 @@ def handle_extended_attributes_file(
             file_key,
             created_at_formatted_string,
             expiry_timestamp,
-            queue_name,
+            extended_attribute_identifier,
             file_status,
             error_details=str(error),
         )

lambdas/filenameprocessor/tests/test_lambda_handler.py

@@ -265,7 +265,7 @@ def test_lambda_handler_extended_attributes_success(self):
         with (
             patch("file_name_processor.uuid4", return_value=test_cases[0].message_id),
             patch(
-                "file_name_processor.copy_file_outside_bucket",
+                "file_name_processor.copy_file_to_external_bucket",
                 side_effect=lambda src_bucket, key, dst_bucket, dst_key: (
                     s3_client.put_object(
                         Bucket=BucketNames.DESTINATION,
@@ -332,7 +332,7 @@ def test_lambda_handler_extended_attributes_failure(self):
         with (
             patch("file_name_processor.uuid4", return_value=test_cases[0].message_id),
             patch(
-                "file_name_processor.copy_file_outside_bucket",
+                "file_name_processor.copy_file_to_external_bucket",
                 side_effect=lambda src_bucket, key, dst_bucket, dst_key: (  # effectively do nothing
                     None,
                 ),

lambdas/filenameprocessor/tests/utils_for_tests/values_for_tests.py

@@ -95,7 +95,6 @@ class MockFileDetails:
     extended_attributes_file = FileDetails(
         vaccine_type="Vaccination_Extended_Attributes", file_number=1, organization_code="X8E5B"
     )
-    # extended_attributes_file = "Vaccination_Extended_Attributes_v1_5_RJ123_20000101T00000001.csv"
 
 
 MOCK_FILE_HEADERS = (

lambdas/shared/src/common/aws_s3_utils.py

@@ -19,7 +19,9 @@ def move_file(bucket_name: str, source_file_key: str, destination_file_key: str)
     logger.info("File moved from %s to %s", source_file_key, destination_file_key)
 
 
-def copy_file_outside_bucket(source_bucket: str, source_key: str, destination_bucket: str, destination_key: str) -> None:
+def copy_file_to_external_bucket(
+    source_bucket: str, source_key: str, destination_bucket: str, destination_key: str
+) -> None:
     s3_client = get_s3_client()
     s3_client.copy_object(
         CopySource={"Bucket": source_bucket, "Key": source_key},

lambdas/shared/tests/test_common/test_s3_utils.py

@@ -90,7 +90,7 @@ def test_move_file_outside_bucket_copies_then_deletes(self):
             self.s3.get_object(Bucket=self.destination_bucket, Key=destination_key)
 
         # Execute copy across buckets
-        aws_s3_utils.copy_file_outside_bucket(
+        aws_s3_utils.copy_file_to_external_bucket(
             source_bucket=self.source_bucket,
             source_key=source_key,
             destination_bucket=self.destination_bucket,