resolve comments

Author: Akol125

lambdas/filenameprocessor/src/constants.py

@@ -11,14 +11,18 @@
 )
 
 SOURCE_BUCKET_NAME = os.getenv("SOURCE_BUCKET_NAME")
+
+# We have used an internal temporary bucket here and an acutal dps bucket will replace this
 DPS_DESTINATION_BUCKET_NAME = os.getenv("ACK_BUCKET_NAME")
+EXPECTED_BUCKET_OWNER_ACCOUNT = os.getenv("ACCOUNT_ID")
 AUDIT_TABLE_NAME = os.getenv("AUDIT_TABLE_NAME")
 AUDIT_TABLE_TTL_DAYS = os.getenv("AUDIT_TABLE_TTL_DAYS")
 VALID_VERSIONS = ["V5"]
 
 VACCINE_TYPE_TO_DISEASES_HASH_KEY = "vacc_to_diseases"
 ODS_CODE_TO_SUPPLIER_SYSTEM_HASH_KEY = "ods_code_to_supplier"
-EXTENDED_ATTRIBUTES_PREFIXES = "Vaccination_Extended_Attributes"
+EXTENDED_ATTRIBUTES_FILE_PREFIXES = "Vaccination_Extended_Attributes"
+DPS_DESTINATION_PREFIX = "dps_destination/"
 
 ERROR_TYPE_TO_STATUS_CODE_MAP = {
     VaccineTypePermissionsError: 403,

lambdas/filenameprocessor/src/file_name_processor.py

@@ -10,14 +10,15 @@
 from uuid import uuid4
 
 from audit_table import upsert_audit_table
-from common.aws_s3_utils import move_file, move_file_outside_bucket
+from common.aws_s3_utils import move_file, move_file_to_external_bucket
 from common.clients import STREAM_NAME, get_s3_client, logger
 from common.log_decorator import logging_decorator
 from common.models.errors import UnhandledAuditTableError
 from constants import (
     DPS_DESTINATION_BUCKET_NAME,
+    DPS_DESTINATION_PREFIX,
     ERROR_TYPE_TO_STATUS_CODE_MAP,
-    EXTENDED_ATTRIBUTES_PREFIXES,
+    EXTENDED_ATTRIBUTES_FILE_PREFIXES,
     SOURCE_BUCKET_NAME,
     FileNotProcessedReason,
     FileStatus,
@@ -76,7 +77,7 @@ def handle_record(record) -> dict:
     s3_response = get_s3_client().get_object(Bucket=bucket_name, Key=file_key)
     created_at_formatted_string, expiry_timestamp = get_creation_and_expiry_times(s3_response)
 
-    if file_key.startswith(EXTENDED_ATTRIBUTES_PREFIXES):
+    if file_key.startswith(EXTENDED_ATTRIBUTES_FILE_PREFIXES):
         return handle_extended_attributes_file(
             file_key,
             bucket_name,
@@ -106,7 +107,7 @@ def handle_unexpected_bucket_name(bucket_name: str, file_key: str) -> dict:
     """Handles scenario where Lambda was not invoked by the data-sources bucket. Should not occur due to terraform
     config and overarching design"""
     try:
-        if file_key.startswith(EXTENDED_ATTRIBUTES_PREFIXES):
+        if file_key.startswith(EXTENDED_ATTRIBUTES_FILE_PREFIXES):
             extended_attribute_identifier = validate_extended_attributes_file_key(file_key)
             logger.error(
                 "Unable to process file %s due to unexpected bucket name %s",
@@ -155,7 +156,9 @@ def handle_unexpected_bucket_name(bucket_name: str, file_key: str) -> dict:
         }
 
 
-def handle_batch_file(file_key, bucket_name, message_id, created_at_formatted_string, expiry_timestamp) -> dict:
+def handle_batch_file(
+    file_key: str, bucket_name: str, message_id: str, created_at_formatted_string: str, expiry_timestamp: str
+) -> dict:
     """
     Processes a single record for batch file.
     Returns a dictionary containing information to be included in the logs.
@@ -236,31 +239,32 @@ def handle_batch_file(file_key, bucket_name, message_id, created_at_formatted_st
 
 
 def handle_extended_attributes_file(
-    file_key, bucket_name, message_id, created_at_formatted_string, expiry_timestamp
+    file_key: str, bucket_name: str, message_id: str, created_at_formatted_string: str, expiry_timestamp: str
 ) -> dict:
     """
     Processes a single record for extended attributes file.
     Returns a dictionary containing information to be included in the logs.
     """
     try:
         extended_attribute_identifier = validate_extended_attributes_file_key(file_key)
-        move_file_outside_bucket(bucket_name, file_key, DPS_DESTINATION_BUCKET_NAME, f"dps_destination/{file_key}")
-        queue_name = extended_attribute_identifier
+        move_file_to_external_bucket(
+            bucket_name, file_key, DPS_DESTINATION_BUCKET_NAME, f"{DPS_DESTINATION_PREFIX}{file_key}"
+        )
 
         upsert_audit_table(
             message_id,
             file_key,
             created_at_formatted_string,
             expiry_timestamp,
-            queue_name,
+            extended_attribute_identifier,
             FileStatus.PROCESSING,
         )
         return {
             "statusCode": 200,
             "message": "Extended Attributes file successfully processed",
             "file_key": file_key,
             "message_id": message_id,
-            "queue_name": queue_name,
+            "queue_name": extended_attribute_identifier,
         }
     except (  # pylint: disable=broad-exception-caught
         VaccineTypePermissionsError,
@@ -273,7 +277,6 @@ def handle_extended_attributes_file(
 
         file_status = get_file_status_for_error(error)
         extended_attribute_identifier = validate_extended_attributes_file_key(file_key)
-        queue_name = extended_attribute_identifier
 
         upsert_audit_table(
             message_id,

lambdas/filenameprocessor/tests/test_lambda_handler.py

@@ -269,7 +269,7 @@ def test_lambda_handler_extended_attributes_success(self):
                 return_value=test_cases[0].ods_code + "_COVID",
             ),
             patch(
-                "file_name_processor.move_file_outside_bucket",
+                "file_name_processor.move_file_to_external_bucket",
                 side_effect=lambda src_bucket, key, dst_bucket, dst_key: (
                     s3_client.put_object(
                         Bucket=BucketNames.DESTINATION,

lambdas/filenameprocessor/tests/utils_for_tests/values_for_tests.py

@@ -95,7 +95,6 @@ class MockFileDetails:
     extended_attributes_file = FileDetails(
         vaccine_type="Vaccination_Extended_Attributes", file_number=1, organization_code="RJ123"
     )
-    # extended_attributes_file = "Vaccination_Extended_Attributes_v1_5_RJ123_20000101T00000001.csv"
 
 
 MOCK_FILE_HEADERS = (

lambdas/shared/src/common/aws_s3_utils.py

@@ -5,7 +5,6 @@
 from common.clients import get_s3_client, logger
 
 EXPECTED_BUCKET_OWNER_ACCOUNT = os.getenv("ACCOUNT_ID")
-DSP_DESTINATION_BUCKET_NAME = os.getenv("ACK_BUCKET_NAME")
 
 
 def move_file(bucket_name: str, source_file_key: str, destination_file_key: str) -> None:
@@ -20,7 +19,9 @@ def move_file(bucket_name: str, source_file_key: str, destination_file_key: str)
     logger.info("File moved from %s to %s", source_file_key, destination_file_key)
 
 
-def move_file_outside_bucket(source_bucket: str, source_key: str, destination_bucket: str, destination_key: str) -> None:
+def move_file_to_external_bucket(
+    source_bucket: str, source_key: str, destination_bucket: str, destination_key: str
+) -> None:
     s3_client = get_s3_client()
     s3_client.copy_object(
         CopySource={"Bucket": source_bucket, "Key": source_key},