move_file

Author: JamesW1-NHS

lambdas/ack_backend/src/update_ack_file.py

@@ -6,6 +6,7 @@
 
 from audit_table import change_audit_table_status_to_processed
 from common.clients import get_s3_client, logger
+from common.utils import move_file
 from constants import (
     ACK_HEADERS,
     BATCH_FILE_ARCHIVE_DIR,
@@ -123,15 +124,3 @@ def update_ack_file(
 
     get_s3_client().upload_fileobj(csv_file_like_object, ack_bucket_name, temp_ack_file_key)
     logger.info("Ack file updated to %s: %s", ack_bucket_name, archive_ack_file_key)
-
-
-def move_file(bucket_name: str, source_file_key: str, destination_file_key: str) -> None:
-    """Moves a file from one location to another within a single S3 bucket by copying and then deleting the file."""
-    s3_client = get_s3_client()
-    s3_client.copy_object(
-        Bucket=bucket_name,
-        CopySource={"Bucket": bucket_name, "Key": source_file_key},
-        Key=destination_file_key,
-    )
-    s3_client.delete_object(Bucket=bucket_name, Key=source_file_key)
-    logger.info("File moved from %s to %s", source_file_key, destination_file_key)

lambdas/ack_backend/tests/test_splunk_logging.py

@@ -62,6 +62,7 @@ def run(self, result=None):
             # The logging_decorator.logger is patched individually in each test to allow for assertions to be made.
             # Any uses of the logger in other files will confound the tests and should be patched here.
             patch("update_ack_file.logger"),
+            patch("common.utils.logger"),
             # Time is incremented by 1.0 for each call to time.time for ease of testing.
             # Range is set to a large number (300) due to many calls being made to time.time for some tests.
             patch(

lambdas/ack_backend/tests/test_update_ack_file_flow.py

@@ -69,26 +69,3 @@ def test_audit_table_updated_correctly_when_ack_process_complete(self):
 
         # Assert: Only check audit table update
         self.mock_change_audit_status.assert_called_once_with(file_key, message_id)
-
-    def test_move_file(self):
-        """VED-167 test that the file has been moved to the appropriate location"""
-        bucket_name = "move-bucket"
-        file_key = "src/move_file_test.csv"
-        dest_key = "dest/move_file_test.csv"
-        self.s3_client.create_bucket(
-            Bucket=bucket_name,
-            CreateBucketConfiguration={"LocationConstraint": "eu-west-2"},
-        )
-        self.s3_client.put_object(Bucket=bucket_name, Key=file_key, Body="dummy content")
-        update_ack_file.move_file(bucket_name, file_key, dest_key)
-        # Assert the destination object exists
-        response = self.s3_client.get_object(Bucket=bucket_name, Key=dest_key)
-        content = response["Body"].read().decode()
-        self.assertEqual(content, "dummy content")
-
-        # Assert the source object no longer exists
-        with self.assertRaises(self.s3_client.exceptions.NoSuchKey):
-            self.s3_client.get_object(Bucket=bucket_name, Key=file_key)
-
-        # Logger assertion (if logger is mocked)
-        self.mock_logger.info.assert_called_with("File moved from %s to %s", file_key, dest_key)

lambdas/filenameprocessor/src/file_name_processor.py

@@ -13,6 +13,7 @@
 from common.clients import STREAM_NAME, get_s3_client, logger
 from common.log_decorator import logging_decorator
 from common.models.errors import UnhandledAuditTableError
+from common.utils import move_file
 from constants import (
     ERROR_TYPE_TO_STATUS_CODE_MAP,
     SOURCE_BUCKET_NAME,
@@ -28,7 +29,7 @@
 )
 from send_sqs_message import make_and_send_sqs_message
 from supplier_permissions import validate_vaccine_type_permissions
-from utils_for_filenameprocessor import get_creation_and_expiry_times, move_file
+from utils_for_filenameprocessor import get_creation_and_expiry_times
 
 
 # NOTE: logging_decorator is applied to handle_record function, rather than lambda_handler, because

lambdas/filenameprocessor/src/utils_for_filenameprocessor.py

@@ -2,7 +2,6 @@
 
 from datetime import timedelta
 
-from common.clients import get_s3_client, logger
 from constants import AUDIT_TABLE_TTL_DAYS
 
 
@@ -12,15 +11,3 @@ def get_creation_and_expiry_times(s3_response: dict) -> (str, int):
     expiry_datetime = creation_datetime + timedelta(days=int(AUDIT_TABLE_TTL_DAYS))
     expiry_timestamp = int(expiry_datetime.timestamp())
     return creation_datetime.strftime("%Y%m%dT%H%M%S00"), expiry_timestamp
-
-
-def move_file(bucket_name: str, source_file_key: str, destination_file_key: str) -> None:
-    """Moves a file from one location to another within a single S3 bucket by copying and then deleting the file."""
-    s3_client = get_s3_client()
-    s3_client.copy_object(
-        Bucket=bucket_name,
-        CopySource={"Bucket": bucket_name, "Key": source_file_key},
-        Key=destination_file_key,
-    )
-    s3_client.delete_object(Bucket=bucket_name, Key=source_file_key)
-    logger.info("File moved from %s to %s", source_file_key, destination_file_key)

lambdas/filenameprocessor/tests/test_utils_for_filenameprocessor.py

@@ -9,7 +9,6 @@
 
 from utils_for_tests.mock_environment_variables import (
     MOCK_ENVIRONMENT_DICT,
-    BucketNames,
 )
 from utils_for_tests.utils_for_filenameprocessor_tests import (
     GenericSetUp,
@@ -20,7 +19,7 @@
 with patch.dict("os.environ", MOCK_ENVIRONMENT_DICT):
     from common.clients import REGION_NAME
     from constants import AUDIT_TABLE_TTL_DAYS
-    from utils_for_filenameprocessor import get_creation_and_expiry_times, move_file
+    from utils_for_filenameprocessor import get_creation_and_expiry_times
 
 s3_client = boto3_client("s3", region_name=REGION_NAME)
 
@@ -50,20 +49,3 @@ def test_get_creation_and_expiry_times(self):
 
         self.assertEqual(created_at_formatted_string, expected_result_created_at)
         self.assertEqual(expires_at, expected_result_expires_at)
-
-    def test_move_file(self):
-        """Tests that move_file correctly moves a file from one location to another within a single S3 bucket"""
-        source_file_key = "test_file_key"
-        destination_file_key = "destination/test_file_key"
-        source_file_content = "test_content"
-        s3_client.put_object(Bucket=BucketNames.SOURCE, Key=source_file_key, Body=source_file_content)
-
-        move_file(BucketNames.SOURCE, source_file_key, destination_file_key)
-
-        keys_of_objects_in_bucket = [
-            obj["Key"] for obj in s3_client.list_objects_v2(Bucket=BucketNames.SOURCE).get("Contents")
-        ]
-        self.assertNotIn(source_file_key, keys_of_objects_in_bucket)
-        self.assertIn(destination_file_key, keys_of_objects_in_bucket)
-        destination_file_content = s3_client.get_object(Bucket=BucketNames.SOURCE, Key=destination_file_key)
-        self.assertEqual(destination_file_content["Body"].read().decode("utf-8"), source_file_content)

lambdas/recordprocessor/src/batch_processor.py

@@ -9,14 +9,15 @@
 
 from audit_table import update_audit_table_status
 from common.clients import logger
+from common.utils import move_file
 from constants import (
     ARCHIVE_DIR_NAME,
     PROCESSING_DIR_NAME,
     SOURCE_BUCKET_NAME,
     FileNotProcessedReason,
     FileStatus,
 )
-from file_level_validation import file_is_empty, file_level_validation, move_file
+from file_level_validation import file_is_empty, file_level_validation
 from mappings import map_target_disease
 from process_row import process_row
 from send_to_kinesis import send_to_kinesis

lambdas/recordprocessor/src/file_level_validation.py

@@ -6,7 +6,8 @@
 from csv import DictReader
 
 from audit_table import update_audit_table_status
-from common.clients import get_s3_client, logger
+from common.clients import logger
+from common.utils import move_file
 from constants import (
     ARCHIVE_DIR_NAME,
     EXPECTED_CSV_HEADERS,
@@ -61,18 +62,6 @@ def get_permitted_operations(supplier: str, vaccine_type: str, allowed_permissio
     return permitted_operations_for_vaccine_type
 
 
-def move_file(bucket_name: str, source_file_key: str, destination_file_key: str) -> None:
-    """Moves a file from one location to another within a single S3 bucket by copying and then deleting the file."""
-    s3_client = get_s3_client()
-    s3_client.copy_object(
-        Bucket=bucket_name,
-        CopySource={"Bucket": bucket_name, "Key": source_file_key},
-        Key=destination_file_key,
-    )
-    s3_client.delete_object(Bucket=bucket_name, Key=source_file_key)
-    logger.info("File moved from %s to %s", source_file_key, destination_file_key)
-
-
 @file_level_validation_logging_decorator
 def file_level_validation(incoming_message_body: dict) -> dict:
     """

lambdas/recordprocessor/tests/test_utils_for_recordprocessor.py

@@ -23,7 +23,6 @@
 )
 
 with patch("os.environ", MOCK_ENVIRONMENT_DICT):
-    from file_level_validation import move_file
     from utils_for_recordprocessor import (
         create_diagnostics_dictionary,
         get_csv_content_dict_reader,
@@ -87,23 +86,6 @@ def test_create_diagnostics_dictionary(self):
             },
         )
 
-    def test_move_file(self):
-        """Tests that move_file correctly moves a file from one location to another within a single S3 bucket"""
-        source_file_key = "test_file_key"
-        destination_file_key = "archive/test_file_key"
-        source_file_content = "test_content"
-        s3_client.put_object(Bucket=BucketNames.SOURCE, Key=source_file_key, Body=source_file_content)
-
-        move_file(BucketNames.SOURCE, source_file_key, destination_file_key)
-
-        keys_of_objects_in_bucket = [
-            obj["Key"] for obj in s3_client.list_objects_v2(Bucket=BucketNames.SOURCE).get("Contents")
-        ]
-        self.assertNotIn(source_file_key, keys_of_objects_in_bucket)
-        self.assertIn(destination_file_key, keys_of_objects_in_bucket)
-        destination_file_content = s3_client.get_object(Bucket=BucketNames.SOURCE, Key=destination_file_key)
-        self.assertEqual(destination_file_content["Body"].read().decode("utf-8"), source_file_content)
-
 
 if __name__ == "__main__":
     unittest.main()

lambdas/shared/src/common/utils.py

@@ -0,0 +1,15 @@
+"""Non-imms Utility Functions"""
+
+from common.clients import get_s3_client, logger
+
+
+def move_file(bucket_name: str, source_file_key: str, destination_file_key: str) -> None:
+    """Moves a file from one location to another within a single S3 bucket by copying and then deleting the file."""
+    s3_client = get_s3_client()
+    s3_client.copy_object(
+        Bucket=bucket_name,
+        CopySource={"Bucket": bucket_name, "Key": source_file_key},
+        Key=destination_file_key,
+    )
+    s3_client.delete_object(Bucket=bucket_name, Key=source_file_key)
+    logger.info("File moved from %s to %s", source_file_key, destination_file_key)