Final tidy up

Author: dlzhry2nhs

lambdas/recordprocessor/src/audit_table.py

@@ -3,8 +3,8 @@
 from typing import Optional
 
 from common.clients import dynamodb_client, logger
+from common.models.errors import UnhandledAuditTableError
 from constants import AUDIT_TABLE_NAME, AuditTableKeys
-from errors import UnhandledAuditTableError
 
 
 def update_audit_table_status(

lambdas/recordprocessor/src/constants.py

@@ -3,12 +3,11 @@
 import os
 from enum import StrEnum
 
+# Once Python projects are moved to /lambdas consider consolidating constants common to batch in
+# /shared/src/common/constants/batch_constants.py
 SOURCE_BUCKET_NAME = os.getenv("SOURCE_BUCKET_NAME")
 ACK_BUCKET_NAME = os.getenv("ACK_BUCKET_NAME")
 AUDIT_TABLE_NAME = os.getenv("AUDIT_TABLE_NAME")
-AUDIT_TABLE_FILENAME_GSI = "filename_index"
-AUDIT_TABLE_QUEUE_NAME_GSI = "queue_name_index"
-FILE_NAME_PROC_LAMBDA_NAME = os.getenv("FILE_NAME_PROC_LAMBDA_NAME")
 
 ARCHIVE_DIR_NAME = "archive"
 PROCESSING_DIR_NAME = "processing"

lambdas/recordprocessor/src/errors.py

@@ -7,7 +7,3 @@ class NoOperationPermissions(Exception):
 
 class InvalidHeaders(Exception):
     """A custom exception for when the file headers are invalid."""
-
-
-class UnhandledAuditTableError(Exception):
-    """A custom exception for when an unexpected error occurs whilst adding the file to the audit table."""

lambdas/recordprocessor/tests/test_audit_table.py

@@ -7,15 +7,13 @@
 from boto3 import client as boto3_client
 from moto import mock_dynamodb
 
-from errors import UnhandledAuditTableError
-from utils_for_recordprocessor_tests.generic_setup_and_teardown import (
-    GenericSetUp,
-    GenericTearDown,
-)
+from common.models.errors import UnhandledAuditTableError
 from utils_for_recordprocessor_tests.mock_environment_variables import (
     MOCK_ENVIRONMENT_DICT,
 )
 from utils_for_recordprocessor_tests.utils_for_recordprocessor_tests import (
+    GenericSetUp,
+    GenericTearDown,
     add_entry_to_table,
 )
 from utils_for_recordprocessor_tests.values_for_recordprocessor_tests import (
@@ -47,11 +45,11 @@ def setUp(self):
         """Set up test values to be used for the tests"""
         self.logger_patcher = patch("audit_table.logger")
         self.mock_logger = self.logger_patcher.start()
-        GenericSetUp(dynamodb_client=dynamodb_client)
+        GenericSetUp(dynamo_db_client=dynamodb_client)
 
     def tearDown(self):
         """Tear down the test values"""
-        GenericTearDown(dynamodb_client=dynamodb_client)
+        GenericTearDown(dynamo_db_client=dynamodb_client)
         self.mock_logger.stop()
 
     @staticmethod

lambdas/recordprocessor/tests/test_process_csv_to_fhir.py

@@ -8,18 +8,16 @@
 import boto3
 from moto import mock_dynamodb, mock_firehose, mock_s3
 
-from tests.utils_for_recordprocessor_tests.generic_setup_and_teardown import (
-    GenericSetUp,
-    GenericTearDown,
-)
-from tests.utils_for_recordprocessor_tests.mock_environment_variables import (
+from utils_for_recordprocessor_tests.mock_environment_variables import (
     MOCK_ENVIRONMENT_DICT,
     BucketNames,
 )
-from tests.utils_for_recordprocessor_tests.utils_for_recordprocessor_tests import (
+from utils_for_recordprocessor_tests.utils_for_recordprocessor_tests import (
+    GenericSetUp,
+    GenericTearDown,
     add_entry_to_table,
 )
-from tests.utils_for_recordprocessor_tests.values_for_recordprocessor_tests import (
+from utils_for_recordprocessor_tests.values_for_recordprocessor_tests import (
     REGION_NAME,
     MockFileDetails,
     ValidMockFileContent,
@@ -46,7 +44,7 @@ def setUp(self) -> None:
         GenericSetUp(
             s3_client=s3_client,
             firehose_client=firehose_client,
-            dynamodb_client=dynamodb_client,
+            dynamo_db_client=dynamodb_client,
         )
 
         redis_getter_patcher = patch("mappings.get_redis_client")
@@ -67,7 +65,7 @@ def tearDown(self) -> None:
         GenericTearDown(
             s3_client=s3_client,
             firehose_client=firehose_client,
-            dynamodb_client=dynamodb_client,
+            dynamo_db_client=dynamodb_client,
         )
 
     @staticmethod

lambdas/recordprocessor/tests/utils_for_recordprocessor_tests/generic_setup_and_teardown.py

@@ -40,6 +40,5 @@ class Sqs:
     "SPLUNK_FIREHOSE_NAME": Firehose.STREAM_NAME,
     "KINESIS_STREAM_NAME": Kinesis.STREAM_NAME,
     "KINESIS_STREAM_ARN": f"arn:aws:kinesis:{REGION_NAME}:123456789012:stream/{Kinesis.STREAM_NAME}",
-    "FIREHOSE_STREAM_NAME": Firehose.STREAM_NAME,
     "AUDIT_TABLE_NAME": "immunisation-batch-internal-dev-audit-table",
 }

lambdas/recordprocessor/tests/utils_for_recordprocessor_tests/mock_environment_variables.py

@@ -24,9 +24,7 @@
 
     from common.clients import REGION_NAME
     from constants import (
-        AUDIT_TABLE_FILENAME_GSI,
         AUDIT_TABLE_NAME,
-        AUDIT_TABLE_QUEUE_NAME_GSI,
         AuditTableKeys,
     )
 
@@ -90,47 +88,8 @@ def __init__(
             dynamo_db_client.create_table(
                 TableName=AUDIT_TABLE_NAME,
                 KeySchema=[{"AttributeName": AuditTableKeys.MESSAGE_ID, "KeyType": "HASH"}],
-                AttributeDefinitions=[
-                    {"AttributeName": AuditTableKeys.MESSAGE_ID, "AttributeType": "S"},
-                    {"AttributeName": AuditTableKeys.FILENAME, "AttributeType": "S"},
-                    {"AttributeName": AuditTableKeys.QUEUE_NAME, "AttributeType": "S"},
-                    {"AttributeName": AuditTableKeys.STATUS, "AttributeType": "S"},
-                ],
+                AttributeDefinitions=[{"AttributeName": AuditTableKeys.MESSAGE_ID, "AttributeType": "S"}],
                 ProvisionedThroughput={"ReadCapacityUnits": 5, "WriteCapacityUnits": 5},
-                GlobalSecondaryIndexes=[
-                    {
-                        "IndexName": AUDIT_TABLE_FILENAME_GSI,
-                        "KeySchema": [
-                            {
-                                "AttributeName": AuditTableKeys.FILENAME,
-                                "KeyType": "HASH",
-                            }
-                        ],
-                        "Projection": {"ProjectionType": "KEYS_ONLY"},
-                        "ProvisionedThroughput": {
-                            "ReadCapacityUnits": 5,
-                            "WriteCapacityUnits": 5,
-                        },
-                    },
-                    {
-                        "IndexName": AUDIT_TABLE_QUEUE_NAME_GSI,
-                        "KeySchema": [
-                            {
-                                "AttributeName": AuditTableKeys.QUEUE_NAME,
-                                "KeyType": "HASH",
-                            },
-                            {
-                                "AttributeName": AuditTableKeys.STATUS,
-                                "KeyType": "RANGE",
-                            },
-                        ],
-                        "Projection": {"ProjectionType": "ALL"},
-                        "ProvisionedThroughput": {
-                            "ReadCapacityUnits": 5,
-                            "WriteCapacityUnits": 5,
-                        },
-                    },
-                ],
             )