redefine audit table

JamesW1-NHS · JamesW1-NHS · commit 5b95bd304e0c · 2025-12-03T15:51:01.000Z
diff --git a/infrastructure/instance/dynamodb.tf b/infrastructure/instance/dynamodb.tf
@@ -24,6 +24,36 @@ resource "aws_dynamodb_table" "audit-table" {
     type = "S"
   }
 
+  attribute {
+    name = "error_details"
+    type = "S"
+  }
+
+  attribute {
+    name = "created_at"
+    type = "S"
+  }
+
+  attribute {
+    name = "completed_at"
+    type = "S"
+  }
+
+  attribute {
+    name = "record_count"
+    type = "N"
+  }
+
+  attribute {
+    name = "records_succeeded"
+    type = "N"
+  }
+
+  attribute {
+    name = "records_failed"
+    type = "N"
+  }
+
   ttl {
     attribute_name = "expires_at"
     enabled        = true
diff --git a/lambdas/ack_backend/src/constants.py b/lambdas/ack_backend/src/constants.py
@@ -32,12 +32,17 @@ class FileStatus:
 class AuditTableKeys:
     """Audit table keys"""
 
-    FILENAME = "filename"
     MESSAGE_ID = "message_id"
+    FILENAME = "filename"
     QUEUE_NAME = "queue_name"
-    RECORD_COUNT = "record_count"
     STATUS = "status"
-    TIMESTAMP = "timestamp"
+    ERROR_DETAILS = "error_details"
+    CREATED_AT = "created_at"
+    COMPLETED_AT = "completed_at"
+    RECORD_COUNT = "record_count"
+    RECORDS_SUCCEEDED = "records_succeeded"
+    RECORDS_FAILED = "records_failed"
+    EXPIRES_AT = "expires_at"
 
 
 ACK_HEADERS = [
diff --git a/lambdas/batch_processor_filter/src/constants.py b/lambdas/batch_processor_filter/src/constants.py
@@ -30,9 +30,14 @@ class FileNotProcessedReason(StrEnum):
 class AuditTableKeys(StrEnum):
     """Audit table keys"""
 
-    FILENAME = "filename"
     MESSAGE_ID = "message_id"
+    FILENAME = "filename"
     QUEUE_NAME = "queue_name"
     STATUS = "status"
-    TIMESTAMP = "timestamp"
     ERROR_DETAILS = "error_details"
+    CREATED_AT = "created_at"
+    COMPLETED_AT = "completed_at"
+    RECORD_COUNT = "record_count"
+    RECORDS_SUCCEEDED = "records_succeeded"
+    RECORDS_FAILED = "records_failed"
+    EXPIRES_AT = "expires_at"
diff --git a/lambdas/filenameprocessor/src/audit_table.py b/lambdas/filenameprocessor/src/audit_table.py
@@ -25,7 +25,7 @@ def upsert_audit_table(
         AuditTableKeys.FILENAME: {"S": file_key},
         AuditTableKeys.QUEUE_NAME: {"S": queue_name},
         AuditTableKeys.STATUS: {"S": file_status},
-        AuditTableKeys.TIMESTAMP: {"S": created_at_formatted_str},
+        AuditTableKeys.CREATED_AT: {"S": created_at_formatted_str},
         AuditTableKeys.EXPIRES_AT: {"N": str(expiry_timestamp)},
     }
 
diff --git a/lambdas/filenameprocessor/src/constants.py b/lambdas/filenameprocessor/src/constants.py
@@ -54,10 +54,14 @@ class FileNotProcessedReason(StrEnum):
 class AuditTableKeys(StrEnum):
     """Audit table keys"""
 
-    FILENAME = "filename"
     MESSAGE_ID = "message_id"
+    FILENAME = "filename"
     QUEUE_NAME = "queue_name"
     STATUS = "status"
-    TIMESTAMP = "timestamp"
-    EXPIRES_AT = "expires_at"
     ERROR_DETAILS = "error_details"
+    CREATED_AT = "created_at"
+    COMPLETED_AT = "completed_at"
+    RECORD_COUNT = "record_count"
+    RECORDS_SUCCEEDED = "records_succeeded"
+    RECORDS_FAILED = "records_failed"
+    EXPIRES_AT = "expires_at"
diff --git a/lambdas/filenameprocessor/tests/test_lambda_handler.py b/lambdas/filenameprocessor/tests/test_lambda_handler.py
@@ -303,7 +303,7 @@ def test_lambda_handler_extended_attributes_success(self):
             item[AuditTableKeys.QUEUE_NAME]["S"], test_cases[0].ods_code + "_" + EXTENDED_ATTRIBUTES_VACC_TYPE
         )
         self.assertEqual(item[AuditTableKeys.STATUS]["S"], "Processed")
-        self.assertEqual(item[AuditTableKeys.TIMESTAMP]["S"], test_cases[0].created_at_formatted_string)
+        self.assertEqual(item[AuditTableKeys.CREATED_AT]["S"], test_cases[0].created_at_formatted_string)
         self.assertEqual(item[AuditTableKeys.EXPIRES_AT]["N"], str(test_cases[0].expires_at))
         # File should be moved to destination/
         dest_key = f"dps_destination/{test_cases[0].file_key}"
@@ -356,7 +356,7 @@ def test_lambda_handler_extended_attributes_failure(self):
         self.assertEqual(
             item[AuditTableKeys.QUEUE_NAME]["S"], test_cases[0].ods_code + "_" + EXTENDED_ATTRIBUTES_VACC_TYPE
         )
-        self.assertEqual(item[AuditTableKeys.TIMESTAMP]["S"], test_cases[0].created_at_formatted_string)
+        self.assertEqual(item[AuditTableKeys.CREATED_AT]["S"], test_cases[0].created_at_formatted_string)
         self.assertEqual(item[AuditTableKeys.STATUS]["S"], "Failed")
         self.assertEqual(
             item[AuditTableKeys.ERROR_DETAILS]["S"],
@@ -418,7 +418,7 @@ def test_lambda_handler_extended_attributes_invalid_key(self):
         self.assertEqual(item[AuditTableKeys.MESSAGE_ID]["S"], test_cases[0].message_id)
         self.assertEqual(item[AuditTableKeys.FILENAME]["S"], invalid_file_key)
         self.assertEqual(item[AuditTableKeys.QUEUE_NAME]["S"], "unknown")
-        self.assertEqual(item[AuditTableKeys.TIMESTAMP]["S"], test_cases[0].created_at_formatted_string)
+        self.assertEqual(item[AuditTableKeys.CREATED_AT]["S"], test_cases[0].created_at_formatted_string)
         self.assertEqual(item[AuditTableKeys.STATUS]["S"], "Failed")
         self.assertEqual(
             item[AuditTableKeys.ERROR_DETAILS]["S"],
@@ -474,7 +474,7 @@ def test_lambda_invalid_file_key_no_other_files_in_queue(self, mock_get_redis_cl
                 "queue_name": {"S": "unknown_unknown"},
                 "status": {"S": "Failed"},
                 "error_details": {"S": "Initial file validation failed: invalid file key"},
-                "timestamp": {"S": file_details.created_at_formatted_string},
+                "created_at": {"S": file_details.created_at_formatted_string},
                 "expires_at": {"N": str(file_details.expires_at)},
             }
         ]
@@ -517,7 +517,7 @@ def test_lambda_invalid_permissions(self, mock_get_redis_client):
                 "queue_name": {"S": "RAVS_RSV"},
                 "status": {"S": "Not processed - Unauthorised"},
                 "error_details": {"S": "Initial file validation failed: RAVS does not have permissions for RSV"},
-                "timestamp": {"S": file_details.created_at_formatted_string},
+                "created_at": {"S": file_details.created_at_formatted_string},
                 "expires_at": {"N": str(file_details.expires_at)},
             }
         ]
@@ -561,7 +561,7 @@ def test_lambda_adds_event_to_audit_table_as_failed_when_unexpected_exception_is
                 "queue_name": {"S": "EMIS_FLU"},
                 "status": {"S": "Failed"},
                 "error_details": {"S": "Some unexpected exception"},
-                "timestamp": {"S": test_file_details.created_at_formatted_string},
+                "created_at": {"S": test_file_details.created_at_formatted_string},
                 "expires_at": {"N": str(test_file_details.expires_at)},
             }
         ]
diff --git a/lambdas/filenameprocessor/tests/utils_for_tests/values_for_tests.py b/lambdas/filenameprocessor/tests/utils_for_tests/values_for_tests.py
@@ -76,7 +76,7 @@ def __init__(
             AuditTableKeys.MESSAGE_ID: {"S": self.message_id},
             AuditTableKeys.FILENAME: {"S": self.file_key},
             AuditTableKeys.QUEUE_NAME: {"S": self.queue_name},
-            AuditTableKeys.TIMESTAMP: {"S": self.created_at_formatted_string},
+            AuditTableKeys.CREATED_AT: {"S": self.created_at_formatted_string},
             AuditTableKeys.EXPIRES_AT: {"N": str(self.expires_at)},
         }
 
diff --git a/lambdas/recordprocessor/src/constants.py b/lambdas/recordprocessor/src/constants.py
@@ -71,13 +71,17 @@ class FileNotProcessedReason(StrEnum):
 class AuditTableKeys:
     """Audit table keys"""
 
-    FILENAME = "filename"
     MESSAGE_ID = "message_id"
+    FILENAME = "filename"
     QUEUE_NAME = "queue_name"
-    RECORD_COUNT = "record_count"
     STATUS = "status"
-    TIMESTAMP = "timestamp"
     ERROR_DETAILS = "error_details"
+    CREATED_AT = "created_at"
+    COMPLETED_AT = "completed_at"
+    RECORD_COUNT = "record_count"
+    RECORDS_SUCCEEDED = "records_succeeded"
+    RECORDS_FAILED = "records_failed"
+    EXPIRES_AT = "expires_at"
 
 
 class Diagnostics:
diff --git a/lambdas/recordprocessor/tests/utils_for_recordprocessor_tests/values_for_recordprocessor_tests.py b/lambdas/recordprocessor/tests/utils_for_recordprocessor_tests/values_for_recordprocessor_tests.py
@@ -197,7 +197,7 @@ def __init__(self, vaccine_type: str, supplier: str, ods_code: str, file_number:
             AuditTableKeys.MESSAGE_ID: {"S": self.message_id},
             AuditTableKeys.FILENAME: {"S": self.file_key},
             AuditTableKeys.QUEUE_NAME: {"S": self.queue_name},
-            AuditTableKeys.TIMESTAMP: {"S": self.created_at_formatted_string},
+            AuditTableKeys.CREATED_AT: {"S": self.created_at_formatted_string},
         }
 
 

Original file line number	Diff line number	Diff line change
`@@ -25,7 +25,7 @@ def upsert_audit_table(`
`25`	`25`	`AuditTableKeys.FILENAME: {"S": file_key},`
`26`	`26`	`AuditTableKeys.QUEUE_NAME: {"S": queue_name},`
`27`	`27`	`AuditTableKeys.STATUS: {"S": file_status},`
`28`		`- AuditTableKeys.TIMESTAMP: {"S": created_at_formatted_str},`
	`28`	`+ AuditTableKeys.CREATED_AT: {"S": created_at_formatted_str},`
`29`	`29`	`AuditTableKeys.EXPIRES_AT: {"N": str(expiry_timestamp)},`
`30`	`30`	`}`
`31`	`31`
Original file line number	Diff line number	Diff line change
`@@ -76,7 +76,7 @@ def __init__(`
`76`	`76`	`AuditTableKeys.MESSAGE_ID: {"S": self.message_id},`
`77`	`77`	`AuditTableKeys.FILENAME: {"S": self.file_key},`
`78`	`78`	`AuditTableKeys.QUEUE_NAME: {"S": self.queue_name},`
`79`		`- AuditTableKeys.TIMESTAMP: {"S": self.created_at_formatted_string},`
	`79`	`+ AuditTableKeys.CREATED_AT: {"S": self.created_at_formatted_string},`
`80`	`80`	`AuditTableKeys.EXPIRES_AT: {"N": str(self.expires_at)},`
`81`	`81`	`}`
`82`	`82`
Original file line number	Diff line number	Diff line change
`@@ -197,7 +197,7 @@ def __init__(self, vaccine_type: str, supplier: str, ods_code: str, file_number:`
`197`	`197`	`AuditTableKeys.MESSAGE_ID: {"S": self.message_id},`
`198`	`198`	`AuditTableKeys.FILENAME: {"S": self.file_key},`
`199`	`199`	`AuditTableKeys.QUEUE_NAME: {"S": self.queue_name},`
`200`		`- AuditTableKeys.TIMESTAMP: {"S": self.created_at_formatted_string},`
	`200`	`+ AuditTableKeys.CREATED_AT: {"S": self.created_at_formatted_string},`
`201`	`201`	`}`
`202`	`202`
`203`	`203`