VED-47-Delta-Expiry (#785)

* DELTA_TTF in terraform

* mock DELTA_TTL

* mock DELTA_TTL II

* test for expiresAt

* DELTA_TTL_DAYS

Author: JamesW1-NHS

delta_backend/src/delta.py

@@ -17,6 +17,7 @@
 failure_queue_url = os.environ["AWS_SQS_QUEUE_URL"]
 delta_table_name = os.environ["DELTA_TABLE_NAME"]
 delta_source = os.environ["SOURCE"]
+delta_ttl_days = os.environ["DELTA_TTL_DAYS"]
 region_name = "eu-west-2"
 logging.basicConfig()
 logger = logging.getLogger()
@@ -73,7 +74,7 @@ def get_imms_id(primary_key: str) -> str:
 
 def get_creation_and_expiry_times(creation_timestamp: float) -> (str, int):
     creation_datetime = datetime.fromtimestamp(creation_timestamp, UTC)
-    expiry_datetime = creation_datetime + timedelta(days=30)
+    expiry_datetime = creation_datetime + timedelta(days=int(delta_ttl_days))
     expiry_timestamp = int(expiry_datetime.timestamp())
     return creation_datetime.isoformat(), expiry_timestamp
 

delta_backend/tests/test_convert.py

@@ -1,6 +1,8 @@
 import json
+import os
 import unittest
 from copy import deepcopy
+from datetime import datetime
 from unittest.mock import patch, Mock
 from moto import mock_aws
 from boto3 import resource as boto3_resource
@@ -11,6 +13,7 @@
 MOCK_ENV_VARS = {
     "AWS_SQS_QUEUE_URL": "https://sqs.eu-west-2.amazonaws.com/123456789012/test-queue",
     "DELTA_TABLE_NAME": "immunisation-batch-internal-dev-audit-test-table",
+    "DELTA_TTL_DAYS": "14",
     "SOURCE": "test-source",
 }
 request_json_data = ValuesForTests.json_data
@@ -82,18 +85,23 @@ def get_event(event_name=EventName.CREATE, operation="operation", supplier="EMIS
     def assert_dynamodb_record(self, operation_flag, action_flag, items, expected_values, expected_imms, response):
         """
         Asserts that a record with the expected structure exists in DynamoDB.
-        Ignores dynamically generated fields like PK, DateTimeStamp, and ExpiresAt.
+        Ignores the dynamically generated field PK.
         Ensures that the 'Imms' field matches exactly.
+        Ensures that the ExpiresAt field has been calculated correctly.
         """
         self.assertTrue(response)
 
-        filtered_items = [
-            {k: v for k, v in item.items() if k not in ["PK", "DateTimeStamp", "ExpiresAt"]}
+        unfiltered_items = [
+            {k: v for k, v in item.items()}
             for item in items
             if item.get("Operation") == operation_flag
             and item.get("Imms", {}).get("ACTION_FLAG") == action_flag
         ]
 
+        filtered_items = [
+            {k: v for k, v in item.items() if k not in ["PK", "DateTimeStamp", "ExpiresAt"]}
+            for item in unfiltered_items
+        ]
         self.assertGreater(len(filtered_items), 0, f"No matching item found for {operation_flag}")
 
         imms_data = filtered_items[0]["Imms"]
@@ -107,6 +115,12 @@ def assert_dynamodb_record(self, operation_flag, action_flag, items, expected_va
             self.assertIn(key, filtered_items[0], f"{key} is missing")
             self.assertEqual(filtered_items[0][key], expected_value, f"{key} mismatch")
 
+        # Check that the value of ExpiresAt is DELTA_TTL_DAYS after DateTimeStamp
+        expected_seconds = int(os.environ["DELTA_TTL_DAYS"]) * 24 * 60 * 60
+        date_time = int(datetime.fromisoformat(unfiltered_items[0]["DateTimeStamp"]).timestamp())
+        expires_at = unfiltered_items[0]["ExpiresAt"]
+        self.assertEqual(expires_at - date_time, expected_seconds)
+
     def test_fhir_converter_json_direct_data(self):
         """it should convert fhir json data to flat json"""
         json_data = json.dumps(ValuesForTests.json_data)

delta_backend/tests/test_delta.py

@@ -13,6 +13,7 @@
 ## @TODO: # Note: Environment variables shared across tests, thus aligned
 os.environ["AWS_SQS_QUEUE_URL"] = TEST_QUEUE_URL
 os.environ["DELTA_TABLE_NAME"] = "my_delta_table"
+os.environ["DELTA_TTL_DAYS"] = "14"
 os.environ["SOURCE"] = "my_source"
 
 from delta import send_message, handler, process_record  # Import after setting environment variables

terraform/delta.tf

@@ -138,6 +138,7 @@ resource "aws_lambda_function" "delta_sync_lambda" {
   environment {
     variables = {
       DELTA_TABLE_NAME     = aws_dynamodb_table.delta-dynamodb-table.name
+      DELTA_TTL_DAYS       = 30
       AWS_SQS_QUEUE_URL    = aws_sqs_queue.dlq.id
       SOURCE               = "IEDS"
       SPLUNK_FIREHOSE_NAME = module.splunk.firehose_stream_name