VED-755: added pagination, remove ieds_check_exist, modify test

Author: Akol125

lambdas/id_sync/src/id_sync.py

@@ -1,18 +1,17 @@
-from typing import Any, Dict
-
-from common.clients import logger, STREAM_NAME
-from common.log_decorator import logging_decorator
-from common.aws_lambda_event import AwsLambdaEvent
-from exceptions.id_sync_exception import IdSyncException
-from record_processor import process_record
-
 """
 - Parses the incoming AWS event into `AwsLambdaEvent` and iterate its `records`.
 - Delegate each record to `process_record` and collect `nhs_number` from each result.
 - If any record has status == "error" raise `IdSyncException` with aggregated nhs_numbers.
 - Any unexpected error is wrapped into `IdSyncException(message="Error processing id_sync event")`.
 """
 
+from typing import Any, Dict
+from common.clients import logger, STREAM_NAME
+from common.log_decorator import logging_decorator
+from common.aws_lambda_event import AwsLambdaEvent
+from exceptions.id_sync_exception import IdSyncException
+from record_processor import process_record
+
 
 @logging_decorator(prefix="id_sync", stream_name=STREAM_NAME)
 def handler(event_data: Dict[str, Any], _context) -> Dict[str, Any]:

lambdas/id_sync/src/ieds_db_operations.py

@@ -16,17 +16,6 @@ def get_ieds_table():
     return ieds_table
 
 
-def ieds_check_exist(id: str) -> bool:
-    """Check if a record exists in the IEDS table for the given ID."""
-    logger.info(f"Check Id exists ID: {id}")
-    items = get_items_from_patient_id(id, 1)
-
-    if items or len(items) > 0:
-        logger.info(f"Found patient ID: {id}")
-        return True
-    return False
-
-
 BATCH_SIZE = 25
 
 
@@ -118,28 +107,62 @@ def ieds_update_patient_id(old_id: str, new_id: str) -> dict:
         )
 
 
-def get_items_from_patient_id(id: str, limit=BATCH_SIZE) -> list:
-    """Get all items for patient ID."""
+def get_items_from_patient_id(id: str, filter_expression=None) -> list:
+    """Query the PatientGSI and paginate through all results.
+
+    - Uses LastEvaluatedKey to page until all items are collected.
+    - If `filter_expression` is provided it will be included as `FilterExpression`.
+    - Raises `IdSyncException` if the DynamoDB response doesn't include 'Items' or
+      an underlying error occurs.
+    """
     logger.info(f"Getting items for patient id: {id}")
     patient_pk = f"Patient#{id}"
-    try:
-        response = get_ieds_table().query(
-            IndexName='PatientGSI',  # query the GSI
-            KeyConditionExpression=Key('PatientPK').eq(patient_pk),
-            Limit=limit
-        )
 
-        if 'Items' not in response or not response['Items']:
+    all_items: list = []
+    last_evaluated_key = None
+    try:
+        while True:
+            query_args = {
+                "IndexName": "PatientGSI",
+                "KeyConditionExpression": Key('PatientPK').eq(patient_pk),
+            }
+            if filter_expression is not None:
+                query_args["FilterExpression"] = filter_expression
+            if last_evaluated_key:
+                query_args["ExclusiveStartKey"] = last_evaluated_key
+
+            response = get_ieds_table().query(**query_args)
+
+            if "Items" not in response:
+                # Unexpected DynamoDB response shape - surface as IdSyncException
+                logger.exception("Unexpected DynamoDB response: missing 'Items'")
+                raise IdSyncException(
+                    message="No Items in DynamoDB response",
+                    nhs_numbers=[patient_pk],
+                    exception=response,
+                )
+
+            items = response.get("Items", [])
+            all_items.extend(items)
+
+            last_evaluated_key = response.get("LastEvaluatedKey")
+            if not last_evaluated_key:
+                break
+
+        if not all_items:
             logger.warning(f"No items found for patient PK: {patient_pk}")
             return []
 
-        return response['Items']
+        return all_items
+
+    except IdSyncException:
+        raise
     except Exception as e:
         logger.exception(f"Error querying items for patient PK: {patient_pk}")
         raise IdSyncException(
             message=f"Error querying items for patient PK: {patient_pk}",
             nhs_numbers=[patient_pk],
-            exception=e
+            exception=e,
         )
 
 

lambdas/id_sync/src/record_processor.py

@@ -2,7 +2,6 @@
 from typing import Dict, Any
 from pds_details import pds_get_patient_id, pds_get_patient_details
 from ieds_db_operations import (
-    ieds_check_exist,
     ieds_update_patient_id,
     extract_patient_resource_from_item,
     get_items_from_patient_id,
@@ -57,20 +56,26 @@ def process_nhs_number(nhs_number: str) -> Dict[str, Any]:
         }
     logger.info("Update patient ID from %s to %s", nhs_number, new_nhs_number)
 
-    if not ieds_check_exist(nhs_number):
-        logger.info("No IEDS record found for: %s", nhs_number)
-        response = {"status": "success", "message": f"No records returned for ID: {nhs_number}"}
-        return response
     try:
         pds_details, ieds_details = fetch_demographic_details(nhs_number)
     except Exception as e:
-        logger.exception("process_nhs_number: %s, aborting update", e)
+        logger.exception("process_nhs_number: failed to fetch demographic details: %s", e)
         return {
                 "status": "error",
                 "message": str(e),
                 "nhs_number": nhs_number,
         }
 
+    # If no IEDS items were returned, nothing to update — return a clear success
+    # message to match existing test expectations.
+    if not ieds_details:
+        logger.info("No IEDS records returned for NHS number: %s", nhs_number)
+        return {
+            "status": "success",
+            "message": f"No records returned for ID: {nhs_number}",
+            "nhs_number": nhs_number,
+        }
+
     # If at least one IEDS item matches demographics, proceed with update
     if not all(demographics_match(pds_details, detail) for detail in ieds_details):
         logger.info("Not all IEDS items matched PDS demographics; skipping update for %s", nhs_number)

lambdas/id_sync/tests/test_ieds_db_operations.py

@@ -533,142 +533,3 @@ def test_get_items_from_patient_id_no_records(self):
 
         # Assert
         self.assertEqual(result, [])
-
-
-class TestIedsCheckExists(TestIedsDbOperations):
-
-    def setUp(self):
-        """Set up test fixtures"""
-        super().setUp()
-
-        # Mock get_items_from_patient_id instead of table.query
-        self.get_items_from_patient_id_patcher = patch('ieds_db_operations.get_items_from_patient_id')
-        self.mock_get_items_from_patient_id = self.get_items_from_patient_id_patcher.start()
-
-    def tearDown(self):
-        """Clean up patches"""
-        super().tearDown()
-
-    def test_ieds_check_exist_record_exists(self):
-        """Test when record exists in IEDS table"""
-        # Arrange
-        patient_id = "test-patient-123"
-        mock_items = [{'PK': 'Patient#test-patient-123', 'PatientPK': 'Patient#test-patient-123'}]
-        self.mock_get_items_from_patient_id.return_value = mock_items
-
-        # Act
-        result = ieds_db_operations.ieds_check_exist(patient_id)
-
-        # Assert
-        self.assertTrue(result)
-
-        # Verify get_items_from_patient_id was called with correct parameters
-        self.mock_get_items_from_patient_id.assert_called_once_with(patient_id, 1)
-
-    def test_ieds_check_exist_record_not_exists(self):
-        """Test when no record exists in IEDS table"""
-        # Arrange
-        patient_id = "test-patient-456"
-        self.mock_get_items_from_patient_id.return_value = []
-
-        # Act
-        result = ieds_db_operations.ieds_check_exist(patient_id)
-
-        # Assert
-        self.assertFalse(result)
-
-        # Verify get_items_from_patient_id was called
-        self.mock_get_items_from_patient_id.assert_called_once_with(patient_id, 1)
-
-    def test_ieds_check_exist_empty_id(self):
-        """Test with empty patient ID"""
-        # Arrange
-        patient_id = ""
-        self.mock_get_items_from_patient_id.return_value = []
-
-        # Act
-        result = ieds_db_operations.ieds_check_exist(patient_id)
-
-        # Assert
-        self.assertFalse(result)
-
-        # Verify get_items_from_patient_id was called with empty ID
-        self.mock_get_items_from_patient_id.assert_called_once_with(patient_id, 1)
-
-    def test_ieds_check_exist_none_id(self):
-        """Test with None patient ID"""
-        # Arrange
-        patient_id = None
-        self.mock_get_items_from_patient_id.return_value = []
-
-        # Act
-        result = ieds_db_operations.ieds_check_exist(patient_id)
-
-        # Assert
-        self.assertFalse(result)
-
-        # Verify get_items_from_patient_id was called with None ID
-        self.mock_get_items_from_patient_id.assert_called_once_with(patient_id, 1)
-
-    def test_ieds_check_exist_query_exception(self):
-        """Test exception handling during get_items_from_patient_id"""
-        # Arrange
-        patient_id = "test-patient-error"
-        test_exception = Exception("DynamoDB query failed")
-        self.mock_get_items_from_patient_id.side_effect = test_exception
-
-        # Act & Assert
-        with self.assertRaises(Exception) as context:
-            ieds_db_operations.ieds_check_exist(patient_id)
-
-        self.assertEqual(str(context.exception), "DynamoDB query failed")
-
-        # Verify get_items_from_patient_id was attempted
-        self.mock_get_items_from_patient_id.assert_called_once_with(patient_id, 1)
-
-    def test_ieds_check_exist_multiple_items_found(self):
-        """Test when multiple items are found (should still return True)"""
-        # Arrange
-        patient_id = "test-patient-multiple"
-        mock_items = [
-            {'PK': 'Patient#test-patient-multiple', 'PatientPK': 'Patient#test-patient-multiple'},
-            {'PK': 'Patient#test-patient-multiple#record1', 'PatientPK': 'Patient#test-patient-multiple'}
-        ]
-        self.mock_get_items_from_patient_id.return_value = mock_items
-
-        # Act
-        result = ieds_db_operations.ieds_check_exist(patient_id)
-
-        # Assert
-        self.assertTrue(result)
-
-        # Verify get_items_from_patient_id was called with limit=1
-        self.mock_get_items_from_patient_id.assert_called_once_with(patient_id, 1)
-
-    def test_ieds_check_exist_single_item_found(self):
-        """Test when exactly one item is found"""
-        # Arrange
-        patient_id = "test-patient-single"
-        mock_items = [{'PK': 'Patient#test-patient-single', 'PatientPK': 'Patient#test-patient-single'}]
-        self.mock_get_items_from_patient_id.return_value = mock_items
-
-        # Act
-        result = ieds_db_operations.ieds_check_exist(patient_id)
-
-        # Assert
-        self.assertTrue(result)
-
-        # Verify get_items_from_patient_id was called
-        self.mock_get_items_from_patient_id.assert_called_once_with(patient_id, 1)
-
-    def test_ieds_check_exist_limit_parameter(self):
-        """Test that the function passes limit=1 to get_items_from_patient_id"""
-        # Arrange
-        patient_id = "test-patient-limit"
-        self.mock_get_items_from_patient_id.return_value = []
-
-        # Act
-        ieds_db_operations.ieds_check_exist(patient_id)
-
-        # Assert - Verify the limit parameter is correctly passed
-        self.mock_get_items_from_patient_id.assert_called_once_with(patient_id, 1)