VED-687 Search API bug RSV Not returned (#721)

implement pagination

Author: nhsdevws

backend/poetry.lock

@@ -3,6 +3,7 @@
 import boto3
 import time
 import simplejson as json
+from clients import logger
 from dataclasses import dataclass
 import botocore.exceptions
 from boto3.dynamodb.conditions import Key, Attr
@@ -34,7 +35,7 @@ def _query_identifier(table, index, pk, identifier, is_present):
                 return queryresponse
 
             if retries > 6:
-                print(f"{identifier}: Crossed {retries} retries")
+                logger.info(f"{identifier}: Crossed {retries} retries")
 
             retries += 1
             # Delay time in milliseconds

backend/src/fhir_batch_repository.py

@@ -13,6 +13,7 @@
 from botocore.config import Config
 from fhir.resources.R4B.immunization import Immunization
 from boto3 import client as boto3_client
+from clients import logger
 
 from authorization import Authorization, UnknownPermission
 from cache import Cache
@@ -640,7 +641,7 @@ def check_vaccine_type_permissions(self, aws_event):
             if len(supplier_system) == 0:
                 raise UnauthorizedSystemError()
             imms_vax_type_perms = get_supplier_permissions(supplier_system)
-            print(f" update imms = {imms_vax_type_perms}")
+            logger.info(f" update imms = {imms_vax_type_perms}")
             if len(imms_vax_type_perms) == 0:
                 raise UnauthorizedVaxError()
             # Return the values needed for later use

backend/src/fhir_controller.py

@@ -1,3 +1,5 @@
+from urllib import response
+from responses import logger
 import simplejson as json
 import os
 import time
@@ -397,20 +399,46 @@ def find_immunizations(self, patient_identifier: str, vaccine_types: list):
         condition = Key("PatientPK").eq(_make_patient_pk(patient_identifier))
         is_not_deleted = Attr("DeletedAt").not_exists() | Attr("DeletedAt").eq("reinstated")
 
-        response = self.table.query(
-            IndexName="PatientGSI",
-            KeyConditionExpression=condition,
-            FilterExpression=is_not_deleted,
-        )
+        raw_items = self.get_all_items(condition, is_not_deleted)
 
-        if "Items" in response:
+        if raw_items:    
             # Filter the response to contain only the requested vaccine types
-            items = [x for x in response["Items"] if x["PatientSK"].split("#")[0] in vaccine_types]
+            items = [x for x in raw_items if x["PatientSK"].split("#")[0] in vaccine_types]
 
             # Return a list of the FHIR immunization resource JSON items
-            return [json.loads(item["Resource"]) for item in items]
+            final_resources = [json.loads(item["Resource"]) for item in items]
+
+            return final_resources
         else:
-            raise UnhandledResponseError(message=f"Unhandled error. Query failed", response=response)
+            logger.warning("no items matched patient_identifier filter!")
+            return []
+
+    def get_all_items(self, condition, is_not_deleted):
+        """Query DynamoDB and paginate through all results."""
+        all_items = []
+        last_evaluated_key = None
+
+        while True:
+            query_args = {
+                "IndexName": "PatientGSI",
+                "KeyConditionExpression": condition,
+                "FilterExpression": is_not_deleted,
+            }
+            if last_evaluated_key:
+                query_args["ExclusiveStartKey"] = last_evaluated_key
+
+            response = self.table.query(**query_args)
+            if "Items" not in response:
+                raise UnhandledResponseError(message="No Items in DynamoDB response", response=response)
+  
+            items = response.get("Items", [])
+            all_items.extend(items)
+
+            last_evaluated_key = response.get("LastEvaluatedKey")
+            if not last_evaluated_key:
+                break
+
+        return all_items
 
     @staticmethod
     def _handle_dynamo_response(response):

backend/src/fhir_repository.py

@@ -20,7 +20,7 @@ def function_info(func):
     @wraps(func)
     def wrapper(*args, **kwargs):
         event = args[0] if args else {}
-        print(f"Event: {event}")
+        logger.info(f"Event: {event}")
         headers = event.get("headers", {})
         correlation_id = headers.get("X-Correlation-ID", "X-Correlation-ID not passed")
         request_id = headers.get("X-Request-ID", "X-Request-ID not passed")
@@ -40,7 +40,7 @@ def wrapper(*args, **kwargs):
         start = time.time()
         try:
             result = func(*args, **kwargs)
-            print(f"Result:{result}")
+            logger.info(f"Result:{result}")
             end = time.time()
             log_data["time_taken"] = f"{round(end - start, 5)}s"
             status = "500"
@@ -56,7 +56,7 @@ def wrapper(*args, **kwargs):
                         record = result_headers["Location"]
                 if result.get("body"):
                     ops_outcome = json.loads(result["body"])
-                    print(f"ops_outcome: {ops_outcome}")
+                    logger.info(f"ops_outcome: {ops_outcome}")
                     if ops_outcome.get("issue"):
                         outcome_body = ops_outcome["issue"][0]
                         status_code = outcome_body["code"]

backend/src/log_structure.py

@@ -1,4 +1,5 @@
 from enum import StrEnum
+from clients import logger
 
 class ApiOperationCode(StrEnum):
     CREATE = "c"
@@ -22,7 +23,7 @@ def _expand_permissions(permissions: list[str]) -> dict[str, list[ApiOperationCo
 
 def validate_permissions(permissions: list[str], operation: ApiOperationCode, vaccine_types: list[str]):
     expanded_permissions = _expand_permissions(permissions)
-    print(f"operation: {operation}, expanded_permissions: {expanded_permissions}, vaccine_types: {vaccine_types}")
+    logger.info(f"operation: {operation}, expanded_permissions: {expanded_permissions}, vaccine_types: {vaccine_types}")
     return all(
         operation in expanded_permissions.get(vaccine_type.lower(), [])
         for vaccine_type in vaccine_types

backend/src/models/utils/permission_checker.py

@@ -6,7 +6,7 @@
 from typing import Optional
 from urllib.parse import parse_qs, urlencode, quote
 
-from clients import redis_client
+from clients import redis_client, logger
 from models.errors import ParameterException
 from models.constants import Constants
 

backend/src/parameter_parser.py

@@ -2,7 +2,6 @@
 import json
 import logging
 import pprint
-import traceback
 import uuid
 
 from aws_lambda_typing import context as context_, events
@@ -53,9 +52,7 @@ def search_imms(event: events.APIGatewayProxyEventV1, controller: FhirController
                 or body_has_immunization_element
             ):
                 return controller.get_immunization_by_identifier(event)
-            response = controller.search_immunizations(event)
-        else:
-            response = controller.search_immunizations(event)
+        response = controller.search_immunizations(event)
 
         result_json = json.dumps(response)
         result_size = len(result_json.encode("utf-8"))

backend/src/search_imms_handler.py

@@ -1,6 +1,6 @@
 import json
 import unittest
-from unittest.mock import create_autospec
+from unittest.mock import create_autospec, patch
 
 from create_imms_handler import create_immunization
 from fhir_controller import FhirController
@@ -11,6 +11,13 @@
 class TestCreateImmunizationById(unittest.TestCase):
     def setUp(self):
         self.controller = create_autospec(FhirController)
+        self.logger_info_patcher = patch("logging.Logger.info")
+        self.mock_logger_info = self.logger_info_patcher.start()
+        self.logger_exception_patcher = patch("logging.Logger.exception")
+        self.mock_logger_exception = self.logger_exception_patcher.start()
+        
+    def tearDown(self):
+        patch.stopall()
 
     def test_create_immunization(self):
         """it should create Immunization"""
@@ -26,7 +33,7 @@ def test_create_immunization(self):
         self.controller.create_immunization.assert_called_once_with(lambda_event)
         self.assertDictEqual(exp_res, act_res)
 
-    def test_handle_exception(self):
+    def test_create_handle_exception(self):
         """unhandled exceptions should result in 500"""
         lambda_event = {"aws": "event"}
         error_msg = "an unhandled error"

backend/tests/test_create_imms.py

@@ -1,6 +1,6 @@
 import json
 import unittest
-from unittest.mock import create_autospec
+from unittest.mock import create_autospec, patch
 
 from delete_imms_handler import delete_immunization
 from fhir_controller import FhirController
@@ -11,6 +11,11 @@
 class TestDeleteImmunizationById(unittest.TestCase):
     def setUp(self):
         self.controller = create_autospec(FhirController)
+        self.logger_exception_patcher = patch("logging.Logger.exception")
+        self.mock_logger_exception = self.logger_exception_patcher.start()
+
+    def tearDown(self):
+        patch.stopall()
 
     def test_delete_immunization(self):
         """it should delete Immunization"""
@@ -26,7 +31,7 @@ def test_delete_immunization(self):
         self.controller.delete_immunization.assert_called_once_with(lambda_event)
         self.assertDictEqual(exp_res, act_res)
 
-    def test_handle_exception(self):
+    def test_delete_handle_exception(self):
         """unhandled exceptions should result in 500"""
         lambda_event = {"pathParameters": {"id": "an-id"}}
         error_msg = "an unhandled error"