Add unittests for cache and logging functions and move cache functions and constants to appropriate files

parent 0719a0d
author Akol125 <akinola.olutola1@nhs.net> 1741732167 +0000
committer Robert Novac <robert.novac@airelogic.com> 1742314388 +0000

READ request - drop obfuscation for s-flag patient
SEARCH request - removed PDS call from search and used Patient resource from last immunisation Contained resource in the final response

Author: AlexandraBenson

.github/workflows/continuous-integration.yml

@@ -8,12 +8,12 @@ jobs:
     if: github.ref == 'refs/heads/master'
     steps:
       - name: Checkout
-        uses: actions/checkout@v2
+        uses: actions/checkout@v4
         with:
           fetch-depth: 0  # This causes all history to be fetched, which is required for calculate-version to function	
 
       - name: Install Python 3.9
-        uses: actions/setup-python@v1
+        uses: actions/setup-python@v5
         with:
           python-version: 3.9
 
@@ -40,5 +40,4 @@ jobs:
           GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
         with:
           tag_name: ${{ env.SPEC_VERSION }}
-          release_name: ${{ env.SPEC_VERSION }}
-
+          release_name: ${{ env.SPEC_VERSION }}

.github/workflows/sonarcloud.yml

@@ -34,7 +34,7 @@ jobs:
         id: filenameprocessor
         continue-on-error: true
         run: |
-          pip install poetry==1.8.4 moto==4.2.11 coverage redis botocore==1.35.49 simplejson pandas freezegun responses structlog fhir.resources jsonpath_ng pydantic==1.10.13 requests aws-lambda-typing cffi pyjwt boto3-stubs-lite[dynamodb]~=1.26.90 python-stdnum==1.20
+          pip install poetry==1.8.4 moto==4.2.11 coverage redis botocore==1.35.49 simplejson pandas freezegun responses structlog fhir.resources jsonpath_ng pydantic==1.10.13 requests aws-lambda-typing cffi pyjwt boto3-stubs-lite[dynamodb]~=1.26.90 python-stdnum==1.20 fakeredis
           poetry run coverage run --source=filenameprocessor -m unittest discover -s filenameprocessor || echo "filenameprocessor tests failed" >> failed_tests.txt
           poetry run coverage xml -o sonarcloud-coverage-filenameprocessor-coverage.xml
 

Makefile

@@ -38,7 +38,7 @@ build-proxy:
 	scripts/build_proxy.sh
 
 #Files to loop over in release
-_dist_include="pytest.ini poetry.lock poetry.toml pyproject.toml Makefile build/. e2e specification sandbox terraform scripts backend delta_backend ack_backend filenameprocessor recordprocessor"
+_dist_include="pytest.ini poetry.lock poetry.toml pyproject.toml Makefile build/. e2e e2e_batch specification sandbox terraform scripts backend delta_backend ack_backend filenameprocessor recordprocessor"
 
 
 #Create /dist/ sub-directory and copy files into directory

azure/templates/post-deploy.yml

@@ -114,6 +114,12 @@ steps:
     displayName: Waiting for TF resources to be UP
     workingDirectory: "$(Pipeline.Workspace)/s/$(SERVICE_NAME)/$(SERVICE_ARTIFACT_NAME)"
 
+  - bash: |
+      pyenv install -s 3.10.8
+      pyenv global 3.10.8
+      python --version
+    displayName: Set Python 3.10
+
   - bash: |
       set -e
       export RELEASE_RELEASEID=$(Build.BuildId)
@@ -166,13 +172,45 @@ steps:
         echo "running: $test_cmd -v -c"
         $test_cmd -v -c
       fi
-
     workingDirectory: "$(Pipeline.Workspace)/s/$(SERVICE_NAME)/$(SERVICE_ARTIFACT_NAME)/e2e"
-    displayName: Run full test suite
+    displayName: Run Full Test Suite
+
+  
+
+  - bash: |
+      set -e
+      if ! [[ "$APIGEE_ENVIRONMENT" == "prod" || "$APIGEE_ENVIRONMENT" == "int" || "$APIGEE_ENVIRONMENT" == *"sandbox" ]]; then
+        echo "Running E2E batch folder test cases"
+ 
+        export AWS_PROFILE="apim-dev"
+        aws_account_no="$(aws sts get-caller-identity --query Account --output text)"
+        echo "Using AWS Account: $aws_account_no"
+        
+        service_name="${FULLY_QUALIFIED_SERVICE_NAME}"
+ 
+        pr_no=$(echo "$service_name" | { grep -oE '[0-9]+$' || true; })
+        if [ -z "$pr_no" ]; then
+          workspace="$APIGEE_ENVIRONMENT"
+        else
+          workspace="pr-$pr_no"
+        fi
+ 
+        poetry install --no-root  # Install dependencies defined in pyproject.toml
+ 
+        ENV="$workspace" poetry run python -m unittest -v -c
+ 
+        echo "E2E batch folder test cases executed successfully"
+      else
+        echo "Skipping E2E batch folder test cases as the environment is prod-int-sandbox"
+      fi
+ 
+    displayName: Run full batch test suite
+    workingDirectory: "$(Pipeline.Workspace)/s/$(SERVICE_NAME)/$(SERVICE_ARTIFACT_NAME)/e2e_batch"
+
 
   - task: PublishTestResults@2
     displayName: 'Publish test results'
     condition: always()
     inputs:
       testResultsFiles: '$(Pipeline.Workspace)/s/$(SERVICE_NAME)/$(SERVICE_ARTIFACT_NAME)/tests/test-report.xml'
-      failTaskOnFailedTests: true
+      failTaskOnFailedTests: true

backend/.envrc.default

@@ -1,3 +1,3 @@
-layout pyenv 3.10.12
+layout pyenv 3.10.16
 
-dotenv
+dotenv

backend/.gitignore

@@ -1,2 +1,2 @@
-build
-.venv
+build
+.venv

backend/src/fhir_service.py

@@ -19,7 +19,7 @@
 from models.field_names import FieldNames
 from models.errors import InvalidPatientId, CustomValidationError, UnhandledResponseError
 from models.fhir_immunization import ImmunizationValidator
-from models.utils.generic_utils import nhs_number_mod11_check, get_occurrence_datetime, create_diagnostics, form_json
+from models.utils.generic_utils import nhs_number_mod11_check, get_occurrence_datetime, create_diagnostics, form_json, get_contained_patient
 from models.constants import Constants
 from models.errors import MandatoryError
 from pds_service import PdsService
@@ -76,7 +76,7 @@ def get_immunization_by_identifier(
             base_url = f"{get_service_url()}/Immunization"
             response = form_json(imms_resp, element, identifier, base_url)
             return response
-
+    
     def get_immunization_by_id(self, imms_id: str, imms_vax_type_perms: str) -> Optional[dict]:
         """
         Get an Immunization by its ID. Return None if not found. If the patient doesn't have an NHS number,
@@ -85,26 +85,19 @@ def get_immunization_by_id(self, imms_id: str, imms_vax_type_perms: str) -> Opti
         if not (imms_resp := self.immunization_repo.get_immunization_by_id(imms_id, imms_vax_type_perms)):
             return None
 
-        # Remove fields rom the imms resource which are not to be returned for read
-        imms_filtered_for_read = Filter.read(imms_resp.get("Resource", {}))
+        # Returns the Immunisation full resource with no obfuscation
+        resource = imms_resp.get("Resource", {})
+        imms_filtered_for_read = Filter.read(resource) if resource else {}
 
-        # Handle s-flag filtering, where applicable
-        if not (nhs_number := obtain_field_value(imms_filtered_for_read, FieldNames.patient_identifier_value)):
-            imms_filtered_for_read_and_s_flag = imms_filtered_for_read
-        else:
-            if patient := self.pds_service.get_patient_details(nhs_number):
-                imms_filtered_for_read_and_s_flag = handle_s_flag(imms_filtered_for_read, patient)
-            else:
-                raise UnhandledResponseError("unable to validate NHS number with downstream service")
 
         return {
             "Version": imms_resp.get("Version", ""),
-            "Resource": Immunization.parse_obj(imms_filtered_for_read_and_s_flag),
+            "Resource": Immunization.parse_obj(imms_filtered_for_read),
         }
 
     def get_immunization_by_id_all(self, imms_id: str, imms: dict) -> Optional[dict]:
         """
-        Get an Immunization by its ID. Return None if not found. If the patient doesn't have an NHS number,
+        Get an Immunization by its ID. Return None if it is not found. If the patient doesn't have an NHS number,
         return the Immunization without calling PDS or checking S flag.
         """
         imms["id"] = imms_id
@@ -323,25 +316,22 @@ def search_immunizations(
             if self.is_valid_date_from(r, date_from) and self.is_valid_date_to(r, date_to)
         ]
 
-        # Check whether the Superseded NHS number present in PDS
-        if pds_patient := self.pds_service.get_patient_details(nhs_number):
-            if pds_patient["identifier"][0]["value"] != nhs_number:
-                return create_diagnostics()
-
         # Create the patient URN for the fullUrl field.
         # NOTE: This UUID is assigned when a SEARCH request is received and used only for referencing the patient
         # resource from immunisation resources within the bundle. The fullUrl value we are using is a urn (hence the
         # FHIR key name of "fullUrl" is somewhat misleading) which cannot be used to locate any externally stored
         # patient resource. This is as agreed with VDS team for backwards compatibility with Immunisation History API.
         patient_full_url = f"urn:uuid:{str(uuid4())}"
-
-        # Filter and amend the immunization resources for the SEARCH response
-        resources_filtered_for_search = [Filter.search(imms, patient_full_url, pds_patient) for imms in resources]
+        
+        imms_patient_record = get_contained_patient(resources[-1]) if resources else None
+        
+        # Filter and amend the immunization resources for the SEARCH response            
+        resources_filtered_for_search = [Filter.search(imms, patient_full_url, []) for imms in resources]
 
         # Add bundle entries for each of the immunization resources
         entries = [
             BundleEntry(
-                resource=Immunization.parse_obj(handle_s_flag(imms, pds_patient)),
+                resource=Immunization.parse_obj(imms),
                 search=BundleEntrySearch(mode="match"),
                 fullUrl=f"https://api.service.nhs.uk/immunisation-fhir-api/Immunization/{imms['id']}",
             )
@@ -352,7 +342,7 @@ def search_immunizations(
         if len(resources) > 0:
             entries.append(
                 BundleEntry(
-                    resource=self.process_patient_for_bundle(pds_patient),
+                    resource=self.process_patient_for_bundle(imms_patient_record),
                     search=BundleEntrySearch(mode="include"),
                     fullUrl=patient_full_url,
                 )

backend/src/filter.py

@@ -1,6 +1,6 @@
 """Functions for filtering a FHIR Immunization Resource"""
 
-from models.utils.generic_utils import is_actor_referencing_contained_resource, get_contained_practitioner
+from models.utils.generic_utils import is_actor_referencing_contained_resource, get_contained_practitioner, get_contained_patient
 from constants import Urls
 
 
@@ -47,7 +47,7 @@ def replace_address_postal_codes(imms: dict) -> dict:
                 # Remove all other keys in the address dictionary
                 keys_to_remove = [key for key in address.keys() if key != "postalCode"]
                 for key in keys_to_remove:
-                    del address[key]    
+                    del address[key]
 
     return imms
 
@@ -59,23 +59,23 @@ def replace_organization_values(imms: dict) -> dict:
     """
     for performer in imms.get("performer", [{}]):
         if performer.get("actor", {}).get("type") == "Organization":
-            
+
             # Obfuscate or set the identifier value and system.
             identifier = performer["actor"].get("identifier", {})
             if identifier.get("value") is not None:
                 identifier["value"] = "N2N9I"
                 identifier["system"] = Urls.ods_organization_code
             if identifier.get("system") is not None:
                 identifier["system"] = Urls.ods_organization_code
-            
+
             # Ensure only 'system' and 'value' remain in identifier
             keys = {"system", "value"}
             keys_to_remove = [key for key in identifier.keys() if key not in keys]
             for key in keys_to_remove:
                 del identifier[key]
-            
+
             # Remove all other fields except 'identifier' in actor
-            keys_to_remove = [key for key in performer["actor"].keys() if key not in ("identifier","type")]
+            keys_to_remove = [key for key in performer["actor"].keys() if key not in ("identifier", "type")]
             for key in keys_to_remove:
                 del performer["actor"][key]
 
@@ -94,7 +94,6 @@ def add_use_to_identifier(imms: dict) -> dict:
 
 class Filter:
     """Functions for filtering a FHIR Immunization Resource"""
-
     @staticmethod
     def read(imms: dict) -> dict:
         """Apply filtering for READ request"""
@@ -105,12 +104,10 @@ def read(imms: dict) -> dict:
     def search(imms: dict, patient_full_url: str, bundle_patient: dict = None) -> dict:
         """Apply filtering for an individual FHIR Immunization Resource as part of SEARCH request"""
         imms = remove_reference_to_contained_practitioner(imms)
+        imms_patient_data = get_contained_patient(imms)
         imms.pop("contained")
-        imms["patient"] = create_reference_to_patient_resource(patient_full_url, bundle_patient)
+        imms["patient"] = create_reference_to_patient_resource(patient_full_url, imms_patient_data)
         imms = add_use_to_identifier(imms)
-        # Location identifier system and value are to be overwritten
-        # (for backwards compatibility with Immunisation History API, as agreed with VDS team)
-        imms["location"] = {"identifier": {"system": "urn:iso:std:iso:3166", "value": "GB"}}
         return imms
 
     @staticmethod

backend/src/get_imms_handler.py

@@ -16,7 +16,7 @@ def get_imms_handler(event, context):
 
 def get_immunization_by_id(event, controller: FhirController):
     try:
-        return controller.get_immunization_by_id(event)
+        return controller.get_immunization_by_id(event)  
     except Exception:  # pylint: disable = broad-exception-caught
         exp_error = create_operation_outcome(
             resource_id=str(uuid.uuid4()),

backend/tests/sample_data/completed_covid19_immunization_event_filtered_for_search_using_bundle_patient_resource.json

@@ -46,9 +46,10 @@
     "display": "AstraZeneca Ltd"
   },
   "location": {
+    "type": "Location",
     "identifier": {
-      "value": "GB",
-      "system": "urn:iso:std:iso:3166"
+      "value": "X99999",
+      "system": "https://fhir.nhs.uk/Id/ods-organization-code"
     }
   },
   "lotNumber": "4120Z001",