Merge branch 'master' into VED-169-Delta-Stream-multi-rec

# Conflicts:
#	delta_backend/src/delta.py

Author: nhsdevws

.github/workflows/sonarcloud.yml

@@ -13,7 +13,6 @@ jobs:
     if: ${{ !contains(github.event.pull_request.labels.*.name, 'dependency') }}
     runs-on: ubuntu-latest
 
-    
     steps:
       - uses: actions/checkout@v4
         with:
@@ -26,10 +25,10 @@ jobs:
         run: |
           aws configure set aws_access_key_id $AWS_ACCESS_KEY_ID
           aws configure set aws_secret_access_key $AWS_SECRET_ACCESS_KEY
-      
+
       - name: Initialize Unit Test Failure Tracker
         run: echo "false" > test_failed.txt
-          
+
       - name: Run unittest with filenameprocessor-coverage
         id: filenameprocessor
         continue-on-error: true
@@ -38,23 +37,20 @@ jobs:
           poetry run coverage run --source=filenameprocessor -m unittest discover -s filenameprocessor || echo "filenameprocessor tests failed" >> failed_tests.txt
           poetry run coverage xml -o sonarcloud-coverage-filenameprocessor-coverage.xml
 
-      
       - name: Run unittest with recordprocessor-coverage
         id: recordprocessor
         continue-on-error: true
         run: |
           poetry run coverage run --source=recordprocessor -m unittest discover -s recordprocessor || echo "recordprocessor tests failed" >> failed_tests.txt
           poetry run coverage xml -o sonarcloud-coverage-recordprocessor-coverage.xml
 
-
       - name: Run unittest with recordforwarder-coverage
         id: recordforwarder
         continue-on-error: true
         run: |
           PYTHONPATH=$(pwd)/backend:$(pwd)/backend/tests poetry run coverage run --source=backend -m unittest discover -s backend/tests -p "*batch*.py" || echo "recordforwarder tests failed" >> failed_tests.txt
           poetry run coverage xml -o sonarcloud-coverage-recordforwarder-coverage.xml
 
-          
       - name: Run unittest with coverage-ack-lambda
         id: acklambda
         continue-on-error: true
@@ -70,7 +66,7 @@ jobs:
           pip install poetry==1.8.4 mypy-boto3-dynamodb==1.35.54 boto3==1.26.165 coverage botocore==1.29.165 jmespath==1.0.1 python-dateutil==2.9.0 urllib3==1.26.20 s3transfer==0.6.2 typing-extensions==4.12.2
           poetry run coverage run --source=delta_backend -m unittest discover -s delta_backend || echo "delta tests failed" >> failed_tests.txt
           poetry run coverage xml -o sonarcloud-coverage-delta.xml
-      
+
       - name: Run unittest with coverage-fhir-api
         id: fhirapi
         continue-on-error: true
@@ -79,6 +75,14 @@ jobs:
           poetry run coverage run --source=backend -m unittest discover -s backend || echo "fhir-api tests failed" >> failed_tests.txt
           poetry run coverage xml -o sonarcloud-coverage.xml
 
+      - name: Run unittest with coverage-mesh-processor
+        id: meshprocessor
+        continue-on-error: true
+        run: |
+          pip install poetry==1.8.4 moto==4.2.11 coverage redis botocore==1.35.49 simplejson responses structlog fhir.resources jsonpath_ng pydantic==1.10.13 requests aws-lambda-typing cffi pyjwt boto3-stubs-lite[dynamodb]~=1.26.90 python-stdnum==1.20
+          poetry run coverage run --source=mesh_processor -m unittest discover -s mesh_processor || echo "mesh_processor tests failed" >> failed_tests.txt
+          poetry run coverage xml -o sonarcloud-mesh_processor-coverage.xml
+
       - name: Run Test Failure Summary
         id: check_failure
         run: |
@@ -98,5 +102,5 @@ jobs:
       - name: SonarCloud Scan
         uses: SonarSource/sonarqube-scan-action@master
         env:
-          GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}  # Needed to get PR information, if any
-          SONAR_TOKEN: ${{ secrets.SONAR_TOKEN }}
+          GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} # Needed to get PR information, if any
+          SONAR_TOKEN: ${{ secrets.SONAR_TOKEN }}

Makefile

@@ -38,7 +38,7 @@ build-proxy:
 	scripts/build_proxy.sh
 
 #Files to loop over in release
-_dist_include="pytest.ini poetry.lock poetry.toml pyproject.toml Makefile build/. e2e e2e_batch specification sandbox terraform scripts backend delta_backend ack_backend filenameprocessor recordprocessor"
+_dist_include="pytest.ini poetry.lock poetry.toml pyproject.toml Makefile build/. e2e e2e_batch specification sandbox terraform scripts backend delta_backend ack_backend filenameprocessor recordprocessor mesh_processor"
 
 
 #Create /dist/ sub-directory and copy files into directory

backend/.envrc.default

@@ -1,3 +1,3 @@
-layout pyenv 3.10.16
+layout pyenv 3.10.12
 
-dotenv
+dotenv

backend/.gitignore

@@ -1,2 +1,2 @@
 build
-.venv
+.venv

backend/src/fhir_service.py

@@ -19,11 +19,10 @@
 from models.field_names import FieldNames
 from models.errors import InvalidPatientId, CustomValidationError, UnhandledResponseError
 from models.fhir_immunization import ImmunizationValidator
-from models.utils.generic_utils import nhs_number_mod11_check, get_occurrence_datetime, create_diagnostics, form_json
+from models.utils.generic_utils import nhs_number_mod11_check, get_occurrence_datetime, create_diagnostics, form_json, get_contained_patient
 from models.constants import Constants
 from models.errors import MandatoryError
 from pds_service import PdsService
-from s_flag_handler import handle_s_flag
 from timer import timed
 from filter import Filter
 
@@ -79,25 +78,23 @@ def get_immunization_by_identifier(
 
     def get_immunization_by_id(self, imms_id: str, imms_vax_type_perms: str) -> Optional[dict]:
         """
-        Get an Immunization by its ID. Return None if not found. If the patient doesn't have an NHS number,
+        Get an Immunization by its ID. Return None if it is not found. If the patient doesn't have an NHS number,
         return the Immunization without calling PDS or checking S flag.
         """
         if not (imms_resp := self.immunization_repo.get_immunization_by_id(imms_id, imms_vax_type_perms)):
             return None
 
         # Returns the Immunisation full resource with no obfuscation
         resource = imms_resp.get("Resource", {})
-        imms_filtered_for_read = Filter.read(resource) if resource else {}
-
 
         return {
             "Version": imms_resp.get("Version", ""),
-            "Resource": Immunization.parse_obj(imms_filtered_for_read),
+            "Resource": Immunization.parse_obj(resource),
         }
 
     def get_immunization_by_id_all(self, imms_id: str, imms: dict) -> Optional[dict]:
         """
-        Get an Immunization by its ID. Return None if it is not found. If the patient doesn't have an NHS number,
+        Get an Immunization by its ID. Return None if not found. If the patient doesn't have an NHS number,
         return the Immunization without calling PDS or checking S flag.
         """
         imms["id"] = imms_id
@@ -264,15 +261,18 @@ def process_patient_for_bundle(patient: dict):
         """
 
         # Remove unwanted top-level fields
-        fields_to_keep = ["id", "resourceType", "identifier", "birthDate"]
+        fields_to_keep = {"resourceType", "identifier"}
         new_patient = {k: v for k, v in patient.items() if k in fields_to_keep}
 
         # Remove unwanted identifier fields
-        new_identifiers = []
-        for identifier in new_patient["identifier"]:
-            identifier_fields_to_keep = ["system", "value"]
-            new_identifiers.append({k: v for k, v in identifier.items() if k in identifier_fields_to_keep})
-        new_patient["identifier"] = new_identifiers
+        identifier_fields_to_keep = {"system", "value"}
+        new_patient["identifier"] = [
+            {k: v for k, v in identifier.items() if k in identifier_fields_to_keep}
+            for identifier in new_patient.get("identifier", [])
+        ]
+
+        if new_patient["identifier"]:
+            new_patient["id"] = new_patient["identifier"][0].get("value")
 
         return new_patient
 
@@ -316,25 +316,22 @@ def search_immunizations(
             if self.is_valid_date_from(r, date_from) and self.is_valid_date_to(r, date_to)
         ]
 
-        # Check whether the Superseded NHS number present in PDS
-        if pds_patient := self.pds_service.get_patient_details(nhs_number):
-            if pds_patient["identifier"][0]["value"] != nhs_number:
-                return create_diagnostics()
-
         # Create the patient URN for the fullUrl field.
         # NOTE: This UUID is assigned when a SEARCH request is received and used only for referencing the patient
         # resource from immunisation resources within the bundle. The fullUrl value we are using is a urn (hence the
         # FHIR key name of "fullUrl" is somewhat misleading) which cannot be used to locate any externally stored
         # patient resource. This is as agreed with VDS team for backwards compatibility with Immunisation History API.
         patient_full_url = f"urn:uuid:{str(uuid4())}"
 
+        imms_patient_record = get_contained_patient(resources[-1]) if resources else None
+
         # Filter and amend the immunization resources for the SEARCH response
-        resources_filtered_for_search = [Filter.search(imms, patient_full_url, pds_patient) for imms in resources]
+        resources_filtered_for_search = [Filter.search(imms, patient_full_url) for imms in resources]
 
         # Add bundle entries for each of the immunization resources
         entries = [
             BundleEntry(
-                resource=Immunization.parse_obj(handle_s_flag(imms, pds_patient)),
+                resource=Immunization.parse_obj(imms),
                 search=BundleEntrySearch(mode="match"),
                 fullUrl=f"https://api.service.nhs.uk/immunisation-fhir-api/Immunization/{imms['id']}",
             )
@@ -345,7 +342,7 @@ def search_immunizations(
         if len(resources) > 0:
             entries.append(
                 BundleEntry(
-                    resource=self.process_patient_for_bundle(pds_patient),
+                    resource=self.process_patient_for_bundle(imms_patient_record),
                     search=BundleEntrySearch(mode="include"),
                     fullUrl=patient_full_url,
                 )

backend/src/filter.py

@@ -1,6 +1,6 @@
 """Functions for filtering a FHIR Immunization Resource"""
 
-from models.utils.generic_utils import is_actor_referencing_contained_resource, get_contained_practitioner
+from models.utils.generic_utils import is_actor_referencing_contained_resource, get_contained_practitioner, get_contained_patient
 from constants import Urls
 
 
@@ -14,7 +14,7 @@ def remove_reference_to_contained_practitioner(imms: dict) -> dict:
 
     # Remove reference to the contained practitioner from imms[performer]
     imms["performer"] = [
-        x for x in imms["performer"] if not is_actor_referencing_contained_resource(x, contained_practitioner["id"])
+         x for x in imms["performer"] if not is_actor_referencing_contained_resource(x, contained_practitioner["id"])
     ]
 
     return imms
@@ -96,25 +96,11 @@ class Filter:
     """Functions for filtering a FHIR Immunization Resource"""
 
     @staticmethod
-    def read(imms: dict) -> dict:
-        """Apply filtering for READ request"""
-        imms.pop("identifier")
-        return imms
-
-    @staticmethod
-    def search(imms: dict, patient_full_url: str, bundle_patient: dict = None) -> dict:
+    def search(imms: dict, patient_full_url: str) -> dict:
         """Apply filtering for an individual FHIR Immunization Resource as part of SEARCH request"""
         imms = remove_reference_to_contained_practitioner(imms)
-        imms.pop("contained")
-        imms["patient"] = create_reference_to_patient_resource(patient_full_url, bundle_patient)
+        imms["patient"] = create_reference_to_patient_resource(patient_full_url, get_contained_patient(imms))
         imms = add_use_to_identifier(imms)
-        return imms
+        imms.pop("contained")
 
-    @staticmethod
-    def s_flag(imms: dict) -> dict:
-        """Apply filtering for patients with 'RESTRICTED' flag"""
-        imms = replace_address_postal_codes(imms)
-        imms = replace_organization_values(imms)
-        if imms.get("location"):
-            imms["location"] = {"identifier": {"system": "urn:iso:std:iso:3166", "value": "GB"}}
         return imms