Merge branch 'master' into VED-789-Schema-Validation

Author: Akol125

.github/dependabot.yml

@@ -7,13 +7,13 @@ version: 2
 updates:
   - package-ecosystem: "docker"
     directories:
-      - "/batch_processor_filter"
       - "/infrastructure/grafana/non-prod/docker"
-      - "/lambdas/filenameprocessor"
       - "/lambdas/ack_backend"
+      - "/lambdas/batch_processor_filter"
       - "/lambdas/delta_backend"
+      - "/lambdas/filenameprocessor"
+      - "/lambdas/mesh_processor"
       - "/lambdas/recordprocessor"
-      - "/mesh_processor"
       - "/sandbox"
     schedule:
       interval: "daily"
@@ -50,14 +50,14 @@ updates:
     directories:
       - "/"
       - "/backend"
-      - "/batch_processor_filter"
-      - "/mesh_processor"
-      - "/lambdas/recordprocessor"
       - "/lambdas/ack_backend"
+      - "/lambdas/batch_processor_filter"
       - "/lambdas/delta_backend"
       - "/lambdas/filenameprocessor"
       - "/lambdas/id_sync"
+      - "/lambdas/mesh_processor"
       - "/lambdas/mns_subscription"
+      - "/lambdas/recordprocessor"
       - "/lambdas/redis_sync"
       - "/lambdas/shared"
       - "/tests/e2e"

.github/workflows/quality-checks.yml

@@ -97,17 +97,6 @@ jobs:
           aws configure set aws_access_key_id $AWS_ACCESS_KEY_ID
           aws configure set aws_secret_access_key $AWS_SECRET_ACCESS_KEY
 
-      - name: Run unittest with batchprocessorfilter-coverage
-        working-directory: batch_processor_filter
-        id: batchprocessorfilter
-        env:
-          PYTHONPATH: ${{ github.workspace }}/batch_processor_filter/src:${{ github.workspace }}/batch_processor_filter/tests
-        continue-on-error: true
-        run: |
-          poetry install
-          poetry run coverage run -m unittest discover || echo "batchprocessorfilter tests failed" >> ../failed_tests.txt
-          poetry run coverage xml -o ../batchprocessorfilter-coverage.xml
-
       - name: Run unittest with recordprocessor-coverage
         working-directory: lambdas/recordprocessor
         id: recordprocessor
@@ -128,7 +117,7 @@ jobs:
         continue-on-error: true
         run: |
           poetry install
-          poetry run coverage run -m unittest discover -p "*batch*.py" || echo "recordforwarder tests failed" >> ../failed_tests.txt
+          poetry run coverage run --source=src -m unittest discover -p "*batch*.py" || echo "recordforwarder tests failed" >> ../failed_tests.txt
           poetry run coverage xml -o ../recordforwarder-coverage.xml
 
       - name: Run unittest with coverage-fhir-api
@@ -139,18 +128,9 @@ jobs:
         continue-on-error: true
         run: |
           poetry install
-          poetry run coverage run -m unittest discover || echo "fhir-api tests failed" >> ../failed_tests.txt
+          poetry run coverage run --source=src -m unittest discover || echo "fhir-api tests failed" >> ../failed_tests.txt
           poetry run coverage xml -o ../backend-coverage.xml
 
-      - name: Run unittest with coverage-mesh-processor
-        working-directory: mesh_processor
-        id: meshprocessor
-        continue-on-error: true
-        run: |
-          poetry install
-          poetry run coverage run -m unittest discover || echo "mesh_processor tests failed" >> ../failed_tests.txt
-          poetry run coverage xml -o ../mesh_processor-coverage.xml
-
       - name: Run unittest with coverage-ack-lambda
         working-directory: lambdas/ack_backend
         id: acklambda
@@ -162,6 +142,17 @@ jobs:
           poetry run coverage run --source=src -m unittest discover || echo "ack-lambda tests failed" >> ../../failed_tests.txt
           poetry run coverage xml -o ../../ack-lambda-coverage.xml
 
+      - name: Run unittest with batchprocessorfilter-coverage
+        working-directory: lambdas/batch_processor_filter
+        id: batchprocessorfilter
+        env:
+          PYTHONPATH: ${{ env.LAMBDA_PATH }}/batch_processor_filter/src:${{ env.LAMBDA_PATH }}/batch_processor_filter/tests:${{ env.SHARED_PATH }}/src
+        continue-on-error: true
+        run: |
+          poetry install
+          poetry run coverage run --source=src -m unittest discover || echo "batchprocessorfilter tests failed" >> ../../failed_tests.txt
+          poetry run coverage xml -o ../../batchprocessorfilter-coverage.xml
+
       - name: Run unittest with coverage-delta
         working-directory: lambdas/delta_backend
         id: delta
@@ -170,7 +161,7 @@ jobs:
         continue-on-error: true
         run: |
           poetry install
-          poetry run coverage run -m unittest discover || echo "delta tests failed" >> ../../failed_tests.txt
+          poetry run coverage run --source=src -m unittest discover || echo "delta tests failed" >> ../../failed_tests.txt
           poetry run coverage xml -o ../../delta-coverage.xml
 
       - name: Run unittest with filenameprocessor-coverage
@@ -181,7 +172,7 @@ jobs:
         continue-on-error: true
         run: |
           poetry install
-          poetry run coverage run -m unittest discover || echo "filenameprocessor tests failed" >> ../../failed_tests.txt
+          poetry run coverage run --source=src -m unittest discover || echo "filenameprocessor tests failed" >> ../../failed_tests.txt
           poetry run coverage xml -o ../../filenameprocessor-coverage.xml
 
       - name: Run unittest with id_sync
@@ -195,6 +186,15 @@ jobs:
           poetry run coverage run --rcfile=.coveragerc --source=src -m unittest discover || echo "id_sync tests failed" >> ../../failed_tests.txt
           poetry run coverage xml -o ../../id_sync-coverage.xml
 
+      - name: Run unittest with coverage-mesh-processor
+        working-directory: lambdas/mesh_processor
+        id: meshprocessor
+        continue-on-error: true
+        run: |
+          poetry install
+          poetry run coverage run --source=src -m unittest discover || echo "mesh_processor tests failed" >> ../../failed_tests.txt
+          poetry run coverage xml -o ../../mesh_processor-coverage.xml
+
       - name: Run unittest with coverage-mns-subscription
         working-directory: lambdas/mns_subscription
         id: mns_subscription
@@ -203,7 +203,7 @@ jobs:
         continue-on-error: true
         run: |
           poetry install
-          poetry run coverage run -m unittest discover || echo "mns_subscription tests failed" >> ../../failed_tests.txt
+          poetry run coverage run --source=src -m unittest discover || echo "mns_subscription tests failed" >> ../../failed_tests.txt
           poetry run coverage report -m
           poetry run coverage xml -o ../../mns_subscription-coverage.xml
 

Makefile

@@ -1,6 +1,6 @@
 SHELL=/usr/bin/env bash -euo pipefail
 
-PYTHON_PROJECT_DIRS_WITH_UNIT_TESTS = backend batch_processor_filter mesh_processor lambdas/ack_backend lambdas/delta_backend lambdas/filenameprocessor lambdas/id_sync lambdas/mns_subscription lambdas/recordprocessor lambdas/redis_sync lambdas/shared
+PYTHON_PROJECT_DIRS_WITH_UNIT_TESTS = backend lambdas/ack_backend lambdas/batch_processor_filter lambdas/delta_backend lambdas/filenameprocessor lambdas/id_sync lambdas/mesh_processor lambdas/mns_subscription lambdas/recordprocessor lambdas/redis_sync lambdas/shared
 PYTHON_PROJECT_DIRS = tests/e2e tests/e2e_batch quality_checks $(PYTHON_PROJECT_DIRS_WITH_UNIT_TESTS)
 
 .PHONY: install lint format format-check clean publish build-proxy release initialise-all-python-venvs update-all-python-dependencies run-all-python-unit-tests build-all-docker-images

backend/Makefile

@@ -9,7 +9,7 @@ test:
 	@PYTHONPATH=src:tests python -m unittest
 
 coverage-run:
-	@PYTHONPATH=src:tests coverage run -m unittest discover
+	@PYTHONPATH=src:tests coverage run --source=src -m unittest discover
 
 coverage-report:
 	coverage report -m 

backend/src/constants.py

@@ -11,7 +11,7 @@ class Urls:
 
     nhs_number = "https://fhir.nhs.uk/Id/nhs-number"
     vaccination_procedure = "https://fhir.hl7.org.uk/StructureDefinition/Extension-UKCore-VaccinationProcedure"
-    snomed = "http://snomed.info/sct"
+    snomed = "http://snomed.info/sct"  # NOSONAR(S5332)
     nhs_number_verification_status_structure_definition = (
         "https://fhir.hl7.org.uk/StructureDefinition/Extension-UKCore-NHSNumberVerificationStatus"
     )

backend/src/models/fhir_immunization_pre_validators.py

@@ -96,6 +96,7 @@ def validate(self):
             self.pre_validate_value_codeable_concept,
             self.pre_validate_extension_length,
             self.pre_validate_vaccination_procedure_code,
+            self.pre_validate_vaccination_procedure_display,
             self.pre_validate_vaccine_code,
         ]
 
@@ -590,7 +591,7 @@ def pre_validate_vaccination_procedure_code(self, values: dict) -> dict:
         (legacy CSV field name: VACCINATION_PROCEDURE_CODE) exists, then it is a non-empty string
         """
         url = "https://fhir.hl7.org.uk/StructureDefinition/Extension-UKCore-" + "VaccinationProcedure"
-        system = "http://snomed.info/sct"
+        system = Urls.snomed
         field_type = "code"
         field_location = generate_field_location_for_extension(url, system, field_type)
         try:
@@ -600,14 +601,30 @@ def pre_validate_vaccination_procedure_code(self, values: dict) -> dict:
         except (KeyError, IndexError):
             pass
 
+    def pre_validate_vaccination_procedure_display(self, values: dict) -> dict:
+        """
+        Pre-validate that, if extension[?(@.url=='https://fhir.hl7.org.uk/StructureDefinition/Extension-UKCore-
+        VaccinationProcedure')].valueCodeableConcept.coding[?(@.system=='http://snomed.info/sct')].display
+        (legacy CSV field name: VACCINATION_PROCEDURE_TERM) exists, then it is a non-empty string
+        """
+        url = "https://fhir.hl7.org.uk/StructureDefinition/Extension-UKCore-" + "VaccinationProcedure"
+        system = Urls.snomed
+        field_type = "display"
+        field_location = generate_field_location_for_extension(url, system, field_type)
+        try:
+            field_value = get_generic_extension_value(values, url, system, field_type)
+            PreValidation.for_string(field_value, field_location)
+        except (KeyError, IndexError):
+            pass
+
     def pre_validate_vaccination_situation_code(self, values: dict) -> dict:
         """
         Pre-validate that, if extension[?(@.url=='https://fhir.hl7.org.uk/StructureDefinition/Extension-UKCore-
         VaccinationSituation')].valueCodeableConcept.coding[?(@.system=='http://snomed.info/sct')].code
         (legacy CSV field name: VACCINATION_SITUATION_CODE) exists, then it is a non-empty string
         """
         url = "https://fhir.hl7.org.uk/StructureDefinition/Extension-UKCore-VaccinationSituation"
-        system = "http://snomed.info/sct"
+        system = Urls.snomed
         field_type = "code"
         field_location = generate_field_location_for_extension(url, system, field_type)
         try:
@@ -623,7 +640,7 @@ def pre_validate_vaccination_situation_display(self, values: dict) -> dict:
         (legacy CSV field name: VACCINATION_SITUATION_TERM) exists, then it is a non-empty string
         """
         url = "https://fhir.hl7.org.uk/StructureDefinition/Extension-UKCore-VaccinationSituation"
-        system = "http://snomed.info/sct"
+        system = Urls.snomed
         field_type = "display"
         field_location = generate_field_location_for_extension(url, system, field_type)
         try:
@@ -702,7 +719,7 @@ def pre_validate_disease_type_coding_codes(self, values: dict) -> dict:
         Pre-validate that, if protocolApplied[0].targetDisease[{i}].coding[?(@.system=='http://snomed.info/sct')].code
         exists, then it is a non-empty string
         """
-        url = "http://snomed.info/sct"
+        url = Urls.snomed
         try:
             for i in range(len(values["protocolApplied"][0]["targetDisease"])):
                 field_location = f"protocolApplied[0].targetDisease[{i}].coding[?(@.system=='{url}')].code"
@@ -761,7 +778,7 @@ def pre_validate_site_coding_code(self, values: dict) -> dict:
         Pre-validate that, if site.coding[?(@.system=='http://snomed.info/sct')].code
         (legacy CSV field name: SITE_OF_VACCINATION_CODE) exists, then it is a non-empty string
         """
-        url = "http://snomed.info/sct"
+        url = Urls.snomed
         field_location = f"site.coding[?(@.system=='{url}')].code"
         try:
             site_coding_code = [x for x in values["site"]["coding"] if x.get("system") == url][0]["code"]
@@ -774,7 +791,7 @@ def pre_validate_site_coding_display(self, values: dict) -> dict:
         Pre-validate that, if site.coding[?(@.system=='http://snomed.info/sct')].display
         (legacy CSV field name: SITE_OF_VACCINATION_TERM) exists, then it is a non-empty string
         """
-        url = "http://snomed.info/sct"
+        url = Urls.snomed
         field_location = f"site.coding[?(@.system=='{url}')].display"
         try:
             field_value = [x for x in values["site"]["coding"] if x.get("system") == url][0]["display"]
@@ -795,7 +812,7 @@ def pre_validate_route_coding_code(self, values: dict) -> dict:
         Pre-validate that, if route.coding[?(@.system=='http://snomed.info/sct')].code
         (legacy CSV field name: ROUTE_OF_VACCINATION_CODE) exists, then it is a non-empty string
         """
-        url = "http://snomed.info/sct"
+        url = Urls.snomed
         field_location = f"route.coding[?(@.system=='{url}')].code"
         try:
             field_value = [x for x in values["route"]["coding"] if x.get("system") == url][0]["code"]
@@ -808,7 +825,7 @@ def pre_validate_route_coding_display(self, values: dict) -> dict:
         Pre-validate that, if route.coding[?(@.system=='http://snomed.info/sct')].display
         (legacy CSV field name: ROUTE_OF_VACCINATION_TERM) exists, then it is a non-empty string
         """
-        url = "http://snomed.info/sct"
+        url = Urls.snomed
         field_location = f"route.coding[?(@.system=='{url}')].display"
         try:
             field_value = [x for x in values["route"]["coding"] if x.get("system") == url][0]["display"]
@@ -951,7 +968,7 @@ def pre_validate_vaccine_code(self, values: dict) -> dict:
         NOTE: vaccineCode is a mandatory FHIR field. A value of None will be rejected by the
         FHIR model before pre-validators are run.
         """
-        url = "http://snomed.info/sct"
+        url = Urls.snomed
         field_location = f"vaccineCode.coding[?(@.system=='{url}')].code"
         try:
             field_value = [x for x in values["vaccineCode"]["coding"] if x.get("system") == url][0]["code"]

backend/tests/test_immunization_pre_validator.py

@@ -1404,6 +1404,14 @@ def test_pre_validate_vaccine_code(self):
                 actual_error_messages,
             )
 
+    def test_pre_validate_vaccination_procedure_display(self):
+        """Test test_pre_validate_vaccination_procedure_display accepts valid values and rejects invalid values"""
+        ValidatorModelTests.test_string_value(
+            self,
+            field_location="extension[?(@.url=='https://fhir.hl7.org.uk/StructureDefinition/Extension-UKCore-VaccinationProcedure')].valueCodeableConcept.coding[?(@.system=='http://snomed.info/sct')].display",
+            valid_strings_to_test=["DUMMY"],
+        )
+
 
 class TestImmunizationModelPreValidationRulesForReduceValidation(unittest.TestCase):
     """Test immunization pre validation rules on the FHIR model using the status="reduce validation" data"""

batch_processor_filter/src/logger.py

@@ -7,10 +7,10 @@
             "path": "backend",
         },
         {
-            "path": "mesh_processor",
+            "path": "lambdas/ack_backend",
         },
         {
-            "path": "lambdas/ack_backend",
+            "path": "lambdas/batch_processor_filter",
         },
         {
             "path": "lambdas/delta_backend",
@@ -21,6 +21,9 @@
         {
             "path": "lambdas/id_sync",
         },
+        {
+            "path": "lambdas/mesh_processor",
+        },
         {
             "path": "lambdas/mns_subscription",
         },