Merge branch 'master' into VED-815-coverage-runs

Author: JamesW1-NHS

.github/dependabot.yml

@@ -7,9 +7,9 @@ version: 2
 updates:
   - package-ecosystem: "docker"
     directories:
-      - "/batch_processor_filter"
       - "/infrastructure/grafana/non-prod/docker"
       - "/lambdas/ack_backend"
+      - "/lambdas/batch_processor_filter"
       - "/lambdas/delta_backend"
       - "/lambdas/filenameprocessor"
       - "/lambdas/mesh_processor"
@@ -50,8 +50,8 @@ updates:
     directories:
       - "/"
       - "/backend"
-      - "/batch_processor_filter"
       - "/lambdas/ack_backend"
+      - "/lambdas/batch_processor_filter"
       - "/lambdas/delta_backend"
       - "/lambdas/filenameprocessor"
       - "/lambdas/id_sync"

.github/workflows/quality-checks.yml

@@ -97,17 +97,6 @@ jobs:
           aws configure set aws_access_key_id $AWS_ACCESS_KEY_ID
           aws configure set aws_secret_access_key $AWS_SECRET_ACCESS_KEY
 
-      - name: Run unittest with batchprocessorfilter-coverage
-        working-directory: batch_processor_filter
-        id: batchprocessorfilter
-        env:
-          PYTHONPATH: ${{ github.workspace }}/batch_processor_filter/src:${{ github.workspace }}/batch_processor_filter/tests
-        continue-on-error: true
-        run: |
-          poetry install
-          poetry run coverage run --source=src -m unittest discover || echo "batchprocessorfilter tests failed" >> ../failed_tests.txt
-          poetry run coverage xml -o ../batchprocessorfilter-coverage.xml
-
       - name: Run unittest with recordprocessor-coverage
         working-directory: lambdas/recordprocessor
         id: recordprocessor
@@ -153,6 +142,17 @@ jobs:
           poetry run coverage run --source=src -m unittest discover || echo "ack-lambda tests failed" >> ../../failed_tests.txt
           poetry run coverage xml -o ../../ack-lambda-coverage.xml
 
+      - name: Run unittest with batchprocessorfilter-coverage
+        working-directory: lambdas/batch_processor_filter
+        id: batchprocessorfilter
+        env:
+          PYTHONPATH: ${{ env.LAMBDA_PATH }}/batch_processor_filter/src:${{ env.LAMBDA_PATH }}/batch_processor_filter/tests:${{ env.SHARED_PATH }}/src
+        continue-on-error: true
+        run: |
+          poetry install
+          poetry run coverage run --source=src -m unittest discover || echo "batchprocessorfilter tests failed" >> ../../failed_tests.txt
+          poetry run coverage xml -o ../../batchprocessorfilter-coverage.xml
+
       - name: Run unittest with coverage-delta
         working-directory: lambdas/delta_backend
         id: delta

Makefile

@@ -1,6 +1,6 @@
 SHELL=/usr/bin/env bash -euo pipefail
 
-PYTHON_PROJECT_DIRS_WITH_UNIT_TESTS = backend batch_processor_filter lambdas/ack_backend lambdas/delta_backend lambdas/filenameprocessor lambdas/id_sync lambdas/mesh_processor lambdas/mns_subscription lambdas/recordprocessor lambdas/redis_sync lambdas/shared
+PYTHON_PROJECT_DIRS_WITH_UNIT_TESTS = backend lambdas/ack_backend lambdas/batch_processor_filter lambdas/delta_backend lambdas/filenameprocessor lambdas/id_sync lambdas/mesh_processor lambdas/mns_subscription lambdas/recordprocessor lambdas/redis_sync lambdas/shared
 PYTHON_PROJECT_DIRS = tests/e2e tests/e2e_batch quality_checks $(PYTHON_PROJECT_DIRS_WITH_UNIT_TESTS)
 
 .PHONY: install lint format format-check clean publish build-proxy release initialise-all-python-venvs update-all-python-dependencies run-all-python-unit-tests build-all-docker-images

backend/src/models/fhir_immunization_pre_validators.py

@@ -96,6 +96,7 @@ def validate(self):
             self.pre_validate_value_codeable_concept,
             self.pre_validate_extension_length,
             self.pre_validate_vaccination_procedure_code,
+            self.pre_validate_vaccination_procedure_display,
             self.pre_validate_vaccine_code,
         ]
 
@@ -590,7 +591,7 @@ def pre_validate_vaccination_procedure_code(self, values: dict) -> dict:
         (legacy CSV field name: VACCINATION_PROCEDURE_CODE) exists, then it is a non-empty string
         """
         url = "https://fhir.hl7.org.uk/StructureDefinition/Extension-UKCore-" + "VaccinationProcedure"
-        system = "http://snomed.info/sct"
+        system = Urls.snomed
         field_type = "code"
         field_location = generate_field_location_for_extension(url, system, field_type)
         try:
@@ -600,14 +601,30 @@ def pre_validate_vaccination_procedure_code(self, values: dict) -> dict:
         except (KeyError, IndexError):
             pass
 
+    def pre_validate_vaccination_procedure_display(self, values: dict) -> dict:
+        """
+        Pre-validate that, if extension[?(@.url=='https://fhir.hl7.org.uk/StructureDefinition/Extension-UKCore-
+        VaccinationProcedure')].valueCodeableConcept.coding[?(@.system=='http://snomed.info/sct')].display
+        (legacy CSV field name: VACCINATION_PROCEDURE_TERM) exists, then it is a non-empty string
+        """
+        url = "https://fhir.hl7.org.uk/StructureDefinition/Extension-UKCore-" + "VaccinationProcedure"
+        system = Urls.snomed
+        field_type = "display"
+        field_location = generate_field_location_for_extension(url, system, field_type)
+        try:
+            field_value = get_generic_extension_value(values, url, system, field_type)
+            PreValidation.for_string(field_value, field_location)
+        except (KeyError, IndexError):
+            pass
+
     def pre_validate_vaccination_situation_code(self, values: dict) -> dict:
         """
         Pre-validate that, if extension[?(@.url=='https://fhir.hl7.org.uk/StructureDefinition/Extension-UKCore-
         VaccinationSituation')].valueCodeableConcept.coding[?(@.system=='http://snomed.info/sct')].code
         (legacy CSV field name: VACCINATION_SITUATION_CODE) exists, then it is a non-empty string
         """
         url = "https://fhir.hl7.org.uk/StructureDefinition/Extension-UKCore-VaccinationSituation"
-        system = "http://snomed.info/sct"
+        system = Urls.snomed
         field_type = "code"
         field_location = generate_field_location_for_extension(url, system, field_type)
         try:
@@ -623,7 +640,7 @@ def pre_validate_vaccination_situation_display(self, values: dict) -> dict:
         (legacy CSV field name: VACCINATION_SITUATION_TERM) exists, then it is a non-empty string
         """
         url = "https://fhir.hl7.org.uk/StructureDefinition/Extension-UKCore-VaccinationSituation"
-        system = "http://snomed.info/sct"
+        system = Urls.snomed
         field_type = "display"
         field_location = generate_field_location_for_extension(url, system, field_type)
         try:
@@ -702,7 +719,7 @@ def pre_validate_disease_type_coding_codes(self, values: dict) -> dict:
         Pre-validate that, if protocolApplied[0].targetDisease[{i}].coding[?(@.system=='http://snomed.info/sct')].code
         exists, then it is a non-empty string
         """
-        url = "http://snomed.info/sct"
+        url = Urls.snomed
         try:
             for i in range(len(values["protocolApplied"][0]["targetDisease"])):
                 field_location = f"protocolApplied[0].targetDisease[{i}].coding[?(@.system=='{url}')].code"
@@ -761,7 +778,7 @@ def pre_validate_site_coding_code(self, values: dict) -> dict:
         Pre-validate that, if site.coding[?(@.system=='http://snomed.info/sct')].code
         (legacy CSV field name: SITE_OF_VACCINATION_CODE) exists, then it is a non-empty string
         """
-        url = "http://snomed.info/sct"
+        url = Urls.snomed
         field_location = f"site.coding[?(@.system=='{url}')].code"
         try:
             site_coding_code = [x for x in values["site"]["coding"] if x.get("system") == url][0]["code"]
@@ -774,7 +791,7 @@ def pre_validate_site_coding_display(self, values: dict) -> dict:
         Pre-validate that, if site.coding[?(@.system=='http://snomed.info/sct')].display
         (legacy CSV field name: SITE_OF_VACCINATION_TERM) exists, then it is a non-empty string
         """
-        url = "http://snomed.info/sct"
+        url = Urls.snomed
         field_location = f"site.coding[?(@.system=='{url}')].display"
         try:
             field_value = [x for x in values["site"]["coding"] if x.get("system") == url][0]["display"]
@@ -795,7 +812,7 @@ def pre_validate_route_coding_code(self, values: dict) -> dict:
         Pre-validate that, if route.coding[?(@.system=='http://snomed.info/sct')].code
         (legacy CSV field name: ROUTE_OF_VACCINATION_CODE) exists, then it is a non-empty string
         """
-        url = "http://snomed.info/sct"
+        url = Urls.snomed
         field_location = f"route.coding[?(@.system=='{url}')].code"
         try:
             field_value = [x for x in values["route"]["coding"] if x.get("system") == url][0]["code"]
@@ -808,7 +825,7 @@ def pre_validate_route_coding_display(self, values: dict) -> dict:
         Pre-validate that, if route.coding[?(@.system=='http://snomed.info/sct')].display
         (legacy CSV field name: ROUTE_OF_VACCINATION_TERM) exists, then it is a non-empty string
         """
-        url = "http://snomed.info/sct"
+        url = Urls.snomed
         field_location = f"route.coding[?(@.system=='{url}')].display"
         try:
             field_value = [x for x in values["route"]["coding"] if x.get("system") == url][0]["display"]
@@ -951,7 +968,7 @@ def pre_validate_vaccine_code(self, values: dict) -> dict:
         NOTE: vaccineCode is a mandatory FHIR field. A value of None will be rejected by the
         FHIR model before pre-validators are run.
         """
-        url = "http://snomed.info/sct"
+        url = Urls.snomed
         field_location = f"vaccineCode.coding[?(@.system=='{url}')].code"
         try:
             field_value = [x for x in values["vaccineCode"]["coding"] if x.get("system") == url][0]["code"]

backend/tests/test_immunization_pre_validator.py

@@ -1404,6 +1404,14 @@ def test_pre_validate_vaccine_code(self):
                 actual_error_messages,
             )
 
+    def test_pre_validate_vaccination_procedure_display(self):
+        """Test test_pre_validate_vaccination_procedure_display accepts valid values and rejects invalid values"""
+        ValidatorModelTests.test_string_value(
+            self,
+            field_location="extension[?(@.url=='https://fhir.hl7.org.uk/StructureDefinition/Extension-UKCore-VaccinationProcedure')].valueCodeableConcept.coding[?(@.system=='http://snomed.info/sct')].display",
+            valid_strings_to_test=["DUMMY"],
+        )
+
 
 class TestImmunizationModelPreValidationRulesForReduceValidation(unittest.TestCase):
     """Test immunization pre validation rules on the FHIR model using the status="reduce validation" data"""

batch_processor_filter/src/logger.py

@@ -9,6 +9,9 @@
         {
             "path": "lambdas/ack_backend",
         },
+        {
+            "path": "lambdas/batch_processor_filter",
+        },
         {
             "path": "lambdas/delta_backend",
         },

batch_processor_filter/src/send_log_to_firehose.py

@@ -1,6 +1,6 @@
 # Define the directory containing the Docker image and calculate its SHA-256 hash for triggering redeployments
 locals {
-  batch_processor_filter_lambda_dir     = abspath("${path.root}/../../batch_processor_filter")
+  batch_processor_filter_lambda_dir     = abspath("${path.root}/../../lambdas/batch_processor_filter")
   batch_processor_filter_lambda_files   = fileset(local.batch_processor_filter_lambda_dir, "**")
   batch_processor_filter_lambda_dir_sha = sha1(join("", [for f in local.batch_processor_filter_lambda_files : filesha1("${local.batch_processor_filter_lambda_dir}/${f}")]))
 }
@@ -15,8 +15,9 @@ resource "aws_ecr_repository" "batch_processor_filter_lambda_repository" {
 
 # Module for building and pushing Docker image to ECR
 module "batch_processor_filter_docker_image" {
-  source  = "terraform-aws-modules/lambda/aws//modules/docker-build"
-  version = "8.1.2"
+  source           = "terraform-aws-modules/lambda/aws//modules/docker-build"
+  version          = "8.1.2"
+  docker_file_path = "./batch_processor_filter/Dockerfile"
 
   create_ecr_repo = false
   ecr_repo        = aws_ecr_repository.batch_processor_filter_lambda_repository.name
@@ -39,9 +40,10 @@ module "batch_processor_filter_docker_image" {
 
   platform      = "linux/amd64"
   use_image_tag = false
-  source_path   = local.batch_processor_filter_lambda_dir
+  source_path   = abspath("${path.root}/../../lambdas")
   triggers = {
-    dir_sha = local.batch_processor_filter_lambda_dir_sha
+    dir_sha        = local.batch_processor_filter_lambda_dir_sha
+    shared_dir_sha = local.shared_dir_sha
   }
 }
 

immunisation-fhir-api.code-workspace

@@ -5,12 +5,31 @@ RUN mkdir -p /home/appuser && \
     echo 'appuser:x:1001:' >> /etc/group && \
     chown -R 1001:1001 /home/appuser && pip install "poetry~=2.1.4"
 
-COPY poetry.lock pyproject.toml README.md ./
+# Install Poetry dependencies
+# Copy batch_processor_filter Poetry files
+COPY ./batch_processor_filter/poetry.lock ./batch_processor_filter/pyproject.toml ./
+
+# Install batch_processor_filter dependencies
+WORKDIR /var/task
 RUN poetry config virtualenvs.create false && poetry install --no-interaction --no-ansi --no-root --only main
+
 # -----------------------------
 FROM base AS build
-COPY src .
+
+# Set working directory back to Lambda task root
+WORKDIR /var/task
+
+# Copy shared source code
+COPY ./shared/src/common ./common
+
+# Copy batch_processor_filter source code
+COPY ./batch_processor_filter/src .
+
+# Set correct permissions
 RUN chmod 644 $(find . -type f) && chmod 755 $(find . -type d)
+
 # Switch to the non-root user for running the container
 USER 1001:1001
+
+# Set the Lambda handler
 CMD ["lambda_handler.lambda_handler"]