Merge branch 'master' into VED-83-int-ref

Author: nhsdevws

.github/workflows/sonarcloud.yml

@@ -13,7 +13,6 @@ jobs:
     if: ${{ !contains(github.event.pull_request.labels.*.name, 'dependency') }}
     runs-on: ubuntu-latest
 
-    
     steps:
       - uses: actions/checkout@v4
         with:
@@ -26,10 +25,10 @@ jobs:
         run: |
           aws configure set aws_access_key_id $AWS_ACCESS_KEY_ID
           aws configure set aws_secret_access_key $AWS_SECRET_ACCESS_KEY
-      
+
       - name: Initialize Unit Test Failure Tracker
         run: echo "false" > test_failed.txt
-          
+
       - name: Run unittest with filenameprocessor-coverage
         id: filenameprocessor
         continue-on-error: true
@@ -38,23 +37,20 @@ jobs:
           poetry run coverage run --source=filenameprocessor -m unittest discover -s filenameprocessor || echo "filenameprocessor tests failed" >> failed_tests.txt
           poetry run coverage xml -o sonarcloud-coverage-filenameprocessor-coverage.xml
 
-      
       - name: Run unittest with recordprocessor-coverage
         id: recordprocessor
         continue-on-error: true
         run: |
           poetry run coverage run --source=recordprocessor -m unittest discover -s recordprocessor || echo "recordprocessor tests failed" >> failed_tests.txt
           poetry run coverage xml -o sonarcloud-coverage-recordprocessor-coverage.xml
 
-
       - name: Run unittest with recordforwarder-coverage
         id: recordforwarder
         continue-on-error: true
         run: |
           PYTHONPATH=$(pwd)/backend:$(pwd)/backend/tests poetry run coverage run --source=backend -m unittest discover -s backend/tests -p "*batch*.py" || echo "recordforwarder tests failed" >> failed_tests.txt
           poetry run coverage xml -o sonarcloud-coverage-recordforwarder-coverage.xml
 
-          
       - name: Run unittest with coverage-ack-lambda
         id: acklambda
         continue-on-error: true
@@ -70,7 +66,7 @@ jobs:
           pip install poetry==1.8.4 mypy-boto3-dynamodb==1.35.54 boto3==1.26.165 coverage botocore==1.29.165 jmespath==1.0.1 python-dateutil==2.9.0 urllib3==1.26.20 s3transfer==0.6.2 typing-extensions==4.12.2
           poetry run coverage run --source=delta_backend -m unittest discover -s delta_backend || echo "delta tests failed" >> failed_tests.txt
           poetry run coverage xml -o sonarcloud-coverage-delta.xml
-      
+
       - name: Run unittest with coverage-fhir-api
         id: fhirapi
         continue-on-error: true
@@ -79,6 +75,14 @@ jobs:
           poetry run coverage run --source=backend -m unittest discover -s backend || echo "fhir-api tests failed" >> failed_tests.txt
           poetry run coverage xml -o sonarcloud-coverage.xml
 
+      - name: Run unittest with coverage-mesh-processor
+        id: meshprocessor
+        continue-on-error: true
+        run: |
+          pip install poetry==1.8.4 moto==4.2.11 coverage redis botocore==1.35.49 simplejson responses structlog fhir.resources jsonpath_ng pydantic==1.10.13 requests aws-lambda-typing cffi pyjwt boto3-stubs-lite[dynamodb]~=1.26.90 python-stdnum==1.20
+          poetry run coverage run --source=mesh_processor -m unittest discover -s mesh_processor || echo "mesh_processor tests failed" >> failed_tests.txt
+          poetry run coverage xml -o sonarcloud-mesh_processor-coverage.xml
+
       - name: Run Test Failure Summary
         id: check_failure
         run: |
@@ -98,5 +102,5 @@ jobs:
       - name: SonarCloud Scan
         uses: SonarSource/sonarqube-scan-action@master
         env:
-          GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}  # Needed to get PR information, if any
-          SONAR_TOKEN: ${{ secrets.SONAR_TOKEN }}
+          GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} # Needed to get PR information, if any
+          SONAR_TOKEN: ${{ secrets.SONAR_TOKEN }}

Makefile

@@ -38,7 +38,7 @@ build-proxy:
 	scripts/build_proxy.sh
 
 #Files to loop over in release
-_dist_include="pytest.ini poetry.lock poetry.toml pyproject.toml Makefile build/. e2e e2e_batch specification sandbox terraform scripts backend delta_backend ack_backend filenameprocessor recordprocessor"
+_dist_include="pytest.ini poetry.lock poetry.toml pyproject.toml Makefile build/. e2e e2e_batch specification sandbox terraform scripts backend delta_backend ack_backend filenameprocessor recordprocessor mesh_processor"
 
 
 #Create /dist/ sub-directory and copy files into directory

filenameprocessor/src/file_key_validation.py

@@ -54,7 +54,7 @@ def validate_file_key(file_key: str) -> tuple[str, str]:
         and version in Constants.VALID_VERSIONS
         and supplier  # Note that if supplier could be identified, this also implies that ODS code is valid
         and is_valid_datetime(timestamp)
-        and extension == "CSV"
+        and ((extension == "CSV") or (extension == "DAT"))  # The DAT extension has been added for MESH file processing
     ):
         raise InvalidFileKeyError("Initial file validation failed: invalid file key")
 

filenameprocessor/tests/test_file_key_validation.py

@@ -93,7 +93,7 @@ def test_validate_file_key(self):
             # File key with missing timestamp
             (VALID_FLU_EMIS_FILE_KEY.replace("20000101T00000001", ""), invalid_file_key_error_message),
             # File key with incorrect extension
-            (VALID_FLU_EMIS_FILE_KEY.replace(".csv", ".dat"), invalid_file_key_error_message),
+            (VALID_FLU_EMIS_FILE_KEY.replace(".csv", ".xlsx"), invalid_file_key_error_message),
         ]
 
         for file_key, expected_result in test_cases_for_failure_scenarios:

immunisation-fhir-api.code-workspace

@@ -17,8 +17,10 @@
 		},
 		{
 			"path": "delta_backend"
+		},
+		{
+			"path": "mesh_processor"
 		}
 	],
 	"settings": {},
-	
-}
+}

mesh-infra/main.tf

@@ -0,0 +1,41 @@
+terraform {
+  required_providers {
+    aws = {
+      source  = "hashicorp/aws"
+      version = "~> 5"
+    }    
+  }
+  backend "s3" {
+    region = "eu-west-2"
+    key    = "state"
+  }
+    required_version = ">= 1.5.0"
+}
+
+
+
+data "aws_vpc" "default" {
+    default = true
+}
+data "aws_subnets" "default" {
+    filter {
+        name   = "vpc-id"
+        values = [data.aws_vpc.default.id]
+    }
+}
+
+module "mesh" {
+  source = "git::https://github.com/nhsdigital/terraform-aws-mesh-client.git//module?ref=v2.1.5"
+
+  name_prefix                    = "local-immunisation"
+  mesh_env                       = "integration"
+  subnet_ids                     = data.aws_subnets.default.ids
+
+  mailbox_ids                    = ["X26OT303"]
+  verify_ssl                     = "true"
+  get_message_max_concurrency    = 10
+  compress_threshold             = 1 * 1024 * 1024
+  handshake_schedule             = "rate(24 hours)"
+
+  account_id                     = 345594581768
+}

mesh_processor/Dockerfile

@@ -0,0 +1,22 @@
+FROM public.ecr.aws/lambda/python:3.10 AS base
+
+# Create a non-root user with a specific UID and GID
+RUN mkdir -p /home/appuser && \
+    echo 'appuser:x:1001:1001::/home/appuser:/sbin/nologin' >> /etc/passwd && \
+    echo 'appuser:x:1001:' >> /etc/group && \
+    chown -R 1001:1001 /home/appuser && pip install "poetry~=1.5.0"
+
+
+
+COPY poetry.lock pyproject.toml README.md ./
+RUN poetry config virtualenvs.create false && poetry install --no-interaction --no-ansi --no-root --only main
+
+
+# -----------------------------
+FROM base AS build
+
+COPY src .
+RUN chmod 644 $(find . -type f) && chmod 755 $(find . -type d)
+# Switch to the non-root user for running the container
+USER 1001:1001
+CMD ["converter.lambda_handler"]

mesh_processor/Makefile

@@ -0,0 +1,11 @@
+build:
+	docker build -t mesh-lambda-build .
+
+package: build
+	mkdir -p build
+	docker run --rm -v $(shell pwd)/build:/build mesh-lambda-build
+
+test:
+	python -m unittest
+
+.PHONY: build package test