init: move recordforwarder, copy src/models

Author: JamesW1-NHS

.github/dependabot.yml

@@ -13,6 +13,7 @@ updates:
       - "/lambdas/delta_backend"
       - "/lambdas/filenameprocessor"
       - "/lambdas/mesh_processor"
+      - "/lambdas/recordforwarder"
       - "/lambdas/recordprocessor"
       - "/sandbox"
     schedule:
@@ -57,6 +58,7 @@ updates:
       - "/lambdas/id_sync"
       - "/lambdas/mesh_processor"
       - "/lambdas/mns_subscription"
+      - "/lambdas/recordforwarder"
       - "/lambdas/recordprocessor"
       - "/lambdas/redis_sync"
       - "/lambdas/shared"

.github/workflows/quality-checks.yml

@@ -97,29 +97,6 @@ jobs:
           aws configure set aws_access_key_id $AWS_ACCESS_KEY_ID
           aws configure set aws_secret_access_key $AWS_SECRET_ACCESS_KEY
 
-      - name: Run unittest with recordprocessor-coverage
-        working-directory: lambdas/recordprocessor
-        id: recordprocessor
-        env:
-          PYTHONPATH: ${{ env.LAMBDA_PATH }}/recordprocessor/src:${{ env.LAMBDA_PATH }}/recordprocessor/tests:${{ env.SHARED_PATH }}/src
-        continue-on-error: true
-        run: |
-          poetry install
-          poetry run coverage run --source=src -m unittest discover || echo "recordprocessor tests failed" >> ../../failed_tests.txt
-          poetry run coverage xml -o ../../recordprocessor-coverage.xml
-
-        # This step is redundant - all of these tests will be run in the backend step below
-      - name: Run unittest with recordforwarder-coverage
-        working-directory: backend
-        id: recordforwarder
-        env:
-          PYTHONPATH: ${{ github.workspace }}/backend/src:${{ github.workspace }}/backend/tests
-        continue-on-error: true
-        run: |
-          poetry install
-          poetry run coverage run --source=src -m unittest discover -p "*batch*.py" || echo "recordforwarder tests failed" >> ../failed_tests.txt
-          poetry run coverage xml -o ../recordforwarder-coverage.xml
-
       - name: Run unittest with coverage-fhir-api
         working-directory: backend
         env:
@@ -207,6 +184,28 @@ jobs:
           poetry run coverage report -m
           poetry run coverage xml -o ../../mns_subscription-coverage.xml
 
+      - name: Run unittest with recordforwarder-coverage
+        working-directory: lambdas/recordforwarder
+        id: recordforwarder
+        env:
+          PYTHONPATH: ${{ env.LAMBDA_PATH }}/recordforwarder/src:${{ env.LAMBDA_PATH }}/recordforwarder/tests:${{ env.SHARED_PATH }}/src
+        continue-on-error: true
+        run: |
+          poetry install
+          poetry run coverage run --source=src -m unittest discover || echo "recordforwarder tests failed" >> ../../failed_tests.txt
+          poetry run coverage xml -o ../../recordforwarder-coverage.xml
+
+      - name: Run unittest with recordprocessor-coverage
+        working-directory: lambdas/recordprocessor
+        id: recordprocessor
+        env:
+          PYTHONPATH: ${{ env.LAMBDA_PATH }}/recordprocessor/src:${{ env.LAMBDA_PATH }}/recordprocessor/tests:${{ env.SHARED_PATH }}/src
+        continue-on-error: true
+        run: |
+          poetry install
+          poetry run coverage run --source=src -m unittest discover || echo "recordprocessor tests failed" >> ../../failed_tests.txt
+          poetry run coverage xml -o ../../recordprocessor-coverage.xml
+
       - name: Run unittest with redis_sync
         working-directory: lambdas/redis_sync
         id: redis_sync

Makefile

@@ -1,6 +1,6 @@
 SHELL=/usr/bin/env bash -euo pipefail
 
-PYTHON_PROJECT_DIRS_WITH_UNIT_TESTS = backend lambdas/ack_backend lambdas/batch_processor_filter lambdas/delta_backend lambdas/filenameprocessor lambdas/id_sync lambdas/mesh_processor lambdas/mns_subscription lambdas/recordprocessor lambdas/redis_sync lambdas/shared
+PYTHON_PROJECT_DIRS_WITH_UNIT_TESTS = backend lambdas/ack_backend lambdas/batch_processor_filter lambdas/delta_backend lambdas/filenameprocessor lambdas/id_sync lambdas/mesh_processor lambdas/mns_subscription lambdas/recordforwarder lambdas/recordprocessor lambdas/redis_sync lambdas/shared
 PYTHON_PROJECT_DIRS = tests/e2e tests/e2e_batch quality_checks $(PYTHON_PROJECT_DIRS_WITH_UNIT_TESTS)
 
 .PHONY: install lint format format-check clean publish build-proxy release initialise-all-python-venvs update-all-python-dependencies run-all-python-unit-tests build-all-docker-images

immunisation-fhir-api.code-workspace

@@ -27,6 +27,9 @@
         {
             "path": "lambdas/mns_subscription",
         },
+        {
+            "path": "lambdas/recordforwarder",
+        },
         {
             "path": "lambdas/recordprocessor",
         },

infrastructure/instance/forwarder_lambda.tf

@@ -1,13 +1,7 @@
 locals {
-  forwarder_lambda_dir    = abspath("${path.root}/../../backend")
-  forwarder_source_path   = local.forwarder_lambda_dir
-  forwarder_path_include  = ["**"]
-  forwarder_path_exclude  = ["**/__pycache__/**"]
-  forwarder_files_include = setunion([for f in local.forwarder_path_include : fileset(local.forwarder_source_path, f)]...)
-  forwarder_files_exclude = setunion([for f in local.forwarder_path_exclude : fileset(local.forwarder_source_path, f)]...)
-  forwarder_files         = sort(setsubtract(local.forwarder_files_include, local.forwarder_files_exclude))
-
-  forwarder_dir_sha = sha1(join("", [for f in local.forwarder_files : filesha1("${local.forwarder_source_path}/${f}")]))
+  forwarder_lambda_dir     = abspath("${path.root}/../../lambdas/recordforwarder")
+  forwarder_lambda_files   = fileset(local.forwarder_lambda_dir, "**")
+  forwarder_lambda_dir_sha = sha1(join("", [for f in local.forwarder_lambda_files : filesha1("${local.forwarder_lambda_dir}/${f}")]))
 }
 
 resource "aws_ecr_repository" "forwarder_lambda_repository" {
@@ -19,12 +13,12 @@ resource "aws_ecr_repository" "forwarder_lambda_repository" {
 }
 
 module "forwarding_docker_image" {
-  source  = "terraform-aws-modules/lambda/aws//modules/docker-build"
-  version = "8.1.2"
+  source           = "terraform-aws-modules/lambda/aws//modules/docker-build"
+  version          = "8.1.2"
+  docker_file_path = "./recordforwarder/Dockerfile"
 
-  create_ecr_repo  = false
-  ecr_repo         = aws_ecr_repository.forwarder_lambda_repository.name
-  docker_file_path = "batch.Dockerfile"
+  create_ecr_repo = false
+  ecr_repo        = aws_ecr_repository.forwarder_lambda_repository.name
   ecr_repo_lifecycle_policy = jsonencode({
     rules = [
       {
@@ -46,7 +40,8 @@ module "forwarding_docker_image" {
   use_image_tag = false
   source_path   = local.forwarder_lambda_dir
   triggers = {
-    dir_sha = local.forwarder_dir_sha
+    dir_sha        = local.forwarder_lambda_dir_sha
+    shared_dir_sha = local.shared_dir_sha
   }
 }
 

backend/batch.Dockerfile lambdas/recordforwarder/Dockerfilebackend/batch.Dockerfile renamed to lambdas/recordforwarder/Dockerfile

@@ -1,23 +1,35 @@
 FROM public.ecr.aws/lambda/python:3.11 AS base
-# Create a non-root user with a specific UID and GID
+
 RUN mkdir -p /home/appuser && \
     echo 'appuser:x:1001:1001::/home/appuser:/sbin/nologin' >> /etc/passwd && \
     echo 'appuser:x:1001:' >> /etc/group && \
     chown -R 1001:1001 /home/appuser && pip install "poetry~=2.1.4"
 
-# -----------------------------
-COPY poetry.lock pyproject.toml README.md ./
+# Install Poetry dependencies
+# Copy recordforwarder Poetry files
+COPY ./recordforwarder/poetry.lock ./recordforwarder/pyproject.toml ./
+
+# Install recordforwarder dependencies
+WORKDIR /var/task
 RUN poetry config virtualenvs.create false && poetry install --no-interaction --no-ansi --no-root --only main
-# -----------------------------
-FROM base AS test
-COPY src src
-COPY tests tests
-RUN poetry install --no-interaction --no-ansi --no-root && \
-    pytest --disable-warnings tests
+
 # -----------------------------
 FROM base AS build
-COPY src .
+
+# Set working directory back to Lambda task root
+WORKDIR /var/task
+
+# Copy shared source code
+COPY ./shared/src/common ./common
+
+# Copy recordforwarder source code
+COPY ./recordforwarder/src .
+
+# Set correct permissions
 RUN chmod 644 $(find . -type f) && chmod 755 $(find . -type d)
+
 # Switch to the non-root user for running the container
 USER 1001:1001
+
+# Set the Lambda handler
 CMD ["forwarding_batch_lambda.forward_lambda_handler"]

lambdas/recordforwarder/Makefile

@@ -0,0 +1,22 @@
+TEST_ENV := @PYTHONPATH=src:tests:../shared/src
+
+build:
+	docker build -t imms-lambda-build -f Dockerfile .
+
+package: build
+	mkdir -p build
+	docker run --rm -v $(shell pwd)/build:/build imms-lambda-build
+
+test:
+	$(TEST_ENV) python -m unittest
+
+coverage-run:
+	$(TEST_ENV) coverage run --source=src -m unittest discover
+
+coverage-report:
+	$(TEST_ENV) coverage report -m
+
+coverage-html:
+	$(TEST_ENV) coverage html
+
+.PHONY: build package test

lambdas/recordforwarder/README.md

@@ -0,0 +1,24 @@
+# About
+
+This document describes the environment setup for the recordforwarder Lambda.
+
+## Setting up the recordforwarder lambda
+
+Note: Paths are relative to this directory, `recordforwarder`.
+
+1. Follow the instructions in the root level README.md to setup the [dependencies](../README.md#environment-setup) and create a [virtual environment](../README.md#) for this folder.
+
+2. Replace the `.env` file in the recordforwarder folder. Note the variables might change in the future. These environment variables will be loaded automatically when using `direnv`.
+
+    ```
+    AWS_PROFILE={your-profile}
+    IMMUNIZATION_ENV={environment}
+    ```
+
+3. Run `poetry install --no-root` to install dependencies.
+
+4. Run `make test` to run unit tests or individual tests by running:
+    ```
+    python -m unittest tests.test_fhir_controller.TestSearchImmunizations
+    python -m unittest tests.test_fhir_controller.TestSearchImmunizations.test_search_immunizations
+    ```

lambdas/recordforwarder/poetry.lock

@@ -0,0 +1,35 @@
+[tool.poetry]
+name        = "recordforwarder"
+version     = "0.1.0"
+description = ""
+authors     = ["Your Name <[email protected]>"]
+readme      = "README.md"
+packages = [
+    {include = "src"},
+    {include = "common", from = "../shared/src"}
+]
+
+[tool.poetry.dependencies]
+python                  = "~3.11"
+"fhir.resources"        = "~7.0.2"
+boto3                   = "~1.40.64"
+boto3-stubs-lite        = {extras = ["dynamodb"], version = "~1.40.64"}
+aws-lambda-typing       = "~2.20.0"
+redis                   = "^4.6.0"
+moto                    = "^5.1.16"
+requests                = "~2.32.5"
+responses               = "~0.25.7"
+pydantic                = "~1.10.13"
+pyjwt                   = "~2.10.1"
+cryptography            = "~46.0.0"
+cffi                    = "~1.17.1"
+jsonpath-ng             = "^1.6.0"
+simplejson              = "^3.20.2"
+structlog               = "^24.1.0"
+python-stdnum           = "^2.1"
+freezegun               = "^1.5.1"
+coverage                = "^7.10.7"
+
+[build-system]
+requires      = ["poetry-core ~= 1.5.0"]
+build-backend = "poetry.core.masonry.api"