VED-788 Refactor ack_backend (#844)

* init

* lint

* smells

* typo

* imports

* cleanup

* sonarcloud

* sonarcloud II

* local Makefile

* tests for get_s3_client

* tests for get_s3_client II

* delete .coverage

* use_ms_precision

* lint

* shared helper functions

* prefix

* missing cache tests

* test_errors

* terraform fixes per VED-810

* review fixes

* review fixes II

* review fixes III

Author: JamesW1-NHS

.github/dependabot.yml

@@ -7,7 +7,7 @@ version: 2
 updates:
   - package-ecosystem: "docker"
     directories:
-      - "/ack_backend"
+      - "/lambdas/ack_backend"
       - "/delta_backend"
       - "/filenameprocessor"
       - "/grafana/non-prod/docker"
@@ -49,7 +49,6 @@ updates:
   - package-ecosystem: "pip"
     directories:
       - "/"
-      - "/ack_backend"
       - "/backend"
       - "/batch_processor_filter"
       - "/delta_backend"
@@ -58,6 +57,7 @@ updates:
       - "/filenameprocessor"
       - "/mesh_processor"
       - "/recordprocessor"
+      - "/lambdas/ack_backend"
       - "/lambdas/redis_sync"
       - "/lambdas/id_sync"
       - "/lambdas/mns_subscription"

.github/workflows/quality-checks.yml

@@ -106,17 +106,6 @@ jobs:
           poetry run coverage run -m unittest discover -p "*batch*.py" || echo "recordforwarder tests failed" >> ../failed_tests.txt
           poetry run coverage xml -o ../recordforwarder-coverage.xml
 
-      - name: Run unittest with coverage-ack-lambda
-        working-directory: ack_backend
-        id: acklambda
-        env:
-          PYTHONPATH: ${{ github.workspace }}/ack_backend/src:${{ github.workspace }}/ack_backend/tests
-        continue-on-error: true
-        run: |
-          poetry install
-          poetry run coverage run -m unittest discover || echo "ack-lambda tests failed" >> ../failed_tests.txt
-          poetry run coverage xml -o ../ack-lambda-coverage.xml
-
       - name: Run unittest with coverage-delta
         working-directory: delta_backend
         id: delta
@@ -148,6 +137,17 @@ jobs:
           poetry run coverage run -m unittest discover || echo "mesh_processor tests failed" >> ../failed_tests.txt
           poetry run coverage xml -o ../mesh_processor-coverage.xml
 
+      - name: Run unittest with coverage-ack-lambda
+        working-directory: lambdas/ack_backend
+        id: acklambda
+        env:
+          PYTHONPATH: ${{ env.LAMBDA_PATH }}/ack_backend/src:${{ github.workspace }}/ack_backend/tests
+        continue-on-error: true
+        run: |
+            poetry install
+            poetry run coverage run --source=src -m unittest discover || echo "ack-lambda tests failed" >> ../../failed_tests.txt
+            poetry run coverage xml -o ../../ack-lambda-coverage.xml
+
       - name: Run unittest with coverage-mns-subscription
         working-directory: lambdas/mns_subscription
         id: mns_subscription

Makefile

@@ -1,6 +1,6 @@
 SHELL=/usr/bin/env bash -euo pipefail
 
-PYTHON_PROJECT_DIRS_WITH_UNIT_TESTS = ack_backend backend batch_processor_filter delta_backend filenameprocessor mesh_processor recordprocessor lambdas/redis_sync lambdas/id_sync lambdas/mns_subscription lambdas/shared
+PYTHON_PROJECT_DIRS_WITH_UNIT_TESTS = backend batch_processor_filter delta_backend filenameprocessor mesh_processor recordprocessor lambdas/ack_backend lambdas/redis_sync lambdas/id_sync lambdas/mns_subscription lambdas/shared
 PYTHON_PROJECT_DIRS = e2e e2e_batch $(PYTHON_PROJECT_DIRS_WITH_UNIT_TESTS)
 
 #Installs dependencies using poetry.

ack_backend/Makefile

@@ -12,9 +12,6 @@
 		{
 			"path": "recordprocessor"
 		},
-		{
-			"path": "ack_backend"
-		},
 		{
 			"path": "delta_backend"
 		},
@@ -27,6 +24,9 @@
 		{
 			"path": "e2e_batch"
 		},
+		{
+			"path": "lambdas/ack_backend"
+		},
         {
             "path": "lambdas/redis_sync"
         },

ack_backend/src/clients.py

@@ -0,0 +1,18 @@
+{
+    "python.analysis.extraPaths": [
+        "./src"
+    ],
+    "python.testing.unittestArgs": [
+        "-v",
+        "-s",
+        "./",
+        "-p",
+        "test_*.py"
+    ],
+    "python.testing.pytestEnabled": false,
+    "python.testing.unittestEnabled": true,
+    "pylint.args": [
+        "--init-hook",
+        "import sys; sys.path.append('./src')"
+    ]
+}

ack_backend/src/errors.py

@@ -5,12 +5,31 @@ RUN mkdir -p /home/appuser && \
     echo 'appuser:x:1001:' >> /etc/group && \
     chown -R 1001:1001 /home/appuser && pip install "poetry~=2.1.4"
 
-COPY poetry.lock pyproject.toml README.md ./
+# Install Poetry dependencies
+# Copy ack_backend Poetry files
+COPY ./ack_backend/poetry.lock ./ack_backend/pyproject.toml ./
+
+# Install ack_backend dependencies
+WORKDIR /var/task
 RUN poetry config virtualenvs.create false && poetry install --no-interaction --no-ansi --no-root --only main
+
 # -----------------------------
 FROM base AS build
-COPY src .
+
+# Set working directory back to Lambda task root
+WORKDIR /var/task
+
+# Copy shared source code
+COPY ./shared/src/common ./common
+
+# Copy ack_backend source code
+COPY ./ack_backend/src .
+
+# Set correct permissions
 RUN chmod 644 $(find . -type f) && chmod 755 $(find . -type d)
+
 # Switch to the non-root user for running the container
 USER 1001:1001
+
+# Set the Lambda handler
 CMD ["ack_processor.lambda_handler"]

immunisation-fhir-api.code-workspace

@@ -0,0 +1,22 @@
+TEST_ENV := @PYTHONPATH=src:tests:../shared/src
+
+build:
+	docker build -t ack-lambda-build .
+
+package: build
+	mkdir -p build
+	docker run --rm -v $(shell pwd)/build:/build ack-lambda-build
+
+test:
+	$(TEST_ENV) python -m unittest
+
+coverage-run:
+	$(TEST_ENV) coverage run --source=src -m unittest discover -v
+
+coverage-report:
+	$(TEST_ENV) coverage report -m
+
+coverage-html:
+	$(TEST_ENV) coverage html
+
+.PHONY: build package