VED-807 Refactor backend (#967)

* init: move recordforwarder, copy src/models

* temp: sonar.cpd.exclusions

* nosonar

* temp: sonar.exclusions

* test: remove Dockerfile path

* test: remove Dockerfile path II

* test: remove Dockerfile path III

* fix source_path

* lint

* init: move batch source and tests, duplicate testing_utils

* backend -> lambdas

* cleanup docker tags

* sonar

* lambdas/backend

* sonar

* source_path

* clients/redis_client

* errors

* ruff

* shared/src/common/models

* TEMP: sonar exclusions

* TEMP: sonar exclusions II

* ruff

* cache

* interim - for merge

* interim - for merge II

* path fix

* backend tests

* ruff

* poetry.lock

* smells

* smell: pydantic.ValidationError

* smell: expression_attribute_values

* ApiValidationError

* errors / api_errors

* errors / api_errors II

* backend -> shared

* :${{ env.SHARED_PATH }}/tests

* recordforwarder -> shared

* sonar

* S5332

* smells

* smells II

* smells III

* constants

* ruff

* errors

* rest of errors

* ruff

* singleton s3 client

* shared s3 client II

* mesh_processor env

* assert bug fix attempt

* move_file

* move_file II

* reworked test_clients

* bucket cleanup

* get_nhs_number

* make_*_pk

* make_*_pk II

* cleanup

* remove cpd.exclusions

* utils -> aws_s3_utils

* backend/src/constants

* backend/src/constants II

* utils -> aws_s3_utils II

* remove PII

* revert PK stuff from generic_utils

Author: JamesW1-NHS

.github/dependabot.yml

@@ -13,6 +13,7 @@ updates:
       - "/lambdas/delta_backend"
       - "/lambdas/filenameprocessor"
       - "/lambdas/mesh_processor"
+      - "/lambdas/recordforwarder"
       - "/lambdas/recordprocessor"
       - "/sandbox"
     schedule:
@@ -49,14 +50,15 @@ updates:
   - package-ecosystem: "pip"
     directories:
       - "/"
-      - "/backend"
+      - "/lambdas/backend"
       - "/lambdas/ack_backend"
       - "/lambdas/batch_processor_filter"
       - "/lambdas/delta_backend"
       - "/lambdas/filenameprocessor"
       - "/lambdas/id_sync"
       - "/lambdas/mesh_processor"
       - "/lambdas/mns_subscription"
+      - "/lambdas/recordforwarder"
       - "/lambdas/recordprocessor"
       - "/lambdas/redis_sync"
       - "/lambdas/shared"

.github/workflows/quality-checks.yml

@@ -97,39 +97,16 @@ jobs:
           aws configure set aws_access_key_id $AWS_ACCESS_KEY_ID
           aws configure set aws_secret_access_key $AWS_SECRET_ACCESS_KEY
 
-      - name: Run unittest with recordprocessor-coverage
-        working-directory: lambdas/recordprocessor
-        id: recordprocessor
-        env:
-          PYTHONPATH: ${{ env.LAMBDA_PATH }}/recordprocessor/src:${{ env.LAMBDA_PATH }}/recordprocessor/tests:${{ env.SHARED_PATH }}/src
-        continue-on-error: true
-        run: |
-          poetry install
-          poetry run coverage run --source=src -m unittest discover || echo "recordprocessor tests failed" >> ../../failed_tests.txt
-          poetry run coverage xml -o ../../recordprocessor-coverage.xml
-
-        # This step is redundant - all of these tests will be run in the backend step below
-      - name: Run unittest with recordforwarder-coverage
-        working-directory: backend
-        id: recordforwarder
-        env:
-          PYTHONPATH: ${{ github.workspace }}/backend/src:${{ github.workspace }}/backend/tests
-        continue-on-error: true
-        run: |
-          poetry install
-          poetry run coverage run --source=src -m unittest discover -p "*batch*.py" || echo "recordforwarder tests failed" >> ../failed_tests.txt
-          poetry run coverage xml -o ../recordforwarder-coverage.xml
-
       - name: Run unittest with coverage-fhir-api
-        working-directory: backend
+        working-directory: lambdas/backend
         env:
-          PYTHONPATH: ${{ github.workspace }}/backend/src:${{ github.workspace }}/backend/tests
+          PYTHONPATH: ${{ env.LAMBDA_PATH }}/backend/src:${{ env.LAMBDA_PATH }}/backend/tests:${{ env.SHARED_PATH }}/src:${{ env.SHARED_PATH }}/tests
         id: fhirapi
         continue-on-error: true
         run: |
           poetry install
-          poetry run coverage run --source=src -m unittest discover || echo "fhir-api tests failed" >> ../failed_tests.txt
-          poetry run coverage xml -o ../backend-coverage.xml
+          poetry run coverage run --source=src -m unittest discover || echo "fhir-api tests failed" >> ../../failed_tests.txt
+          poetry run coverage xml -o ../../backend-coverage.xml
 
       - name: Run unittest with coverage-ack-lambda
         working-directory: lambdas/ack_backend
@@ -189,6 +166,8 @@ jobs:
       - name: Run unittest with coverage-mesh-processor
         working-directory: lambdas/mesh_processor
         id: meshprocessor
+        env:
+          PYTHONPATH: ${{ env.LAMBDA_PATH }}/mesh_processor/src:${{ env.LAMBDA_PATH }}/mesh_processor/tests:${{ env.SHARED_PATH }}/src
         continue-on-error: true
         run: |
           poetry install
@@ -207,6 +186,28 @@ jobs:
           poetry run coverage report -m
           poetry run coverage xml -o ../../mns_subscription-coverage.xml
 
+      - name: Run unittest with recordforwarder-coverage
+        working-directory: lambdas/recordforwarder
+        id: recordforwarder
+        env:
+          PYTHONPATH: ${{ env.LAMBDA_PATH }}/recordforwarder/src:${{ env.LAMBDA_PATH }}/recordforwarder/tests:${{ env.SHARED_PATH }}/src:${{ env.SHARED_PATH }}/tests
+        continue-on-error: true
+        run: |
+          poetry install
+          poetry run coverage run --source=src -m unittest discover || echo "recordforwarder tests failed" >> ../../failed_tests.txt
+          poetry run coverage xml -o ../../recordforwarder-coverage.xml
+
+      - name: Run unittest with recordprocessor-coverage
+        working-directory: lambdas/recordprocessor
+        id: recordprocessor
+        env:
+          PYTHONPATH: ${{ env.LAMBDA_PATH }}/recordprocessor/src:${{ env.LAMBDA_PATH }}/recordprocessor/tests:${{ env.SHARED_PATH }}/src
+        continue-on-error: true
+        run: |
+          poetry install
+          poetry run coverage run --source=src -m unittest discover || echo "recordprocessor tests failed" >> ../../failed_tests.txt
+          poetry run coverage xml -o ../../recordprocessor-coverage.xml
+
       - name: Run unittest with redis_sync
         working-directory: lambdas/redis_sync
         id: redis_sync

Makefile

@@ -1,6 +1,6 @@
 SHELL=/usr/bin/env bash -euo pipefail
 
-PYTHON_PROJECT_DIRS_WITH_UNIT_TESTS = backend lambdas/ack_backend lambdas/batch_processor_filter lambdas/delta_backend lambdas/filenameprocessor lambdas/id_sync lambdas/mesh_processor lambdas/mns_subscription lambdas/recordprocessor lambdas/redis_sync lambdas/shared
+PYTHON_PROJECT_DIRS_WITH_UNIT_TESTS = lambdas/backend lambdas/ack_backend lambdas/batch_processor_filter lambdas/delta_backend lambdas/filenameprocessor lambdas/id_sync lambdas/mesh_processor lambdas/mns_subscription lambdas/recordforwarder lambdas/recordprocessor lambdas/redis_sync lambdas/shared
 PYTHON_PROJECT_DIRS = tests/e2e tests/e2e_batch quality_checks $(PYTHON_PROJECT_DIRS_WITH_UNIT_TESTS)
 
 .PHONY: install lint format format-check clean publish build-proxy release initialise-all-python-venvs update-all-python-dependencies run-all-python-unit-tests build-all-docker-images