Merge branch 'VED-81-Number-Update' into VED-480-Number-Update-terraform

Author: JamesW1-NHS

.github/workflows/sonarcloud.yml

@@ -22,13 +22,11 @@ jobs:
           fetch-depth: 0
 
       - name: Install poetry
-        run: pip install poetry==1.8.4
+        run: pip install poetry==2.1.2
 
       - uses: actions/setup-python@v5
         with:
-          python-version: |
-            3.10
-            3.11
+          python-version: 3.11
           cache: 'poetry'
       - name: Debug shared folder
         run: |
@@ -49,7 +47,6 @@ jobs:
         id: filenameprocessor
         continue-on-error: true
         run: |
-          poetry env use 3.11
           poetry install
           poetry run coverage run -m unittest discover || echo "filenameprocessor tests failed" >> ../failed_tests.txt
           poetry run coverage xml -o ../filenameprocessor-coverage.xml
@@ -59,7 +56,6 @@ jobs:
         id: recordprocessor
         continue-on-error: true
         run: |
-          poetry env use 3.11
           poetry install
           poetry run coverage run -m unittest discover || echo "recordprocessor tests failed" >> ../failed_tests.txt
           poetry run coverage xml -o ../recordprocessor-coverage.xml
@@ -72,7 +68,6 @@ jobs:
           PYTHONPATH: ${{ github.workspace }}/backend/src:${{ github.workspace }}/backend/tests
         continue-on-error: true
         run: |
-          poetry env use 3.11
           poetry install
           poetry run coverage run -m unittest discover -s "./tests" -p "*batch*.py" || echo "recordforwarder tests failed" >> ../failed_tests.txt
           poetry run coverage xml -o ../recordforwarder-coverage.xml
@@ -82,7 +77,6 @@ jobs:
         id: acklambda
         continue-on-error: true
         run: |
-          poetry env use 3.10
           poetry install
           poetry run coverage run -m unittest discover || echo "ack-lambda tests failed" >> ../failed_tests.txt
           poetry run coverage xml -o ../ack-lambda-coverage.xml
@@ -94,10 +88,10 @@ jobs:
             PYTHONPATH: delta_backend/src:delta_backend/tests
         continue-on-error: true
         run: |
-          poetry env use 3.11
           poetry install
           poetry run coverage run -m unittest discover || echo "delta tests failed" >> ../failed_tests.txt
           poetry run coverage xml -o ../delta-coverage.xml
+          cat ../delta-coverage.xml
 
       - name: Run unittest with coverage-fhir-api
         working-directory: backend
@@ -106,7 +100,6 @@ jobs:
         id: fhirapi
         continue-on-error: true
         run: |
-          poetry env use 3.11
           poetry install
           poetry run coverage run -m unittest discover || echo "fhir-api tests failed" >> ../failed_tests.txt
           poetry run coverage xml -o ../backend-coverage.xml
@@ -116,7 +109,6 @@ jobs:
         id: meshprocessor
         continue-on-error: true
         run: |
-          poetry env use 3.10
           poetry install
           poetry run coverage run -m unittest discover || echo "mesh_processor tests failed" >> ../failed_tests.txt
           poetry run coverage xml -o ../mesh_processor-coverage.xml
@@ -128,7 +120,6 @@ jobs:
           PYTHONPATH: ${{ github.workspace }}/redis_sync/src:${{ github.workspace }}/redis_sync/tests
         continue-on-error: true
         run: |
-            poetry env use 3.11
             poetry install
             poetry run coverage run -m unittest discover || echo "redis_sync tests failed" >> ../failed_tests.txt
             poetry run coverage xml -o ../redis_sync-coverage.xml
@@ -140,18 +131,10 @@ jobs:
           PYTHONPATH: ${{ env.SHARED_PATH }}
         continue-on-error: true
         run: |
-            echo "shared coverage - Current directory: $(pwd)"
             poetry env use 3.11
             poetry install
-            poetry run coverage run -m unittest discover || echo "shared tests failed" >> ../../failed_tests.txt
+            poetry run coverage run --source=src -m unittest discover -s tests -p "test_*.py" -v || echo "shared tests failed" >> ../../failed_tests.txt
             poetry run coverage xml -o ../../shared-coverage.xml
-            ls ../../shared-coverage.xml
-            # show director of ../.. as absolute path
-            echo "Parent directory of shared-coverage.xml: $(pwd)/../../"
-            #check files created
-            if [ ! -f ../../shared-coverage.xml ]; then
-              echo "shared-coverage.xml not found"
-            fi
 
       - name: Run unittest with id_sync
         working-directory: lambdas/id_sync
@@ -160,22 +143,14 @@ jobs:
           PYTHONPATH: ${{ env.LAMBDA_PATH }}/id_sync/src:${{ env.LAMBDA_PATH }}/id_sync/tests:${{ env.SHARED_PATH }}
         continue-on-error: true
         run: |
-            echo "id_sync coverage - Current directory: $(pwd)"
             poetry env use 3.11
             poetry install
             poetry run coverage run -m unittest discover || echo "id_sync tests failed" >> ../../failed_tests.txt
             poetry run coverage xml -o ../../id_sync-coverage.xml
-            #check files created
-            if [ ! -f ../../id_sync-coverage.xml ]; then
-              echo "id_sync-coverage.xml not found, exiting with error"
-            fi
 
       - name: Run Test Failure Summary
         id: check_failure
         run: |
-          echo "Checking for test failures..."
-          ls *-coverage.xml
-          cat shared-coverage.xml || echo "shared-coverage.xml not found"
           if [ -s failed_tests.txt ]; then
             echo "The following tests failed:"
             cat failed_tests.txt

.gitignore

@@ -29,6 +29,4 @@ openapi.json
 !**/.vscode/settings.json.default
 
 devtools/volume/
-backend/tests/.coverage
-redis_sync/.vscode/settings.json.default
-lambdas/shared/.coverage
+**/.coverage

.tool-versions

@@ -0,0 +1,3 @@
+poetry 2.1.2
+nodejs 23.11.0
+python 3.11.12

Makefile

@@ -1,5 +1,8 @@
 SHELL=/usr/bin/env bash -euo pipefail
 
+PYTHON_PROJECT_DIRS_WITH_UNIT_TESTS = ack_backend backend delta_backend filenameprocessor mesh_processor recordprocessor redis_sync
+PYTHON_PROJECT_DIRS = e2e e2e_batch $(PYTHON_PROJECT_DIRS_WITH_UNIT_TESTS)
+
 #Installs dependencies using poetry.
 install-python:
 	poetry lock --no-update
@@ -38,7 +41,7 @@ build-proxy:
 	scripts/build_proxy.sh
 
 #Files to loop over in release
-_dist_include="pytest.ini poetry.lock poetry.toml pyproject.toml Makefile build/. e2e e2e_batch specification sandbox terraform scripts backend delta_backend ack_backend filenameprocessor recordprocessor mesh_processor redis_sync lambdas/id_sync"
+_dist_include="pytest.ini poetry.lock poetry.toml pyproject.toml Makefile build/. specification sandbox terraform scripts $(PYTHON_PROJECT_DIRS)"
 
 
 #Create /dist/ sub-directory and copy files into directory
@@ -85,3 +88,39 @@ test-prod:
 
 setup-python-envs:
 	scripts/setup-python-envs.sh
+
+initialise-all-python-venvs:
+	for dir in $(PYTHON_PROJECT_DIRS); do ( \
+		cd $$dir && \
+		pwd && \
+		rm -rf .venv && \
+		python -m venv .venv && \
+		source .venv/bin/activate && \
+		poetry install --no-root && \
+		deactivate \
+	); done
+
+update-all-python-dependencies:
+	for dir in $(PYTHON_PROJECT_DIRS); do ( \
+		cd $$dir && \
+		pwd && \
+		source .venv/bin/activate && \
+		poetry update && \
+		deactivate \
+	); done
+
+run-all-python-unit-tests:
+	for dir in $(PYTHON_PROJECT_DIRS_WITH_UNIT_TESTS); do ( \
+		cd $$dir && \
+		pwd && \
+		source .venv/bin/activate && \
+		poetry run make test && \
+		deactivate \
+	); done
+
+build-all-docker-images:
+	for dir in $(PYTHON_PROJECT_DIRS_WITH_UNIT_TESTS); do \
+		for dockerfile in $$(ls $$dir/*Dockerfile); do \
+			echo $$dockerfile && docker build --file $$dockerfile $$dir; \
+		done; \
+	done

ack_backend/Dockerfile

@@ -1,10 +1,10 @@
-FROM public.ecr.aws/lambda/python:3.10 AS base
+FROM public.ecr.aws/lambda/python:3.11 AS base
 
 RUN mkdir -p /home/appuser && \
     echo 'appuser:x:1001:1001::/home/appuser:/sbin/nologin' >> /etc/passwd && \
     echo 'appuser:x:1001:' >> /etc/group && \
-    chown -R 1001:1001 /home/appuser && pip install "poetry~=1.5.0"
-    
+    chown -R 1001:1001 /home/appuser && pip install "poetry~=2.1.2"
+
 COPY poetry.lock pyproject.toml README.md ./
 RUN poetry config virtualenvs.create false && poetry install --no-interaction --no-ansi --no-root --only main
 # -----------------------------