Update remaining Lambda functions to Python 3.11. Update dependencies. Add Makefile scripts to simplify setup. Add .tool-versions for asdf users.

Author: mfjarvis

.github/workflows/sonarcloud.yml

@@ -18,13 +18,11 @@ jobs:
           fetch-depth: 0
 
       - name: Install poetry
-        run: pip install poetry==1.8.4
+        run: pip install poetry==2.1.2
 
       - uses: actions/setup-python@v5
         with:
-          python-version: |
-            3.10
-            3.11
+          python-version: 3.11
           cache: 'poetry'
 
       - name: Set up AWS credentials
@@ -40,7 +38,6 @@ jobs:
         id: filenameprocessor
         continue-on-error: true
         run: |
-          poetry env use 3.11
           poetry install
           poetry run coverage run -m unittest discover || echo "filenameprocessor tests failed" >> ../failed_tests.txt
           poetry run coverage xml -o ../filenameprocessor-coverage.xml
@@ -50,7 +47,6 @@ jobs:
         id: recordprocessor
         continue-on-error: true
         run: |
-          poetry env use 3.11
           poetry install
           poetry run coverage run -m unittest discover || echo "recordprocessor tests failed" >> ../failed_tests.txt
           poetry run coverage xml -o ../recordprocessor-coverage.xml
@@ -63,7 +59,6 @@ jobs:
           PYTHONPATH: ${{ github.workspace }}/backend/src:${{ github.workspace }}/backend/tests
         continue-on-error: true
         run: |
-          poetry env use 3.11
           poetry install
           poetry run coverage run -m unittest discover -s "./tests" -p "*batch*.py" || echo "recordforwarder tests failed" >> ../failed_tests.txt
           poetry run coverage xml -o ../recordforwarder-coverage.xml
@@ -73,7 +68,6 @@ jobs:
         id: acklambda
         continue-on-error: true
         run: |
-          poetry env use 3.10
           poetry install
           poetry run coverage run -m unittest discover || echo "ack-lambda tests failed" >> ../failed_tests.txt
           poetry run coverage xml -o ../ack-lambda-coverage.xml
@@ -85,7 +79,6 @@ jobs:
             PYTHONPATH: delta_backend/src:delta_backend/tests
         continue-on-error: true
         run: |
-          poetry env use 3.11
           poetry install
           poetry run coverage run -m unittest discover || echo "delta tests failed" >> ../failed_tests.txt
           poetry run coverage xml -o ../delta-coverage.xml
@@ -97,7 +90,6 @@ jobs:
         id: fhirapi
         continue-on-error: true
         run: |
-          poetry env use 3.11
           poetry install
           poetry run coverage run -m unittest discover || echo "fhir-api tests failed" >> ../failed_tests.txt
           poetry run coverage xml -o ../backend-coverage.xml
@@ -107,7 +99,6 @@ jobs:
         id: meshprocessor
         continue-on-error: true
         run: |
-          poetry env use 3.11
           poetry install
           poetry run coverage run -m unittest discover || echo "mesh_processor tests failed" >> ../failed_tests.txt
           poetry run coverage xml -o ../mesh_processor-coverage.xml
@@ -119,7 +110,6 @@ jobs:
           PYTHONPATH: ${{ github.workspace }}/redis_sync/src:${{ github.workspace }}/redis_sync/tests
         continue-on-error: true
         run: |
-            poetry env use 3.11
             poetry install
             poetry run coverage run -m unittest discover || echo "redis_sync tests failed" >> ../failed_tests.txt
             poetry run coverage xml -o ../redis_sync-coverage.xml

.tool-versions

@@ -0,0 +1,3 @@
+poetry 2.1.2
+nodejs 23.11.0
+python 3.11.12

Makefile

@@ -1,5 +1,8 @@
 SHELL=/usr/bin/env bash -euo pipefail
 
+PYTHON_PROJECT_DIRS_WITH_UNIT_TESTS = ack_backend backend delta_backend filenameprocessor mesh_processor recordprocessor redis_sync
+PYTHON_PROJECT_DIRS = e2e e2e_batch $(PYTHON_PROJECT_DIRS_WITH_UNIT_TESTS)
+
 #Installs dependencies using poetry.
 install-python:
 	poetry lock --no-update
@@ -38,7 +41,7 @@ build-proxy:
 	scripts/build_proxy.sh
 
 #Files to loop over in release
-_dist_include="pytest.ini poetry.lock poetry.toml pyproject.toml Makefile build/. e2e e2e_batch specification sandbox terraform scripts backend delta_backend ack_backend filenameprocessor recordprocessor mesh_processor redis_sync"
+_dist_include="pytest.ini poetry.lock poetry.toml pyproject.toml Makefile build/. specification sandbox terraform scripts $(PYTHON_PROJECT_DIRS)"
 
 
 #Create /dist/ sub-directory and copy files into directory
@@ -85,3 +88,32 @@ test-prod:
 
 setup-python-envs:
 	scripts/setup-python-envs.sh
+
+initialise-all-python-venvs:
+	for dir in $(PYTHON_PROJECT_DIRS); do ( \
+		cd $$dir && \
+		pwd && \
+		rm -rf .venv && \
+		python -m venv .venv && \
+		source .venv/bin/activate && \
+		poetry install --no-root && \
+		deactivate \
+	); done
+
+update-all-python-dependencies:
+	for dir in $(PYTHON_PROJECT_DIRS); do ( \
+		cd $$dir && \
+		pwd && \
+		source .venv/bin/activate && \
+		poetry update && \
+		deactivate \
+	); done
+
+run-all-python-unit-tests:
+	for dir in $(PYTHON_PROJECT_DIRS_WITH_UNIT_TESTS); do ( \
+		cd $$dir && \
+		pwd && \
+		source .venv/bin/activate && \
+		poetry run make test && \
+		deactivate \
+	); done

ack_backend/Dockerfile

@@ -1,10 +1,10 @@
-FROM public.ecr.aws/lambda/python:3.10 AS base
+FROM public.ecr.aws/lambda/python:3.11 AS base
 
 RUN mkdir -p /home/appuser && \
     echo 'appuser:x:1001:1001::/home/appuser:/sbin/nologin' >> /etc/passwd && \
     echo 'appuser:x:1001:' >> /etc/group && \
-    chown -R 1001:1001 /home/appuser && pip install "poetry~=1.5.0"
-    
+    chown -R 1001:1001 /home/appuser && pip install "poetry~=2.1.2"
+
 COPY poetry.lock pyproject.toml README.md ./
 RUN poetry config virtualenvs.create false && poetry install --no-interaction --no-ansi --no-root --only main
 # -----------------------------