VED-305 sonarcloud isolation env

.github/scripts/run_test copy.sh.save

@@ -0,0 +1,49 @@
+#!/bin/bash
+set -e
+
+PYTHON_VERSION="$1"
+DESCRIPTION="$2"
+COVERAGE_XML="sonarcloud-coverage-$3.xml"
+
+# show cwd
+echo "Current working directory: $(pwd)"
+# list contents of venv
+
+ls -la .
+
+########### debug
+echo "Which Python:"
+which python$PYTHON_VERSION || echo "python$PYTHON_VERSION not found in PATH"
+
+echo "Available interpreters:"
+pyenv versions || true
+
+echo "Using Python Version $PYTHON_VERSION"
+poetry config virtualenvs.in-project true
+##############
+
+# Only create/use the env and install if .venv does not exist
+echo "Checking for Poetry virtual environment (.venv)"
+poetry env list
+if [ ! -d ".venv" ]; then
+  echo "Creating virtual environment (.venv) with Poetry"
+  poetry env use "$PYTHON_VERSION"
+
+  ########### debug
+  echo "Poetry environment info:"
+  poetry env info || true
+  ###########
+
+  poetry install
+  poetry env list
+else
+  echo "Using cached virtual environment (.venv)"
+fi
+
+if poetry run coverage run -m unittest discover; then
+  echo "$DESCRIPTION tests passed"
+else
+  echo "$DESCRIPTION tests failed" >> ../failed_tests.txt
+fi
+
+poetry run coverage xml -o "../$COVERAGE_XML"

.github/scripts/run_test.sh

@@ -0,0 +1,11 @@
+#!/bin/bash
+set -e
+
+PYTHON_VERSION="$1"
+DESCRIPTION="$2"
+COVERAGE_XML="sonarcloud-coverage-$3.xml"
+
+poetry env use "$PYTHON_VERSION"
+poetry install
+poetry run coverage run -m unittest discover || echo "$DESCRIPTION tests failed" >> ../failed_tests.txt
+poetry run coverage xml -o "../$COVERAGE_XML"

.github/workflows/sonarcloud.yml

@@ -6,17 +6,43 @@ on:
       - master
   pull_request:
     types: [labeled, opened, synchronize, reopened, unlabeled]
+env:
+  SCRIPT_FOLDER: ./.github/scripts
+  RUN_TEST: source ../.github/scripts/run_test.sh
 
 jobs:
   sonarcloud:
     name: SonarCloud
     runs-on: ubuntu-latest
+    strategy:
+      matrix:
+        python-version: ['3.11', '3.10']
 
     steps:
       - uses: actions/checkout@v4
         with:
           fetch-depth: 0
 
+      - name: Chmod .github directory
+        run: |
+          chmod +x $SCRIPT_FOLDER/*.sh
+
+      - name: Install poetry
+        run: pip install poetry==1.8.4
+
+      - uses: actions/setup-python@v5
+        with:
+          python-version: ${{ matrix.python-version }}
+          cache: 'poetry'
+
+      - name: Cache global Poetry virtualenvs
+        uses: actions/cache@v4
+        with:
+          path: ~/.cache/pypoetry/virtualenvs
+          key: ${{ runner.os }}-poetry-global-venv-py${{ matrix.python-version }}-${{ hashFiles('**/poetry.lock') }}
+          restore-keys: |
+            ${{ runner.os }}-poetry-global-venv-py${{ matrix.python-version }}-
+
       - name: Set up AWS credentials
         env:
           AWS_ACCESS_KEY_ID: "FOOBARKEY"
@@ -25,71 +51,102 @@ jobs:
           aws configure set aws_access_key_id $AWS_ACCESS_KEY_ID
           aws configure set aws_secret_access_key $AWS_SECRET_ACCESS_KEY
 
-      - name: Initialize Unit Test Failure Tracker
-        run: echo "false" > test_failed.txt
-
       - name: Run unittest with filenameprocessor-coverage
+        if: matrix.python-version == '3.10'
+        working-directory: filenameprocessor
         id: filenameprocessor
         continue-on-error: true
         run: |
-          pip install poetry==1.8.4 moto==4.2.11 coverage redis botocore==1.35.49 simplejson pandas freezegun responses structlog fhir.resources jsonpath_ng pydantic==1.10.13 requests aws-lambda-typing cffi pyjwt boto3-stubs-lite[dynamodb]~=1.26.90 python-stdnum==1.20 fakeredis
-          poetry run coverage run --source=filenameprocessor -m unittest discover -s filenameprocessor || echo "filenameprocessor tests failed" >> failed_tests.txt
-          poetry run coverage xml -o sonarcloud-coverage-filenameprocessor-coverage.xml
+          poetry env use 3.10
+          poetry install
+          poetry run coverage run -m unittest discover || echo "filenameprocessor tests failed" >> ../failed_tests.txt
+          poetry run coverage xml -o ../sonarcloud-coverage-filenameprocessor-coverage.xml
 
       - name: Run unittest with recordprocessor-coverage
+        if: matrix.python-version == '3.10'  
+        working-directory: recordprocessor
         id: recordprocessor
         continue-on-error: true
         run: |
-          poetry run coverage run --source=recordprocessor -m unittest discover -s recordprocessor || echo "recordprocessor tests failed" >> failed_tests.txt
-          poetry run coverage xml -o sonarcloud-coverage-recordprocessor-coverage.xml
+          poetry env use 3.10
+          poetry install
+          poetry run coverage run -m unittest discover || echo "recordprocessor tests failed" >> ../failed_tests.txt
+          poetry run coverage xml -o ../sonarcloud-coverage-recordprocessor-coverage.xml
 
+        # This step is redundant - all of these tests will be run in the backend step below
       - name: Run unittest with recordforwarder-coverage
+        if: matrix.python-version == '3.11'
+        working-directory: backend
         id: recordforwarder
         continue-on-error: true
         run: |
-          pip install poetry==1.8.4 moto==5.1.4
-          PYTHONPATH=$(pwd)/backend:$(pwd)/backend/tests poetry run coverage run --source=backend -m unittest discover -s backend/tests -p "*batch*.py" || echo "recordforwarder tests failed" >> failed_tests.txt
-          poetry run coverage xml -o sonarcloud-coverage-recordforwarder-coverage.xml
+          poetry env use 3.11
+          poetry install
+          poetry run coverage run -m unittest discover -s "./tests" -p "*batch*.py" || echo "recordforwarder tests failed" >> ../failed_tests.txt
+          poetry run coverage xml -o ../sonarcloud-coverage-recordforwarder-coverage.xml
 
       - name: Run unittest with coverage-ack-lambda
+        if: matrix.python-version == '3.10'
+        working-directory: ack_backend
         id: acklambda
         continue-on-error: true
         run: |
-          pip install poetry==1.8.4 coverage moto==4.2.11 freezegun
-          poetry run coverage run --source=ack_backend -m unittest discover -s ack_backend || echo "ack-lambda tests failed" >> failed_tests.txt
-          poetry run coverage xml -o sonarcloud-coverage-ack-lambda.xml
+          poetry env use 3.10
+          poetry install
+          poetry run coverage run -m unittest discover || echo "ack-lambda tests failed" >> ../failed_tests.txt
+          poetry run coverage xml -o ../sonarcloud-coverage-ack-lambda.xml
 
       - name: Run unittest with coverage-delta
+        if: matrix.python-version == '3.11'
+        working-directory: delta_backend
         id: delta
         env:
             PYTHONPATH: delta_backend/src:delta_backend/tests
         continue-on-error: true
         run: |
-          pip install poetry==1.8.4 moto==5.1.4 mypy-boto3-dynamodb==1.35.54 boto3==1.26.165 coverage botocore==1.29.165 jmespath==1.0.1 python-dateutil==2.9.0 urllib3==1.26.20 s3transfer==0.6.2 typing-extensions==4.12.2
-          poetry run coverage run --source=delta_backend -m unittest discover -s delta_backend || echo "delta tests failed" >> failed_tests.txt
-          poetry run coverage xml -o sonarcloud-coverage-delta.xml
+          poetry env use 3.11
+          poetry install
+          poetry run coverage run -m unittest discover || echo "delta tests failed" >> ../failed_tests.txt
+          poetry run coverage xml -o ../sonarcloud-coverage-delta.xml
+
+
+      - name: Run unittest with coverage-delta2
+        if: matrix.python-version == '3.11'
+        working-directory: delta_backend
+        id: delta2
+        env:
+            PYTHONPATH: delta_backend/src:delta_backend/tests
+        continue-on-error: true
+        run: |
+          $RUN_TEST 3.11 delta_backend delta_backend
 
       - name: Run unittest with coverage-fhir-api
+        if: matrix.python-version == '3.11'
+        working-directory: backend
         id: fhirapi
         continue-on-error: true
         run: |
-          pip install poetry==1.8.4 moto==5.1.4 coverage redis botocore==1.35.49 simplejson responses structlog fhir.resources jsonpath_ng pydantic==1.10.13 requests aws-lambda-typing cffi pyjwt boto3-stubs-lite[dynamodb]~=1.26.90 python-stdnum==1.20
-          poetry run coverage run --source=backend -m unittest discover -s backend || echo "fhir-api tests failed" >> failed_tests.txt
-          poetry run coverage xml -o sonarcloud-coverage.xml
+          poetry env use 3.11
+          poetry install
+          poetry run coverage run -m unittest discover || echo "fhir-api tests failed" >> ../failed_tests.txt
+          poetry run coverage xml -o ../sonarcloud-coverage.xml
 
       - name: Run unittest with coverage-mesh-processor
+        if: matrix.python-version == '3.10'
+        working-directory: mesh_processor
         id: meshprocessor
         continue-on-error: true
         run: |
-          pip install poetry==1.8.4 moto==4.2.11 coverage redis botocore==1.35.49 simplejson responses structlog fhir.resources jsonpath_ng pydantic==1.10.13 requests aws-lambda-typing cffi pyjwt boto3-stubs-lite[dynamodb]~=1.26.90 python-stdnum==1.20
-          poetry run coverage run --source=mesh_processor -m unittest discover -s mesh_processor || echo "mesh_processor tests failed" >> failed_tests.txt
-          poetry run coverage xml -o sonarcloud-mesh_processor-coverage.xml
+          poetry env use 3.10
+          poetry install
+          poetry run coverage run -m unittest discover || echo "mesh_processor tests failed" >> ../failed_tests.txt
+          poetry run coverage xml -o ../sonarcloud-mesh_processor-coverage.xml
 
       - name: Run Test Failure Summary
         id: check_failure
         run: |
           if [ -s failed_tests.txt ]; then
-            echo "The following tests failed:"
+            echo "The following ${{ matrix.python-version }} tests failed:"
             cat failed_tests.txt
 
             while IFS= read -r line; do
@@ -98,11 +155,19 @@ jobs:
 
             exit 1
           else
-            echo "All tests passed."
+            echo "All ${{ matrix.python-version }} tests passed."
           fi
 
+  sonarcloud_scan:
+    name: SonarCloud Scan
+    runs-on: ubuntu-latest
+    needs: sonarcloud
+    steps:
+      - uses: actions/checkout@v4
+        with:
+          fetch-depth: 0
       - name: SonarCloud Scan
         uses: SonarSource/sonarqube-scan-action@master
         env:
           GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} # Needed to get PR information, if any
-          SONAR_TOKEN: ${{ secrets.SONAR_TOKEN }}
+          SONAR_TOKEN: ${{ secrets.SONAR_TOKEN }}