Merge remote branch into VED-789-Schema-Validation

Author: Akol125

.github/dependabot.yml

@@ -7,12 +7,13 @@ version: 2
 updates:
   - package-ecosystem: "docker"
     directories:
+      - "/batch_processor_filter"
+      - "/infrastructure/grafana/non-prod/docker"
+      - "/lambdas/filenameprocessor"
       - "/lambdas/ack_backend"
       - "/lambdas/delta_backend"
-      - "/filenameprocessor"
-      - "/infrastructure/grafana/non-prod/docker"
+      - "/lambdas/recordprocessor"
       - "/mesh_processor"
-      - "/recordprocessor"
       - "/sandbox"
     schedule:
       interval: "daily"
@@ -50,17 +51,17 @@ updates:
       - "/"
       - "/backend"
       - "/batch_processor_filter"
-      - "/tests/e2e"
-      - "/tests/e2e_batch"
-      - "/filenameprocessor"
       - "/mesh_processor"
-      - "/recordprocessor"
+      - "/lambdas/recordprocessor"
       - "/lambdas/ack_backend"
       - "/lambdas/delta_backend"
-      - "/lambdas/redis_sync"
+      - "/lambdas/filenameprocessor"
       - "/lambdas/id_sync"
       - "/lambdas/mns_subscription"
+      - "/lambdas/redis_sync"
       - "/lambdas/shared"
+      - "/tests/e2e"
+      - "/tests/e2e_batch"
     schedule:
       interval: "daily"
     open-pull-requests-limit: 1

.github/workflows/deploy-backend.yml

@@ -137,6 +137,7 @@ jobs:
         if: ${{ inputs.environment == 'dev' && inputs.create_mns_subscription }}
         working-directory: "./lambdas/mns_subscription"
         env:
+          APIGEE_ENVIRONMENT: int
           SQS_ARN: ${{ env.ID_SYNC_QUEUE_ARN }}
         run: |
           poetry install --no-root

.github/workflows/pr-teardown.yml

@@ -63,6 +63,7 @@ jobs:
       - name: Unsubscribe MNS
         working-directory: "./lambdas/mns_subscription"
         env:
+          APIGEE_ENVIRONMENT: int
           SQS_ARN: ${{ env.ID_SYNC_QUEUE_ARN }}
         run: |
           poetry install --no-root

.github/workflows/quality-checks.yml

@@ -97,15 +97,6 @@ jobs:
           aws configure set aws_access_key_id $AWS_ACCESS_KEY_ID
           aws configure set aws_secret_access_key $AWS_SECRET_ACCESS_KEY
 
-      - name: Run unittest with filenameprocessor-coverage
-        working-directory: filenameprocessor
-        id: filenameprocessor
-        continue-on-error: true
-        run: |
-          poetry install
-          poetry run coverage run -m unittest discover || echo "filenameprocessor tests failed" >> ../failed_tests.txt
-          poetry run coverage xml -o ../filenameprocessor-coverage.xml
-
       - name: Run unittest with batchprocessorfilter-coverage
         working-directory: batch_processor_filter
         id: batchprocessorfilter
@@ -118,15 +109,15 @@ jobs:
           poetry run coverage xml -o ../batchprocessorfilter-coverage.xml
 
       - name: Run unittest with recordprocessor-coverage
-        working-directory: recordprocessor
+        working-directory: lambdas/recordprocessor
         id: recordprocessor
         env:
-          PYTHONPATH: ${{ github.workspace }}/recordprocessor/src:${{ github.workspace }}/recordprocessor/tests
+          PYTHONPATH: ${{ env.LAMBDA_PATH }}/recordprocessor/src:${{ env.LAMBDA_PATH }}/recordprocessor/tests:${{ env.SHARED_PATH }}/src
         continue-on-error: true
         run: |
           poetry install
-          poetry run coverage run -m unittest discover || echo "recordprocessor tests failed" >> ../failed_tests.txt
-          poetry run coverage xml -o ../recordprocessor-coverage.xml
+          poetry run coverage run --source=src -m unittest discover || echo "recordprocessor tests failed" >> ../../failed_tests.txt
+          poetry run coverage xml -o ../../recordprocessor-coverage.xml
 
         # This step is redundant - all of these tests will be run in the backend step below
       - name: Run unittest with recordforwarder-coverage
@@ -182,6 +173,28 @@ jobs:
           poetry run coverage run -m unittest discover || echo "delta tests failed" >> ../../failed_tests.txt
           poetry run coverage xml -o ../../delta-coverage.xml
 
+      - name: Run unittest with filenameprocessor-coverage
+        working-directory: lambdas/filenameprocessor
+        id: filenameprocessor
+        env:
+          PYTHONPATH: ${{ env.LAMBDA_PATH }}/filenameprocessor/src:${{ env.LAMBDA_PATH }}/filenameprocessor/tests:${{ env.SHARED_PATH }}/src
+        continue-on-error: true
+        run: |
+          poetry install
+          poetry run coverage run -m unittest discover || echo "filenameprocessor tests failed" >> ../../failed_tests.txt
+          poetry run coverage xml -o ../../filenameprocessor-coverage.xml
+
+      - name: Run unittest with id_sync
+        working-directory: lambdas/id_sync
+        id: id_sync
+        env:
+          PYTHONPATH: ${{ env.LAMBDA_PATH }}/id_sync/src:${{ env.LAMBDA_PATH }}/id_sync/tests:${{ env.SHARED_PATH }}/src
+        continue-on-error: true
+        run: |
+          poetry install
+          poetry run coverage run --rcfile=.coveragerc --source=src -m unittest discover || echo "id_sync tests failed" >> ../../failed_tests.txt
+          poetry run coverage xml -o ../../id_sync-coverage.xml
+
       - name: Run unittest with coverage-mns-subscription
         working-directory: lambdas/mns_subscription
         id: mns_subscription
@@ -216,17 +229,6 @@ jobs:
           poetry run coverage run --rcfile=.coveragerc --source=src -m unittest discover -s tests -p "test_*.py" -v || echo "shared tests failed" >> ../../failed_tests.txt
           poetry run coverage xml -o ../../shared-coverage.xml
 
-      - name: Run unittest with id_sync
-        working-directory: lambdas/id_sync
-        id: id_sync
-        env:
-          PYTHONPATH: ${{ env.LAMBDA_PATH }}/id_sync/src:${{ env.LAMBDA_PATH }}/id_sync/tests:${{ env.SHARED_PATH }}/src
-        continue-on-error: true
-        run: |
-          poetry install
-          poetry run coverage run --rcfile=.coveragerc --source=src -m unittest discover || echo "id_sync tests failed" >> ../../failed_tests.txt
-          poetry run coverage xml -o ../../id_sync-coverage.xml
-
       - name: Run Test Failure Summary
         id: check_failure
         run: |

Makefile

@@ -1,6 +1,6 @@
 SHELL=/usr/bin/env bash -euo pipefail
 
-PYTHON_PROJECT_DIRS_WITH_UNIT_TESTS = backend batch_processor_filter filenameprocessor mesh_processor recordprocessor lambdas/ack_backend lambdas/delta_backend lambdas/redis_sync lambdas/id_sync lambdas/mns_subscription lambdas/shared
+PYTHON_PROJECT_DIRS_WITH_UNIT_TESTS = backend batch_processor_filter mesh_processor lambdas/ack_backend lambdas/delta_backend lambdas/filenameprocessor lambdas/id_sync lambdas/mns_subscription lambdas/recordprocessor lambdas/redis_sync lambdas/shared
 PYTHON_PROJECT_DIRS = tests/e2e tests/e2e_batch quality_checks $(PYTHON_PROJECT_DIRS_WITH_UNIT_TESTS)
 
 .PHONY: install lint format format-check clean publish build-proxy release initialise-all-python-venvs update-all-python-dependencies run-all-python-unit-tests build-all-docker-images