VED-803 Refactor filenameprocessor (#929)

* init

* terraform fmt

* mappings -> shared

* exclude for http:

* Dockerfile

* utils

* log_firehose

* log_firehose II

* log_firehose III

* log_firehose

* mappings not shared

* ruff

* sqs_client

* get_delta_table

* ruff format

* exclude_snomed_urls

* noqa S5332

* noqa S5332 II

* NOSONAR(S5332)

* init: move

* init: pyproject.toml

* fmt

* empty commit to trigger github actions

* revert to pre-802

* batch.Dockerfile

* rm batch.Dockerfile

* clients, errors, redis_client

* ruff

* Empty commit to kick github actions (stale terraform plan)

* note on global_redis_client

* log_decorator

* singleton redis_cache

* GenericSetUp/TearDown

* from tests.

* ruff

* removed lambda_client

Author: JamesW1-NHS

.github/dependabot.yml

@@ -8,8 +8,8 @@ updates:
   - package-ecosystem: "docker"
     directories:
       - "/batch_processor_filter"
-      - "/filenameprocessor"
       - "/infrastructure/grafana/non-prod/docker"
+      - "/lambdas/filenameprocessor"
       - "/lambdas/ack_backend"
       - "/lambdas/delta_backend"
       - "/lambdas/recordprocessor"
@@ -51,17 +51,17 @@ updates:
       - "/"
       - "/backend"
       - "/batch_processor_filter"
-      - "/tests/e2e"
-      - "/tests/e2e_batch"
-      - "/filenameprocessor"
       - "/mesh_processor"
       - "/lambdas/recordprocessor"
       - "/lambdas/ack_backend"
       - "/lambdas/delta_backend"
-      - "/lambdas/redis_sync"
+      - "/lambdas/filenameprocessor"
       - "/lambdas/id_sync"
       - "/lambdas/mns_subscription"
+      - "/lambdas/redis_sync"
       - "/lambdas/shared"
+      - "/tests/e2e"
+      - "/tests/e2e_batch"
     schedule:
       interval: "daily"
     open-pull-requests-limit: 1

.github/workflows/quality-checks.yml

@@ -97,15 +97,6 @@ jobs:
           aws configure set aws_access_key_id $AWS_ACCESS_KEY_ID
           aws configure set aws_secret_access_key $AWS_SECRET_ACCESS_KEY
 
-      - name: Run unittest with filenameprocessor-coverage
-        working-directory: filenameprocessor
-        id: filenameprocessor
-        continue-on-error: true
-        run: |
-          poetry install
-          poetry run coverage run -m unittest discover || echo "filenameprocessor tests failed" >> ../failed_tests.txt
-          poetry run coverage xml -o ../filenameprocessor-coverage.xml
-
       - name: Run unittest with batchprocessorfilter-coverage
         working-directory: batch_processor_filter
         id: batchprocessorfilter
@@ -182,6 +173,28 @@ jobs:
           poetry run coverage run -m unittest discover || echo "delta tests failed" >> ../../failed_tests.txt
           poetry run coverage xml -o ../../delta-coverage.xml
 
+      - name: Run unittest with filenameprocessor-coverage
+        working-directory: lambdas/filenameprocessor
+        id: filenameprocessor
+        env:
+          PYTHONPATH: ${{ env.LAMBDA_PATH }}/filenameprocessor/src:${{ env.LAMBDA_PATH }}/filenameprocessor/tests:${{ env.SHARED_PATH }}/src
+        continue-on-error: true
+        run: |
+          poetry install
+          poetry run coverage run -m unittest discover || echo "filenameprocessor tests failed" >> ../../failed_tests.txt
+          poetry run coverage xml -o ../../filenameprocessor-coverage.xml
+
+      - name: Run unittest with id_sync
+        working-directory: lambdas/id_sync
+        id: id_sync
+        env:
+          PYTHONPATH: ${{ env.LAMBDA_PATH }}/id_sync/src:${{ env.LAMBDA_PATH }}/id_sync/tests:${{ env.SHARED_PATH }}/src
+        continue-on-error: true
+        run: |
+          poetry install
+          poetry run coverage run --rcfile=.coveragerc --source=src -m unittest discover || echo "id_sync tests failed" >> ../../failed_tests.txt
+          poetry run coverage xml -o ../../id_sync-coverage.xml
+
       - name: Run unittest with coverage-mns-subscription
         working-directory: lambdas/mns_subscription
         id: mns_subscription
@@ -216,17 +229,6 @@ jobs:
           poetry run coverage run --rcfile=.coveragerc --source=src -m unittest discover -s tests -p "test_*.py" -v || echo "shared tests failed" >> ../../failed_tests.txt
           poetry run coverage xml -o ../../shared-coverage.xml
 
-      - name: Run unittest with id_sync
-        working-directory: lambdas/id_sync
-        id: id_sync
-        env:
-          PYTHONPATH: ${{ env.LAMBDA_PATH }}/id_sync/src:${{ env.LAMBDA_PATH }}/id_sync/tests:${{ env.SHARED_PATH }}/src
-        continue-on-error: true
-        run: |
-          poetry install
-          poetry run coverage run --rcfile=.coveragerc --source=src -m unittest discover || echo "id_sync tests failed" >> ../../failed_tests.txt
-          poetry run coverage xml -o ../../id_sync-coverage.xml
-
       - name: Run Test Failure Summary
         id: check_failure
         run: |

Makefile

@@ -1,6 +1,6 @@
 SHELL=/usr/bin/env bash -euo pipefail
 
-PYTHON_PROJECT_DIRS_WITH_UNIT_TESTS = backend batch_processor_filter filenameprocessor mesh_processor lambdas/recordprocessor lambdas/ack_backend lambdas/delta_backend lambdas/redis_sync lambdas/id_sync lambdas/mns_subscription lambdas/shared
+PYTHON_PROJECT_DIRS_WITH_UNIT_TESTS = backend batch_processor_filter mesh_processor lambdas/ack_backend lambdas/delta_backend lambdas/filenameprocessor lambdas/id_sync lambdas/mns_subscription lambdas/recordprocessor lambdas/redis_sync lambdas/shared
 PYTHON_PROJECT_DIRS = tests/e2e tests/e2e_batch quality_checks $(PYTHON_PROJECT_DIRS_WITH_UNIT_TESTS)
 
 .PHONY: install lint format format-check clean publish build-proxy release initialise-all-python-venvs update-all-python-dependencies run-all-python-unit-tests build-all-docker-images