VED-802 Refactor delta_backend (#925)

* init

* terraform fmt

* mappings -> shared

* exclude for http:

* Dockerfile

* utils

* log_firehose

* log_firehose II

* log_firehose III

* log_firehose

* mappings not shared

* ruff

* sqs_client

* get_delta_table

* ruff format

* exclude_snomed_urls

* noqa S5332

* noqa S5332 II

* NOSONAR(S5332)

* review changes

* test: touch file without removing NOSONAR

* test: remove NOSONAR

* PoC: sonar exclusions: filepaths

* resubmit with comments

* ruff

* restored NOSONAR directives

Author: JamesW1-NHS

.github/dependabot.yml

@@ -8,7 +8,7 @@ updates:
   - package-ecosystem: "docker"
     directories:
       - "/lambdas/ack_backend"
-      - "/delta_backend"
+      - "/lambdas/delta_backend"
       - "/filenameprocessor"
       - "/infrastructure/grafana/non-prod/docker"
       - "/mesh_processor"
@@ -50,13 +50,13 @@ updates:
       - "/"
       - "/backend"
       - "/batch_processor_filter"
-      - "/delta_backend"
       - "/tests/e2e"
       - "/tests/e2e_batch"
       - "/filenameprocessor"
       - "/mesh_processor"
       - "/recordprocessor"
       - "/lambdas/ack_backend"
+      - "/lambdas/delta_backend"
       - "/lambdas/redis_sync"
       - "/lambdas/id_sync"
       - "/lambdas/mns_subscription"

.github/workflows/quality-checks.yml

@@ -140,17 +140,6 @@ jobs:
           poetry run coverage run -m unittest discover -p "*batch*.py" || echo "recordforwarder tests failed" >> ../failed_tests.txt
           poetry run coverage xml -o ../recordforwarder-coverage.xml
 
-      - name: Run unittest with coverage-delta
-        working-directory: delta_backend
-        id: delta
-        env:
-          PYTHONPATH: delta_backend/src:delta_backend/tests
-        continue-on-error: true
-        run: |
-          poetry install
-          poetry run coverage run -m unittest discover || echo "delta tests failed" >> ../failed_tests.txt
-          poetry run coverage xml -o ../delta-coverage.xml
-
       - name: Run unittest with coverage-fhir-api
         working-directory: backend
         env:
@@ -182,6 +171,17 @@ jobs:
           poetry run coverage run --source=src -m unittest discover || echo "ack-lambda tests failed" >> ../../failed_tests.txt
           poetry run coverage xml -o ../../ack-lambda-coverage.xml
 
+      - name: Run unittest with coverage-delta
+        working-directory: lambdas/delta_backend
+        id: delta
+        env:
+          PYTHONPATH: ${{ env.LAMBDA_PATH }}/delta_backend/src:${{ env.LAMBDA_PATH }}/delta_backend/tests:${{ env.SHARED_PATH }}/src
+        continue-on-error: true
+        run: |
+          poetry install
+          poetry run coverage run -m unittest discover || echo "delta tests failed" >> ../../failed_tests.txt
+          poetry run coverage xml -o ../../delta-coverage.xml
+
       - name: Run unittest with coverage-mns-subscription
         working-directory: lambdas/mns_subscription
         id: mns_subscription

Makefile

@@ -1,6 +1,6 @@
 SHELL=/usr/bin/env bash -euo pipefail
 
-PYTHON_PROJECT_DIRS_WITH_UNIT_TESTS = backend batch_processor_filter delta_backend filenameprocessor mesh_processor recordprocessor lambdas/ack_backend lambdas/redis_sync lambdas/id_sync lambdas/mns_subscription lambdas/shared
+PYTHON_PROJECT_DIRS_WITH_UNIT_TESTS = backend batch_processor_filter filenameprocessor mesh_processor recordprocessor lambdas/ack_backend lambdas/delta_backend lambdas/redis_sync lambdas/id_sync lambdas/mns_subscription lambdas/shared
 PYTHON_PROJECT_DIRS = tests/e2e tests/e2e_batch quality_checks $(PYTHON_PROJECT_DIRS_WITH_UNIT_TESTS)
 
 .PHONY: install lint format format-check clean publish build-proxy release initialise-all-python-venvs update-all-python-dependencies run-all-python-unit-tests build-all-docker-images

delta_backend/src/log_firehose.py

@@ -12,9 +12,6 @@
         {
             "path": "recordprocessor",
         },
-        {
-            "path": "delta_backend",
-        },
         {
             "path": "mesh_processor",
         },
@@ -27,6 +24,9 @@
         {
             "path": "lambdas/ack_backend",
         },
+        {
+            "path": "lambdas/delta_backend",
+        },
         {
             "path": "lambdas/redis_sync",
         },

delta_backend/src/utils.py

@@ -1,5 +1,5 @@
 locals {
-  delta_lambda_dir = abspath("${path.root}/../../delta_backend")
+  delta_lambda_dir = abspath("${path.root}/../../lambdas/delta_backend")
   delta_files      = fileset(local.delta_lambda_dir, "**")
   delta_dir_sha    = sha1(join("", [for f in local.delta_files : filesha1("${local.delta_lambda_dir}/${f}")]))
   function_name    = "delta"
@@ -16,8 +16,9 @@ resource "aws_ecr_repository" "delta_lambda_repository" {
 }
 
 module "delta_docker_image" {
-  source  = "terraform-aws-modules/lambda/aws//modules/docker-build"
-  version = "8.1.0"
+  source           = "terraform-aws-modules/lambda/aws//modules/docker-build"
+  version          = "8.1.0"
+  docker_file_path = "./delta_backend/Dockerfile"
 
   create_ecr_repo = false
   ecr_repo        = "${local.prefix}-delta-lambda-repo"
@@ -40,9 +41,10 @@ module "delta_docker_image" {
 
   platform      = "linux/amd64"
   use_image_tag = false
-  source_path   = local.delta_lambda_dir
+  source_path   = abspath("${path.root}/../../lambdas")
   triggers = {
-    dir_sha = local.delta_dir_sha
+    dir_sha        = local.delta_dir_sha
+    shared_dir_sha = local.shared_dir_sha
   }
 
 }

delta_backend/tests/test_log_firehose.py

@@ -9,7 +9,7 @@ def setUp(self):
         # Patch logger and firehose_client
         self.logger_patcher = patch("common.log_decorator.logger")
         self.mock_logger = self.logger_patcher.start()
-        self.firehose_patcher = patch("common.log_decorator.firehose_client")
+        self.firehose_patcher = patch("common.log_firehose.firehose_client")
         self.mock_firehose = self.firehose_patcher.start()
 
     def tearDown(self):