shared tests pass

nhsdevws · nhsdevws · commit 1303f87a842f · 2025-07-18T09:06:02.000+01:00
diff --git a/.github/dependabot.yml b/.github/dependabot.yml
@@ -59,6 +59,7 @@ updates:
       - "/recordprocessor"
       - "/redis_sync"
       - "/lambdas/id_sync"
+      - "/lambdas/shared"
     schedule:
       interval: "daily"
     open-pull-requests-limit: 1
diff --git a/.github/workflows/sonarcloud.yml b/.github/workflows/sonarcloud.yml
@@ -124,6 +124,18 @@ jobs:
             poetry run coverage run -m unittest discover || echo "redis_sync tests failed" >> ../failed_tests.txt
             poetry run coverage xml -o ../redis_sync-coverage.xml
 
+      - name: Run unittest with shared
+        working-directory: lambdas/shared
+        id: shared
+        env:
+          PYTHONPATH: ${{ github.workspace }}/lambdas/shared/src:${{ github.workspace }}/lambdas/shared/tests
+        continue-on-error: true
+        run: |
+            poetry env use 3.11
+            poetry install
+            poetry run coverage run -m unittest discover || echo "shared tests failed" >> ../../failed_tests.txt
+            poetry run coverage xml -o ../../shared-coverage.xml
+
       - name: Run unittest with id_sync
         working-directory: lambdas/id_sync
         id: id_sync
diff --git a/immunisation-fhir-api.code-workspace b/immunisation-fhir-api.code-workspace
@@ -32,6 +32,9 @@
         },
         {
             "path": "lambdas/id_sync"
+        },
+        {
+            "path": "lambdas/shared"
         }
     ],
 	"settings": {},
diff --git a/lambdas/shared/.vscode/settings.json.default b/lambdas/shared/.vscode/settings.json.default
@@ -0,0 +1,18 @@
+{
+    "python.analysis.extraPaths": [
+        "./src"
+    ],
+    "python.testing.unittestArgs": [
+        "-v",
+        "-s",
+        "./",
+        "-p",
+        "test_*.py"
+    ],
+    "python.testing.pytestEnabled": false,
+    "python.testing.unittestEnabled": true,
+    "pylint.args": [
+        "--init-hook",
+        "import sys; sys.path.append('./src')"
+    ]
+}
diff --git a/lambdas/shared/Makefile b/lambdas/shared/Makefile
@@ -0,0 +1,17 @@
+
+test:
+	@PYTHONPATH=src python -m unittest discover -s tests -p "test_*.py" -v
+
+test-list:
+	@PYTHONPATH=src:tests:tests python -m unittest discover -s tests/common -p "test_*.py" --verbose | grep test_
+
+coverage-run:
+	@PYTHONPATH=src:tests coverage run -m unittest discover -v
+
+coverage-report:
+	coverage report -m
+
+coverage-html:
+	coverage html
+
+.PHONY: build package test
diff --git a/lambdas/shared/README.md b/lambdas/shared/README.md
@@ -0,0 +1,44 @@
+# Redis Sync Lambda
+
+## Overview
+
+**Redis Sync** is an AWS Lambda function designed to monitor a configuration S3 bucket and synchronize its contents with an ElastiCache Redis instance. Whenever a new configuration file is uploaded or updated in the S3 bucket, the Lambda function processes the file, applies any required transformations, and uploads the result to the Redis cache.
+
+## Features
+
+- **S3 Event Driven:** Automatically triggered by S3 events (e.g., file uploads or updates) in the config bucket.
+- **Transformation Support:** Applies custom transformation logic to configuration files before caching.
+- **Redis Integration:** Uploads processed configuration data to ElastiCache Redis for fast, centralized access.
+- **Logging:** Provides detailed logging for monitoring and troubleshooting.
+
+## How It Works
+
+1. **S3 Event Trigger:** The Lambda is triggered by S3 events on the config bucket.
+2. **File Processing:** The Lambda reads the new or updated file from S3.
+3. **Transformation:** If required, the file content is transformed to the appropriate format.
+4. **Redis Upload:** The transformed data is uploaded to the Redis cache under a key corresponding to the file.
+5. **Monitoring:** Logs are generated for each step, aiding in monitoring and debugging.
+
+## Configuration
+
+- **Environment Variables:**
+  - `CONFIG_BUCKET_NAME`: Name of the S3 bucket to monitor.
+  - `AWS_REGION`: AWS region for S3 and Redis.
+  - `REDIS_HOST`: Redis endpoint.
+  - `REDIS_PORT`: Redis port (default: 6379).
+
+## Usage
+
+1. **Deploy the Lambda** using your preferred IaC tool (e.g., Terraform, AWS SAM).
+2. **Configure S3 event notifications** to trigger the Lambda on object creation or update.
+3. **Ensure Redis and S3 permissions** are set for the Lambda execution role.
+
+## Development
+
+- Code is located in the `src/` directory.
+- Unit tests are in the `tests/` directory.
+- Use the provided Makefile and Dockerfile for building, testing, and packaging.
+
+## License
+
+This project is maintained by NHS. See [LICENSE](../LICENSE) for details.
diff --git a/lambdas/shared/poetry.lock b/lambdas/shared/poetry.lock
diff --git a/lambdas/shared/pyproject.toml b/lambdas/shared/pyproject.toml
@@ -0,0 +1,34 @@
+[project]
+name = "shared"
+version = "0.1.0"
+description = "Shared utilities for NHS Immunisation FHIR API"
+authors = [
+    {name = "nhsdevws",email = "stephen.wates1@nhs.net"}
+]
+
+[tool.poetry]
+name = "shared"
+version = "0.1.0"
+description = ""
+authors = ["s.wates <stephen.wates1@nhs.net>"]
+readme = "README.md"
+packages = [
+    {include = "src"},
+    {include = "common", from = "src"}
+]
+
+[tool.poetry.dependencies]
+python = ">=3.11"
+boto3 = "~1.38.29"
+mypy-boto3-dynamodb = "^1.26.164"
+moto = "~5.1.5"
+python-stdnum = "^2.1"
+coverage = "^7.8.0"
+redis = "^4.6.0"
+
+[tool.poetry.group.dev.dependencies]
+coverage = "^7.8.0"
+
+[build-system]
+requires = ["poetry-core"]
+build-backend = "poetry.core.masonry.api"
diff --git a/lambdas/shared/src/__init__.py b/lambdas/shared/src/__init__.py
diff --git a/lambdas/shared/src/common/clients.py b/lambdas/shared/src/common/clients.py
@@ -1,11 +1,21 @@
 import os
 import logging
+import redis
 from boto3 import client as boto3_client
 
+
 logging.basicConfig(level="INFO")
 logger = logging.getLogger()
 logger.setLevel("INFO")
 
+STREAM_NAME = os.getenv("SPLUNK_FIREHOSE_NAME", "firehose-name-not-defined")
+CONFIG_BUCKET_NAME = os.getenv("CONFIG_BUCKET_NAME", "variconfig-bucketable-not-defined")
+
 REGION_NAME = os.getenv("AWS_REGION", "eu-west-2")
+REDIS_HOST = os.getenv("REDIS_HOST", "")
+REDIS_PORT = os.getenv("REDIS_PORT", 6379)
+
 s3_client = boto3_client("s3", region_name=REGION_NAME)
 firehose_client = boto3_client("firehose", region_name=REGION_NAME)
+logger.info(f"Connecting to Redis at {REDIS_HOST}:{REDIS_PORT}")
+redis_client = redis.StrictRedis(host=REDIS_HOST, port=REDIS_PORT, decode_responses=True)
diff --git a/lambdas/shared/src/common/log_decorator.py b/lambdas/shared/src/common/log_decorator.py
@@ -8,7 +8,7 @@
 import time
 from datetime import datetime
 from functools import wraps
-from clients import logger, firehose_client
+from common.clients import logger, firehose_client
 
 
 def send_log_to_firehose(stream_name, log_data: dict) -> None:
@@ -36,6 +36,7 @@ def logging_decorator(prefix: str, stream_name):
     def decorator(func):
         @wraps(func)
         def wrapper(*args, **kwargs):
+            logger.info("Starting function: %s", func.__name__)
             base_log_data = {
                 "function_name": f"{prefix}_{func.__name__}",
                 "date_time": str(datetime.now())
diff --git a/lambdas/shared/tests/test_common/__init__.py b/lambdas/shared/tests/test_common/__init__.py
diff --git a/lambdas/shared/tests/test_common/test_clients.py b/lambdas/shared/tests/test_common/test_clients.py
@@ -0,0 +1,89 @@
+import unittest
+from unittest.mock import patch
+import common as clients
+import importlib
+
+
+class TestClients(unittest.TestCase):
+
+    BUCKET_NAME = "default-bucket"
+    AWS_REGION = "eu-west-2"
+    REDIS_HOST = "mock-redis-host"
+    REDIS_PORT = 6379
+
+    def setUp(self):
+        self.boto3_client_patch = patch("boto3.client")
+        self.mock_boto3_client = self.boto3_client_patch.start()
+        self.logging_patch = patch("logging.getLogger")
+        self.mock_logging = self.logging_patch.start()
+        self.logger_info_patcher = patch("logging.Logger.info")
+        self.mock_logger_info = self.logger_info_patcher.start()
+        self.getenv_patch = patch("os.getenv")
+        self.mock_getenv = self.getenv_patch.start()
+        self.mock_getenv.side_effect = lambda key, default=None: {
+            "CONFIG_BUCKET_NAME": self.BUCKET_NAME,
+            "AWS_REGION": self.AWS_REGION,
+            "REDIS_HOST": self.REDIS_HOST,
+            "REDIS_PORT": self.REDIS_PORT
+        }.get(key, default)
+
+        self.redis_patch = patch("redis.StrictRedis")
+        self.mock_redis = self.redis_patch.start()
+
+        self.mock_redis.return_value = self.mock_redis
+        self.mock_boto3_client.return_value = self.mock_boto3_client
+        self.mock_boto3_client.return_value.send_message = {}
+
+    def tearDown(self):
+        patch.stopall()
+
+    def test_os_environ(self):
+        # Test if environment variables are set correctly
+        importlib.reload(clients)
+        self.assertEqual(clients.CONFIG_BUCKET_NAME, self.BUCKET_NAME)
+        self.assertEqual(clients.REGION_NAME, self.AWS_REGION)
+        self.assertEqual(clients.REDIS_HOST, self.REDIS_HOST)
+        self.assertEqual(clients.REDIS_PORT, self.REDIS_PORT)
+
+    def test_boto3_client(self):
+        ''' Test boto3 client is created with correct parameters '''
+        importlib.reload(clients)
+        self.mock_boto3_client.assert_any_call("s3", region_name=self.AWS_REGION)
+
+    def test_firehose_client(self):
+        ''' Test firehose client is created with correct parameters '''
+        importlib.reload(clients)
+        self.mock_boto3_client.assert_any_call("firehose", region_name=self.AWS_REGION)
+
+    def test_redis_client(self):
+        ''' Test redis client is created with correct parameters '''
+        importlib.reload(clients)
+        self.mock_redis.assert_called_once_with(
+            host=self.REDIS_HOST,
+            port=self.REDIS_PORT,
+            decode_responses=True
+        )
+
+    def test_logging_setup(self):
+        ''' Test logging is set up correctly '''
+        importlib.reload(clients)
+        self.assertTrue(hasattr(clients, 'logger'))
+
+        def test_logging_configuration(self):
+            ''' Test logging configuration '''
+            importlib.reload(clients)
+            clients.logger.setLevel.assert_called_once_with("INFO")
+
+    def test_redis_client_initialization(self):
+        ''' Test redis client initialization '''
+        importlib.reload(clients)
+        self.mock_redis.assert_called_once_with(host=self.REDIS_HOST, port=self.REDIS_PORT, decode_responses=True)
+        self.assertTrue(hasattr(clients, 'redis_client'))
+        self.assertIsInstance(clients.redis_client, self.mock_redis.return_value.__class__)
+
+    def test_logging_initialization(self):
+        ''' Test logging initialization '''
+        importlib.reload(clients)
+        self.mock_logging.assert_called_once_with()
+        self.assertTrue(hasattr(clients, 'logger'))
+        clients.logger.setLevel.assert_any_call("INFO")
diff --git a/lambdas/shared/tests/test_common/test_log_decorator.py b/lambdas/shared/tests/test_common/test_log_decorator.py
diff --git a/redis_sync/src/clients.py b/redis_sync/src/clients.py

Original file line number	Diff line number	Diff line change
`@@ -32,6 +32,9 @@`
`32`	`32`	`},`
`33`	`33`	`{`
`34`	`34`	`"path": "lambdas/id_sync"`
	`35`	`+ },`
	`36`	`+ {`
	`37`	`+ "path": "lambdas/shared"`
`35`	`38`	`}`
`36`	`39`	`],`
`37`	`40`	`"settings": {},`