common folder patching

Author: nhsdevws

Makefile

@@ -38,7 +38,7 @@ build-proxy:
 	scripts/build_proxy.sh
 
 #Files to loop over in release
-_dist_include="pytest.ini poetry.lock poetry.toml pyproject.toml Makefile build/. e2e e2e_batch specification sandbox terraform scripts backend delta_backend ack_backend filenameprocessor recordprocessor mesh_processor redis_sync id_sync"
+_dist_include="pytest.ini poetry.lock poetry.toml pyproject.toml Makefile build/. e2e e2e_batch specification sandbox terraform scripts backend delta_backend ack_backend filenameprocessor recordprocessor mesh_processor redis_sync lambdas/id_sync"
 
 
 #Create /dist/ sub-directory and copy files into directory

lambdas/id_sync/Dockerfile

@@ -9,17 +9,12 @@ RUN mkdir -p /home/appuser && \
 # Install Poetry as root
 COPY poetry.lock pyproject.toml README.md ./
 RUN poetry config virtualenvs.create false && poetry install --no-interaction --no-ansi --no-root --only main
-# -----------------------------
-FROM base AS test
-COPY src src
-COPY tests tests
-RUN poetry install --no-interaction --no-ansi --no-root && \
-    pytest --disable-warnings tests
 
 # -----------------------------
 FROM base AS build
 COPY src .
+COPY ../shared/src/common ./common
 RUN chmod 644 $(find . -type f) && chmod 755 $(find . -type d)
 # Build as non-root user
 USER 1001:1001
-CMD ["id_sync.handler"]
+CMD ["id_sync.handler"]

lambdas/id_sync/Makefile

@@ -1,6 +1,6 @@
 
 test:
-	@PYTHONPATH=src:tests python -m unittest
+	@PYTHONPATH=src:tests:../shared/src python -m unittest
 
 coverage-run:
 	coverage run -m unittest discover -v

lambdas/id_sync/pyproject.toml

@@ -13,7 +13,8 @@ description = ""
 authors = ["s.wates <[email protected]>"]
 readme = "README.md"
 packages = [
-    {include = "src"}
+    {include = "src"},
+    {include = "common", from = "../shared/src/common"}
 ]
 
 [tool.poetry.dependencies]

lambdas/id_sync/src/clients.py

@@ -13,3 +13,4 @@
 REDIS_PORT = os.getenv("REDIS_PORT", 6379)
 s3_client = boto3_client("s3", region_name=REGION_NAME)
 firehose_client = boto3_client("firehose", region_name=REGION_NAME)
+

lambdas/id_sync/src/id_sync.py

@@ -1,5 +1,6 @@
-from clients import logger
-from log_decorator import logging_decorator
+from common.clients import logger
+from clients import STREAM_NAME
+from common.log_decorator import logging_decorator
 from record_processor import process_record
 
 '''
@@ -8,7 +9,7 @@
     This module processes S3 events and iterates through each record to process them individually.'''
 
 
-@logging_decorator(prefix="id_sync")
+@logging_decorator(prefix="id_sync", stream_name=STREAM_NAME)
 def handler(event, _):
 
     try:

lambdas/id_sync/tests/test_handler.py

@@ -16,7 +16,7 @@ def setUp(self):
         self.record_processor_patcher = patch("id_sync.process_record")
         self.mock_record_processor = self.record_processor_patcher.start()
         # patch log_decorator to pass through
-        self.mock_log_decorator = patch("log_decorator.logging_decorator", lambda prefix=None: (lambda f: f)).start()
+        self.mock_log_decorator = patch("common.log_decorator.logging_decorator", lambda prefix=None: (lambda f: f)).start()
 
     def tearDown(self):
         patch.stopall()

lambdas/shared/src/common/__init__.py

@@ -0,0 +1,11 @@
+import os
+import logging
+from boto3 import client as boto3_client
+
+logging.basicConfig(level="INFO")
+logger = logging.getLogger()
+logger.setLevel("INFO")
+
+REGION_NAME = os.getenv("AWS_REGION", "eu-west-2")
+s3_client = boto3_client("s3", region_name=REGION_NAME)
+firehose_client = boto3_client("firehose", region_name=REGION_NAME)

lambdas/shared/src/common/clients.py

@@ -8,31 +8,31 @@
 import time
 from datetime import datetime
 from functools import wraps
-from clients import firehose_client, logger, STREAM_NAME
+from clients import logger, firehose_client
 
 
-def send_log_to_firehose(log_data: dict) -> None:
+def send_log_to_firehose(stream_name, log_data: dict) -> None:
     """Sends the log_message to Firehose"""
     try:
         record = {"Data": json.dumps({"event": log_data}).encode("utf-8")}
-        response = firehose_client.put_record(DeliveryStreamName=STREAM_NAME, Record=record)
+        response = firehose_client.put_record(DeliveryStreamName=stream_name, Record=record)
         logger.info("Log sent to Firehose: %s", response)
     except Exception as error:  # pylint:disable = broad-exception-caught
         logger.exception("Error sending log to Firehose: %s", error)
 
 
-def generate_and_send_logs(
-    start_time, base_log_data: dict, additional_log_data: dict, is_error_log: bool = False
-) -> None:
+def generate_and_send_logs(stream_name,
+                           start_time, base_log_data: dict, additional_log_data: dict, is_error_log: bool = False
+                           ) -> None:
     """Generates log data which includes the base_log_data, additional_log_data, and time taken (calculated using the
     current time and given start_time) and sends them to Cloudwatch and Firehose."""
     log_data = {**base_log_data, "time_taken": f"{round(time.time() - start_time, 5)}s", **additional_log_data}
     log_function = logger.error if is_error_log else logger.info
     log_function(json.dumps(log_data))
-    send_log_to_firehose(log_data)
+    send_log_to_firehose(stream_name, log_data)
 
 
-def logging_decorator(prefix="id_sync"):
+def logging_decorator(prefix: str, stream_name):
     def decorator(func):
         @wraps(func)
         def wrapper(*args, **kwargs):
@@ -43,11 +43,13 @@ def wrapper(*args, **kwargs):
             start_time = time.time()
             try:
                 result = func(*args, **kwargs)
-                generate_and_send_logs(start_time, base_log_data, additional_log_data=result)
+                generate_and_send_logs(stream_name,
+                                       start_time, base_log_data, additional_log_data=result)
                 return result
             except Exception as e:
                 additional_log_data = {"statusCode": 500, "error": str(e)}
-                generate_and_send_logs(start_time, base_log_data, additional_log_data, is_error_log=True)
+                generate_and_send_logs(stream_name,
+                                       start_time, base_log_data, additional_log_data, is_error_log=True)
                 raise
         return wrapper
     return decorator