[PRMP-178] Reporting ECS task isn't producing all logs (#781)

* [PRMT-609]- refactored csv_to_staging_metadata

* [PRMT-609]- added space

* [PRMT-609]- fixed spaces

* [PRMT-609]- fixed spaces

* [PRMT-609]- testing singleton

* [PRMT-609]- fixed tests

---------

Co-authored-by: Robert Gaskin <[email protected]>

Author: PedroSoaresNHS

lambdas/tests/unit/conftest.py

@@ -6,6 +6,7 @@
 from enum import Enum
 
 import pytest
+from utils.audit_logging_setup import LoggingService
 from botocore.exceptions import ClientError
 from models.document_reference import DocumentReference
 from models.pds_models import Patient, PatientDetails
@@ -366,3 +367,18 @@ def expect_not_to_raise(exception, message_when_fail=""):
 def mock_jwt_encode(mocker):
     decoded_token = {"selected_organisation": {"org_ods_code": "ODS123"}}
     yield mocker.patch("jwt.decode", return_value=decoded_token)
+
+@pytest.fixture(autouse=True)
+def reset_logging_singletons():
+    LoggingService._instances.clear()
+
+@pytest.fixture(autouse=True)
+def attach_caplog_handler(caplog):
+    for instance in LoggingService._instances.values():
+        instance.logger.addHandler(caplog.handler)
+    yield
+    for instance in LoggingService._instances.values():
+        try:
+            instance.logger.removeHandler(caplog.handler)
+        except Exception:
+            pass

lambdas/tests/unit/handlers/test_manage_nrl_pointer_handler.py

@@ -5,7 +5,6 @@
 from models.sqs.nrl_sqs_message import NrlSqsMessage
 from utils.exceptions import NrlApiException
 
-
 @pytest.fixture
 def mock_service(mocker):
     mocked_class = mocker.patch("handlers.manage_nrl_pointer_handler.NrlApiService")
@@ -57,8 +56,7 @@ def test_failed_to_create_a_pointer(mock_service, context, set_env, caplog):
     mock_service.create_new_pointer.side_effect = NrlApiException("test exception")
     with pytest.raises(NrlApiException):
         lambda_handler(event, context)
-    actual_log = caplog.records[-2].msg
-    assert actual_log == expected_log
+    assert any(expected_log in record.message for record in caplog.records)
     mock_service.create_new_pointer.assert_called_once()
 
 

lambdas/utils/audit_logging_setup.py

@@ -1,39 +1,50 @@
 import logging
-
+import sys
 from utils.logging_formatter import LoggingFormatter
 
+
 class LoggingService:
-    audit_logger = None
+    _instances = {}
+
+    def __new__(cls, name, *args, **kwargs):
+        if name not in cls._instances:
+            instance = super().__new__(cls)
+            cls._instances[name] = instance
+        return cls._instances[name]
 
     def __init__(self, name):
+        if getattr(self, '_initialized', False):
+            return
+
         self.name = name
         self.logger = logging.getLogger(name)
         self.logger.setLevel(logging.INFO)
-        self.formatter = LoggingFormatter()
-        logging.Formatter.format = self.formatter.format
+
+        if not self.logger.handlers:
+            handler = logging.StreamHandler(sys.stdout)
+            handler.setFormatter(LoggingFormatter())
+            self.logger.addHandler(handler)
+
+        self.logger.propagate = False
+
+        self._initialized = True
 
     def info(self, message, custom_args: dict = None, *args, **kwargs):
-        self.logger.info(message, extra={"custom_args": custom_args}, *args, **kwargs)
+        self.logger.info(message, extra={"custom_args": custom_args or {}}, *args, **kwargs)
 
     def error(self, message, custom_args: dict = None, *args, **kwargs):
-        self.logger.error(message, extra={"custom_args": custom_args}, *args, **kwargs)
+        self.logger.error(message, extra={"custom_args": custom_args or {}}, *args, **kwargs)
 
     def warning(self, message, custom_args: dict = None, *args, **kwargs):
-        self.logger.warning(
-            message, extra={"custom_args": custom_args}, *args, **kwargs
-        )
+        self.logger.warning(message, extra={"custom_args": custom_args or {}}, *args, **kwargs)
 
     def debug(self, message, custom_args: dict = None, *args, **kwargs):
-        self.logger.debug(message, extra={"custom_args": custom_args}, *args, **kwargs)
+        self.logger.debug(message, extra={"custom_args": custom_args or {}}, *args, **kwargs)
 
-    def exception(
-        self, message, custom_args: dict = None, *args, exc_info=True, **kwargs
-    ):
+    def exception(self, message, custom_args: dict = None, *args, exc_info=True, **kwargs):
         self.logger.exception(
-            message, exc_info, extra={"custom_args": custom_args}, *args, **kwargs
+            message, exc_info=exc_info, extra={"custom_args": custom_args or {}}, *args, **kwargs
         )
 
     def critical(self, message, custom_args: dict = None, *args, **kwargs):
-        self.logger.critical(
-            message, extra={"custom_args": custom_args}, *args, **kwargs
-        )
+        self.logger.critical(message, extra={"custom_args": custom_args or {}}, *args, **kwargs)

lambdas/utils/logging_formatter.py

@@ -3,6 +3,7 @@
 
 from utils.request_context import request_context
 
+
 class LoggingFormatter(logging.Formatter):
     """
     A JSON formatter that automatically adds key information from the
@@ -29,4 +30,4 @@ def format(self, record: logging.LogRecord) -> str:
         if record.__dict__.get("custom_args", {}) is not None:
             log_content.update(record.__dict__.get("custom_args", {}))
 
-        return json.dumps(log_content)
+        return json.dumps(log_content)