Sonarcloud and firehose mocking.

Author: nhsdevws

.github/workflows/sonarcloud.yml

@@ -61,9 +61,11 @@ jobs:
 
       - name: Run unittest with coverage-delta
         id: delta
+        env:
+            PYTHONPATH: delta_backend/src:delta_backend/tests
         continue-on-error: true
         run: |
-          pip install poetry==1.8.4 mypy-boto3-dynamodb==1.35.54 boto3==1.26.165 coverage botocore==1.29.165 jmespath==1.0.1 python-dateutil==2.9.0 urllib3==1.26.20 s3transfer==0.6.2 typing-extensions==4.12.2
+          pip install poetry==1.8.4 moto==4.2.11 mypy-boto3-dynamodb==1.35.54 boto3==1.26.165 coverage botocore==1.29.165 jmespath==1.0.1 python-dateutil==2.9.0 urllib3==1.26.20 s3transfer==0.6.2 typing-extensions==4.12.2
           poetry run coverage run --source=delta_backend -m unittest discover -s delta_backend || echo "delta tests failed" >> failed_tests.txt
           poetry run coverage xml -o sonarcloud-coverage-delta.xml
 

delta_backend/pyproject.toml

@@ -4,6 +4,9 @@ version = "0.1.0"
 description = ""
 authors = ["Your Name <[email protected]>"]
 readme = "README.md"
+packages = [
+    {include = "src"}
+]
 
 [tool.poetry.dependencies]
 python = "~3.10"

delta_backend/src/Converter.py

@@ -14,20 +14,17 @@
     get_valid_address,
 )
 
-# Converter variables
-FHIRData = ""
-SchemaFile = {}
-imms = []
-Converted = {}
-ErrorRecords = []
-
 
 # Converter
 class Converter:
 
     def __init__(self, fhir_data):
-        self.FHIRData = fhir_data  # Store JSON data directly
-        self.SchemaFile = ConversionLayout.ConvertLayout
+        #Converter variables
+        self.imms = []
+        self.converted = {}
+        self.error_records = []
+        self.fhir_data = fhir_data  # Store JSON data directly
+        self.schema_file = ConversionLayout.ConvertLayout
 
     # create a FHIR  parser - uses fhir json data from delta 
     # (helper methods to extract values from the nested FHIR structure)
@@ -56,7 +53,7 @@ def _convertData(self, ConversionValidate, expression, dataParser, json_data):
         except Exception as e:
             message = "Data get value Unexpected exception [%s]: %s" % (e.__class__.__name__, e)
             p = {"code": ExceptionMessages.PARSING_ERROR, "message": message}
-            ErrorRecords.append(p)
+            self.error_records.append(p)
             return p
 
         for conversionValue in conversionValues:
@@ -66,26 +63,26 @@ def _convertData(self, ConversionValidate, expression, dataParser, json_data):
             if "address" in FHIRFieldName or "performer" in FHIRFieldName or "name" in FHIRFieldName:
                 convertedData = self.extract_patient_details(json_data, FlatFieldName)
             if convertedData is not None:
-                Converted[FlatFieldName] = convertedData
+                self.converted[FlatFieldName] = convertedData
 
     # run the conversion against the data
     def runConversion(self, json_data, summarise=False, report_unexpected_exception=True):
         try:
-            dataParser = self._getFHIRParser(self.FHIRData)
+            dataParser = self._getFHIRParser(self.fhir_data)
         except Exception as e:
             if report_unexpected_exception:
                 message = "FHIR Parser Unexpected exception [%s]: %s" % (e.__class__.__name__, e)
                 p = {"code": 0, "message": message}
-                ErrorRecords.append(p)
+                self.error_records.append(p)
                 return p
 
         try:
-            schemaParser = self._getSchemaParser(self.SchemaFile)
+            schemaParser = self._getSchemaParser(self.schema_file)
         except Exception as e:
             if report_unexpected_exception:
                 message = "Schema Parser Unexpected exception [%s]: %s" % (e.__class__.__name__, e)
                 p = {"code": 0, "message": message}
-                ErrorRecords.append(p)
+                self.error_records.append(p)
                 return p
 
         try:
@@ -94,7 +91,7 @@ def runConversion(self, json_data, summarise=False, report_unexpected_exception=
             if report_unexpected_exception:
                 message = "Expression Checker Unexpected exception [%s]: %s" % (e.__class__.__name__, e)
                 p = {"code": 0, "message": message}
-                ErrorRecords.append(p)
+                self.error_records.append(p)
                 return p
 
         # get list of expressions
@@ -104,17 +101,17 @@ def runConversion(self, json_data, summarise=False, report_unexpected_exception=
             if report_unexpected_exception:
                 message = "Expression Getter Unexpected exception [%s]: %s" % (e.__class__.__name__, e)
                 p = {"code": 0, "message": message}
-                ErrorRecords.append(p)
+                self.error_records.append(p)
                 return p
 
         for conversion in conversions:
             rows = self._convertData(ConversionValidate, conversion, dataParser, json_data)
 
-        imms.append(Converted)
-        return imms
+        self.imms.append(self.converted)
+        return self.imms
 
     def getErrorRecords(self):
-        return ErrorRecords
+        return self.error_records
 
     def extract_patient_details(self, json_data, FlatFieldName):
         if not hasattr(self, "_cached_values"):

delta_backend/src/delta.py

@@ -28,7 +28,7 @@ def send_message(record):
         sqs_client.send_message(QueueUrl=failure_queue_url, MessageBody=json.dumps(message_body))
         logger.info("Record saved successfully to the DLQ")
     except ClientError as e:
-        logger.info(f"Error sending record to DLQ: {e}")
+        logger.error(f"Error sending record to DLQ: {e}")
 
 
 def get_vaccine_type(patientsk) -> str:
@@ -156,4 +156,7 @@ def handler(event, context):
         log_data["operation_outcome"] = operation_outcome
         firehose_log["event"] = log_data
         firehose_logger.send_log(firehose_log)
-        raise Exception(f"Delta Lambda failure: {e}")
+        return {
+            "statusCode": 500,
+            "body": "Records not processed",
+        }

delta_backend/tests/sample_data/__init__.py

@@ -0,0 +1,164 @@
+"""
+# This module provides a function to generate a sample FHIR Immunization resource
+"""
+def get_test_data_resource():
+    """
+    The returned resource includes details about the practitioner, patient,
+    vaccine code, location, and other relevant fields.
+    """
+    return {
+    "resourceType": "Immunization",
+    "contained": [
+        {
+            "resourceType": "Practitioner",
+            "id": "Pract1",
+            "name": [
+                {
+                    "family": "O'Reilly",
+                    "given": ["Ellena"]
+                }
+            ]
+        },
+        {
+            "resourceType": "Patient",
+            "id": "Pat1",
+            "identifier": [
+                {
+                    "system": "https://fhir.nhs.uk/Id/nhs-number",
+                    "value": "9674963871"
+                }
+            ],
+            "name": [
+                {
+                    "family": "GREIR",
+                    "given": ["SABINA"]
+                }
+            ],
+            "gender": "female",
+            "birthDate": "2019-01-31",
+            "address": [
+                {
+                    "postalCode": "GU14 6TU"
+                }
+            ]
+        }
+    ],
+    "extension": [
+        {
+            "url": "https://fhir.hl7.org.uk/StructureDefinition/Extension-UKCore-VaccinationProcedure",
+            "valueCodeableConcept": {
+                "coding": [
+                    {
+                        "system": "http://snomed.info/sct",
+                        "code": "1303503001",
+                        "display":
+                        "Administration of vaccine product containing only Human orthopneumovirus antigen (procedure)"
+                    }
+                ]
+            }
+        }
+    ],
+    "identifier": [
+        {
+            "system": "https://www.ravs.england.nhs.uk/",
+            "value": "0001_RSV_v5_RUN_2_CDFDPS-742_valid_dose_1"
+        }
+    ],
+    "status": "completed",
+    "vaccineCode": {
+        "coding": [
+            {
+                "system": "http://snomed.info/sct",
+                "code": "42605811000001109",
+                "display":
+                "Abrysvo vaccine powder and solvent for solution for injection 0.5ml vials (Pfizer Ltd) (product)"
+            }
+        ]
+    },
+    "patient": {
+        "reference": "#Pat1"
+    },
+    "occurrenceDateTime": "2024-06-10T18:33:25+00:00",
+    "recorded": "2024-06-10T18:33:25+00:00",
+    "primarySource": True,
+    "manufacturer": {
+        "display": "Pfizer"
+    },
+    "location": {
+        "type": "Location",
+        "identifier": {
+            "value": "J82067",
+            "system": "https://fhir.nhs.uk/Id/ods-organization-code"
+        }
+    },
+    "lotNumber": "RSVTEST",
+    "expirationDate": "2024-12-31",
+    "site": {
+        "coding": [
+            {
+                "system": "http://snomed.info/sct",
+                "code": "368208006",
+                "display": "Left upper arm structure (body structure)"
+            }
+        ]
+    },
+    "route": {
+        "coding": [
+            {
+                "system": "http://snomed.info/sct",
+                "code": "78421000",
+                "display": "Intramuscular route (qualifier value)"
+            }
+        ]
+    },
+    "doseQuantity": {
+        "value": 0.5,
+        "unit": "Milliliter (qualifier value)",
+        "system": "http://unitsofmeasure.org",
+        "code": "258773002"
+    },
+    "performer": [
+        {
+            "actor": {
+                "reference": "#Pract1"
+            }
+        },
+        {
+            "actor": {
+                "type": "Organization",
+                "identifier": {
+                    "system": "https://fhir.nhs.uk/Id/ods-organization-code",
+                    "value": "X0X0X"
+                }
+            }
+        }
+    ],
+    "reasonCode": [
+        {
+            "coding": [
+                {
+                    "code": "Test",
+                    "system": "http://snomed.info/sct"
+                }
+            ]
+        }
+    ],
+    "protocolApplied": [
+        {
+            "targetDisease": [
+                {
+                    "coding": [
+                        {
+                            "system": "http://snomed.info/sct",
+                            "code": "840539006",
+                            "display": "Disease caused by severe acute respiratory syndrome coronavirus 2"
+                        }
+                    ]
+                }
+            ],
+            "doseNumberPositiveInt": 1
+        }
+    ],
+    "id": "ca8ba2c6-2383-4465-b456-c1174c21cf31"
+    }
+    

delta_backend/tests/sample_data/test_resource_data.py

@@ -8,7 +8,7 @@
 from unittest.mock import patch, Mock
 from moto import mock_dynamodb, mock_sqs
 from boto3 import resource as boto3_resource, client as boto3_client
-from tests.utils_for_converter_tests import ValuesForTests, ErrorValuesForTests
+from utils_for_converter_tests import ValuesForTests, ErrorValuesForTests
 from botocore.config import Config
 from pathlib import Path
 from SchemaParser import SchemaParser
@@ -24,7 +24,6 @@
 request_json_data = ValuesForTests.json_data
 with patch.dict("os.environ", MOCK_ENV_VARS):
     from delta import handler, Converter
-    from Converter import imms, ErrorRecords
 
 
 @patch.dict("os.environ", MOCK_ENV_VARS, clear=True)
@@ -65,6 +64,16 @@ def setUp(self):
                 },
             ],
         )
+        self.logger_info_patcher = patch("logging.Logger.info")
+        self.mock_logger_info = self.logger_info_patcher.start()
+
+        self.logger_exception_patcher = patch("logging.Logger.exception")
+        self.mock_logger_exception = self.logger_exception_patcher.start()
+
+    def tearDown(self):
+        self.logger_exception_patcher.stop()
+        self.logger_info_patcher.stop()
+
 
     @staticmethod
     def get_event(event_name="INSERT", operation="operation", supplier="EMIS"):
@@ -101,7 +110,6 @@ def assert_dynamodb_record(self, operation_flag, items, expected_values, expecte
 
     def test_fhir_converter_json_direct_data(self):
         """it should convert fhir json data to flat json"""
-        imms.clear()
         json_data = json.dumps(ValuesForTests.json_data)
 
         start = time.time()
@@ -130,7 +138,6 @@ def test_fhir_converter_json_error_scenario(self):
         error_test_cases = [ErrorValuesForTests.missing_json, ErrorValuesForTests.json_dob_error]
 
         for test_case in error_test_cases:
-            imms.clear()
             json_data = json.dumps(test_case)
 
             start = time.time()
@@ -167,8 +174,6 @@ def test_handler_imms_convert_to_flat_json(self):
 
         for test_case in expected_action_flags:
             with self.subTest(test_case["Operation"]):
-                imms.clear()
-
                 event = self.get_event(operation=test_case["Operation"])
 
                 response = handler(event, None)
@@ -256,7 +261,7 @@ def test_get_conversions_exception(self, mock_get_conversions):
         response = converter.runConversion(ValuesForTests.json_data)
 
         # Check if the error message was added to ErrorRecords
-        self.assertEqual(len(converter.getErrorRecords()), 3)
+        self.assertEqual(len(converter.getErrorRecords()), 1)
         self.assertIn(
             "FHIR Parser Unexpected exception [JSONDecodeError]: Expecting value: line 1 column 1 (char 0)",
             converter.getErrorRecords()[0]["message"],
@@ -268,7 +273,6 @@ def test_get_conversions_exception(self, mock_get_conversions):
     def test_conversion_exceptions(self, mock_get_key_value, mock_get_conversions):
         mock_get_conversions.side_effect = Exception("Error while getting conversions")
         mock_get_key_value.side_effect = Exception("Key value retrieval failed")
-        ErrorRecords.clear()
         converter = Converter(fhir_data="some_data")
 
         schema = {