VED-235 Deliver imms as list (not string) (#407)

* Dose Amount, ACTION FLAG, Remove Array

* check conversion

Author: nhsdevws

delta_backend/poetry.lock

@@ -3,7 +3,6 @@
 # Root and base type expression checker functions
 import ExceptionMessages
 from datetime import datetime,timedelta, timezone
-from zoneinfo import ZoneInfo
 import re
 from LookUpData import LookUpData
 
@@ -259,7 +258,7 @@ def _convertToDefaultTo(self, expressionRule, fieldName, fieldValue, summarise,
         try:
             if fieldValue == "":
                 return expressionRule
-            return fieldValue
+            return str(fieldValue)
         except Exception as e:
             if report_unexpected_exception:
                 message = ExceptionMessages.MESSAGES[ExceptionMessages.UNEXPECTED_EXCEPTION] % (e.__class__.__name__, e)

delta_backend/src/ConversionChecker.py

@@ -19,7 +19,6 @@
 class Converter:
 
     def __init__(self, fhir_data):
-        self.imms = []
         self.converted = {}
         self.error_records = []
         self.fhir_data = fhir_data  # Store JSON data directly
@@ -123,11 +122,10 @@ def runConversion(self, json_data, summarise=False, report_unexpected_exception=
         if error_records:
             error_summary = error_records
         else:
-            error_summary = ""
+            error_summary = []
         self.converted["CONVERSION_ERRORS"] = error_summary
 
-        self.imms.append(self.converted)
-        return self.imms
+        return self.converted
 
     def getErrorRecords(self):
         return self.error_records

delta_backend/src/Converter.py

@@ -68,13 +68,12 @@ def handler(event, context):
                 supplier_system = new_image["SupplierSystem"]["S"]
                 if supplier_system not in ("DPSFULL", "DPSREDUCED"):
                     operation = new_image["Operation"]["S"]
-                    if operation == "CREATE":
-                        operation = "NEW"
+                    action_flag = "NEW" if operation == "CREATE" else operation
                     resource_json = json.loads(new_image["Resource"]["S"])
                     FHIRConverter = Converter(json.dumps(resource_json))
                     flat_json = FHIRConverter.runConversion(resource_json)  # Get the flat JSON
                     error_records = FHIRConverter.getErrorRecords()
-                    flat_json[0]["ACTION_FLAG"] = operation
+                    flat_json["ACTION_FLAG"] = action_flag
                     response = delta_table.put_item(
                         Item={
                             "PK": str(uuid.uuid4()),
@@ -84,7 +83,7 @@ def handler(event, context):
                             "SupplierSystem": supplier_system,
                             "DateTimeStamp": approximate_creation_time.isoformat(),
                             "Source": delta_source,
-                            "Imms": str(flat_json),
+                            "Imms": flat_json,
                             "ExpiresAt": expiry_time_epoch,
                         }
                     )

delta_backend/src/delta.py

@@ -34,11 +34,11 @@
 
 # Convert JSON list of dicts to CSV
 if result:
-    keys = result[0].keys()
+    keys = result.keys()
     with open(csv_path, "w", newline="") as f:
         writer = csv.DictWriter(f, fieldnames=keys)
         writer.writeheader()
-        writer.writerows(result)
+        writer.writerow(result)
 
 print("CSV saved to:", csv_path)
 print("JSON saved to:", json_path)

delta_backend/tests/check_conversion.py

@@ -1,18 +1,10 @@
 import json
 import unittest
-import os
-import time
-from datetime import datetime, timedelta
-from decimal import Decimal
 from copy import deepcopy
-from unittest import TestCase
 from unittest.mock import patch, Mock
 from moto import mock_dynamodb, mock_sqs
-from boto3 import resource as boto3_resource, client as boto3_client
+from boto3 import resource as boto3_resource
 from tests.utils_for_converter_tests import ValuesForTests, ErrorValuesForTests
-from botocore.config import Config
-from pathlib import Path
-from zoneinfo import ZoneInfo
 from SchemaParser import SchemaParser
 from Converter import Converter
 from ConversionChecker import ConversionChecker, RecordError
@@ -112,7 +104,7 @@ def get_event(event_name="INSERT", operation="operation", supplier="EMIS"):
         """Returns test event data."""
         return ValuesForTests.get_event(event_name, operation, supplier)
 
-    def assert_dynamodb_record(self, operation_flag, items, expected_values, expected_imms, response):
+    def assert_dynamodb_record(self, operation_flag, action_flag, items, expected_values, expected_imms, response):
         """
         Asserts that a record with the expected structure exists in DynamoDB.
         Ignores dynamically generated fields like PK, DateTimeStamp, and ExpiresAt.
@@ -125,16 +117,17 @@ def assert_dynamodb_record(self, operation_flag, items, expected_values, expecte
             {k: v for k, v in item.items() if k not in ["PK", "DateTimeStamp", "ExpiresAt"]}
             for item in items
             if item.get("Operation") == operation_flag
+            and item.get("Imms", {}).get("ACTION_FLAG") == action_flag
         ]
 
         self.assertGreater(len(filtered_items), 0, f"No matching item found for {operation_flag}")
 
         imms_data = filtered_items[0]["Imms"]
-        self.assertIsInstance(imms_data, str)
+        self.assertIsInstance(imms_data, dict)
         self.assertGreater(len(imms_data), 0)
 
         # Check Imms JSON structure matches exactly
-        self.assertEqual(imms_data, str(expected_imms), "Imms data does not match expected JSON structure")
+        self.assertEqual(imms_data, expected_imms, "Imms data does not match expected JSON structure")
 
         for key, expected_value in expected_values.items():
             self.assertIn(key, filtered_items[0], f"{key} is missing")
@@ -169,7 +162,6 @@ def test_fhir_converter_json_error_scenario(self):
             errorRecords = FHIRConverter.getErrorRecords()
 
             # Check if bad data creates error records
-            print(f"Error Test Case, {len(errorRecords)}")
             self.assertTrue(len(errorRecords) > 0)
 
     def test_handler_imms_convert_to_flat_json(self):
@@ -195,7 +187,12 @@ def test_handler_imms_convert_to_flat_json(self):
                 expected_imms = ValuesForTests.get_expected_imms(test_case["EXPECTED_ACTION_FLAG"])
 
                 self.assert_dynamodb_record(
-                    test_case["EXPECTED_ACTION_FLAG"], items, expected_values, expected_imms, response
+                    test_case["Operation"],
+                    test_case["EXPECTED_ACTION_FLAG"],
+                    items,
+                    expected_values,
+                    expected_imms,
+                    response
                 )
 
                 result = self.table.scan()
@@ -435,7 +432,6 @@ def test_convert_to_date(self, MockLookUpData):
 
         # 7 Validate all error logs of various responses
         messages = [err["message"] for err in checker.errorRecords]
-        print(f"Error Test Case, {messages}")
 
         self.assertIn("Value is not a string", messages)
 
@@ -486,7 +482,6 @@ def test_convert_to_date_time(self, MockLookUpData):
         self.assertEqual(result, "")
 
         messages = [err["message"] for err in checker.errorRecords]
-        print(f"Error Test Case, {messages}")
 
         self.assertIn("Unexpected exception [ValueError]", messages[0])
         self.assertIn("Unsupported Format or offset", messages[1])
@@ -676,7 +671,7 @@ def _run_test(self, expected_forename):
         """Helper function to run the test"""
         self.converter = Converter(json.dumps(request_json_data))
         flat_json = self.converter.runConversion(request_json_data, False, True)
-        self.assertEqual(flat_json[0]["PERSON_FORENAME"], expected_forename)
+        self.assertEqual(flat_json["PERSON_FORENAME"], expected_forename)
 
 class TestPersonSurNameToFlatJson(unittest.TestCase):
 
@@ -764,7 +759,7 @@ def _run_test_surname(self, expected_forename):
         """Helper function to run the test"""
         self.converter = Converter(json.dumps(request_json_data))
         flat_json = self.converter.runConversion(request_json_data, False, True)
-        self.assertEqual(flat_json[0]["PERSON_SURNAME"], expected_forename)
+        self.assertEqual(flat_json["PERSON_SURNAME"], expected_forename)
 
 
 class TestPersonPostalCodeToFlatJson(unittest.TestCase):
@@ -853,7 +848,7 @@ def _run_postal_code_test(self, expected_postal_code):
         """Helper function to run the test"""
         self.converter = Converter(json.dumps(request_json_data))
         flat_json = self.converter.runConversion(request_json_data, False, True)
-        self.assertEqual(flat_json[0]["PERSON_POSTCODE"], expected_postal_code)
+        self.assertEqual(flat_json["PERSON_POSTCODE"], expected_postal_code)
 
 
 class TestPersonSiteCodeToFlatJson(unittest.TestCase):
@@ -972,7 +967,7 @@ def _run_site_code_test(self, expected_site_code):
         """Helper function to run the test"""
         self.converter = Converter(json.dumps(request_json_data))
         flat_json = self.converter.runConversion(request_json_data, False, True)
-        self.assertEqual(flat_json[0].get("SITE_CODE"), expected_site_code)
+        self.assertEqual(flat_json.get("SITE_CODE"), expected_site_code)
 
 
 class TestPersonSiteUriToFlatJson(unittest.TestCase):
@@ -1063,7 +1058,7 @@ def _run_site_uri_test(self, expected_site_code):
         """Helper function to run the test"""
         self.converter = Converter(json.dumps(request_json_data))
         flat_json = self.converter.runConversion(request_json_data, False, True)
-        self.assertEqual(flat_json[0].get("SITE_CODE_TYPE_URI"), expected_site_code)
+        self.assertEqual(flat_json.get("SITE_CODE_TYPE_URI"), expected_site_code)
 
 
 class TestPractitionerForeNameToFlatJson(unittest.TestCase):
@@ -1179,7 +1174,7 @@ def _run_practitioner_test(self, expected_forename):
         """Helper function to run the test"""
         self.converter = Converter(json.dumps(request_json_data))
         flat_json = self.converter.runConversion(request_json_data, False, True)
-        self.assertEqual(flat_json[0]["PERFORMING_PROFESSIONAL_FORENAME"], expected_forename)
+        self.assertEqual(flat_json["PERFORMING_PROFESSIONAL_FORENAME"], expected_forename)
 
 
 class TestPractitionerSurNameToFlatJson(unittest.TestCase):
@@ -1293,7 +1288,7 @@ def _run_test_practitioner_surname(self, expected_forename):
         """Helper function to run the test"""
         self.converter = Converter(json.dumps(request_json_data))
         flat_json = self.converter.runConversion(request_json_data, False, True)
-        self.assertEqual(flat_json[0]["PERFORMING_PROFESSIONAL_SURNAME"], expected_forename)
+        self.assertEqual(flat_json["PERFORMING_PROFESSIONAL_SURNAME"], expected_forename)
 
     if __name__ == "__main__":
         unittest.main()

delta_backend/tests/test_convert_to_flat_json.py

@@ -78,9 +78,9 @@ def test_runConversion_success(self, mock_checker, mock_schema_parser, mock_fhir
 
         result = converter.runConversion(json_data={"occurrenceDateTime": "2023-01-01T12:00:00"})
 
-        self.assertEqual(len(result), 1)
-        self.assertIn("someFlatField", result[0])
-        self.assertEqual(result[0]["someFlatField"], "converted_value")
+        self.assertEqual(len(result), 2)
+        self.assertIn("someFlatField", result)
+        self.assertEqual(result["someFlatField"], "converted_value")
 
     @patch('Converter.extract_person_names', return_value=("John", "Doe"))
     @patch('Converter.get_valid_address', return_value="12345")

delta_backend/tests/test_converter.py

@@ -167,8 +167,7 @@ def get_event(event_name="INSERT", operation="CREATE", supplier="EMIS"):
     @staticmethod
     def get_expected_imms(expected_action_flag):
         """Returns expected Imms JSON data with the given action flag."""
-        return [
-            {
+        return {
                 "NHS_NUMBER": "9000000009",
                 "PERSON_FORENAME": "Sam",
                 "PERSON_SURNAME": "Trailor",
@@ -203,12 +202,10 @@ def get_expected_imms(expected_action_flag):
                 "INDICATION_CODE": "443684005",
                 "LOCATION_CODE": "EC1111",
                 "LOCATION_CODE_TYPE_URI": "https://fhir.nhs.uk/Id/ods-organization-code",
-                "CONVERSION_ERRORS": ''
+                "CONVERSION_ERRORS": []
             }
-        ]
 
-    expected_imms = [
-        {
+    expected_imms = {
             "NHS_NUMBER": "9000000009",
             "PERSON_FORENAME": "Sam",
             "PERSON_SURNAME": "Trailor",
@@ -243,12 +240,10 @@ def get_expected_imms(expected_action_flag):
             "INDICATION_CODE": "443684005",
             "LOCATION_CODE": "EC1111",
             "LOCATION_CODE_TYPE_URI": "https://fhir.nhs.uk/Id/ods-organization-code",
-            "CONVERSION_ERRORS": ''
+            "CONVERSION_ERRORS": []
         }
-    ]
 
-    expected_imms2 = [
-        {
+    expected_imms2 = {
             "NHS_NUMBER": "9000000009",
             "PERSON_FORENAME": "Sam",
             "PERSON_SURNAME": "Trailor",
@@ -283,9 +278,8 @@ def get_expected_imms(expected_action_flag):
             "INDICATION_CODE": "443684005",
             "LOCATION_CODE": "EC1111",
             "LOCATION_CODE_TYPE_URI": "https://fhir.nhs.uk/Id/ods-organization-code",
-            "CONVERSION_ERRORS": ''
+            "CONVERSION_ERRORS": []
         }
-    ]
 
 
 class ErrorValuesForTests:
@@ -437,6 +431,6 @@ def get_expected_imms_error_output(expected_action_flag):
                 "INDICATION_CODE": "443684005",
                 "LOCATION_CODE": "E712",
                 "LOCATION_CODE_TYPE_URI": "https://fhir.nhs.uk/Id/ods-organization-code",
-                "CONVERSION_ERRORS": ''
+                "CONVERSION_ERRORS": []
             }
         ]