VED-222-FHIR-FLAT-JSON (#398)

* refactor test

* changed specs

* snake_case

* corrections

* empty string

* date conversion

* coverage

* add milliseconds logic

* datetime conversion

* unit test and coverage

---------

Co-authored-by: Matt Jarvis <[email protected]>

Author: Akol125

delta_backend/Dockerfile

@@ -1,4 +1,4 @@
-FROM public.ecr.aws/lambda/python:3.10 AS base
+FROM public.ecr.aws/lambda/python:3.11 AS base
 
 # Create a non-root user with a specific UID and GID
 RUN mkdir -p /home/appuser && \

delta_backend/poetry.lock

@@ -9,7 +9,7 @@ packages = [
 ]
 
 [tool.poetry.dependencies]
-python = "~3.10"
+python = "~3.11"
 boto3 = "~1.26.90"
 mypy-boto3-dynamodb = "^1.26.164"
 moto = "~4.2.11"

delta_backend/pyproject.toml

@@ -2,7 +2,7 @@
 # Handles the transformation logic for each field based on the schema
 # Root and base type expression checker functions
 import ExceptionMessages
-from datetime import datetime,timezone
+from datetime import datetime,timedelta, timezone
 from zoneinfo import ZoneInfo
 import re
 from LookUpData import LookUpData
@@ -122,7 +122,6 @@ def _convertToDate(self, expressionRule, fieldName, fieldValue, summarise, repor
                     self._log_error(fieldName, fieldValue, "Value is not a string")
                 return ""
             try:
-                fieldValue = re.sub(r"\.\d+(?=[+-]\d{2}:\d{2}$)", "", fieldValue)  # Remove milliseconds
                 dt = datetime.fromisoformat(fieldValue)
                 return dt.strftime(expressionRule)
             except ValueError as e:
@@ -136,40 +135,29 @@ def _convertToDateTime(self, expressionRule, fieldName, fieldValue, summarise, r
         if not fieldValue:
             return ""
 
-        # Reject partial dates like "2024" or "2024-05"
-        if re.match(r"^\d{4}(-\d{2})?$", fieldValue):
-            raise RecordError(
-                ExceptionMessages.RECORD_CHECK_FAILED,
-                f"{fieldName} rejected: partial datetime not accepted.",
-                f"Invalid partial datetime: {fieldValue}",
-            )
         try:
             dt = datetime.fromisoformat(fieldValue)
-        except ValueError:
+            if dt.tzinfo is None:
+                dt = dt.replace(tzinfo=timezone.utc)
+        except Exception as e:
             if report_unexpected_exception:
-                return f"Unexpected format: {fieldValue}"
+                self._log_error(fieldName, fieldValue, e)
+            return ""
 
         # Allow only +00:00 or +01:00 offsets (UTC and BST) and reject unsupported timezones
         offset = dt.utcoffset()
-        allowed_offsets = [ZoneInfo("UTC").utcoffset(dt),
-                           ZoneInfo("Europe/London").utcoffset(dt)]
-        if offset not in allowed_offsets:
-            raise RecordError(
-                ExceptionMessages.RECORD_CHECK_FAILED,
-                f"{fieldName} rejected: unsupported timezone.",
-                f"Unsupported offset: {offset}",
-            )
-
-        # Convert to UTC
-        dt_utc = dt.astimezone(ZoneInfo("UTC")).replace(microsecond=0)
-
-        format_str = expressionRule.replace("format:", "")
+        allowed_offsets = [timedelta(hours=0), timedelta(hours=1)]
+        if offset is not None and offset not in allowed_offsets:
+            if report_unexpected_exception:
+                self._log_error(fieldName, fieldValue, "Unsupported Format or offset")
+            return ""
 
-        if format_str == "csv-utc":
-            formatted = dt_utc.strftime("%Y%m%dT%H%M%S%z")
-            return formatted.replace("+0000", "00").replace("+0100", "01")
+        # remove microseconds
+        dt_format = dt.replace(microsecond=0)
 
-        return dt_utc.strftime(format_str)
+        formatted = dt_format.strftime("%Y%m%dT%H%M%S%z")
+        return formatted.replace("+0000", "00").replace("+0100", "01")
+    
 
     # Not Empty Validate - Returns exactly what is in the extracted fields no parsing or logic needed
     def _convertToNotEmpty(self, expressionRule, fieldName, fieldValue, summarise, report_unexpected_exception):

delta_backend/src/ConversionChecker.py

@@ -68,7 +68,7 @@
       "expression": {
         "expressionName": "Date Convert",
         "expressionType": "DATETIME",
-        "expressionRule": "format:csv-utc"
+        "expressionRule": "fhir-date"
       }
     },
     {

delta_backend/src/ConversionLayout.py

@@ -137,7 +137,12 @@ def extract_patient_details(self, json_data, FlatFieldName):
             self._cached_values = {}
 
         if not self._cached_values:
-            occurrence_time = datetime.fromisoformat(json_data.get("occurrenceDateTime", ""))
+            try:
+                occurrence_time = datetime.fromisoformat(json_data.get("occurrenceDateTime", ""))
+            except Exception as e:
+                    message = "DateTime conversion error [%s]: %s" % (e.__class__.__name__, e)
+                    error = self._log_error(message, code=ExceptionMessages.UNEXPECTED_EXCEPTION)
+                    return error
             patient = get_patient(json_data)
             if not patient:
                 return None

delta_backend/src/Converter.py

@@ -72,8 +72,8 @@
   "patient": {
     "reference": "#Pat1"
   },
-  "occurrenceDateTime": "2021-02-07T13:28:17+00:00",
-  "recorded": "2025-02-07",
+  "occurrenceDateTime": "2021-05-07T12:24:37+00:00",
+  "recorded": "2025-01-01T00:00:00.000000",
   "primarySource": "True",
   "manufacturer": {
     "display": "AstraZeneca Ltd"
@@ -86,7 +86,8 @@
     }
   },
   "lotNumber": "4120Z001",
-  "expirationDate": "2024-10-12",
+  "expirationDate": "2024-10-05",
+
   "site": {
     "coding": [
       {

delta_backend/tests/sample_data/fhir_sample.json

@@ -153,7 +153,7 @@ def test_fhir_converter_json_direct_data(self):
         self.assertEqual(flatJSON, expected_imms)
 
         errorRecords = FHIRConverter.getErrorRecords()
-        
+
         self.assertEqual(len(errorRecords), 0)
 
     def test_fhir_converter_json_error_scenario(self):
@@ -461,22 +461,52 @@ def test_convert_to_date_time(self, MockLookUpData):
 
         checker = ConversionChecker(dataParser, summarise=False, report_unexpected_exception=True)
 
-        valid_date_time = "2022-01-01T12:00:00+00:00"
-        result = checker._convertToDateTime("%Y%m%dT%H%M%S", "fieldName", valid_date_time, False, True)
-        self.assertEqual(result, "20220101T120000")
+        valid_date_time = "2025-01-01T12:00:00+00:00"
+        result = checker._convertToDateTime("fhir-date", "fieldName", valid_date_time, False, True)
+        self.assertEqual(result, "20250101T12000000")
+
+        valid_fhir_date = "2025-01-01T13:28:17+00:00"
+        result = checker._convertToDateTime("fhir-date", "fieldName", valid_fhir_date, False, True)
+        self.assertEqual(result, "20250101T13281700")
 
-        valid_csv_utc = "2022-01-01T13:28:17+00:00"
-        result = checker._convertToDateTime("format:csv-utc", "fieldName", valid_csv_utc, False, True)
-        self.assertEqual(result, "20220101T13281700")
+        valid_fhir_date = "2022-01-01"
+        result = checker._convertToDateTime("", "fieldName", valid_fhir_date, False, True)
+        self.assertEqual(result, "20220101T00000000")
+
+        valid_fhir_date = "2025-05-01T13:28:17+01:00"
+        result = checker._convertToDateTime("fhir-date", "fieldName", valid_fhir_date, False, True)
+        self.assertEqual(result, "20250501T13281701")
 
         invalid_date_time = "invalid_date_time"
-        result = checker._convertToDateTime("format:%Y%m%dT%H%M%S", "fieldName", invalid_date_time, False, True)
-        self.assertEqual(result, "Unexpected format: invalid_date_time")
+        result = checker._convertToDateTime("fhir-date", "fieldName", invalid_date_time, False, True)
+        self.assertEqual(result, "")
+
+        valid_date_time = "2025-01-01T12:00:00+03:00"
+        result = checker._convertToDateTime("fhir-date", "fieldName", valid_date_time, False, True)
+        self.assertEqual(result, "")
+
+        messages = [err["message"] for err in checker.errorRecords]
+        print(f"Error Test Case, {messages}")
+
+        self.assertIn("Unexpected exception [ValueError]", messages[0])
+        self.assertIn("Unsupported Format or offset", messages[1])
+
+        # Confirm Total Errors Per conversion
+        self.assertEqual(len(checker.errorRecords), 2)
 
-        # Empty input returns blank
-        result = checker._convertToDateTime("format:%Y%m%dT%H%M%S", "fieldName", "", False, True)
+        # Test for value Error
+        checker._log_error = Mock()
+
+        valid_fhir_date = "2022-01"
+        result = checker._convertToDateTime("fhir-date", "fieldName", valid_fhir_date, False, True)
         self.assertEqual(result, "")
 
+        # ensure we logged exactly that ValueError
+        checker._log_error.assert_called_once()
+        field, value, err = checker._log_error.call_args[0]
+        self.assertEqual((field, value), ("fieldName", valid_fhir_date))
+        self.assertIsInstance(err, ValueError)
+
     @patch("ConversionChecker.LookUpData")
     def test_convert_to_boolean(self, MockLookUpData):
         dataParser = Mock()

delta_backend/tests/test_convert_to_flat_json.py

@@ -1,6 +1,7 @@
 import unittest
 from unittest.mock import patch, MagicMock
 from Converter import Converter
+import ExceptionMessages
 
 
 class TestConverter(unittest.TestCase):
@@ -109,6 +110,21 @@ def test_extract_patient_details(self, mock_practitioner, mock_site_code, mock_p
         result = converter.extract_patient_details(json_data, "PERFORMING_PROFESSIONAL_SURNAME")
         self.assertEqual(result, "Smith")
 
+        # Pass a malformed datetime to trigger the exception
+        json_data = {"occurrenceDateTime": "invalid-date-time"}
+   
+        with patch.object(Converter, '_log_error', return_value="mocked-error") as mock_log_error:
+            converter = Converter({})
+            result = converter.extract_patient_details(json_data, "PERSON_SURNAME")
+
+            mock_log_error.assert_called_once()
+            error_message = mock_log_error.call_args[0][0]
+            error_code = mock_log_error.call_args[1]["code"]
+
+            self.assertIn("DateTime conversion error", error_message)
+            self.assertIn("ValueError", error_message)
+            self.assertEqual(error_code, ExceptionMessages.UNEXPECTED_EXCEPTION)
+            self.assertEqual(result, "mocked-error")
 
 if __name__ == "__main__":
     unittest.main()