datetime conversion

Author: Akol125

delta_backend/Dockerfile

@@ -1,4 +1,4 @@
-FROM public.ecr.aws/lambda/python:3.10 AS base
+FROM public.ecr.aws/lambda/python:3.11 AS base
 
 # Create a non-root user with a specific UID and GID
 RUN mkdir -p /home/appuser && \

delta_backend/poetry.lock

@@ -9,7 +9,7 @@ packages = [
 ]
 
 [tool.poetry.dependencies]
-python = "~3.10"
+python = "~3.11"
 boto3 = "~1.26.90"
 mypy-boto3-dynamodb = "^1.26.164"
 moto = "~4.2.11"

delta_backend/pyproject.toml

@@ -2,7 +2,7 @@
 # Handles the transformation logic for each field based on the schema
 # Root and base type expression checker functions
 import ExceptionMessages
-from datetime import datetime,timezone
+from datetime import datetime,timedelta, timezone
 from zoneinfo import ZoneInfo
 import re
 from LookUpData import LookUpData
@@ -136,40 +136,36 @@ def _convertToDateTime(self, expressionRule, fieldName, fieldValue, summarise, r
         if not fieldValue:
             return ""
 
-        # Reject partial dates like "2024" or "2024-05"
-        if re.match(r"^\d{4}(-\d{2})?$", fieldValue):
-            raise RecordError(
-                ExceptionMessages.RECORD_CHECK_FAILED,
-                f"{fieldName} rejected: partial datetime not accepted.",
-                f"Invalid partial datetime: {fieldValue}",
-            )
         try:
-            dt = datetime.fromisoformat(fieldValue)
-        except ValueError:
+            if not expressionRule:
+                # Handle YYYYMMDD format to 
+                dt = datetime.strptime(fieldValue, "%Y%m%d")
+                dt = dt.replace(hour=0, minute=0, second=0) 
+            else:
+                dt = datetime.fromisoformat(fieldValue)
+                if dt.tzinfo is None:
+                    dt = dt.replace(tzinfo=timezone.utc)
+        except Exception as e:
             if report_unexpected_exception:
-                return f"Unexpected format: {fieldValue}"
+                self._log_error(fieldName, fieldValue, e)
+            return ""
 
         # Allow only +00:00 or +01:00 offsets (UTC and BST) and reject unsupported timezones
         offset = dt.utcoffset()
-        allowed_offsets = [ZoneInfo("UTC").utcoffset(dt),
-                           ZoneInfo("Europe/London").utcoffset(dt)]
-        if offset not in allowed_offsets:
-            raise RecordError(
-                ExceptionMessages.RECORD_CHECK_FAILED,
-                f"{fieldName} rejected: unsupported timezone.",
-                f"Unsupported offset: {offset}",
-            )
+        allowed_offsets = [timedelta(hours=0), timedelta(hours=1)]
+        if offset is not None and offset not in allowed_offsets:
+            if report_unexpected_exception:
+                self._log_error(fieldName, fieldValue, "Unsupported Format or offset")
+            return "" 
 
         # Convert to UTC
-        dt_utc = dt.astimezone(ZoneInfo("UTC")).replace(microsecond=0)
-
-        format_str = expressionRule.replace("format:", "")
+        dt_utc = dt.replace(microsecond=0)
 
-        if format_str == "csv-utc":
+        if expressionRule == "csv-utc":
             formatted = dt_utc.strftime("%Y%m%dT%H%M%S%z")
             return formatted.replace("+0000", "00").replace("+0100", "01")
 
-        return dt_utc.strftime(format_str)
+        return dt_utc.strftime("%Y%m%dT%H%M%S%z")
 
     # Not Empty Validate - Returns exactly what is in the extracted fields no parsing or logic needed
     def _convertToNotEmpty(self, expressionRule, fieldName, fieldValue, summarise, report_unexpected_exception):

delta_backend/src/ConversionChecker.py

@@ -68,7 +68,7 @@
       "expression": {
         "expressionName": "Date Convert",
         "expressionType": "DATETIME",
-        "expressionRule": "format:csv-utc"
+        "expressionRule": "csv-utc"
       }
     },
     {

delta_backend/src/ConversionLayout.py

@@ -137,7 +137,12 @@ def extract_patient_details(self, json_data, FlatFieldName):
             self._cached_values = {}
 
         if not self._cached_values:
-            occurrence_time = datetime.fromisoformat(json_data.get("occurrenceDateTime", ""))
+            try:
+                occurrence_time = datetime.fromisoformat(json_data.get("occurrenceDateTime", ""))
+            except Exception as e:
+                    message = "DateTime conversion error [%s]: %s" % (e.__class__.__name__, e)
+                    error = self._log_error(message, code=ExceptionMessages.UNEXPECTED_EXCEPTION)
+                    return error
             patient = get_patient(json_data)
             if not patient:
                 return None

delta_backend/src/Converter.py

@@ -72,8 +72,8 @@
   "patient": {
     "reference": "#Pat1"
   },
-  "occurrenceDateTime": "2021-02-07T13:28:17+00:00",
-  "recorded": "2025-02-07",
+  "occurrenceDateTime": "2021-05-07T12:24:37.67867+01:00",
+  "recorded": "2025-01-01T00:00:00.00",
   "primarySource": "True",
   "manufacturer": {
     "display": "AstraZeneca Ltd"
@@ -86,7 +86,7 @@
     }
   },
   "lotNumber": "4120Z001",
-  "expirationDate": "2024-10-12",
+  "expirationDate": "2024-10-05",
   "site": {
     "coding": [
       {

delta_backend/tests/sample_data/fhir_sample.json

@@ -462,19 +462,19 @@ def test_convert_to_date_time(self, MockLookUpData):
         checker = ConversionChecker(dataParser, summarise=False, report_unexpected_exception=True)
 
         valid_date_time = "2022-01-01T12:00:00+00:00"
-        result = checker._convertToDateTime("%Y%m%dT%H%M%S", "fieldName", valid_date_time, False, True)
-        self.assertEqual(result, "20220101T120000")
+        result = checker._convertToDateTime("csv-utc", "fieldName", valid_date_time, False, True)
+        self.assertEqual(result, "20220101T12000000")
 
         valid_csv_utc = "2022-01-01T13:28:17+00:00"
-        result = checker._convertToDateTime("format:csv-utc", "fieldName", valid_csv_utc, False, True)
+        result = checker._convertToDateTime("csv-utc", "fieldName", valid_csv_utc, False, True)
         self.assertEqual(result, "20220101T13281700")
 
         invalid_date_time = "invalid_date_time"
-        result = checker._convertToDateTime("format:%Y%m%dT%H%M%S", "fieldName", invalid_date_time, False, True)
-        self.assertEqual(result, "Unexpected format: invalid_date_time")
+        result = checker._convertToDateTime("csv-utc", "fieldName", invalid_date_time, False, True)
+        self.assertEqual(result, "")
 
         # Empty input returns blank
-        result = checker._convertToDateTime("format:%Y%m%dT%H%M%S", "fieldName", "", False, True)
+        result = checker._convertToDateTime("csv-utc", "fieldName", "", False, True)
         self.assertEqual(result, "")
 
     @patch("ConversionChecker.LookUpData")