VED-950 Add timeliness checks (#1076)

Author: dlzhry2nhs

lambdas/shared/src/common/data_quality/checker.py

@@ -1,39 +1,48 @@
 from dataclasses import dataclass
+from datetime import datetime, timezone
 
 from pydantic import ValidationError
 
 from common.data_quality.completeness import DataQualityCompletenessChecker, MissingFields
 from common.data_quality.models.immunization_batch_row_model import ImmunizationBatchRowModel
+from common.data_quality.timeliness import (
+    TimelinessCheckOutput,
+    get_ingested_timeliness_seconds,
+    get_recorded_timeliness_days,
+)
 from common.models.fhir_converter.converter import Converter
 
 
 @dataclass
 class DataQualityOutput:
     missing_fields: MissingFields
     invalid_fields: list[str]
-    timeliness: dict[str, int]
+    timeliness: TimelinessCheckOutput
+    validation_datetime: str
 
 
 class DataQualityChecker:
     """Runs data quality checks against an Immunisation and creates a Data Quality Output object"""
 
     def __init__(
         self,
-        completeness_checker: DataQualityCompletenessChecker,
         is_batch_csv: bool,
     ):
-        self.completeness_checker = completeness_checker
+        self.completeness_checker = DataQualityCompletenessChecker()
         self.data_quality_model = ImmunizationBatchRowModel
         self.is_batch_csv = is_batch_csv
 
     def run_checks(self, immunisation: dict) -> DataQualityOutput:
+        dq_checks_executed_timestamp = datetime.now(timezone.utc)
+
         if not self.is_batch_csv:
             immunisation = Converter(fhir_data=immunisation).run_conversion()
 
         return DataQualityOutput(
             missing_fields=self._check_completeness(immunisation),
             invalid_fields=self._check_validity(immunisation),
-            timeliness=self._check_timeliness(immunisation),
+            timeliness=self._check_timeliness(immunisation, dq_checks_executed_timestamp),
+            validation_datetime=dq_checks_executed_timestamp.isoformat(timespec="milliseconds").replace("+00:00", "Z"),
         )
 
     def _check_completeness(self, immunisation: dict) -> MissingFields:
@@ -56,5 +65,9 @@ def _check_validity(self, immunisation: dict) -> list[str]:
 
         return fields_with_errors
 
-    def _check_timeliness(self, immunisation: dict) -> dict[str, int]:
-        pass
+    @staticmethod
+    def _check_timeliness(immunisation: dict, datetime_now: datetime) -> TimelinessCheckOutput:
+        return TimelinessCheckOutput(
+            recorded_timeliness_days=get_recorded_timeliness_days(immunisation),
+            ingested_timeliness_seconds=get_ingested_timeliness_seconds(immunisation, datetime_now),
+        )

lambdas/shared/src/common/data_quality/timeliness.py

@@ -1,6 +1,25 @@
 """Module containing functions to aid with timeliness checks"""
 
 import datetime
+import math
+from dataclasses import dataclass
+from typing import Optional
+
+
+@dataclass
+class TimelinessCheckOutput:
+    """
+    The required outputs for the data quality team for timeliness checks.
+
+    recorded_timeliness_days - the number of days between the vaccination taking place and being recorded into the
+    clinical system
+
+    ingested_timeliness_seconds - the seconds between the vaccination taking place and the record being provided to the
+    Immunisation FHIR API/batch processor for ingestion
+    """
+
+    recorded_timeliness_days: Optional[int]
+    ingested_timeliness_seconds: Optional[int]
 
 
 def parse_csv_date(d: str) -> datetime.date:
@@ -18,3 +37,48 @@ def parse_csv_datetime(d: str) -> datetime.datetime:
             return datetime.datetime.strptime(d, "%Y%m%dT%H%M%S").replace(tzinfo=datetime.timezone.utc)
         case _:
             raise ValueError("Invalid datetime format provided")
+
+
+def get_recorded_date_from_immunisation(immunization: dict) -> Optional[datetime.date]:
+    try:
+        parsed_recorded_date = parse_csv_date(immunization.get("RECORDED_DATE", ""))
+    except ValueError:
+        # Completeness and validity checks will catch these issues separately. Just return None here.
+        return None
+
+    return parsed_recorded_date
+
+
+def get_occurrence_datetime_from_immunisation(immunization: dict) -> Optional[datetime.datetime]:
+    try:
+        parsed_recorded_date = parse_csv_datetime(immunization.get("DATE_AND_TIME", ""))
+    except ValueError:
+        # Completeness and validity checks will catch these issues separately. Just return None here.
+        return None
+
+    return parsed_recorded_date
+
+
+def get_recorded_timeliness_days(immunisation: dict) -> Optional[int]:
+    """Gets the time delta in days between the recorded date and occurrence date. Returns None if either of the fields
+    are not provided or are invalid dates."""
+    recorded_date = get_recorded_date_from_immunisation(immunisation)
+    occurrence_date_time = get_occurrence_datetime_from_immunisation(immunisation)
+
+    if recorded_date is None or occurrence_date_time is None:
+        return None
+
+    occurrence_date = datetime.date(occurrence_date_time.year, occurrence_date_time.month, occurrence_date_time.day)
+
+    return (recorded_date - occurrence_date).days
+
+
+def get_ingested_timeliness_seconds(immunisation: dict, datetime_now: datetime.datetime) -> Optional[int]:
+    """Gets the time delta in seconds between the time of ingestion into the system and the vaccination occurrence
+    datetime. Returns None if the occurrence datetime field is not provided or is invalid."""
+    occurrence_date_time = get_occurrence_datetime_from_immunisation(immunisation)
+
+    if not occurrence_date_time:
+        return None
+
+    return math.floor((datetime_now - occurrence_date_time).total_seconds())

lambdas/shared/tests/test_common/data_quality/sample_values.py

@@ -64,8 +64,8 @@
         ]
     },
     "patient": {"reference": "#Pat1"},
-    "occurrenceDateTime": "2021-02-07T13:28:17+00:00",
-    "recorded": "2021-02-07T13:28:17+00:00",
+    "occurrenceDateTime": "2024-05-11T12:00:00+00:00",
+    "recorded": "2024-05-15",
     "primarySource": True,
     "manufacturer": {"display": "AstraZeneca Ltd"},
     "location": {
@@ -76,7 +76,7 @@
         },
     },
     "lotNumber": "4120Z001",
-    "expirationDate": "2021-07-02",
+    "expirationDate": "2024-12-02",
     "site": {
         "coding": [
             {
@@ -99,7 +99,7 @@
         "value": str(Decimal(0.5)),
         "unit": "milliliter",
         "system": "http://snomed.info/sct",
-        "code": "ml",
+        "code": "258773002",
     },
     "performer": [
         {"actor": {"reference": "#Pract1"}},
@@ -147,7 +147,7 @@
     "ACTION_FLAG": "UPDATE",
     "PERFORMING_PROFESSIONAL_FORENAME": "ALICE",
     "PERFORMING_PROFESSIONAL_SURNAME": "SMITH",
-    "RECORDED_DATE": "20250306",
+    "RECORDED_DATE": "20240515",
     "PRIMARY_SOURCE": "True",
     "VACCINATION_PROCEDURE_CODE": "1324681000000101",
     "VACCINATION_PROCEDURE_TERM": "Procedure Term",

lambdas/shared/tests/test_common/data_quality/test_checker.py

@@ -3,9 +3,8 @@
 import unittest
 from unittest.mock import patch
 
-from common.data_quality.checker import DataQualityChecker
-from common.data_quality.completeness import DataQualityCompletenessChecker
-from test_common.data_quality.sample_values import VALID_BATCH_IMMUNISATION
+from common.data_quality.checker import DataQualityChecker, DataQualityOutput
+from test_common.data_quality.sample_values import VALID_BATCH_IMMUNISATION, VALID_FHIR_IMMUNISATION
 
 
 class TestDataQualityChecker(unittest.TestCase):
@@ -15,9 +14,23 @@ def setUp(self):
         self.mock_date_today = date_today_patcher.start()
         self.mock_date_today.date.today.return_value = datetime.date(2024, 5, 20)
 
-        completeness_checker = DataQualityCompletenessChecker()
-        self.batch_dq_checker = DataQualityChecker(completeness_checker, is_batch_csv=True)
-        self.fhir_json_dq_checker = DataQualityChecker(completeness_checker, is_batch_csv=False)
+        # Fix datetime.now
+        self.mock_fixed_datetime = datetime.datetime(2024, 5, 20, 14, 12, 30, 123, tzinfo=datetime.timezone.utc)
+        datetime_now_patcher = patch("common.data_quality.checker.datetime", wraps=datetime.datetime)
+        self.mock_datetime_now = datetime_now_patcher.start()
+        self.mock_datetime_now.now.return_value = self.mock_fixed_datetime
+
+        self.batch_dq_checker = DataQualityChecker(is_batch_csv=True)
+        self.fhir_json_dq_checker = DataQualityChecker(is_batch_csv=False)
+
+    def assert_successful_result(self, result: DataQualityOutput) -> None:
+        self.assertEqual([], result.missing_fields.optional_fields)
+        self.assertEqual([], result.missing_fields.mandatory_fields)
+        self.assertEqual([], result.missing_fields.required_fields)
+        self.assertEqual([], result.invalid_fields)
+        self.assertEqual(4, result.timeliness.recorded_timeliness_days)
+        self.assertEqual(785550, result.timeliness.ingested_timeliness_seconds)
+        self.assertEqual("2024-05-20T14:12:30.000Z", result.validation_datetime)
 
     def test_check_validity_returns_empty_list_when_data_is_valid(self):
         validation_result = self.batch_dq_checker._check_validity(VALID_BATCH_IMMUNISATION)
@@ -76,3 +89,77 @@ def test_check_validity_returns_list_of_multiple_invalid_fields_for_multiple_fai
         self.assertEqual(
             ["NHS_NUMBER", "PERSON_POSTCODE", "EXPIRY_DATE", "DOSE_AMOUNT", "INDICATION_CODE"], validation_result
         )
+
+    def test_check_timeliness_calculates_the_timeliness_diffs(self):
+        timeliness_output = self.batch_dq_checker._check_timeliness(VALID_BATCH_IMMUNISATION, self.mock_fixed_datetime)
+
+        self.assertEqual(4, timeliness_output.recorded_timeliness_days)
+        self.assertEqual(785550, timeliness_output.ingested_timeliness_seconds)
+
+    def test_check_timeliness_returns_none_for_recorded_timeliness_when_relevant_field_invalid(self):
+        invalid_batch_imms_payload = copy.deepcopy(VALID_BATCH_IMMUNISATION)
+        invalid_batch_imms_payload["RECORDED_DATE"] = ""
+
+        timeliness_output = self.batch_dq_checker._check_timeliness(invalid_batch_imms_payload, self.mock_fixed_datetime)
+
+        self.assertIsNone(timeliness_output.recorded_timeliness_days)
+        self.assertEqual(785550, timeliness_output.ingested_timeliness_seconds)
+
+    def test_check_timeliness_returns_none_for_both_when_date_and_time_field_invalid(self):
+        invalid_batch_imms_payload = copy.deepcopy(VALID_BATCH_IMMUNISATION)
+        invalid_batch_imms_payload["DATE_AND_TIME"] = "20245"
+
+        timeliness_output = self.batch_dq_checker._check_timeliness(invalid_batch_imms_payload, self.mock_fixed_datetime)
+
+        self.assertIsNone(timeliness_output.recorded_timeliness_days)
+        self.assertIsNone(timeliness_output.ingested_timeliness_seconds)
+
+    def test_run_checks_returns_correct_output_for_valid_data_for_csv_payload(self):
+        result = self.batch_dq_checker.run_checks(VALID_BATCH_IMMUNISATION)
+        self.assert_successful_result(result)
+
+    def test_run_checks_returns_correct_output_for_valid_data_for_fhir_payload(self):
+        result = self.fhir_json_dq_checker.run_checks(VALID_FHIR_IMMUNISATION)
+        self.assert_successful_result(result)
+
+    def test_run_checks_returns_correct_output_for_invalid_data_for_csv_payload(self):
+        invalid_batch_imms_payload = copy.deepcopy(VALID_BATCH_IMMUNISATION)
+        invalid_batch_imms_payload["NHS_NUMBER"] = "12345678901"
+        invalid_batch_imms_payload["RECORDED_DATE"] = "20240137"
+        invalid_batch_imms_payload["PERSON_DOB"] = ""
+        invalid_batch_imms_payload["DOSE_AMOUNT"] = "6.789"
+        invalid_batch_imms_payload["BATCH_NUMBER"] = ""
+
+        result = self.batch_dq_checker.run_checks(invalid_batch_imms_payload)
+
+        self.assertEqual([], result.missing_fields.optional_fields)
+        self.assertEqual(["PERSON_DOB"], result.missing_fields.mandatory_fields)
+        self.assertEqual(["BATCH_NUMBER"], result.missing_fields.required_fields)
+
+        # Fields which are subject to validation and are also empty will appear in both the completeness and validity
+        # checks e.g. PERSON_DOB
+        self.assertEqual(["NHS_NUMBER", "PERSON_DOB", "DOSE_AMOUNT"], result.invalid_fields)
+        self.assertIsNone(result.timeliness.recorded_timeliness_days)
+        self.assertEqual(785550, result.timeliness.ingested_timeliness_seconds)
+
+    def test_run_checks_returns_correct_output_for_invalid_data_for_fhir_payload(self):
+        invalid_fhir_imms_payload = copy.deepcopy(VALID_FHIR_IMMUNISATION)
+        invalid_fhir_imms_payload["contained"][1]["identifier"][0]["value"] = "12345678901"
+        invalid_fhir_imms_payload["recorded"] = "2024-01-37"
+        del invalid_fhir_imms_payload["contained"][1]["birthDate"]
+        invalid_fhir_imms_payload["doseQuantity"]["value"] = "6.789"
+        invalid_fhir_imms_payload["lotNumber"] = ""
+
+        result = self.fhir_json_dq_checker.run_checks(invalid_fhir_imms_payload)
+
+        self.assertEqual([], result.missing_fields.optional_fields)
+        # Worth noting that due to the use of the fhir converter, invalid dates will be mapped to an empty string hence
+        # will also show up here where they would not in batch validation
+        self.assertEqual(["PERSON_DOB", "RECORDED_DATE"], result.missing_fields.mandatory_fields)
+        self.assertEqual(["BATCH_NUMBER"], result.missing_fields.required_fields)
+
+        # Fields which are subject to validation and are also empty will appear in both the completeness and validity
+        # checks e.g. PERSON_DOB
+        self.assertEqual(["NHS_NUMBER", "PERSON_DOB", "DOSE_AMOUNT"], result.invalid_fields)
+        self.assertIsNone(result.timeliness.recorded_timeliness_days)
+        self.assertEqual(785550, result.timeliness.ingested_timeliness_seconds)