VED-948: Add completeness checker (#1073)

Author: edhall-nhs

lambdas/filenameprocessor/src/file_name_processor.py

@@ -6,7 +6,6 @@
 (ODS code has multiple lengths)
 """
 
-import argparse
 from uuid import uuid4
 
 from audit_table import upsert_audit_table
@@ -264,7 +263,6 @@ def handle_extended_attributes_file(
             FileStatus.PROCESSING,
         )
 
-        # TODO: agree the prefix with DPS
         dest_file_key = f"{DPS_DESTINATION_PREFIX}/{file_key}"
         copy_file_to_external_bucket(
             bucket_name,
@@ -339,18 +337,3 @@ def lambda_handler(event: dict, context) -> None:  # pylint: disable=unused-argu
         handle_record(record)
 
     logger.info("Filename processor lambda task completed")
-
-
-def run_local():
-    parser = argparse.ArgumentParser("file_name_processor")
-    parser.add_argument("--bucket", required=True, help="Bucket name.", type=str)
-    parser.add_argument("--key", required=True, help="Object key.", type=str)
-    args = parser.parse_args()
-
-    event = {"Records": [{"s3": {"bucket": {"name": args.bucket}, "object": {"key": args.key}}}]}
-    print(event)
-    print(lambda_handler(event=event, context={}))
-
-
-if __name__ == "__main__":
-    run_local()

lambdas/filenameprocessor/tests/test_lambda_handler.py

@@ -1,7 +1,6 @@
 """Tests for lambda_handler"""
 
 import json
-import sys
 from contextlib import ExitStack
 from copy import deepcopy
 from json import loads as json_loads
@@ -778,37 +777,3 @@ def test_unexpected_bucket_name_and_filename_validation_fails(self):
             self.assertIn("Unable to process file", args[0])
             self.assertIn(invalid_file_key, args)
             self.assertIn("unknown-bucket", args)
-
-
-class TestMainEntryPoint(TestCase):
-    def test_run_local_constructs_event_and_calls_lambda_handler(self):
-        test_args = [
-            "file_name_processor.py",
-            "--bucket",
-            "test-bucket",
-            "--key",
-            "some/path/file.csv",
-        ]
-
-        expected_event = {
-            "Records": [
-                {
-                    "s3": {
-                        "bucket": {"name": "test-bucket"},
-                        "object": {"key": "some/path/file.csv"},
-                    }
-                }
-            ]
-        }
-
-        with (
-            patch.object(sys, "argv", test_args),
-            patch("file_name_processor.lambda_handler") as mock_lambda_handler,
-            patch("file_name_processor.print") as mock_print,
-        ):
-            import file_name_processor
-
-            file_name_processor.run_local()
-
-            mock_lambda_handler.assert_called_once_with(event=expected_event, context={})
-            mock_print.assert_called()

lambdas/shared/src/common/data_quality/checker.py

@@ -1,14 +1,13 @@
 from dataclasses import dataclass
 
-from common.data_quality.completeness import DataQualityCompletenessChecker
+from common.data_quality.completeness import DataQualityCompletenessChecker, MissingFields
 from common.data_quality.validator import DataQualityValidator
 from common.models.fhir_converter.converter import Converter
-from common.models.fhir_converter.mappings import ActionFlag
 
 
 @dataclass
 class DataQualityOutput:
-    incomplete_fields: dict[str, list[str]]
+    missing_fields: MissingFields
     invalid_fields: list[str]
     timeliness: dict[str, int]
 
@@ -18,33 +17,31 @@ class DataQualityChecker:
 
     def __init__(
         self,
-        immunisation: dict,
-        action_flag: ActionFlag,
         completeness_checker: DataQualityCompletenessChecker,
-        data_quality_validator: DataQualityValidator,
         is_batch_csv: bool,
     ):
-        self.immunisation = immunisation
-        self.fhir_converter = Converter(fhir_data=immunisation, action_flag=action_flag)
         self.completeness_checker = completeness_checker
-        self.data_quality_validator = data_quality_validator
         self.is_batch_csv = is_batch_csv
 
-    def run_checks(self) -> DataQualityOutput:
+    def run_checks(self, immunisation: dict) -> DataQualityOutput:
+        data_quality_validator = DataQualityValidator()
+
         if not self.is_batch_csv:
-            self.immunisation = self.fhir_converter.run_conversion()
+            immunisation = Converter(fhir_data=immunisation).run_conversion()
 
         return DataQualityOutput(
-            incomplete_fields=self._check_completeness(),
-            invalid_fields=self._check_validity(),
-            timeliness=self._check_timeliness(),
+            missing_fields=self._check_completeness(immunisation),
+            invalid_fields=self._check_validity(immunisation, data_quality_validator),
+            timeliness=self._check_timeliness(immunisation),
         )
 
-    def _check_completeness(self) -> dict[str, list[str]]:
-        pass
+    def _check_completeness(self, immunisation: dict) -> MissingFields:
+        return self.completeness_checker.run_checks(immunisation)
 
-    def _check_validity(self) -> list[str]:
+    @staticmethod
+    def _check_validity(immunisation: dict, data_quality_validator: DataQualityValidator) -> list[str]:
         pass
 
-    def _check_timeliness(self) -> dict[str, int]:
+    @staticmethod
+    def _check_timeliness(immunisation: dict) -> dict[str, int]:
         pass

lambdas/shared/src/common/data_quality/completeness.py

@@ -1,3 +1,68 @@
+from dataclasses import dataclass
+
+required_fields = [
+    "NHS_NUMBER",
+    "VACCINATION_PROCEDURE_TERM",
+    "DOSE_SEQUENCE",
+    "VACCINE_PRODUCT_CODE",
+    "VACCINE_PRODUCT_TERM",
+    "VACCINE_MANUFACTURER",
+    "BATCH_NUMBER",
+    "EXPIRY_DATE",
+    "SITE_OF_VACCINATION_CODE",
+    "SITE_OF_VACCINATION_TERM",
+    "ROUTE_OF_VACCINATION_CODE",
+    "ROUTE_OF_VACCINATION_TERM",
+    "DOSE_AMOUNT",
+    "DOSE_UNIT_CODE",
+    "DOSE_UNIT_TERM",
+    "INDICATION_CODE",
+]
+mandatory_fields = [
+    "PERSON_FORENAME",
+    "PERSON_SURNAME",
+    "PERSON_DOB",
+    "PERSON_GENDER_CODE",
+    "PERSON_POSTCODE",
+    "DATE_AND_TIME",
+    "SITE_CODE",
+    "SITE_CODE_TYPE_URI",
+    "UNIQUE_ID",
+    "UNIQUE_ID_URI",
+    "ACTION_FLAG",
+    "RECORDED_DATE",
+    "PRIMARY_SOURCE",
+    "VACCINATION_PROCEDURE_CODE",
+    "LOCATION_CODE",
+    "LOCATION_CODE_TYPE_URI",
+]
+optional_fields = [
+    "PERFORMING_PROFESSIONAL_FORENAME",
+    "PERFORMING_PROFESSIONAL_SURNAME",
+]
+
+
+@dataclass
+class MissingFields:
+    required_fields: list[str]
+    mandatory_fields: list[str]
+    optional_fields: list[str]
+
+
 class DataQualityCompletenessChecker:
-    def check_completeness(self, immunisation: dict) -> dict[str, list[str]]:
-        pass
+    def run_checks(self, immunisation: dict) -> MissingFields:
+        return MissingFields(
+            required_fields=self._get_missing_fields(immunisation, required_fields),
+            mandatory_fields=self._get_missing_fields(immunisation, mandatory_fields),
+            optional_fields=self._get_missing_fields(immunisation, optional_fields),
+        )
+
+    @staticmethod
+    def _get_missing_fields(immunisation: dict, list_of_fields: list[str]) -> list[str]:
+        missing_fields = []
+        for field in list_of_fields:
+            exists = immunisation.get(field)
+            if not exists:
+                missing_fields.append(field)
+
+        return missing_fields