VED-789: Gen Validation Schema (#916)

* build validation engine and refactor fhir and batch journey

* refactor csv incoming data to dict

* add test for validation and resolve reviews

Author: Akol125

lambdas/shared/Makefile

@@ -1,5 +1,10 @@
+TEST_ENV := @PYTHONPATH=src:tests:src/common
+
 test:
-	@PYTHONPATH=src:../ python -m unittest discover -s tests -p "test_*.py" -v
+	$(TEST_ENV) python -m unittest discover -s tests -p "test_*.py" -v
+
+test-validator:
+	$(TEST_ENV) python -m unittest discover -s tests/test_common/validator -p "test_*.py" -v
 
 test-list:
 	@PYTHONPATH=src:tests python -m unittest discover -s tests -p "test_*.py" --verbose | grep test_

lambdas/shared/pyproject.toml

@@ -32,4 +32,4 @@ coverage = "^7.10.7"
 
 [build-system]
 requires = ["poetry-core"]
-build-backend = "poetry.core.masonry.api"
+build-backend = "poetry.core.masonry.api"

lambdas/shared/src/common/aws_lambda_event.py

@@ -1,5 +1,5 @@
 from enum import Enum
-from typing import Any, Dict
+from typing import Any
 
 
 class AwsEventType(Enum):
@@ -10,7 +10,7 @@ class AwsEventType(Enum):
 
 
 class AwsLambdaEvent:
-    def __init__(self, event: Dict[str, Any]):
+    def __init__(self, event: dict[str, Any]):
         self.event_source = None
         self.event_type = AwsEventType.UNKNOWN
         self.event_source = event.get("eventSource")

lambdas/shared/src/common/cache.py

@@ -1,5 +1,4 @@
 import json
-from typing import Optional
 
 
 class Cache:
@@ -19,7 +18,7 @@ def put(self, key: str, value: dict):
         self.cache[key] = value
         self._overwrite()
 
-    def get(self, key: str) -> Optional[dict]:
+    def get(self, key: str) -> dict | None:
         return self.cache.get(key, None)
 
     def delete(self, key: str):

lambdas/shared/src/common/log_firehose.py

@@ -1,9 +1,6 @@
 import json
 
-from common.clients import (
-    firehose_client,
-    logger,
-)
+from common.clients import firehose_client, logger
 
 
 # Not keen on including blocking calls in function code to forward log data to Splunk (via Firehose)

lambdas/shared/src/common/validator/__init__.py

@@ -0,0 +1,62 @@
+from enum import IntEnum, StrEnum
+
+
+# Used for error report messages in DQ Reporter to categorize error levels
+class ErrorLevels(IntEnum):
+    CRITICAL_ERROR = 0
+    WARNING = 1
+    NOTIFICATION = 2
+
+
+MESSAGES = {
+    ErrorLevels.CRITICAL_ERROR: "Critical Validation Error",
+    ErrorLevels.WARNING: "Non-Critical Validation Error",
+    ErrorLevels.NOTIFICATION: "Quality Issue Found",
+}
+
+
+# Constants for error report messages used in Expression Checker
+class MessageLabel(StrEnum):
+    EXPECTED_LABEL = "Expected- "
+    FOUND_LABEL = "Found- "
+    VALUE_MISMATCH_MSG = "Value does not equal expected value, "
+
+
+class ExceptionLevels(IntEnum):
+    UNEXPECTED_EXCEPTION = 0
+    VALUE_CHECK_FAILED = 1
+    HEADER_CHECK_FAILED = 2
+    RECORD_LENGTH_CHECK_FAILED = 3
+    VALUE_PREDICATE_FALSE = 4
+    RECORD_CHECK_FAILED = 5
+    RECORD_PREDICATE_FALSE = 6
+    UNIQUE_CHECK_FAILED = 7
+    ASSERT_CHECK_FAILED = 8
+    FINALLY_ASSERT_CHECK_FAILED = 9
+    PARSING_ERROR = 10
+    PARENT_FAILED = 11
+    KEY_CHECK_FAILED = 12
+
+
+MESSAGES = {
+    ExceptionLevels.UNEXPECTED_EXCEPTION: "Unexpected exception [%s]: %s",
+    ExceptionLevels.VALUE_CHECK_FAILED: "Value Check Failed",
+    ExceptionLevels.HEADER_CHECK_FAILED: "Header Check Failed",
+    ExceptionLevels.RECORD_LENGTH_CHECK_FAILED: "Record Length Check Failed",
+    ExceptionLevels.RECORD_CHECK_FAILED: "Record Check Failed",
+    ExceptionLevels.VALUE_PREDICATE_FALSE: "Value Predicate False",
+    ExceptionLevels.RECORD_PREDICATE_FALSE: "Record Predicate False",
+    ExceptionLevels.UNIQUE_CHECK_FAILED: "Unique Check Failed",
+    ExceptionLevels.ASSERT_CHECK_FAILED: "Assert Check Failed",
+    ExceptionLevels.FINALLY_ASSERT_CHECK_FAILED: "Finally Assert Check Failed",
+    ExceptionLevels.PARSING_ERROR: "Parsing Error",
+    ExceptionLevels.PARENT_FAILED: "Parent Failed",
+    ExceptionLevels.KEY_CHECK_FAILED: "Key Check Failed",
+}
+
+
+class DataType(StrEnum):
+    FHIR = "FHIR"
+    FHIRJSON = "FHIRJSON"
+    CSV = "CSV"
+    CSVROW = "CSVROW"

lambdas/shared/src/common/validator/constants/__init__.py

@@ -0,0 +1,12 @@
+error_report = {
+    "eventId": "",
+    "validationDate": "",
+    "validated": "true",
+    "results": {
+        "totalErrors": 0,
+        "completeness": {"errors": 0, "fields": []},
+        "consistency": {"errors": 0, "fields": []},
+        "validity": {"errors": 0, "fields": []},
+        "timeliness_processed": 0,
+    },
+}