VED-386: recordprocessor filevalidation logic and dependency updates

Author: Akol125

.github/workflows/sonarcloud.yml

@@ -40,7 +40,7 @@ jobs:
         id: filenameprocessor
         continue-on-error: true
         run: |
-          poetry env use 3.10
+          poetry env use 3.11
           poetry install
           poetry run coverage run -m unittest discover || echo "filenameprocessor tests failed" >> ../failed_tests.txt
           poetry run coverage xml -o ../filenameprocessor-coverage.xml
@@ -50,7 +50,7 @@ jobs:
         id: recordprocessor
         continue-on-error: true
         run: |
-          poetry env use 3.10
+          poetry env use 3.11
           poetry install
           poetry run coverage run -m unittest discover || echo "recordprocessor tests failed" >> ../failed_tests.txt
           poetry run coverage xml -o ../recordprocessor-coverage.xml

filenameprocessor/src/constants.py

@@ -60,10 +60,11 @@ class Constants:
         "8J1100001": "PINNACLE",
         "8HK48": "SONAR",
         "YGA": "TPP",
+        "V0V8L":"MAVIS",
+        "X8E5B":"RAVS",
         "0DE": "AGEM-NIVS",
         "0DF": "NIMS",
         "8HA94": "EVA",
-        "X26": "RAVS",
         "YGMYH": "MEDICAL_DIRECTOR",
         "W00": "WELSH_DA_1",
         "W000": "WELSH_DA_2",

recordprocessor/Dockerfile

@@ -1,4 +1,4 @@
-FROM public.ecr.aws/lambda/python:3.10 AS base
+FROM public.ecr.aws/lambda/python:3.11 AS base
 
 # Create a non-root user with a specific UID and GID
 RUN mkdir -p /home/appuser && \

recordprocessor/src/constants.py

@@ -97,3 +97,8 @@ class ActionFlag(Enum):
     NEW = Permission.CREATE
     UPDATE = Permission.UPDATE
     DELETE = Permission.DELETE
+
+
+class AllowedPermission(Enum):
+    CRUD = {"C", "U", "D"}
+    CRUDS = {"C", "U", "D"}

recordprocessor/src/file_level_validation.py

@@ -10,7 +10,7 @@
 from errors import InvalidHeaders, NoOperationPermissions
 from logging_decorator import file_level_validation_logging_decorator
 from audit_table import change_audit_table_status_to_processed, get_next_queued_file_details
-from constants import SOURCE_BUCKET_NAME, EXPECTED_CSV_HEADERS, Permission, ActionFlag
+from constants import SOURCE_BUCKET_NAME, EXPECTED_CSV_HEADERS, Permission, ActionFlag, AllowedPermission
 
 
 def validate_content_headers(csv_content_reader) -> None:
@@ -43,21 +43,14 @@ def validate_action_flag_permissions(
         return set()
 
     # Get allowed permission in single letters from allowed_permissions_list
-    allowed_ops = set()
-    for perm in allowed_permissions_list:
-        if perm.startswith(f"{vaccine_type}."):
-            allowed_ops.update(perm.split(".")[1])
-
     allowed_ops = set()
     for perm in allowed_permissions_list:
         if not perm.startswith(f"{vaccine_type}."):
             continue
 
         _, op_code = perm.split(".")
-        if op_code == "CRUD":
-            allowed_ops.update({"C", "R", "U", "D"})
-        elif op_code == "CRUDS":
-            allowed_ops.update({"C", "R", "U", "D", "S"})
+        if op_code in AllowedPermission.__members__:
+            allowed_ops.update(AllowedPermission[op_code].value)
         else:
             allowed_ops.add(op_code)
 

recordprocessor/src/unique_permission.py

@@ -1,13 +1,31 @@
 import pandas as pd
 from io import StringIO
+import logging
+
+
+logger = logging.getLogger(__name__)
 
 
 def get_unique_action_flags_from_s3(csv_data):
     """
     Reads the CSV file from an S3 bucket and returns a set of unique ACTION_FLAG values.
     """
     # Load content into a pandas DataFrame
-    df = pd.read_csv(StringIO(csv_data), delimiter="|", usecols=["ACTION_FLAG"])
+    try:
+        df = pd.read_csv(StringIO(csv_data), delimiter="|", usecols=["ACTION_FLAG"])
+    except ValueError:
+        logger.warning("ACTION_FLAG column missing or malformed in file.")
+        return set()
+
+    if "ACTION_FLAG" not in df.columns:
+        logger.warning("ACTION_FLAG column is missing in file.")
+        return set()
     # Get unique ACTION_FLAG values in one step
-    unique_action_flags = set(df["ACTION_FLAG"].str.upper().unique())
+    unique_action_flags = set(
+        df["ACTION_FLAG"]
+        .dropna()
+        .astype(str)
+        .str.upper()
+        .unique()
+    )
     return unique_action_flags

recordprocessor/tests/test_file_level_validation.py

@@ -94,7 +94,7 @@ def test_validate_action_flag_permissions(self):
 
         # Case: Supplier has no permissions to perform any of the requested operations
         # Test case tuples are stuctured as (vaccine_type, vaccine_permissions, file_content)
-        test_cases = [
+        invalid_cases = [
             # FLU, no permissions
             ("FLU", ["FLU.U", "COVID19.CRUDS"], valid_content_new_and_delete_lowercase),
             # COVID19, no permissions
@@ -103,11 +103,21 @@ def test_validate_action_flag_permissions(self):
             ("RSV", ["FLU.C", "FLU.U"], valid_content_update_and_delete_lowercase),
         ]
 
-        for vaccine_type, vaccine_permissions, file_content in test_cases:
+        for vaccine_type, vaccine_permissions, file_content in invalid_cases:
             with self.subTest():
                 with self.assertRaises(NoOperationPermissions):
                     validate_action_flag_permissions("TEST_SUPPLIER", vaccine_type, vaccine_permissions, file_content)
 
+        no_flag_cases = [
+            ("FLU", ["FLU.C"], valid_file_content.replace("new", "").replace("update", "")),
+            ("COVID19", ["COVID19.CRUD"], valid_file_content.replace("new", "INVALID").replace("update", "")),
+            ]
+
+        for vaccine_type, permissions, file_content in no_flag_cases:
+            with self.subTest(f"{vaccine_type} with invalid or missing ACTION_FLAGs"):
+                result = validate_action_flag_permissions("TEST_SUPPLIER", vaccine_type, permissions, file_content)
+                self.assertEqual(result, set())
+
 
 if __name__ == "__main__":
     unittest.main()