test

Author: nhsdevws

e2e_batch/Makefile

@@ -1,4 +1,4 @@
 -include .env
 
-run-immunization-batch:
-	ENVIRONMENT=$(environment) poetry run python -m unittest -v -c
+test:
+	ENVIRONMENT=$(ENVIRONMENT) poetry run python -m unittest -v -c

e2e_batch/README.md

@@ -0,0 +1,35 @@
+# End-to-End Batch Test Suite (test_e2e_batch.py)
+
+This test suite provides automated end-to-end (E2E) testing for the Immunisation FHIR API batch processing pipeline. It verifies that batch file submissions are correctly processed, acknowledged, and validated across the system.
+
+## Overview
+- Framework: Python unittest
+- Purpose: Simulate real-world batch file submissions, poll for acknowledgements, and validate processing results.
+- Test Scenarios: Defined in the scenarios module and enabled in setUp().
+- Key Features:
+- - Uploads test batch files to S3.
+- - Waits for and validates ACK (acknowledgement) files.
+- - Cleans up SQS queues and test artifacts after each run.
+
+## Test Flow
+1. Setup (setUp)
+- Loads and enables a set of test scenarios.
+- Prepares test data for batch submission.
+2. Test Execution (test_batch_submission)
+- Uploads ALL enabled test files to S3.
+- Polls for ALL ACK responses and forwarded files.
+- Validates the content and structure of ACK files.
+3. Teardown (tearDown)
+- Cleans up SQS queues and any generated test files.
+
+## Key Functions
+- send_files(tests): Uploads enabled test files to the S3 input bucket.
+- poll_for_responses(tests, max_timeout): Polls for ACKs and processed files, with a timeout.
+- validate_responses(tests): Validates the content of ACK files and checks for expected outcomes.
+
+## How to Run
+1. Ensure all dependencies and environment variables are set (see project root README).
+2. Run tests from vscode debugger or from makefile using 
+```
+make test
+```

e2e_batch/clients.py

@@ -3,7 +3,10 @@
 """
 
 import logging
-from constants import (environment, REGION)
+from constants import (
+    environment, REGION,
+    batch_fifo_queue_name, ack_metadata_queue_name, audit_table_name
+    )
 from boto3 import client as boto3_client, resource as boto3_resource
 
 
@@ -13,8 +16,12 @@
 s3_client = boto3_client("s3", region_name=REGION)
 
 dynamodb = boto3_resource("dynamodb", region_name=REGION)
-table_name = f"imms-{environment}-imms-events"
-table = dynamodb.Table(table_name)
+sqs_client = boto3_client('sqs', region_name=REGION)
+events_table_name = f"imms-{environment}-imms-events"
+events_table = dynamodb.Table(events_table_name)
+audit_table = dynamodb.Table(audit_table_name)
+batch_fifo_queue_url = sqs_client.get_queue_url(QueueName=batch_fifo_queue_name)['QueueUrl']
+ack_metadata_queue_url = sqs_client.get_queue_url(QueueName=ack_metadata_queue_name)['QueueUrl']
 # Logger
 logging.basicConfig(level="INFO")
 logger = logging.getLogger()

e2e_batch/constants.py

@@ -12,30 +12,110 @@
 POST_VALIDATION_ERROR = "Validation errors: contained[?(@.resourceType=='Patient')].name[0].given is a mandatory field"
 DUPLICATE = "The provided identifier:"
 ACK_PREFIX = "ack/"
+TEMP_ACK_PREFIX = "TempAck/"
 HEADER_RESPONSE_CODE_COLUMN = "HEADER_RESPONSE_CODE"
 FILE_NAME_VAL_ERROR = "Infrastructure Level Response Value - Processing Error"
 CONFIG_BUCKET = "imms-internal-dev-supplier-config"
 PERMISSIONS_CONFIG_FILE_KEY = "permissions_config.json"
+RAVS_URI = "https://www.ravs.england.nhs.uk/"
+batch_fifo_queue_name = f"imms-{environment}-batch-file-created-queue.fifo"
+ack_metadata_queue_name = f"imms-{environment}-ack-metadata-queue.fifo"
+audit_table_name = f"immunisation-batch-{environment}-audit-table"
 
+class EventName():
+    CREATE = "INSERT"
+    UPDATE = "MODIFY"
+    DELETE_LOGICAL = "MODIFY"
+    DELETE_PHYSICAL = "REMOVE"
 
-def create_row(unique_id, fore_name, dose_amount, action_flag, header):
+
+class Operation():
+    CREATE = "CREATE"
+    UPDATE = "UPDATE"
+    DELETE_LOGICAL = "DELETE"
+    DELETE_PHYSICAL = "REMOVE"
+
+
+class ActionFlag():
+    CREATE = "NEW"
+    UPDATE = "UPDATE"
+    DELETE_LOGICAL = "DELETE"
+    NONE = "NONE"
+
+
+class InfResult:
+    SUCCESS = "Success"
+    PARTIAL_SUCCESS = "Partial Success"
+    FATAL_ERROR = "Fatal Error"
+
+
+class BusRowResult():
+    SUCCESS = "OK"
+    FATAL_ERROR = "Fatal Error"
+    IMMS_NOT_FOUND = "Immunization resource does not exist"
+    NONE = "NONE"
+
+
+class OperationOutcome:
+    IMMS_NOT_FOUND = "Immunization resource does not exist"
+    TEST = "TEST"
+
+
+class OpMsgs:
+    VALIDATION_ERROR = "Validation errors"
+    MISSING_MANDATORY_FIELD = "is a mandatory field"
+    DOSE_QUANTITY_NOT_NUMBER = "doseQuantity.value must be a number"
+    IMM_NOT_EXIST = "Immunization resource does not exist"
+    IDENTIFIER_PROVIDED = "The provided identifier:"
+    INVALID_DATE_FORMAT = "is not in the correct format"
+
+
+class DestinationType:
+    INF = ACK_PREFIX
+    BUS = FORWARDEDFILE_PREFIX
+
+
+class ActionSequence():
+    def __init__(self, desc: str, actions: list[ActionFlag], outcome: ActionFlag = None):
+        self.actions = actions
+        self.description = desc
+        self.outcome = outcome if outcome else actions[-1]
+
+
+class PermPair():
+    def __init__(self, ods_code: str, permissions: str):
+        self.ods_code = ods_code
+        self.permissions = permissions
+
+
+class TestSet():
+    CREATE_OK = ActionSequence("Create. OK", [ActionFlag.CREATE])
+    UPDATE_OK = ActionSequence("Update. OK", [ActionFlag.CREATE, ActionFlag.UPDATE])
+    DELETE_OK = ActionSequence("Delete. OK", [ActionFlag.CREATE, ActionFlag.UPDATE, ActionFlag.DELETE_LOGICAL])
+    REINSTATE_OK = ActionSequence("Reinstate. OK", [ActionFlag.CREATE, ActionFlag.DELETE_LOGICAL, ActionFlag.UPDATE])
+    DELETE_FAIL = ActionSequence("Delete without Create. Fail", [ActionFlag.DELETE_LOGICAL])
+    UPDATE_FAIL = ActionSequence("Update without Create. Fail", [ActionFlag.UPDATE], outcome=ActionFlag.NONE)
+
+
+def create_row(unique_id, dose_amount, action_flag: str, header, inject_char=None):
     """Helper function to create a single row with the specified UNIQUE_ID and ACTION_FLAG."""
 
+    name = "James" if not inject_char else b'Jam\xe9s'
     return {
         header: "9732928395",
-        "PERSON_FORENAME": fore_name,
-        "PERSON_SURNAME": "James",
+        "PERSON_FORENAME": "PHYLIS",
+        "PERSON_SURNAME": name,
         "PERSON_DOB": "20080217",
         "PERSON_GENDER_CODE": "0",
         "PERSON_POSTCODE": "WD25 0DZ",
         "DATE_AND_TIME": datetime.now(timezone.utc).strftime("%Y%m%dT%H%M%S"),
         "SITE_CODE": "RVVKC",
         "SITE_CODE_TYPE_URI": "https://fhir.nhs.uk/Id/ods-organization-code",
         "UNIQUE_ID": unique_id,
-        "UNIQUE_ID_URI": "https://www.ravs.england.nhs.uk/",
+        "UNIQUE_ID_URI": RAVS_URI,
         "ACTION_FLAG": action_flag,
         "PERFORMING_PROFESSIONAL_FORENAME": "PHYLIS",
-        "PERFORMING_PROFESSIONAL_SURNAME": "James",
+        "PERFORMING_PROFESSIONAL_SURNAME": name,
         "RECORDED_DATE": datetime.now(timezone.utc).strftime("%Y%m%d"),
         "PRIMARY_SOURCE": "TRUE",
         "VACCINATION_PROCEDURE_CODE": "956951000000104",
@@ -57,32 +137,3 @@ def create_row(unique_id, fore_name, dose_amount, action_flag, header):
         "LOCATION_CODE": "RJC02",
         "LOCATION_CODE_TYPE_URI": "https://fhir.nhs.uk/Id/ods-organization-code",
     }
-
-
-def create_permissions_json(value):
-    return {
-        "all_permissions": {
-            "DPSFULL": ["RSV_FULL", "COVID19_FULL", "FLU_FULL", "MMR_FULL"],
-            "DPSREDUCED": ["COVID19_FULL", "FLU_FULL", "MMR_FULL"],
-            "EMIS": [value, "RSV_FULL"],
-            "PINNACLE": ["COVID19_UPDATE", "RSV_FULL"],
-            "SONAR": "",
-            "TPP": [""],
-            "AGEM-NIVS": [""],
-            "NIMS": [""],
-            "EVA": [""],
-            "RAVS": [""],
-            "MEDICAL_DIRECTOR": [""],
-            "WELSH_DA_1": [""],
-            "WELSH_DA_2": [""],
-            "NORTHERN_IRELAND_DA": [""],
-            "SCOTLAND_DA": [""],
-            "COVID19_VACCINE_RESOLUTION_SERVICEDESK": [""],
-        },
-        "definitions:": {
-            "FULL": "Full permissions to create, update and delete a batch record",
-            "CREATE": "Permission to create a batch record",
-            "UPDATE": "Permission to update a batch record",
-            "DELETE": "Permission to delete a batch record",
-        },
-    }