VED-770-e2e-batch-rewrite

azure/templates/post-deploy.yml

@@ -225,7 +225,6 @@ steps:
 
     displayName: Run full batch test suite
     workingDirectory: "$(Pipeline.Workspace)/s/$(SERVICE_NAME)/$(SERVICE_ARTIFACT_NAME)/e2e_batch"
-    condition: eq(1, 2) # Disable task but make this step visible in the pipeline
 
   - task: PublishTestResults@2
     displayName: 'Publish test results'

e2e_batch/Makefile

@@ -1,4 +1,4 @@
 -include .env
 
-run-immunization-batch:
-	ENVIRONMENT=$(environment) poetry run python -m unittest -v -c
+test:
+    ENVIRONMENT=$(ENVIRONMENT) poetry run python -m unittest -v -c

e2e_batch/README.md

@@ -0,0 +1,38 @@
+# End-to-End Batch Test Suite (test_e2e_batch.py)
+
+This test suite provides automated end-to-end (E2E) testing for the Immunisation FHIR API batch processing pipeline. It verifies that batch file submissions are correctly processed, acknowledged, and validated across the system.
+
+## Overview
+- Framework: Python unittest
+- Purpose: Simulate real-world batch file submissions, poll for acknowledgements, and validate processing results.
+- Test Scenarios: Defined in the scenarios module and enabled in setUp().
+- Key Features:
+- - Uploads test batch files to S3.
+- - Waits for and validates ACK (acknowledgement) files.
+- - Cleans up SQS queues and test artifacts after each run.
+
+## Test Flow
+1. Setup (setUp)
+- Loads and enables a set of test scenarios.
+- Prepares test data for batch submission.
+2. Test Execution (test_batch_submission)
+- Uploads ALL enabled test files to S3.
+- Polls for ALL ACK responses and forwarded files.
+- Validates the content and structure of ACK files.
+3. Teardown (tearDown)
+- Cleans up SQS queues and any generated test files.
+
+## Key Functions
+- send_files(tests): Uploads enabled test files to the S3 input bucket.
+- poll_for_responses(tests, max_timeout): Polls for ACKs and processed files, with a timeout.
+- validate_responses(tests): Validates the content of ACK files and checks for expected outcomes.
+
+## How to Run
+1. Ensure all dependencies and environment variables are set (see project root README).
+2. Update `.env` file with contents indicated in `PR-NNN.env`, modified for PR
+3. Update `.env` with referrence to the appropriate AWS config profile `AWS_PROFILE={your-aws-profile}`
+4. Update the apigee app to match the required PR-NNN
+5. Run tests from vscode debugger or from makefile using 
+```
+make test
+```

e2e_batch/clients.py

@@ -3,7 +3,10 @@
 """
 
 import logging
-from constants import (environment, REGION)
+from constants import (
+    environment, REGION,
+    batch_fifo_queue_name, ack_metadata_queue_name, audit_table_name
+    )
 from boto3 import client as boto3_client, resource as boto3_resource
 
 
@@ -13,8 +16,12 @@
 s3_client = boto3_client("s3", region_name=REGION)
 
 dynamodb = boto3_resource("dynamodb", region_name=REGION)
-table_name = f"imms-{environment}-imms-events"
-table = dynamodb.Table(table_name)
+sqs_client = boto3_client('sqs', region_name=REGION)
+events_table_name = f"imms-{environment}-imms-events"
+events_table = dynamodb.Table(events_table_name)
+audit_table = dynamodb.Table(audit_table_name)
+batch_fifo_queue_url = sqs_client.get_queue_url(QueueName=batch_fifo_queue_name)['QueueUrl']
+ack_metadata_queue_url = sqs_client.get_queue_url(QueueName=ack_metadata_queue_name)['QueueUrl']
 # Logger
 logging.basicConfig(level="INFO")
 logger = logging.getLogger()

e2e_batch/constants.py

@@ -1,5 +1,4 @@
 import os
-from datetime import datetime, timezone
 
 environment = os.environ.get("ENVIRONMENT", "internal-dev")
 REGION = "eu-west-2"
@@ -12,77 +11,87 @@
 POST_VALIDATION_ERROR = "Validation errors: contained[?(@.resourceType=='Patient')].name[0].given is a mandatory field"
 DUPLICATE = "The provided identifier:"
 ACK_PREFIX = "ack/"
+TEMP_ACK_PREFIX = "TempAck/"
 HEADER_RESPONSE_CODE_COLUMN = "HEADER_RESPONSE_CODE"
 FILE_NAME_VAL_ERROR = "Infrastructure Level Response Value - Processing Error"
 CONFIG_BUCKET = "imms-internal-dev-supplier-config"
 PERMISSIONS_CONFIG_FILE_KEY = "permissions_config.json"
+RAVS_URI = "https://www.ravs.england.nhs.uk/"
+batch_fifo_queue_name = f"imms-{environment}-batch-file-created-queue.fifo"
+ack_metadata_queue_name = f"imms-{environment}-ack-metadata-queue.fifo"
+audit_table_name = f"immunisation-batch-{environment}-audit-table"
 
 
-def create_row(unique_id, fore_name, dose_amount, action_flag, header):
-    """Helper function to create a single row with the specified UNIQUE_ID and ACTION_FLAG."""
-
-    return {
-        header: "9732928395",
-        "PERSON_FORENAME": fore_name,
-        "PERSON_SURNAME": "James",
-        "PERSON_DOB": "20080217",
-        "PERSON_GENDER_CODE": "0",
-        "PERSON_POSTCODE": "WD25 0DZ",
-        "DATE_AND_TIME": datetime.now(timezone.utc).strftime("%Y%m%dT%H%M%S"),
-        "SITE_CODE": "RVVKC",
-        "SITE_CODE_TYPE_URI": "https://fhir.nhs.uk/Id/ods-organization-code",
-        "UNIQUE_ID": unique_id,
-        "UNIQUE_ID_URI": "https://www.ravs.england.nhs.uk/",
-        "ACTION_FLAG": action_flag,
-        "PERFORMING_PROFESSIONAL_FORENAME": "PHYLIS",
-        "PERFORMING_PROFESSIONAL_SURNAME": "James",
-        "RECORDED_DATE": datetime.now(timezone.utc).strftime("%Y%m%d"),
-        "PRIMARY_SOURCE": "TRUE",
-        "VACCINATION_PROCEDURE_CODE": "956951000000104",
-        "VACCINATION_PROCEDURE_TERM": "RSV vaccination in pregnancy (procedure)",
-        "DOSE_SEQUENCE": "1",
-        "VACCINE_PRODUCT_CODE": "42223111000001107",
-        "VACCINE_PRODUCT_TERM": "Quadrivalent influenza vaccine (Sanofi Pasteur)",
-        "VACCINE_MANUFACTURER": "Sanofi Pasteur",
-        "BATCH_NUMBER": "BN92478105653",
-        "EXPIRY_DATE": "20240915",
-        "SITE_OF_VACCINATION_CODE": "368209003",
-        "SITE_OF_VACCINATION_TERM": "Right arm",
-        "ROUTE_OF_VACCINATION_CODE": "1210999013",
-        "ROUTE_OF_VACCINATION_TERM": "Intradermal use",
-        "DOSE_AMOUNT": dose_amount,
-        "DOSE_UNIT_CODE": "2622896019",
-        "DOSE_UNIT_TERM": "Inhalation - unit of product usage",
-        "INDICATION_CODE": "1037351000000105",
-        "LOCATION_CODE": "RJC02",
-        "LOCATION_CODE_TYPE_URI": "https://fhir.nhs.uk/Id/ods-organization-code",
-    }
-
-
-def create_permissions_json(value):
-    return {
-        "all_permissions": {
-            "DPSFULL": ["RSV_FULL", "COVID19_FULL", "FLU_FULL", "MMR_FULL"],
-            "DPSREDUCED": ["COVID19_FULL", "FLU_FULL", "MMR_FULL"],
-            "EMIS": [value, "RSV_FULL"],
-            "PINNACLE": ["COVID19_UPDATE", "RSV_FULL"],
-            "SONAR": "",
-            "TPP": [""],
-            "AGEM-NIVS": [""],
-            "NIMS": [""],
-            "EVA": [""],
-            "RAVS": [""],
-            "MEDICAL_DIRECTOR": [""],
-            "WELSH_DA_1": [""],
-            "WELSH_DA_2": [""],
-            "NORTHERN_IRELAND_DA": [""],
-            "SCOTLAND_DA": [""],
-            "COVID19_VACCINE_RESOLUTION_SERVICEDESK": [""],
-        },
-        "definitions:": {
-            "FULL": "Full permissions to create, update and delete a batch record",
-            "CREATE": "Permission to create a batch record",
-            "UPDATE": "Permission to update a batch record",
-            "DELETE": "Permission to delete a batch record",
-        },
-    }
+class EventName:
+    CREATE = "INSERT"
+    UPDATE = "MODIFY"
+    DELETE_LOGICAL = "MODIFY"
+    DELETE_PHYSICAL = "REMOVE"
+
+
+class Operation:
+    CREATE = "CREATE"
+    UPDATE = "UPDATE"
+    DELETE_LOGICAL = "DELETE"
+    DELETE_PHYSICAL = "REMOVE"
+
+
+class ActionFlag:
+    CREATE = "NEW"
+    UPDATE = "UPDATE"
+    DELETE_LOGICAL = "DELETE"
+    NONE = "NONE"
+
+
+class InfResult:
+    SUCCESS = "Success"
+    PARTIAL_SUCCESS = "Partial Success"
+    FATAL_ERROR = "Fatal Error"
+
+
+class BusRowResult:
+    SUCCESS = "OK"
+    FATAL_ERROR = "Fatal Error"
+    IMMS_NOT_FOUND = "Immunization resource does not exist"
+    NONE = "NONE"
+
+
+class OperationOutcome:
+    IMMS_NOT_FOUND = "Immunization resource does not exist"
+    TEST = "TEST"
+
+
+class OpMsgs:
+    VALIDATION_ERROR = "Validation errors"
+    MISSING_MANDATORY_FIELD = "is a mandatory field"
+    DOSE_QUANTITY_NOT_NUMBER = "doseQuantity.value must be a number"
+    IMM_NOT_EXIST = "Immunization resource does not exist"
+    IDENTIFIER_PROVIDED = "The provided identifier:"
+    INVALID_DATE_FORMAT = "is not in the correct format"
+
+
+class DestinationType:
+    INF = ACK_PREFIX
+    BUS = FORWARDEDFILE_PREFIX
+
+
+class ActionSequence:
+    def __init__(self, desc: str, actions: list[ActionFlag], outcome: ActionFlag = None):
+        self.actions = actions
+        self.description = desc
+        self.outcome = outcome if outcome else actions[-1]
+
+
+class PermPair:
+    def __init__(self, ods_code: str, permissions: str):
+        self.ods_code = ods_code
+        self.permissions = permissions
+
+
+class TestSet:
+    CREATE_OK = ActionSequence("Create. OK", [ActionFlag.CREATE])
+    UPDATE_OK = ActionSequence("Update. OK", [ActionFlag.CREATE, ActionFlag.UPDATE])
+    DELETE_OK = ActionSequence("Delete. OK", [ActionFlag.CREATE, ActionFlag.UPDATE, ActionFlag.DELETE_LOGICAL])
+    REINSTATE_OK = ActionSequence("Reinstate. OK", [ActionFlag.CREATE, ActionFlag.DELETE_LOGICAL, ActionFlag.UPDATE])
+    DELETE_FAIL = ActionSequence("Delete without Create. Fail", [ActionFlag.DELETE_LOGICAL])
+    UPDATE_FAIL = ActionSequence("Update without Create. Fail", [ActionFlag.UPDATE], outcome=ActionFlag.NONE)