Skip to content

Commit 95dcfbc

Browse files
nhsdevwsnhsdevws
authored
VED-770-e2e-batch-rewrite (#849)
* Purge SQS in development PR branch only * comments env & create_with_cp1252_encoded_character * poetry lock * remove VaxSupplierPerms. 600 s timeout. readme
1 parent c4986ba commit 95dcfbc

File tree

11 files changed

+844
-459
lines changed

11 files changed

+844
-459
lines changed

‎azure/templates/post-deploy.yml‎

Lines changed: 0 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -225,7 +225,6 @@ steps:
225225
226226
displayName: Run full batch test suite
227227
workingDirectory: "$(Pipeline.Workspace)/s/$(SERVICE_NAME)/$(SERVICE_ARTIFACT_NAME)/e2e_batch"
228-
condition: eq(1, 2) # Disable task but make this step visible in the pipeline
229228
230229
- task: PublishTestResults@2
231230
displayName: 'Publish test results'

‎e2e_batch/Makefile‎

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -1,4 +1,4 @@
11
-include .env
22

3-
run-immunization-batch:
4-
ENVIRONMENT=$(environment) poetry run python -m unittest -v -c
3+
test:
4+
ENVIRONMENT=$(ENVIRONMENT) poetry run python -m unittest -v -c

‎e2e_batch/README.md‎

Lines changed: 38 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,38 @@
1+
# End-to-End Batch Test Suite (test_e2e_batch.py)
2+
3+
This test suite provides automated end-to-end (E2E) testing for the Immunisation FHIR API batch processing pipeline. It verifies that batch file submissions are correctly processed, acknowledged, and validated across the system.
4+
5+
## Overview
6+
- Framework: Python unittest
7+
- Purpose: Simulate real-world batch file submissions, poll for acknowledgements, and validate processing results.
8+
- Test Scenarios: Defined in the scenarios module and enabled in setUp().
9+
- Key Features:
10+
- - Uploads test batch files to S3.
11+
- - Waits for and validates ACK (acknowledgement) files.
12+
- - Cleans up SQS queues and test artifacts after each run.
13+
14+
## Test Flow
15+
1. Setup (setUp)
16+
- Loads and enables a set of test scenarios.
17+
- Prepares test data for batch submission.
18+
2. Test Execution (test_batch_submission)
19+
- Uploads ALL enabled test files to S3.
20+
- Polls for ALL ACK responses and forwarded files.
21+
- Validates the content and structure of ACK files.
22+
3. Teardown (tearDown)
23+
- Cleans up SQS queues and any generated test files.
24+
25+
## Key Functions
26+
- send_files(tests): Uploads enabled test files to the S3 input bucket.
27+
- poll_for_responses(tests, max_timeout): Polls for ACKs and processed files, with a timeout.
28+
- validate_responses(tests): Validates the content of ACK files and checks for expected outcomes.
29+
30+
## How to Run
31+
1. Ensure all dependencies and environment variables are set (see project root README).
32+
2. Update `.env` file with contents indicated in `PR-NNN.env`, modified for PR
33+
3. Update `.env` with referrence to the appropriate AWS config profile `AWS_PROFILE={your-aws-profile}`
34+
4. Update the apigee app to match the required PR-NNN
35+
5. Run tests from vscode debugger or from makefile using
36+
```
37+
make test
38+
```

‎e2e_batch/clients.py‎

Lines changed: 10 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -3,7 +3,10 @@
33
"""
44

55
import logging
6-
from constants import (environment, REGION)
6+
from constants import (
7+
environment, REGION,
8+
batch_fifo_queue_name, ack_metadata_queue_name, audit_table_name
9+
)
710
from boto3 import client as boto3_client, resource as boto3_resource
811

912

@@ -13,8 +16,12 @@
1316
s3_client = boto3_client("s3", region_name=REGION)
1417

1518
dynamodb = boto3_resource("dynamodb", region_name=REGION)
16-
table_name = f"imms-{environment}-imms-events"
17-
table = dynamodb.Table(table_name)
19+
sqs_client = boto3_client('sqs', region_name=REGION)
20+
events_table_name = f"imms-{environment}-imms-events"
21+
events_table = dynamodb.Table(events_table_name)
22+
audit_table = dynamodb.Table(audit_table_name)
23+
batch_fifo_queue_url = sqs_client.get_queue_url(QueueName=batch_fifo_queue_name)['QueueUrl']
24+
ack_metadata_queue_url = sqs_client.get_queue_url(QueueName=ack_metadata_queue_name)['QueueUrl']
1825
# Logger
1926
logging.basicConfig(level="INFO")
2027
logger = logging.getLogger()

‎e2e_batch/constants.py‎

Lines changed: 78 additions & 69 deletions
Original file line numberDiff line numberDiff line change
@@ -1,5 +1,4 @@
11
import os
2-
from datetime import datetime, timezone
32

43
environment = os.environ.get("ENVIRONMENT", "internal-dev")
54
REGION = "eu-west-2"
@@ -12,77 +11,87 @@
1211
POST_VALIDATION_ERROR = "Validation errors: contained[?(@.resourceType=='Patient')].name[0].given is a mandatory field"
1312
DUPLICATE = "The provided identifier:"
1413
ACK_PREFIX = "ack/"
14+
TEMP_ACK_PREFIX = "TempAck/"
1515
HEADER_RESPONSE_CODE_COLUMN = "HEADER_RESPONSE_CODE"
1616
FILE_NAME_VAL_ERROR = "Infrastructure Level Response Value - Processing Error"
1717
CONFIG_BUCKET = "imms-internal-dev-supplier-config"
1818
PERMISSIONS_CONFIG_FILE_KEY = "permissions_config.json"
19+
RAVS_URI = "https://www.ravs.england.nhs.uk/"
20+
batch_fifo_queue_name = f"imms-{environment}-batch-file-created-queue.fifo"
21+
ack_metadata_queue_name = f"imms-{environment}-ack-metadata-queue.fifo"
22+
audit_table_name = f"immunisation-batch-{environment}-audit-table"
1923

2024

21-
def create_row(unique_id, fore_name, dose_amount, action_flag, header):
22-
"""Helper function to create a single row with the specified UNIQUE_ID and ACTION_FLAG."""
23-
24-
return {
25-
header: "9732928395",
26-
"PERSON_FORENAME": fore_name,
27-
"PERSON_SURNAME": "James",
28-
"PERSON_DOB": "20080217",
29-
"PERSON_GENDER_CODE": "0",
30-
"PERSON_POSTCODE": "WD25 0DZ",
31-
"DATE_AND_TIME": datetime.now(timezone.utc).strftime("%Y%m%dT%H%M%S"),
32-
"SITE_CODE": "RVVKC",
33-
"SITE_CODE_TYPE_URI": "https://fhir.nhs.uk/Id/ods-organization-code",
34-
"UNIQUE_ID": unique_id,
35-
"UNIQUE_ID_URI": "https://www.ravs.england.nhs.uk/",
36-
"ACTION_FLAG": action_flag,
37-
"PERFORMING_PROFESSIONAL_FORENAME": "PHYLIS",
38-
"PERFORMING_PROFESSIONAL_SURNAME": "James",
39-
"RECORDED_DATE": datetime.now(timezone.utc).strftime("%Y%m%d"),
40-
"PRIMARY_SOURCE": "TRUE",
41-
"VACCINATION_PROCEDURE_CODE": "956951000000104",
42-
"VACCINATION_PROCEDURE_TERM": "RSV vaccination in pregnancy (procedure)",
43-
"DOSE_SEQUENCE": "1",
44-
"VACCINE_PRODUCT_CODE": "42223111000001107",
45-
"VACCINE_PRODUCT_TERM": "Quadrivalent influenza vaccine (Sanofi Pasteur)",
46-
"VACCINE_MANUFACTURER": "Sanofi Pasteur",
47-
"BATCH_NUMBER": "BN92478105653",
48-
"EXPIRY_DATE": "20240915",
49-
"SITE_OF_VACCINATION_CODE": "368209003",
50-
"SITE_OF_VACCINATION_TERM": "Right arm",
51-
"ROUTE_OF_VACCINATION_CODE": "1210999013",
52-
"ROUTE_OF_VACCINATION_TERM": "Intradermal use",
53-
"DOSE_AMOUNT": dose_amount,
54-
"DOSE_UNIT_CODE": "2622896019",
55-
"DOSE_UNIT_TERM": "Inhalation - unit of product usage",
56-
"INDICATION_CODE": "1037351000000105",
57-
"LOCATION_CODE": "RJC02",
58-
"LOCATION_CODE_TYPE_URI": "https://fhir.nhs.uk/Id/ods-organization-code",
59-
}
60-
61-
62-
def create_permissions_json(value):
63-
return {
64-
"all_permissions": {
65-
"DPSFULL": ["RSV_FULL", "COVID19_FULL", "FLU_FULL", "MMR_FULL"],
66-
"DPSREDUCED": ["COVID19_FULL", "FLU_FULL", "MMR_FULL"],
67-
"EMIS": [value, "RSV_FULL"],
68-
"PINNACLE": ["COVID19_UPDATE", "RSV_FULL"],
69-
"SONAR": "",
70-
"TPP": [""],
71-
"AGEM-NIVS": [""],
72-
"NIMS": [""],
73-
"EVA": [""],
74-
"RAVS": [""],
75-
"MEDICAL_DIRECTOR": [""],
76-
"WELSH_DA_1": [""],
77-
"WELSH_DA_2": [""],
78-
"NORTHERN_IRELAND_DA": [""],
79-
"SCOTLAND_DA": [""],
80-
"COVID19_VACCINE_RESOLUTION_SERVICEDESK": [""],
81-
},
82-
"definitions:": {
83-
"FULL": "Full permissions to create, update and delete a batch record",
84-
"CREATE": "Permission to create a batch record",
85-
"UPDATE": "Permission to update a batch record",
86-
"DELETE": "Permission to delete a batch record",
87-
},
88-
}
25+
class EventName:
26+
CREATE = "INSERT"
27+
UPDATE = "MODIFY"
28+
DELETE_LOGICAL = "MODIFY"
29+
DELETE_PHYSICAL = "REMOVE"
30+
31+
32+
class Operation:
33+
CREATE = "CREATE"
34+
UPDATE = "UPDATE"
35+
DELETE_LOGICAL = "DELETE"
36+
DELETE_PHYSICAL = "REMOVE"
37+
38+
39+
class ActionFlag:
40+
CREATE = "NEW"
41+
UPDATE = "UPDATE"
42+
DELETE_LOGICAL = "DELETE"
43+
NONE = "NONE"
44+
45+
46+
class InfResult:
47+
SUCCESS = "Success"
48+
PARTIAL_SUCCESS = "Partial Success"
49+
FATAL_ERROR = "Fatal Error"
50+
51+
52+
class BusRowResult:
53+
SUCCESS = "OK"
54+
FATAL_ERROR = "Fatal Error"
55+
IMMS_NOT_FOUND = "Immunization resource does not exist"
56+
NONE = "NONE"
57+
58+
59+
class OperationOutcome:
60+
IMMS_NOT_FOUND = "Immunization resource does not exist"
61+
TEST = "TEST"
62+
63+
64+
class OpMsgs:
65+
VALIDATION_ERROR = "Validation errors"
66+
MISSING_MANDATORY_FIELD = "is a mandatory field"
67+
DOSE_QUANTITY_NOT_NUMBER = "doseQuantity.value must be a number"
68+
IMM_NOT_EXIST = "Immunization resource does not exist"
69+
IDENTIFIER_PROVIDED = "The provided identifier:"
70+
INVALID_DATE_FORMAT = "is not in the correct format"
71+
72+
73+
class DestinationType:
74+
INF = ACK_PREFIX
75+
BUS = FORWARDEDFILE_PREFIX
76+
77+
78+
class ActionSequence:
79+
def __init__(self, desc: str, actions: list[ActionFlag], outcome: ActionFlag = None):
80+
self.actions = actions
81+
self.description = desc
82+
self.outcome = outcome if outcome else actions[-1]
83+
84+
85+
class PermPair:
86+
def __init__(self, ods_code: str, permissions: str):
87+
self.ods_code = ods_code
88+
self.permissions = permissions
89+
90+
91+
class TestSet:
92+
CREATE_OK = ActionSequence("Create. OK", [ActionFlag.CREATE])
93+
UPDATE_OK = ActionSequence("Update. OK", [ActionFlag.CREATE, ActionFlag.UPDATE])
94+
DELETE_OK = ActionSequence("Delete. OK", [ActionFlag.CREATE, ActionFlag.UPDATE, ActionFlag.DELETE_LOGICAL])
95+
REINSTATE_OK = ActionSequence("Reinstate. OK", [ActionFlag.CREATE, ActionFlag.DELETE_LOGICAL, ActionFlag.UPDATE])
96+
DELETE_FAIL = ActionSequence("Delete without Create. Fail", [ActionFlag.DELETE_LOGICAL])
97+
UPDATE_FAIL = ActionSequence("Update without Create. Fail", [ActionFlag.UPDATE], outcome=ActionFlag.NONE)

0 commit comments

Comments
 (0)