Skip to content

Commit d4b1183

Browse files
committed
comments env & create_with_cp1252_encoded_character
1 parent ca1b99f commit d4b1183

File tree

7 files changed

+193
-178
lines changed

7 files changed

+193
-178
lines changed

e2e_batch/constants.py

Lines changed: 7 additions & 50 deletions
Original file line numberDiff line numberDiff line change
@@ -1,5 +1,4 @@
11
import os
2-
from datetime import datetime, timezone
32

43
environment = os.environ.get("ENVIRONMENT", "internal-dev")
54
REGION = "eu-west-2"
@@ -23,21 +22,21 @@
2322
audit_table_name = f"immunisation-batch-{environment}-audit-table"
2423

2524

26-
class EventName():
25+
class EventName:
2726
CREATE = "INSERT"
2827
UPDATE = "MODIFY"
2928
DELETE_LOGICAL = "MODIFY"
3029
DELETE_PHYSICAL = "REMOVE"
3130

3231

33-
class Operation():
32+
class Operation:
3433
CREATE = "CREATE"
3534
UPDATE = "UPDATE"
3635
DELETE_LOGICAL = "DELETE"
3736
DELETE_PHYSICAL = "REMOVE"
3837

3938

40-
class ActionFlag():
39+
class ActionFlag:
4140
CREATE = "NEW"
4241
UPDATE = "UPDATE"
4342
DELETE_LOGICAL = "DELETE"
@@ -50,7 +49,7 @@ class InfResult:
5049
FATAL_ERROR = "Fatal Error"
5150

5251

53-
class BusRowResult():
52+
class BusRowResult:
5453
SUCCESS = "OK"
5554
FATAL_ERROR = "Fatal Error"
5655
IMMS_NOT_FOUND = "Immunization resource does not exist"
@@ -76,65 +75,23 @@ class DestinationType:
7675
BUS = FORWARDEDFILE_PREFIX
7776

7877

79-
class ActionSequence():
78+
class ActionSequence:
8079
def __init__(self, desc: str, actions: list[ActionFlag], outcome: ActionFlag = None):
8180
self.actions = actions
8281
self.description = desc
8382
self.outcome = outcome if outcome else actions[-1]
8483

8584

86-
class PermPair():
85+
class PermPair:
8786
def __init__(self, ods_code: str, permissions: str):
8887
self.ods_code = ods_code
8988
self.permissions = permissions
9089

9190

92-
class TestSet():
91+
class TestSet:
9392
CREATE_OK = ActionSequence("Create. OK", [ActionFlag.CREATE])
9493
UPDATE_OK = ActionSequence("Update. OK", [ActionFlag.CREATE, ActionFlag.UPDATE])
9594
DELETE_OK = ActionSequence("Delete. OK", [ActionFlag.CREATE, ActionFlag.UPDATE, ActionFlag.DELETE_LOGICAL])
9695
REINSTATE_OK = ActionSequence("Reinstate. OK", [ActionFlag.CREATE, ActionFlag.DELETE_LOGICAL, ActionFlag.UPDATE])
9796
DELETE_FAIL = ActionSequence("Delete without Create. Fail", [ActionFlag.DELETE_LOGICAL])
9897
UPDATE_FAIL = ActionSequence("Update without Create. Fail", [ActionFlag.UPDATE], outcome=ActionFlag.NONE)
99-
100-
101-
def create_row(unique_id, dose_amount, action_flag: str, header, inject_char=None):
102-
"""Helper function to create a single row with the specified UNIQUE_ID and ACTION_FLAG."""
103-
104-
name = "James" if not inject_char else b'Jam\xe9s'
105-
return {
106-
header: "9732928395",
107-
"PERSON_FORENAME": "PHYLIS",
108-
"PERSON_SURNAME": name,
109-
"PERSON_DOB": "20080217",
110-
"PERSON_GENDER_CODE": "0",
111-
"PERSON_POSTCODE": "WD25 0DZ",
112-
"DATE_AND_TIME": datetime.now(timezone.utc).strftime("%Y%m%dT%H%M%S"),
113-
"SITE_CODE": "RVVKC",
114-
"SITE_CODE_TYPE_URI": "https://fhir.nhs.uk/Id/ods-organization-code",
115-
"UNIQUE_ID": unique_id,
116-
"UNIQUE_ID_URI": RAVS_URI,
117-
"ACTION_FLAG": action_flag,
118-
"PERFORMING_PROFESSIONAL_FORENAME": "PHYLIS",
119-
"PERFORMING_PROFESSIONAL_SURNAME": name,
120-
"RECORDED_DATE": datetime.now(timezone.utc).strftime("%Y%m%d"),
121-
"PRIMARY_SOURCE": "TRUE",
122-
"VACCINATION_PROCEDURE_CODE": "956951000000104",
123-
"VACCINATION_PROCEDURE_TERM": "RSV vaccination in pregnancy (procedure)",
124-
"DOSE_SEQUENCE": "1",
125-
"VACCINE_PRODUCT_CODE": "42223111000001107",
126-
"VACCINE_PRODUCT_TERM": "Quadrivalent influenza vaccine (Sanofi Pasteur)",
127-
"VACCINE_MANUFACTURER": "Sanofi Pasteur",
128-
"BATCH_NUMBER": "BN92478105653",
129-
"EXPIRY_DATE": "20240915",
130-
"SITE_OF_VACCINATION_CODE": "368209003",
131-
"SITE_OF_VACCINATION_TERM": "Right arm",
132-
"ROUTE_OF_VACCINATION_CODE": "1210999013",
133-
"ROUTE_OF_VACCINATION_TERM": "Intradermal use",
134-
"DOSE_AMOUNT": dose_amount,
135-
"DOSE_UNIT_CODE": "2622896019",
136-
"DOSE_UNIT_TERM": "Inhalation - unit of product usage",
137-
"INDICATION_CODE": "1037351000000105",
138-
"LOCATION_CODE": "RJC02",
139-
"LOCATION_CODE_TYPE_URI": "https://fhir.nhs.uk/Id/ods-organization-code",
140-
}

e2e_batch/pr-NNN.env

Lines changed: 1 addition & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -6,6 +6,4 @@ SERVICE_BASE_PATH=immunisation-fhir-api-pr-NNN
66
AUDIT_TABLE_NAME=immunisation-batch-pr-NNN-audit-table
77
PR_NUMBER=NNN
88

9-
AWS_SECRET_ACCESS_KEY=AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
10-
AWS_ACCESS_KEY_ID=AAAAAAAAAAAAAAAAAAAA
11-
AWS_SESSION_TOKEN=*************************************************************************
9+
AWS_PROFILE={your-aws-profile}

e2e_batch/pyproject.toml

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -8,5 +8,5 @@ readme = "README.md"
88

99
[tool.poetry.dependencies]
1010
python = "~3.11"
11-
boto3 = "~1.38.42"
12-
pandas = "^2.3.0"
11+
boto3 = "~1.40.28"
12+
pandas = "^2.3.2"

e2e_batch/scenarios.py

Lines changed: 40 additions & 56 deletions
Original file line numberDiff line numberDiff line change
@@ -3,20 +3,16 @@
33
from vax_suppliers import TestPair, OdsVax
44
from constants import (
55
ActionFlag, BusRowResult, DestinationType, Operation,
6-
create_row,
7-
SOURCE_BUCKET,
86
ACK_BUCKET,
9-
TEMP_ACK_PREFIX,
107
RAVS_URI,
118
OperationOutcome
129
)
1310
from utils import (
1411
poll_s3_file_pattern, fetch_pk_and_operation_from_dynamodb,
1512
validate_fatal_error,
16-
delete_file_from_s3,
17-
delete_filename_from_audit_table,
18-
delete_filename_from_events_table,
19-
get_file_content_from_s3
13+
get_file_content_from_s3,
14+
aws_cleanup,
15+
create_row,
2016
)
2117
from clients import logger
2218
from errors import DynamoDBMismatchError
@@ -39,18 +35,19 @@ def __init__(self, scenario: dict):
3935
self.description: str = scenario.get("description", "")
4036
self.ods_vax: OdsVax = scenario.get("ods_vax")
4137
self.actions: list[TestAction] = scenario.get("actions", [])
42-
ods_vax = self.ods_vax
43-
self.ods = ods_vax.ods_code
44-
self.vax = ods_vax.vax
38+
self.ods = self.ods_vax.ods_code
39+
self.vax = self.ods_vax.vax
4540
self.dose_amount: float = scenario.get("dose_amount", 0.5)
46-
self.inject_char = scenario.get("test_encoding", False)
41+
self.inject_cp1252 = scenario.get("create_with_cp1252_encoded_character", False)
4742
self.header = scenario.get("header", "NHS_NUMBER")
4843
self.version = scenario.get("version", 5)
4944
self.operation_outcome = scenario.get("operation_outcome", "")
50-
# initialise attribs to be set later
51-
self.ack_keys = {DestinationType.INF: None, DestinationType.BUS: None}
52-
self.key = None # TODO is identifier and key the same
5345
self.enabled = scenario.get("enabled", False)
46+
self.ack_keys = {DestinationType.INF: None, DestinationType.BUS: None}
47+
# initialise attribs to be set later
48+
self.key = None # S3 key of the uploaded file
49+
self.file_name = None # name of the generated CSV file
50+
self.identifier = None # unique identifier of subject in the CSV file rows
5451

5552
def get_poll_destinations(self, pending: bool) -> bool:
5653
# loop through keys in test (inf and bus)
@@ -113,14 +110,14 @@ def check_bus_file_content(self):
113110
elif row_HEADER_RESPONSE_CODE == "Fatal Error":
114111
validate_fatal_error(desc, row, i, operation_outcome)
115112

116-
def generate_csv_file_good(self):
113+
def generate_csv_file(self):
117114

118115
self.file_name = self.get_file_name(self.vax, self.ods, self.version)
119116
logger.info(f"Test \"{self.name}\" File {self.file_name}")
120117
data = []
121118
self.identifier = str(uuid.uuid4())
122119
for action in self.actions:
123-
row = create_row(self.identifier, self.dose_amount, action.action, self.header, self.inject_char)
120+
row = create_row(self.identifier, self.dose_amount, action.action, self.header, self.inject_cp1252)
124121
logger.info(f" > {action.action} - {self.vax}/{self.ods} - {self.identifier}")
125122
data.append(row)
126123
df = pd.DataFrame(data)
@@ -129,48 +126,35 @@ def generate_csv_file_good(self):
129126

130127
def get_file_name(self, vax_type, ods, version="5"):
131128
timestamp = datetime.now(timezone.utc).strftime("%Y%m%dT%H%M%S00")
132-
# timestamp = timestamp[:-3]
133129
return f"{vax_type}_Vaccinations_v{version}_{ods}_{timestamp}.csv"
134130

135131
def cleanup(self):
136-
if self.key:
137-
archive_file = f"archive/{self.key}"
138-
if not delete_file_from_s3(SOURCE_BUCKET, archive_file):
139-
logger.warning(f"S3 delete fail {SOURCE_BUCKET}: {archive_file}")
140-
delete_filename_from_audit_table(self.key)
141-
delete_filename_from_events_table(self.identifier)
142-
for ack_key in self.ack_keys.values():
143-
if ack_key:
144-
if not delete_file_from_s3(ACK_BUCKET, ack_key):
145-
logger.warning(f"s3 delete fail {ACK_BUCKET}: {ack_key}")
146-
# cleanup TEMP_ACK
147-
delete_file_from_s3(ACK_BUCKET, TEMP_ACK_PREFIX)
148-
149-
150-
class TestCases:
151-
def __init__(self, test_cases: dict):
152-
self.test_cases = []
153-
# scenarios = scenario.get(environment)
154-
for s in test_cases:
155-
self.test_cases.append(TestCase(s))
156-
157-
def enable_tests(self, names: list[str]):
158-
for name in names:
159-
for test in self.test_cases:
160-
if test.name == name:
161-
test.enabled = True
162-
break
163-
else:
164-
raise Exception(f"Test case with name '{name}' not found.")
165-
166-
def generate_csv_files_good(self) -> list[TestCase]:
167-
"""Generate CSV files based on a list of Test Rules."""
168-
ret = []
169-
for seed_data in self.test_cases:
170-
if seed_data.enabled:
171-
seed_data.generate_csv_file_good()
172-
ret.append(seed_data)
173-
return ret
132+
aws_cleanup(self.key, self.identifier, self.ack_keys)
133+
134+
135+
def create_test_cases(test_case_dict: dict) -> list[TestCase]:
136+
"""Initialize test cases from a dictionary."""
137+
return [TestCase(name) for name in test_case_dict]
138+
139+
140+
def enable_tests(test_cases: list[TestCase], names: list[str]) -> None:
141+
"""Enable only the test cases with the given names."""
142+
for name in names:
143+
for test in test_cases:
144+
if test.name == name:
145+
test.enabled = True
146+
break
147+
else:
148+
raise Exception(f"Test case with name '{name}' not found.")
149+
150+
151+
def generate_csv_files(test_cases: list[TestCase]) -> list[TestCase]:
152+
"""Generate CSV files for all enabled test cases."""
153+
ret = []
154+
for test in test_cases:
155+
if test.enabled:
156+
test.generate_csv_file()
157+
ret.append(test)
174158

175159

176160
scenarios = {
@@ -220,7 +204,7 @@ def generate_csv_files_good(self) -> list[TestCase]:
220204
"ods_vax": TestPair.YGA_MENACWY_CRUDS,
221205
"operation_outcome": ActionFlag.CREATE,
222206
"actions": [TestAction(ActionFlag.CREATE)],
223-
"test_encoding": True
207+
"create_with_cp1252_encoded_character": True
224208
}
225209
],
226210
"ref": []

e2e_batch/test_e2e_batch.py

Lines changed: 13 additions & 18 deletions
Original file line numberDiff line numberDiff line change
@@ -5,53 +5,48 @@
55
get_file_content_from_s3,
66
check_ack_file_content,
77
validate_row_count,
8+
purge_sqs_queues,
9+
delete_file_from_s3
810
)
911

10-
from clients import logger, sqs_client, batch_fifo_queue_url, ack_metadata_queue_url
11-
from scenarios import scenarios, TestCases, TestCase
12+
from clients import logger
13+
from scenarios import scenarios, TestCase, create_test_cases, enable_tests, generate_csv_files
1214

1315
from constants import (
1416
SOURCE_BUCKET,
1517
INPUT_PREFIX,
1618
ACK_BUCKET,
1719
environment,
18-
DestinationType
20+
DestinationType,
21+
TEMP_ACK_PREFIX
1922
)
2023

2124

2225
class TestE2EBatch(unittest.TestCase):
2326
def setUp(self):
24-
test_data = TestCases(scenarios["dev"])
25-
test_data.enable_tests([
27+
self.tests: list[TestCase] = create_test_cases(scenarios["dev"])
28+
enable_tests(self.tests, [
2629
"Successful Create",
2730
"Successful Update",
2831
"Successful Delete",
2932
"Create with 1252 char",
3033
"Failed Update",
3134
"Failed Delete",
3235
])
33-
self.tests: list[TestCase] = test_data.generate_csv_files_good()
36+
generate_csv_files(self.tests)
3437

3538
def tearDown(self):
3639
logger.info("Cleanup...")
3740
for test in self.tests:
3841
test.cleanup()
39-
40-
try:
41-
# only purge if ENVIRONMENT=pr-* or dev
42-
if environment.startswith("pr-"):
43-
sqs_client.purge_queue(QueueUrl=batch_fifo_queue_url)
44-
sqs_client.purge_queue(QueueUrl=ack_metadata_queue_url)
45-
except sqs_client.exceptions.PurgeQueueInProgress:
46-
logger.error("SQS purge already in progress. Try again later.")
47-
except Exception as e:
48-
logger.error(f"SQS Purge error: {e}")
42+
delete_file_from_s3(ACK_BUCKET, TEMP_ACK_PREFIX)
43+
purge_sqs_queues()
4944

5045
@unittest.skipIf(environment == "ref", "Skip for ref")
5146
def test_batch_submission(self):
5247
"""Test all scenarios and submit as batch."""
5348
start_time = time.time()
54-
max_timeout = 1200 # seconds)
49+
max_timeout = 1200 # seconds
5550

5651
send_files(self.tests)
5752

@@ -91,7 +86,7 @@ def poll_for_responses(tests: list[TestCase], max_timeout=1200) -> bool:
9186
return True
9287

9388

94-
def validate_responses(tests: TestCases):
89+
def validate_responses(tests: list[TestCase]):
9590
start_time = time.time()
9691
count = 0
9792
expected_count = len(tests) * 2

0 commit comments

Comments
 (0)