Skip to content

Commit e76c6e8

Browse files
nhsdevwsnhsdevws
committed
Parked
1 parent 3000b36 commit e76c6e8

File tree

5 files changed

+204
-177
lines changed

5 files changed

+204
-177
lines changed

‎azure/templates/post-deploy.yml‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -236,7 +236,7 @@ steps:
236236
237237
displayName: e2e serial
238238
workingDirectory: "$(Pipeline.Workspace)/s/$(SERVICE_NAME)/$(SERVICE_ARTIFACT_NAME)/e2e_batch"
239-
# condition: eq(1, 1) # Disable task but make this step visible in the pipeline
239+
condition: eq(2, 1) # Disable task but make this step visible in the pipeline
240240
241241
- bash: |
242242
pyenv local 3.11

‎e2e_batch/Makefile‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -2,7 +2,7 @@
22

33
APIGEE_ACCESS_TOKEN ?= $(shell export SSO_LOGIN_URL=https://login.apigee.com && eval get_token -u $(APIGEE_USERNAME))
44
AWS_DOMAIN_NAME=https://$(shell make -C ../terraform -s output name=service_domain_name || true)
5-
PARALLEL_WORKERS=2
5+
PARALLEL_WORKERS=4
66

77
print-token:
88
@echo "APIGEE_ACCESS_TOKEN=$(APIGEE_ACCESS_TOKEN)"

‎e2e_batch/test_e2e_batch.py‎

Lines changed: 22 additions & 168 deletions
Original file line numberDiff line numberDiff line change
@@ -1,4 +1,3 @@
1-
import asyncio
21
import unittest
32
from utils import (
43
generate_csv,
@@ -7,236 +6,91 @@
76
wait_for_ack_file,
87
check_ack_file_content,
98
validate_row_count,
10-
upload_config_file,
11-
generate_csv_with_ordered_100000_rows,
12-
verify_final_ack_file,
139
delete_file_from_s3
1410
)
1511

1612
from constants import (
1713
SOURCE_BUCKET,
1814
INPUT_PREFIX,
1915
ACK_BUCKET,
20-
PRE_VALIDATION_ERROR,
21-
POST_VALIDATION_ERROR,
2216
DUPLICATE,
23-
FILE_NAME_VAL_ERROR,
2417
environment
2518
)
2619

2720

28-
class TestE2EBatch(unittest.IsolatedAsyncioTestCase):
29-
async def asyncSetUp(self):
21+
class TestE2EBatch(unittest.TestCase):
22+
def setUp(self):
3023
self.uploaded_files = [] # Tracks uploaded input keys
3124
self.ack_files = [] # Tracks ack keys
3225

33-
async def asyncTearDown(self):
26+
def tearDown(self):
3427
for file_key in self.uploaded_files:
3528
delete_file_from_s3(SOURCE_BUCKET, file_key)
3629
for ack_key in self.ack_files:
3730
delete_file_from_s3(ACK_BUCKET, ack_key)
3831

3932
if environment != "ref":
40-
async def test_create_success(self):
33+
def test_create_success(self):
4134
"""Test CREATE scenario."""
42-
input_file = generate_csv("PHYLIS", "0.3",
43-
action_flag="CREATE", offset=1,
44-
vax_type="COVID19", ods="8HA94")
35+
input_file = generate_csv("PHYLIS", "0.3", action_flag="CREATE", offset=1, vax_type="COVID19", ods="8HA94")
4536

46-
key = await upload_file_to_s3(input_file, SOURCE_BUCKET, INPUT_PREFIX)
37+
key = upload_file_to_s3(input_file, SOURCE_BUCKET, INPUT_PREFIX)
4738
self.uploaded_files.append(key)
4839

49-
ack_key = await wait_for_ack_file(None, input_file)
40+
ack_key = wait_for_ack_file(None, input_file)
5041
self.ack_files.append(ack_key)
5142

5243
validate_row_count(input_file, ack_key)
5344

54-
ack_content = await get_file_content_from_s3(ACK_BUCKET, ack_key)
45+
ack_content = get_file_content_from_s3(ACK_BUCKET, ack_key)
5546
check_ack_file_content(ack_content, "OK", None, "CREATE")
5647

57-
async def test_duplicate_create(self):
48+
def test_duplicate_create(self):
5849
"""Test DUPLICATE scenario."""
5950

60-
input_file = generate_csv("PHYLIS", "0.3",
61-
action_flag="CREATE", same_id=True, offset=2,
62-
vax_type="FLU", ods="8HK48")
51+
input_file = generate_csv("PHYLIS", "0.3", action_flag="CREATE", same_id=True, offset=2,
52+
vax_type="COVID19", ods="8HA94")
6353

64-
key = await upload_file_to_s3(input_file, SOURCE_BUCKET, INPUT_PREFIX)
54+
key = upload_file_to_s3(input_file, SOURCE_BUCKET, INPUT_PREFIX)
6555
self.uploaded_files.append(key)
6656

67-
ack_key = await wait_for_ack_file(None, input_file)
57+
ack_key = wait_for_ack_file(None, input_file)
6858
self.ack_files.append(ack_key)
6959

7060
validate_row_count(input_file, ack_key)
7161

72-
ack_content = await get_file_content_from_s3(ACK_BUCKET, ack_key)
62+
ack_content = get_file_content_from_s3(ACK_BUCKET, ack_key)
7363
check_ack_file_content(ack_content, "Fatal Error", DUPLICATE, "CREATE")
7464

75-
async def test_update_success(self):
65+
def test_update_success(self):
7666
"""Test UPDATE scenario."""
7767
input_file = generate_csv("PHYLIS", "0.5", action_flag="UPDATE",
7868
offset=3, vax_type="MMR", ods="V0V8L")
7969

80-
key = await upload_file_to_s3(input_file, SOURCE_BUCKET, INPUT_PREFIX)
70+
key = upload_file_to_s3(input_file, SOURCE_BUCKET, INPUT_PREFIX)
8171
self.uploaded_files.append(key)
8272

83-
ack_key = await wait_for_ack_file(None, input_file)
73+
ack_key = wait_for_ack_file(None, input_file)
8474
self.ack_files.append(ack_key)
8575

8676
validate_row_count(input_file, ack_key)
8777

88-
ack_content = await get_file_content_from_s3(ACK_BUCKET, ack_key)
78+
ack_content = get_file_content_from_s3(ACK_BUCKET, ack_key)
8979
check_ack_file_content(ack_content, "OK", None, "UPDATE")
9080

91-
async def test_reinstated_success(self):
81+
def test_reinstated_success(self):
9282
"""Test REINSTATED scenario."""
9383
input_file = generate_csv("PHYLIS", "0.5",
9484
action_flag="REINSTATED", offset=4,
9585
vax_type="HPV", ods="DPSREDUCED")
9686

97-
key = await upload_file_to_s3(input_file, SOURCE_BUCKET, INPUT_PREFIX)
87+
key = upload_file_to_s3(input_file, SOURCE_BUCKET, INPUT_PREFIX)
9888
self.uploaded_files.append(key)
9989

100-
ack_key = await wait_for_ack_file(None, input_file)
90+
ack_key = wait_for_ack_file(None, input_file)
10191
self.ack_files.append(ack_key)
10292

10393
validate_row_count(input_file, ack_key)
10494

105-
ack_content = await get_file_content_from_s3(ACK_BUCKET, ack_key)
95+
ack_content = get_file_content_from_s3(ACK_BUCKET, ack_key)
10696
check_ack_file_content(ack_content, "OK", None, "reinstated")
107-
108-
async def test_update_reinstated_success(self):
109-
"""Test UPDATE-REINSTATED scenario."""
110-
input_file = generate_csv("PHYLIS", "0.5",
111-
action_flag="UPDATE-REINSTATED", offset=5,
112-
vax_type="MENACWY", ods="DPSFULL")
113-
114-
key = await upload_file_to_s3(input_file, SOURCE_BUCKET, INPUT_PREFIX)
115-
self.uploaded_files.append(key)
116-
117-
ack_key = await wait_for_ack_file(None, input_file)
118-
self.ack_files.append(ack_key)
119-
120-
validate_row_count(input_file, ack_key)
121-
122-
ack_content = await get_file_content_from_s3(ACK_BUCKET, ack_key)
123-
check_ack_file_content(ack_content, "OK", None, "update-reinstated")
124-
125-
async def test_delete_success(self):
126-
"""Test DELETE scenario."""
127-
input_file = generate_csv("PHYLIS", "0.8",
128-
action_flag="DELETE", offset=6,
129-
vax_type="MMR", ods="V0V8L")
130-
131-
key = await upload_file_to_s3(input_file, SOURCE_BUCKET, INPUT_PREFIX)
132-
self.uploaded_files.append(key)
133-
134-
ack_key = await wait_for_ack_file(None, input_file)
135-
self.ack_files.append(ack_key)
136-
137-
validate_row_count(input_file, ack_key)
138-
139-
ack_content = await get_file_content_from_s3(ACK_BUCKET, ack_key)
140-
check_ack_file_content(ack_content, "OK", None, "DELETE")
141-
142-
async def test_pre_validation_error(self):
143-
"""Test PRE-VALIDATION error scenario."""
144-
input_file = generate_csv("PHYLIS", "TRUE", action_flag="CREATE",
145-
offset=7, vax_type="MMR", ods="X8E5B")
146-
147-
key = await upload_file_to_s3(input_file, SOURCE_BUCKET, INPUT_PREFIX)
148-
self.uploaded_files.append(key)
149-
150-
ack_key = await wait_for_ack_file(None, input_file)
151-
self.ack_files.append(ack_key)
152-
153-
validate_row_count(input_file, ack_key)
154-
155-
ack_content = await get_file_content_from_s3(ACK_BUCKET, ack_key)
156-
check_ack_file_content(ack_content, "Fatal Error", PRE_VALIDATION_ERROR, None)
157-
158-
async def test_post_validation_error(self):
159-
"""Test POST-VALIDATION error scenario."""
160-
input_file = generate_csv("", "0.3", action_flag="CREATE",
161-
offset=8, vax_type="3IN1", ods="YGJ")
162-
163-
key = await upload_file_to_s3(input_file, SOURCE_BUCKET, INPUT_PREFIX)
164-
self.uploaded_files.append(key)
165-
166-
ack_key = await wait_for_ack_file(None, input_file)
167-
self.ack_files.append(ack_key)
168-
169-
ack_content = await get_file_content_from_s3(ACK_BUCKET, ack_key)
170-
check_ack_file_content(ack_content, "Fatal Error", POST_VALIDATION_ERROR, None)
171-
172-
async def test_file_name_validation_error(self):
173-
"""Test FILE-NAME-VALIDATION error scenario."""
174-
input_file = generate_csv("PHYLIS", "0.3", action_flag="CREATE",
175-
file_key=True, offset=9,
176-
vax_type="HPV", ods="YGA")
177-
key = await upload_file_to_s3(input_file, SOURCE_BUCKET, INPUT_PREFIX)
178-
self.uploaded_files.append(key)
179-
180-
ack_key = await wait_for_ack_file(True, input_file)
181-
self.ack_files.append(ack_key)
182-
183-
ack_content = await get_file_content_from_s3(ACK_BUCKET, ack_key)
184-
check_ack_file_content(ack_content, "Failure", FILE_NAME_VAL_ERROR, None)
185-
186-
async def test_header_name_validation_error(self):
187-
"""Test HEADER-NAME-VALIDATION error scenario."""
188-
input_file = generate_csv("PHYLIS", "0.3", action_flag="CREATE",
189-
headers="NH_NUMBER", offset=10,
190-
vax_type="3IN1", ods="YGMYW")
191-
192-
key = await upload_file_to_s3(input_file, SOURCE_BUCKET, INPUT_PREFIX)
193-
self.uploaded_files.append(key)
194-
195-
ack_key = await wait_for_ack_file(True, input_file)
196-
self.ack_files.append(ack_key)
197-
198-
ack_content = await get_file_content_from_s3(ACK_BUCKET, ack_key)
199-
check_ack_file_content(ack_content, "Failure", FILE_NAME_VAL_ERROR, None)
200-
201-
# This test updates the permissions_config.json file from the imms-internal-dev-supplier-config
202-
# S3 bucket shared across multiple environments (PR environments, internal-dev, int, and ref).
203-
# Running this may modify permissions in these environments, causing unintended side effects.
204-
@unittest.skip("Modifies shared S3 permissions configuration")
205-
async def test_invalid_permission(self):
206-
"""Test INVALID-PERMISSION error scenario."""
207-
await upload_config_file("MMR_FULL") # permissions_config.json is updated here
208-
await asyncio.sleep(20)
209-
210-
input_file = generate_csv("PHYLIS", "0.3", action_flag="CREATE",
211-
offset=11, vax_type="PINNACLE", ods="8J1100001")
212-
213-
key = await upload_file_to_s3(input_file, SOURCE_BUCKET, INPUT_PREFIX)
214-
self.uploaded_files.append(key)
215-
216-
ack_key = await wait_for_ack_file(True, input_file)
217-
self.ack_files.append(ack_key)
218-
219-
ack_content = await get_file_content_from_s3(ACK_BUCKET, ack_key)
220-
check_ack_file_content(ack_content, "Failure", FILE_NAME_VAL_ERROR, None)
221-
222-
await upload_config_file("COVID19_FULL")
223-
await asyncio.sleep(20)
224-
225-
else:
226-
async def test_end_to_end_speed_test_with_100000_rows(self):
227-
"""Test end_to_end_speed_test_with_100000_rows scenario with full integration"""
228-
input_file = generate_csv_with_ordered_100000_rows(12,
229-
vax_type="COVID19", ods="DPSFULL")
230-
231-
key = await upload_file_to_s3(input_file, SOURCE_BUCKET, INPUT_PREFIX)
232-
self.uploaded_files.append(key)
233-
234-
final_ack_key = await wait_for_ack_file(None, input_file, timeout=1800)
235-
self.ack_files.append(final_ack_key)
236-
237-
response = await verify_final_ack_file(final_ack_key)
238-
assert response is True
239-
240-
241-
if __name__ == "__main__":
242-
unittest.main()

0 commit comments

Comments
 (0)