Skip to content

Commit 1f2be81

Browse files
bethany-kish-nhsbethany-kish-nhs
authored
[PRMT-542] Add test for bulk upload - 300 patients with 3 files (#770)
1 parent f2b5a36 commit 1f2be81

File tree

11 files changed

+148
-9
lines changed

11 files changed

+148
-9
lines changed

‎.github/workflows/base-e2e-backendtest.yml‎

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -44,10 +44,10 @@ jobs:
4444
mask-aws-account-id: true
4545
aws-region: ${{ vars.AWS_REGION }}
4646

47-
- name: Set up Python ${{ inputs.python_version }}
47+
- name: Set up Python 3.11
4848
uses: actions/setup-python@v5
4949
with:
50-
python-version: ${{ inputs.python_version }}
50+
python-version: 3.11
5151

5252
- name: Make virtual environment
5353
run: |

‎.github/workflows/base-run-bulk-upload.yml‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -32,7 +32,7 @@ on:
3232
description: "How many files per patient to generate."
3333
required: false
3434
type: "string"
35-
default: "1"
35+
default: "3"
3636
secrets:
3737
AWS_ASSUME_ROLE:
3838
required: true

‎.github/workflows/bulk-upload-300-test.yml‎

Lines changed: 74 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,74 @@
1+
# .github/workflows/terraform-dev
2+
name: "Test: Bulk Upload Validator - 300 patients 3 files"
3+
4+
on:
5+
workflow_dispatch:
6+
inputs:
7+
build_branch:
8+
description: "Branch with bulk upload tests."
9+
required: true
10+
type: "string"
11+
environment:
12+
description: "Which Environment type are we using"
13+
required: true
14+
type: "string"
15+
default: "development"
16+
sandbox:
17+
description: "Sandbox to run the test on."
18+
required: true
19+
type: "string"
20+
secrets:
21+
AWS_ASSUME_ROLE:
22+
required: true
23+
24+
permissions:
25+
id-token: write
26+
contents: read
27+
28+
jobs:
29+
bulk-upload-test:
30+
runs-on: ubuntu-latest
31+
environment: ${{ inputs.environment }}
32+
steps:
33+
- name: Checkout
34+
uses: actions/checkout@v5
35+
with:
36+
repository: "nhsconnect/national-document-repository"
37+
ref: ${{ inputs.build_branch }}
38+
39+
- name: AWS Role
40+
uses: aws-actions/configure-aws-credentials@v4
41+
with:
42+
role-to-assume: ${{ secrets.AWS_ASSUME_ROLE }}
43+
role-skip-session-tagging: true
44+
mask-aws-account-id: true
45+
aws-region: ${{ vars.AWS_REGION }}
46+
47+
- name: Set up Python 3.11
48+
uses: actions/setup-python@v5
49+
with:
50+
python-version: 3.11
51+
52+
- name: Make virtual environment
53+
run: |
54+
make env
55+
56+
- name: Start virtual environment
57+
run: |
58+
source ./lambdas/venv/bin/activate
59+
echo PATH=$PATH >> $GITHUB_ENV
60+
61+
- name: Get E2e Test Variables
62+
run: |
63+
LG_DYNAMO_STORE="${SANDBOX}_LloydGeorgeReferenceMetadata"
64+
LG_UNSTITCHED_STORE="${SANDBOX}_UnstitchedLloydGeorgeReferenceMetadata"
65+
BULK_UPLOAD_REPORT_STORE="${SANDBOX}_BulkUploadReport"
66+
echo "LG_METADATA_TABLE=$LG_DYNAMO_STORE" >> $GITHUB_ENV
67+
echo "LG_UNSTITCHED_TABLE=$LG_UNSTITCHED_STORE" >> $GITHUB_ENV
68+
echo "BULK_REPORT_TABLE=$BULK_UPLOAD_REPORT_STORE" >> $GITHUB_ENV
69+
env:
70+
SANDBOX: ${{ inputs.sandbox }}
71+
72+
- name: Bulk Upload Test
73+
run: |
74+
make test-bulk-upload-e2e

‎.github/workflows/full-deploy-to-sandbox.yml‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -118,6 +118,6 @@ jobs:
118118
sandbox: ${{ inputs.sandbox }}
119119
combi_settings: "combi300"
120120
base_branch: ${{ inputs.build_branch }}
121-
file_count: "1"
121+
file_count: "3"
122122
secrets:
123123
AWS_ASSUME_ROLE: ${{ secrets.AWS_ASSUME_ROLE }}

‎.github/workflows/run-bulk-upload-dev.yml‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -24,7 +24,7 @@ on:
2424
description: "How many files per patient"
2525
required: true
2626
type: "string"
27-
default: "1"
27+
default: "3"
2828

2929
jobs:
3030
bulk_upload:

‎.github/workflows/run-bulk-upload-pre-prod.yml‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -19,7 +19,7 @@ on:
1919
description: "How many files per patient"
2020
required: true
2121
type: "string"
22-
default: "1"
22+
default: "3"
2323

2424
jobs:
2525
bulk_upload:

‎.github/workflows/run-bulk-upload-test.yml‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -19,7 +19,7 @@ on:
1919
description: "How many files per patient"
2020
required: true
2121
type: "string"
22-
default: "1"
22+
default: "3"
2323

2424
jobs:
2525
bulk_upload:

‎Makefile‎

Lines changed: 5 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -60,10 +60,13 @@ check-packages:
6060
./lambdas/venv/bin/pip-audit -r $(ALERTING_REQUIREMENTS)
6161

6262
test-api-e2e:
63-
cd ./lambdas && ./venv/bin/python3 -m pytest tests/e2e -vv
63+
cd ./lambdas && ./venv/bin/python3 -m pytest tests/e2e/api -vv
6464

6565
test-api-e2e-snapshots:
66-
cd ./lambdas && ./venv/bin/python3 -m pytest tests/e2e --snapshot-update
66+
cd ./lambdas && ./venv/bin/python3 -m pytest tests/e2e/api --snapshot-update
67+
68+
test-bulk-upload-e2e:
69+
cd ./lambdas && ./venv/bin/python3 -m pytest tests/e2e/bulk_upload -vv
6770

6871
test-unit:
6972
cd ./lambdas && ./venv/bin/python3 -m pytest tests/unit

‎lambdas/tests/e2e/bulk_upload/__init__.py‎

Whitespace-only changes.

‎lambdas/tests/e2e/bulk_upload/test_bulk_upload_300_output.py‎

Lines changed: 59 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,59 @@
1+
from services.base.dynamo_service import DynamoDBService
2+
from tests.e2e.conftest import (
3+
LG_METADATA_TABLE ,
4+
LG_UNSTITCHED_TABLE,
5+
BULK_REPORT_TABLE,
6+
)
7+
8+
dynamo_service = DynamoDBService()
9+
10+
def test_bulk_upload_300_patients_with_3_files():
11+
12+
#assert bulk upload report table values
13+
bulk_upload_table = dynamo_service.scan_whole_table(BULK_REPORT_TABLE, "Reason, UploadStatus, NhsNumber")
14+
assert len(bulk_upload_table) == 868
15+
16+
complete_uploads = []
17+
failed_uploads = []
18+
name_mismatch_rejections = []
19+
dob_mismatch_rejections = []
20+
21+
for item in bulk_upload_table:
22+
if item.get("UploadStatus") == "complete":
23+
complete_uploads.append(item)
24+
elif item.get("UploadStatus") == "failed":
25+
failed_uploads.append(item)
26+
if item.get("Reason") == "Patient name does not match our records":
27+
name_mismatch_rejections.append(item)
28+
elif item.get("Reason") == "Patient DoB does not match our records":
29+
dob_mismatch_rejections.append(item)
30+
31+
assert len(complete_uploads) == 765
32+
assert len(failed_uploads) == 103
33+
assert len(name_mismatch_rejections) == 6
34+
assert len(dob_mismatch_rejections) == 27
35+
36+
#assert lloyd george metadata values, this will also validate the files were stitched
37+
lg_metadata_table = dynamo_service.scan_whole_table(LG_METADATA_TABLE, "CurrentGpOds, NhsNumber")
38+
assert len(lg_metadata_table) == 255
39+
40+
dece_records = []
41+
susp_records = []
42+
rest_records = []
43+
44+
for item in lg_metadata_table:
45+
ods_code = item.get("CurrentGpOds")
46+
if ods_code == "DECE":
47+
dece_records.append(item)
48+
elif ods_code == "SUSP":
49+
susp_records.append(item)
50+
elif ods_code == "REST":
51+
rest_records.append(item)
52+
53+
assert len(dece_records) == 4
54+
assert len(susp_records) == 4
55+
assert len(rest_records) == 9
56+
57+
#assert unstitched metadata contains original unstitched files
58+
lg_unstitched_metadata_table = dynamo_service.scan_whole_table(LG_UNSTITCHED_TABLE, "CurrentGpOds")
59+
assert len(lg_unstitched_metadata_table) == 765

0 commit comments

Comments
 (0)