diff --git a/.gitignore b/.gitignore index 735c385b6..e06864c72 100644 --- a/.gitignore +++ b/.gitignore @@ -108,4 +108,4 @@ lambdas/tests/e2e/apim/*.pdf tests/bulk-upload/output/** tests/bulk-upload/test_patients_data/** -tests/bulk-upload/scripts/scenario_report.txt \ No newline at end of file +tests/bulk-upload/scripts/scenario_report.txt diff --git a/Makefile b/Makefile index 11013a969..61d1a3a79 100644 --- a/Makefile +++ b/Makefile @@ -104,11 +104,25 @@ else cd ./lambdas && ./venv/bin/python3 -m pytest tests/e2e/api --ignore=tests/e2e/api/fhir --snapshot-update endif +initiate-bulk-upload: +ifeq ($(CONTAINER), true) + cd ./lambdas && PYTHONPATH=. poetry run pytest tests/e2e/bulk_upload/upload -vv +else + cd ./lambdas && ./venv/bin/python3 -m pytest tests/e2e/bulk_upload/upload -vv +endif + test-bulk-upload-e2e: ifeq ($(CONTAINER), true) - cd ./lambdas && PYTHONPATH=. poetry run pytest tests/e2e/bulk_upload -vv + cd ./lambdas && PYTHONPATH=. poetry run pytest tests/e2e/bulk_upload/check -vv +else + cd ./lambdas && ./venv/bin/python3 -m pytest tests/e2e/bulk_upload/check -vv +endif + +test-bulk-upload-e2e-snapshots: +ifeq ($(CONTAINER), true) + cd ./lambdas && PYTHONPATH=. poetry run pytest tests/e2e/bulk_upload/check --snapshot-update else - cd ./lambdas && ./venv/bin/python3 -m pytest tests/e2e/bulk_upload -vv + cd ./lambdas && ./venv/bin/python3 -m pytest tests/e2e/bulk_upload/check --snapshot-update endif test-unit: diff --git a/lambdas/tests/e2e/api/fhir/conftest.py b/lambdas/tests/e2e/api/fhir/conftest.py index 28e759e12..d55a816e0 100644 --- a/lambdas/tests/e2e/api/fhir/conftest.py +++ b/lambdas/tests/e2e/api/fhir/conftest.py @@ -7,7 +7,7 @@ import pytest import requests -from lambdas.tests.e2e.helpers.data_helper import PdmDataHelper +from tests.e2e.helpers.data_helper import PdmDataHelper pdm_data_helper = PdmDataHelper() @@ -27,9 +27,7 @@ def test_data(): pdm_data_helper.tidyup(record) -def fetch_with_retry_mtls( - session, url, headers, condition_func=None, max_retries=5, delay=10 -): +def fetch_with_retry_mtls(session, url, headers, condition_func=None, max_retries=5, delay=10): retries = 0 while retries < max_retries: response = session.get(url, headers=headers) @@ -47,9 +45,7 @@ def fetch_with_retry_mtls( raise Exception("Condition not met within retry limit") -def create_mtls_session( - client_cert_path=CLIENT_CERT_PATH, client_key_path=CLIENT_KEY_PATH -): +def create_mtls_session(client_cert_path=CLIENT_CERT_PATH, client_key_path=CLIENT_KEY_PATH): session = requests.Session() session.cert = (client_cert_path, client_key_path) return session @@ -111,9 +107,7 @@ def create_and_store_pdm_record( size: int | None = None, ): """Helper to create metadata and resource for a record.""" - record = pdm_data_helper.build_record( - nhs_number=nhs_number, doc_status=doc_status, size=size - ) + record = pdm_data_helper.build_record(nhs_number=nhs_number, doc_status=doc_status, size=size) test_data.append(record) pdm_data_helper.create_metadata(record) pdm_data_helper.create_resource(record) diff --git a/lambdas/tests/e2e/api/fhir/test_mtls_routing.py b/lambdas/tests/e2e/api/fhir/test_mtls_routing.py index 6256f166b..d94039641 100644 --- a/lambdas/tests/e2e/api/fhir/test_mtls_routing.py +++ b/lambdas/tests/e2e/api/fhir/test_mtls_routing.py @@ -1,7 +1,7 @@ import os import uuid -from lambdas.tests.e2e.api.fhir.conftest import get_pdm_document_reference +from tests.e2e.api.fhir.conftest import get_pdm_document_reference UNAUTHORISED_CLIENT_CERT_PATH = os.environ.get("UNAUTHORISED_CLIENT_CERT_PATH") UNAUTHORISED_CLIENT_KEY_PATH = os.environ.get("UNAUTHORISED_CLIENT_KEY_PATH") @@ -9,9 +9,7 @@ def test_mtls_invalid_common_name(): record_id = str(uuid.uuid4()) - response = get_pdm_document_reference( - record_id, UNAUTHORISED_CLIENT_CERT_PATH, UNAUTHORISED_CLIENT_KEY_PATH - ) + response = get_pdm_document_reference(record_id, UNAUTHORISED_CLIENT_CERT_PATH, UNAUTHORISED_CLIENT_KEY_PATH) assert response.status_code == 400 data = response.json() diff --git a/lambdas/tests/e2e/api/fhir/test_retrieve_document_fhir_api.py b/lambdas/tests/e2e/api/fhir/test_retrieve_document_fhir_api.py index 5cafe6b7d..942721e65 100644 --- a/lambdas/tests/e2e/api/fhir/test_retrieve_document_fhir_api.py +++ b/lambdas/tests/e2e/api/fhir/test_retrieve_document_fhir_api.py @@ -2,13 +2,13 @@ import uuid import pytest -from tests.e2e.helpers.data_helper import PdmDataHelper -from lambdas.tests.e2e.api.fhir.conftest import ( +from tests.e2e.api.fhir.conftest import ( PDM_S3_BUCKET, create_and_store_pdm_record, get_pdm_document_reference, ) +from tests.e2e.helpers.data_helper import PdmDataHelper pdm_data_helper = PdmDataHelper() @@ -16,9 +16,7 @@ @pytest.mark.parametrize("file_size", [None, 10 * 1024 * 1024]) def test_file_retrieval(test_data, file_size): """Test retrieval for small and large files.""" - pdm_record = create_and_store_pdm_record( - test_data, size=file_size if file_size else None - ) + pdm_record = create_and_store_pdm_record(test_data, size=file_size if file_size else None) response = get_pdm_document_reference(pdm_record["id"]) assert response.status_code == 200 @@ -30,10 +28,7 @@ def test_file_retrieval(test_data, file_size): expected_bytes = b"Sample PDF Content" assert decoded_data == expected_bytes else: - expected_presign_uri = ( - f"https://{PDM_S3_BUCKET}.s3.eu-west-2.amazonaws.com/" - f"{pdm_record['nhs_number']}/{pdm_record['id']}" - ) + expected_presign_uri = f"https://{PDM_S3_BUCKET}.s3.eu-west-2.amazonaws.com/{pdm_record['nhs_number']}/{pdm_record['id']}" assert expected_presign_uri in json["content"][0]["attachment"]["url"] @@ -43,9 +38,7 @@ def test_file_retrieval(test_data, file_size): (str(uuid.uuid4()), 404, "RESOURCE_NOT_FOUND", "Document reference not found"), ], ) -def test_retrieve_edge_cases( - record_id, expected_status, expected_code, expected_diagnostics -): +def test_retrieve_edge_cases(record_id, expected_status, expected_code, expected_diagnostics): response = get_pdm_document_reference(record_id) assert response.status_code == expected_status @@ -75,9 +68,7 @@ def test_forbidden_with_invalid_cert(test_data, temp_cert_and_key): # Use an invalid cert that is trusted by TLS but fails truststore validation cert_path, key_path = temp_cert_and_key - response = get_pdm_document_reference( - pdm_record["id"], client_cert_path=cert_path, client_key_path=key_path - ) + response = get_pdm_document_reference(pdm_record["id"], client_cert_path=cert_path, client_key_path=key_path) body = response.json() assert response.status_code == 403 diff --git a/lambdas/tests/e2e/api/fhir/test_search_patient_fhir_api.py b/lambdas/tests/e2e/api/fhir/test_search_patient_fhir_api.py index 3d24b0907..e15e9b75b 100644 --- a/lambdas/tests/e2e/api/fhir/test_search_patient_fhir_api.py +++ b/lambdas/tests/e2e/api/fhir/test_search_patient_fhir_api.py @@ -1,23 +1,20 @@ import pytest -from lambdas.tests.e2e.api.fhir.conftest import ( +from tests.e2e.api.fhir.conftest import ( MTLS_ENDPOINT, PDM_SNOMED, create_and_store_pdm_record, create_mtls_session, ) -from lambdas.tests.e2e.conftest import APIM_ENDPOINT -from lambdas.tests.e2e.helpers.data_helper import PdmDataHelper +from tests.e2e.conftest import APIM_ENDPOINT +from tests.e2e.helpers.data_helper import PdmDataHelper pdm_data_helper = PdmDataHelper() def search_document_reference(nhs_number, client_cert_path=None, client_key_path=None): """Helper to perform search by NHS number with optional mTLS certs.""" - url = ( - f"https://{MTLS_ENDPOINT}/DocumentReference?" - f"subject:identifier=https://fhir.nhs.uk/Id/nhs-number|{nhs_number}" - ) + url = f"https://{MTLS_ENDPOINT}/DocumentReference?subject:identifier=https://fhir.nhs.uk/Id/nhs-number|{nhs_number}" headers = { "X-Correlation-Id": "1234", } @@ -56,11 +53,7 @@ def test_search_patient_details(test_data): # Find the entry with the matching record_id matching_entry = next( - ( - e - for e in entries - if e["resource"].get("id") == f"{PDM_SNOMED}~{expected_record_id}" - ), + (e for e in entries if e["resource"].get("id") == f"{PDM_SNOMED}~{expected_record_id}"), None, ) assert matching_entry @@ -73,10 +66,7 @@ def test_search_patient_details(test_data): def test_multiple_cancelled_search_patient_details(test_data): - record_ids = [ - create_and_store_pdm_record(test_data, doc_status="cancelled")["id"] - for _ in range(2) - ] + record_ids = [create_and_store_pdm_record(test_data, doc_status="cancelled")["id"] for _ in range(2)] response = search_document_reference("9912003071") assert response.status_code == 200 @@ -88,11 +78,7 @@ def test_multiple_cancelled_search_patient_details(test_data): # Validate all created records exist and have status cancelled for record_id in record_ids: entry = next( - ( - e - for e in entries - if e["resource"].get("id") == f"{PDM_SNOMED}~{record_id}" - ), + (e for e in entries if e["resource"].get("id") == f"{PDM_SNOMED}~{record_id}"), None, ) assert entry @@ -106,9 +92,7 @@ def test_multiple_cancelled_search_patient_details(test_data): ("123", 400, "INVALID_SEARCH_DATA", "Invalid patient number 123"), ], ) -def test_search_edge_cases( - nhs_number, expected_status, expected_code, expected_diagnostics -): +def test_search_edge_cases(nhs_number, expected_status, expected_code, expected_diagnostics): response = search_document_reference(nhs_number) assert response.status_code == expected_status @@ -127,9 +111,7 @@ def test_search_patient_unauthorized_mtls(test_data, temp_cert_and_key): # Use an invalid cert that is trusted by TLS but fails truststore validation cert_path, key_path = temp_cert_and_key - response = search_document_reference( - "9912003071", client_cert_path=cert_path, client_key_path=key_path - ) + response = search_document_reference("9912003071", client_cert_path=cert_path, client_key_path=key_path) body = response.json() assert response.status_code == 403 diff --git a/lambdas/tests/e2e/api/fhir/test_upload_document_fhir_api_failure.py b/lambdas/tests/e2e/api/fhir/test_upload_document_fhir_api_failure.py index f5c360d27..5648943c5 100644 --- a/lambdas/tests/e2e/api/fhir/test_upload_document_fhir_api_failure.py +++ b/lambdas/tests/e2e/api/fhir/test_upload_document_fhir_api_failure.py @@ -6,12 +6,12 @@ import pytest import requests -from lambdas.tests.e2e.api.fhir.conftest import ( +from tests.e2e.api.fhir.conftest import ( MTLS_ENDPOINT, create_mtls_session, fetch_with_retry_mtls, ) -from lambdas.tests.e2e.helpers.data_helper import PdmDataHelper +from tests.e2e.helpers.data_helper import PdmDataHelper pdm_data_helper = PdmDataHelper() @@ -59,9 +59,7 @@ def test_create_document_virus(test_data): } # Attach EICAR data - eicar_string = ( - r"X5O!P%@AP[4\PZX54(P^)7CC)7}$EICAR-STANDARD-ANTIVIRUS-TEST-FILE!$H+H*" - ) + eicar_string = r"X5O!P%@AP[4\PZX54(P^)7CC)7}$EICAR-STANDARD-ANTIVIRUS-TEST-FILE!$H+H*" record["data"] = base64.b64encode(eicar_string.encode()).decode() payload = pdm_data_helper.create_upload_payload(record) @@ -79,9 +77,7 @@ def condition(response_json): "final", ) - raw_retrieve_response = retrieve_document_with_retry( - upload_response["id"], condition - ) + raw_retrieve_response = retrieve_document_with_retry(upload_response["id"], condition) retrieve_response = raw_retrieve_response.json() assert retrieve_response["docStatus"] == "cancelled" @@ -104,9 +100,7 @@ def condition(response_json): ), ], ) -def test_search_edge_cases( - nhs_number, expected_status, expected_code, expected_diagnostics -): +def test_search_edge_cases(nhs_number, expected_status, expected_code, expected_diagnostics): record = { "ods": "H81109", "nhs_number": f"{nhs_number}", @@ -143,9 +137,7 @@ def test_forbidden_with_invalid_cert(temp_cert_and_key): url = f"https://{MTLS_ENDPOINT}/DocumentReference" headers = {"Authorization": "Bearer 123", "X-Correlation-Id": "1234"} - response = requests.post( - url, headers=headers, cert=(cert_path, key_path), data=payload - ) + response = requests.post(url, headers=headers, cert=(cert_path, key_path), data=payload) body = response.json() assert response.status_code == 403 assert body["message"] == "Forbidden" @@ -172,6 +164,4 @@ def test_create_document_with_invalid_author_returns_error(test_data, author_pay response_json = raw_upload_response.json() assert raw_upload_response.status_code == 400 assert response_json["resourceType"] == "OperationOutcome" - assert ( - response_json["issue"][0]["details"]["coding"][0]["code"] == "VALIDATION_ERROR" - ) + assert response_json["issue"][0]["details"]["coding"][0]["code"] == "VALIDATION_ERROR" diff --git a/lambdas/tests/e2e/api/fhir/test_upload_document_fhir_api_success.py b/lambdas/tests/e2e/api/fhir/test_upload_document_fhir_api_success.py index 47ac4320d..deffa771e 100644 --- a/lambdas/tests/e2e/api/fhir/test_upload_document_fhir_api_success.py +++ b/lambdas/tests/e2e/api/fhir/test_upload_document_fhir_api_success.py @@ -2,14 +2,14 @@ import logging import os -from lambdas.tests.e2e.api.fhir.conftest import ( +from tests.e2e.api.fhir.conftest import ( MTLS_ENDPOINT, PDM_SNOMED, create_mtls_session, fetch_with_retry_mtls, ) -from lambdas.tests.e2e.conftest import APIM_ENDPOINT -from lambdas.tests.e2e.helpers.data_helper import PdmDataHelper +from tests.e2e.conftest import APIM_ENDPOINT +from tests.e2e.helpers.data_helper import PdmDataHelper pdm_data_helper = PdmDataHelper() @@ -53,18 +53,13 @@ def test_create_document_base64(test_data): # Validate attachment URL upload_response = raw_upload_response.json() attachment_url = upload_response["content"][0]["attachment"]["url"] - assert ( - f"https://{APIM_ENDPOINT}/national-document-repository/FHIR/R4/DocumentReference/{PDM_SNOMED}~" - in attachment_url - ) + assert f"https://{APIM_ENDPOINT}/national-document-repository/FHIR/R4/DocumentReference/{PDM_SNOMED}~" in attachment_url def condition(response_json): logging.info(response_json) return response_json["content"][0]["attachment"].get("data", False) - raw_retrieve_response = retrieve_document_with_retry( - upload_response["id"], condition - ) + raw_retrieve_response = retrieve_document_with_retry(upload_response["id"], condition) retrieve_response = raw_retrieve_response.json() # Validate base64 data @@ -105,9 +100,7 @@ def test_create_document_without_author_or_type(test_data): sample_pdf_path = os.path.join(os.path.dirname(__file__), "files", "dummy.pdf") with open(sample_pdf_path, "rb") as f: record["data"] = base64.b64encode(f.read()).decode("utf-8") - payload = pdm_data_helper.create_upload_payload( - record=record, exclude=["type", "author"] - ) + payload = pdm_data_helper.create_upload_payload(record=record, exclude=["type", "author"]) for field in ["type", "author"]: assert field not in payload diff --git a/lambdas/tests/e2e/api/test_login_api.py b/lambdas/tests/e2e/api/test_login_api.py index 107dbc082..6673fbf60 100644 --- a/lambdas/tests/e2e/api/test_login_api.py +++ b/lambdas/tests/e2e/api/test_login_api.py @@ -1,15 +1,14 @@ import pytest + from tests.e2e.helpers.data_helper import LloydGeorgeDataHelper -from tests.e2e.helpers.lloyd_george_mockcis2_helper import LloydGeorgeMockcis2Helper +from tests.e2e.helpers.mockcis2_helper import MockCis2Helper # Note this is testing a mock, but this test is valuable to ensure the login code is working for other tests data_helper = LloydGeorgeDataHelper() -@pytest.mark.skipif( - data_helper.workspace == "pre-prod", reason="CIS2 login is not mocked in pre-prod" -) +@pytest.mark.skipif(data_helper.workspace == "pre-prod", reason="CIS2 login is not mocked in pre-prod") @pytest.mark.parametrize( "ods_code, role_to_assume, expected_granted_role", [ @@ -19,9 +18,7 @@ ], ) def test_login(ods_code, role_to_assume, expected_granted_role): - login_helper = LloydGeorgeMockcis2Helper( - ods=ods_code, repository_role=role_to_assume - ) + login_helper = MockCis2Helper(ods=ods_code, repository_role=role_to_assume) login_helper.generate_mockcis2_token() assert login_helper.user_role == expected_granted_role assert login_helper.user_token is not None diff --git a/lambdas/tests/e2e/api/test_retrieve_document_api.py b/lambdas/tests/e2e/api/test_retrieve_document_api.py index 4f464c7b5..047768441 100644 --- a/lambdas/tests/e2e/api/test_retrieve_document_api.py +++ b/lambdas/tests/e2e/api/test_retrieve_document_api.py @@ -3,6 +3,7 @@ import requests from syrupy.filters import paths + from tests.e2e.conftest import ( API_ENDPOINT, API_KEY, @@ -62,9 +63,7 @@ def test_large_file(test_data, snapshot_json): expected_presign_uri = f"https://{LLOYD_GEORGE_S3_BUCKET}.s3.eu-west-2.amazonaws.com/{lloyd_george_record['nhs_number']}/{lloyd_george_record['id']}" assert expected_presign_uri in json["content"][0]["attachment"]["url"] - assert json == snapshot_json( - exclude=paths("date", "id", "content.0.attachment.url") - ) + assert json == snapshot_json(exclude=paths("date", "id", "content.0.attachment.url")) def test_no_file_found(snapshot_json): diff --git a/lambdas/tests/e2e/api/test_search_patient_api.py b/lambdas/tests/e2e/api/test_search_patient_api.py index ad7c7c79f..fb564d690 100644 --- a/lambdas/tests/e2e/api/test_search_patient_api.py +++ b/lambdas/tests/e2e/api/test_search_patient_api.py @@ -4,6 +4,7 @@ import requests from syrupy.filters import paths + from tests.e2e.conftest import API_ENDPOINT, API_KEY, APIM_ENDPOINT, LLOYD_GEORGE_SNOMED from tests.e2e.helpers.data_helper import LloydGeorgeDataHelper @@ -33,8 +34,7 @@ def test_search_patient_details(test_data, snapshot_json): attachment_url = bundle["entry"][0]["resource"]["content"][0]["attachment"]["url"] assert ( - f"https://{APIM_ENDPOINT}/national-document-repository/FHIR/R4/DocumentReference/{LLOYD_GEORGE_SNOMED}~" - in attachment_url + f"https://{APIM_ENDPOINT}/national-document-repository/FHIR/R4/DocumentReference/{LLOYD_GEORGE_SNOMED}~" in attachment_url ) assert bundle == snapshot_json( @@ -80,15 +80,9 @@ def test_multiple_cancelled_search_patient_details(test_data, snapshot_json): bundle = response.json() assert bundle["entry"][0] == snapshot_json( - exclude=paths( - "resource.id", "resource.date", "resource.content.0.attachment.url" - ) - ) + exclude=paths("resource.id", "resource.date", "resource.content.0.attachment.url")) assert bundle["entry"][1] == snapshot_json( - exclude=paths( - "resource.id", "resource.date", "resource.content.0.attachment.url" - ) - ) + exclude=paths("resource.id", "resource.date", "resource.content.0.attachment.url")) def test_no_records(snapshot_json): diff --git a/lambdas/tests/e2e/api/test_upload_document_api.py b/lambdas/tests/e2e/api/test_upload_document_api.py index cc9319bc2..4b929753e 100644 --- a/lambdas/tests/e2e/api/test_upload_document_api.py +++ b/lambdas/tests/e2e/api/test_upload_document_api.py @@ -5,6 +5,7 @@ import requests from syrupy.filters import paths + from tests.e2e.conftest import ( API_ENDPOINT, API_KEY, @@ -84,9 +85,7 @@ def test_create_document_base64(test_data, snapshot_json): lloyd_george_record["id"] = upload_response["id"].split("~")[1] test_data.append(lloyd_george_record) - retrieve_url = ( - f"https://{API_ENDPOINT}/FhirDocumentReference/{upload_response['id']}" - ) + retrieve_url = f"https://{API_ENDPOINT}/FhirDocumentReference/{upload_response['id']}" def condition(response_json): logging.info(response_json) @@ -97,19 +96,14 @@ def condition(response_json): attachment_url = upload_response["content"][0]["attachment"]["url"] assert ( - f"https://{APIM_ENDPOINT}/national-document-repository/FHIR/R4/DocumentReference/{LLOYD_GEORGE_SNOMED}~" - in attachment_url + f"https://{APIM_ENDPOINT}/national-document-repository/FHIR/R4/DocumentReference/{LLOYD_GEORGE_SNOMED}~" in attachment_url ) base64_data = retrieve_response["content"][0]["attachment"]["data"] assert base64.b64decode(base64_data, validate=True) - assert upload_response == snapshot_json( - exclude=paths("id", "date", "content.0.attachment.url") - ) - assert retrieve_response == snapshot_json( - exclude=paths("id", "date", "content.0.attachment.data") - ) + assert upload_response == snapshot_json(exclude=paths("id", "date", "content.0.attachment.url")) + assert retrieve_response == snapshot_json(exclude=paths("id", "date", "content.0.attachment.data")) def test_create_document_presign(test_data, snapshot_json): @@ -134,9 +128,7 @@ def test_create_document_presign(test_data, snapshot_json): presign_response = requests.put(presign_uri, files=files) assert presign_response.status_code == 200 - retrieve_url = ( - f"https://{API_ENDPOINT}/FhirDocumentReference/{upload_response['id']}" - ) + retrieve_url = f"https://{API_ENDPOINT}/FhirDocumentReference/{upload_response['id']}" def condition(response_json): logging.info(response_json) @@ -152,9 +144,7 @@ def condition(response_json): assert upload_response == snapshot_json(exclude=paths("id", "date")) assert retrieve_response == snapshot_json( - exclude=paths( - "id", "date", "content.0.attachment.url", "content.0.attachment.size" - ) + exclude=paths("id", "date", "content.0.attachment.url", "content.0.attachment.size") ) @@ -165,9 +155,7 @@ def test_create_document_virus(test_data, snapshot_json): lloyd_george_record["nhs_number"] = "9730154260" # Attach EICAR data - eicar_string = ( - r"X5O!P%@AP[4\PZX54(P^)7CC)7}$EICAR-STANDARD-ANTIVIRUS-TEST-FILE!$H+H*" - ) + eicar_string = r"X5O!P%@AP[4\PZX54(P^)7CC)7}$EICAR-STANDARD-ANTIVIRUS-TEST-FILE!$H+H*" lloyd_george_record["data"] = base64.b64encode(eicar_string.encode()).decode() payload = create_upload_payload(lloyd_george_record) @@ -179,9 +167,7 @@ def test_create_document_virus(test_data, snapshot_json): lloyd_george_record["id"] = upload_response["id"].split("~")[1] test_data.append(lloyd_george_record) - retrieve_url = ( - f"https://{API_ENDPOINT}/FhirDocumentReference/{upload_response['id']}" - ) + retrieve_url = f"https://{API_ENDPOINT}/FhirDocumentReference/{upload_response['id']}" # Poll until processing/scan completes def condition(response_json): @@ -194,9 +180,7 @@ def condition(response_json): raw_retrieve_response = fetch_with_retry(retrieve_url, condition) retrieve_response = raw_retrieve_response.json() - assert upload_response == snapshot_json( - exclude=paths("id", "date", "content.0.attachment.url") - ) + assert upload_response == snapshot_json(exclude=paths("id", "date", "content.0.attachment.url")) assert retrieve_response == snapshot_json(exclude=paths("id", "date")) diff --git a/lambdas/tests/e2e/apim/test_apim_client.py b/lambdas/tests/e2e/apim/test_apim_client.py index 5a7c78443..698e3ee3c 100644 --- a/lambdas/tests/e2e/apim/test_apim_client.py +++ b/lambdas/tests/e2e/apim/test_apim_client.py @@ -4,6 +4,7 @@ import pytest import requests + from tests.e2e.helpers.data_helper import PdmDataHelper pdm_data_helper = PdmDataHelper() @@ -16,9 +17,7 @@ def test_ping(nhsd_apim_proxy_url): @pytest.mark.nhsd_apim_authorization(access="application", level="level3") -def test_app_level3_access_search( - test_data, nhsd_apim_proxy_url, nhsd_apim_auth_headers -): +def test_app_level3_access_search(test_data, nhsd_apim_proxy_url, nhsd_apim_auth_headers): # Put PDM record pdm_record = pdm_data_helper.build_record() test_data.append(pdm_record) @@ -28,8 +27,7 @@ def test_app_level3_access_search( # Search record via APIM resp = requests.get( - nhsd_apim_proxy_url - + "/DocumentReference?subject:identifier=https://fhir.nhs.uk/Id/nhs-number%7C9912003071", + nhsd_apim_proxy_url + "/DocumentReference?subject:identifier=https://fhir.nhs.uk/Id/nhs-number%7C9912003071", headers=nhsd_apim_auth_headers, ) assert resp.status_code == 200 @@ -49,9 +47,7 @@ def test_app_level3_access_search( @pytest.mark.nhsd_apim_authorization(access="application", level="level3") -def test_app_level3_access_retrieve( - test_data, nhsd_apim_proxy_url, nhsd_apim_auth_headers -): +def test_app_level3_access_retrieve(test_data, nhsd_apim_proxy_url, nhsd_apim_auth_headers): # Put PDM record pdm_record = pdm_data_helper.build_record() test_data.append(pdm_record) @@ -77,9 +73,7 @@ def test_app_level3_access_retrieve( @pytest.mark.nhsd_apim_authorization(access="application", level="level3") -def test_app_level3_access_upload( - test_data, nhsd_apim_proxy_url, nhsd_apim_auth_headers -): +def test_app_level3_access_upload(test_data, nhsd_apim_proxy_url, nhsd_apim_auth_headers): # Build PDM record sample_pdf_path = os.path.join(os.path.dirname(__file__), "files", "dummy.pdf") with open(sample_pdf_path, "rb") as f: diff --git a/lambdas/tests/e2e/bulk_upload/check/__snapshots__/test_accept_bulk_upload/test_general_accepted_ingestions[General-Accepted-Multi File Upload 1of3-9730786976].json b/lambdas/tests/e2e/bulk_upload/check/__snapshots__/test_accept_bulk_upload/test_general_accepted_ingestions[General-Accepted-Multi File Upload 1of3-9730786976].json new file mode 100644 index 000000000..653f6e227 --- /dev/null +++ b/lambdas/tests/e2e/bulk_upload/check/__snapshots__/test_accept_bulk_upload/test_general_accepted_ingestions[General-Accepted-Multi File Upload 1of3-9730786976].json @@ -0,0 +1,69 @@ +{ + "_description": "General-Accepted-Multi File Upload 1of3", + "bulk_upload_report": { + "NhsNumber": "9730786976", + "PdsOdsCode": "M85143", + "UploadStatus": "complete", + "UploaderOdsCode": "M85143" + }, + "metadata": { + "Author": "M85143", + "ContentType": "application/pdf", + "CurrentGpOds": "M85143", + "Custodian": "M85143", + "DocStatus": "final", + "DocumentScanCreation": "2023-01-01", + "DocumentSnomedCodeType": "16521000000101", + "FileName": "1of1_Lloyd_George_Record_[Lena Rena LAWLER]_[9730786976]_[24-07-1924].pdf", + "FileSize": "Decimal('578652')", + "NhsNumber": "9730786976", + "Status": "current", + "Uploaded": true, + "Version": "1", + "VirusScannerResult": "Clean" + }, + "unstitched": [ + { + "Author": "M85143", + "ContentType": "application/pdf", + "Custodian": "M85143", + "DocStatus": "final", + "DocumentScanCreation": "2023-01-01", + "DocumentSnomedCodeType": "16521000000101", + "FileSize": "Decimal('173760')", + "NhsNumber": "9730786976", + "Status": "current", + "Uploaded": true, + "Version": "1", + "VirusScannerResult": "Clean" + }, + { + "Author": "M85143", + "ContentType": "application/pdf", + "Custodian": "M85143", + "DocStatus": "final", + "DocumentScanCreation": "2023-01-01", + "DocumentSnomedCodeType": "16521000000101", + "FileSize": "Decimal('173760')", + "NhsNumber": "9730786976", + "Status": "current", + "Uploaded": true, + "Version": "1", + "VirusScannerResult": "Clean" + }, + { + "Author": "M85143", + "ContentType": "application/pdf", + "Custodian": "M85143", + "DocStatus": "final", + "DocumentScanCreation": "2023-01-01", + "DocumentSnomedCodeType": "16521000000101", + "FileSize": "Decimal('173760')", + "NhsNumber": "9730786976", + "Status": "current", + "Uploaded": true, + "Version": "1", + "VirusScannerResult": "Clean" + } + ] +} diff --git a/lambdas/tests/e2e/bulk_upload/check/__snapshots__/test_accept_bulk_upload/test_general_accepted_ingestions[General-Accepted-Multi File Upload 2of3-9730786976].json b/lambdas/tests/e2e/bulk_upload/check/__snapshots__/test_accept_bulk_upload/test_general_accepted_ingestions[General-Accepted-Multi File Upload 2of3-9730786976].json new file mode 100644 index 000000000..6ae839f66 --- /dev/null +++ b/lambdas/tests/e2e/bulk_upload/check/__snapshots__/test_accept_bulk_upload/test_general_accepted_ingestions[General-Accepted-Multi File Upload 2of3-9730786976].json @@ -0,0 +1,69 @@ +{ + "_description": "General-Accepted-Multi File Upload 2of3", + "bulk_upload_report": { + "NhsNumber": "9730786976", + "PdsOdsCode": "M85143", + "UploadStatus": "complete", + "UploaderOdsCode": "M85143" + }, + "metadata": { + "Author": "M85143", + "ContentType": "application/pdf", + "CurrentGpOds": "M85143", + "Custodian": "M85143", + "DocStatus": "final", + "DocumentScanCreation": "2023-01-01", + "DocumentSnomedCodeType": "16521000000101", + "FileName": "1of1_Lloyd_George_Record_[Lena Rena LAWLER]_[9730786976]_[24-07-1924].pdf", + "FileSize": "Decimal('578652')", + "NhsNumber": "9730786976", + "Status": "current", + "Uploaded": true, + "Version": "1", + "VirusScannerResult": "Clean" + }, + "unstitched": [ + { + "Author": "M85143", + "ContentType": "application/pdf", + "Custodian": "M85143", + "DocStatus": "final", + "DocumentScanCreation": "2023-01-01", + "DocumentSnomedCodeType": "16521000000101", + "FileSize": "Decimal('173760')", + "NhsNumber": "9730786976", + "Status": "current", + "Uploaded": true, + "Version": "1", + "VirusScannerResult": "Clean" + }, + { + "Author": "M85143", + "ContentType": "application/pdf", + "Custodian": "M85143", + "DocStatus": "final", + "DocumentScanCreation": "2023-01-01", + "DocumentSnomedCodeType": "16521000000101", + "FileSize": "Decimal('173760')", + "NhsNumber": "9730786976", + "Status": "current", + "Uploaded": true, + "Version": "1", + "VirusScannerResult": "Clean" + }, + { + "Author": "M85143", + "ContentType": "application/pdf", + "Custodian": "M85143", + "DocStatus": "final", + "DocumentScanCreation": "2023-01-01", + "DocumentSnomedCodeType": "16521000000101", + "FileSize": "Decimal('173760')", + "NhsNumber": "9730786976", + "Status": "current", + "Uploaded": true, + "Version": "1", + "VirusScannerResult": "Clean" + } + ] +} diff --git a/lambdas/tests/e2e/bulk_upload/check/__snapshots__/test_accept_bulk_upload/test_general_accepted_ingestions[General-Accepted-Multi File Upload 3of3-9730786976].json b/lambdas/tests/e2e/bulk_upload/check/__snapshots__/test_accept_bulk_upload/test_general_accepted_ingestions[General-Accepted-Multi File Upload 3of3-9730786976].json new file mode 100644 index 000000000..a5afd9d9c --- /dev/null +++ b/lambdas/tests/e2e/bulk_upload/check/__snapshots__/test_accept_bulk_upload/test_general_accepted_ingestions[General-Accepted-Multi File Upload 3of3-9730786976].json @@ -0,0 +1,69 @@ +{ + "_description": "General-Accepted-Multi File Upload 3of3", + "bulk_upload_report": { + "NhsNumber": "9730786976", + "PdsOdsCode": "M85143", + "UploadStatus": "complete", + "UploaderOdsCode": "M85143" + }, + "metadata": { + "Author": "M85143", + "ContentType": "application/pdf", + "CurrentGpOds": "M85143", + "Custodian": "M85143", + "DocStatus": "final", + "DocumentScanCreation": "2023-01-01", + "DocumentSnomedCodeType": "16521000000101", + "FileName": "1of1_Lloyd_George_Record_[Lena Rena LAWLER]_[9730786976]_[24-07-1924].pdf", + "FileSize": "Decimal('578652')", + "NhsNumber": "9730786976", + "Status": "current", + "Uploaded": true, + "Version": "1", + "VirusScannerResult": "Clean" + }, + "unstitched": [ + { + "Author": "M85143", + "ContentType": "application/pdf", + "Custodian": "M85143", + "DocStatus": "final", + "DocumentScanCreation": "2023-01-01", + "DocumentSnomedCodeType": "16521000000101", + "FileSize": "Decimal('173760')", + "NhsNumber": "9730786976", + "Status": "current", + "Uploaded": true, + "Version": "1", + "VirusScannerResult": "Clean" + }, + { + "Author": "M85143", + "ContentType": "application/pdf", + "Custodian": "M85143", + "DocStatus": "final", + "DocumentScanCreation": "2023-01-01", + "DocumentSnomedCodeType": "16521000000101", + "FileSize": "Decimal('173760')", + "NhsNumber": "9730786976", + "Status": "current", + "Uploaded": true, + "Version": "1", + "VirusScannerResult": "Clean" + }, + { + "Author": "M85143", + "ContentType": "application/pdf", + "Custodian": "M85143", + "DocStatus": "final", + "DocumentScanCreation": "2023-01-01", + "DocumentSnomedCodeType": "16521000000101", + "FileSize": "Decimal('173760')", + "NhsNumber": "9730786976", + "Status": "current", + "Uploaded": true, + "Version": "1", + "VirusScannerResult": "Clean" + } + ] +} diff --git a/lambdas/tests/e2e/bulk_upload/check/__snapshots__/test_accept_bulk_upload/test_general_accepted_ingestions[General-Accepted-Pre-processor Hyphenated Nhs Number-9730154406].json b/lambdas/tests/e2e/bulk_upload/check/__snapshots__/test_accept_bulk_upload/test_general_accepted_ingestions[General-Accepted-Pre-processor Hyphenated Nhs Number-9730154406].json new file mode 100644 index 000000000..e36d2dc55 --- /dev/null +++ b/lambdas/tests/e2e/bulk_upload/check/__snapshots__/test_accept_bulk_upload/test_general_accepted_ingestions[General-Accepted-Pre-processor Hyphenated Nhs Number-9730154406].json @@ -0,0 +1,26 @@ +{ + "_description": "General-Accepted-Pre-processor Hyphenated Nhs Number", + "bulk_upload_report": { + "NhsNumber": "9730154406", + "PdsOdsCode": "H81109", + "UploadStatus": "complete", + "UploaderOdsCode": "H81109" + }, + "metadata": { + "Author": "H81109", + "ContentType": "application/pdf", + "CurrentGpOds": "H81109", + "Custodian": "H81109", + "DocStatus": "final", + "DocumentScanCreation": "2023-01-01", + "DocumentSnomedCodeType": "16521000000101", + "FileName": "1of1_Lloyd_George_Record_[MEIXIANG XIAOLIAN FONG]_[9730154406]_[27-11-1944].pdf", + "FileSize": "Decimal('173760')", + "NhsNumber": "9730154406", + "Status": "current", + "Uploaded": true, + "Version": "1", + "VirusScannerResult": "Clean" + }, + "unstitched": [] +} diff --git a/lambdas/tests/e2e/bulk_upload/check/__snapshots__/test_accept_bulk_upload/test_general_accepted_ingestions[General-Accepted-Pre-processor LloydGeorge Name Edit-9730786925].json b/lambdas/tests/e2e/bulk_upload/check/__snapshots__/test_accept_bulk_upload/test_general_accepted_ingestions[General-Accepted-Pre-processor LloydGeorge Name Edit-9730786925].json new file mode 100644 index 000000000..61e2feb0a --- /dev/null +++ b/lambdas/tests/e2e/bulk_upload/check/__snapshots__/test_accept_bulk_upload/test_general_accepted_ingestions[General-Accepted-Pre-processor LloydGeorge Name Edit-9730786925].json @@ -0,0 +1,26 @@ +{ + "_description": "General-Accepted-Pre-processor LloydGeorge Name Edit", + "bulk_upload_report": { + "NhsNumber": "9730786925", + "PdsOdsCode": "M85143", + "UploadStatus": "complete", + "UploaderOdsCode": "M85143" + }, + "metadata": { + "Author": "M85143", + "ContentType": "application/pdf", + "CurrentGpOds": "M85143", + "Custodian": "M85143", + "DocStatus": "final", + "DocumentScanCreation": "2023-01-01", + "DocumentSnomedCodeType": "16521000000101", + "FileName": "1of1_Lloyd_George_Record_[Harry Adam TWIST]_[9730786925]_[01-04-1925].pdf", + "FileSize": "Decimal('173760')", + "NhsNumber": "9730786925", + "Status": "current", + "Uploaded": true, + "Version": "1", + "VirusScannerResult": "Clean" + }, + "unstitched": [] +} diff --git a/lambdas/tests/e2e/bulk_upload/check/__snapshots__/test_accept_bulk_upload/test_general_accepted_ingestions[General-Accepted-Pre-processor Mixed Spacing in Page-9730154775].json b/lambdas/tests/e2e/bulk_upload/check/__snapshots__/test_accept_bulk_upload/test_general_accepted_ingestions[General-Accepted-Pre-processor Mixed Spacing in Page-9730154775].json new file mode 100644 index 000000000..addcf354d --- /dev/null +++ b/lambdas/tests/e2e/bulk_upload/check/__snapshots__/test_accept_bulk_upload/test_general_accepted_ingestions[General-Accepted-Pre-processor Mixed Spacing in Page-9730154775].json @@ -0,0 +1,26 @@ +{ + "_description": "General-Accepted-Pre-processor Mixed Spacing in Page", + "bulk_upload_report": { + "NhsNumber": "9730154775", + "PdsOdsCode": "H81109", + "UploadStatus": "complete", + "UploaderOdsCode": "H81109" + }, + "metadata": { + "Author": "H81109", + "ContentType": "application/pdf", + "CurrentGpOds": "H81109", + "Custodian": "H81109", + "DocStatus": "final", + "DocumentScanCreation": "2023-01-01", + "DocumentSnomedCodeType": "16521000000101", + "FileName": "1of1_Lloyd_George_Record_[Andrea Gladys HOAKES]_[9730154775]_[09-01-2009].pdf", + "FileSize": "Decimal('173760')", + "NhsNumber": "9730154775", + "Status": "current", + "Uploaded": true, + "Version": "1", + "VirusScannerResult": "Clean" + }, + "unstitched": [] +} diff --git a/lambdas/tests/e2e/bulk_upload/check/__snapshots__/test_accept_bulk_upload/test_general_accepted_ingestions[General-Accepted-Pre-processor Mixed brackets-9730154090].json b/lambdas/tests/e2e/bulk_upload/check/__snapshots__/test_accept_bulk_upload/test_general_accepted_ingestions[General-Accepted-Pre-processor Mixed brackets-9730154090].json new file mode 100644 index 000000000..89892fb46 --- /dev/null +++ b/lambdas/tests/e2e/bulk_upload/check/__snapshots__/test_accept_bulk_upload/test_general_accepted_ingestions[General-Accepted-Pre-processor Mixed brackets-9730154090].json @@ -0,0 +1,26 @@ +{ + "_description": "General-Accepted-Pre-processor Mixed brackets", + "bulk_upload_report": { + "NhsNumber": "9730154090", + "PdsOdsCode": "H81109", + "UploadStatus": "complete", + "UploaderOdsCode": "H81109" + }, + "metadata": { + "Author": "H81109", + "ContentType": "application/pdf", + "CurrentGpOds": "H81109", + "Custodian": "H81109", + "DocStatus": "final", + "DocumentScanCreation": "2023-01-01", + "DocumentSnomedCodeType": "16521000000101", + "FileName": "1of1_Lloyd_George_Record_[Tarina Sybill LEVY]_[9730154090]_[14-09-1961].pdf", + "FileSize": "Decimal('173760')", + "NhsNumber": "9730154090", + "Status": "current", + "Uploaded": true, + "Version": "1", + "VirusScannerResult": "Clean" + }, + "unstitched": [] +} diff --git a/lambdas/tests/e2e/bulk_upload/check/__snapshots__/test_accept_bulk_upload/test_general_accepted_ingestions[General-Accepted-Pre-processor Spacing Nhs Number-9730154724].json b/lambdas/tests/e2e/bulk_upload/check/__snapshots__/test_accept_bulk_upload/test_general_accepted_ingestions[General-Accepted-Pre-processor Spacing Nhs Number-9730154724].json new file mode 100644 index 000000000..2f4b0b95d --- /dev/null +++ b/lambdas/tests/e2e/bulk_upload/check/__snapshots__/test_accept_bulk_upload/test_general_accepted_ingestions[General-Accepted-Pre-processor Spacing Nhs Number-9730154724].json @@ -0,0 +1,26 @@ +{ + "_description": "General-Accepted-Pre-processor Spacing Nhs Number", + "bulk_upload_report": { + "NhsNumber": "9730154724", + "PdsOdsCode": "H81109", + "UploadStatus": "complete", + "UploaderOdsCode": "H81109" + }, + "metadata": { + "Author": "H81109", + "ContentType": "application/pdf", + "CurrentGpOds": "H81109", + "Custodian": "H81109", + "DocStatus": "final", + "DocumentScanCreation": "2023-01-01", + "DocumentSnomedCodeType": "16521000000101", + "FileName": "1of1_Lloyd_George_Record_[GILALA NYSSA RUBENSTEIN]_[9730154724]_[18-07-2020].pdf", + "FileSize": "Decimal('173760')", + "NhsNumber": "9730154724", + "Status": "current", + "Uploaded": true, + "Version": "1", + "VirusScannerResult": "Clean" + }, + "unstitched": [] +} diff --git a/lambdas/tests/e2e/bulk_upload/check/__snapshots__/test_accept_bulk_upload/test_general_accepted_ingestions[General-Accepted-Single File Upload-9730787026].json b/lambdas/tests/e2e/bulk_upload/check/__snapshots__/test_accept_bulk_upload/test_general_accepted_ingestions[General-Accepted-Single File Upload-9730787026].json new file mode 100644 index 000000000..1d3507935 --- /dev/null +++ b/lambdas/tests/e2e/bulk_upload/check/__snapshots__/test_accept_bulk_upload/test_general_accepted_ingestions[General-Accepted-Single File Upload-9730787026].json @@ -0,0 +1,26 @@ +{ + "_description": "General-Accepted-Single File Upload", + "bulk_upload_report": { + "NhsNumber": "9730787026", + "PdsOdsCode": "M85143", + "UploadStatus": "complete", + "UploaderOdsCode": "M85143" + }, + "metadata": { + "Author": "M85143", + "ContentType": "application/pdf", + "CurrentGpOds": "M85143", + "Custodian": "M85143", + "DocStatus": "final", + "DocumentScanCreation": "2023-01-01", + "DocumentSnomedCodeType": "16521000000101", + "FileName": "1of1_Lloyd_George_Record_[Dora Celia HAMNET]_[9730787026]_[07-07-1926].pdf", + "FileSize": "Decimal('173760')", + "NhsNumber": "9730787026", + "Status": "current", + "Uploaded": true, + "Version": "1", + "VirusScannerResult": "Clean" + }, + "unstitched": [] +} diff --git a/lambdas/tests/e2e/bulk_upload/check/__snapshots__/test_accept_bulk_upload/test_remap_and_fixed[9730787492-Remap and Fixed].json b/lambdas/tests/e2e/bulk_upload/check/__snapshots__/test_accept_bulk_upload/test_remap_and_fixed[9730787492-Remap and Fixed].json new file mode 100644 index 000000000..51f9e2403 --- /dev/null +++ b/lambdas/tests/e2e/bulk_upload/check/__snapshots__/test_accept_bulk_upload/test_remap_and_fixed[9730787492-Remap and Fixed].json @@ -0,0 +1,28 @@ +{ + "_description": "Remap and Fixed", + "bulk_upload_report": { + "FilePath": "accept-remap-and-fixed/test/9730787492/1of1_Lloyd_George_Record_[Bob SPEAR]_[9730787492]_[05-08-1968].pdf", + "NhsNumber": "9730787492", + "PdsOdsCode": "M85143", + "StoredFileName": "test/9730787492/1of1_Lloyd_George_Record_[Bob SPEAR]_[9730787492]_[05-08-1968].pdf", + "UploadStatus": "complete", + "UploaderOdsCode": "M85143" + }, + "metadata": { + "Author": "M85143", + "ContentType": "application/pdf", + "CurrentGpOds": "M85143", + "Custodian": "M85143", + "DocStatus": "final", + "DocumentScanCreation": "2022-01-01", + "DocumentSnomedCodeType": "16521000000101", + "FileName": "1of1_Lloyd_George_Record_[Bob SPEAR]_[9730787492]_[05-08-1968].pdf", + "FileSize": "Decimal('173760')", + "NhsNumber": "9730787492", + "Status": "current", + "Uploaded": true, + "Version": "1", + "VirusScannerResult": "Clean" + }, + "unstitched": [] +} diff --git a/lambdas/tests/e2e/bulk_upload/check/__snapshots__/test_accept_bulk_upload/test_usb_accepted_ingestions[USB-Accepted-1of1 in filename-9730153817].json b/lambdas/tests/e2e/bulk_upload/check/__snapshots__/test_accept_bulk_upload/test_usb_accepted_ingestions[USB-Accepted-1of1 in filename-9730153817].json new file mode 100644 index 000000000..d2f1386d7 --- /dev/null +++ b/lambdas/tests/e2e/bulk_upload/check/__snapshots__/test_accept_bulk_upload/test_usb_accepted_ingestions[USB-Accepted-1of1 in filename-9730153817].json @@ -0,0 +1,28 @@ +{ + "_description": "USB-Accepted-1of1 in filename", + "bulk_upload_report": { + "FilePath": "accept-usb/9730153817 Haley Glenda RUDKIN 07-Aug-2023/fgdgdf2f4e4dtd_B_1_Lloyd_George_Record_NOS_1of1_9730153817.pdf", + "NhsNumber": "9730153817", + "PdsOdsCode": "H81109", + "StoredFileName": "9730153817 Haley Glenda RUDKIN 07-Aug-2023/1of1_Lloyd_George_Record_[Haley Glenda RUDKIN]_[9730153817]_[07-08-2023].pdf", + "UploadStatus": "complete", + "UploaderOdsCode": "M85143" + }, + "metadata": { + "Author": "M85143", + "ContentType": "application/pdf", + "CurrentGpOds": "H81109", + "Custodian": "H81109", + "DocStatus": "final", + "DocumentScanCreation": "2023-01-01", + "DocumentSnomedCodeType": "16521000000101", + "FileName": "1of1_Lloyd_George_Record_[Haley Glenda RUDKIN]_[9730153817]_[07-08-2023].pdf", + "FileSize": "Decimal('173760')", + "NhsNumber": "9730153817", + "Status": "current", + "Uploaded": true, + "Version": "1", + "VirusScannerResult": "Clean" + }, + "unstitched": [] +} diff --git a/lambdas/tests/e2e/bulk_upload/check/__snapshots__/test_accept_bulk_upload/test_usb_accepted_ingestions[USB-Accepted-sent straight to queue (skips validation if already valid)-9730153760].json b/lambdas/tests/e2e/bulk_upload/check/__snapshots__/test_accept_bulk_upload/test_usb_accepted_ingestions[USB-Accepted-sent straight to queue (skips validation if already valid)-9730153760].json new file mode 100644 index 000000000..b17550672 --- /dev/null +++ b/lambdas/tests/e2e/bulk_upload/check/__snapshots__/test_accept_bulk_upload/test_usb_accepted_ingestions[USB-Accepted-sent straight to queue (skips validation if already valid)-9730153760].json @@ -0,0 +1,28 @@ +{ + "_description": "USB-Accepted-sent straight to queue (skips validation if already valid)", + "bulk_upload_report": { + "FilePath": "accept-usb/9730153760/1of1_Lloyd_George_Record_[Ruby Sandra MANTON]_[9730153760]_[08-01-2003].pdf", + "NhsNumber": "9730153760", + "PdsOdsCode": "H81109", + "StoredFileName": "9730153760/1of1_Lloyd_George_Record_[Ruby Sandra MANTON]_[9730153760]_[08-01-2003].pdf", + "UploadStatus": "complete", + "UploaderOdsCode": "M85143" + }, + "metadata": { + "Author": "M85143", + "ContentType": "application/pdf", + "CurrentGpOds": "H81109", + "Custodian": "H81109", + "DocStatus": "final", + "DocumentScanCreation": "2023-01-01", + "DocumentSnomedCodeType": "16521000000101", + "FileName": "1of1_Lloyd_George_Record_[Ruby Sandra MANTON]_[9730153760]_[08-01-2003].pdf", + "FileSize": "Decimal('173760')", + "NhsNumber": "9730153760", + "Status": "current", + "Uploaded": true, + "Version": "1", + "VirusScannerResult": "Clean" + }, + "unstitched": [] +} diff --git a/lambdas/tests/e2e/bulk_upload/check/__snapshots__/test_reject_bulk_upload/test_general_reject_ingestions[General-Rejected-Corrupted-9730788022].json b/lambdas/tests/e2e/bulk_upload/check/__snapshots__/test_reject_bulk_upload/test_general_reject_ingestions[General-Rejected-Corrupted-9730788022].json new file mode 100644 index 000000000..2ed762485 --- /dev/null +++ b/lambdas/tests/e2e/bulk_upload/check/__snapshots__/test_reject_bulk_upload/test_general_reject_ingestions[General-Rejected-Corrupted-9730788022].json @@ -0,0 +1,14 @@ +{ + "_description": "General-Rejected-Corrupted", + "bulk_upload_report": { + "FilePath": "reject/test/9730788022/1of1_Lloyd_George_Record_[Cath YOUDS]_[9730788022]_[05-05-1965].pdf", + "NhsNumber": "9730788022", + "PdsOdsCode": "M85143", + "Reason": "One or more of the files were corrupt", + "StoredFileName": "test/9730788022/1of1_Lloyd_George_Record_[Cath YOUDS]_[9730788022]_[05-05-1965].pdf", + "UploadStatus": "failed", + "UploaderOdsCode": "M85143" + }, + "metadata": null, + "unstitched": null +} diff --git a/lambdas/tests/e2e/bulk_upload/check/__snapshots__/test_reject_bulk_upload/test_general_reject_ingestions[General-Rejected-Infected-9730788529].json b/lambdas/tests/e2e/bulk_upload/check/__snapshots__/test_reject_bulk_upload/test_general_reject_ingestions[General-Rejected-Infected-9730788529].json new file mode 100644 index 000000000..e670fbdd1 --- /dev/null +++ b/lambdas/tests/e2e/bulk_upload/check/__snapshots__/test_reject_bulk_upload/test_general_reject_ingestions[General-Rejected-Infected-9730788529].json @@ -0,0 +1,14 @@ +{ + "_description": "General-Rejected-Infected", + "bulk_upload_report": { + "FilePath": "reject/1of1_Lloyd_George_Record_[Moses Ivan WELLS]_[9730788529]_[20-03-1994].pdf", + "NhsNumber": "9730788529", + "PdsOdsCode": "M85143", + "Reason": "One or more of the files failed virus scanner check", + "StoredFileName": "1of1_Lloyd_George_Record_[Moses Ivan WELLS]_[9730788529]_[20-03-1994].pdf", + "UploadStatus": "failed", + "UploaderOdsCode": "M85143" + }, + "metadata": null, + "unstitched": null +} diff --git a/lambdas/tests/e2e/bulk_upload/check/__snapshots__/test_reject_bulk_upload/test_general_reject_ingestions[General-Rejected-Invalid NHS number-9000000061].json b/lambdas/tests/e2e/bulk_upload/check/__snapshots__/test_reject_bulk_upload/test_general_reject_ingestions[General-Rejected-Invalid NHS number-9000000061].json new file mode 100644 index 000000000..433a06b45 --- /dev/null +++ b/lambdas/tests/e2e/bulk_upload/check/__snapshots__/test_reject_bulk_upload/test_general_reject_ingestions[General-Rejected-Invalid NHS number-9000000061].json @@ -0,0 +1,14 @@ +{ + "_description": "General-Rejected-Invalid NHS number", + "bulk_upload_report": { + "FilePath": "01 of 02_Lloyd_George_Record_Jane Stevens_900_000_0001_11.10.2010.pdf", + "NhsNumber": "9000000061", + "PdsOdsCode": "", + "Reason": "Invalid NHS number", + "StoredFileName": "01 of 02_Lloyd_George_Record_Jane Stevens_900_000_0001_11.10.2010.pdf", + "UploadStatus": "failed", + "UploaderOdsCode": "H81109" + }, + "metadata": null, + "unstitched": null +} diff --git a/lambdas/tests/e2e/bulk_upload/check/__snapshots__/test_reject_bulk_upload/test_general_reject_ingestions[General-Rejected-Invalid date format-9000000051].json b/lambdas/tests/e2e/bulk_upload/check/__snapshots__/test_reject_bulk_upload/test_general_reject_ingestions[General-Rejected-Invalid date format-9000000051].json new file mode 100644 index 000000000..f7d8bb5b7 --- /dev/null +++ b/lambdas/tests/e2e/bulk_upload/check/__snapshots__/test_reject_bulk_upload/test_general_reject_ingestions[General-Rejected-Invalid date format-9000000051].json @@ -0,0 +1,14 @@ +{ + "_description": "General-Rejected-Invalid date format", + "bulk_upload_report": { + "FilePath": "01 of 02_Lloyd_George_Record_Jaimie Stevens_9000000001_300.4.2010.pdf", + "NhsNumber": "9000000051", + "PdsOdsCode": "", + "Reason": "Could not find a valid date in the filename", + "StoredFileName": "01 of 02_Lloyd_George_Record_Jaimie Stevens_9000000001_300.4.2010.pdf", + "UploadStatus": "failed", + "UploaderOdsCode": "H81109" + }, + "metadata": null, + "unstitched": null +} diff --git a/lambdas/tests/e2e/bulk_upload/check/__snapshots__/test_reject_bulk_upload/test_general_reject_ingestions[General-Rejected-Missing 'lloyd george record'-9500000001].json b/lambdas/tests/e2e/bulk_upload/check/__snapshots__/test_reject_bulk_upload/test_general_reject_ingestions[General-Rejected-Missing 'lloyd george record'-9500000001].json new file mode 100644 index 000000000..1acd57e09 --- /dev/null +++ b/lambdas/tests/e2e/bulk_upload/check/__snapshots__/test_reject_bulk_upload/test_general_reject_ingestions[General-Rejected-Missing 'lloyd george record'-9500000001].json @@ -0,0 +1,14 @@ +{ + "_description": "General-Rejected-Missing 'lloyd george record'", + "bulk_upload_report": { + "FilePath": "1093/01 of 02_Phineas Stevens_9000000001_22.10.2010.pdf", + "NhsNumber": "9500000001", + "PdsOdsCode": "", + "Reason": "Invalid Lloyd_George_Record separator", + "StoredFileName": "1093/01 of 02_Phineas Stevens_9000000001_22.10.2010.pdf", + "UploadStatus": "failed", + "UploaderOdsCode": "H81109" + }, + "metadata": null, + "unstitched": null +} diff --git a/lambdas/tests/e2e/bulk_upload/check/__snapshots__/test_reject_bulk_upload/test_general_reject_ingestions[General-Rejected-Nhs Not in PDS-9596333437].json b/lambdas/tests/e2e/bulk_upload/check/__snapshots__/test_reject_bulk_upload/test_general_reject_ingestions[General-Rejected-Nhs Not in PDS-9596333437].json new file mode 100644 index 000000000..5c4a2bca5 --- /dev/null +++ b/lambdas/tests/e2e/bulk_upload/check/__snapshots__/test_reject_bulk_upload/test_general_reject_ingestions[General-Rejected-Nhs Not in PDS-9596333437].json @@ -0,0 +1,14 @@ +{ + "_description": "General-Rejected-Nhs Not in PDS", + "bulk_upload_report": { + "FilePath": "reject/1of1_Lloyd_George_Record_[Josh Lucien PLATT]_[9596333437]_[23-10-1979].pdf", + "NhsNumber": "9596333437", + "PdsOdsCode": "", + "Reason": "Could not find the given patient on PDS", + "StoredFileName": "1of1_Lloyd_George_Record_[Josh Lucien PLATT]_[9596333437]_[23-10-1979].pdf", + "UploadStatus": "failed", + "UploaderOdsCode": "M85143" + }, + "metadata": null, + "unstitched": null +} diff --git a/lambdas/tests/e2e/bulk_upload/check/__snapshots__/test_reject_bulk_upload/test_general_reject_ingestions[General-Rejected-incorrect NHS number or date format - Invalid Format-9000000201].json b/lambdas/tests/e2e/bulk_upload/check/__snapshots__/test_reject_bulk_upload/test_general_reject_ingestions[General-Rejected-incorrect NHS number or date format - Invalid Format-9000000201].json new file mode 100644 index 000000000..e83b4e7a8 --- /dev/null +++ b/lambdas/tests/e2e/bulk_upload/check/__snapshots__/test_reject_bulk_upload/test_general_reject_ingestions[General-Rejected-incorrect NHS number or date format - Invalid Format-9000000201].json @@ -0,0 +1,14 @@ +{ + "_description": "General-Rejected-incorrect NHS number or date format - Invalid Format", + "bulk_upload_report": { + "FilePath": "01 of 02_Lloyd_George_Record_Andrew Stevens_9000000001_12.10.21.pdf", + "NhsNumber": "9000000201", + "PdsOdsCode": "", + "Reason": "Incorrect NHS number or date format", + "StoredFileName": "01 of 02_Lloyd_George_Record_Andrew Stevens_9000000001_12.10.21.pdf", + "UploadStatus": "failed", + "UploaderOdsCode": "H81109" + }, + "metadata": null, + "unstitched": null +} diff --git a/lambdas/tests/e2e/bulk_upload/check/__snapshots__/test_reject_bulk_upload/test_general_reject_ingestions[General-Rejected-incorrect NHS number or date format-9000000071].json b/lambdas/tests/e2e/bulk_upload/check/__snapshots__/test_reject_bulk_upload/test_general_reject_ingestions[General-Rejected-incorrect NHS number or date format-9000000071].json new file mode 100644 index 000000000..58af9fc7d --- /dev/null +++ b/lambdas/tests/e2e/bulk_upload/check/__snapshots__/test_reject_bulk_upload/test_general_reject_ingestions[General-Rejected-incorrect NHS number or date format-9000000071].json @@ -0,0 +1,14 @@ +{ + "_description": "General-Rejected-incorrect NHS number or date format", + "bulk_upload_report": { + "FilePath": "01 of 02_Lloyd_George_Record_Patricia Stevens_9000000001_2_11.10.2010.pdf", + "NhsNumber": "9000000071", + "PdsOdsCode": "", + "Reason": "Incorrect NHS number or date format", + "StoredFileName": "01 of 02_Lloyd_George_Record_Patricia Stevens_9000000001_2_11.10.2010.pdf", + "UploadStatus": "failed", + "UploaderOdsCode": "H81109" + }, + "metadata": null, + "unstitched": null +} diff --git a/lambdas/tests/e2e/bulk_upload/check/__snapshots__/test_reject_bulk_upload/test_general_reject_ingestions[General-Rejected-incorrect document path format-9000000081].json b/lambdas/tests/e2e/bulk_upload/check/__snapshots__/test_reject_bulk_upload/test_general_reject_ingestions[General-Rejected-incorrect document path format-9000000081].json new file mode 100644 index 000000000..3ed24ec7d --- /dev/null +++ b/lambdas/tests/e2e/bulk_upload/check/__snapshots__/test_reject_bulk_upload/test_general_reject_ingestions[General-Rejected-incorrect document path format-9000000081].json @@ -0,0 +1,14 @@ +{ + "_description": "General-Rejected-incorrect document path format", + "bulk_upload_report": { + "FilePath": "103/01 02_Lloyd_George_Record_Charles Stevens_9000000001_22.10.2010.pdf", + "NhsNumber": "9000000081", + "PdsOdsCode": "", + "Reason": "Incorrect document path format", + "StoredFileName": "103/01 02_Lloyd_George_Record_Charles Stevens_9000000001_22.10.2010.pdf", + "UploadStatus": "failed", + "UploaderOdsCode": "H81109" + }, + "metadata": null, + "unstitched": null +} diff --git a/lambdas/tests/e2e/bulk_upload/check/__snapshots__/test_reject_bulk_upload/test_general_reject_ingestions[General-Rejected-invalid date format - Invalid Month-9000000101].json b/lambdas/tests/e2e/bulk_upload/check/__snapshots__/test_reject_bulk_upload/test_general_reject_ingestions[General-Rejected-invalid date format - Invalid Month-9000000101].json new file mode 100644 index 000000000..c41270c68 --- /dev/null +++ b/lambdas/tests/e2e/bulk_upload/check/__snapshots__/test_reject_bulk_upload/test_general_reject_ingestions[General-Rejected-invalid date format - Invalid Month-9000000101].json @@ -0,0 +1,14 @@ +{ + "_description": "General-Rejected-invalid date format - Invalid Month", + "bulk_upload_report": { + "FilePath": "01 of 02_Lloyd_George_Record_Steph Stevens_9000000001_12.25.2010.pdf", + "NhsNumber": "9000000101", + "PdsOdsCode": "", + "Reason": "Invalid date format", + "StoredFileName": "01 of 02_Lloyd_George_Record_Steph Stevens_9000000001_12.25.2010.pdf", + "UploadStatus": "failed", + "UploaderOdsCode": "H81109" + }, + "metadata": null, + "unstitched": null +} diff --git a/lambdas/tests/e2e/bulk_upload/check/__snapshots__/test_reject_bulk_upload/test_remap_and_fixed[123456789-Prevent Fixed on File and NhsNumber].json b/lambdas/tests/e2e/bulk_upload/check/__snapshots__/test_reject_bulk_upload/test_remap_and_fixed[123456789-Prevent Fixed on File and NhsNumber].json new file mode 100644 index 000000000..14efd8624 --- /dev/null +++ b/lambdas/tests/e2e/bulk_upload/check/__snapshots__/test_reject_bulk_upload/test_remap_and_fixed[123456789-Prevent Fixed on File and NhsNumber].json @@ -0,0 +1,5 @@ +{ + "_description": "Prevent Fixed on File and NhsNumber", + "bulk_upload_report": null, + "metadata": null +} diff --git a/lambdas/tests/e2e/bulk_upload/check/__snapshots__/test_reject_bulk_upload/test_remap_and_fixed_reject[123456789-Prevent Fixed on File and NhsNumber].json b/lambdas/tests/e2e/bulk_upload/check/__snapshots__/test_reject_bulk_upload/test_remap_and_fixed_reject[123456789-Prevent Fixed on File and NhsNumber].json new file mode 100644 index 000000000..14efd8624 --- /dev/null +++ b/lambdas/tests/e2e/bulk_upload/check/__snapshots__/test_reject_bulk_upload/test_remap_and_fixed_reject[123456789-Prevent Fixed on File and NhsNumber].json @@ -0,0 +1,5 @@ +{ + "_description": "Prevent Fixed on File and NhsNumber", + "bulk_upload_report": null, + "metadata": null +} diff --git a/lambdas/tests/e2e/bulk_upload/check/__snapshots__/test_reject_bulk_upload/test_usb_reject_ingestions[USB-Rejected-Could not find a valid date in the filename-9000000005].json b/lambdas/tests/e2e/bulk_upload/check/__snapshots__/test_reject_bulk_upload/test_usb_reject_ingestions[USB-Rejected-Could not find a valid date in the filename-9000000005].json new file mode 100644 index 000000000..f1c07fc1b --- /dev/null +++ b/lambdas/tests/e2e/bulk_upload/check/__snapshots__/test_reject_bulk_upload/test_usb_reject_ingestions[USB-Rejected-Could not find a valid date in the filename-9000000005].json @@ -0,0 +1,14 @@ +{ + "_description": "USB-Rejected-Could not find a valid date in the filename?", + "bulk_upload_report": { + "FilePath": "9730153819 07-Aug-2020/fgdgdf2f4e4dtd_B_1_Lloyd_George_Record_NOS_1of1_9000000005.pdf", + "NhsNumber": "9000000005", + "PdsOdsCode": "", + "Reason": "Could not find a valid date in the filename", + "StoredFileName": "9730153819 07-Aug-2020/fgdgdf2f4e4dtd_B_1_Lloyd_George_Record_NOS_1of1_9000000005.pdf", + "UploadStatus": "failed", + "UploaderOdsCode": "M85143" + }, + "metadata": null, + "unstitched": null +} diff --git a/lambdas/tests/e2e/bulk_upload/check/__snapshots__/test_reject_bulk_upload/test_usb_reject_ingestions[USB-Rejected-Could not find a valid date in the filename.-9000000045].json b/lambdas/tests/e2e/bulk_upload/check/__snapshots__/test_reject_bulk_upload/test_usb_reject_ingestions[USB-Rejected-Could not find a valid date in the filename.-9000000045].json new file mode 100644 index 000000000..acf9d8c7f --- /dev/null +++ b/lambdas/tests/e2e/bulk_upload/check/__snapshots__/test_reject_bulk_upload/test_usb_reject_ingestions[USB-Rejected-Could not find a valid date in the filename.-9000000045].json @@ -0,0 +1,14 @@ +{ + "_description": "USB-Rejected-Could not find a valid date in the filename.", + "bulk_upload_report": { + "FilePath": "1232323232 12-Aug-1999 Catherine Higgins/fgdgdf2f4e4dtd_B_1_Lloyd_George_Record_NOS_1of1_9000000045.pdf", + "NhsNumber": "9000000045", + "PdsOdsCode": "", + "Reason": "Could not find a valid date in the filename", + "StoredFileName": "1232323232 12-Aug-1999 Catherine Higgins/fgdgdf2f4e4dtd_B_1_Lloyd_George_Record_NOS_1of1_9000000045.pdf", + "UploadStatus": "failed", + "UploaderOdsCode": "M85143" + }, + "metadata": null, + "unstitched": null +} diff --git a/lambdas/tests/e2e/bulk_upload/check/__snapshots__/test_reject_bulk_upload/test_usb_reject_ingestions[USB-Rejected-Could not find a valid date in the filename.-9000000055].json b/lambdas/tests/e2e/bulk_upload/check/__snapshots__/test_reject_bulk_upload/test_usb_reject_ingestions[USB-Rejected-Could not find a valid date in the filename.-9000000055].json new file mode 100644 index 000000000..810a01e05 --- /dev/null +++ b/lambdas/tests/e2e/bulk_upload/check/__snapshots__/test_reject_bulk_upload/test_usb_reject_ingestions[USB-Rejected-Could not find a valid date in the filename.-9000000055].json @@ -0,0 +1,14 @@ +{ + "_description": "USB-Rejected-Could not find a valid date in the filename.", + "bulk_upload_report": { + "FilePath": "1253617284 Adam Thomas 07-Nov-99/fgdgdf2f4e4dtd_B_1_Lloyd_George_Record_NOS_1of1_9000000055.pdf", + "NhsNumber": "9000000055", + "PdsOdsCode": "", + "Reason": "Could not find a valid date in the filename", + "StoredFileName": "1253617284 Adam Thomas 07-Nov-99/fgdgdf2f4e4dtd_B_1_Lloyd_George_Record_NOS_1of1_9000000055.pdf", + "UploadStatus": "failed", + "UploaderOdsCode": "M85143" + }, + "metadata": null, + "unstitched": null +} diff --git a/lambdas/tests/e2e/bulk_upload/check/__snapshots__/test_reject_bulk_upload/test_usb_reject_ingestions[USB-Rejected-Could not find a valid date in the filename.-9000000099].json b/lambdas/tests/e2e/bulk_upload/check/__snapshots__/test_reject_bulk_upload/test_usb_reject_ingestions[USB-Rejected-Could not find a valid date in the filename.-9000000099].json new file mode 100644 index 000000000..22a1aa9e5 --- /dev/null +++ b/lambdas/tests/e2e/bulk_upload/check/__snapshots__/test_reject_bulk_upload/test_usb_reject_ingestions[USB-Rejected-Could not find a valid date in the filename.-9000000099].json @@ -0,0 +1,14 @@ +{ + "_description": "USB-Rejected-Could not find a valid date in the filename.", + "bulk_upload_report": { + "FilePath": "2516273843 Fiona Smith/fgdgdf2f4e4dtd_B_1_Lloyd_George_Record_NOS_1of1_9000000099.pdf", + "NhsNumber": "9000000099", + "PdsOdsCode": "", + "Reason": "Could not find a valid date in the filename", + "StoredFileName": "2516273843 Fiona Smith/fgdgdf2f4e4dtd_B_1_Lloyd_George_Record_NOS_1of1_9000000099.pdf", + "UploadStatus": "failed", + "UploaderOdsCode": "M85143" + }, + "metadata": null, + "unstitched": null +} diff --git a/lambdas/tests/e2e/bulk_upload/check/__snapshots__/test_reject_bulk_upload/test_usb_reject_ingestions[USB-Rejected-File extension .tiff is not supported-9000000001].json b/lambdas/tests/e2e/bulk_upload/check/__snapshots__/test_reject_bulk_upload/test_usb_reject_ingestions[USB-Rejected-File extension .tiff is not supported-9000000001].json new file mode 100644 index 000000000..488cb3ce9 --- /dev/null +++ b/lambdas/tests/e2e/bulk_upload/check/__snapshots__/test_reject_bulk_upload/test_usb_reject_ingestions[USB-Rejected-File extension .tiff is not supported-9000000001].json @@ -0,0 +1,14 @@ +{ + "_description": "USB-Rejected-File extension .tiff is not supported", + "bulk_upload_report": { + "FilePath": "9730153752 Dawn FLING 19-Sep-2011/fgdgdf2f4e4dtd_B_1_Lloyd_George_tiff_NOS_9730153752.tiff", + "NhsNumber": "9000000001", + "PdsOdsCode": "", + "Reason": "File extension .tiff is not supported", + "StoredFileName": "9730153752 Dawn FLING 19-Sep-2011/fgdgdf2f4e4dtd_B_1_Lloyd_George_tiff_NOS_9730153752.tiff", + "UploadStatus": "failed", + "UploaderOdsCode": "M85143" + }, + "metadata": null, + "unstitched": null +} diff --git a/lambdas/tests/e2e/bulk_upload/check/__snapshots__/test_reject_bulk_upload/test_usb_reject_ingestions[USB-Rejected-Invalid NHS number-9000000088].json b/lambdas/tests/e2e/bulk_upload/check/__snapshots__/test_reject_bulk_upload/test_usb_reject_ingestions[USB-Rejected-Invalid NHS number-9000000088].json new file mode 100644 index 000000000..054cfca31 --- /dev/null +++ b/lambdas/tests/e2e/bulk_upload/check/__snapshots__/test_reject_bulk_upload/test_usb_reject_ingestions[USB-Rejected-Invalid NHS number-9000000088].json @@ -0,0 +1,14 @@ +{ + "_description": "USB-Rejected-Invalid NHS number", + "bulk_upload_report": { + "FilePath": "Stewart Smith 07-Sep-1998/fgdgdf2f4e4dtd_B_1_Lloyd_George_Record_NOS_1of1_9000000088.pdf", + "NhsNumber": "9000000088", + "PdsOdsCode": "", + "Reason": "Invalid NHS number", + "StoredFileName": "Stewart Smith 07-Sep-1998/fgdgdf2f4e4dtd_B_1_Lloyd_George_Record_NOS_1of1_9000000088.pdf", + "UploadStatus": "failed", + "UploaderOdsCode": "M85143" + }, + "metadata": null, + "unstitched": null +} diff --git a/lambdas/tests/e2e/bulk_upload/check/__snapshots__/test_reject_bulk_upload/test_usb_reject_ingestions[USB-Rejected-Invalid date format-9000000044].json b/lambdas/tests/e2e/bulk_upload/check/__snapshots__/test_reject_bulk_upload/test_usb_reject_ingestions[USB-Rejected-Invalid date format-9000000044].json new file mode 100644 index 000000000..b222e2e9b --- /dev/null +++ b/lambdas/tests/e2e/bulk_upload/check/__snapshots__/test_reject_bulk_upload/test_usb_reject_ingestions[USB-Rejected-Invalid date format-9000000044].json @@ -0,0 +1,14 @@ +{ + "_description": "USB-Rejected-Invalid date format", + "bulk_upload_report": { + "FilePath": "1234567895 Lillian Percy 07-24-1999/fgdgdf2f4e4dtd_B_1_Lloyd_George_Record_NOS_1of1_9000000044.pdf", + "NhsNumber": "9000000044", + "PdsOdsCode": "", + "Reason": "Invalid date format", + "StoredFileName": "1234567895 Lillian Percy 07-24-1999/fgdgdf2f4e4dtd_B_1_Lloyd_George_Record_NOS_1of1_9000000044.pdf", + "UploadStatus": "failed", + "UploaderOdsCode": "M85143" + }, + "metadata": null, + "unstitched": null +} diff --git a/lambdas/tests/e2e/bulk_upload/check/__snapshots__/test_reject_bulk_upload/test_usb_reject_ingestions[USB-Rejected-Invalid patient name-9000000008].json b/lambdas/tests/e2e/bulk_upload/check/__snapshots__/test_reject_bulk_upload/test_usb_reject_ingestions[USB-Rejected-Invalid patient name-9000000008].json new file mode 100644 index 000000000..92c42a7ad --- /dev/null +++ b/lambdas/tests/e2e/bulk_upload/check/__snapshots__/test_reject_bulk_upload/test_usb_reject_ingestions[USB-Rejected-Invalid patient name-9000000008].json @@ -0,0 +1,14 @@ +{ + "_description": "USB-Rejected-Invalid patient name", + "bulk_upload_report": { + "FilePath": "9730153 Louisa Gables 07-Nov-1999/fgdgdf2f4e4dtd_B_1_Lloyd_George_Record_NOS_1of1_9000000008.pdf", + "NhsNumber": "9000000008", + "PdsOdsCode": "", + "Reason": "Invalid NHS number", + "StoredFileName": "9730153 Louisa Gables 07-Nov-1999/fgdgdf2f4e4dtd_B_1_Lloyd_George_Record_NOS_1of1_9000000008.pdf", + "UploadStatus": "failed", + "UploaderOdsCode": "M85143" + }, + "metadata": null, + "unstitched": null +} diff --git a/lambdas/tests/e2e/bulk_upload/check/__snapshots__/test_reject_bulk_upload/test_usb_reject_ingestions[USB-Rejected-Invalid patient name-9000000067].json b/lambdas/tests/e2e/bulk_upload/check/__snapshots__/test_reject_bulk_upload/test_usb_reject_ingestions[USB-Rejected-Invalid patient name-9000000067].json new file mode 100644 index 000000000..db4a5090c --- /dev/null +++ b/lambdas/tests/e2e/bulk_upload/check/__snapshots__/test_reject_bulk_upload/test_usb_reject_ingestions[USB-Rejected-Invalid patient name-9000000067].json @@ -0,0 +1,14 @@ +{ + "_description": "USB-Rejected-Invalid patient name", + "bulk_upload_report": { + "FilePath": "12-Aug-1944 Catherine Higgins 1992323232/fgdgdf2f4e4dtd_B_1_Lloyd_George_Record_NOS_1of1_9000000067.pdf", + "NhsNumber": "9000000067", + "PdsOdsCode": "", + "Reason": "Invalid patient name", + "StoredFileName": "12-Aug-1944 Catherine Higgins 1992323232/fgdgdf2f4e4dtd_B_1_Lloyd_George_Record_NOS_1of1_9000000067.pdf", + "UploadStatus": "failed", + "UploaderOdsCode": "M85143" + }, + "metadata": null, + "unstitched": null +} diff --git a/lambdas/tests/e2e/bulk_upload/check/__snapshots__/test_reject_bulk_upload/test_usb_reject_ingestions[USB-Rejected-Multi-part documents are not supported-9000000003].json b/lambdas/tests/e2e/bulk_upload/check/__snapshots__/test_reject_bulk_upload/test_usb_reject_ingestions[USB-Rejected-Multi-part documents are not supported-9000000003].json new file mode 100644 index 000000000..1144e566c --- /dev/null +++ b/lambdas/tests/e2e/bulk_upload/check/__snapshots__/test_reject_bulk_upload/test_usb_reject_ingestions[USB-Rejected-Multi-part documents are not supported-9000000003].json @@ -0,0 +1,14 @@ +{ + "_description": "USB-Rejected-Multi-part documents are not supported", + "bulk_upload_report": { + "FilePath": "9730153973 Sue Lara TEMPLE 25-Jul-2010/fgdgdf2f4e4dtd_B_1_Lloyd_George_Record_NOS_1of7_9730153973.pdf", + "NhsNumber": "9000000003", + "PdsOdsCode": "", + "Reason": "Multi-part documents are not supported", + "StoredFileName": "9730153973 Sue Lara TEMPLE 25-Jul-2010/fgdgdf2f4e4dtd_B_1_Lloyd_George_Record_NOS_1of7_9730153973.pdf", + "UploadStatus": "failed", + "UploaderOdsCode": "M85143" + }, + "metadata": null, + "unstitched": null +} diff --git a/lambdas/tests/e2e/bulk_upload/check/__snapshots__/test_reject_bulk_upload/test_usb_reject_ingestions[USB-Rejected-no space between name and dob-9000000066].json b/lambdas/tests/e2e/bulk_upload/check/__snapshots__/test_reject_bulk_upload/test_usb_reject_ingestions[USB-Rejected-no space between name and dob-9000000066].json new file mode 100644 index 000000000..712e30071 --- /dev/null +++ b/lambdas/tests/e2e/bulk_upload/check/__snapshots__/test_reject_bulk_upload/test_usb_reject_ingestions[USB-Rejected-no space between name and dob-9000000066].json @@ -0,0 +1,14 @@ +{ + "_description": "USB-Rejected-no space between name and dob", + "bulk_upload_report": { + "FilePath": "9725362748 Geraldine Ellis07-Sep-1999/fgdgdf2f4e4dtd_B_1_Lloyd_George_Record_NOS_1of1_9000000066.pdf", + "NhsNumber": "9000000066", + "PdsOdsCode": "", + "Reason": "Could not find a valid date in the filename", + "StoredFileName": "9725362748 Geraldine Ellis07-Sep-1999/fgdgdf2f4e4dtd_B_1_Lloyd_George_Record_NOS_1of1_9000000066.pdf", + "UploadStatus": "failed", + "UploaderOdsCode": "M85143" + }, + "metadata": null, + "unstitched": null +} diff --git a/lambdas/tests/e2e/bulk_upload/check/__snapshots__/test_scod_bulk_upload/test_scod_ingestions[SCOD-Accept-9730787158].json b/lambdas/tests/e2e/bulk_upload/check/__snapshots__/test_scod_bulk_upload/test_scod_ingestions[SCOD-Accept-9730787158].json new file mode 100644 index 000000000..91964f69f --- /dev/null +++ b/lambdas/tests/e2e/bulk_upload/check/__snapshots__/test_scod_bulk_upload/test_scod_ingestions[SCOD-Accept-9730787158].json @@ -0,0 +1,27 @@ +{ + "_description": "SCOD-Accept", + "bulk_upload_report": { + "FilePath": "expedite/1of1_Lloyd_George_Record_[Felix Alfred VOS]_[9730787158]_[03-01-2016].pdf", + "NhsNumber": "9730787158", + "PdsOdsCode": "M85143", + "StoredFileName": "expedite/1of1_Lloyd_George_Record_[Felix Alfred VOS]_[9730787158]_[03-01-2016].pdf", + "UploadStatus": "complete", + "UploaderOdsCode": "expedite" + }, + "metadata": { + "Author": "expedite", + "ContentType": "application/pdf", + "CurrentGpOds": "M85143", + "Custodian": "M85143", + "DocStatus": "final", + "DocumentSnomedCodeType": "16521000000101", + "FileName": "1of1_Lloyd_George_Record_[Felix Alfred VOS]_[9730787158]_[03-01-2016].pdf", + "FileSize": "Decimal('173760')", + "NhsNumber": "9730787158", + "Status": "current", + "Uploaded": true, + "Version": "1", + "VirusScannerResult": "Clean" + }, + "unstitched": [] +} diff --git a/lambdas/tests/e2e/bulk_upload/check/__snapshots__/test_scod_bulk_upload/test_scod_ingestions[SCOD-Reject-9730787159].json b/lambdas/tests/e2e/bulk_upload/check/__snapshots__/test_scod_bulk_upload/test_scod_ingestions[SCOD-Reject-9730787159].json new file mode 100644 index 000000000..c41adf1ea --- /dev/null +++ b/lambdas/tests/e2e/bulk_upload/check/__snapshots__/test_scod_bulk_upload/test_scod_ingestions[SCOD-Reject-9730787159].json @@ -0,0 +1,14 @@ +{ + "_description": "SCOD-Reject", + "bulk_upload_report": { + "FilePath": "expedite/1of1_Lloyd_George_Record_[Felix Alfred VOS]_[9730787159]_[03-01-2016].pdf", + "NhsNumber": "9730787159", + "PdsOdsCode": "", + "Reason": "Invalid NHS number format", + "StoredFileName": "expedite/1of1_Lloyd_George_Record_[Felix Alfred VOS]_[9730787159]_[03-01-2016].pdf", + "UploadStatus": "failed", + "UploaderOdsCode": "expedite" + }, + "metadata": null, + "unstitched": [] +} diff --git a/lambdas/tests/e2e/bulk_upload/check/test_accept_bulk_upload.py b/lambdas/tests/e2e/bulk_upload/check/test_accept_bulk_upload.py new file mode 100644 index 000000000..0bc1b8566 --- /dev/null +++ b/lambdas/tests/e2e/bulk_upload/check/test_accept_bulk_upload.py @@ -0,0 +1,160 @@ +from pathlib import Path + +import pytest +from syrupy.filters import paths + +from services.base.dynamo_service import DynamoDBService +from tests.e2e.bulk_upload.conftest import ( + get_all_entries_from_table_by_nhs_number, + read_metadata_csv, + get_entry_from_table_by_nhs_number, +) +from tests.e2e.helpers.data_helper import LloydGeorgeDataHelper + +dynamo_service = DynamoDBService() + +lloyd_george_data_helper = LloydGeorgeDataHelper() + +datadir = Path(__file__).parent.parent / "data" +general_accept_test_data = read_metadata_csv(datadir, "accept-metadata.csv") +usb_accept_test_data = read_metadata_csv(datadir, "accept-metadata-usb.csv") + +general_accept_test_cases = [] +for record in general_accept_test_data: + general_accept_test_cases.append((record["Test-Description"], record["NHS-NO"])) + +usb_accept_test_cases = [] +for record in usb_accept_test_data: + usb_accept_test_cases.append((record["Test-Description"], record["NHS-NO"])) + + +@pytest.fixture(scope="session", autouse=True) +def bulk_upload_table_records(): + return { + "lloyd_george_records": lloyd_george_data_helper.scan_lloyd_george_table(), + "bulk_upload_report_records": lloyd_george_data_helper.scan_bulk_upload_report_table(), + "unstitched_records": lloyd_george_data_helper.scan_unstitch_table(), + } + + +@pytest.mark.parametrize( + "description, nhs_number", + general_accept_test_cases, +) +def test_general_accepted_ingestions( + nhs_number, + description, + snapshot_json, + bulk_upload_table_records, +) -> None: + records_by_num = { + "_description": description, + "metadata": get_entry_from_table_by_nhs_number(nhs_number, bulk_upload_table_records["lloyd_george_records"]), + "bulk_upload_report": get_entry_from_table_by_nhs_number( + nhs_number, bulk_upload_table_records["bulk_upload_report_records"] + ), + "unstitched": get_all_entries_from_table_by_nhs_number(nhs_number, + bulk_upload_table_records["unstitched_records"]), + } + + def generate_unstitched_exclusions(num): + return [ + f"unstitched.{i}.{key}" + for i in range(num) + for key in ["Created", "FileLocation", "ID", "S3VersionID", "S3FileKey", "LastUpdated", "FileName"] + ] + + assert records_by_num == snapshot_json( + exclude=paths( + "metadata.S3FileKey", + "metadata.ID", + "metadata.Created", + "metadata.FileLocation", + "metadata.LastUpdated", + "metadata.S3VersionID", + "bulk_upload_report.Date", + "bulk_upload_report.ID", + "bulk_upload_report.Timestamp", + "bulk_upload_report.FilePath", + "bulk_upload_report.StoredFileName", + "bulk_upload_report.Created", + *generate_unstitched_exclusions(3), + ) + ) + metadata = records_by_num.get("metadata") or {} + metadata_s3_key = str(metadata.get("S3FileKey")) + metadata_s3_version = str(metadata.get("S3VersionID")) + assert lloyd_george_data_helper.check_record_exists_in_s3_with_version(metadata_s3_key, metadata_s3_version) + + +@pytest.mark.parametrize( + "description, nhs_number", + usb_accept_test_cases, +) +def test_usb_accepted_ingestions( + nhs_number, + description, + snapshot_json, + bulk_upload_table_records, +) -> None: + records_by_num = { + "_description": description, + "metadata": get_entry_from_table_by_nhs_number(nhs_number, bulk_upload_table_records["lloyd_george_records"]), + "bulk_upload_report": get_entry_from_table_by_nhs_number( + nhs_number, bulk_upload_table_records["bulk_upload_report_records"] + ), + "unstitched": get_all_entries_from_table_by_nhs_number(nhs_number, + bulk_upload_table_records["unstitched_records"]), + } + assert records_by_num == snapshot_json( + exclude=paths( + "metadata.S3FileKey", + "metadata.ID", + "metadata.Created", + "metadata.FileLocation", + "metadata.S3VersionID", + "metadata.LastUpdated", + "bulk_upload_report.Date", + "bulk_upload_report.ID", + "bulk_upload_report.Timestamp", + ) + ) + metadata = records_by_num.get("metadata") or {} + metadata_s3_key = str(metadata.get("S3FileKey")) + metadata_s3_version = str(metadata.get("S3VersionID")) + assert lloyd_george_data_helper.check_record_exists_in_s3_with_version(metadata_s3_key, metadata_s3_version) + + +@pytest.mark.parametrize("nhs_number,description", [("9730787492", "Remap and Fixed")]) +def test_remap_and_fixed( + nhs_number, + description, + snapshot_json, + bulk_upload_table_records, +) -> None: + records_by_num = { + "_description": description, + "metadata": get_entry_from_table_by_nhs_number(nhs_number, bulk_upload_table_records["lloyd_george_records"]), + "bulk_upload_report": get_entry_from_table_by_nhs_number( + nhs_number, bulk_upload_table_records["bulk_upload_report_records"] + ), + "unstitched": get_all_entries_from_table_by_nhs_number(nhs_number, + bulk_upload_table_records["unstitched_records"]), + } + assert records_by_num == snapshot_json( + exclude=paths( + "metadata.S3FileKey", + "metadata.S3VersionID", + "metadata.ID", + "metadata.Created", + "metadata.FileLocation", + "metadata.LastUpdated", + "bulk_upload_report.Date", + "bulk_upload_report.ID", + "bulk_upload_report.Timestamp", + ) + ) + metadata = records_by_num.get("metadata") or {} + metadata_s3_key = str(metadata.get("S3FileKey")) + metadata_s3_version = str(metadata.get("S3VersionID")) + assert lloyd_george_data_helper.check_record_exists_in_s3_with_version(metadata_s3_key, metadata_s3_version) diff --git a/lambdas/tests/e2e/bulk_upload/check/test_reject_bulk_upload.py b/lambdas/tests/e2e/bulk_upload/check/test_reject_bulk_upload.py new file mode 100644 index 000000000..b22df54da --- /dev/null +++ b/lambdas/tests/e2e/bulk_upload/check/test_reject_bulk_upload.py @@ -0,0 +1,119 @@ +from pathlib import Path + +import boto3 +import pytest +from syrupy.filters import paths + +from services.base.dynamo_service import DynamoDBService +from tests.e2e.bulk_upload.conftest import read_metadata_csv, get_entry_from_table_by_nhs_number +from tests.e2e.helpers.data_helper import LloydGeorgeDataHelper + +dynamo_service = DynamoDBService() + +lloyd_george_data_helper = LloydGeorgeDataHelper() + +s3 = boto3.client("s3") +datadir = Path(__file__).parent.parent / "data" +general_reject_test_data = read_metadata_csv(datadir, "reject-metadata.csv") +usb_reject_test_data = read_metadata_csv(datadir, "reject-metadata-usb.csv") + +general_reject_test_cases = [] +for record in general_reject_test_data: + general_reject_test_cases.append((record["Test-Description"], record["NHS-NO"])) + +usb_reject_test_cases = [] +for record in usb_reject_test_data: + usb_reject_test_cases.append((record["Test-Description"], record["NHS-NO"])) + + +@pytest.fixture(scope="session", autouse=True) +def bulk_upload_table_records(): + return { + "lloyd_george_records": lloyd_george_data_helper.scan_lloyd_george_table(), + "bulk_upload_report_records": lloyd_george_data_helper.scan_bulk_upload_report_table(), + "unstitched_records": lloyd_george_data_helper.scan_unstitch_table(), + } + + +@pytest.mark.parametrize( + "description, nhs_number", + general_reject_test_cases, +) +def test_general_reject_ingestions( + nhs_number, + description, + snapshot_json, + bulk_upload_table_records, +) -> None: + records_by_num = { + "_description": description, + "metadata": get_entry_from_table_by_nhs_number(nhs_number, bulk_upload_table_records["lloyd_george_records"]), + "bulk_upload_report": get_entry_from_table_by_nhs_number( + nhs_number, bulk_upload_table_records["bulk_upload_report_records"] + ), + "unstitched": get_entry_from_table_by_nhs_number(nhs_number, bulk_upload_table_records["unstitched_records"]), + } + assert records_by_num == snapshot_json( + exclude=paths( + "metadata.LastUpdated", + "bulk_upload_report.Date", + "bulk_upload_report.ID", + "bulk_upload_report.Timestamp", + ) + ) + + +@pytest.mark.parametrize( + "description, nhs_number", + usb_reject_test_cases, +) +def test_usb_reject_ingestions( + nhs_number, + description, + snapshot_json, + bulk_upload_table_records, +) -> None: + records_by_num = { + "_description": description, + "metadata": get_entry_from_table_by_nhs_number(nhs_number, bulk_upload_table_records["lloyd_george_records"]), + "bulk_upload_report": get_entry_from_table_by_nhs_number( + nhs_number, bulk_upload_table_records["bulk_upload_report_records"] + ), + "unstitched": get_entry_from_table_by_nhs_number(nhs_number, bulk_upload_table_records["unstitched_records"]), + } + assert records_by_num == snapshot_json( + exclude=paths( + "metadata.LastUpdated", + "bulk_upload_report.Date", + "bulk_upload_report.ID", + "bulk_upload_report.Timestamp", + ) + ) + + +@pytest.mark.parametrize("nhs_number,description", [("123456789", "Prevent Fixed on File and NhsNumber")]) +def test_remap_and_fixed_reject( + nhs_number, + description, + snapshot_json, + bulk_upload_table_records, +) -> None: + records_by_num = { + "_description": description, + "metadata": get_entry_from_table_by_nhs_number(nhs_number, bulk_upload_table_records["lloyd_george_records"]), + "bulk_upload_report": get_entry_from_table_by_nhs_number( + nhs_number, bulk_upload_table_records["bulk_upload_report_records"] + ), + } + assert records_by_num == snapshot_json( + exclude=paths( + "metadata.S3FileKey", + "metadata.ID", + "metadata.Created", + "metadata.FileLocation", + "metadata.LastUpdated", + "bulk_upload_report.Date", + "bulk_upload_report.ID", + "bulk_upload_report.Timestamp", + ) + ) diff --git a/lambdas/tests/e2e/bulk_upload/check/test_scod_bulk_upload.py b/lambdas/tests/e2e/bulk_upload/check/test_scod_bulk_upload.py new file mode 100644 index 000000000..a1d0ae72e --- /dev/null +++ b/lambdas/tests/e2e/bulk_upload/check/test_scod_bulk_upload.py @@ -0,0 +1,74 @@ +from pathlib import Path + +import pytest +from syrupy.filters import paths + +from services.base.dynamo_service import DynamoDBService +from tests.e2e.bulk_upload.conftest import ( + get_all_entries_from_table_by_nhs_number, + read_metadata_csv, + get_entry_from_table_by_nhs_number, +) +from tests.e2e.helpers.data_helper import LloydGeorgeDataHelper + +dynamo_service = DynamoDBService() + +lloyd_george_data_helper = LloydGeorgeDataHelper() + +datadir = Path(__file__).parent.parent / "data" +accept_test_data = read_metadata_csv(datadir, "scod.csv") + +accept_test_cases = [] +for record in accept_test_data: + accept_test_cases.append((record["Test-Description"], record["NHS-NO"])) + + +@pytest.fixture(scope="session", autouse=True) +def bulk_upload_table_records(): + return { + "lloyd_george_records": lloyd_george_data_helper.scan_lloyd_george_table(), + "bulk_upload_report_records": lloyd_george_data_helper.scan_bulk_upload_report_table(), + "unstitched_records": lloyd_george_data_helper.scan_unstitch_table(), + } + + +@pytest.mark.parametrize( + "description, nhs_number", + accept_test_cases, +) +def test_scod_ingestions( + nhs_number, + description, + snapshot_json, + bulk_upload_table_records, +) -> None: + records_by_num = { + "_description": description, + "metadata": get_entry_from_table_by_nhs_number(nhs_number, bulk_upload_table_records["lloyd_george_records"]), + "bulk_upload_report": get_entry_from_table_by_nhs_number( + nhs_number, bulk_upload_table_records["bulk_upload_report_records"] + ), + "unstitched": get_all_entries_from_table_by_nhs_number(nhs_number, + bulk_upload_table_records["unstitched_records"]), + } + assert records_by_num == snapshot_json( + exclude=paths( + "metadata.S3FileKey", + "metadata.ID", + "metadata.Created", + "metadata.FileLocation", + "metadata.LastUpdated", + "metadata.S3VersionID", + "metadata.DocumentScanCreation", + "bulk_upload_report.Date", + "bulk_upload_report.ID", + "bulk_upload_report.Timestamp", + ) + ) + metadata = records_by_num.get("metadata") or {} + if "Reject" not in description: + metadata_s3_key = metadata.get("S3FileKey") + s3_version_id = metadata.get("S3VersionID") + + exists = lloyd_george_data_helper.check_record_exists_in_s3_with_version(metadata_s3_key, s3_version_id) + assert exists diff --git a/lambdas/tests/e2e/bulk_upload/conftest.py b/lambdas/tests/e2e/bulk_upload/conftest.py new file mode 100644 index 000000000..dc4fd296a --- /dev/null +++ b/lambdas/tests/e2e/bulk_upload/conftest.py @@ -0,0 +1,51 @@ +import csv +from pathlib import Path +from typing import List, Dict + +import boto3 + + +def read_metadata_csv(datadir: Path, metadata_file: str) -> List[Dict]: + metadatafile = datadir / metadata_file + with open(metadatafile, newline="") as csvfile: + reader = csv.DictReader(csvfile) + rows = [row for row in reader] + return rows + + +def get_entry_from_table_by_nhs_number(nhs_number, table_entries): + for entry in table_entries: + if entry.get("NhsNumber") == nhs_number: + return entry + return None + + +def get_all_entries_from_table_by_nhs_number(nhs_number, table_entries): + matching_entries = [] + for entry in table_entries: + if entry.get("NhsNumber") == nhs_number: + matching_entries.append(entry) + return matching_entries + + +def empty_table(table_name): + dynamodb = boto3.resource("dynamodb") + table = dynamodb.Table(table_name) + + response = table.scan(ProjectionExpression="ID") + items = response.get("Items", []) + + # Delete initial page + with table.batch_writer() as batch: + for item in items: + batch.delete_item(Key={"ID": item["ID"]}) + + # Continue if paginated + while "LastEvaluatedKey" in response: + response = table.scan(ProjectionExpression="ID", ExclusiveStartKey=response["LastEvaluatedKey"]) + items = response.get("Items", []) + with table.batch_writer() as batch: + for item in items: + batch.delete_item(Key={"ID": item["ID"]}) + + print(f"Emptied table {table_name}") diff --git a/lambdas/tests/e2e/bulk_upload/data/accept-metadata-usb.csv b/lambdas/tests/e2e/bulk_upload/data/accept-metadata-usb.csv new file mode 100644 index 000000000..fc5dbe384 --- /dev/null +++ b/lambdas/tests/e2e/bulk_upload/data/accept-metadata-usb.csv @@ -0,0 +1,3 @@ +Test-Description,FILEPATH,PAGE COUNT,GP-PRACTICE-CODE,NHS-NO,SECTION,SUB-SECTION,SCAN-DATE,SCAN-ID,USER-ID,UPLOAD +USB-Accepted-sent straight to queue (skips validation if already valid),9730153760/1of1_Lloyd_George_Record_[Ruby Sandra MANTON]_[9730153760]_[08-01-2003].pdf,1,M85143,9730153760,LG,,01/01/2023,NEC,NEC,23/11/2023 +USB-Accepted-1of1 in filename,9730153817 Haley Glenda RUDKIN 07-Aug-2023/fgdgdf2f4e4dtd_B_1_Lloyd_George_Record_NOS_1of1_9730153817.pdf,1,M85143,9730153817,LG,,01/01/2023,NEC,NEC,23/11/2023 diff --git a/lambdas/tests/e2e/bulk_upload/data/accept-metadata.csv b/lambdas/tests/e2e/bulk_upload/data/accept-metadata.csv new file mode 100644 index 000000000..c75630d9e --- /dev/null +++ b/lambdas/tests/e2e/bulk_upload/data/accept-metadata.csv @@ -0,0 +1,10 @@ +Test-Description,FILEPATH,PAGE COUNT,GP-PRACTICE-CODE,NHS-NO,SECTION,SUB-SECTION,SCAN-DATE,SCAN-ID,USER-ID,UPLOAD +General-Accepted-Single File Upload,9730787026/1of1_Lloyd_George_Record_[Dora Celia HAMNET]_[9730787026]_[07-07-1926].pdf,1,M85143,9730787026,LG,,01/01/2023,NEC,NEC,21/11/2025 +General-Accepted-Multi File Upload 1of3,9730786976/1of3_Lloyd_George_Record_[Lena Rena LAWLER]_[9730786976]_[24-07-1924].pdf,1,M85143,9730786976,LG,,01/01/2023,NEC,NEC,05/12/2025 +General-Accepted-Multi File Upload 2of3,9730786976/2of3_Lloyd_George_Record_[Lena Rena LAWLER]_[9730786976]_[24-07-1924].pdf,2,M85143,9730786976,LG,,01/01/2023,NEC,NEC,05/12/2025 +General-Accepted-Multi File Upload 3of3,9730786976/3of3_Lloyd_George_Record_[Lena Rena LAWLER]_[9730786976]_[24-07-1924].pdf,3,M85143,9730786976,LG,,01/01/2023,NEC,NEC,05/12/2025 +General-Accepted-Pre-processor LloydGeorge Name Edit,9730786925/1of1_Ll0yd_George_Record_[Harry Adam TWIST]_[9730786925]_[01-04-1925].pdf,1,M85143,9730786925,LG,,01/01/2023,NEC,NEC,05/12/2025 +General-Accepted-Pre-processor Mixed Spacing in Page,01of 01_Lloyd_George_Record_Andrea Gladys HOAKES_9730154775_09-01-2009.pdf,1,H81109,9730154775,LG,,01/01/2023,NEC,NEC,23/11/2023 +General-Accepted-Pre-processor Spacing Nhs Number,01 of 01_Lloyd_George_Record_GILALA NYSSA RUBENSTEIN_973 015 472 4_18-07-2020.pdf,1,H81109,9730154724,LG,,01/01/2023,NEC,NEC,23/11/2023 +General-Accepted-Pre-processor Hyphenated Nhs Number,01 of 01_Lloyd_George_Record_MEIXIANG XIAOLIAN FONG_973-0154-406_27-11-1944.pdf,1,H81109,9730154406,LG,,01/01/2023,NEC,NEC,23/11/2023 +General-Accepted-Pre-processor Mixed brackets,01 of 01_Lloyd George Record_[Tarina Sybill LEVY]_9730154090_14-09-1961.pdf,1,H81109,9730154090,LG,,01/01/2023,NEC,NEC,23/11/2023 diff --git a/lambdas/tests/e2e/bulk_upload/data/accept-remap-and-fixed.csv b/lambdas/tests/e2e/bulk_upload/data/accept-remap-and-fixed.csv new file mode 100644 index 000000000..cf01d4675 --- /dev/null +++ b/lambdas/tests/e2e/bulk_upload/data/accept-remap-and-fixed.csv @@ -0,0 +1,2 @@ +Test-Description,file,PAGE COUNT,nhsnumber,SECTION,SUB-SECTION,SCAN-DATE,SCAN-ID,USER-ID,UPLOAD +Remap and Fixed,test/9730787492/1of1_Lloyd_George_Record_[Bob SPEAR]_[9730787492]_[05-08-1968].pdf,1,9730787492,LG,,01/01/2023,NEC,NEC,21/11/2025 diff --git a/lambdas/tests/e2e/bulk_upload/data/pdf/valid.pdf b/lambdas/tests/e2e/bulk_upload/data/pdf/valid.pdf new file mode 100644 index 000000000..1ba3baa9a Binary files /dev/null and b/lambdas/tests/e2e/bulk_upload/data/pdf/valid.pdf differ diff --git a/lambdas/tests/e2e/bulk_upload/data/reject-fixed.csv b/lambdas/tests/e2e/bulk_upload/data/reject-fixed.csv new file mode 100644 index 000000000..239b48956 --- /dev/null +++ b/lambdas/tests/e2e/bulk_upload/data/reject-fixed.csv @@ -0,0 +1,2 @@ +Test-Description,FILEPATH,PAGE COUNT,NHS-NO,SECTION,SUB-SECTION,scandate,SCAN-ID,USER-ID,UPLOAD +Prevent Fixed on File and NhsNumber,test/9730787603/1of1_Lloyd_George_Record_[Earl Noel MCGLUE]_[9730787603]_[26-06-1963].pdf,1,M85143,9730787603,LG,,01/01/2023,NEC,NEC,21/11/2025 diff --git a/lambdas/tests/e2e/bulk_upload/data/reject-metadata-usb.csv b/lambdas/tests/e2e/bulk_upload/data/reject-metadata-usb.csv new file mode 100644 index 000000000..88edd5eaa --- /dev/null +++ b/lambdas/tests/e2e/bulk_upload/data/reject-metadata-usb.csv @@ -0,0 +1,12 @@ +Test-Description,FILEPATH,PAGE COUNT,GP-PRACTICE-CODE,NHS-NO,SECTION,SUB-SECTION,SCAN-DATE,SCAN-ID,USER-ID,UPLOAD +USB-Rejected-File extension .tiff is not supported,9730153752 Dawn FLING 19-Sep-2011/fgdgdf2f4e4dtd_B_1_Lloyd_George_tiff_NOS_9730153752.tiff,1,M85143,9000000001,LG,,01/01/2023,NEC,NEC,23/11/2023 +USB-Rejected-Multi-part documents are not supported,9730153973 Sue Lara TEMPLE 25-Jul-2010/fgdgdf2f4e4dtd_B_1_Lloyd_George_Record_NOS_1of7_9730153973.pdf,1,M85143,9000000003,LG,,01/01/2023,NEC,NEC,23/11/2023 +USB-Rejected-Could not find a valid date in the filename?,9730153819 07-Aug-2020/fgdgdf2f4e4dtd_B_1_Lloyd_George_Record_NOS_1of1_9000000005.pdf,1,M85143,9000000005,LG,,01/01/2023,NEC,NEC,23/11/2023 +USB-Rejected-Invalid patient name,9730153 Louisa Gables 07-Nov-1999/fgdgdf2f4e4dtd_B_1_Lloyd_George_Record_NOS_1of1_9000000008.pdf,1,M85143,9000000008,LG,,01/01/2023,NEC,NEC,23/11/2023 +USB-Rejected-Invalid date format,1234567895 Lillian Percy 07-24-1999/fgdgdf2f4e4dtd_B_1_Lloyd_George_Record_NOS_1of1_9000000044.pdf,1,M85143,9000000044,LG,,01/01/2023,NEC,NEC,23/11/2023 +USB-Rejected-Could not find a valid date in the filename.,1253617284 Adam Thomas 07-Nov-99/fgdgdf2f4e4dtd_B_1_Lloyd_George_Record_NOS_1of1_9000000055.pdf,1,M85143,9000000055,LG,,01/01/2023,NEC,NEC,23/11/2023 +USB-Rejected-no space between name and dob,9725362748 Geraldine Ellis07-Sep-1999/fgdgdf2f4e4dtd_B_1_Lloyd_George_Record_NOS_1of1_9000000066.pdf,1,M85143,9000000066,LG,,01/01/2023,NEC,NEC,23/11/2023 +USB-Rejected-Invalid NHS number,Stewart Smith 07-Sep-1998/fgdgdf2f4e4dtd_B_1_Lloyd_George_Record_NOS_1of1_9000000088.pdf,1,M85143,9000000088,LG,,01/01/2023,NEC,NEC,23/11/2023 +USB-Rejected-Could not find a valid date in the filename.,2516273843 Fiona Smith/fgdgdf2f4e4dtd_B_1_Lloyd_George_Record_NOS_1of1_9000000099.pdf,1,M85143,9000000099,LG,,01/01/2023,NEC,NEC,23/11/2023 +USB-Rejected-Could not find a valid date in the filename.,1232323232 12-Aug-1999 Catherine Higgins/fgdgdf2f4e4dtd_B_1_Lloyd_George_Record_NOS_1of1_9000000045.pdf,1,M85143,9000000045,LG,,01/01/2023,NEC,NEC,23/11/2023 +USB-Rejected-Invalid patient name,12-Aug-1944 Catherine Higgins 1992323232/fgdgdf2f4e4dtd_B_1_Lloyd_George_Record_NOS_1of1_9000000067.pdf,1,M85143,9000000067,LG,,01/01/2023,NEC,NEC,23/11/2023 diff --git a/lambdas/tests/e2e/bulk_upload/data/reject-metadata.csv b/lambdas/tests/e2e/bulk_upload/data/reject-metadata.csv new file mode 100644 index 000000000..cb919029f --- /dev/null +++ b/lambdas/tests/e2e/bulk_upload/data/reject-metadata.csv @@ -0,0 +1,11 @@ +Test-Description,FILEPATH,PAGE COUNT,GP-PRACTICE-CODE,NHS-NO,SECTION,SUB-SECTION,SCAN-DATE,SCAN-ID,USER-ID,UPLOAD +General-Rejected-invalid date format - Invalid Month,01 of 02_Lloyd_George_Record_Steph Stevens_9000000001_12.25.2010.pdf,1,H81109,9000000101,LG,,01/01/2023,NEC,NEC,23/11/2023 +General-Rejected-incorrect NHS number or date format - Invalid Format,01 of 02_Lloyd_George_Record_Andrew Stevens_9000000001_12.10.21.pdf,1,H81109,9000000201,LG,,01/01/2023,NEC,NEC,23/11/2023 +General-Rejected-Invalid date format,01 of 02_Lloyd_George_Record_Jaimie Stevens_9000000001_300.4.2010.pdf,1,H81109,9000000051,LG,,01/01/2023,NEC,NEC,23/11/2023 +General-Rejected-Invalid NHS number,01 of 02_Lloyd_George_Record_Jane Stevens_900_000_0001_11.10.2010.pdf,1,H81109,9000000061,LG,,01/01/2023,NEC,NEC,23/11/2023 +General-Rejected-incorrect NHS number or date format,01 of 02_Lloyd_George_Record_Patricia Stevens_9000000001_2_11.10.2010.pdf,1,H81109,9000000071,LG,,01/01/2023,NEC,NEC,23/11/2023 +General-Rejected-incorrect document path format,103/01 02_Lloyd_George_Record_Charles Stevens_9000000001_22.10.2010.pdf,1,H81109,9000000081,LG,,01/01/2023,NEC,NEC,23/11/2023 +General-Rejected-Missing 'lloyd george record',1093/01 of 02_Phineas Stevens_9000000001_22.10.2010.pdf,1,H81109,9500000001,LG,,01/01/2023,NEC,NEC,23/11/2023 +General-Rejected-Nhs Not in PDS,1of1_Lloyd_George_Record_[Josh Lucien PLATT]_[9596333437]_[23-10-1979].pdf,1,M85143,9596333437,LG,,01/01/2023,NEC,NEC,29/10/2025 +General-Rejected-Infected,1of1_Lloyd_George_Record_[Moses Ivan WELLS]_[9730788529]_[20-03-1994].pdf,1,M85143,9730788529,LG,,01/01/2023,NEC,NEC,29/10/2025 +General-Rejected-Corrupted,test/9730788022/1of1_Lloyd_George_Record_[Cath YOUDS]_[9730788022]_[05-05-1965].pdf,1,M85143,9730788022,LG,,01/01/2023,NEC,NEC,21/11/2025 diff --git a/lambdas/tests/e2e/bulk_upload/data/scod.csv b/lambdas/tests/e2e/bulk_upload/data/scod.csv new file mode 100644 index 000000000..6568f2595 --- /dev/null +++ b/lambdas/tests/e2e/bulk_upload/data/scod.csv @@ -0,0 +1,3 @@ +Test-Description,FILEPATH,NHS-NO +SCOD-Accept,expedite/1of1_Lloyd_George_Record_[Felix Alfred VOS]_[9730787158]_[03-01-2016].pdf,9730787158 +SCOD-Reject,expedite/1of1_Lloyd_George_Record_[Felix Alfred VOS]_[9730787159]_[03-01-2016].pdf,9730787159 diff --git a/lambdas/tests/e2e/bulk_upload/test_bulk_upload_300_output.py b/lambdas/tests/e2e/bulk_upload/test_bulk_upload_300_output.py deleted file mode 100644 index 1b4ee1415..000000000 --- a/lambdas/tests/e2e/bulk_upload/test_bulk_upload_300_output.py +++ /dev/null @@ -1,62 +0,0 @@ -from services.base.dynamo_service import DynamoDBService -from tests.e2e.conftest import BULK_REPORT_TABLE, LG_METADATA_TABLE, LG_UNSTITCHED_TABLE - -dynamo_service = DynamoDBService() - - -def test_bulk_upload_300_patients_with_3_files(): - - # assert bulk upload report table values - bulk_upload_table = dynamo_service.scan_whole_table( - BULK_REPORT_TABLE, "Reason, UploadStatus, NhsNumber" - ) - assert len(bulk_upload_table) == 868 - - complete_uploads = [] - failed_uploads = [] - name_mismatch_rejections = [] - dob_mismatch_rejections = [] - - for item in bulk_upload_table: - if item.get("UploadStatus") == "complete": - complete_uploads.append(item) - elif item.get("UploadStatus") == "failed": - failed_uploads.append(item) - if item.get("Reason") == "Patient name does not match our records": - name_mismatch_rejections.append(item) - elif item.get("Reason") == "Patient DoB does not match our records": - dob_mismatch_rejections.append(item) - - assert len(complete_uploads) == 765 - assert len(failed_uploads) == 103 - assert len(name_mismatch_rejections) == 6 - assert len(dob_mismatch_rejections) == 27 - - # assert lloyd george metadata values, this will also validate the files were stitched - lg_metadata_table = dynamo_service.scan_whole_table( - LG_METADATA_TABLE, "CurrentGpOds, NhsNumber" - ) - assert len(lg_metadata_table) == 255 - - dece_records = [] - susp_records = [] - rest_records = [] - - for item in lg_metadata_table: - ods_code = item.get("CurrentGpOds") - if ods_code == "DECE": - dece_records.append(item) - elif ods_code == "SUSP": - susp_records.append(item) - elif ods_code == "REST": - rest_records.append(item) - - assert len(dece_records) == 4 - assert len(susp_records) == 4 - assert len(rest_records) == 9 - - # assert unstitched metadata contains original unstitched files - lg_unstitched_metadata_table = dynamo_service.scan_whole_table( - LG_UNSTITCHED_TABLE, "CurrentGpOds" - ) - assert len(lg_unstitched_metadata_table) == 765 diff --git a/lambdas/tests/e2e/bulk_upload/upload/test_initiate_bulk_upload.py b/lambdas/tests/e2e/bulk_upload/upload/test_initiate_bulk_upload.py new file mode 100644 index 000000000..f6983d7d5 --- /dev/null +++ b/lambdas/tests/e2e/bulk_upload/upload/test_initiate_bulk_upload.py @@ -0,0 +1,179 @@ +import tempfile +from pathlib import Path + +import boto3 +import pytest + +from tests.e2e.bulk_upload.conftest import empty_table +from tests.e2e.bulk_upload.conftest import read_metadata_csv +from tests.e2e.helpers.data_helper import LloydGeorgeDataHelper + +s3_client = boto3.client("s3") +lambda_client = boto3.client("lambda") +datadir = Path(__file__).parent.parent / "data" + +lloyd_george_helper = LloydGeorgeDataHelper() + + +@pytest.fixture(scope="module", autouse=True) +def cleanup_databases(): + empty_table(lloyd_george_helper.bulk_upload_table) + empty_table(lloyd_george_helper.unstitched_table) + empty_table(lloyd_george_helper.dynamo_table) + + +def upload_record(record: str, pdf_path: Path, prefix=None, infected=False): + s3_key = (prefix + "/" + record) if prefix else record + + if infected: + virus_scanner_result_value = "Infected" + else: + virus_scanner_result_value = "Clean" + virus_scanner_date_value = "2023-11-23T15:50:33Z" + + with open(pdf_path, "rb") as f: + lloyd_george_helper.upload_to_staging_directory(s3_key, f.read()) + lloyd_george_helper.add_virus_scan_tag(s3_key, virus_scanner_result_value, virus_scanner_date_value) + + +def generate_corrupt_file(source_file: Path) -> Path: + data = bytearray(source_file.read_bytes()) + + for offset in range(min(3, len(data))): + data[offset] ^= 0xFF + + _, tmp_path = tempfile.mkstemp(suffix=source_file.suffix) + Path(tmp_path).write_bytes(data) + + return Path(tmp_path) + + +def upload_corrupted_record(record: str, pdf_path: Path, prefix="reject"): + s3_key = prefix + "/" + record + path = generate_corrupt_file(pdf_path) + + virus_scanner_result_value = "Clean" + virus_scanner_date_value = "2023-11-23T15:50:33Z" + + with open(path, "rb") as f: + lloyd_george_helper.upload_to_staging_directory(s3_key, f.read()) + lloyd_george_helper.add_virus_scan_tag(s3_key, virus_scanner_result_value, virus_scanner_date_value) + + +def run_bulk_upload(metadata_filename, pre_format_type="general"): + payload = {"inputFileLocation": metadata_filename, "preFormatType": pre_format_type} + response = lloyd_george_helper.run_bulk_upload(payload=payload) + return response + + +def run_bulk_upload_with_remappings_and_fixes(metada_filename, remappings, fixed_values): + payload = {"inputFileLocation": metada_filename, "metadataFieldRemappings": remappings, "fixedValues": fixed_values} + response = lloyd_george_helper.run_bulk_upload(payload=payload) + return response + + +def test_run_bulk_upload_accepted(): + rows = read_metadata_csv(datadir, "accept-metadata.csv") + for row in rows: + pdf_filepath = datadir / "pdf" / "valid.pdf" + upload_record(row["FILEPATH"], pdf_filepath, "accept") + metadata_filepath = datadir / "accept-metadata.csv" + s3_key = "accept/index.csv" + + with open(metadata_filepath, "rb") as file: + file_content = file.read() + lloyd_george_helper.upload_to_staging_directory(s3_key, file_content) + run_bulk_upload(s3_key) + + +def test_run_bulk_upload_accepted_usb(): + rows = read_metadata_csv(datadir, "accept-metadata-usb.csv") + for row in rows: + pdf_filepath = datadir / "pdf" / "valid.pdf" + upload_record(row["FILEPATH"], pdf_filepath, "accept-usb") + metadata_filepath = datadir / "accept-metadata-usb.csv" + s3_key = "accept-usb/index.csv" + + with open(metadata_filepath, "rb") as file: + file_content = file.read() + lloyd_george_helper.upload_to_staging_directory(s3_key, file_content) + run_bulk_upload(s3_key, "usb") + + +def test_run_bulk_upload_rejected(): + rows = read_metadata_csv(datadir, "reject-metadata.csv") + for row in rows: + pdf_filepath = datadir / "pdf" / "valid.pdf" + if "Infected" in row["Test-Description"]: + upload_record(row["FILEPATH"], pdf_filepath, "reject", True) + elif "Corrupted" in row["Test-Description"]: + upload_corrupted_record(row["FILEPATH"], pdf_filepath, "reject") + else: + upload_record(row["FILEPATH"], pdf_filepath, "reject") + metadata_filepath = datadir / "reject-metadata.csv" + s3_key = "reject/index.csv" + + with open(metadata_filepath, "rb") as file: + file_content = file.read() + lloyd_george_helper.upload_to_staging_directory(s3_key, file_content) + + run_bulk_upload(s3_key) + + +def test_run_bulk_upload_rejected_usb(): + rows = read_metadata_csv(datadir, "reject-metadata-usb.csv") + for row in rows: + pdf_filepath = datadir / "pdf" / "valid.pdf" + upload_record(row["FILEPATH"], pdf_filepath, "reject-usb") + metadata_filepath = datadir / "reject-metadata-usb.csv" + s3_key = "reject-usb/index.csv" + + with open(metadata_filepath, "rb") as file: + file_content = file.read() + lloyd_george_helper.upload_to_staging_directory(s3_key, file_content) + run_bulk_upload(s3_key, "usb") + + +def test_run_bulk_upload_remap_and_fixed(): + rows = read_metadata_csv(datadir, "accept-remap-and-fixed.csv") + for row in rows: + pdf_filepath = datadir / "pdf" / "valid.pdf" + upload_record(row["file"], pdf_filepath, "accept-remap-and-fixed") + metadata_filepath = datadir / "accept-remap-and-fixed.csv" + s3_key = "accept-remap-and-fixed/metadata.csv" + + with open(metadata_filepath, "rb") as file: + file_content = file.read() + lloyd_george_helper.upload_to_staging_directory(s3_key, file_content) + + metdata_field_remappings = { + "NHS-NO": "nhsnumber", + "FILEPATH": "file", + } + fixed_values = {"GP-PRACTICE-CODE": "M85143", "SCAN-DATE": "01/01/2022"} + run_bulk_upload_with_remappings_and_fixes(s3_key, metdata_field_remappings, fixed_values) + + +def test_run_bulk_upload_fixed_reject(): + rows = read_metadata_csv(datadir, "reject-fixed.csv") + for row in rows: + pdf_filepath = datadir / "pdf" / "valid.pdf" + upload_record(row["FILEPATH"], pdf_filepath, "reject-fixed") + metadata_filepath = datadir / "reject-fixed.csv" + s3_key = "reject-fixed/index.csv" + + with open(metadata_filepath, "rb") as file: + file_content = file.read() + lloyd_george_helper.upload_to_staging_directory(s3_key, file_content) + + metdata_field_remappings = {} + + fixed_values = {"NHS-NO": "123456789", "FILEPATH": "myfile.pdf"} + run_bulk_upload_with_remappings_and_fixes(s3_key, metdata_field_remappings, fixed_values) + + +def test_run_bulk_upload_scod(): + rows = read_metadata_csv(datadir, "scod.csv") + for row in rows: + pdf_filepath = datadir / "pdf" / "valid.pdf" + upload_record(row["FILEPATH"], pdf_filepath) diff --git a/lambdas/tests/e2e/conftest.py b/lambdas/tests/e2e/conftest.py index 49dcbc54e..1612542af 100644 --- a/lambdas/tests/e2e/conftest.py +++ b/lambdas/tests/e2e/conftest.py @@ -4,6 +4,7 @@ import pytest import requests from syrupy.extensions.json import JSONSnapshotExtension + from tests.e2e.helpers.data_helper import LloydGeorgeDataHelper data_helper = LloydGeorgeDataHelper() @@ -11,10 +12,10 @@ LLOYD_GEORGE_SNOMED = data_helper.snomed_code API_ENDPOINT = data_helper.api_endpoint API_KEY = os.environ.get("NDR_API_KEY") + LG_METADATA_TABLE = data_helper.dynamo_table -LG_UNSTITCHED_TABLE = os.environ.get("LG_UNSTITCHED_TABLE") -BULK_REPORT_TABLE = os.environ.get("BULK_REPORT_TABLE") LLOYD_GEORGE_S3_BUCKET = data_helper.s3_bucket + APIM_ENDPOINT = data_helper.apim_url diff --git a/lambdas/tests/e2e/helpers/data_helper.py b/lambdas/tests/e2e/helpers/data_helper.py index 20656bb88..8ce8e46cd 100644 --- a/lambdas/tests/e2e/helpers/data_helper.py +++ b/lambdas/tests/e2e/helpers/data_helper.py @@ -4,18 +4,20 @@ import os import uuid +import boto3 from enums.snomed_codes import SnomedCodes + from services.base.dynamo_service import DynamoDBService from services.base.s3_service import S3Service class DataHelper: def __init__( - self, - table_name: str, - bucket_name: str, - snomed_code: str, - record_type: str, + self, + table_name: str, + bucket_name: str, + snomed_code: str, + record_type: str, ): self.workspace = os.environ.get("AWS_WORKSPACE", "") self.dynamo_table = None @@ -41,9 +43,7 @@ def build_env(self, table_name, bucket_name): "ndr-test": "internal-qa.api.service.nhs.uk", "ndr-dev": "internal-dev.api.service.nhs.uk", } - self.apim_url = apim_map.get( - str(self.workspace), "internal-dev.api.service.nhs.uk" - ) + self.apim_url = apim_map.get(str(self.workspace), "internal-dev.api.service.nhs.uk") domain = ( "national-document-repository.nhs.uk" @@ -52,17 +52,13 @@ def build_env(self, table_name, bucket_name): ) self.api_endpoint = ( - f"api.{self.workspace}.{domain}" - if self.workspace in {"pre-prod", "ndr-test"} - else f"api-{self.workspace}.{domain}" + f"api.{self.workspace}.{domain}" if self.workspace in {"pre-prod", + "ndr-test"} else f"api-{self.workspace}.{domain}" ) self.mtls_endpoint = f"mtls.{self.workspace}.{domain}" - def build_record( - self, nhs_number="9912003071", data=None, doc_status=None, size=None - ): - """Helper to create a PDM record dictionary.""" + def build_record(self, nhs_number="9912003071", data=None, doc_status=None, size=None): record = { "id": str(uuid.uuid4()), "nhs_number": nhs_number, @@ -107,9 +103,7 @@ def create_resource(self, record): ) def retrieve_document_reference(self, record): - return self.dynamo_service.get_item( - table_name=self.dynamo_table, key={"ID": record["id"]} - ) + return self.dynamo_service.get_item(table_name=self.dynamo_table, key={"ID": record["id"]}) def create_upload_payload(self, record, exclude=[], return_json=False): """Helper to build DocumentReference payload.""" @@ -206,9 +200,75 @@ def tidyup(self, record): class LloydGeorgeDataHelper(DataHelper): def __init__(self): + self.bulk_upload_table_name = "BulkUploadReport" + self.metadata_processor_lambda_name = "BulkUploadMetadataProcessor" + self.unstitched_table_name = "UnstitchedLloydGeorgeReferenceMetadata" + self.staging_bucket = "staging-bulk-store" + self.bulk_upload_table = None + self.unstitched_table = None + self.metadata_processor_lambda = None + + self.lambda_client = boto3.client("lambda") + self.s3_client = boto3.client("s3") + super().__init__( "LloydGeorgeReferenceMetadata", "lloyd-george-store", SnomedCodes.LLOYD_GEORGE.value.code, "Lloyd_George_Record", ) + + def build_env(self, table_name, bucket_name): + super().build_env(table_name, bucket_name) + self.bulk_upload_table = f"{self.workspace}_{self.bulk_upload_table_name}" + self.metadata_processor_lambda = f"{self.workspace}_{self.metadata_processor_lambda_name}" + self.unstitched_table = f"{self.workspace}_{self.unstitched_table_name}" + self.staging_bucket = f"{self.workspace}-{self.staging_bucket}" + + def scan_bulk_upload_report_table(self): + return self.dynamo_service.scan_whole_table(self.bulk_upload_table or "") + + def scan_unstitch_table(self): + return self.dynamo_service.scan_whole_table(self.unstitched_table or "") + + def run_bulk_upload(self, payload): + payload = json.dumps(payload) + response = self.lambda_client.invoke( + FunctionName=self.metadata_processor_lambda, InvocationType="RequestResponse", Payload=payload + ) + return response + + def upload_to_staging_directory(self, key, body): + self.s3_service.save_or_create_file(self.staging_bucket, key, body) + + def add_virus_scan_tag(self, key, result, date): + self.s3_client.put_object_tagging( + Bucket=self.staging_bucket, + Key=key, + Tagging={ + "TagSet": [ + {"Key": "scan-result", "Value": result}, + {"Key": "scan-date", "Value": date}, + ] + }, + ) + + def scan_lloyd_george_table(self): + return self.dynamo_service.scan_whole_table(self.dynamo_table or "") + + def check_record_exists_in_s3(self, key): + return self.s3_service.get_head_object(self.s3_bucket or "", key) + + def check_record_exists_in_s3_with_version(self, key, version_id): + s3_client = boto3.client("s3") + try: + if version_id: + _ = s3_client.head_object(Bucket=self.s3_bucket, Key=key, VersionId=version_id) + else: + _ = s3_client.head_object(Bucket=self.s3_bucket, Key=key) + return True + except s3_client.exceptions.ClientError as e: + if e.response["Error"]["Code"] == "404": + return False + else: + raise diff --git a/lambdas/tests/e2e/helpers/lloyd_george_mockcis2_helper.py b/lambdas/tests/e2e/helpers/mockcis2_helper.py similarity index 91% rename from lambdas/tests/e2e/helpers/lloyd_george_mockcis2_helper.py rename to lambdas/tests/e2e/helpers/mockcis2_helper.py index fbd153e3e..aa3bdf739 100644 --- a/lambdas/tests/e2e/helpers/lloyd_george_mockcis2_helper.py +++ b/lambdas/tests/e2e/helpers/mockcis2_helper.py @@ -4,12 +4,13 @@ from urllib.parse import parse_qs, quote, urlparse import requests + from tests.e2e.helpers.data_helper import LloydGeorgeDataHelper data_helper = LloydGeorgeDataHelper() -class LloydGeorgeMockcis2Helper: +class MockCis2Helper: def __init__(self, ods, repository_role): self.mock_key = os.environ.get("MOCK_CIS2_KEY") or "" self.api_endpoint = data_helper.api_endpoint @@ -24,9 +25,7 @@ def generate_mockcis2_token(self): pass def get_state(self): - login_response = requests.get( - f"https://{self.api_endpoint}/Auth/Login", allow_redirects=False - ) + login_response = requests.get(f"https://{self.api_endpoint}/Auth/Login", allow_redirects=False) location = login_response.headers.get("Location") parsed_url = urlparse(location)