[PRMP-631] Bulk Upload E2e Tests

Makefile

@@ -93,11 +93,25 @@ else
 	cd ./lambdas && ./venv/bin/python3 -m pytest tests/e2e/api --ignore=tests/e2e/api/fhir --snapshot-update
 endif
 
+initiate-bulk-upload:
+ifeq ($(CONTAINER), true)
+	cd ./lambdas && PYTHONPATH=. poetry run pytest tests/e2e/bulk_upload/upload -vv
+else
+	cd ./lambdas && ./venv/bin/python3 -m pytest tests/e2e/bulk_upload/upload -vv
+endif
+
 test-bulk-upload-e2e:
 ifeq ($(CONTAINER), true)
-	cd ./lambdas && PYTHONPATH=. poetry run pytest tests/e2e/bulk_upload -vv
+	cd ./lambdas && PYTHONPATH=. poetry run pytest tests/e2e/bulk_upload/check -vv
+else
+	cd ./lambdas && ./venv/bin/python3 -m pytest tests/e2e/bulk_upload/check -vv
+endif
+
+test-bulk-upload-e2e-snapshots:
+ifeq ($(CONTAINER), true)
+	cd ./lambdas && PYTHONPATH=. poetry run pytest tests/e2e/bulk_upload/check --snapshot-update
 else
-	cd ./lambdas && ./venv/bin/python3 -m pytest tests/e2e/bulk_upload -vv
+	cd ./lambdas && ./venv/bin/python3 -m pytest tests/e2e/bulk_upload/check --snapshot-update
 endif
 
 test-unit:

lambdas/tests/e2e/api/fhir/conftest.py

@@ -7,7 +7,7 @@
 import pytest
 import requests
 
-from lambdas.tests.e2e.helpers.data_helper import PdmDataHelper
+from tests.e2e.helpers.patient_record_data_helper import PdmDataHelper
 
 pdm_data_helper = PdmDataHelper()
 
@@ -28,7 +28,7 @@ def test_data():
 
 
 def fetch_with_retry_mtls(
-    session, url, headers, condition_func=None, max_retries=5, delay=10
+        session, url, headers, condition_func=None, max_retries=5, delay=10
 ):
     retries = 0
     while retries < max_retries:
@@ -48,7 +48,7 @@ def fetch_with_retry_mtls(
 
 
 def create_mtls_session(
-    client_cert_path=CLIENT_CERT_PATH, client_key_path=CLIENT_KEY_PATH
+        client_cert_path=CLIENT_CERT_PATH, client_key_path=CLIENT_KEY_PATH
 ):
     session = requests.Session()
     session.cert = (client_cert_path, client_key_path)
@@ -105,10 +105,10 @@ def get_pdm_document_reference(record_id, client_cert_path=None, client_key_path
 
 
 def create_and_store_pdm_record(
-    test_data,
-    nhs_number: str = "9912003071",
-    doc_status: str | None = None,
-    size: int | None = None,
+        test_data,
+        nhs_number: str = "9912003071",
+        doc_status: str | None = None,
+        size: int | None = None,
 ):
     """Helper to create metadata and resource for a record."""
     record = pdm_data_helper.build_record(

lambdas/tests/e2e/api/fhir/test_mtls_routing.py

@@ -1,7 +1,7 @@
 import os
 import uuid
 
-from lambdas.tests.e2e.api.fhir.conftest import get_pdm_document_reference
+from tests.e2e.api.fhir.conftest import get_pdm_document_reference
 
 UNAUTHORISED_CLIENT_CERT_PATH = os.environ.get("UNAUTHORISED_CLIENT_CERT_PATH")
 UNAUTHORISED_CLIENT_KEY_PATH = os.environ.get("UNAUTHORISED_CLIENT_KEY_PATH")

lambdas/tests/e2e/api/fhir/test_retrieve_document_fhir_api.py

@@ -2,13 +2,13 @@
 import uuid
 
 import pytest
-from tests.e2e.helpers.data_helper import PdmDataHelper
 
-from lambdas.tests.e2e.api.fhir.conftest import (
+from tests.e2e.api.fhir.conftest import (
     PDM_S3_BUCKET,
     create_and_store_pdm_record,
     get_pdm_document_reference,
 )
+from tests.e2e.helpers.patient_record_data_helper import PdmDataHelper
 
 pdm_data_helper = PdmDataHelper()
 
@@ -44,7 +44,7 @@ def test_file_retrieval(test_data, file_size):
     ],
 )
 def test_retrieve_edge_cases(
-    record_id, expected_status, expected_code, expected_diagnostics
+        record_id, expected_status, expected_code, expected_diagnostics
 ):
     response = get_pdm_document_reference(record_id)
     assert response.status_code == expected_status

lambdas/tests/e2e/api/fhir/test_search_patient_fhir_api.py

@@ -1,13 +1,13 @@
 import pytest
 
-from lambdas.tests.e2e.api.fhir.conftest import (
+from tests.e2e.api.fhir.conftest import (
     MTLS_ENDPOINT,
     PDM_SNOMED,
     create_and_store_pdm_record,
     create_mtls_session,
 )
-from lambdas.tests.e2e.conftest import APIM_ENDPOINT
-from lambdas.tests.e2e.helpers.data_helper import PdmDataHelper
+from tests.e2e.conftest import APIM_ENDPOINT
+from tests.e2e.helpers.patient_record_data_helper import PdmDataHelper
 
 pdm_data_helper = PdmDataHelper()
 
@@ -67,8 +67,8 @@ def test_search_patient_details(test_data):
 
     attachment_url = matching_entry["resource"]["content"][0]["attachment"]["url"]
     assert (
-        f"https://{APIM_ENDPOINT}/national-document-repository/FHIR/R4/DocumentReference/{PDM_SNOMED}~{expected_record_id}"
-        in attachment_url
+            f"https://{APIM_ENDPOINT}/national-document-repository/FHIR/R4/DocumentReference/{PDM_SNOMED}~{expected_record_id}"
+            in attachment_url
     )
 
 
@@ -107,7 +107,7 @@ def test_multiple_cancelled_search_patient_details(test_data):
     ],
 )
 def test_search_edge_cases(
-    nhs_number, expected_status, expected_code, expected_diagnostics
+        nhs_number, expected_status, expected_code, expected_diagnostics
 ):
     response = search_document_reference(nhs_number)
     assert response.status_code == expected_status

lambdas/tests/e2e/api/fhir/test_upload_document_fhir_api_failure.py

@@ -6,12 +6,12 @@
 import pytest
 import requests
 
-from lambdas.tests.e2e.api.fhir.conftest import (
+from tests.e2e.api.fhir.conftest import (
     MTLS_ENDPOINT,
     create_mtls_session,
     fetch_with_retry_mtls,
 )
-from lambdas.tests.e2e.helpers.data_helper import PdmDataHelper
+from tests.e2e.helpers.patient_record_data_helper import PdmDataHelper
 
 pdm_data_helper = PdmDataHelper()
 
@@ -91,21 +91,21 @@ def condition(response_json):
     "nhs_number,expected_status,expected_code,expected_diagnostics",
     [
         (
-            "9999999993",
-            400,
-            "VALIDATION_ERROR",
-            "Failed to parse document upload request data",
+                "9999999993",
+                400,
+                "VALIDATION_ERROR",
+                "Failed to parse document upload request data",
         ),
         (
-            "123",
-            400,
-            "VALIDATION_ERROR",
-            "Failed to parse document upload request data",
+                "123",
+                400,
+                "VALIDATION_ERROR",
+                "Failed to parse document upload request data",
         ),
     ],
 )
 def test_search_edge_cases(
-    nhs_number, expected_status, expected_code, expected_diagnostics
+        nhs_number, expected_status, expected_code, expected_diagnostics
 ):
     record = {
         "ods": "H81109",
@@ -173,5 +173,5 @@ def test_create_document_with_invalid_author_returns_error(test_data, author_pay
     assert raw_upload_response.status_code == 400
     assert response_json["resourceType"] == "OperationOutcome"
     assert (
-        response_json["issue"][0]["details"]["coding"][0]["code"] == "VALIDATION_ERROR"
+            response_json["issue"][0]["details"]["coding"][0]["code"] == "VALIDATION_ERROR"
     )

lambdas/tests/e2e/api/fhir/test_upload_document_fhir_api_success.py

@@ -2,14 +2,14 @@
 import logging
 import os
 
-from lambdas.tests.e2e.api.fhir.conftest import (
+from tests.e2e.api.fhir.conftest import (
     MTLS_ENDPOINT,
     PDM_SNOMED,
     create_mtls_session,
     fetch_with_retry_mtls,
 )
-from lambdas.tests.e2e.conftest import APIM_ENDPOINT
-from lambdas.tests.e2e.helpers.data_helper import PdmDataHelper
+from tests.e2e.conftest import APIM_ENDPOINT
+from tests.e2e.helpers.patient_record_data_helper import PdmDataHelper
 
 pdm_data_helper = PdmDataHelper()
 
@@ -54,8 +54,8 @@ def test_create_document_base64(test_data):
     upload_response = raw_upload_response.json()
     attachment_url = upload_response["content"][0]["attachment"]["url"]
     assert (
-        f"https://{APIM_ENDPOINT}/national-document-repository/FHIR/R4/DocumentReference/{PDM_SNOMED}~"
-        in attachment_url
+            f"https://{APIM_ENDPOINT}/national-document-repository/FHIR/R4/DocumentReference/{PDM_SNOMED}~"
+            in attachment_url
     )
 
     def condition(response_json):

lambdas/tests/e2e/api/test_login_api.py

@@ -1,6 +1,7 @@
 import pytest
-from tests.e2e.helpers.data_helper import LloydGeorgeDataHelper
+
 from tests.e2e.helpers.lloyd_george_mockcis2_helper import LloydGeorgeMockcis2Helper
+from tests.e2e.helpers.patient_record_data_helper import LloydGeorgeDataHelper
 
 # Note this is testing a mock, but this test is valuable to ensure the login code is working for other tests
 

lambdas/tests/e2e/api/test_retrieve_document_api.py

@@ -3,13 +3,14 @@
 
 import requests
 from syrupy.filters import paths
+
 from tests.e2e.conftest import (
     API_ENDPOINT,
     API_KEY,
     LLOYD_GEORGE_S3_BUCKET,
     LLOYD_GEORGE_SNOMED,
 )
-from tests.e2e.helpers.data_helper import LloydGeorgeDataHelper
+from tests.e2e.helpers.patient_record_data_helper import LloydGeorgeDataHelper
 
 data_helper = LloydGeorgeDataHelper()
 

lambdas/tests/e2e/api/test_search_patient_api.py

@@ -4,8 +4,9 @@
 
 import requests
 from syrupy.filters import paths
+
 from tests.e2e.conftest import API_ENDPOINT, API_KEY, APIM_ENDPOINT, LLOYD_GEORGE_SNOMED
-from tests.e2e.helpers.data_helper import LloydGeorgeDataHelper
+from tests.e2e.helpers.patient_record_data_helper import LloydGeorgeDataHelper
 
 data_helper = LloydGeorgeDataHelper()
 
@@ -33,8 +34,8 @@ def test_search_patient_details(test_data, snapshot_json):
 
     attachment_url = bundle["entry"][0]["resource"]["content"][0]["attachment"]["url"]
     assert (
-        f"https://{APIM_ENDPOINT}/national-document-repository/FHIR/R4/DocumentReference/{LLOYD_GEORGE_SNOMED}~"
-        in attachment_url
+            f"https://{APIM_ENDPOINT}/national-document-repository/FHIR/R4/DocumentReference/{LLOYD_GEORGE_SNOMED}~"
+            in attachment_url
     )
 
     assert bundle == snapshot_json(

lambdas/tests/e2e/api/test_upload_document_api.py

@@ -5,6 +5,7 @@
 
 import requests
 from syrupy.filters import paths
+
 from tests.e2e.conftest import (
     API_ENDPOINT,
     API_KEY,
@@ -13,7 +14,7 @@
     LLOYD_GEORGE_SNOMED,
     fetch_with_retry,
 )
-from tests.e2e.helpers.data_helper import LloydGeorgeDataHelper
+from tests.e2e.helpers.patient_record_data_helper import LloydGeorgeDataHelper
 
 data_helper = LloydGeorgeDataHelper()
 
@@ -97,8 +98,8 @@ def condition(response_json):
 
     attachment_url = upload_response["content"][0]["attachment"]["url"]
     assert (
-        f"https://{APIM_ENDPOINT}/national-document-repository/FHIR/R4/DocumentReference/{LLOYD_GEORGE_SNOMED}~"
-        in attachment_url
+            f"https://{APIM_ENDPOINT}/national-document-repository/FHIR/R4/DocumentReference/{LLOYD_GEORGE_SNOMED}~"
+            in attachment_url
     )
 
     base64_data = retrieve_response["content"][0]["attachment"]["data"]

lambdas/tests/e2e/apim/test_apim_client.py

@@ -4,7 +4,9 @@
 
 import pytest
 import requests
-from tests.e2e.helpers.data_helper import PdmDataHelper
+
+from tests.e2e.conftest import PDM_SNOMED
+from tests.e2e.helpers.patient_record_data_helper import PdmDataHelper
 
 pdm_data_helper = PdmDataHelper()
 PDM_SNOMED = pdm_data_helper.snomed_code
@@ -16,9 +18,7 @@ def test_ping(nhsd_apim_proxy_url):
 
 
 @pytest.mark.nhsd_apim_authorization(access="application", level="level3")
-def test_app_level3_access_search(
-    test_data, nhsd_apim_proxy_url, nhsd_apim_auth_headers
-):
+def test_app_level3_access_search(test_data, nhsd_apim_proxy_url, nhsd_apim_auth_headers):
     # Put PDM record
     pdm_record = pdm_data_helper.build_record()
     test_data.append(pdm_record)
@@ -28,8 +28,7 @@ def test_app_level3_access_search(
 
     # Search record via APIM
     resp = requests.get(
-        nhsd_apim_proxy_url
-        + "/DocumentReference?subject:identifier=https://fhir.nhs.uk/Id/nhs-number%7C9912003071",
+        nhsd_apim_proxy_url + "/DocumentReference?subject:identifier=https://fhir.nhs.uk/Id/nhs-number%7C9912003071",
         headers=nhsd_apim_auth_headers,
     )
     assert resp.status_code == 200
@@ -49,9 +48,7 @@ def test_app_level3_access_search(
 
 
 @pytest.mark.nhsd_apim_authorization(access="application", level="level3")
-def test_app_level3_access_retrieve(
-    test_data, nhsd_apim_proxy_url, nhsd_apim_auth_headers
-):
+def test_app_level3_access_retrieve(test_data, nhsd_apim_proxy_url, nhsd_apim_auth_headers):
     # Put PDM record
     pdm_record = pdm_data_helper.build_record()
     test_data.append(pdm_record)
@@ -77,9 +74,7 @@ def test_app_level3_access_retrieve(
 
 
 @pytest.mark.nhsd_apim_authorization(access="application", level="level3")
-def test_app_level3_access_upload(
-    test_data, nhsd_apim_proxy_url, nhsd_apim_auth_headers
-):
+def test_app_level3_access_upload(test_data, nhsd_apim_proxy_url, nhsd_apim_auth_headers):
     # Build PDM record
     sample_pdf_path = os.path.join(os.path.dirname(__file__), "files", "dummy.pdf")
     with open(sample_pdf_path, "rb") as f: