[NDR-289] E2E tests for Raw data field. (#857)

Author: jameslinnell

lambdas/tests/e2e/api/fhir/conftest.py

@@ -7,7 +7,7 @@
 import pytest
 import requests
 
-from lambdas.tests.e2e.helpers.pdm_data_helper import PdmDataHelper
+from lambdas.tests.e2e.helpers.data_helper import PdmDataHelper
 
 pdm_data_helper = PdmDataHelper()
 

lambdas/tests/e2e/api/fhir/test_retrieve_document_fhir_api.py

@@ -4,7 +4,7 @@
 
 import pytest
 import requests
-from tests.e2e.helpers.pdm_data_helper import PdmDataHelper
+from tests.e2e.helpers.data_helper import PdmDataHelper
 
 from lambdas.tests.e2e.api.fhir.conftest import (
     MTLS_ENDPOINT,

lambdas/tests/e2e/api/fhir/test_search_patient_fhir_api.py

@@ -10,7 +10,7 @@
     create_mtls_session,
 )
 from lambdas.tests.e2e.conftest import APIM_ENDPOINT
-from lambdas.tests.e2e.helpers.pdm_data_helper import PdmDataHelper
+from lambdas.tests.e2e.helpers.data_helper import PdmDataHelper
 
 pdm_data_helper = PdmDataHelper()
 
@@ -41,7 +41,9 @@ def search_document_reference(nhs_number):
     return session.get(url, headers=headers)
 
 
-def create_and_store_record(test_data, nhs_number="9912003071", doc_status=None):
+def create_and_store_record(
+    test_data, nhs_number="9912003071", doc_status: str | None = None
+):
     """Helper to create metadata and resource for a record."""
     record = build_pdm_record(nhs_number=nhs_number, doc_status=doc_status)
     test_data.append(record)
@@ -61,7 +63,7 @@ def test_search_patient_details(test_data):
 
     attachment_url = bundle["entry"][0]["resource"]["content"][0]["attachment"]["url"]
     assert (
-        f"https://{APIM_ENDPOINT}/national-document-repository/DocumentReference/{PDM_SNOMED}~"
+        f"https://{APIM_ENDPOINT}/national-document-repository/FHIR/R4/DocumentReference/{PDM_SNOMED}~"
         in attachment_url
     )
 

lambdas/tests/e2e/api/fhir/test_upload_document_fhir_api.py

@@ -12,9 +12,10 @@
     create_mtls_session,
     fetch_with_retry_mtls,
 )
+from lambdas.tests.e2e.helpers.data_helper import PdmDataHelper
 
 
-def create_upload_payload(pdm_record):
+def create_upload_payload(record):
     """Helper to build DocumentReference payload."""
     payload = {
         "resourceType": "DocumentReference",
@@ -30,21 +31,21 @@ def create_upload_payload(pdm_record):
         "subject": {
             "identifier": {
                 "system": "https://fhir.nhs.uk/Id/nhs-number",
-                "value": pdm_record["nhs_number"],
+                "value": record["nhs_number"],
             }
         },
         "author": [
             {
                 "identifier": {
                     "system": "https://fhir.nhs.uk/Id/ods-organization-code",
-                    "value": pdm_record["ods"],
+                    "value": record["ods"],
                 }
             }
         ],
         "custodian": {
             "identifier": {
                 "system": "https://fhir.nhs.uk/Id/ods-organization-code",
-                "value": pdm_record["ods"],
+                "value": record["ods"],
             }
         },
         "content": [
@@ -53,13 +54,13 @@ def create_upload_payload(pdm_record):
                     "creation": "2023-01-01",
                     "contentType": "application/pdf",
                     "language": "en-GB",
-                    "title": "1of1_pdm_record_[Paula Esme VESEY]_[9730153973]_[22-01-1960].pdf",
+                    "title": "1of1_record_[Paula Esme VESEY]_[9730153973]_[22-01-1960].pdf",
                 }
             }
         ],
     }
-    if "data" in pdm_record:
-        payload["content"][0]["attachment"]["data"] = pdm_record["data"]
+    if "data" in record:
+        payload["content"][0]["attachment"]["data"] = record["data"]
     return json.dumps(payload)
 
 
@@ -84,15 +85,15 @@ def retrieve_document_with_retry(session, doc_id, condition):
 
 
 def test_create_document_base64(test_data):
-    pdm_record = {
+    record = {
         "ods": "H81109",
         "nhs_number": "9912003071",
     }
 
     sample_pdf_path = os.path.join(os.path.dirname(__file__), "files", "dummy.pdf")
     with open(sample_pdf_path, "rb") as f:
-        pdm_record["data"] = base64.b64encode(f.read()).decode("utf-8")
-    payload = create_upload_payload(pdm_record)
+        record["data"] = base64.b64encode(f.read()).decode("utf-8")
+    payload = create_upload_payload(record)
 
     session = create_mtls_session()
     raw_upload_response = upload_document(session, payload)
@@ -103,8 +104,8 @@ def test_create_document_base64(test_data):
     attachment_url = upload_response["content"][0]["attachment"]["url"]
     assert f"/DocumentReference/{PDM_SNOMED}~" in attachment_url
 
-    pdm_record["id"] = raw_upload_response.json()["id"].split("~")[1]
-    test_data.append(pdm_record)
+    record["id"] = raw_upload_response.json()["id"].split("~")[1]
+    test_data.append(record)
 
     def condition(response_json):
         logging.info(response_json)
@@ -121,15 +122,15 @@ def condition(response_json):
 
 
 def test_create_document_presign_fails():
-    pdm_record = {
+    record = {
         "ods": "H81109",
         "nhs_number": "9912003071",
     }
 
     sample_pdf_path = os.path.join(os.path.dirname(__file__), "files", "big-dummy.pdf")
     with open(sample_pdf_path, "rb") as f:
-        pdm_record["data"] = base64.b64encode(f.read()).decode("utf-8")
-    payload = create_upload_payload(pdm_record)
+        record["data"] = base64.b64encode(f.read()).decode("utf-8")
+    payload = create_upload_payload(record)
 
     session = create_mtls_session()
     upload_response = upload_document(session, payload)
@@ -138,18 +139,18 @@ def test_create_document_presign_fails():
 
 
 def test_create_document_virus(test_data):
-    pdm_record = {
+    record = {
         "ods": "H81109",
         "nhs_number": "9730154260",
     }
-    payload = create_upload_payload(pdm_record)
+    payload = create_upload_payload(record)
     session = create_mtls_session()
 
     retrieve_response = upload_document(session, payload)
     upload_response = retrieve_response.json()
 
-    pdm_record["id"] = upload_response["id"].split("~")[1]
-    test_data.append(pdm_record)
+    record["id"] = upload_response["id"].split("~")[1]
+    test_data.append(record)
 
     # Presigned upload
     presign_uri = upload_response["content"][0]["attachment"]["url"]
@@ -191,15 +192,15 @@ def condition(response_json):
 def test_search_edge_cases(
     nhs_number, expected_status, expected_code, expected_diagnostics
 ):
-    pdm_record = {
+    record = {
         "ods": "H81109",
         "nhs_number": f"{nhs_number}",
     }
 
     sample_pdf_bytes = b"Sample PDF Content"
-    pdm_record["data"] = base64.b64encode(sample_pdf_bytes).decode("utf-8")
+    record["data"] = base64.b64encode(sample_pdf_bytes).decode("utf-8")
 
-    payload = create_upload_payload(pdm_record)
+    payload = create_upload_payload(record)
     session = create_mtls_session()
     response = upload_document(session, payload)
     assert response.status_code == expected_status
@@ -213,15 +214,15 @@ def test_search_edge_cases(
 
 
 def test_forbidden_with_invalid_cert(temp_cert_and_key):
-    pdm_record = {
+    record = {
         "ods": "H81109",
         "nhs_number": "9912003071",
     }
 
     sample_pdf_bytes = b"Sample PDF Content"
-    pdm_record["data"] = base64.b64encode(sample_pdf_bytes).decode("utf-8")
+    record["data"] = base64.b64encode(sample_pdf_bytes).decode("utf-8")
 
-    payload = create_upload_payload(pdm_record)
+    payload = create_upload_payload(record)
 
     # Use an invalid cert that is trusted by TLS but fails truststore validation
     cert_path, key_path = temp_cert_and_key
@@ -234,3 +235,28 @@ def test_forbidden_with_invalid_cert(temp_cert_and_key):
     body = response.json()
     assert response.status_code == 403
     assert body["message"] == "Forbidden"
+
+
+def test_create_document_saves_raw(test_data):
+    record = {
+        "ods": "H81109",
+        "nhs_number": "9912003071",
+    }
+
+    sample_pdf_path = os.path.join(os.path.dirname(__file__), "files", "dummy.pdf")
+    with open(sample_pdf_path, "rb") as f:
+        record["data"] = base64.b64encode(f.read()).decode("utf-8")
+    payload = create_upload_payload(record)
+
+    session = create_mtls_session()
+    raw_upload_response = upload_document(session, payload)
+    assert raw_upload_response.status_code == 200
+
+    json_response = raw_upload_response.json()
+    record["id"] = json_response.get("id")
+    pdm_data_helper = PdmDataHelper()
+    doc_ref = pdm_data_helper.retrieve_document_reference(record=record)
+    assert "Item" in doc_ref
+    assert "RawRequest" in doc_ref["Item"]
+    assert doc_ref["Item"]["RawRequest"]
+    assert doc_ref["Item"]["RawRequest"] == payload

lambdas/tests/e2e/api/test_retrieve_document_api.py

@@ -9,7 +9,7 @@
     LLOYD_GEORGE_S3_BUCKET,
     LLOYD_GEORGE_SNOMED,
 )
-from tests.e2e.helpers.lloyd_george_data_helper import LloydGeorgeDataHelper
+from tests.e2e.helpers.data_helper import LloydGeorgeDataHelper
 
 data_helper = LloydGeorgeDataHelper()
 

lambdas/tests/e2e/api/test_search_patient_api.py

@@ -5,7 +5,7 @@
 import requests
 from syrupy.filters import paths
 from tests.e2e.conftest import API_ENDPOINT, API_KEY, APIM_ENDPOINT, LLOYD_GEORGE_SNOMED
-from tests.e2e.helpers.lloyd_george_data_helper import LloydGeorgeDataHelper
+from tests.e2e.helpers.data_helper import LloydGeorgeDataHelper
 
 data_helper = LloydGeorgeDataHelper()
 
@@ -33,7 +33,7 @@ def test_search_patient_details(test_data, snapshot_json):
 
     attachment_url = bundle["entry"][0]["resource"]["content"][0]["attachment"]["url"]
     assert (
-        f"https://{APIM_ENDPOINT}/national-document-repository/DocumentReference/{LLOYD_GEORGE_SNOMED}~"
+        f"https://{APIM_ENDPOINT}/national-document-repository/FHIR/R4/DocumentReference/{LLOYD_GEORGE_SNOMED}~"
         in attachment_url
     )
 

lambdas/tests/e2e/api/test_upload_document_api.py

@@ -13,7 +13,7 @@
     LLOYD_GEORGE_SNOMED,
     fetch_with_retry,
 )
-from tests.e2e.helpers.lloyd_george_data_helper import LloydGeorgeDataHelper
+from tests.e2e.helpers.data_helper import LloydGeorgeDataHelper
 
 data_helper = LloydGeorgeDataHelper()
 
@@ -97,7 +97,7 @@ def condition(response_json):
 
     attachment_url = upload_response["content"][0]["attachment"]["url"]
     assert (
-        f"https://{APIM_ENDPOINT}/national-document-repository/DocumentReference/{LLOYD_GEORGE_SNOMED}~"
+        f"https://{APIM_ENDPOINT}/national-document-repository/FHIR/R4/DocumentReference/{LLOYD_GEORGE_SNOMED}~"
         in attachment_url
     )
 
@@ -194,3 +194,25 @@ def condition(response_json):
 
     assert upload_response == snapshot_json(exclude=paths("id", "date"))
     assert retrieve_response == snapshot_json(exclude=paths("id", "date"))
+
+
+def test_create_document_does_not_save_raw(test_data):
+    lloyd_george_record = {}
+    lloyd_george_record["ods"] = "H81109"
+    lloyd_george_record["nhs_number"] = "9449303304"
+
+    sample_pdf_path = os.path.join(os.path.dirname(__file__), "files", "dummy.pdf")
+    with open(sample_pdf_path, "rb") as f:
+        lloyd_george_record["data"] = base64.b64encode(f.read()).decode("utf-8")
+    payload = create_upload_payload(lloyd_george_record)
+
+    url = f"https://{API_ENDPOINT}/FhirDocumentReference"
+    headers = {"Authorization": "Bearer 123", "X-Api-Key": API_KEY}
+
+    retrieve_response = requests.post(url, headers=headers, data=payload)
+    json_response = retrieve_response.json()
+
+    lloyd_george_record["id"] = json_response.get("id")
+    doc_ref = data_helper.retrieve_document_reference(record=lloyd_george_record)
+    assert "Item" in doc_ref
+    assert "RawRequest" not in doc_ref["Item"]

lambdas/tests/e2e/conftest.py

@@ -4,7 +4,7 @@
 import pytest
 import requests
 from syrupy.extensions.json import JSONSnapshotExtension
-from tests.e2e.helpers.lloyd_george_data_helper import LloydGeorgeDataHelper
+from tests.e2e.helpers.data_helper import LloydGeorgeDataHelper
 
 data_helper = LloydGeorgeDataHelper()