VED-719: Fix validation errors.

Author: mfjarvis

azure/azure-prod-proxy-release-pipeline.yml

@@ -43,10 +43,11 @@ extends:
     prod_producer_approval: true
     apigee_deployments:
       - environment: ref
-        stage_name: Deploy Ref Proxy (TEST TO BE REMOVED)
+        depends_on: []
         jinja_templates:
           DOMAIN_ENDPOINT: https://${{ parameters.sub_environment }}.example.com
       - environment: prod
-        stage_name: Deploy Prod Proxy
+        depends_on:
+        - ref
         jinja_templates:
           DOMAIN_ENDPOINT: https://${{ parameters.sub_environment }}.imms.prod.vds.platform.nhs.uk

mesh_processor/tests/upload_large_multipart_file.py

@@ -0,0 +1,80 @@
+from csv import DictWriter
+from io import TextIOWrapper
+
+import boto3
+from smart_open import open
+
+
+s3 = boto3.client("s3")
+
+
+def upload_test(bucket: str, key: str):
+    # with open(f"s3://{bucket}/{key}", "wb", transport_params={"client": s3}) as output_file:
+    with open(key, "wb") as output_file:
+        output_file.write(b"----------------------------641963206721457550558305\r\n")
+        output_file.write(b'Content-Disposition: form-data; name="File"; filename="HPV_Vaccinations_v5_TESTING_20250716T15453800.csv"\r\n')
+        output_file.write(b"Content-Type: text/csv\r\n")
+        output_file.write(b"\r\n")
+        dict_writer = DictWriter(
+            TextIOWrapper(output_file, encoding="utf-8", newline="", write_through=True),
+            fieldnames=["NHS_NUMBER", "PERSON_FORENAME", "PERSON_SURNAME", "PERSON_DOB", "PERSON_GENDER_CODE", "PERSON_POSTCODE", "DATE_AND_TIME", "SITE_CODE", "SITE_CODE_TYPE_URI", "UNIQUE_ID", "UNIQUE_ID_URI", "ACTION_FLAG", "PERFORMING_PROFESSIONAL_FORENAME", "PERFORMING_PROFESSIONAL_SURNAME", "RECORDED_DATE", "PRIMARY_SOURCE", "VACCINATION_PROCEDURE_CODE", "VACCINATION_PROCEDURE_TERM", "DOSE_SEQUENCE", "VACCINE_PRODUCT_CODE", "VACCINE_PRODUCT_TERM", "VACCINE_MANUFACTURER", "BATCH_NUMBER", "EXPIRY_DATE", "SITE_OF_VACCINATION_CODE", "SITE_OF_VACCINATION_TERM", "ROUTE_OF_VACCINATION_CODE", "ROUTE_OF_VACCINATION_TERM", "DOSE_AMOUNT", "DOSE_UNIT_CODE", "DOSE_UNIT_TERM", "INDICATION_CODE", "LOCATION_CODE", "LOCATION_CODE_TYPE_URI"],
+            delimiter="|",
+        )
+        dict_writer.writeheader()
+        dict_writer.writerow({
+            "NHS_NUMBER": "9626382228",
+            "PERSON_FORENAME": "Ashleigh",
+            "PERSON_SURNAME": "Wells",
+            "PERSON_DOB": "19970423",
+            "PERSON_GENDER_CODE": "1",
+            "PERSON_POSTCODE": "L6G 1AF",
+            "DATE_AND_TIME": "20250710T140606",
+            "SITE_CODE": "RJ1",
+            "SITE_CODE_TYPE_URI": "https://fhir.nhs.uk/Id/ods-organization-code",
+            "UNIQUE_ID": "c8cd1dbc-e7e4-4c3c-ae24-7303e43e2e0f",
+            "UNIQUE_ID_URI": "https://www.ravs.england.nhs.uk/",
+            "ACTION_FLAG": "NEW",
+            "PERFORMING_PROFESSIONAL_FORENAME": "Rita",
+            "PERFORMING_PROFESSIONAL_SURNAME": "Harding",
+            "RECORDED_DATE": "2025-07-16T15:45:38+00:00",
+            "PRIMARY_SOURCE": "TRUE",
+            "VACCINATION_PROCEDURE_CODE": "956951000000104",
+            "VACCINATION_PROCEDURE_TERM": "RSV vaccination in pregnancy (procedure)",
+            "DOSE_SEQUENCE": "1",
+            "VACCINE_PRODUCT_CODE": "42223111000001107",
+            "VACCINE_PRODUCT_TERM": "Quadrivalent influenza vaccine (split virion, inactivated) suspension for injection 0.5ml pre-filled syringes (Sanofi Pasteur)",
+            "VACCINE_MANUFACTURER": "Sanofi Pasteur",
+            "BATCH_NUMBER": "BN92478105653",
+            "EXPIRY_DATE": "20250721",
+            "SITE_OF_VACCINATION_CODE": "368209003",
+            "SITE_OF_VACCINATION_TERM": "Right arm",
+            "ROUTE_OF_VACCINATION_CODE": "1210999013",
+            "ROUTE_OF_VACCINATION_TERM": "Intradermal use",
+            "DOSE_AMOUNT": "0.3",
+            "DOSE_UNIT_CODE": "2622896019",
+            "DOSE_UNIT_TERM": "Inhalation - unit of product usage",
+            "INDICATION_CODE": "1037351000000105",
+            "LOCATION_CODE": "RJC02",
+            "LOCATION_CODE_TYPE_URI": "https://fhir.nhs.uk/Id/ods-organization-code",
+        })
+        output_file.write(b"\r\n")
+        output_file.write(b"----------------------------641963206721457550558305--\r\n")
+
+
+def upload_large_multipart_file(bucket: str, key: str) -> None:
+    with open(f"s3://{bucket}/{key}", "wb", transport_params={"client": s3}) as f:
+    # with open(key, "wb") as f:
+        f.write(b"----------------------------641963206721457550558305\r\n")
+        f.write(b'Content-Disposition: form-data; name="File"; filename="HPV_Vaccinations_v5_INVALID_20250721T15453800.csv"\r\n')
+        f.write(b"Content-Type: text/csv\r\n")
+        f.write(b"\r\n")
+        f.write(b"NHS_NUMBER|PERSON_FORENAME|PERSON_SURNAME|PERSON_DOB|PERSON_GENDER_CODE|PERSON_POSTCODE|DATE_AND_TIME|SITE_CODE|SITE_CODE_TYPE_URI|UNIQUE_ID|UNIQUE_ID_URI|ACTION_FLAG|PERFORMING_PROFESSIONAL_FORENAME|PERFORMING_PROFESSIONAL_SURNAME|RECORDED_DATE|PRIMARY_SOURCE|VACCINATION_PROCEDURE_CODE|VACCINATION_PROCEDURE_TERM|DOSE_SEQUENCE|VACCINE_PRODUCT_CODE|VACCINE_PRODUCT_TERM|VACCINE_MANUFACTURER|BATCH_NUMBER|EXPIRY_DATE|SITE_OF_VACCINATION_CODE|SITE_OF_VACCINATION_TERM|ROUTE_OF_VACCINATION_CODE|ROUTE_OF_VACCINATION_TERM|DOSE_AMOUNT|DOSE_UNIT_CODE|DOSE_UNIT_TERM|INDICATION_CODE|LOCATION_CODE|LOCATION_CODE_TYPE_URI\n")
+        for i in range(1000000):
+            f.write(b'"9626382228"|"Ashleigh"|"Wells"|"19970423"|"1"|"L6G 1AF"|"20250710T140606"|"RJ1"|"https://fhir.nhs.uk/Id/ods-organization-code"|"c8cd1dbc-e7e4-4c3c-ae24-7303e43e2e0f"|"https://www.ravs.england.nhs.uk/"|"INVALID"|"Rita"|"Harding"|"2025-07-16T15:45:38+00:00"|"TRUE"|"956951000000104"|"RSV vaccination in pregnancy (procedure)"|"1"|"42223111000001107"|"Quadrivalent influenza vaccine (split virion, inactivated) suspension for injection 0.5ml pre-filled syringes (Sanofi Pasteur)"|"Sanofi Pasteur"|"BN92478105653"|"20250721"|"368209003"|"Right arm"|"1210999013"|"Intradermal use"|"0.3"|"2622896019"|"Inhalation - unit of product usage"|"1037351000000105"|"RJC02"|"https://fhir.nhs.uk/Id/ods-organization-code"\n')
+        f.write(b"\r\n")
+        f.write(b"----------------------------641963206721457550558305--\r\n")
+
+
+if __name__ == '__main__':
+    upload_large_multipart_file("imms-dev-mesh", "jarvis-test-file.dat")
+    # upload_test("test_files", "test_file_1.csv")