Merge branch 'master' into feature/VED-810-backend-deploy-pipeline

Author: dlzhry2nhs

azure/templates/post-deploy.yml

@@ -225,7 +225,6 @@ steps:
 
     displayName: Run full batch test suite
     workingDirectory: "$(Pipeline.Workspace)/s/$(SERVICE_NAME)/$(SERVICE_ARTIFACT_NAME)/e2e_batch"
-    condition: eq(1, 2) # Disable task but make this step visible in the pipeline
 
   - task: PublishTestResults@2
     displayName: 'Publish test results'

backend/tests/utils/generic_utils.py

@@ -6,6 +6,8 @@
 from decimal import Decimal
 from typing import Literal, Any
 from jsonpath_ng.ext import parse
+from datetime import datetime, date
+from typing import Union, List
 
 
 def load_json_data(filename: str):
@@ -91,3 +93,21 @@ def update_contained_resource_field(
         {field_to_update: update_value}
     )
     return json_data
+
+def format_date_types(dates: List[Union[date, datetime]], mode: str = "auto") -> List[str]:
+    """
+    Accepts a list of date or datetime objects and returns them as strings:
+    - datetime → ISO 8601 string with timezone if present
+    - date → 'YYYY-MM-DD'
+    """
+    formatted = []
+
+    for future_date in dates:
+        if mode == "datetime":
+            formatted.append(future_date.isoformat())  # full datetime with timezone
+        elif mode == "date":
+            formatted.append(future_date.strftime('%Y-%m-%d'))  # just date
+        else:
+            raise TypeError(f"Unsupported type {type(future_date)}; expected date or datetime.")
+
+    return formatted

backend/tests/utils/test_generic_utils.py

@@ -2,7 +2,10 @@
 
 import unittest
 from src.models.utils.generic_utils import form_json
-from tests.utils.generic_utils import load_json_data
+from tests.utils.generic_utils import load_json_data, format_date_types
+
+import unittest
+from datetime import datetime, date
 
 
 class TestFormJson(unittest.TestCase):
@@ -70,3 +73,25 @@ def test_elements_whitespace_and_case_are_handled(self):
         )
         self.assertEqual(res["id"], self.response["id"])
         self.assertEqual(res["meta"]["versionId"], self.response["version"])
+
+
+class TestFormatFutureDates(unittest.TestCase):
+    def test_date_mode_formats_dates_and_datetimes(self):
+        inputs = [date(2100, 1, 2), datetime(2100, 1, 3, 12, 0, 0)]
+        expected = ["2100-01-02", "2100-01-03"]
+        self.assertEqual(format_date_types(inputs, mode="date"), expected)
+
+    def test_datetime_mode_formats_dates_and_datetimes(self):
+        inputs = [date(2100, 1, 2), datetime(2100, 1, 3, 12, 0, 0)]
+        expected = ["2100-01-02", "2100-01-03T12:00:00"]
+        self.assertEqual(format_date_types(inputs, mode="datetime"), expected)
+
+    def test_default_auto_mode_is_currently_unsupported(self):
+        # Current implementation raises TypeError when mode is not 'date' or 'datetime'
+        inputs = [date(2100, 1, 2)]
+        with self.assertRaises(TypeError):
+            format_date_types(inputs)  # default mode is 'auto'
+
+
+if __name__ == "__main__":
+    unittest.main()

backend/tests/utils/values_for_tests.py

@@ -2,6 +2,9 @@
 
 from dataclasses import dataclass
 from decimal import Decimal
+from .generic_utils import format_date_types
+from datetime import datetime, timedelta
+
 
 # Lists of data types for 'invalid data type' testing
 integers = [-1, 0, 1]
@@ -291,16 +294,20 @@ class InvalidValues:
         "2000-02-30",  # Invalid combination of month and day
     ]
 
-    for_future_dates = [
-        "2100-01-01",  # Year in future
-        "2050-12-31",  # Year in future
-        "2029-06-15",  # Year in future
+    now = datetime.now()
+    sample_inputs = [
+        now + timedelta(days=1),
+        now + timedelta(days=365),
+        now + timedelta(days=730)
     ]
 
+    for_future_dates = format_date_types(sample_inputs, mode = "date")
+
     # Strings which are not in acceptable date time format
     for_date_time_string_formats_for_relaxed_timezone = [
         "",  # Empty string
         "invalid",  # Invalid format
+        *format_date_types(sample_inputs, mode = "datetime"),
         "20000101",  # Date digits only (i.e. without hypens)
         "20000101000000",  # Date and time digits only
         "200001010000000000",  # Date, time and timezone digits only
@@ -392,4 +399,3 @@ class InvalidValues:
     ]
 
     invalid_dose_quantity = {"value": 2, "unit": "ml", "code": "258773002"}
- 

e2e_batch/Makefile

@@ -1,4 +1,4 @@
 -include .env
 
-run-immunization-batch:
-	ENVIRONMENT=$(environment) poetry run python -m unittest -v -c
+test:
+    ENVIRONMENT=$(ENVIRONMENT) poetry run python -m unittest -v -c

e2e_batch/README.md

@@ -0,0 +1,38 @@
+# End-to-End Batch Test Suite (test_e2e_batch.py)
+
+This test suite provides automated end-to-end (E2E) testing for the Immunisation FHIR API batch processing pipeline. It verifies that batch file submissions are correctly processed, acknowledged, and validated across the system.
+
+## Overview
+- Framework: Python unittest
+- Purpose: Simulate real-world batch file submissions, poll for acknowledgements, and validate processing results.
+- Test Scenarios: Defined in the scenarios module and enabled in setUp().
+- Key Features:
+- - Uploads test batch files to S3.
+- - Waits for and validates ACK (acknowledgement) files.
+- - Cleans up SQS queues and test artifacts after each run.
+
+## Test Flow
+1. Setup (setUp)
+- Loads and enables a set of test scenarios.
+- Prepares test data for batch submission.
+2. Test Execution (test_batch_submission)
+- Uploads ALL enabled test files to S3.
+- Polls for ALL ACK responses and forwarded files.
+- Validates the content and structure of ACK files.
+3. Teardown (tearDown)
+- Cleans up SQS queues and any generated test files.
+
+## Key Functions
+- send_files(tests): Uploads enabled test files to the S3 input bucket.
+- poll_for_responses(tests, max_timeout): Polls for ACKs and processed files, with a timeout.
+- validate_responses(tests): Validates the content of ACK files and checks for expected outcomes.
+
+## How to Run
+1. Ensure all dependencies and environment variables are set (see project root README).
+2. Update `.env` file with contents indicated in `PR-NNN.env`, modified for PR
+3. Update `.env` with referrence to the appropriate AWS config profile `AWS_PROFILE={your-aws-profile}`
+4. Update the apigee app to match the required PR-NNN
+5. Run tests from vscode debugger or from makefile using 
+```
+make test
+```

e2e_batch/clients.py

@@ -3,7 +3,10 @@
 """
 
 import logging
-from constants import (environment, REGION)
+from constants import (
+    environment, REGION,
+    batch_fifo_queue_name, ack_metadata_queue_name, audit_table_name
+    )
 from boto3 import client as boto3_client, resource as boto3_resource
 
 
@@ -13,8 +16,12 @@
 s3_client = boto3_client("s3", region_name=REGION)
 
 dynamodb = boto3_resource("dynamodb", region_name=REGION)
-table_name = f"imms-{environment}-imms-events"
-table = dynamodb.Table(table_name)
+sqs_client = boto3_client('sqs', region_name=REGION)
+events_table_name = f"imms-{environment}-imms-events"
+events_table = dynamodb.Table(events_table_name)
+audit_table = dynamodb.Table(audit_table_name)
+batch_fifo_queue_url = sqs_client.get_queue_url(QueueName=batch_fifo_queue_name)['QueueUrl']
+ack_metadata_queue_url = sqs_client.get_queue_url(QueueName=ack_metadata_queue_name)['QueueUrl']
 # Logger
 logging.basicConfig(level="INFO")
 logger = logging.getLogger()

e2e_batch/constants.py

@@ -1,5 +1,4 @@
 import os
-from datetime import datetime, timezone
 
 environment = os.environ.get("ENVIRONMENT", "internal-dev")
 REGION = "eu-west-2"
@@ -12,77 +11,87 @@
 POST_VALIDATION_ERROR = "Validation errors: contained[?(@.resourceType=='Patient')].name[0].given is a mandatory field"
 DUPLICATE = "The provided identifier:"
 ACK_PREFIX = "ack/"
+TEMP_ACK_PREFIX = "TempAck/"
 HEADER_RESPONSE_CODE_COLUMN = "HEADER_RESPONSE_CODE"
 FILE_NAME_VAL_ERROR = "Infrastructure Level Response Value - Processing Error"
 CONFIG_BUCKET = "imms-internal-dev-supplier-config"
 PERMISSIONS_CONFIG_FILE_KEY = "permissions_config.json"
+RAVS_URI = "https://www.ravs.england.nhs.uk/"
+batch_fifo_queue_name = f"imms-{environment}-batch-file-created-queue.fifo"
+ack_metadata_queue_name = f"imms-{environment}-ack-metadata-queue.fifo"
+audit_table_name = f"immunisation-batch-{environment}-audit-table"
 
 
-def create_row(unique_id, fore_name, dose_amount, action_flag, header):
-    """Helper function to create a single row with the specified UNIQUE_ID and ACTION_FLAG."""
-
-    return {
-        header: "9732928395",
-        "PERSON_FORENAME": fore_name,
-        "PERSON_SURNAME": "James",
-        "PERSON_DOB": "20080217",
-        "PERSON_GENDER_CODE": "0",
-        "PERSON_POSTCODE": "WD25 0DZ",
-        "DATE_AND_TIME": datetime.now(timezone.utc).strftime("%Y%m%dT%H%M%S"),
-        "SITE_CODE": "RVVKC",
-        "SITE_CODE_TYPE_URI": "https://fhir.nhs.uk/Id/ods-organization-code",
-        "UNIQUE_ID": unique_id,
-        "UNIQUE_ID_URI": "https://www.ravs.england.nhs.uk/",
-        "ACTION_FLAG": action_flag,
-        "PERFORMING_PROFESSIONAL_FORENAME": "PHYLIS",
-        "PERFORMING_PROFESSIONAL_SURNAME": "James",
-        "RECORDED_DATE": datetime.now(timezone.utc).strftime("%Y%m%d"),
-        "PRIMARY_SOURCE": "TRUE",
-        "VACCINATION_PROCEDURE_CODE": "956951000000104",
-        "VACCINATION_PROCEDURE_TERM": "RSV vaccination in pregnancy (procedure)",
-        "DOSE_SEQUENCE": "1",
-        "VACCINE_PRODUCT_CODE": "42223111000001107",
-        "VACCINE_PRODUCT_TERM": "Quadrivalent influenza vaccine (Sanofi Pasteur)",
-        "VACCINE_MANUFACTURER": "Sanofi Pasteur",
-        "BATCH_NUMBER": "BN92478105653",
-        "EXPIRY_DATE": "20240915",
-        "SITE_OF_VACCINATION_CODE": "368209003",
-        "SITE_OF_VACCINATION_TERM": "Right arm",
-        "ROUTE_OF_VACCINATION_CODE": "1210999013",
-        "ROUTE_OF_VACCINATION_TERM": "Intradermal use",
-        "DOSE_AMOUNT": dose_amount,
-        "DOSE_UNIT_CODE": "2622896019",
-        "DOSE_UNIT_TERM": "Inhalation - unit of product usage",
-        "INDICATION_CODE": "1037351000000105",
-        "LOCATION_CODE": "RJC02",
-        "LOCATION_CODE_TYPE_URI": "https://fhir.nhs.uk/Id/ods-organization-code",
-    }
-
-
-def create_permissions_json(value):
-    return {
-        "all_permissions": {
-            "DPSFULL": ["RSV_FULL", "COVID19_FULL", "FLU_FULL", "MMR_FULL"],
-            "DPSREDUCED": ["COVID19_FULL", "FLU_FULL", "MMR_FULL"],
-            "EMIS": [value, "RSV_FULL"],
-            "PINNACLE": ["COVID19_UPDATE", "RSV_FULL"],
-            "SONAR": "",
-            "TPP": [""],
-            "AGEM-NIVS": [""],
-            "NIMS": [""],
-            "EVA": [""],
-            "RAVS": [""],
-            "MEDICAL_DIRECTOR": [""],
-            "WELSH_DA_1": [""],
-            "WELSH_DA_2": [""],
-            "NORTHERN_IRELAND_DA": [""],
-            "SCOTLAND_DA": [""],
-            "COVID19_VACCINE_RESOLUTION_SERVICEDESK": [""],
-        },
-        "definitions:": {
-            "FULL": "Full permissions to create, update and delete a batch record",
-            "CREATE": "Permission to create a batch record",
-            "UPDATE": "Permission to update a batch record",
-            "DELETE": "Permission to delete a batch record",
-        },
-    }
+class EventName:
+    CREATE = "INSERT"
+    UPDATE = "MODIFY"
+    DELETE_LOGICAL = "MODIFY"
+    DELETE_PHYSICAL = "REMOVE"
+
+
+class Operation:
+    CREATE = "CREATE"
+    UPDATE = "UPDATE"
+    DELETE_LOGICAL = "DELETE"
+    DELETE_PHYSICAL = "REMOVE"
+
+
+class ActionFlag:
+    CREATE = "NEW"
+    UPDATE = "UPDATE"
+    DELETE_LOGICAL = "DELETE"
+    NONE = "NONE"
+
+
+class InfResult:
+    SUCCESS = "Success"
+    PARTIAL_SUCCESS = "Partial Success"
+    FATAL_ERROR = "Fatal Error"
+
+
+class BusRowResult:
+    SUCCESS = "OK"
+    FATAL_ERROR = "Fatal Error"
+    IMMS_NOT_FOUND = "Immunization resource does not exist"
+    NONE = "NONE"
+
+
+class OperationOutcome:
+    IMMS_NOT_FOUND = "Immunization resource does not exist"
+    TEST = "TEST"
+
+
+class OpMsgs:
+    VALIDATION_ERROR = "Validation errors"
+    MISSING_MANDATORY_FIELD = "is a mandatory field"
+    DOSE_QUANTITY_NOT_NUMBER = "doseQuantity.value must be a number"
+    IMM_NOT_EXIST = "Immunization resource does not exist"
+    IDENTIFIER_PROVIDED = "The provided identifier:"
+    INVALID_DATE_FORMAT = "is not in the correct format"
+
+
+class DestinationType:
+    INF = ACK_PREFIX
+    BUS = FORWARDEDFILE_PREFIX
+
+
+class ActionSequence:
+    def __init__(self, desc: str, actions: list[ActionFlag], outcome: ActionFlag = None):
+        self.actions = actions
+        self.description = desc
+        self.outcome = outcome if outcome else actions[-1]
+
+
+class PermPair:
+    def __init__(self, ods_code: str, permissions: str):
+        self.ods_code = ods_code
+        self.permissions = permissions
+
+
+class TestSet:
+    CREATE_OK = ActionSequence("Create. OK", [ActionFlag.CREATE])
+    UPDATE_OK = ActionSequence("Update. OK", [ActionFlag.CREATE, ActionFlag.UPDATE])
+    DELETE_OK = ActionSequence("Delete. OK", [ActionFlag.CREATE, ActionFlag.UPDATE, ActionFlag.DELETE_LOGICAL])
+    REINSTATE_OK = ActionSequence("Reinstate. OK", [ActionFlag.CREATE, ActionFlag.DELETE_LOGICAL, ActionFlag.UPDATE])
+    DELETE_FAIL = ActionSequence("Delete without Create. Fail", [ActionFlag.DELETE_LOGICAL])
+    UPDATE_FAIL = ActionSequence("Update without Create. Fail", [ActionFlag.UPDATE], outcome=ActionFlag.NONE)