Quick fix for batch e2e tests

Author: dlzhry2nhs

tests/e2e_batch/Makefile

@@ -1,4 +1,4 @@
--include .env
-
 test:
-    ENVIRONMENT=$(ENVIRONMENT) poetry run python -m unittest -v -c
+	poetry run python -m unittest -v -c
+
+.PHONY: test

tests/e2e_batch/README.md

@@ -8,9 +8,9 @@ This test suite provides automated end-to-end (E2E) testing for the Immunisation
 - Purpose: Simulate real-world batch file submissions, poll for acknowledgements, and validate processing results.
 - Test Scenarios: Defined in the scenarios module and enabled in setUp().
 - Key Features:
--   - Uploads test batch files to S3.
--   - Waits for and validates ACK (acknowledgement) files.
--   - Cleans up SQS queues and test artifacts after each run.
+    - Uploads test batch files to S3.
+    - Waits for and validates ACK (acknowledgement) files.
+    - Cleans up SQS queues and test artifacts after each run.
 
 ## Test Flow
 
@@ -39,7 +39,7 @@ This test suite provides automated end-to-end (E2E) testing for the Immunisation
 
 1. Ensure all dependencies and environment variables are set (see project root README).
 2. Update `.env` file with contents indicated in `PR-NNN.env`, modified for PR
-3. Update `.env` with referrence to the appropriate AWS config profile `AWS_PROFILE={your-aws-profile}`
+3. Update `.env` with reference to the appropriate AWS config profile `AWS_PROFILE={your-aws-profile}`
 4. Update the apigee app to match the required PR-NNN
 5. Run tests from vscode debugger or from makefile using
 

tests/e2e_batch/pr-NNN.env

@@ -1,9 +1,2 @@
 ENVIRONMENT=pr-NNN
-REGION=eu-west-2
-PROXY_NAME=immunisation-fhir-api-pr-NNN
-APIGEE_ENVIRONMENT=internal-dev
-SERVICE_BASE_PATH=immunisation-fhir-api-pr-NNN
-AUDIT_TABLE_NAME=immunisation-batch-pr-NNN-audit-table
-PR_NUMBER=NNN
-
-AWS_PROFILE={your-aws-profile}
+AWS_PROFILE={your-aws-profile}

tests/e2e_batch/scenarios.py

@@ -42,6 +42,7 @@ class TestCase:
     def __init__(self, scenario: dict):
         self.name: str = scenario.get("name", "Unnamed Test Case")
         self.description: str = scenario.get("description", "")
+        self.is_failure_scenario = scenario.get("is_failure_scenario", False)
         self.ods_vax: OdsVax = scenario.get("ods_vax")
         self.actions: list[TestAction] = scenario.get("actions", [])
         self.ods = self.ods_vax.ods_code
@@ -207,6 +208,7 @@ def generate_csv_files(test_cases: list[TestCase]) -> list[TestCase]:
                     expected_operation_outcome=OperationOutcome.IMMS_NOT_FOUND,
                 )
             ],
+            "is_failure_scenario": True,
             "operation_outcome": ActionFlag.NONE,
         },
         {
@@ -220,6 +222,7 @@ def generate_csv_files(test_cases: list[TestCase]) -> list[TestCase]:
                     expected_operation_outcome=OperationOutcome.IMMS_NOT_FOUND,
                 )
             ],
+            "is_failure_scenario": True,
             "operation_outcome": ActionFlag.NONE,
         },
         {

tests/e2e_batch/test_e2e_batch.py

@@ -28,6 +28,8 @@
 )
 
 
+# For consideration - these tests are suboptimal and difficult to read and extend. Consider integrating the automation
+# test repo into this one in future (it currently lives elsewhere but uses a nice readable framework to drive the tests)
 class TestE2EBatch(unittest.TestCase):
     def setUp(self):
         self.tests: list[TestCase] = create_test_cases(scenarios["dev"])
@@ -97,27 +99,28 @@ def poll_for_responses(tests: list[TestCase], max_timeout=1200) -> bool:
 
 def validate_responses(tests: list[TestCase]):
     start_time = time.time()
-    count = 0
-    expected_count = len(tests) * 2
+    current_ack_file_count = 0
+
+    # We expect an INF Ack (high-level i.e. file accepted or not) and a BUS Ack (containing all failure rows) for each
+    # batch file
+    expected_ack_file_count = len(tests) * 2
     errors = False
     try:
         for test in tests:
             logger.info(f"Validation for Test: {test.name} ")
             # Validate the ACK file
             if test.ack_keys[DestinationType.INF]:
-                count += 1
+                current_ack_file_count += 1
                 inf_ack_content = get_file_content_from_s3(ACK_BUCKET, test.ack_keys[DestinationType.INF])
                 check_ack_file_content(test.name, inf_ack_content, "Success", None, test.operation_outcome)
             else:
                 logger.error(f"INF ACK file not found for test: {test.name}")
                 errors = True
 
             if test.ack_keys[DestinationType.BUS]:
-                count += 1
+                current_ack_file_count += 1
                 validate_row_count(
-                    f"{test.name} - bus",
-                    test.file_name,
-                    test.ack_keys[DestinationType.BUS],
+                    f"{test.name} - bus", test.file_name, test.ack_keys[DestinationType.BUS], test.is_failure_scenario
                 )
 
                 test.check_bus_file_content()
@@ -131,10 +134,12 @@ def validate_responses(tests: list[TestCase]):
         logger.error(f"Error during validation: {e}")
         errors = True
     finally:
-        if count == expected_count:
+        if current_ack_file_count == expected_ack_file_count:
             logger.info("All responses subject to validation.")
         else:
-            logger.error(f"{count} of {expected_count} responses subject to validation.")
+            logger.error(f"{current_ack_file_count} of {expected_ack_file_count} responses subject to validation.")
         logger.info(f"Time: {time.time() - start_time:.1f} seconds")
-        assert count == expected_count, f"Only {count} of {expected_count} responses subject to validation."
+        assert current_ack_file_count == expected_ack_file_count, (
+            f"Only {current_ack_file_count} of {expected_ack_file_count} responses subject to validation."
+        )
         assert not errors, "Errors found during validation."

tests/e2e_batch/utils.py

@@ -307,14 +307,16 @@ def fetch_pk_and_operation_from_dynamodb(identifier_pk):
         return "ERROR"
 
 
-def validate_row_count(desc, source_file_name, ack_file_name):
+def validate_row_count(desc: str, source_file_name: str, ack_file_name: str, is_failure_scenario: bool) -> None:
     """
     Compare the row count of a file in one S3 bucket with a file in another S3 bucket.
     Raises:
         AssertionError: If the row counts do not match.
     """
-    source_file_row_count = fetch_row_count(SOURCE_BUCKET, f"archive/{source_file_name}")
+    # The BUS Ack will only add failed rows to the final file.
+    source_file_row_count = fetch_row_count(SOURCE_BUCKET, f"archive/{source_file_name}") if is_failure_scenario else 1
     ack_file_row_count = fetch_row_count(ACK_BUCKET, ack_file_name)
+
     assert source_file_row_count == ack_file_row_count, (
         f"{desc}. Row count mismatch: Input ({source_file_row_count}) vs Ack ({ack_file_row_count})"
     )