Merge pull request #351 from NHSDigital/AMB-2347

AMB-2347 : Immunisation Batch End to End Automation  Testing REF

Author: ASubaran

azure/templates/post-deploy.yml

@@ -179,7 +179,7 @@ steps:
 
   - bash: |
       set -e
-      if ! [[ "$APIGEE_ENVIRONMENT" == "ref" || "$APIGEE_ENVIRONMENT" == "prod" || "$APIGEE_ENVIRONMENT" == "int" || "$APIGEE_ENVIRONMENT" == *"sandbox" ]]; then
+      if ! [[ "$APIGEE_ENVIRONMENT" == "prod" || "$APIGEE_ENVIRONMENT" == "int" || "$APIGEE_ENVIRONMENT" == *"sandbox" ]]; then
         echo "Running E2E batch folder test cases"
  
         export AWS_PROFILE="apim-dev"
@@ -201,7 +201,7 @@ steps:
 
         echo "E2E batch folder test cases executed successfully"
       else
-        echo "Skipping E2E batch folder test cases as the environment is ref-prod-int-sandbox"
+        echo "Skipping E2E batch folder test cases as the environment is prod-int-sandbox"
       fi
 
     displayName: Run full batch test suite

e2e/extract_failed_tests.py

@@ -0,0 +1,72 @@
+import boto3
+
+# Get the DynamoDB table name
+TABLE_NAME = "imms-internal-dev-imms-events"
+
+if not TABLE_NAME:
+    raise ValueError("DYNAMODB_TABLE_NAME environment variable is not set")
+
+# Initialize DynamoDB client
+dynamodb = boto3.resource("dynamodb")
+table = dynamodb.Table(TABLE_NAME)
+
+
+def get_primary_keys():
+    """Retrieve the primary key schema of the table."""
+    response = table.key_schema
+    return [key["AttributeName"] for key in response]
+
+
+def get_total_count():
+    """Get the total count of items in the table, handling pagination."""
+    total_count = 0
+    last_evaluated_key = None
+
+    while True:
+        if last_evaluated_key:
+            response = table.scan(Select="COUNT", ExclusiveStartKey=last_evaluated_key)
+        else:
+            response = table.scan(Select="COUNT")
+
+        total_count += response.get("Count", 0)
+        last_evaluated_key = response.get("LastEvaluatedKey")
+
+        if not last_evaluated_key:
+            break
+
+    return total_count
+
+
+def clear_dynamodb():
+    """Deletes all items from the DynamoDB table, handling pagination."""
+    print(f"Clearing DynamoDB table: {TABLE_NAME}")
+
+    primary_keys = get_primary_keys()
+    if not primary_keys:
+        raise ValueError("Unable to retrieve primary key schema")
+
+    total_count = get_total_count()
+    print(f"Total items before deletion: {total_count}")
+
+    deleted_count = 0
+
+    while True:
+        scan = table.scan()
+        items = scan.get("Items", [])
+
+        if not items:
+            break
+
+        with table.batch_writer() as batch:
+            for item in items:
+                key = {pk: item[pk] for pk in primary_keys}
+                batch.delete_item(Key=key)
+                deleted_count += 1
+
+        print(f"Deleted {len(items)} items...")
+
+    print(f"Total {deleted_count} items deleted from DynamoDB")
+
+
+if __name__ == "__main__":
+    clear_dynamodb()

e2e_batch/clear_dynamodb.py

@@ -1,9 +1,6 @@
 import os
-# import logging
+
 from datetime import datetime, timezone
-# logging.basicConfig(level="INFO")
-# logger = logging.getLogger()
-# logger.setLevel("INFO")
 from clients import logger
 
 env_value = os.environ.get("ENV", "internal-dev")
@@ -16,6 +13,7 @@
 POST_VALIDATION_ERROR = "Validation errors: contained[?(@.resourceType=='Patient')].name[0].given is a mandatory field"
 DUPLICATE = "The provided identifier:"
 ACK_PREFIX = "ack/"
+HEADER_RESPONSE_CODE_COLUMN = "HEADER_RESPONSE_CODE"
 FILE_NAME_VAL_ERROR = "Infrastructure Level Response Value - Processing Error"
 CONFIG_BUCKET = "imms-internal-dev-supplier-config"
 PERMISSIONS_CONFIG_FILE_KEY = "permissions_config.json"

e2e_batch/constants.py

@@ -7,7 +7,9 @@
     wait_for_ack_file,
     check_ack_file_content,
     validate_row_count,
-    upload_config_file
+    upload_config_file,
+    generate_csv_with_ordered_100000_rows,
+    verify_final_ack_file,
 )
 from constants import (
     SOURCE_BUCKET,
@@ -17,109 +19,121 @@
     POST_VALIDATION_ERROR,
     DUPLICATE,
     FILE_NAME_VAL_ERROR,
+    env_value,
 )
 
 
 class TestE2EBatch(unittest.TestCase):
+    if env_value != "ref":
 
-    def test_create_success(self):
-        """Test CREATE scenario."""
-        input_file = generate_csv("PHYLIS", "0.3", action_flag="CREATE")
-        upload_file_to_s3(input_file, SOURCE_BUCKET, INPUT_PREFIX)
-        ack_key = wait_for_ack_file(None, input_file)
-        validate_row_count(input_file, ack_key)
-        ack_content = get_file_content_from_s3(ACK_BUCKET, ack_key)
-        check_ack_file_content(ack_content, "OK", None, "CREATE")
-
-    def test_duplicate_create(self):
-        """Test DUPLICATE scenario."""
-        input_file = generate_csv("PHYLIS", "0.3", action_flag="CREATE", same_id=True)
-        upload_file_to_s3(input_file, SOURCE_BUCKET, INPUT_PREFIX)
-        ack_key = wait_for_ack_file(None, input_file)
-        validate_row_count(input_file, ack_key)
-        ack_content = get_file_content_from_s3(ACK_BUCKET, ack_key)
-        check_ack_file_content(ack_content, "Fatal Error", DUPLICATE, "CREATE")
-
-    def test_update_success(self):
-        """Test UPDATE scenario."""
-        input_file = generate_csv("PHYLIS", "0.5", action_flag="UPDATE")
-        upload_file_to_s3(input_file, SOURCE_BUCKET, INPUT_PREFIX)
-        ack_key = wait_for_ack_file(None, input_file)
-        validate_row_count(input_file, ack_key)
-        ack_content = get_file_content_from_s3(ACK_BUCKET, ack_key)
-        check_ack_file_content(ack_content, "OK", None, "UPDATE")
-
-    def test_reinstated_success(self):
-        """Test REINSTATED scenario."""
-        input_file = generate_csv("PHYLIS", "0.5", action_flag="REINSTATED")
-        upload_file_to_s3(input_file, SOURCE_BUCKET, INPUT_PREFIX)
-        ack_key = wait_for_ack_file(None, input_file)
-        validate_row_count(input_file, ack_key)
-        ack_content = get_file_content_from_s3(ACK_BUCKET, ack_key)
-        check_ack_file_content(ack_content, "OK", None, "reinstated")
-
-    def test_update_reinstated_success(self):
-        """Test UPDATE-REINSTATED scenario."""
-        input_file = generate_csv("PHYLIS", "0.5", action_flag="UPDATE-REINSTATED")
-        upload_file_to_s3(input_file, SOURCE_BUCKET, INPUT_PREFIX)
-        ack_key = wait_for_ack_file(None, input_file)
-        validate_row_count(input_file, ack_key)
-        ack_content = get_file_content_from_s3(ACK_BUCKET, ack_key)
-        check_ack_file_content(ack_content, "OK", None, "update-reinstated")
-
-    def test_delete_success(self):
-        """Test DELETE scenario."""
-        input_file = generate_csv("PHYLIS", "0.8", action_flag="DELETE")
-        upload_file_to_s3(input_file, SOURCE_BUCKET, INPUT_PREFIX)
-        ack_key = wait_for_ack_file(None, input_file)
-        validate_row_count(input_file, ack_key)
-        ack_content = get_file_content_from_s3(ACK_BUCKET, ack_key)
-        check_ack_file_content(ack_content, "OK", None, "DELETE")
-
-    def test_pre_validation_error(self):
-        """Test PRE-VALIDATION error scenario."""
-        input_file = generate_csv("PHYLIS", "TRUE", action_flag="CREATE")
-        upload_file_to_s3(input_file, SOURCE_BUCKET, INPUT_PREFIX)
-        ack_key = wait_for_ack_file(None, input_file)
-        validate_row_count(input_file, ack_key)
-        ack_content = get_file_content_from_s3(ACK_BUCKET, ack_key)
-        check_ack_file_content(ack_content, "Fatal Error", PRE_VALIDATION_ERROR, None)
-
-    def test_post_validation_error(self):
-        """Test POST-VALIDATION error scenario."""
-        input_file = generate_csv("", "0.3", action_flag="CREATE")
-        upload_file_to_s3(input_file, SOURCE_BUCKET, INPUT_PREFIX)
-        ack_key = wait_for_ack_file(None, input_file)
-        ack_content = get_file_content_from_s3(ACK_BUCKET, ack_key)
-        check_ack_file_content(ack_content, "Fatal Error", POST_VALIDATION_ERROR, None)
-
-    def test_file_name_validation_error(self):
-        """Test FILE-NAME-VALIDATION error scenario."""
-        input_file = generate_csv("PHYLIS", "0.3", action_flag="CREATE", file_key=True)
-        upload_file_to_s3(input_file, SOURCE_BUCKET, INPUT_PREFIX)
-        ack_key = wait_for_ack_file(True, input_file)
-        ack_content = get_file_content_from_s3(ACK_BUCKET, ack_key)
-        check_ack_file_content(ack_content, "Failure", FILE_NAME_VAL_ERROR, None)
-
-    def test_header_name_validation_error(self):
-        """Test HEADER-NAME-VALIDATION error scenario."""
-        input_file = generate_csv("PHYLIS", "0.3", action_flag="CREATE", headers="NH_NUMBER")
-        upload_file_to_s3(input_file, SOURCE_BUCKET, INPUT_PREFIX)
-        ack_key = wait_for_ack_file(True, input_file)
-        ack_content = get_file_content_from_s3(ACK_BUCKET, ack_key)
-        check_ack_file_content(ack_content, "Failure", FILE_NAME_VAL_ERROR, None)
-
-    def test_invalid_permission(self):
-        """Test INVALID-PERMISSION error scenario."""
-        upload_config_file("MMR_FULL")
-        time.sleep(20)
-        input_file = generate_csv("PHYLIS", "0.3", action_flag="CREATE")
-        upload_file_to_s3(input_file, SOURCE_BUCKET, INPUT_PREFIX)
-        ack_key = wait_for_ack_file(True, input_file)
-        ack_content = get_file_content_from_s3(ACK_BUCKET, ack_key)
-        check_ack_file_content(ack_content, "Failure", FILE_NAME_VAL_ERROR, None)
-        upload_config_file("COVID19_FULL")
-        time.sleep(20)
+        def test_create_success(self):
+            """Test CREATE scenario."""
+            input_file = generate_csv("PHYLIS", "0.3", action_flag="CREATE")
+            upload_file_to_s3(input_file, SOURCE_BUCKET, INPUT_PREFIX)
+            ack_key = wait_for_ack_file(None, input_file)
+            validate_row_count(input_file, ack_key)
+            ack_content = get_file_content_from_s3(ACK_BUCKET, ack_key)
+            check_ack_file_content(ack_content, "OK", None, "CREATE")
+
+        def test_duplicate_create(self):
+            """Test DUPLICATE scenario."""
+            input_file = generate_csv("PHYLIS", "0.3", action_flag="CREATE", same_id=True)
+            upload_file_to_s3(input_file, SOURCE_BUCKET, INPUT_PREFIX)
+            ack_key = wait_for_ack_file(None, input_file)
+            validate_row_count(input_file, ack_key)
+            ack_content = get_file_content_from_s3(ACK_BUCKET, ack_key)
+            check_ack_file_content(ack_content, "Fatal Error", DUPLICATE, "CREATE")
+
+        def test_update_success(self):
+            """Test UPDATE scenario."""
+            input_file = generate_csv("PHYLIS", "0.5", action_flag="UPDATE")
+            upload_file_to_s3(input_file, SOURCE_BUCKET, INPUT_PREFIX)
+            ack_key = wait_for_ack_file(None, input_file)
+            validate_row_count(input_file, ack_key)
+            ack_content = get_file_content_from_s3(ACK_BUCKET, ack_key)
+            check_ack_file_content(ack_content, "OK", None, "UPDATE")
+
+        def test_reinstated_success(self):
+            """Test REINSTATED scenario."""
+            input_file = generate_csv("PHYLIS", "0.5", action_flag="REINSTATED")
+            upload_file_to_s3(input_file, SOURCE_BUCKET, INPUT_PREFIX)
+            ack_key = wait_for_ack_file(None, input_file)
+            validate_row_count(input_file, ack_key)
+            ack_content = get_file_content_from_s3(ACK_BUCKET, ack_key)
+            check_ack_file_content(ack_content, "OK", None, "reinstated")
+
+        def test_update_reinstated_success(self):
+            """Test UPDATE-REINSTATED scenario."""
+            input_file = generate_csv("PHYLIS", "0.5", action_flag="UPDATE-REINSTATED")
+            upload_file_to_s3(input_file, SOURCE_BUCKET, INPUT_PREFIX)
+            ack_key = wait_for_ack_file(None, input_file)
+            validate_row_count(input_file, ack_key)
+            ack_content = get_file_content_from_s3(ACK_BUCKET, ack_key)
+            check_ack_file_content(ack_content, "OK", None, "update-reinstated")
+
+        def test_delete_success(self):
+            """Test DELETE scenario."""
+            input_file = generate_csv("PHYLIS", "0.8", action_flag="DELETE")
+            upload_file_to_s3(input_file, SOURCE_BUCKET, INPUT_PREFIX)
+            ack_key = wait_for_ack_file(None, input_file)
+            validate_row_count(input_file, ack_key)
+            ack_content = get_file_content_from_s3(ACK_BUCKET, ack_key)
+            check_ack_file_content(ack_content, "OK", None, "DELETE")
+
+        def test_pre_validation_error(self):
+            """Test PRE-VALIDATION error scenario."""
+            input_file = generate_csv("PHYLIS", "TRUE", action_flag="CREATE")
+            upload_file_to_s3(input_file, SOURCE_BUCKET, INPUT_PREFIX)
+            ack_key = wait_for_ack_file(None, input_file)
+            validate_row_count(input_file, ack_key)
+            ack_content = get_file_content_from_s3(ACK_BUCKET, ack_key)
+            check_ack_file_content(ack_content, "Fatal Error", PRE_VALIDATION_ERROR, None)
+
+        def test_post_validation_error(self):
+            """Test POST-VALIDATION error scenario."""
+            input_file = generate_csv("", "0.3", action_flag="CREATE")
+            upload_file_to_s3(input_file, SOURCE_BUCKET, INPUT_PREFIX)
+            ack_key = wait_for_ack_file(None, input_file)
+            ack_content = get_file_content_from_s3(ACK_BUCKET, ack_key)
+            check_ack_file_content(ack_content, "Fatal Error", POST_VALIDATION_ERROR, None)
+
+        def test_file_name_validation_error(self):
+            """Test FILE-NAME-VALIDATION error scenario."""
+            input_file = generate_csv("PHYLIS", "0.3", action_flag="CREATE", file_key=True)
+            upload_file_to_s3(input_file, SOURCE_BUCKET, INPUT_PREFIX)
+            ack_key = wait_for_ack_file(True, input_file)
+            ack_content = get_file_content_from_s3(ACK_BUCKET, ack_key)
+            check_ack_file_content(ack_content, "Failure", FILE_NAME_VAL_ERROR, None)
+
+        def test_header_name_validation_error(self):
+            """Test HEADER-NAME-VALIDATION error scenario."""
+            input_file = generate_csv("PHYLIS", "0.3", action_flag="CREATE", headers="NH_NUMBER")
+            upload_file_to_s3(input_file, SOURCE_BUCKET, INPUT_PREFIX)
+            ack_key = wait_for_ack_file(True, input_file)
+            ack_content = get_file_content_from_s3(ACK_BUCKET, ack_key)
+            check_ack_file_content(ack_content, "Failure", FILE_NAME_VAL_ERROR, None)
+
+        def test_invalid_permission(self):
+            """Test INVALID-PERMISSION error scenario."""
+            upload_config_file("MMR_FULL")
+            time.sleep(20)
+            input_file = generate_csv("PHYLIS", "0.3", action_flag="CREATE")
+            upload_file_to_s3(input_file, SOURCE_BUCKET, INPUT_PREFIX)
+            ack_key = wait_for_ack_file(True, input_file)
+            ack_content = get_file_content_from_s3(ACK_BUCKET, ack_key)
+            check_ack_file_content(ack_content, "Failure", FILE_NAME_VAL_ERROR, None)
+            upload_config_file("COVID19_FULL")
+            time.sleep(20)
+
+    else:
+
+        def test_end_to_end_speed_test_with_100000_rows(self):
+            """Test end_to_end_speed_test_with_100000_rows scenario with full integration"""
+            input_file = generate_csv_with_ordered_100000_rows(None)
+            upload_file_to_s3(input_file, SOURCE_BUCKET, INPUT_PREFIX)
+            final_ack_key = wait_for_ack_file(None, input_file, timeout=1800)
+            response = verify_final_ack_file(final_ack_key)
+            assert response is True
 
 
 if __name__ == "__main__":