parallel tests

Author: nhsdevws

e2e_batch/Makefile

@@ -4,12 +4,19 @@
 
 APIGEE_ACCESS_TOKEN ?= $(shell export SSO_LOGIN_URL=https://login.apigee.com && eval get_token -u $(APIGEE_USERNAME))
 AWS_DOMAIN_NAME=https://$(shell make -C ../terraform -s output name=service_domain_name || true)
+PARALLEL_WORKERS=4
 
 print-token:
 	@echo "APIGEE_ACCESS_TOKEN=$(APIGEE_ACCESS_TOKEN)"
 
 run-immunization-batch:
-	ENVIRONMENT=$(environment) poetry run python -m unittest -v -c
+# 	ENVIRONMENT=$(environment) poetry run python -m unittest -v -c
+	ENVIRONMENT=$(ENVIRONMENT) poetry run unittest-parallel -v -j $(PARALLEL_WORKERS)
 
 test:
-	ENVIRONMENT=$(environment) poetry run python -m unittest -v -c
+# 	@echo "PR_NUMBER=$(PR_NUMBER)"
+# 	@echo "ENVIRONMENT=$(ENVIRONMENT)"
+# 	ENVIRONMENT=$(ENVIRONMENT) poetry run python -m unittest -v -c
+	ENVIRONMENT=$(ENVIRONMENT) poetry run unittest-parallel -v -j $(PARALLEL_WORKERS)
+
+

e2e_batch/constants.py

@@ -17,6 +17,10 @@
 CONFIG_BUCKET = "imms-internal-dev-supplier-config"
 PERMISSIONS_CONFIG_FILE_KEY = "permissions_config.json"
 
+print(f"SAW. Debug {SOURCE_BUCKET}")
+print(f"SAW. Debug {ACK_BUCKET}")
+print(f"SAW. Debug {environment}")
+
 
 def create_row(unique_id, fore_name, dose_amount, action_flag, header):
     """Helper function to create a single row with the specified UNIQUE_ID and ACTION_FLAG."""

e2e_batch/per_test.py

@@ -18,7 +18,10 @@ def monitor(nameMarker):
             "marker": nameMarker,
             "elapsed_time": elapsed_ms
         }
+        logger.info(f"Perf end [{nameMarker}]: {elapsed_ms} ms")
+
         logger.info(json.dumps(out_msg))
         del marker[nameMarker]  # <-- remove the marker after logging
     else:
+        logger.info(f"Perf start [{nameMarker}]")
         marker[nameMarker] = t

e2e_batch/poetry.lock

@@ -5,8 +5,15 @@ description = "End-to-end tests for immunization-batch"
 authors = ["Your Name <[email protected]>"]
 license = "MIT"
 readme = "README.md"
+package-mode = false
 
 [tool.poetry.dependencies]
 python                  = "~3.11"
 boto3                   = "~1.38.42"
 pandas                  = "^2.3.0"
+dotenv = "^0.9.9"
+unittest-parallel = "^1.7.5"
+
+[tool.poetry.group.dev.dependencies]
+unittest-parallel = "^1.7.5"
+

e2e_batch/pyproject.toml

@@ -1,5 +1,6 @@
 import time
 import unittest
+
 from utils import (
     generate_csv,
     upload_file_to_s3,
@@ -24,9 +25,13 @@
     FILE_NAME_VAL_ERROR,
     environment
 )
+# load dotenv for run from desktop
+# from dotenv import load_dotenv
+# load_dotenv()
 
 
 class TestE2EBatch(unittest.TestCase):
+
     def setUp(self):
         self.uploaded_files = []  # Tracks uploaded input keys
         self.ack_files = []       # Tracks ack keys
@@ -41,9 +46,8 @@ def tearDown(self):
         def test_create_success(self):
             """Test CREATE scenario."""
             monitor("test_create_success")
-            input_file = generate_csv("PHYLIS", "0.3",
-                                      "RSV", "YGM41",
-                                      action_flag="CREATE")
+            input_file = generate_csv("PHYLIS", "0.3", "CREATE",
+                                      "RSV", "YGM41")
 
             key = upload_file_to_s3(input_file, SOURCE_BUCKET, INPUT_PREFIX)
             self.uploaded_files.append(key)
@@ -239,7 +243,6 @@ def test_invalid_permission(self):
         def test_end_to_end_speed_test_with_100000_rows(self):
             monitor("test_end_to_end_speed_test_with_100000_rows")
             """Test end_to_end_speed_test_with_100000_rows scenario with full integration"""
-            file_name = f"RSV_Vaccinations_v5_YGM41_{timestamp}.csv" if not file_name else file_name
             input_file = generate_csv_with_ordered_100000_rows(None)
 
             key = upload_file_to_s3(input_file, SOURCE_BUCKET, INPUT_PREFIX)