[PRMT-229] Renamed updated bulk upload workflow to performance bulk upload (#668)

Author: PedroSoaresNHS

…rkflows/updated-base-run-bulk-upload.yml …ows/performance-base-run-bulk-upload.yml.github/workflows/updated-base-run-bulk-upload.yml renamed to .github/workflows/performance-base-run-bulk-upload.yml .github/workflows/updated-base-run-bulk-upload.yml renamed to .github/workflows/performance-base-run-bulk-upload.yml

@@ -1,4 +1,4 @@
-name: "Updated Base Bulk Upload - Execute a Bulk Upload"
+name: "Performance Base Bulk Upload - Execute a Bulk Upload"
 
 permissions:
   pull-requests: write
@@ -71,7 +71,7 @@ jobs:
 
       - name: Setup Bulk Upload
         run: |
-          python updated_setup_bulk_upload.py \
+          python performance_setup_bulk_upload.py \
             --environment "${{ inputs.sandbox }}" \
             --delete-table \
             --download-data \
@@ -85,7 +85,7 @@ jobs:
 
       - name: Run Bulk Upload
         run: |
-          python run_bulk_upload.py \
+          python performance_run_bulk_upload.py \
             --environment "${{ inputs.sandbox }}" \
             --start-bulk-upload
         working-directory: ./tests/bulk-upload/scripts

…rkflows/updated-run-bulk-upload-test.yml …ows/performance-run-bulk-upload-test.yml.github/workflows/updated-run-bulk-upload-test.yml renamed to .github/workflows/performance-run-bulk-upload-test.yml .github/workflows/updated-run-bulk-upload-test.yml renamed to .github/workflows/performance-run-bulk-upload-test.yml

@@ -1,4 +1,4 @@
-name: Updated Run Bulk Upload - Test
+name: Performance Run Bulk Upload - Test
 
 permissions:
   pull-requests: write
@@ -31,7 +31,7 @@ on:
 
 jobs:
   bulk_upload:
-    uses: ./.github/workflows/updated-base-run-bulk-upload.yml
+    uses: ./.github/workflows/performance-base-run-bulk-upload.yml
     with:
       environment: development
       sandbox: "${{ inputs.sandbox }}"

lambdas/handlers/bulk_upload_handler.py

@@ -27,7 +27,7 @@ def lambda_handler(event, _context):
     validation_strict_mode = validation_strict_mode_flag_object[
         FeatureFlags.LLOYD_GEORGE_VALIDATION_STRICT_MODE_ENABLED.value
     ]
-    pds_fhir_always_true = os.getenv("PDS_FHIR_ALWAYS_TRUE", "false").lower() == "true"
+    bypass_pds = os.getenv("BYPASS_PDS", "false").lower() == "true"
 
     if validation_strict_mode:
         logger.info("Lloyd George validation strict mode is enabled")
@@ -43,7 +43,7 @@ def lambda_handler(event, _context):
         ).create_api_gateway_response()
 
     bulk_upload_service = BulkUploadService(
-        strict_mode=validation_strict_mode, pds_fhir_always_true=pds_fhir_always_true
+        strict_mode=validation_strict_mode, bypass_pds=bypass_pds
     )
 
     try:

lambdas/services/bulk_upload_service.py

@@ -50,7 +50,7 @@
 
 
 class BulkUploadService:
-    def __init__(self, strict_mode, pds_fhir_always_true=False):
+    def __init__(self, strict_mode, bypass_pds=False):
         self.dynamo_repository = BulkUploadDynamoRepository()
         self.sqs_repository = BulkUploadSqsRepository()
         self.bulk_upload_s3_repository = BulkUploadS3Repository()
@@ -59,7 +59,7 @@ def __init__(self, strict_mode, pds_fhir_always_true=False):
         self.unhandled_messages = []
         self.file_path_cache = {}
         self.pdf_stitching_queue_url = os.environ["PDF_STITCHING_SQS_URL"]
-        self.pds_fhir_always_true = pds_fhir_always_true
+        self.bypass_pds = bypass_pds
 
     def process_message_queue(self, records: list):
         for index, message in enumerate(records, start=1):
@@ -137,7 +137,7 @@ def handle_sqs_message(self, message: dict):
             patient_ods_code = (
                 pds_patient_details.get_ods_code_or_inactive_status_for_gp()
             )
-            if not self.pds_fhir_always_true:
+            if not self.bypass_pds:
                 if not self.strict_mode:
                     (
                         name_validation_accepted_reason,

lambdas/utils/utilities.py

@@ -74,7 +74,7 @@ def get_pds_service() -> PatientSearch:
         return PdsApiService(ssm_service, auth_service)
     else:
         return MockPdsApiService(
-            always_pass_mock=os.getenv("PDS_FHIR_ALWAYS_TRUE") in ["True", "true"]
+            always_pass_mock=os.getenv("BYPASS_PDS", "false").lower() == "true"
         )
 
 

tests/bulk-upload/scripts/performance_run_bulk_upload.py

@@ -0,0 +1,72 @@
+import argparse
+import json
+import sys
+
+import boto3
+
+
+def invoke_lambda(lambda_name, payload={}):
+    client = boto3.client("lambda")
+    response = client.invoke(
+        FunctionName=lambda_name,
+        InvocationType="RequestResponse",
+        Payload=json.dumps(payload),
+    )
+    return json.loads(response["Payload"].read())
+
+
+def update_lambda_environment_variables(lambda_name, new_variables):
+    session = boto3.Session()
+    lambda_client = session.client("lambda")
+
+    response = lambda_client.get_function_configuration(FunctionName=lambda_name)
+    current_environment = response["Environment"]["Variables"]
+
+    updated_environment = current_environment.copy()
+    updated_environment.update(new_variables)
+
+    lambda_client.update_function_configuration(
+        FunctionName=lambda_name, Environment={"Variables": updated_environment}
+    )
+
+
+if __name__ == "__main__":
+    parser = argparse.ArgumentParser(description="Run Bulk Upload Script")
+
+    parser.add_argument(
+        "--environment",
+        help="The name of the environment",
+    )
+    parser.add_argument(
+        "--disable-pds-stub",
+        action="store_true",
+        help="Disable the PDS Stub",
+    )
+    parser.add_argument(
+        "--start-bulk-upload",
+        action="store_true",
+        help="Start the Bulk Upload",
+    )
+
+    args = parser.parse_args()
+
+    if not args.environment:
+        args.environment = input("Please enter the name of the environment: ")
+
+    bulk_upload_lambda_name = f"{args.environment}_BulkUploadLambda"
+    search_lambda_name = f"{args.environment}_SearchPatientDetailsLambda"
+    if args.disable_pds_stub or (
+        sys.stdin.isatty()
+        and input("Would you like to disable the FHIR Stub: ").lower() == "y"
+    ):
+        new_variables = {"PDS_FHIR_IS_STUBBED": "false"}
+        update_lambda_environment_variables(bulk_upload_lambda_name, new_variables)
+        update_lambda_environment_variables(search_lambda_name, new_variables)
+    else:
+        new_variables = {"PDS_FHIR_IS_STUBBED": "true", "BYPASS_PDS": "true"}
+        update_lambda_environment_variables(bulk_upload_lambda_name, new_variables)
+        update_lambda_environment_variables(search_lambda_name, new_variables)
+    if args.start_bulk_upload or input(
+        "Would you like to start the Bulk Upload Process:"
+    ):
+        invoke_lambda(f"{args.environment}_BulkUploadMetadataLambda")

…oad/scripts/updated_setup_bulk_upload.py …scripts/performance_setup_bulk_upload.pytests/bulk-upload/scripts/updated_setup_bulk_upload.py renamed to tests/bulk-upload/scripts/performance_setup_bulk_upload.py tests/bulk-upload/scripts/updated_setup_bulk_upload.py renamed to tests/bulk-upload/scripts/performance_setup_bulk_upload.py

@@ -35,4 +35,4 @@ def invoke_lambda(lambda_name, payload={}):
     if args.start_bulk_upload or input(
         "Would you like to start the Bulk Upload Process:"
     ):
-        invoke_lambda(f"{args.environment}_BulkUploadMetadataLambda")
+        invoke_lambda(f"{args.environment}_BulkUploadMetadataLambda")

tests/bulk-upload/scripts/run_bulk_upload.py

@@ -5,7 +5,7 @@
 from unittest.mock import patch
 
 import pytest
-from updated_setup_bulk_upload import (
+from performance_setup_bulk_upload import (
     S3_STORAGE_CLASS,
     STAGING_BUCKET,
     build_file_path,
@@ -58,7 +58,7 @@ def test_generate_nhs_number(
     mocker, input_nhs_number, mock_return_values, expected_output
 ):
     mocker.patch(
-        "updated_setup_bulk_upload.pairing_nhs_number_digit",
+        "performance_setup_bulk_upload.pairing_nhs_number_digit",
         side_effect=mock_return_values,
     )
     result = generate_nhs_number(input_nhs_number)
@@ -123,20 +123,20 @@ def test_build_file_path(nhs_number, file_name, expected):
 
 def test_create_test_file_keys_and_metadata_rows_calls(mocker):
     mock_generate_random_name = mocker.patch(
-        "updated_setup_bulk_upload.generate_random_name", return_value="Alice"
+        "performance_setup_bulk_upload.generate_random_name", return_value="Alice"
     )
     mock_generate_nhs_number = mocker.patch(
-        "updated_setup_bulk_upload.generate_nhs_number",
+        "performance_setup_bulk_upload.generate_nhs_number",
         side_effect=lambda number: number + "1",
     )
     mock_generate_file_name = mocker.patch(
-        "updated_setup_bulk_upload.generate_file_name", return_value="file1.pdf"
+        "performance_setup_bulk_upload.generate_file_name", return_value="file1.pdf"
     )
     mock_build_file_path = mocker.patch(
-        "updated_setup_bulk_upload.build_file_path", return_value="NH123/file1.pdf"
+        "performance_setup_bulk_upload.build_file_path", return_value="NH123/file1.pdf"
     )
     mock_build_metadata_csv_row = mocker.patch(
-        "updated_setup_bulk_upload.build_metadata_csv_row", return_value="row"
+        "performance_setup_bulk_upload.build_metadata_csv_row", return_value="row"
     )
 
     requested_patients_number = 2
@@ -186,11 +186,13 @@ def test_upload_source_file_to_staging_calls(mocker):
     )
     file_key = "1234567890/test.pdf"
 
-    mock_client = mocker.patch("updated_setup_bulk_upload.client")
-    mocker.patch("updated_setup_bulk_upload.STAGING_BUCKET", "test-bucket")
-    mocker.patch("updated_setup_bulk_upload.S3_STORAGE_CLASS", "INTELLIGENT_TIERING")
+    mock_client = mocker.patch("performance_setup_bulk_upload.client")
+    mocker.patch("performance_setup_bulk_upload.STAGING_BUCKET", "test-bucket")
     mocker.patch(
-        "updated_setup_bulk_upload.S3_SCAN_TAGS",
+        "performance_setup_bulk_upload.S3_STORAGE_CLASS", "INTELLIGENT_TIERING"
+    )
+    mocker.patch(
+        "performance_setup_bulk_upload.S3_SCAN_TAGS",
         [{"Key": "ScanStatus", "Value": "Clean"}],
     )
 
@@ -203,15 +205,15 @@ def test_upload_source_file_to_staging_calls(mocker):
 def test_delete_file_from_staging_calls_client_delete_object_once():
     file_key = "some/file/key.pdf"
 
-    with patch("updated_setup_bulk_upload.client") as mock_client:
+    with patch("performance_setup_bulk_upload.client") as mock_client:
         delete_file_from_staging(file_key)
 
         calls = mock_client.delete_object.call_count
         assert calls == 1, f"Expected 1 call to delete_object, got {calls}"
 
 
 def test_upload_lg_files_to_staging_calls_client_methods(mocker):
-    mock_client = mocker.patch("updated_setup_bulk_upload.client")
+    mock_client = mocker.patch("performance_setup_bulk_upload.client")
 
     lg_file_keys = ["file1.pdf", "file2.pdf", "file3.pdf"]
     source_pdf_file_key = "source.pdf"
@@ -302,7 +304,7 @@ def test_build_metadata_csv_row():
 def test_upload_metadata_to_s3_calls_put_object_with_correct_args(mocker):
     test_body = "some,csv,content\n1,2,3"
 
-    mock_client = mocker.patch("updated_setup_bulk_upload.client")
+    mock_client = mocker.patch("performance_setup_bulk_upload.client")
 
     upload_metadata_to_s3(test_body)