Merge branch 'master' into VED-737-Search-API--element-throws-error-message

Author: nhsdevws

.github/workflows/deploy-blue-green.yml

@@ -1,18 +1,20 @@
 name: Deploy Blue Green - INT
 
 on:
-  pull_request:
-    types: [closed]
-    branches: [master]
+  push:
+    branches:
+      - release-2025-08-12
 
 jobs:
   deploy-green:
     uses: ./.github/workflows/deploy-template.yml
     with:
-      environment: green
-
+      apigee_environment: int
+      environment: preprod
+      sub_environment: int-green
   deploy-blue:
-    needs: deploy-green
     uses: ./.github/workflows/deploy-template.yml
     with:
-      environment: blue
+      apigee_environment: int
+      environment: preprod
+      sub_environment: int-blue

.github/workflows/deploy-template.yml

@@ -1,10 +1,17 @@
-name: Deploy to INT and run E2e test
+name: Deploy Template
+
 on:
   workflow_call:
     inputs:
+      apigee_environment:
+        required: true
+        type: string
       environment:
         required: true
         type: string
+      sub_environment:
+        required: true
+        type: string
 
 jobs:
   terraform-plan:
@@ -13,8 +20,8 @@ jobs:
       id-token: write
       contents: read
     steps:
-      - name: Debug OIDC
-        uses: aws-actions/configure-aws-credentials@v4
+      - name: Connect to AWS
+        uses: aws-actions/configure-aws-credentials@7474bc4690e29a8392af63c5b98e7449536d5c3a
         with:
           aws-region: eu-west-2
           role-to-assume: arn:aws:iam::${{ vars.AWS_ACCOUNT_ID }}:role/auto-ops
@@ -24,26 +31,24 @@ jobs:
         run: aws sts get-caller-identity
 
       - name: Checkout
-        uses: actions/checkout@v5
+        uses: actions/checkout@08c6903cd8c0fde910a37f88322edcfb5dd907a8
         with:
           fetch-depth: 1
 
-      - uses: hashicorp/setup-terraform@v3
+      - uses: hashicorp/setup-terraform@b9cd54a3c349d3f38e8881555d616ced269862dd
         with:
           terraform_version: "1.12.2"
 
       - name: Terraform Init
         working-directory: ${{ vars.TERRAFORM_DIR_PATH }}
-        run: |
-          export ENVIRONMENT=${{ inputs.environment }}
-          make init
+        run: make init apigee_environment=${{ inputs.apigee_environment }} environment=${{ inputs.environment }} sub_environment=${{ inputs.sub_environment }}
 
       - name: Terraform Plan
         working-directory: ${{ vars.TERRAFORM_DIR_PATH }}
-        run: |
-          make plan environment=${{ inputs.environment }} aws_account_name=int
-
+        run: make plan apigee_environment=${{ inputs.apigee_environment }} environment=${{ inputs.environment }} sub_environment=${{ inputs.sub_environment }}
+        # TODO - save the plan and use it in the apply step
   terraform-apply:
+    if: ${{ vars.SKIP_APPLY != 'true' }}
     needs: terraform-plan
     runs-on: ubuntu-latest
     permissions:
@@ -53,55 +58,61 @@ jobs:
       name: int
     steps:
       - name: Checkout
-        uses: actions/checkout@v5
+        uses: actions/checkout@08c6903cd8c0fde910a37f88322edcfb5dd907a8
 
-      - uses: aws-actions/configure-aws-credentials@v4
+      - uses: aws-actions/configure-aws-credentials@7474bc4690e29a8392af63c5b98e7449536d5c3a
         with:
           aws-region: eu-west-2
           role-to-assume: arn:aws:iam::${{ vars.AWS_ACCOUNT_ID }}:role/auto-ops
           role-session-name: github-actions
 
-      - uses: hashicorp/setup-terraform@v3
+      - uses: hashicorp/setup-terraform@b9cd54a3c349d3f38e8881555d616ced269862dd
         with:
           terraform_version: "1.12.2"
 
       - name: Terraform Init
         working-directory: ${{ vars.TERRAFORM_DIR_PATH }}
-        run: |
-          export ENVIRONMENT=${{ inputs.environment }}
-          make init
+        run: make init apigee_environment=${{ inputs.apigee_environment }} environment=${{ inputs.environment }} sub_environment=${{ inputs.sub_environment }}
 
       - name: Terraform Apply
         working-directory: ${{ vars.TERRAFORM_DIR_PATH }}
-        run: |
-          make apply environment=${{ inputs.environment }} aws_account_name=int
-
+        run: make apply apigee_environment=${{ inputs.apigee_environment }} environment=${{ inputs.environment }} sub_environment=${{ inputs.sub_environment }}
+        # TODO - use a saved plan from the plan step
   e2e-tests:
+    if: ${{ vars.RUN_E2E == 'true' && inputs.sub_environment == vars.ACTIVE_ENVIRONMENT }}
     needs: terraform-apply
-    if: ${{ vars.RUN_E2E == 'true' || inputs.environment == vars.ACTIVE_ENVIRONMENT }}
     runs-on: ubuntu-latest
     permissions:
       id-token: write
       contents: read
     steps:
       - name: Checkout
-        uses: actions/checkout@v5
+        uses: actions/checkout@08c6903cd8c0fde910a37f88322edcfb5dd907a8
 
-      - uses: aws-actions/configure-aws-credentials@v4
+      - uses: aws-actions/configure-aws-credentials@7474bc4690e29a8392af63c5b98e7449536d5c3a
         with:
           aws-region: eu-west-2
           role-to-assume: arn:aws:iam::${{ vars.AWS_ACCOUNT_ID }}:role/auto-ops
           role-session-name: github-actions
 
+      - uses: hashicorp/setup-terraform@b9cd54a3c349d3f38e8881555d616ced269862dd
+        with:
+          terraform_version: "1.12.2"
+
+      - name: Terraform Init
+        working-directory: ${{ vars.TERRAFORM_DIR_PATH }}
+        run: make init apigee_environment=${{ inputs.apigee_environment }} environment=${{ inputs.environment }} sub_environment=${{ inputs.sub_environment }}
+
       - name: Set up Python
         uses: actions/setup-python@v5
         with:
           python-version: "3.11"
 
       - name: Install Poetry
         run: |
-          curl -sSL https://install.python-poetry.org | python3 -
+          curl -sSL https://install.python-poetry.org | python3 - --version 2.1.4
           echo "$HOME/.local/bin" >> $GITHUB_PATH
+          poetry --version
 
       - name: Set Poetry to use Python 3.11
         working-directory: ${{ vars.E2E_DIR_PATH }}
@@ -113,27 +124,39 @@ jobs:
         run: |
           poetry install --no-root
 
+      - name: Install oathtool
+        run: sudo apt-get update && sudo apt-get install -y oathtool
+
+      - name: Get JWT token for apigee
+        env:
+          APIGEE_USERNAME: ${{ vars.APIGEE_USERNAME }}
+          APIGEE_PASSWORD: ${{ secrets.APIGEE_PASSWORD }}
+          APIGEE_OAUTH_TOKEN: ${{ secrets.APIGEE_BASIC_AUTH_TOKEN }}
+          APIGEE_OTP_SECRET: ${{ secrets.APIGEE_OTP_KEY }}
+        run: |
+          CODE=$(oathtool --totp -b "$APIGEE_OTP_SECRET")
+          echo "::add-mask::$CODE"
+          echo "Requesting access token from Apigee..."
+          response=$(curl -s -X POST "https://login.apigee.com/oauth/token" \
+            -H "Content-Type: application/x-www-form-urlencoded" \
+            -H "Accept: application/json;charset=utf-8" \
+            -H "Authorization: Basic $APIGEE_BASIC_AUTH_TOKEN" \
+            -d "username=$APIGEE_USERNAME&password=$APIGEE_PASSWORD&mfa_token=$CODE&grant_type=password")
+          token=$(echo "$response" | jq -e -r '.access_token')
+          if [[ -z "$token" ]]; then
+            echo "Failed to retrieve access token"
+            exit 1
+          fi
+          echo "::add-mask::$token"
+          echo "APIGEE_ACCESS_TOKEN=$token" >> $GITHUB_ENV
+
       - name: Run e2e tests
         working-directory: ${{ vars.E2E_DIR_PATH }}
+        env:
+          APIGEE_ACCESS_TOKEN: ${{ env.APIGEE_ACCESS_TOKEN }}
+          APIGEE_USERNAME: [email protected]
         run: |
-          apigee_token=$(aws ssm get-parameter \
-            --name "/imms/apigee/non-prod/token" \
-            --with-decryption \
-            --query "Parameter.Value" \
-            --output text)
-
-          status_api_key=$(aws ssm get-parameter \
-            --name "/imms/apigee/non-prod/status-api-key" \
-            --with-decryption \
-            --query "Parameter.Value" \
-            --output text)
-
-          export APIGEE_ACCESS_TOKEN=$apigee_token
-          export [email protected]
-          export APIGEE_ENVIRONMENT=int
-          export STATUS_API_KEY=$status_api_key
-          export PROXY_NAME=immunisation-fhir-api-internal-dev
-          export SERVICE_BASE_PATH=immunisation-fhir-api/FHIR/R4
-          export SSO_LOGIN_URL=https://login.apigee.com
-
+          export APIGEE_ENVIRONMENT=internal-dev
+          export PROXY_NAME=immunisation-fhir-api-${{ inputs.sub_environment }}
+          export SERVICE_BASE_PATH=immunisation-fhir-api/FHIR/R4-${{ inputs.sub_environment }}
           make run-immunization

.github/workflows/sonarcloud.yml

@@ -64,7 +64,7 @@ jobs:
         continue-on-error: true
         run: |
           poetry install
-          poetry run coverage run -m unittest discover -s "./tests" -p "*batch*.py" || echo "recordforwarder tests failed" >> ../failed_tests.txt
+          poetry run coverage run -m unittest discover -p "*batch*.py" || echo "recordforwarder tests failed" >> ../failed_tests.txt
           poetry run coverage xml -o ../recordforwarder-coverage.xml
 
       - name: Run unittest with coverage-ack-lambda

.tool-versions

@@ -1,3 +1,3 @@
-poetry 2.1.2
+poetry 2.1.4
 nodejs 23.11.0
 python 3.11.12

azure/azure-pr-teardown-pipeline.yml

@@ -47,8 +47,8 @@ jobs:
           export AWS_PROFILE=apim-dev
 
           cd terraform
-          make init environment="dev" sub_environment="$WORKSPACE" bucket_name="immunisation-internal-dev-terraform-state-files"
-          make workspace sub_environment="$WORKSPACE"
+          make init apigee_environment=internal-dev environment=dev sub_environment="$WORKSPACE"
+          make workspace apigee_environment=internal-dev environment=dev sub_environment="$WORKSPACE"
 
           # Extract values from Terraform state before destroying
           ID_SYNC_QUEUE_ARN=$(make -s output name=id_sync_queue_arn)
@@ -76,6 +76,6 @@ jobs:
           export AWS_PROFILE=apim-dev
 
           cd terraform
-          make destroy environment="dev" sub_environment="$WORKSPACE" bucket_name="immunisation-internal-dev-terraform-state-files"
+          make destroy apigee_environment=internal-dev environment=dev sub_environment="$WORKSPACE"
         displayName: Destroy terraform PR workspace and linked resources
-        retryCountOnTaskFailure: 2
+        retryCountOnTaskFailure: 2

azure/templates/post-deploy.yml

@@ -60,7 +60,7 @@ steps:
         echo pr_no: $pr_no
 
         cd terraform
-        make init
+        make init environment=${{ parameters.aws_account_type }} sub_environment=$workspace
         make apply environment=${{ parameters.aws_account_type }} sub_environment=$workspace
 
         AWS_DOMAIN_NAME=$(make -s output name=service_domain_name)

azure/templates/post-prod-deploy.yml

@@ -26,7 +26,6 @@ steps:
       set -e
       if ! [[ $APIGEE_ENVIRONMENT =~ .*-*sandbox ]]; then
         export AWS_PROFILE=apim-dev
-        aws_account_no="$(aws sts get-caller-identity --query Account --output text)"
 
         service_name=$(FULLY_QUALIFIED_SERVICE_NAME)
 
@@ -35,12 +34,12 @@ steps:
         echo sandbox with following parameters:
         echo workspace: $workspace
         echo AWS environment: $APIGEE_ENVIRONMENT
-        
+
           cd terraform
 
-        make init
-        make apply aws_account_no=${aws_account_no} environment=$workspace
+        make init environment=${{ parameters.aws_account_type }} sub_environment=$workspace
+        make apply environment=${{ parameters.aws_account_type }} sub_environment=$workspace
       fi
     displayName: Apply Terraform
     workingDirectory: "$(Pipeline.Workspace)/s/$(SERVICE_NAME)"
-    retryCountOnTaskFailure: 2
+    retryCountOnTaskFailure: 2

backend/src/batch/__init__.py

@@ -0,0 +1,31 @@
+import copy
+
+from models.errors import MessageNotSuccessfulError
+
+
+class BatchFilenameToEventsMapper:
+    FILENAME_NOT_PRESENT_ERROR_MSG = "Filename data was not present"
+
+    def __init__(self):
+        self._filename_to_events_map: dict[str, list[dict]] = {}
+
+    def add_event(self, event: dict) -> None:
+        filename_key = self._make_key(event)
+
+        if filename_key not in self._filename_to_events_map:
+            self._filename_to_events_map[filename_key] = [event]
+            return
+
+        self._filename_to_events_map[filename_key].append(event)
+
+    def get_map(self) -> dict[str, list[dict]]:
+        return copy.deepcopy(self._filename_to_events_map)
+
+    def _make_key(self, event: dict) -> str:
+        file_key = event.get("file_key")
+        created_at_string = event.get("created_at_formatted_string")
+
+        if not file_key or not created_at_string:
+            raise MessageNotSuccessfulError(self.FILENAME_NOT_PRESENT_ERROR_MSG)
+
+        return f"{file_key}_{created_at_string}"

backend/src/batch/batch_filename_to_events_mapper.py

@@ -5,6 +5,8 @@
 import base64
 import time
 import logging
+
+from batch.batch_filename_to_events_mapper import BatchFilenameToEventsMapper
 from fhir_batch_repository import create_table
 from fhir_batch_controller import ImmunizationBatchController, make_batch_controller
 from clients import sqs_client
@@ -59,7 +61,7 @@ def forward_lambda_handler(event, _):
     """Forward each row to the Imms API"""
     logger.info("Processing started")
     table = create_table()
-    array_of_messages = []
+    filename_to_events_mapper = BatchFilenameToEventsMapper()
     array_of_identifiers = []
     controller = make_batch_controller()
 
@@ -101,23 +103,20 @@ def forward_lambda_handler(event, _):
                 array_of_identifiers.append(identifier)
 
             imms_id = forward_request_to_dynamo(incoming_message_body, table, identifier_already_present, controller)
-            array_of_messages.append({**base_outgoing_message_body, "imms_id": imms_id})
+            filename_to_events_mapper.add_event({**base_outgoing_message_body, "imms_id": imms_id})
 
         except Exception as error:  # pylint: disable = broad-exception-caught
-            array_of_messages.append(
+            filename_to_events_mapper.add_event(
                 {**base_outgoing_message_body, "diagnostics": create_diagnostics_dictionary(error)}
             )
             logger.error("Error processing message: %s", error)
 
     # Send to SQS
-    sqs_message_body = json.dumps(array_of_messages)
-    message_len = len(sqs_message_body)
-    logger.info(f"total message length:{message_len}")
-    message_group_id = f"{file_key}_{created_at_formatted_string}"
-    if message_len < 256 * 1024:
-        sqs_client.send_message(QueueUrl=QUEUE_URL, MessageBody=sqs_message_body, MessageGroupId=message_group_id)
-    else:
-        logger.info("Message size exceeds 256 KB limit.Sending to sqs failed")
+    for filename_key, events in filename_to_events_mapper.get_map().items():
+        sqs_message_body = json.dumps(events)
+        logger.info(f"total message length:{len(sqs_message_body)}")
+
+        sqs_client.send_message(QueueUrl=QUEUE_URL, MessageBody=sqs_message_body, MessageGroupId=filename_key)
 
 
 if __name__ == "__main__":