Merge pull request #6280 from NHSDigital/next

Author: thomasleese

.github/pull_request_template.md

@@ -1,9 +1,7 @@
-## Screenshots
-
-## Pre-release tasks
+...
 
-- ...
+[Jira Issue - MAV-XXX](https://nhsd-jira.digital.nhs.uk/browse/MAV-XXX)
 
-## Post-release tasks
+## Screenshots
 
-- ...
+...

.github/workflows/build-and-push-image.yml

@@ -29,12 +29,16 @@ jobs:
     strategy:
       fail-fast: true
       matrix:
-        image_type: ["webapp", "ops"]
+        image_type: [webapp, ops]
     permissions:
       id-token: write
     outputs:
-      webapp-build-needed: ${{ steps.check-dev-image.outputs.webapp-build-needed || steps.check-prod-image.outputs.webapp-build-needed }}
-      ops-build-needed: ${{ steps.check-dev-image.outputs.ops-build-needed || steps.check-prod-image.outputs.ops-build-needed }}
+      webapp-build-needed: >-
+        ${{ steps.check-dev-image.outputs.webapp-build-needed ||
+        steps.check-prod-image.outputs.webapp-build-needed }}
+      ops-build-needed: >-
+        ${{ steps.check-dev-image.outputs.ops-build-needed ||
+        steps.check-prod-image.outputs.ops-build-needed }}
     steps:
       - name: Configure AWS Dev Credentials
         uses: aws-actions/configure-aws-credentials@v6
@@ -44,7 +48,8 @@ jobs:
       - name: Check if dev image exists
         id: check-dev-image
         run: |
-          if aws ecr describe-images --repository-name mavis/${{ matrix.image_type }} --image-ids "imageTag=$git_ref" > /dev/null 2>&1; then
+          if aws ecr describe-images --repository-name mavis/${{ matrix.image_type }} \
+              --image-ids "imageTag=$git_ref" > /dev/null 2>&1; then
             echo "Dev image with given tag already exists"
           else
             echo "Dev image does not exist. Build needed"
@@ -60,7 +65,8 @@ jobs:
         if: env.PUSH_IMAGE_TO_PRODUCTION == 'true'
         id: check-prod-image
         run: |
-          if aws ecr describe-images --repository-name mavis/${{ matrix.image_type }} --image-ids "imageTag=$git_ref" > /dev/null 2>&1; then
+          if aws ecr describe-images --repository-name mavis/${{ matrix.image_type }} \
+              --image-ids "imageTag=$git_ref" > /dev/null 2>&1; then
             echo "Production image with given tag already exists"
           else
             echo "Production image does not exist. Build needed"
@@ -75,57 +81,32 @@ jobs:
     steps:
       - name: Set aws roles
         id: determine-aws-roles
+        # yamllint disable rule:line-length
         run: |
           if [ "$PUSH_IMAGE_TO_PRODUCTION" = "true" ]; then
             echo 'aws-roles=["arn:aws:iam::393416225559:role/GithubDeployMavisAndInfrastructure", "arn:aws:iam::820242920762:role/GithubDeployMavisAndInfrastructure"]' >> "$GITHUB_OUTPUT"
           else
             echo 'aws-roles=["arn:aws:iam::393416225559:role/GithubDeployMavisAndInfrastructure"]' >> "$GITHUB_OUTPUT"
           fi
-  build:
-    needs: check-image-presence
-    if: needs.check-image-presence.outputs.webapp-build-needed == 'true' || needs.check-image-presence.outputs.ops-build-needed  == 'true'
+        # yamllint enable rule:line-length
+  build-and-push:
+    needs: [check-image-presence, define-matrix]
+    if: >-
+      needs.check-image-presence.outputs.webapp-build-needed == 'true' ||
+      needs.check-image-presence.outputs.ops-build-needed == 'true'
     runs-on: ubuntu-latest
     permissions:
       id-token: write
+    strategy:
+      matrix:
+        aws-role: ${{ fromJSON(needs.define-matrix.outputs.aws-roles) }}
     steps:
       - name: Checkout code
         uses: actions/checkout@v6
         with:
           ref: ${{ env.git_ref }}
       - name: Write build SHA
         run: git rev-parse HEAD > public/sha
-      - name: Build webapp docker image
-        run: docker build -t "mavis-webapp:latest" .
-      - name: Save web image
-        run: docker save -o image.tar mavis-webapp:latest
-      - name: Upload web image
-        uses: actions/upload-artifact@v7
-        with:
-          name: webapp-image
-          path: image.tar
-      - name: Build ops docker image
-        run: docker build -f ops.Dockerfile -t "mavis-ops:latest" .
-      - name: Save ops image
-        run: docker save -o image.tar mavis-ops:latest
-      - name: Upload ops image
-        uses: actions/upload-artifact@v7
-        with:
-          name: ops-image
-          path: image.tar
-  push:
-    runs-on: ubuntu-latest
-    needs: [build, define-matrix]
-    permissions:
-      id-token: write
-    strategy:
-      matrix:
-        aws-role: ${{ fromJSON(needs.define-matrix.outputs.aws-roles) }}
-        image_type: ["webapp", "ops"]
-    steps:
-      - name: Download Docker image
-        uses: actions/download-artifact@v8
-        with:
-          name: ${{ matrix.image_type }}-image
       - name: Configure AWS Credentials
         uses: aws-actions/configure-aws-credentials@v6
         with:
@@ -134,9 +115,34 @@ jobs:
       - name: Login to ECR
         id: login-ecr
         uses: aws-actions/amazon-ecr-login@v2
-      - name: Load Docker image
-        run: docker load -i image.tar
-      - name: Tag Docker image
-        run: docker tag mavis-${{ matrix.image_type }}:latest "${{ steps.login-ecr.outputs.registry }}/mavis/${{ matrix.image_type }}":"$git_ref"
-      - name: Push Docker image
-        run: docker push "${{ steps.login-ecr.outputs.registry }}/mavis/${{ matrix.image_type }}":"$git_ref"
+      - name: Set up Docker Buildx
+        uses: docker/setup-buildx-action@v4
+        # yamllint disable rule:line-length
+      - name: Build and push webapp image
+        if: needs.check-image-presence.outputs.webapp-build-needed == 'true'
+        uses: docker/build-push-action@v7
+        with:
+          context: .
+          push: true
+          tags: ${{ steps.login-ecr.outputs.registry }}/mavis/webapp:${{ env.git_ref }}
+          cache-from:
+            type=registry,ref=${{ steps.login-ecr.outputs.registry }}/mavis/webapp:buildcache
+          cache-to:
+            type=registry,ref=${{ steps.login-ecr.outputs.registry
+            }}/mavis/webapp:buildcache,mode=max,image-manifest=true,oci-mediatypes=true
+      - name: Build and push ops image
+        if: needs.check-image-presence.outputs.ops-build-needed == 'true'
+        uses: docker/build-push-action@v7
+        with:
+          context: .
+          file: ops.Dockerfile
+          push: true
+          tags: ${{ steps.login-ecr.outputs.registry }}/mavis/ops:${{ env.git_ref }}
+          build-args: |
+            REPOSITORY=${{ steps.login-ecr.outputs.registry }}/mavis/webapp
+            IMAGE_TAG=${{ env.git_ref }}
+          cache-from: type=registry,ref=${{ steps.login-ecr.outputs.registry }}/mavis/ops:buildcache
+          cache-to:
+            type=registry,ref=${{ steps.login-ecr.outputs.registry
+            }}/mavis/ops:buildcache,mode=max,image-manifest=true,oci-mediatypes=true
+            # yamllint enable rule:line-length

.github/workflows/call-end-to-end-tests.yml

@@ -0,0 +1,45 @@
+name: Call end-to-end tests
+
+on:
+  workflow_call:
+    inputs:
+      cross_service_tests:
+        required: true
+        type: boolean
+      endpoint:
+        required: true
+        type: string
+      github_ref:
+        required: true
+        type: string
+    secrets:
+      HTTP_AUTH_TOKEN_FOR_TESTS:
+        description: HTTP Basic Auth token for the environment under test
+        required: false
+      MAVIS_TESTING_REPO_ACCESS_TOKEN:
+        description: Access token for the manage-vaccinations-in-schools-testing repository
+        required: false
+      IMMS_API_KEY_FOR_TESTS:
+        description: API key to use NHS Immunisations API
+        required: false
+      IMMS_API_KID_FOR_TESTS:
+        description: API KID to use NHS Immunisations API
+        required: false
+      IMMS_API_PEM_FOR_TESTS:
+        description: API PEM to use NHS Immunisations API
+        required: false
+
+jobs:
+  call-tests:
+    # yamllint disable-line rule:line-length
+    uses: NHSDigital/manage-vaccinations-in-schools-testing/.github/workflows/end-to-end-tests.yaml@main
+    with:
+      cross_service_tests: ${{ inputs.cross_service_tests }}
+      github_ref: ${{ inputs.github_ref }}
+      endpoint: ${{ inputs.endpoint }}
+    secrets:
+      HTTP_AUTH_TOKEN_FOR_TESTS: ${{ secrets.HTTP_AUTH_TOKEN_FOR_TESTS }}
+      MAVIS_TESTING_REPO_ACCESS_TOKEN: ${{ secrets.MAVIS_TESTING_REPO_ACCESS_TOKEN }}
+      IMMS_API_KEY: ${{ secrets.IMMS_API_KEY_FOR_TESTS }}
+      IMMS_API_KID: ${{ secrets.IMMS_API_KID_FOR_TESTS }}
+      IMMS_API_PEM: ${{ secrets.IMMS_API_PEM_FOR_TESTS }}

.github/workflows/continuous-deployment.yml

@@ -12,7 +12,6 @@ jobs:
       id-token: write
     uses: ./.github/workflows/test.yml
   deploy:
-    needs: test
     strategy:
       fail-fast: false
       matrix:
@@ -23,18 +22,29 @@ jobs:
     with:
       environment: ${{ matrix.environment }}
       server_types: all
+      run_pre_deploy_migrations: false
+  end-to-end-tests:
+    needs: [deploy]
+    uses: ./.github/workflows/call-end-to-end-tests.yml
+    secrets: inherit
+    with:
+      cross_service_tests: true
+      endpoint: https://qa.mavistesting.com
+      github_ref: main
   slack-notification:
-    needs: [deploy, test]
+    needs: [deploy, test, end-to-end-tests]
     runs-on: ubuntu-latest
     permissions: {}
     if: ${{ !cancelled() }}
     steps:
-      - name: Send Slack notification
+      - name: Send Slack notification about failed deployment
         if: needs.test.result == 'failure' || needs.deploy.result == 'failure'
         uses: slackapi/slack-github-action@91efab103c0de0a537f72a35f6b8cda0ee76bf0a
         with:
-          webhook: ${{ secrets.SLACK_MAVIS_ALERTS_WEBHOOK_URL }}
+          webhook: ${{ secrets.SLACK_MAVIS_TECH_WEBHOOK_URL }}
           webhook-type: incoming-webhook
+          errors: true
+          # yamllint disable rule:line-length
           payload: |
             text: ":rotating_light: Continuous Deployment failed :rotating_light:"
             blocks:
@@ -43,3 +53,21 @@ jobs:
                   type: "mrkdwn"
                   text: ":rotating_light: *Continuous Deployment failed* :rotating_light:\n\n \
                           <${{ github.server_url }}/${{ github.repository }}/actions/runs/${{ github.run_id }}|View workflow run>"
+          # yamllint enable rule:line-length
+      - name: Send Slack notification about failed E2E tests
+        if: needs.end-to-end-tests.result == 'failure'
+        uses: slackapi/slack-github-action@91efab103c0de0a537f72a35f6b8cda0ee76bf0a
+        with:
+          webhook: ${{ secrets.SLACK_MAVIS_TECH_WEBHOOK_URL }}
+          webhook-type: incoming-webhook
+          errors: true
+          # yamllint disable rule:line-length
+          payload: |
+            text: ":rotating_light: E2E tests failed on next :rotating_light:"
+            blocks:
+              - type: "section"
+                text:
+                  type: "mrkdwn"
+                  text: ":rotating_light: *E2E tests failed on next* :rotating_light:\n\n \
+                          <${{ github.server_url }}/${{ github.repository }}/actions/runs/${{ github.run_id }}|View workflow run>"
+          # yamllint enable rule:line-length

.github/workflows/create_dockerized_db.yml

@@ -72,13 +72,18 @@ jobs:
       - name: Login to ECR
         id: login-ecr
         uses: aws-actions/amazon-ecr-login@v2
+      # yamllint disable rule:line-length
       - name: get github ref short
         id: github-ref
         run: |
-          git_ref=$(git rev-parse ${{ inputs.github_ref || github.ref_name == 'next' && 'origin/next' || github.ref_name }} )
+          git_ref=$(git rev-parse ${{ inputs.github_ref || github.ref_name == 'next' && 'origin/next' || github.ref_name }})
           echo "ref=$git_ref" >> "$GITHUB_OUTPUT"
       - name: Commit postgres container with database
-        run: |
-          docker commit database "${{ steps.login-ecr.outputs.registry }}/mavis/development/postgres_db:${{ steps.github-ref.outputs.ref }}"
+        run: >-
+          docker commit database "${{ steps.login-ecr.outputs.registry
+          }}/mavis/development/postgres_db:${{ steps.github-ref.outputs.ref }}"
       - name: Push image
-        run: docker push "${{ steps.login-ecr.outputs.registry }}/mavis/development/postgres_db:${{ steps.github-ref.outputs.ref }}"
+        run: >-
+          docker push "${{ steps.login-ecr.outputs.registry }}/mavis/development/postgres_db:${{
+          steps.github-ref.outputs.ref }}"
+      # yamllint enable rule:line-length

.github/workflows/data-replication-pipeline.yml

@@ -1,5 +1,7 @@
 name: Deploy data replication
-run-name: Deploy data replication from ${{ inputs.git_ref_to_deploy || github.ref_name }} to ${{ inputs.environment }}
+run-name: >-
+  Deploy data replication from ${{ inputs.git_ref_to_deploy || github.ref_name }} to ${{
+  inputs.environment }}
 
 on:
   workflow_dispatch:
@@ -16,10 +18,10 @@ on:
           - sandbox-alpha
           - sandbox-beta
       git_ref_to_deploy:
-        description:
-          | # Use blank unicode character (U+2800) to force line-break
+        description: | # Use blank unicode character (U+2800) to force line-break
           Use code from: ⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀
-          (Git ref to deploy, for example, a tag, branch name or commit SHA. Will use workflow ref if not provided.)
+          (Git ref to deploy, for example, a tag, branch name or commit SHA. Will use workflow ref
+          if not provided.)
         type: string
   workflow_call:
     inputs:
@@ -37,9 +39,10 @@ permissions: {}
 env:
   environment: ${{ inputs.environment }}
   git_ref_to_deploy: ${{ inputs.git_ref_to_deploy || github.sha }}
-  aws_role: ${{ inputs.environment == 'production'
-    && 'arn:aws:iam::820242920762:role/GithubDeployDataReplicationInfrastructure'
-    || 'arn:aws:iam::393416225559:role/GithubDeployDataReplicationInfrastructure' }}
+  aws_role:
+    ${{ inputs.environment == 'production' &&
+    'arn:aws:iam::820242920762:role/GithubDeployDataReplicationInfrastructure' ||
+    'arn:aws:iam::393416225559:role/GithubDeployDataReplicationInfrastructure' }}
   aws_account_id: ${{ inputs.environment == 'production' && '820242920762' || '393416225559' }}
 
 concurrency:
@@ -124,12 +127,18 @@ jobs:
         id: create-task-definition
         uses: aws-actions/amazon-ecs-render-task-definition@v1
         with:
-          task-definition-family: "mavis-data-replication-task-definition-${{ inputs.environment }}-template"
-          container-name: "application"
-          image: "${{ env.aws_account_id }}.dkr.ecr.eu-west-2.amazonaws.com/mavis/webapp@${{ steps.get-image-digest.outputs.digest }}"
+          task-definition-family:
+            mavis-data-replication-task-definition-${{ inputs.environment }}-template
+          container-name: application
+          # yamllint disable-line rule:line-length
+          image:
+            ${{ env.aws_account_id }}.dkr.ecr.eu-west-2.amazonaws.com/mavis/webapp@${{
+            steps.get-image-digest.outputs.digest }}
           environment-variables: ${{ steps.parse-environment-variables.outputs.parsed_env_vars }}
       - name: Rename task definition file
-        run: mv ${{ steps.create-task-definition.outputs.task-definition }} ${{ runner.temp }}/data-replication-task-definition.json
+        run: >-
+          mv ${{ steps.create-task-definition.outputs.task-definition }} ${{ runner.temp
+          }}/data-replication-task-definition.json
       - name: Upload artifact for data-replication task definition
         uses: actions/upload-artifact@v7
         with:
@@ -148,6 +157,7 @@ jobs:
         with:
           webhook: ${{ secrets.SLACK_MAVIS_TECH_WEBHOOK_URL }}
           webhook-type: incoming-webhook
+          # yamllint disable rule:line-length
           payload: |
             text: ":hourglass: Approval required :hourglass:"
             blocks:
@@ -161,6 +171,7 @@ jobs:
                     text: "<${{ github.server_url }}/${{ github.repository }}/actions/runs/${{ github.run_id }}|View workflow run>"
                   - type: "mrkdwn"
                     text: "*Triggered by:*\n${{ github.actor }}"
+          # yamllint enable rule:line-length
 
   approve-deployments:
     name: Wait for approval if required
@@ -202,6 +213,7 @@ jobs:
           force-new-deployment: true
           wait-for-service-stability: true
       - name: Check if deployment was successful
+        # yamllint disable rule:line-length
         run: |
           current_task_definition_arn=$(aws ecs describe-services --cluster "mavis-$environment-data-replication" --services "mavis-$environment-data-replication" --query services[0].deployments[0].taskDefinition | jq -r ".")
           new_task_definition_arn=${{ steps.ecs-deploy.outputs.task-definition-arn }}
@@ -211,3 +223,4 @@ jobs:
             echo "Task definition arns don't match, likely due to a rollback to the previous version. Deployment failed."
             exit 1
           fi
+        # yamllint enable rule:line-length