From 301f5398b5dc0803e9a455c7beff41edbcfd0201 Mon Sep 17 00:00:00 2001 From: Michael Pond Date: Mon, 1 Dec 2025 13:31:15 -0500 Subject: [PATCH 1/8] adding files for xmas time! --- .github/workflows/helmfile_dev_apply.yaml | 131 ++++++++++++++++++++++ scripts/lambdaParamStoreUpdatesDev.sh | 39 +++++++ 2 files changed, 170 insertions(+) create mode 100644 .github/workflows/helmfile_dev_apply.yaml create mode 100644 scripts/lambdaParamStoreUpdatesDev.sh diff --git a/.github/workflows/helmfile_dev_apply.yaml b/.github/workflows/helmfile_dev_apply.yaml new file mode 100644 index 000000000..86b17dec6 --- /dev/null +++ b/.github/workflows/helmfile_dev_apply.yaml @@ -0,0 +1,131 @@ +name: Dev - Helmfile Apply + +on: + workflow_dispatch: + push: + branches: + - dev + paths: + - "helmfile/**" + - ".github/workflows/helmfile_dev_apply.yaml" + +env: + GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} + HELMFILE_FILE_PATH: ${{ github.workspace }}/helmfile + DEV_AWS_ACCOUNT: ${{ secrets.DEV_AWS_ACCOUNT_ID }} + OP_SERVICE_ACCOUNT_TOKEN: ${{ secrets.OP_SERVICE_ACCOUNT_TOKEN_DEV }} + ACCOUNT_ID: ${{ secrets.DEV_AWS_ACCOUNT_ID }} + AWS_DEFAULT_REGION: ca-central-1 + +permissions: + id-token: write + contents: write + pull-requests: write + +jobs: + helmfile-apply: + runs-on: ubuntu-latest + steps: + - name: Inject token authentication + run: | + git config --global url."https://x-access-token:${{ secrets.GITHUB_TOKEN }}@github.com/".insteadOf "https://github.com/" + + - name: Configure credentials to Notify using OIDC + uses: aws-actions/configure-aws-credentials@ececac1a45f3b08a01d2dd070d28d111c5fe6722 # v4.1.0 + with: + role-to-assume: arn:aws:iam::${{env.ACCOUNT_ID}}:role/notification-manifests-apply + role-session-name: NotifyManifestsApplyDev + aws-region: ${{ env.AWS_DEFAULT_REGION }} + + - name: Checkout + uses: actions/checkout@f43a0e5ff2bd294095638e18286ca9a3d1956744 # v3.6.0 + with: + fetch-depth: 0 + + - name: Setup helmfile + uses: mamezou-tech/setup-helmfile@fd46979d2984c886929c416fbdf859b1c5efa0ea # v2.1.0 + with: + install-kubectl: yes + install-helm: yes + helmfile-version: "v0.151.0" + helm-s3-plugin-version: "v0.16.2" + + - name: Install OpenVPN + run: | + sudo apt update + sudo apt install -y openvpn openvpn-systemd-resolved + + - name: Install 1Pass CLI + run: | + curl -o 1pass.deb https://downloads.1password.com/linux/debian/amd64/stable/1password-cli-amd64-latest.deb + sudo dpkg -i 1pass.deb + + - name: Setup Terraform tools + uses: cds-snc/terraform-tools-setup@v1 + env: + CONFTEST_VERSION: 0.30.0 + TERRAFORM_VERSION: 1.9.5 + TERRAGRUNT_VERSION: 0.66.9 + TF_SUMMARIZE_VERSION: 0.2.3 + + - name: Retrieve VPN Config + run: | + scripts/createVPNConfig.sh dev 2> /dev/null + + - name: Connect to VPN + uses: "kota65535/github-openvpn-connect-action@cd2ed8a90cc7b060dc4e001143e811b5f7ea0af5" # v3.1.0 + with: + config_file: /var/tmp/dev.ovpn + echo_config: false + + - name: Configure kubeconfig + run: | + aws eks update-kubeconfig --name notification-canada-ca-dev-eks-cluster --alias dev + + - name: Load Context Variables + run: | + ./helmfile/getContext.sh -g + + - name: Run Database Upgrade Helmfile Sync + uses: ./.github/actions/db-migration + with: + environment: "dev" + namespace: "notification-canada-ca" + app_label: "notify-database" + timeout: "400s" + + - name: Helmfile apply + if: ${{ success() }} + id: helmfile_apply + run: | + pushd helmfile + helmfile --environment dev -l 'tier=crd' apply + helmfile --environment dev -l 'app!=notify-database,tier!=crd' apply + popd + + - name: Save ENV vars and secrets to AWS Param Store + if: ${{ success() }} + run: | + sleep 20 + source ./scripts/lambdaParamStoreUpdatesDev.sh -g + echo DIFF=$DIFF + echo "ENV_DIFF=$DIFF" >> $GITHUB_ENV + + - name: Force api-lambda to redeploy on environment changes + if: env.ENV_DIFF != '0' && ${{ success() }} + uses: ./.github/actions/update-lambda-function + with: + alias-name: latest + function-name: api-lambda-dev + + clear-cache: + needs: helmfile-apply + uses: ./.github/workflows/clear-notify-cache.yaml + with: + environment: dev + cache_clear_user_name: CACHE_CLEAR_USER + api_url: https://api.dev.notification.cdssandbox.xyz + secrets: + CACHE_CLEAR_CLIENT_SECRET: ${{ secrets.DEV_CACHE_CLEAR_CLIENT_SECRET }} + + \ No newline at end of file diff --git a/scripts/lambdaParamStoreUpdatesDev.sh b/scripts/lambdaParamStoreUpdatesDev.sh new file mode 100644 index 000000000..a90ed7b2e --- /dev/null +++ b/scripts/lambdaParamStoreUpdatesDev.sh @@ -0,0 +1,39 @@ +#!/bin/bash + +set -euo pipefail + +NAMESPACE="notification-canada-ca" +params_buffer="" + +get_env_and_secrets() { + local deployment_name=$1 + local secret_name=${2:-$deployment_name} + + while IFS= read -r env_var; do + [[ -n "$env_var" ]] && params_buffer+="$env_var"$'\n' + done < <(kubectl get deployment "$deployment_name" -n "$NAMESPACE" -o jsonpath='{.spec.template.spec.containers[*].env[*]}' \ + | jq -r 'select(.value != null and .value != "") | .name + "=" + .value') + + while IFS= read -r secret_var; do + [[ -n "$secret_var" ]] && params_buffer+="$secret_var"$'\n' + done < <(kubectl get secret "$secret_name" -n "$NAMESPACE" -o jsonpath='{.data}' \ + | jq -r 'to_entries[] | "\(.key)=\(.value | @base64d)"') +} + +# API +params_buffer="" +get_env_and_secrets "notify-api" +params_api=$(printf '%s' "$params_buffer" | sed '/^$/d' | sort -u) +aws ssm get-parameters --region ca-central-1 --with-decryption --names ENVIRONMENT_VARIABLES --query 'Parameters[*].Value' --output text > .previous.env +aws ssm put-parameter --region ca-central-1 --name ENVIRONMENT_VARIABLES --type SecureString --key-id alias/aws/ssm --value "$params_api" --tier "Intelligent-Tiering" --overwrite +aws ssm get-parameters --region ca-central-1 --with-decryption --names ENVIRONMENT_VARIABLES --query 'Parameters[*].Value' --output text > .new.env +DIFF_OUTPUT=$(diff -B .new.env .previous.env || true) +DIFF=$(printf '%s' "$DIFF_OUTPUT" | wc -l | tr -d ' ') +echo "DIFF=$DIFF" +export DIFF + +# ADMIN +params_buffer="" +get_env_and_secrets "notify-admin" +params_admin=$(printf '%s' "$params_buffer" | sed '/^$/d' | sort -u) +aws ssm put-parameter --region ca-central-1 --name ENVIRONMENT_VARIABLES_ADMIN --type SecureString --key-id alias/aws/ssm --value "$params_admin" --tier "Intelligent-Tiering" --overwrite From 4282b27bb4c52dd3fd1c86b14c737c5b6434c809 Mon Sep 17 00:00:00 2001 From: Michael Pond Date: Tue, 2 Dec 2025 10:07:25 -0500 Subject: [PATCH 2/8] adding dispatch workflow --- ...ply.yaml => dev_branch_deploy_images.yaml} | 73 ++++--------------- 1 file changed, 16 insertions(+), 57 deletions(-) rename .github/workflows/{helmfile_dev_apply.yaml => dev_branch_deploy_images.yaml} (55%) diff --git a/.github/workflows/helmfile_dev_apply.yaml b/.github/workflows/dev_branch_deploy_images.yaml similarity index 55% rename from .github/workflows/helmfile_dev_apply.yaml rename to .github/workflows/dev_branch_deploy_images.yaml index 86b17dec6..9b1f5fa69 100644 --- a/.github/workflows/helmfile_dev_apply.yaml +++ b/.github/workflows/dev_branch_deploy_images.yaml @@ -1,45 +1,39 @@ -name: Dev - Helmfile Apply +name: Dev - Deploy dev branch images on: - workflow_dispatch: - push: - branches: - - dev - paths: - - "helmfile/**" - - ".github/workflows/helmfile_dev_apply.yaml" + repository_dispatch: + types: [deploy-dev-branch-images] env: GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} HELMFILE_FILE_PATH: ${{ github.workspace }}/helmfile DEV_AWS_ACCOUNT: ${{ secrets.DEV_AWS_ACCOUNT_ID }} - OP_SERVICE_ACCOUNT_TOKEN: ${{ secrets.OP_SERVICE_ACCOUNT_TOKEN_DEV }} ACCOUNT_ID: ${{ secrets.DEV_AWS_ACCOUNT_ID }} AWS_DEFAULT_REGION: ca-central-1 permissions: id-token: write - contents: write - pull-requests: write + contents: read jobs: - helmfile-apply: + deploy-dev-images: runs-on: ubuntu-latest steps: - name: Inject token authentication run: | git config --global url."https://x-access-token:${{ secrets.GITHUB_TOKEN }}@github.com/".insteadOf "https://github.com/" - - name: Configure credentials to Notify using OIDC + - name: Configure credentials to Notify dev using OIDC uses: aws-actions/configure-aws-credentials@ececac1a45f3b08a01d2dd070d28d111c5fe6722 # v4.1.0 with: - role-to-assume: arn:aws:iam::${{env.ACCOUNT_ID}}:role/notification-manifests-apply + role-to-assume: arn:aws:iam::${{ env.ACCOUNT_ID }}:role/notification-manifests-apply role-session-name: NotifyManifestsApplyDev aws-region: ${{ env.AWS_DEFAULT_REGION }} - - name: Checkout + - name: Checkout manifests repo uses: actions/checkout@f43a0e5ff2bd294095638e18286ca9a3d1956744 # v3.6.0 with: + ref: dev fetch-depth: 0 - name: Setup helmfile @@ -55,19 +49,6 @@ jobs: sudo apt update sudo apt install -y openvpn openvpn-systemd-resolved - - name: Install 1Pass CLI - run: | - curl -o 1pass.deb https://downloads.1password.com/linux/debian/amd64/stable/1password-cli-amd64-latest.deb - sudo dpkg -i 1pass.deb - - - name: Setup Terraform tools - uses: cds-snc/terraform-tools-setup@v1 - env: - CONFTEST_VERSION: 0.30.0 - TERRAFORM_VERSION: 1.9.5 - TERRAGRUNT_VERSION: 0.66.9 - TF_SUMMARIZE_VERSION: 0.2.3 - - name: Retrieve VPN Config run: | scripts/createVPNConfig.sh dev 2> /dev/null @@ -94,38 +75,16 @@ jobs: app_label: "notify-database" timeout: "400s" - - name: Helmfile apply - if: ${{ success() }} - id: helmfile_apply + - name: Helmfile apply for dev images run: | pushd helmfile helmfile --environment dev -l 'tier=crd' apply helmfile --environment dev -l 'app!=notify-database,tier!=crd' apply popd - - name: Save ENV vars and secrets to AWS Param Store - if: ${{ success() }} - run: | - sleep 20 - source ./scripts/lambdaParamStoreUpdatesDev.sh -g - echo DIFF=$DIFF - echo "ENV_DIFF=$DIFF" >> $GITHUB_ENV - - - name: Force api-lambda to redeploy on environment changes - if: env.ENV_DIFF != '0' && ${{ success() }} - uses: ./.github/actions/update-lambda-function - with: - alias-name: latest - function-name: api-lambda-dev - - clear-cache: - needs: helmfile-apply - uses: ./.github/workflows/clear-notify-cache.yaml - with: - environment: dev - cache_clear_user_name: CACHE_CLEAR_USER - api_url: https://api.dev.notification.cdssandbox.xyz - secrets: - CACHE_CLEAR_CLIENT_SECRET: ${{ secrets.DEV_CACHE_CLEAR_CLIENT_SECRET }} - - \ No newline at end of file + # Optionally, force lambda to pick up latest dev image tag if needed + # - name: Force api-lambda-dev to redeploy + # uses: ./.github/actions/update-lambda-function + # with: + # alias-name: latest + # function-name: api-lambda-dev From 60a2347a73c21946e7919fb02c0b0d90a294af68 Mon Sep 17 00:00:00 2001 From: Michael Pond Date: Tue, 2 Dec 2025 10:34:57 -0500 Subject: [PATCH 3/8] ok! looking good... this should deploy dev branch code/images to dev --- .../workflows/dev_branch_deploy_images.yaml | 45 +++++++++++++++---- 1 file changed, 37 insertions(+), 8 deletions(-) diff --git a/.github/workflows/dev_branch_deploy_images.yaml b/.github/workflows/dev_branch_deploy_images.yaml index 9b1f5fa69..a1e870a3b 100644 --- a/.github/workflows/dev_branch_deploy_images.yaml +++ b/.github/workflows/dev_branch_deploy_images.yaml @@ -10,6 +10,7 @@ env: DEV_AWS_ACCOUNT: ${{ secrets.DEV_AWS_ACCOUNT_ID }} ACCOUNT_ID: ${{ secrets.DEV_AWS_ACCOUNT_ID }} AWS_DEFAULT_REGION: ca-central-1 + REGISTRY: ${{ secrets.DEV_AWS_ACCOUNT_ID }}.dkr.ecr.ca-central-1.amazonaws.com/notify permissions: id-token: write @@ -78,13 +79,41 @@ jobs: - name: Helmfile apply for dev images run: | pushd helmfile - helmfile --environment dev -l 'tier=crd' apply - helmfile --environment dev -l 'app!=notify-database,tier!=crd' apply + helmfile --environment dev -l 'tier=crd' apply -- \ + --set image.repository="${REGISTRY}/api" \ + --set image.tag="latest" + helmfile --environment dev -l 'app!=notify-database,tier!=crd' apply -- \ + --set image.repository="${REGISTRY}/api" \ + --set image.tag="latest" popd - # Optionally, force lambda to pick up latest dev image tag if needed - # - name: Force api-lambda-dev to redeploy - # uses: ./.github/actions/update-lambda-function - # with: - # alias-name: latest - # function-name: api-lambda-dev + - name: Deploy api-lambda from latest image tag + run: | + set -euo pipefail + IMAGE_NAME="api-lambda" + # Find the latest image tag for api-lambda in the dev ECR repo + LATEST_TAG=$(aws ecr describe-images \ + --repository-name "notify/${IMAGE_NAME}" \ + --query 'sort_by(imageDetails,& imagePushedAt)[-1].imageTags[0]' \ + --output text) + + if [ -z "$LATEST_TAG" ] || [ "$LATEST_TAG" = "None" ]; then + echo "No image tag found in ECR for $IMAGE_NAME" >&2 + exit 1 + fi + + IMAGE_URI="$REGISTRY/${IMAGE_NAME}:$LATEST_TAG" + echo "Deploying api-lambda with image: $IMAGE_URI" >&2 + + aws lambda update-function-code \ + --function-name api-lambda-dev \ + --image-uri "$IMAGE_URI" > /dev/null 2>&1 + + aws lambda wait function-updated --function-name api-lambda-dev + VERSION="$(aws lambda publish-version --function-name api-lambda-dev | jq -r '.Version')" + + aws lambda update-alias \ + --function-name api-lambda-dev \ + --name latest \ + --function-version "$VERSION" > /dev/null 2>&1 + From d237f95c2dcc498c8000ba3460c3f7300265df0d Mon Sep 17 00:00:00 2001 From: Michael Pond Date: Tue, 2 Dec 2025 10:59:13 -0500 Subject: [PATCH 4/8] udpating the database upgrade and helmfile apply stuff to use the right images, at the right times --- .../workflows/dev_branch_deploy_images.yaml | 62 +++++++++++++++---- 1 file changed, 50 insertions(+), 12 deletions(-) diff --git a/.github/workflows/dev_branch_deploy_images.yaml b/.github/workflows/dev_branch_deploy_images.yaml index a1e870a3b..02840f3b0 100644 --- a/.github/workflows/dev_branch_deploy_images.yaml +++ b/.github/workflows/dev_branch_deploy_images.yaml @@ -69,21 +69,59 @@ jobs: ./helmfile/getContext.sh -g - name: Run Database Upgrade Helmfile Sync - uses: ./.github/actions/db-migration - with: - environment: "dev" - namespace: "notification-canada-ca" - app_label: "notify-database" - timeout: "400s" + id: db_migration + run: | + set -euo pipefail + ENVIRONMENT="dev" + NAMESPACE="notification-canada-ca" + APP_LABEL="notify-database" + DB_ARGS="upgrade" + + pushd helmfile + helmfile --environment "$ENVIRONMENT" -l app="$APP_LABEL" --state-values-set DB_ARGS="$DB_ARGS" sync + + IMAGE_TAG=$(helm get values "$APP_LABEL" -n "$NAMESPACE" -o json | jq -r '.image.tag') + RELEASE_REVISION=$(helm list -n "$NAMESPACE" -o json | jq -r ".[] | select(.name==\"$APP_LABEL\") | .revision") + + echo "image_tag=${IMAGE_TAG}" >> "$GITHUB_OUTPUT" + echo "release_revision=${RELEASE_REVISION}" >> "$GITHUB_OUTPUT" + popd + + - name: Wait for Database Migration Job Completion + env: + IMAGE_TAG: ${{ steps.db_migration.outputs.image_tag }} + RELEASE_REVISION: ${{ steps.db_migration.outputs.release_revision }} + run: | + set -euo pipefail + NAMESPACE="notification-canada-ca" + TIMEOUT="400s" + + echo "Using image tag: ${IMAGE_TAG}" + echo "Using release revision: ${RELEASE_REVISION}" - - name: Helmfile apply for dev images + kubectl wait --for=condition=complete "job/notify-db-migration-job-${IMAGE_TAG}-${RELEASE_REVISION}" \ + -n "$NAMESPACE" \ + --timeout="$TIMEOUT" + + - name: Helmfile apply for all other releases run: | pushd helmfile - helmfile --environment dev -l 'tier=crd' apply -- \ - --set image.repository="${REGISTRY}/api" \ - --set image.tag="latest" - helmfile --environment dev -l 'app!=notify-database,tier!=crd' apply -- \ - --set image.repository="${REGISTRY}/api" \ + helmfile --environment dev -l 'tier=crd' apply + helmfile --environment dev -l 'app!=notify-database,app!=notify-api,app!=notify-admin,app!=notify-document-download,tier!=crd' apply + popd + + - name: Helmfile apply for dev images (api, admin, document-download) + strategy: + fail-fast: false + matrix: + app: + - api + - admin + - document-download + run: | + pushd helmfile + helmfile --environment dev -l "app=notify-${{ matrix.app }}" apply -- \ + --set image.repository="${REGISTRY}/${{ matrix.app }}" \ --set image.tag="latest" popd From a30a0ce47fceaa53e3645b50461aef4ffa73faa6 Mon Sep 17 00:00:00 2001 From: Michael Pond Date: Tue, 2 Dec 2025 11:10:27 -0500 Subject: [PATCH 5/8] making sure proper ecr and tag for DB upgrade --- .github/workflows/dev_branch_deploy_images.yaml | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/.github/workflows/dev_branch_deploy_images.yaml b/.github/workflows/dev_branch_deploy_images.yaml index 02840f3b0..1d1b05baa 100644 --- a/.github/workflows/dev_branch_deploy_images.yaml +++ b/.github/workflows/dev_branch_deploy_images.yaml @@ -78,9 +78,13 @@ jobs: DB_ARGS="upgrade" pushd helmfile - helmfile --environment "$ENVIRONMENT" -l app="$APP_LABEL" --state-values-set DB_ARGS="$DB_ARGS" sync + helmfile --environment "$ENVIRONMENT" -l app="$APP_LABEL" \ + --state-values-set DB_ARGS="$DB_ARGS" \ + apply -- \ + --set image.repository="${REGISTRY}/notify-database" \ + --set image.tag="latest" - IMAGE_TAG=$(helm get values "$APP_LABEL" -n "$NAMESPACE" -o json | jq -r '.image.tag') + IMAGE_TAG="latest" RELEASE_REVISION=$(helm list -n "$NAMESPACE" -o json | jq -r ".[] | select(.name==\"$APP_LABEL\") | .revision") echo "image_tag=${IMAGE_TAG}" >> "$GITHUB_OUTPUT" From f5f27578f8fdb14ab19f2360584de84697795672 Mon Sep 17 00:00:00 2001 From: Michael Pond Date: Tue, 2 Dec 2025 11:18:34 -0500 Subject: [PATCH 6/8] adding param management scripting --- .github/workflows/dev_branch_deploy_images.yaml | 10 ++++++++++ 1 file changed, 10 insertions(+) diff --git a/.github/workflows/dev_branch_deploy_images.yaml b/.github/workflows/dev_branch_deploy_images.yaml index 1d1b05baa..3ed0f5e41 100644 --- a/.github/workflows/dev_branch_deploy_images.yaml +++ b/.github/workflows/dev_branch_deploy_images.yaml @@ -129,6 +129,15 @@ jobs: --set image.tag="latest" popd + - name: Save ENV vars and secrets to AWS Param Store + if: ${{ success() }} + run: | + # wait for the secrets and env vars to be available + sleep 20 + source ./scripts/lambdaParamStoreUpdatesDev.sh -g + echo DIFF=$DIFF + echo "ENV_DIFF=$DIFF" >> $GITHUB_ENV + - name: Deploy api-lambda from latest image tag run: | set -euo pipefail @@ -159,3 +168,4 @@ jobs: --name latest \ --function-version "$VERSION" > /dev/null 2>&1 + From 451d83ab44004a93e2d5510e1ce651853a534d4f Mon Sep 17 00:00:00 2001 From: "copilot-swe-agent[bot]" <198982749+Copilot@users.noreply.github.com> Date: Tue, 2 Dec 2025 16:30:17 +0000 Subject: [PATCH 7/8] Initial plan From b22e19f003f1a0802cf9c8f3e938cb6c9150ce32 Mon Sep 17 00:00:00 2001 From: "copilot-swe-agent[bot]" <198982749+Copilot@users.noreply.github.com> Date: Tue, 2 Dec 2025 16:33:07 +0000 Subject: [PATCH 8/8] Remove fail-fast: false to prevent partial deployments Co-authored-by: P0NDER0SA <32133001+P0NDER0SA@users.noreply.github.com> --- .github/workflows/dev_branch_deploy_images.yaml | 1 - 1 file changed, 1 deletion(-) diff --git a/.github/workflows/dev_branch_deploy_images.yaml b/.github/workflows/dev_branch_deploy_images.yaml index 3ed0f5e41..07b6c16e3 100644 --- a/.github/workflows/dev_branch_deploy_images.yaml +++ b/.github/workflows/dev_branch_deploy_images.yaml @@ -116,7 +116,6 @@ jobs: - name: Helmfile apply for dev images (api, admin, document-download) strategy: - fail-fast: false matrix: app: - api