ppg AMI factory (prod, schedule) #37
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| # EL package-test AMI factory: Oracle Linux + Rocky Linux. | |
| # Bakes OL and Rocky 8/9/10 x86_64+arm64 AMIs with Packer over AWS Session | |
| # Manager (no inbound SSH), authenticated via GitHub OIDC (no static keys), and | |
| # promotes each via a fresh-boot smoke test. Consumer: pg.cd molecule jobs use | |
| # the static AMI pins in vars/moleculeEnvPPG.groovy, refreshed weekly by the | |
| # update-molecule-env job through an auto-opened PR. | |
| # Rocky lineage roots are seeded once per combo via dispatch (just ci-seed-rocky). | |
| # | |
| # Triggers: PR validate-only (pull_request -> check job, no AWS), recipe change | |
| # (paths), weekly security rebake (cron), manual. | |
| # Actions are pinned to commit SHAs; the trailing "# vX.Y.Z" records the tag. | |
| name: ppg-ami-factory | |
| run-name: ppg AMI factory (${{ inputs.env || 'prod' }}, ${{ github.event_name }}) | |
| on: | |
| workflow_dispatch: | |
| inputs: | |
| combos: | |
| description: 'JSON matrix include list, e.g. [{"os":"rocky","os_major":"9","arch":"x86_64","seed":true}]; os defaults to oraclelinux, seed to false' | |
| required: false | |
| default: '[{"os":"oraclelinux","os_major":"9","arch":"x86_64"}]' | |
| env: | |
| description: 'prod (real factory tags) or test (isolated tags, never consumed by pg.cd)' | |
| required: false | |
| default: 'prod' | |
| type: choice | |
| options: [prod, test] | |
| push: | |
| branches: [master] | |
| paths: | |
| - 'ppg/packer/**' | |
| - '.github/workflows/ppg-ami-factory.yml' | |
| pull_request: | |
| paths: | |
| - 'ppg/packer/**' | |
| - '.github/workflows/ppg-ami-factory.yml' | |
| schedule: | |
| - cron: '0 6 * * 1' # weekly Mon 06:00 UTC - absorb OL + Rocky security errata | |
| permissions: | |
| contents: read # workflow-level default: read-only | |
| concurrency: | |
| group: ppg-ami-factory-${{ github.ref }} | |
| cancel-in-progress: false | |
| env: | |
| AWS_REGION: eu-central-1 | |
| FACTORY_ENV: ${{ github.event.inputs.env || 'prod' }} | |
| jobs: | |
| # No-AWS gate: a malformed template or bad combo fails HERE, before bake spins | |
| # up any EC2. Runs scripts/check.sh, the SAME file `just check` runs, so the | |
| # CI and local gates cannot drift. No id-token, so this job can never assume | |
| # the factory role. | |
| check: | |
| name: Validate templates (no AWS) | |
| runs-on: ubuntu-latest | |
| timeout-minutes: 10 | |
| permissions: | |
| contents: read | |
| steps: | |
| - uses: actions/checkout@34e114876b0b11c390a56381ad16ebd13914f8d5 # v4.3.1 | |
| - name: Setup Packer | |
| uses: hashicorp/setup-packer@3286471d6cc6756d056a0b199fea5e0becdbc189 # v3.3.0 | |
| with: | |
| version: '1.15.4' # exact, matches the bake job | |
| - name: fmt-check + validate (fail before any EC2 spend) | |
| working-directory: ppg/packer | |
| run: | | |
| set -euo pipefail | |
| bash scripts/check.sh | |
| bake: | |
| name: ${{ matrix.os || 'oraclelinux' }} ${{ matrix.os_major }} ${{ matrix.arch }} | |
| needs: check # no EC2 launches until templates validate | |
| if: github.event_name != 'pull_request' # PRs run the no-AWS check only | |
| runs-on: ubuntu-latest | |
| timeout-minutes: 60 | |
| permissions: | |
| contents: read | |
| id-token: write # ONLY this job mints the OIDC token | |
| strategy: | |
| fail-fast: false | |
| max-parallel: 12 | |
| matrix: | |
| include: ${{ fromJSON(github.event.inputs.combos || '[{"os":"oraclelinux","os_major":"8","arch":"x86_64"},{"os":"oraclelinux","os_major":"8","arch":"arm64"},{"os":"oraclelinux","os_major":"9","arch":"x86_64"},{"os":"oraclelinux","os_major":"9","arch":"arm64"},{"os":"oraclelinux","os_major":"10","arch":"x86_64"},{"os":"oraclelinux","os_major":"10","arch":"arm64"},{"os":"rocky","os_major":"8","arch":"x86_64"},{"os":"rocky","os_major":"8","arch":"arm64"},{"os":"rocky","os_major":"9","arch":"x86_64"},{"os":"rocky","os_major":"9","arch":"arm64"},{"os":"rocky","os_major":"10","arch":"x86_64"},{"os":"rocky","os_major":"10","arch":"arm64"}]') }} | |
| steps: | |
| - uses: actions/checkout@34e114876b0b11c390a56381ad16ebd13914f8d5 # v4.3.1 | |
| - name: Configure AWS credentials (OIDC) | |
| uses: aws-actions/configure-aws-credentials@7474bc4690e29a8392af63c5b98e7449536d5c3a # v4.3.1 | |
| with: | |
| # No repo secret needed: falls back to the prod OIDC role ARN when the secret is unset. | |
| # Security is the role's OIDC trust (master-only), not ARN secrecy; a fork may override. | |
| role-to-assume: ${{ secrets.PPG_AMI_FACTORY_ROLE_ARN || 'arn:aws:iam::119175775298:role/percona-ci-platform-gha-ppg-ami-factory' }} | |
| role-session-name: ppg-ami-factory-${{ github.run_id }}-${{ github.run_attempt }} | |
| aws-region: ${{ env.AWS_REGION }} | |
| - name: Install Session Manager plugin | |
| run: | | |
| set -euo pipefail | |
| if ! command -v session-manager-plugin >/dev/null; then | |
| curl -fsSL "https://s3.amazonaws.com/session-manager-downloads/plugin/latest/ubuntu_64bit/session-manager-plugin.deb" -o /tmp/smp.deb | |
| sudo dpkg -i /tmp/smp.deb | |
| fi | |
| session-manager-plugin --version | |
| - name: Setup Packer | |
| uses: hashicorp/setup-packer@3286471d6cc6756d056a0b199fea5e0becdbc189 # v3.3.0 | |
| with: | |
| # Exact version, never a constraint: setup-packer uses this string verbatim | |
| # as the tool-cache path segment, so a space/'>' in a constraint like | |
| # ">= 1.12.0" breaks the post-install `packer version` check. Bump deliberately. | |
| version: '1.15.4' | |
| - name: Packer init | |
| working-directory: ppg/packer | |
| run: packer init . | |
| - name: Build candidate AMI (${{ matrix.os || 'oraclelinux' }} ${{ matrix.os_major }} ${{ matrix.arch }}, env=${{ env.FACTORY_ENV }}) | |
| id: build | |
| working-directory: ppg/packer | |
| # Matrix values via env, not inline ${{ }}, so they reach packer as data | |
| # (quoted shell vars) and cannot break out of the run shell. | |
| env: | |
| OS: ${{ matrix.os || 'oraclelinux' }} | |
| SEED: ${{ matrix.seed || false }} | |
| OS_MAJOR: ${{ matrix.os_major }} | |
| ARCH: ${{ matrix.arch }} | |
| run: | | |
| set -euo pipefail | |
| rm -f manifest.json | |
| packer build -color=false \ | |
| -var "os=${OS}" \ | |
| -var "seed=${SEED}" \ | |
| -var "os_major=${OS_MAJOR}" \ | |
| -var "arch=${ARCH}" \ | |
| -var "region=${AWS_REGION}" \ | |
| -var "env=${FACTORY_ENV}" . | |
| AMI=$(python3 -c "import json;print(json.load(open('manifest.json'))['builds'][-1]['artifact_id'].split(':')[-1])") | |
| echo "ami=$AMI" >> "$GITHUB_OUTPUT" | |
| echo "built candidate: $AMI" | |
| { | |
| echo "### ${OS} ${OS_MAJOR} ${ARCH}" | |
| echo "- built: $AMI" | |
| } >> "$GITHUB_STEP_SUMMARY" | |
| - name: Smoke test (boot + install) | |
| working-directory: ppg/packer | |
| env: | |
| AMI: ${{ steps.build.outputs.ami }} | |
| OS: ${{ matrix.os || 'oraclelinux' }} | |
| OS_MAJOR: ${{ matrix.os_major }} | |
| ARCH: ${{ matrix.arch }} | |
| run: | | |
| set -euo pipefail | |
| echo "smoke candidate: $AMI" | |
| # Deregister ONLY on a real boot/install failure (never on a later promote-tag failure). | |
| ( cd smoke && packer init . && packer build -color=false \ | |
| -var "candidate_ami=$AMI" -var "os=${OS}" -var "os_major=${OS_MAJOR}" \ | |
| -var "arch=${ARCH}" -var "region=${AWS_REGION}" . ) \ | |
| || { echo "smoke (boot+install) failed; deregistering $AMI + its snapshots"; aws ec2 deregister-image --delete-associated-snapshots --region "$AWS_REGION" --image-id "$AMI"; echo "- smoke: FAIL (deregistered $AMI)" >> "$GITHUB_STEP_SUMMARY"; exit 1; } | |
| echo "- smoke: pass" >> "$GITHUB_STEP_SUMMARY" | |
| - name: Promote (env=${{ env.FACTORY_ENV }}) | |
| working-directory: ppg/packer | |
| env: | |
| AMI: ${{ steps.build.outputs.ami }} | |
| run: | | |
| set -euo pipefail | |
| pr=ppg-package-test; [ "$FACTORY_ENV" = "test" ] && pr=ppg-test-package-test | |
| # Promote SEPARATELY with retry: a transient tag failure must NOT delete a smoke-passed AMI. | |
| for a in 1 2 3 4 5; do | |
| aws ec2 create-tags --region "$AWS_REGION" --resources "$AMI" --tags Key=role,Value="$pr" Key=smoke,Value=passed && { echo "promoted $AMI -> $pr"; break; } | |
| [ "$a" = 5 ] && { echo "promotion FAILED after retries for smoke-passed $AMI; left intact for manual promote"; echo "- promote: FAILED ($AMI left for manual promote)" >> "$GITHUB_STEP_SUMMARY"; exit 1; } | |
| echo "promote attempt $a failed; retry in $((a * 5))s"; sleep $((a * 5)) | |
| done | |
| echo "- promote: $pr" >> "$GITHUB_STEP_SUMMARY" | |
| update-molecule-env: | |
| name: Update moleculeEnvPPG.groovy (OL/Rocky AMI IDs) | |
| needs: bake | |
| # Weekly runs only, never manual/PR/push, where a partial or test-env combo | |
| # list must not drive the pins. !cancelled() rather than success(): a partial | |
| # bake still refreshes the pins, because the script resolves from the promote | |
| # tags, so a failed combo simply keeps last week's AMI while the rest advance. | |
| if: github.event_name == 'schedule' && !cancelled() && needs.bake.result != 'skipped' | |
| runs-on: ubuntu-latest | |
| timeout-minutes: 10 | |
| permissions: | |
| contents: write | |
| pull-requests: write | |
| id-token: write | |
| steps: | |
| - uses: actions/checkout@34e114876b0b11c390a56381ad16ebd13914f8d5 # v4.3.1 | |
| with: | |
| ref: master | |
| - name: Configure AWS credentials (OIDC) | |
| uses: aws-actions/configure-aws-credentials@7474bc4690e29a8392af63c5b98e7449536d5c3a # v4.3.1 | |
| with: | |
| role-to-assume: ${{ secrets.PPG_AMI_FACTORY_ROLE_ARN || 'arn:aws:iam::119175775298:role/percona-ci-platform-gha-ppg-ami-factory' }} | |
| role-session-name: ppg-ami-factory-update-env-${{ github.run_id }}-${{ github.run_attempt }} | |
| aws-region: ${{ env.AWS_REGION }} | |
| - name: Resolve latest OL/Rocky AMIs and rewrite moleculeEnvPPG.groovy | |
| run: | | |
| set -euo pipefail | |
| python3 -m pip install --quiet 'boto3==1.43.74' 'botocore==1.43.74' | |
| python3 ppg/packer/scripts/update_molecule_env.py | |
| - name: Commit and open a PR if changed | |
| # master rejects direct pushes (protected branch, changes must go through | |
| # a pull request), so the refresh is published as a PR on one fixed rolling | |
| # branch. Force-push keeps that branch current; an already-open PR is | |
| # reused. Needs pull-requests: write plus the repo setting "Allow GitHub | |
| # Actions to create and approve pull requests". | |
| env: | |
| GH_TOKEN: ${{ github.token }} | |
| run: | | |
| set -euo pipefail | |
| if git diff --quiet -- vars/moleculeEnvPPG.groovy; then | |
| echo "no AMI ID changes; nothing to commit" | |
| exit 0 | |
| fi | |
| branch=ppg-ami-refresh | |
| git config user.name "ppg-ami-factory-bot" | |
| git config user.email "actions@github.com" | |
| git checkout -B "$branch" | |
| git add vars/moleculeEnvPPG.groovy | |
| git commit -m "Auto-update OL/Rocky molecule AMI IDs (weekly ppg-ami-factory refresh)" | |
| # The rolling branch is bot-owned and rebuilt from master every week; | |
| # manual commits on it do not survive the next refresh. | |
| git push --force origin "HEAD:$branch" | |
| # Count only same-repo PRs: an unqualified --head also matches a fork | |
| # PR whose head branch happens to share the name, which would skip | |
| # creation here and silently leave the refresh unpublished. | |
| open_prs=$(gh pr list --head "$branch" --base master --state open \ | |
| --json number,headRepositoryOwner \ | |
| --jq '[.[] | select(.headRepositoryOwner.login == "'"${GITHUB_REPOSITORY_OWNER}"'")] | length') | |
| if [ "$open_prs" = "0" ]; then | |
| gh pr create --head "$branch" --base master \ | |
| --title "Auto-update OL/Rocky molecule AMI IDs (weekly refresh)" \ | |
| --body "Weekly ppg-ami-factory refresh: OL/Rocky pins in vars/moleculeEnvPPG.groovy updated to the newest promoted AMIs (tag role=ppg-package-test). Opened automatically by the update-molecule-env job." | |
| else | |
| echo "open PR for $branch already exists; branch force-updated" | |
| fi | |
| notify: | |
| # update-molecule-env is listed so its failures (AMI resolution, PR creation) | |
| # alert too; with needs: bake alone they would stay Slack-silent. | |
| needs: [bake, update-molecule-env] | |
| # failure() is true when ANY ancestor failed, including a check failure on a | |
| # pull_request run where bake was skipped. Never ping the webhook for PR lint. | |
| if: failure() && github.event_name != 'pull_request' | |
| runs-on: ubuntu-latest | |
| # Job-level so the step `if` below reads it reliably; a step-scoped env can be | |
| # unavailable to its own step's `if`, which would skip the alert silently. | |
| env: | |
| SLACK_WEBHOOK: ${{ secrets.RELEASES_CI_SLACK_WEBHOOK }} | |
| steps: | |
| - name: Slack on failure | |
| # Skip cleanly when the webhook secret is unset (e.g. fork tests), so a real | |
| # bake failure is not masked by a second "missing webhook" failure. | |
| if: env.SLACK_WEBHOOK != '' | |
| uses: slackapi/slack-github-action@91efab103c0de0a537f72a35f6b8cda0ee76bf0a # v2.1.1 | |
| with: | |
| webhook: ${{ env.SLACK_WEBHOOK }} | |
| webhook-type: incoming-webhook | |
| payload: | | |
| text: "PPG AMI factory FAILED - ${{ github.server_url }}/${{ github.repository }}/actions/runs/${{ github.run_id }}" |