ppg-ami-factory.yml

# EL package-test AMI factory: Oracle Linux + Rocky Linux.
# Bakes OL and Rocky 8/9/10 x86_64+arm64 AMIs with Packer over AWS Session
# Manager (no inbound SSH), authenticated via GitHub OIDC (no static keys), and
# promotes each via a fresh-boot smoke test. Consumer: pg.cd molecule jobs use
# the static AMI pins in vars/moleculeEnvPPG.groovy, refreshed weekly by the
# update-molecule-env job through an auto-opened PR.
# Rocky lineage roots are seeded once per combo via dispatch (just ci-seed-rocky).
#
# Triggers: PR validate-only (pull_request -> check job, no AWS), recipe change
# (paths), weekly security rebake (cron), manual.
# Actions are pinned to commit SHAs; the trailing "# vX.Y.Z" records the tag.

name: ppg-ami-factory
run-name: ppg AMI factory (${{ inputs.env || 'prod' }}, ${{ github.event_name }})

on:
  workflow_dispatch:
    inputs:
      combos:
        description: 'JSON matrix include list, e.g. [{"os":"rocky","os_major":"9","arch":"x86_64","seed":true}]; os defaults to oraclelinux, seed to false'
        required: false
        default: '[{"os":"oraclelinux","os_major":"9","arch":"x86_64"}]'
      env:
        description: 'prod (real factory tags) or test (isolated tags, never consumed by pg.cd)'
        required: false
        default: 'prod'
        type: choice
        options: [prod, test]
  push:
    branches: [master]
    paths:
      - 'ppg/packer/**'
      - '.github/workflows/ppg-ami-factory.yml'
  pull_request:
    paths:
      - 'ppg/packer/**'
      - '.github/workflows/ppg-ami-factory.yml'
  schedule:
    - cron: '0 6 * * 1' # weekly Mon 06:00 UTC - absorb OL + Rocky security errata

permissions:
  contents: read # workflow-level default: read-only

concurrency:
  group: ppg-ami-factory-${{ github.ref }}
  cancel-in-progress: false

env:
  AWS_REGION: eu-central-1
  FACTORY_ENV: ${{ github.event.inputs.env || 'prod' }}

jobs:
  # No-AWS gate: a malformed template or bad combo fails HERE, before bake spins
  # up any EC2. Runs scripts/check.sh, the SAME file `just check` runs, so the
  # CI and local gates cannot drift. No id-token, so this job can never assume
  # the factory role.
  check:
    name: Validate templates (no AWS)
    runs-on: ubuntu-latest
    timeout-minutes: 10
    permissions:
      contents: read
    steps:
      - uses: actions/checkout@34e114876b0b11c390a56381ad16ebd13914f8d5 # v4.3.1

      - name: Setup Packer
        uses: hashicorp/setup-packer@3286471d6cc6756d056a0b199fea5e0becdbc189 # v3.3.0
        with:
          version: '1.15.4' # exact, matches the bake job

      - name: fmt-check + validate (fail before any EC2 spend)
        working-directory: ppg/packer
        run: |
          set -euo pipefail
          bash scripts/check.sh

  bake:
    name: ${{ matrix.os || 'oraclelinux' }} ${{ matrix.os_major }} ${{ matrix.arch }}
    needs: check # no EC2 launches until templates validate
    if: github.event_name != 'pull_request' # PRs run the no-AWS check only
    runs-on: ubuntu-latest
    timeout-minutes: 60
    permissions:
      contents: read
      id-token: write # ONLY this job mints the OIDC token
    strategy:
      fail-fast: false
      max-parallel: 12
      matrix:
        include: ${{ fromJSON(github.event.inputs.combos || '[{"os":"oraclelinux","os_major":"8","arch":"x86_64"},{"os":"oraclelinux","os_major":"8","arch":"arm64"},{"os":"oraclelinux","os_major":"9","arch":"x86_64"},{"os":"oraclelinux","os_major":"9","arch":"arm64"},{"os":"oraclelinux","os_major":"10","arch":"x86_64"},{"os":"oraclelinux","os_major":"10","arch":"arm64"},{"os":"rocky","os_major":"8","arch":"x86_64"},{"os":"rocky","os_major":"8","arch":"arm64"},{"os":"rocky","os_major":"9","arch":"x86_64"},{"os":"rocky","os_major":"9","arch":"arm64"},{"os":"rocky","os_major":"10","arch":"x86_64"},{"os":"rocky","os_major":"10","arch":"arm64"}]') }}
    steps:
      - uses: actions/checkout@34e114876b0b11c390a56381ad16ebd13914f8d5 # v4.3.1

      - name: Configure AWS credentials (OIDC)
        uses: aws-actions/configure-aws-credentials@7474bc4690e29a8392af63c5b98e7449536d5c3a # v4.3.1
        with:
          # No repo secret needed: falls back to the prod OIDC role ARN when the secret is unset.
          # Security is the role's OIDC trust (master-only), not ARN secrecy; a fork may override.
          role-to-assume: ${{ secrets.PPG_AMI_FACTORY_ROLE_ARN || 'arn:aws:iam::119175775298:role/percona-ci-platform-gha-ppg-ami-factory' }}
          role-session-name: ppg-ami-factory-${{ github.run_id }}-${{ github.run_attempt }}
          aws-region: ${{ env.AWS_REGION }}

      - name: Install Session Manager plugin
        run: |
          set -euo pipefail
          if ! command -v session-manager-plugin >/dev/null; then
            curl -fsSL "https://s3.amazonaws.com/session-manager-downloads/plugin/latest/ubuntu_64bit/session-manager-plugin.deb" -o /tmp/smp.deb
            sudo dpkg -i /tmp/smp.deb
          fi
          session-manager-plugin --version

      - name: Setup Packer
        uses: hashicorp/setup-packer@3286471d6cc6756d056a0b199fea5e0becdbc189 # v3.3.0
        with:
          # Exact version, never a constraint: setup-packer uses this string verbatim
          # as the tool-cache path segment, so a space/'>' in a constraint like
          # ">= 1.12.0" breaks the post-install `packer version` check. Bump deliberately.
          version: '1.15.4'

      - name: Packer init
        working-directory: ppg/packer
        run: packer init .

      - name: Build candidate AMI (${{ matrix.os || 'oraclelinux' }} ${{ matrix.os_major }} ${{ matrix.arch }}, env=${{ env.FACTORY_ENV }})
        id: build
        working-directory: ppg/packer
        # Matrix values via env, not inline ${{ }}, so they reach packer as data
        # (quoted shell vars) and cannot break out of the run shell.
        env:
          OS: ${{ matrix.os || 'oraclelinux' }}
          SEED: ${{ matrix.seed || false }}
          OS_MAJOR: ${{ matrix.os_major }}
          ARCH: ${{ matrix.arch }}
        run: |
          set -euo pipefail
          rm -f manifest.json
          packer build -color=false \
            -var "os=${OS}" \
            -var "seed=${SEED}" \
            -var "os_major=${OS_MAJOR}" \
            -var "arch=${ARCH}" \
            -var "region=${AWS_REGION}" \
            -var "env=${FACTORY_ENV}" .
          AMI=$(python3 -c "import json;print(json.load(open('manifest.json'))['builds'][-1]['artifact_id'].split(':')[-1])")
          echo "ami=$AMI" >> "$GITHUB_OUTPUT"
          echo "built candidate: $AMI"
          {
            echo "### ${OS} ${OS_MAJOR} ${ARCH}"
            echo "- built: $AMI"
          } >> "$GITHUB_STEP_SUMMARY"

      - name: Smoke test (boot + install)
        working-directory: ppg/packer
        env:
          AMI: ${{ steps.build.outputs.ami }}
          OS: ${{ matrix.os || 'oraclelinux' }}
          OS_MAJOR: ${{ matrix.os_major }}
          ARCH: ${{ matrix.arch }}
        run: |
          set -euo pipefail
          echo "smoke candidate: $AMI"
          # Deregister ONLY on a real boot/install failure (never on a later promote-tag failure).
          ( cd smoke && packer init . && packer build -color=false \
              -var "candidate_ami=$AMI" -var "os=${OS}" -var "os_major=${OS_MAJOR}" \
              -var "arch=${ARCH}" -var "region=${AWS_REGION}" . ) \
          || { echo "smoke (boot+install) failed; deregistering $AMI + its snapshots"; aws ec2 deregister-image --delete-associated-snapshots --region "$AWS_REGION" --image-id "$AMI"; echo "- smoke: FAIL (deregistered $AMI)" >> "$GITHUB_STEP_SUMMARY"; exit 1; }
          echo "- smoke: pass" >> "$GITHUB_STEP_SUMMARY"

      - name: Promote (env=${{ env.FACTORY_ENV }})
        working-directory: ppg/packer
        env:
          AMI: ${{ steps.build.outputs.ami }}
        run: |
          set -euo pipefail
          pr=ppg-package-test; [ "$FACTORY_ENV" = "test" ] && pr=ppg-test-package-test
          # Promote SEPARATELY with retry: a transient tag failure must NOT delete a smoke-passed AMI.
          for a in 1 2 3 4 5; do
            aws ec2 create-tags --region "$AWS_REGION" --resources "$AMI" --tags Key=role,Value="$pr" Key=smoke,Value=passed && { echo "promoted $AMI -> $pr"; break; }
            [ "$a" = 5 ] && { echo "promotion FAILED after retries for smoke-passed $AMI; left intact for manual promote"; echo "- promote: FAILED ($AMI left for manual promote)" >> "$GITHUB_STEP_SUMMARY"; exit 1; }
            echo "promote attempt $a failed; retry in $((a * 5))s"; sleep $((a * 5))
          done
          echo "- promote: $pr" >> "$GITHUB_STEP_SUMMARY"

  update-molecule-env:
    name: Update moleculeEnvPPG.groovy (OL/Rocky AMI IDs)
    needs: bake
    # Weekly runs only, never manual/PR/push, where a partial or test-env combo
    # list must not drive the pins. !cancelled() rather than success(): a partial
    # bake still refreshes the pins, because the script resolves from the promote
    # tags, so a failed combo simply keeps last week's AMI while the rest advance.
    if: github.event_name == 'schedule' && !cancelled() && needs.bake.result != 'skipped'
    runs-on: ubuntu-latest
    timeout-minutes: 10
    permissions:
      contents: write
      pull-requests: write
      id-token: write
    steps:
      - uses: actions/checkout@34e114876b0b11c390a56381ad16ebd13914f8d5 # v4.3.1
        with:
          ref: master

      - name: Configure AWS credentials (OIDC)
        uses: aws-actions/configure-aws-credentials@7474bc4690e29a8392af63c5b98e7449536d5c3a # v4.3.1
        with:
          role-to-assume: ${{ secrets.PPG_AMI_FACTORY_ROLE_ARN || 'arn:aws:iam::119175775298:role/percona-ci-platform-gha-ppg-ami-factory' }}
          role-session-name: ppg-ami-factory-update-env-${{ github.run_id }}-${{ github.run_attempt }}
          aws-region: ${{ env.AWS_REGION }}

      - name: Resolve latest OL/Rocky AMIs and rewrite moleculeEnvPPG.groovy
        run: |
          set -euo pipefail
          python3 -m pip install --quiet 'boto3==1.43.74' 'botocore==1.43.74'
          python3 ppg/packer/scripts/update_molecule_env.py

      - name: Commit and open a PR if changed
        # master rejects direct pushes (protected branch, changes must go through
        # a pull request), so the refresh is published as a PR on one fixed rolling
        # branch. Force-push keeps that branch current; an already-open PR is
        # reused. Needs pull-requests: write plus the repo setting "Allow GitHub
        # Actions to create and approve pull requests".
        env:
          GH_TOKEN: ${{ github.token }}
        run: |
          set -euo pipefail
          if git diff --quiet -- vars/moleculeEnvPPG.groovy; then
            echo "no AMI ID changes; nothing to commit"
            exit 0
          fi
          branch=ppg-ami-refresh
          git config user.name "ppg-ami-factory-bot"
          git config user.email "actions@github.com"
          git checkout -B "$branch"
          git add vars/moleculeEnvPPG.groovy
          git commit -m "Auto-update OL/Rocky molecule AMI IDs (weekly ppg-ami-factory refresh)"
          # The rolling branch is bot-owned and rebuilt from master every week;
          # manual commits on it do not survive the next refresh.
          git push --force origin "HEAD:$branch"
          # Count only same-repo PRs: an unqualified --head also matches a fork
          # PR whose head branch happens to share the name, which would skip
          # creation here and silently leave the refresh unpublished.
          open_prs=$(gh pr list --head "$branch" --base master --state open \
            --json number,headRepositoryOwner \
            --jq '[.[] | select(.headRepositoryOwner.login == "'"${GITHUB_REPOSITORY_OWNER}"'")] | length')
          if [ "$open_prs" = "0" ]; then
            gh pr create --head "$branch" --base master \
              --title "Auto-update OL/Rocky molecule AMI IDs (weekly refresh)" \
              --body "Weekly ppg-ami-factory refresh: OL/Rocky pins in vars/moleculeEnvPPG.groovy updated to the newest promoted AMIs (tag role=ppg-package-test). Opened automatically by the update-molecule-env job."
          else
            echo "open PR for $branch already exists; branch force-updated"
          fi

  notify:
    # update-molecule-env is listed so its failures (AMI resolution, PR creation)
    # alert too; with needs: bake alone they would stay Slack-silent.
    needs: [bake, update-molecule-env]
    # failure() is true when ANY ancestor failed, including a check failure on a
    # pull_request run where bake was skipped. Never ping the webhook for PR lint.
    if: failure() && github.event_name != 'pull_request'
    runs-on: ubuntu-latest
    # Job-level so the step `if` below reads it reliably; a step-scoped env can be
    # unavailable to its own step's `if`, which would skip the alert silently.
    env:
      SLACK_WEBHOOK: ${{ secrets.RELEASES_CI_SLACK_WEBHOOK }}
    steps:
      - name: Slack on failure
        # Skip cleanly when the webhook secret is unset (e.g. fork tests), so a real
        # bake failure is not masked by a second "missing webhook" failure.
        if: env.SLACK_WEBHOOK != ''
        uses: slackapi/slack-github-action@91efab103c0de0a537f72a35f6b8cda0ee76bf0a # v2.1.1
        with:
          webhook: ${{ env.SLACK_WEBHOOK }}
          webhook-type: incoming-webhook
          payload: |
            text: "PPG AMI factory FAILED - ${{ github.server_url }}/${{ github.repository }}/actions/runs/${{ github.run_id }}"