prod.docs.plus.yml

# =============================================================================
# Production CI/CD Pipeline — Quality Gates + Blue-Green Deployment
# =============================================================================
#
# Pipeline:
#   1. Quality Gates (parallel): lint, typecheck, security
#   2. Build Verification: extension + Next.js builds (smoke test)
#   3. Deploy Gate: parses commit message with a strict regex (anti-footgun)
#   4. Deploy: docker compose build on the prod host → blue-green rollout
#
# Triggers:
#   - push to main: full pipeline. Deploy chain gated by triage outputs
#     (`app_deploy`, `has_observability`, `has_uptime_kuma`). Ordered:
#     app → observability → uptime-kuma (fail-fast).
#   - pull_request to main: quality gates + build verification only.
#   - schedule (Sunday 00:00 UTC): security audit + all five extension suites.
#   - workflow_dispatch: manual run with `force_deploy` / `skip_quality_gates`.
#
# Architectural notes (decided 2026-05, see commit history):
#   - Images are built on the prod self-hosted runner (`prod.docs.plus`).
#     Disk pressure is mitigated by a pre-build disk guard, not by pushing
#     to a registry. If pressure resurfaces, revisit M3 (ghcr.io push).
#   - Rollback uses an on-disk tag stash on the prod host:
#     /opt/projects/prod.docs.plus/.deploy/last-good-tag.
#   - All third-party actions (workflow + composite) are pinned to commit
#     SHA. Renovate/Dependabot should bump them; never use floating tags.
# =============================================================================

name: CI/CD Production

on:
  push:
    branches: [main]
  pull_request:
    branches: [main]
  schedule:
    # Weekly security scan + unfiltered extension suites (Sunday 00:00 UTC)
    - cron: '0 0 * * 0'
  workflow_dispatch:
    inputs:
      skip_quality_gates:
        description: 'Skip quality gates (emergency deploy)'
        required: false
        default: false
        type: boolean
      force_deploy:
        description: 'Force deployment (bypass commit-message gate)'
        required: false
        default: false
        type: boolean
      dry_run_subject:
        description: 'Simulated commit subject to observe trigger gating (no deploy on dispatch)'
        required: false
        default: ''
        type: string

# Two concurrency groups:
#   - quality-gates can be cancelled freely (cheap to redo)
#   - deploy MUST finish or rollback (mid-deploy SIGTERM corrupts blue-green state)
concurrency:
  group: ${{ github.workflow }}-${{ github.ref }}-quality
  cancel-in-progress: true

# Default to bash so set -e/-o pipefail behavior is consistent across runners.
defaults:
  run:
    shell: bash

# Workflow-level least privilege; per-job overrides where needed.
permissions:
  contents: read

env:
  ENV_SOURCE: /opt/projects/prod.docs.plus/.env
  ENV_FILE: .env.production
  COMPOSE_FILE: docker-compose.prod.yml
  DEPLOY_TAG: ${{ github.sha }}
  # Where the prod host stashes the last successfully deployed SHA for rollback.
  DEPLOY_STATE_DIR: /opt/projects/prod.docs.plus/.deploy
  LAST_GOOD_TAG_FILE: /opt/projects/prod.docs.plus/.deploy/last-good-tag

jobs:
  # ===========================================================================
  # STAGE −1 — TRIAGE (runs parse-build-trigger.sh; exposes skip_app_ci output)
  # ===========================================================================
  # Skipped on `schedule` (weekly security scan has no commit message to parse).
  # All app-CI entry jobs gate on `needs.triage.outputs.skip_app_ci != 'true'`,
  # replacing the six per-job `startsWith(…, '(build): observability')` guards.
  # ===========================================================================
  triage:
    name: 🧭 Triage build trigger
    runs-on: ubuntu-latest
    timeout-minutes: 2
    if: github.event_name != 'schedule'
    permissions:
      contents: read
    outputs:
      triggered: ${{ steps.t.outputs.triggered }}
      domains: ${{ steps.t.outputs.domains }}
      has_back: ${{ steps.t.outputs.has_back }}
      has_front: ${{ steps.t.outputs.has_front }}
      has_observability: ${{ steps.t.outputs.has_observability }}
      has_uptime_kuma: ${{ steps.t.outputs.has_uptime_kuma }}
      app_deploy: ${{ steps.t.outputs.app_deploy }}
      skip_app_ci: ${{ steps.t.outputs.skip_app_ci }}
    steps:
      - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1
      - name: 🧭 Parse trigger
        id: t
        env:
          COMMIT_MSG: ${{ github.event.head_commit.message || inputs.dry_run_subject }}
        run: bash .github/scripts/parse-build-trigger.sh

  # ===========================================================================
  # STAGE 0 — CHANGE DETECTION (cheap; gates the expensive extension suite)
  # ===========================================================================
  # The clean-room extension suite (~14 min) is the pipeline's long pole and is
  # a hard deploy gate. It only needs to run when something that affects an
  # extension build/test actually changed. This job emits a boolean the
  # extension-tests job keys off; lint/typecheck/security stay always-on
  # because they are fast and repo-global (typecheck still catches extension
  # TYPE regressions on every push even when the Cypress suite is skipped).
  # ===========================================================================
  changes:
    name: 🔎 Detect Changes
    runs-on: ubuntu-latest
    timeout-minutes: 5
    needs: [triage]
    # Skip on `(build): observability` infra-only commits (skip_app_ci=true from
    # triage). The weekly cron DOES run this — it is the only unfiltered
    # full-matrix run, so a suite nobody touches still gets exercised. `triage`
    # is skipped on schedule (it parses a commit message), hence `!cancelled()`
    # and the empty-string outputs it leaves behind.
    if: |
      !cancelled() &&
      needs.triage.result != 'failure' &&
      needs.triage.outputs.skip_app_ci != 'true'
    permissions:
      contents: read
    outputs:
      changed_extensions: ${{ steps.assemble.outputs.list }}

    steps:
      - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1

      - id: filter
        uses: dorny/paths-filter@ceb8a2b8f2d89434be7ff52d3de7ec3738c5cc9d # v4.0.3
        with:
          filters: .github/filters/extensions.yaml

      - id: assemble
        env:
          # Fallback (NEW.md §7.4): no reliable diff base -> treat all five as changed.
          # Covers manual dispatch AND a first push / history rewrite (before == all-zeros SHA).
          # `schedule` is deliberate, not a fallback: the weekly cron is the only
          # automatic run that is not path-filtered, so it exercises all five.
          FORCE_ALL: ${{ github.event_name == 'workflow_dispatch' || github.event_name == 'schedule' || github.event.before == '0000000000000000000000000000000000000000' }}
          R_INDENT: ${{ steps.filter.outputs.extension-indent }}
          R_HYPERLINK: ${{ steps.filter.outputs.extension-hyperlink }}
          R_HMM: ${{ steps.filter.outputs.extension-hypermultimedia }}
          R_INLINE: ${{ steps.filter.outputs.extension-inline-code }}
          R_PLACEHOLDER: ${{ steps.filter.outputs.extension-placeholder }}
        run: |
          set -euo pipefail
          all='["extension-indent","extension-hyperlink","extension-hypermultimedia","extension-inline-code","extension-placeholder"]'
          if [ "${FORCE_ALL}" = "true" ]; then echo "list=${all}" >> "$GITHUB_OUTPUT"; exit 0; fi
          sel=()
          [ "${R_INDENT}" = "true" ] && sel+=('"extension-indent"')
          [ "${R_HYPERLINK}" = "true" ] && sel+=('"extension-hyperlink"')
          [ "${R_HMM}" = "true" ] && sel+=('"extension-hypermultimedia"')
          [ "${R_INLINE}" = "true" ] && sel+=('"extension-inline-code"')
          [ "${R_PLACEHOLDER}" = "true" ] && sel+=('"extension-placeholder"')
          if [ "${#sel[@]}" -eq 0 ]; then echo "list=[]" >> "$GITHUB_OUTPUT"; else
            IFS=,; echo "list=[${sel[*]}]" >> "$GITHUB_OUTPUT"; fi

  # ===========================================================================
  # STAGE 1 — QUALITY GATES (parallel, fast feedback)
  # ===========================================================================
  lint:
    needs: [triage]
    # Skip on infra-only commits (skip_app_ci=true) or schedule.
    if: |
      github.event_name != 'schedule' &&
      needs.triage.outputs.skip_app_ci != 'true'
    uses: ./.github/workflows/quality-lint.yml

  typecheck:
    name: 📝 Type Check
    runs-on: ubuntu-latest
    timeout-minutes: 15
    needs: [triage]
    # Skip on infra-only commits (skip_app_ci=true) or schedule.
    if: |
      github.event_name != 'schedule' &&
      needs.triage.outputs.skip_app_ci != 'true'
    permissions:
      contents: read

    steps:
      - name: 📦 Checkout
        uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1

      - name: 🥟 Setup Environment
        uses: ./.github/actions/setup-bun

      - name: 🔧 Build Extensions (required for types)
        uses: ./.github/actions/build-extensions

      - name: 📝 Type Check All
        run: bun run typecheck

  security:
    name: 🔒 Security Audit
    runs-on: ubuntu-latest
    timeout-minutes: 10
    needs: [triage]
    # Keeps its schedule escape (weekly scan) even when triage is skipped.
    if: |
      always() &&
      (github.event_name == 'schedule' || needs.triage.outputs.skip_app_ci != 'true')
    permissions:
      contents: read

    steps:
      - name: 📦 Checkout
        uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1

      - name: 🥟 Setup Environment
        uses: ./.github/actions/setup-bun
        with:
          ignore-scripts: 'true'

      - name: 🔍 Bun Audit
        run: |
          set -o pipefail
          echo "🔍 Checking for known vulnerabilities..."

          # `bun audit`, NOT `bun pm audit` — the latter is not a command, so it
          # printed `bun pm` help, the JSON capture failed, and the fallback `{}`
          # made this gate report zero for every run it ever made.
          bun audit 2>&1 | tee audit-results.txt || true
          bun audit --json > audit-results.json 2>/dev/null || true

          # The gate fails when the report is missing or reshaped, so a future
          # Bun change breaks the build instead of silently passing it.
          bun .github/scripts/audit-gate.ts audit-results.json .github/audit-allowlist.json

      - name: 📤 Upload Audit Results
        if: always()
        uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1
        with:
          name: security-audit-${{ github.sha }}
          path: |
            audit-results.txt
            audit-results.json
          retention-days: 30

  back-validation:
    name: 🔧 Backend Validation
    needs: [triage]
    # `app_deploy` is the value the deploy job gates on, and the deploy always
    # runs the Prisma migration and rolls all three backend services. Gating on
    # `has_back` alone let a `(build): front` push deploy backend code no test
    # had ever touched. `has_back` stays so `(build): back no-deploy` still runs.
    if: needs.triage.outputs.has_back == 'true' || needs.triage.outputs.app_deploy == 'true'
    uses: ./.github/workflows/backend-ci.yml

  extension-tests:
    name: 🧪 Ext (${{ matrix.ext }})
    needs: [triage, changes, back-validation]
    # Run for a `front` trigger OR a normal app-CI commit; only changed extensions.
    if: |
      always() &&
      needs.triage.outputs.skip_app_ci != 'true' &&
      needs.back-validation.result != 'failure' &&
      needs.changes.outputs.changed_extensions != '[]' &&
      (needs.triage.outputs.has_front == 'true' || needs.triage.outputs.triggered != 'true' || github.event_name == 'pull_request')
    strategy:
      fail-fast: false
      matrix:
        # `|| '[]'` guards the runtime throw if `changes` is skipped and the output is '' (actionlint won't catch this).
        ext: ${{ fromJson(needs.changes.outputs.changed_extensions || '[]') }}
    runs-on: ubuntu-latest
    timeout-minutes: 25
    permissions:
      contents: read
    steps:
      - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1
      - uses: ./.github/actions/setup-bun
      - uses: ./.github/actions/build-extensions
        with:
          extensions: ${{ matrix.ext }}
      - uses: ./.github/actions/setup-cypress
      - name: 🧪 Clean-room suite (${{ matrix.ext }})
        env:
          EXTENSION_DIST_READY: '1'
          EXT_ONLY: ${{ matrix.ext }}
        run: bash scripts/run-tests.sh --extensions
      - name: ✈️ Preflight (${{ matrix.ext }})
        env:
          EXT_ONLY: ${{ matrix.ext }}
        run: bash scripts/extension-preflight.sh

  webapp-unit-tests:
    name: 🧪 Webapp Unit Tests
    runs-on: ubuntu-latest
    timeout-minutes: 10
    needs: [triage, back-validation]
    if: |
      always() &&
      github.event_name != 'schedule' &&
      needs.triage.outputs.skip_app_ci != 'true' &&
      needs.back-validation.result != 'failure' &&
      (needs.triage.outputs.has_front == 'true' || needs.triage.outputs.triggered != 'true' || github.event_name == 'pull_request')
    permissions:
      contents: read

    steps:
      - name: 📦 Checkout
        uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1

      - name: 🥟 Setup Environment
        uses: ./.github/actions/setup-bun

      # Webapp tests import built extension dist (e.g. @docs.plus/extension-hyperlink
      # via useHyperlinkEditorForm); build them first or the suite fails to resolve.
      - name: 🔧 Build Extensions (required for webapp imports)
        uses: ./.github/actions/build-extensions

      - name: 🧪 Jest (webapp)
        # Some webapp modules construct a Supabase client at import time (utils
        # barrel → ensureAnonymousSession), which throws without these. Dummy
        # fallbacks mirror the build job; tests never make real Supabase calls.
        env:
          NEXT_PUBLIC_SUPABASE_URL: ${{ secrets.NEXT_PUBLIC_SUPABASE_URL || 'http://localhost:54321' }}
          NEXT_PUBLIC_SUPABASE_ANON_KEY: ${{ secrets.NEXT_PUBLIC_SUPABASE_ANON_KEY || 'dummy-key' }}
        run: bun run --filter @docs.plus/webapp test

  # ===========================================================================
  # STAGE 2 — BUILD VERIFICATION (smoke test, no artifacts produced)
  # ===========================================================================
  # Note: this job intentionally does NOT build Docker images. The deploy job
  # rebuilds them on the prod host anyway (decided 2026-05); duplicating the
  # docker build here would just slow the pipeline without sharing cache. The
  # webapp/admin Next.js compile here catches type/build regressions early.
  # ===========================================================================
  build:
    name: 🏗️ Build Verification
    runs-on: ubuntu-latest
    timeout-minutes: 35
    needs:
      [
        triage,
        changes,
        lint,
        typecheck,
        security,
        extension-tests,
        webapp-unit-tests,
        back-validation
      ]
    if: |
      always() &&
      github.event_name != 'schedule' &&
      needs.triage.outputs.skip_app_ci != 'true' &&
      needs.changes.result == 'success' &&
      (needs.lint.result == 'success' || (github.event_name == 'workflow_dispatch' && inputs.skip_quality_gates)) &&
      (needs.typecheck.result == 'success' || (github.event_name == 'workflow_dispatch' && inputs.skip_quality_gates)) &&
      (needs.security.result == 'success' || (github.event_name == 'workflow_dispatch' && inputs.skip_quality_gates)) &&
      (needs.back-validation.result == 'success' || needs.back-validation.result == 'skipped' || (github.event_name == 'workflow_dispatch' && inputs.skip_quality_gates)) &&
      (needs.extension-tests.result == 'success' || needs.extension-tests.result == 'skipped' || (github.event_name == 'workflow_dispatch' && inputs.skip_quality_gates)) &&
      (needs.webapp-unit-tests.result == 'success' || needs.webapp-unit-tests.result == 'skipped' || (github.event_name == 'workflow_dispatch' && inputs.skip_quality_gates))
    permissions:
      contents: read

    steps:
      - name: 📦 Checkout
        uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1

      - name: 🥟 Setup Environment
        uses: ./.github/actions/setup-bun

      - name: 🔧 Build TipTap Extensions
        uses: ./.github/actions/build-extensions

      - name: 🏗️ Build Webapp
        run: bun run --filter @docs.plus/webapp build:ci
        env:
          NEXT_PUBLIC_SUPABASE_URL: ${{ secrets.NEXT_PUBLIC_SUPABASE_URL || 'http://localhost:54321' }}
          NEXT_PUBLIC_SUPABASE_ANON_KEY: ${{ secrets.NEXT_PUBLIC_SUPABASE_ANON_KEY || 'dummy-key' }}

      - name: 🏗️ Build Admin Dashboard
        run: bun run --filter @docs.plus/admin-dashboard build:ci
        env:
          NEXT_PUBLIC_SUPABASE_URL: ${{ secrets.NEXT_PUBLIC_SUPABASE_URL || 'http://localhost:54321' }}
          NEXT_PUBLIC_SUPABASE_ANON_KEY: ${{ secrets.NEXT_PUBLIC_SUPABASE_ANON_KEY || 'dummy-key' }}
          NEXT_PUBLIC_API_URL: ${{ secrets.NEXT_PUBLIC_API_URL || 'http://localhost:3003' }}
          NEXT_PUBLIC_APP_URL: ${{ secrets.NEXT_PUBLIC_APP_URL || 'http://localhost:3000' }}

  # ===========================================================================
  # STAGE 3 — PRODUCTION DEPLOYMENT (deploy gate inlined — no cross-job output)
  # ===========================================================================
  # The deploy decision lives directly on this job's `if`. `app_deploy` already
  # encodes `app_pipeline && !no_deploy`, so `(build): back front no-deploy` →
  # app_deploy=false → no deploy. `always()` lets the `if` evaluate even when an
  # upstream is skipped; `needs.build.result == 'success'` is the real fail-fast
  # gate. (Inlined from a former deploy-gate job whose constant output did not
  # propagate reliably through `needs.deploy-gate.outputs` under `always()`.)
  # The job-level `…-deploy` group only serializes a SECOND run's deploy behind
  # this one. It does NOT exempt the job from the workflow-level `…-quality`
  # group, so a later push still cancels an in-flight roll — the pre-existing,
  # accepted blue-green exposure, same residual as `uptime-kuma-deploy`.
  # ===========================================================================
  deploy:
    name: 🚀 Deploy Production
    runs-on: prod.docs.plus
    # 45, not 30: replicas are now retired serially and the worker's stop grace
    # is 150s, so a five-service roll plus a five-service rollback no longer
    # fits 30m. A mid-deploy SIGTERM here is worse than a slow deploy.
    timeout-minutes: 45
    needs: [triage, build]
    concurrency:
      group: ${{ github.workflow }}-deploy
      cancel-in-progress: false
    if: |
      always() &&
      needs.build.result == 'success' &&
      ((github.event_name == 'push' && needs.triage.outputs.app_deploy == 'true') ||
       (github.event_name == 'workflow_dispatch' && inputs.force_deploy))
    environment:
      name: production
      url: https://docs.plus
    permissions:
      contents: read

    steps:
      - name: 📦 Checkout Code
        uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1
        with:
          fetch-depth: 1

      - name: 🔐 Prepare Environment
        run: |
          # deploy_service is written to disk rather than defined inline because
          # the rollback step runs in its own shell and must take the identical
          # route — a second copy of this function is a drift waiting to drain
          # the fleet. Written first so a later failure here can still source it.
          cat > "${RUNNER_TEMP}/deploy-service.sh" <<'DEPLOY_SVC'
          # Rolls one compose service onto the new image without ever dropping
          # below its current backend count. Returns non-zero (and retires
          # nothing) if the new replicas do not come up healthy.
          deploy_service() {
            local SERVICE="$1" TARGET="$2"
            local -a COMPOSE=(docker compose -f "${COMPOSE_FILE}" --env-file "${ENV_FILE}")
            local -a OLD_IDS=() NEW_IDS=()
            local id name

            # Project-scoped on purpose: these ids are stopped and removed below,
            # and `docker ps --filter com.docker.compose.service=` also matches an
            # identically-named service in any other compose project on the host.
            mapfile -t OLD_IDS < <("${COMPOSE[@]}" ps -q "${SERVICE}")
            local SCALE_UP=$(( ${#OLD_IDS[@]} + TARGET ))
            # Clamp to one spare generation. A rollback over a half-failed forward
            # deploy sees TARGET stale replicas already running and would otherwise
            # ask for 3x the footprint on a host that is already unhealthy. If the
            # clamp bites, no new container appears and the NEW_IDS check below
            # fails loudly instead of the host running out of memory quietly.
            local MAX_SCALE=$(( TARGET * 2 ))
            if [ "${SCALE_UP}" -gt "${MAX_SCALE}" ]; then
              echo "::warning::${SERVICE} already runs ${#OLD_IDS[@]} replicas (target ${TARGET}); capping scale at ${MAX_SCALE}. Retire strays before redeploying."
              SCALE_UP=${MAX_SCALE}
            fi

            echo ""
            echo "📦 Deploying ${SERVICE} (running: ${#OLD_IDS[@]}, target: ${TARGET})..."

            # --no-recreate is load-bearing. DEPLOY_TAG changes the service config
            # hash, so a plain `up --scale` recreates the replicas that are
            # currently serving and takes the fleet to zero before the new ones bind.
            echo "⬆️  Starting ${TARGET} new replicas alongside the old (scale ${SCALE_UP})..."
            "${COMPOSE[@]}" up -d --no-deps --no-recreate --scale "${SERVICE}=${SCALE_UP}" "${SERVICE}"

            for id in $("${COMPOSE[@]}" ps -q "${SERVICE}"); do
              case " ${OLD_IDS[*]} " in
                *" ${id} "*) ;;
                *) NEW_IDS+=("${id}") ;;
              esac
            done
            if [ "${#NEW_IDS[@]}" -ne "${TARGET}" ]; then
              echo "::error::Expected ${TARGET} new ${SERVICE} containers, found ${#NEW_IDS[@]}. Retiring nothing."
              "${COMPOSE[@]}" ps "${SERVICE}"
              return 1
            fi

            # Health is checked per new container id. A count would pass on the
            # first tick — the old replicas are already healthy and, unlike
            # before, they stay healthy for the whole wait.
            echo "⏳ Waiting for ${#NEW_IDS[@]} new ${SERVICE} replicas to report healthy..."
            local READY=0 i
            for i in {1..60}; do
              READY=1
              for id in "${NEW_IDS[@]}"; do
                [ "$(docker inspect -f '{{.State.Health.Status}}' "${id}" 2>/dev/null)" = healthy ] || READY=0
              done
              [ "${READY}" -eq 1 ] && break
              [ $((i % 10)) -eq 0 ] && echo "   ... attempt ${i}/60"
              sleep 2
            done
            if [ "${READY}" -ne 1 ]; then
              echo "::error::New ${SERVICE} replicas never reported healthy after 120s. Old replicas left running — no traffic was drained."
              "${COMPOSE[@]}" ps "${SERVICE}"
              return 1
            fi
            echo "✅ ${#NEW_IDS[@]} new ${SERVICE} replicas healthy"

            # One at a time, so the service never loses more than one backend.
            # A bare `docker stop` honours the container's StopTimeout, which
            # compose sets from stop_grace_period — 150s for the worker (its
            # jobs need it), 30s elsewhere. A flat --time would SIGKILL the worker.
            for id in "${OLD_IDS[@]}"; do
              name=$(docker inspect -f '{{.Name}}' "${id}" 2>/dev/null || echo "${id}")
              echo "🛑 Retiring ${name}..."
              docker stop "${id}" >/dev/null || true
              docker rm "${id}" >/dev/null || true
              sleep 2
            done
          }
          DEPLOY_SVC

          # Compose `--env-file` is the single source of truth. We do NOT also
          # `set -a; source` it elsewhere — that path was double-loading and
          # leaking vars to subshells unintentionally.
          cp "${ENV_SOURCE}" "${ENV_FILE}"
          echo "DEPLOY_TAG=${DEPLOY_TAG}" >> "${ENV_FILE}"

          # Error capture must never silently regress: both DSNs feed GlitchTip.
          # Require a real first character — dotenv strips quotes, so KEY="" or
          # whitespace-only values are empty at runtime.
          for k in GLITCHTIP_DSN NEXT_PUBLIC_GLITCHTIP_DSN; do
            if ! grep -Eq "^${k}=[\"']?[^\"'[:space:]]" "${ENV_FILE}"; then
              echo "::error::${k} is missing or empty in ${ENV_SOURCE}. Set it to the GlitchTip project DSN (https://glitchtip.docs.plus) before deploying."
              exit 1
            fi
          done

          # Stash the previous successful tag (if any) for the rollback step.
          mkdir -p "${DEPLOY_STATE_DIR}"
          if [ -f "${LAST_GOOD_TAG_FILE}" ]; then
            PREVIOUS_TAG=$(cat "${LAST_GOOD_TAG_FILE}")
            echo "PREVIOUS_TAG=${PREVIOUS_TAG}" >> "$GITHUB_ENV"
            echo "ℹ️ Previous good tag: ${PREVIOUS_TAG}"
          else
            echo "PREVIOUS_TAG=" >> "$GITHUB_ENV"
            echo "ℹ️ No previous good tag stashed (first deploy or fresh state dir)"
          fi
          echo "✅ Environment ready"

      - name: 💾 Pre-deploy disk guard
        run: |
          echo "📊 Disk before prune:"
          df -h / | tail -1

          # Free space proactively. Without this, --no-cache builds can fill
          # the root volume between deploys and silently OOM/ENOSPC the build
          # step (job ends in <2min with no error). Runs before build, not after.
          docker image prune -af --filter "until=24h" 2>/dev/null || true
          docker builder prune -af --filter "until=24h" 2>/dev/null || true

          # Hard guard: refuse to build when <10 GB free. Fail loud here
          # rather than fail silently mid-build.
          AVAIL_KB=$(df --output=avail / | tail -1)
          AVAIL_GB=$((AVAIL_KB / 1024 / 1024))
          echo "📊 Disk after prune: ${AVAIL_GB} GB free"
          if [ "${AVAIL_GB}" -lt 10 ]; then
            echo "::error::Less than 10 GB free on /. Aborting deploy. SSH to host and run 'docker system prune -af --volumes'."
            df -h /
            docker system df
            exit 1
          fi

      - name: 📂 Verify build context (monorepo root)
        run: |
          if [ ! -d packages/email-templates ]; then
            echo "::error::packages/email-templates missing. Build context must be repo root (context: .). Check checkout includes the workspace."
            exit 1
          fi
          if ! grep -q 'email-templates' apps/hocuspocus.server/docker/Dockerfile.bun; then
            echo "::error::apps/hocuspocus.server/docker/Dockerfile.bun must COPY packages/email-templates."
            exit 1
          fi
          if ! grep -q 'email-templates' apps/webapp/docker/Dockerfile.bun; then
            echo "::error::apps/webapp/docker/Dockerfile.bun must COPY packages/email-templates."
            exit 1
          fi
          echo "✅ Build context OK (repo root, email-templates present)"

      - name: 🏗️ Build Docker Images
        env:
          DOCKER_BUILDKIT: '1'
          COMPOSE_DOCKER_CLI_BUILD: '1'
        run: |
          echo "🔨 Building images with tag: ${DEPLOY_TAG}"

          # hocuspocus-server and hocuspocus-worker share `docsplus-hocuspocus`;
          # building both via compose with --no-cache duplicates context transfer
          # and ties up the bake plan. Build via hocuspocus-server only; the
          # worker reuses the resulting tag at `up` time.
          #
          # --no-cache: required as long as the prod entrypoint script changes
          # are layered late in the Dockerfile and we don't yet have stable
          # layer ordering. If/when entrypoint COPY moves to the last layer,
          # we can drop --no-cache and gain ~5 min per deploy.
          docker compose -f "${COMPOSE_FILE}" --env-file "${ENV_FILE}" \
            build --no-cache rest-api hocuspocus-server

          docker compose -f "${COMPOSE_FILE}" --env-file "${ENV_FILE}" \
            build --parallel webapp admin-dashboard

          echo "✅ Images built"

      - name: 🔧 Ensure Infrastructure
        run: |
          # First step that mutates the prod host — everything before (env guard,
          # disk guard, image build) leaves production untouched, and the failure
          # notification keys off this marker to avoid a false rollback page.
          echo "DEPLOY_STARTED=true" >> "$GITHUB_ENV"
          echo "🔧 Ensuring infrastructure..."

          docker network create docsplus-network 2>/dev/null || true

          # No --no-recreate: neither service interpolates DEPLOY_TAG, so compose
          # recreates them only when their compose config actually changed.
          # --no-recreate made every edge/queue config change inert after boot.
          docker compose -f "${COMPOSE_FILE}" --env-file "${ENV_FILE}" \
            up -d traefik redis

          # Force-start Traefik if somehow not running
          if ! docker ps --filter "name=traefik" --filter "status=running" --format '{{.Names}}' | grep -q traefik; then
            echo "⚠️ Traefik not running, starting..."
            docker compose -f "${COMPOSE_FILE}" --env-file "${ENV_FILE}" up -d traefik
            sleep 15
          fi

          # Wait for healthy
          echo "⏳ Waiting for Traefik..."
          for i in {1..30}; do
            if docker ps --filter "name=traefik" --filter "health=healthy" --format '{{.Names}}' | grep -q traefik; then
              echo "✅ Traefik healthy"
              break
            fi
            [ "${i}" -eq 30 ] && echo "⚠️ Traefik health timeout, continuing..."
            sleep 2
          done

      - name: 🗄️ Migrate database (one-shot gate)
        run: |
          # Single prod migration runner: fail-fast BEFORE any app replica scales
          # up, so a bad migration aborts the deploy and the old version keeps
          # serving. App replicas have RUN_MIGRATIONS=0 — this is the only path.
          # `run --rm` runs once and propagates the migrate exit code.
          echo "🗄️ Running database migrations (one-shot, fail-fast)..."
          docker compose -f "${COMPOSE_FILE}" --env-file "${ENV_FILE}" \
            run --rm --no-deps migrate
          echo "✅ Migrations applied"

      - name: 🚀 Deploy Services (rolling replace)
        run: |
          echo "🚀 Starting rolling deployment..."
          source "${RUNNER_TEMP}/deploy-service.sh"

          # Callees before callers, so a consumer always understands the newest
          # shape before a producer emits it. Workers before servers for the job
          # shape (compose declares 2 worker replicas — a scale of 1 silently
          # halved the fleet on every deploy); hocuspocus-server before rest-api
          # because REST forwards content applies to its internal :4003 endpoint.
          deploy_service "hocuspocus-worker" 2
          deploy_service "webapp" 2
          deploy_service "hocuspocus-server" 2
          deploy_service "rest-api" 2
          deploy_service "admin-dashboard" 1

          echo ""
          echo "✅ All services deployed"

      - name: 🩺 Verify Deployment
        run: |
          echo "🩺 Verifying deployment..."
          sleep 10

          # Infrastructure check
          echo "📊 Infrastructure:"
          for svc in traefik docsplus-redis; do
            if docker ps --filter "name=${svc}" --filter "status=running" --format '{{.Names}}' | grep -q "${svc}"; then
              echo "  ✅ ${svc}: running"
            else
              echo "  ❌ ${svc}: NOT running"
              docker logs "${svc}" --tail 30 2>/dev/null || true
              exit 1
            fi
          done

          # Service running + healthy check
          echo "📊 Services:"
          for svc in webapp rest-api hocuspocus-server hocuspocus-worker admin-dashboard; do
            RUNNING=$(docker ps --filter "label=com.docker.compose.service=${svc}" --filter "status=running" --format "{{.Names}}" | wc -l)
            HEALTHY=$(docker ps --filter "label=com.docker.compose.service=${svc}" --filter "health=healthy" --format "{{.Names}}" | wc -l)

            if [ "${RUNNING}" -gt 0 ]; then
              echo "  ✅ ${svc}: ${RUNNING} running, ${HEALTHY} healthy"
            else
              echo "  ❌ ${svc}: NOT running"
              exit 1
            fi
          done

          # Internal smoke test — hit container health endpoints via the
          # docker network, NOT via the public DNS+TLS stack. A transient
          # ACME / Let's Encrypt hiccup must not trigger a false-fail rollback.
          echo ""
          echo "🔍 Internal smoke tests..."
          smoke() {
            local SVC="$1" PORT="$2" PATH_="$3"
            if docker compose -f "${COMPOSE_FILE}" --env-file "${ENV_FILE}" exec -T "${SVC}" \
              bun -e "fetch('http://localhost:${PORT}${PATH_}').then(r => r.ok ? process.exit(0) : process.exit(1)).catch(() => process.exit(1))"; then
              echo "  ✅ ${SVC} internal health"
            else
              echo "  ❌ ${SVC} internal health"
              return 1
            fi
          }
          smoke webapp 3000 /api/health
          smoke rest-api 4000 /health
          smoke hocuspocus-server 4001 /health
          smoke hocuspocus-worker 4002 /health
          smoke admin-dashboard 3100 /api/health

          # Public-URL probe is now informational only — does NOT fail the deploy.
          # Real public-availability monitoring belongs in uptime-kuma, not here.
          echo ""
          echo "🌐 Public URL probe (informational):"
          PUBLIC_CODE=$(curl -sf -o /dev/null -w "%{http_code}" --max-time 10 https://docs.plus/ 2>/dev/null || echo "000")
          echo "  https://docs.plus/ → ${PUBLIC_CODE}"
          API_CODE=$(curl -sf -o /dev/null -w "%{http_code}" --max-time 10 https://prodback.docs.plus/api/health 2>/dev/null || echo "000")
          echo "  https://prodback.docs.plus/api/health → ${API_CODE}"

          echo ""
          echo "✅ Deployment verified"

      - name: 📁 Sync compose files for break-glass
        if: success()
        run: |
          # Replaces the previous "Sync Production Directory" step which re-`up`'d
          # services from a different cwd — that re-up broke the blue-green
          # guarantee. Now we only COPY the active compose + env files to a
          # stable path so a human SSH'd in can run, e.g.:
          #   cd /opt/projects/prod.docs.plus/.deploy/current
          #   docker compose -f docker-compose.prod.yml --env-file .env.production ps
          # without having to know the runner's _work directory.
          mkdir -p "${DEPLOY_STATE_DIR}/current"
          cp "${COMPOSE_FILE}" "${DEPLOY_STATE_DIR}/current/${COMPOSE_FILE}"
          cp "${ENV_FILE}" "${DEPLOY_STATE_DIR}/current/${ENV_FILE}"
          echo "✅ Synced compose+env to ${DEPLOY_STATE_DIR}/current/"

      - name: 💾 Stash this tag as last-good
        # Only on success — failure path is handled by the rollback step.
        if: success()
        run: |
          # Persist current tag for the next deploy's rollback target.
          mkdir -p "${DEPLOY_STATE_DIR}"
          # Keep the previous one as last-good-tag.previous for one-step-back debugging.
          if [ -f "${LAST_GOOD_TAG_FILE}" ]; then
            cp "${LAST_GOOD_TAG_FILE}" "${LAST_GOOD_TAG_FILE}.previous"
          fi
          echo "${DEPLOY_TAG}" > "${LAST_GOOD_TAG_FILE}"
          echo "✅ Stashed last-good-tag = ${DEPLOY_TAG}"

      - name: 🧹 Cleanup
        if: success()
        continue-on-error: true # cleanup failure shouldn't fail an otherwise green deploy
        run: |
          docker image prune -f
          docker image prune -f --filter "until=24h" 2>/dev/null || true
          echo "✅ Cleanup complete"

      - name: 📊 Summary
        if: success()
        run: |
          echo "======================================"
          echo "✅ DEPLOYMENT SUCCESSFUL"
          echo "======================================"
          echo "Tag:           ${DEPLOY_TAG}"
          echo "Previous tag:  ${PREVIOUS_TAG:-<none>}"
          echo ""
          echo "Services:"
          docker ps --format "table {{.Names}}\t{{.Status}}\t{{.Ports}}" | grep -E "(traefik|docsplus|webapp|rest-api|hocuspocus)" | head -15
          echo ""
          echo "URLs:"
          echo "  - https://docs.plus"
          echo "  - https://prodback.docs.plus"
          echo "======================================"

      - name: 📌 Grafana deploy annotation
        if: success()
        run: |
          # Grafana publishes no host port (traefik-only); reach it via its
          # docker-network IP from the host. Never fails the deploy.
          OBS_ENV=/opt/projects/prod.docs.plus/.env.observability
          GRAFANA_PASS=$(grep '^GRAFANA_ADMIN_PASSWORD=' "${OBS_ENV}" 2>/dev/null | cut -d= -f2- || true)
          GRAFANA_IP=$(docker inspect -f '{{range .NetworkSettings.Networks}}{{.IPAddress}}{{end}}' docsplus-grafana 2>/dev/null || true)
          if [ -z "${GRAFANA_PASS}" ] || [ -z "${GRAFANA_IP}" ]; then
            echo "::warning::Grafana unreachable; skipping deploy annotation"; exit 0
          fi
          curl -sf --max-time 10 -u "admin:${GRAFANA_PASS}" \
            -H 'Content-Type: application/json' \
            -d "{\"text\":\"deploy ${DEPLOY_TAG}\",\"tags\":[\"deploy\"]}" \
            "http://${GRAFANA_IP}:3000/api/annotations" >/dev/null \
            && echo "✅ Deploy annotation posted" \
            || echo "::warning::Grafana deploy annotation failed (non-fatal)"

      - name: 🚨 Rollback on Failure
        id: rollback
        if: failure()
        run: |
          echo "⚠️ Deployment failed — attempting rollback..."

          if [ -z "${PREVIOUS_TAG:-}" ]; then
            echo "::warning::No PREVIOUS_TAG stashed — cannot auto-rollback."
            echo "📊 Current state:"
            docker ps --format "table {{.Names}}\t{{.Status}}" | head -15
            exit 0
          fi

          echo "↩️  Rolling back to: ${PREVIOUS_TAG}"

          # Multi-image precondition (A2): ALL service images for the previous
          # tag must still exist locally. The cleanup step honors --filter
          # until=24h, so within a 24h window this works reliably; outside
          # that window we fail loudly rather than partially-rollback into a
          # mixed-version cluster.
          MISSING=()
          for img in docsplus-webapp docsplus-rest-api docsplus-hocuspocus docsplus-admin; do
            if ! docker image inspect "${img}:${PREVIOUS_TAG}" >/dev/null 2>&1; then
              MISSING+=("${img}:${PREVIOUS_TAG}")
            fi
          done
          if [ "${#MISSING[@]}" -gt 0 ]; then
            echo "::error::Cannot auto-rollback. Missing images for previous tag:"
            for img in "${MISSING[@]}"; do
              echo "  - ${img}"
            done
            echo "Manual recovery: bring traffic back via the existing healthy containers."
            docker compose -f "${COMPOSE_FILE}" --env-file "${ENV_FILE}" up -d --no-recreate 2>/dev/null || true
            exit 1
          fi

          # Override DEPLOY_TAG in the env file and re-deploy with previous images.
          sed -i.bak "s|^DEPLOY_TAG=.*|DEPLOY_TAG=${PREVIOUS_TAG}|" "${ENV_FILE}"
          rm -f "${ENV_FILE}.bak"

          # Identical route to the forward deploy, by construction — a
          # --force-recreate here restarts every service at once and takes the
          # fleet to zero, which is the outage this path exists to end.
          if [ ! -f "${RUNNER_TEMP}/deploy-service.sh" ]; then
            echo "::error::deploy-service.sh missing — the run failed before environment prep."
            exit 1
          fi
          source "${RUNNER_TEMP}/deploy-service.sh"

          # Keep going past a failed service so the rest still get their old
          # image back; the smoke tests below decide the step's exit code.
          ROLLBACK_OK=1
          deploy_service "hocuspocus-worker" 2 || ROLLBACK_OK=0
          deploy_service "webapp" 2 || ROLLBACK_OK=0
          deploy_service "hocuspocus-server" 2 || ROLLBACK_OK=0
          deploy_service "rest-api" 2 || ROLLBACK_OK=0
          deploy_service "admin-dashboard" 1 || ROLLBACK_OK=0

          # Post-rollback verification (A3). Give containers a moment to bind
          # ports + pass first healthcheck, then run the same internal smoke
          # set the forward path runs. If rollback itself can't come healthy,
          # we want the workflow to fail RED so the on-call sees it instead
          # of a misleading "rollback complete" green check.
          echo ""
          echo "⏳ Waiting 30s for rolled-back containers to settle..."
          sleep 30

          smoke() {
            local SVC="$1" PORT="$2" PATH_="$3"
            if docker compose -f "${COMPOSE_FILE}" --env-file "${ENV_FILE}" exec -T "${SVC}" \
              bun -e "fetch('http://localhost:${PORT}${PATH_}').then(r => r.ok ? process.exit(0) : process.exit(1)).catch(() => process.exit(1))"; then
              echo "  ✅ ${SVC} internal health (post-rollback)"
              return 0
            else
              echo "  ❌ ${SVC} internal health (post-rollback)"
              return 1
            fi
          }

          smoke webapp 3000 /api/health || ROLLBACK_OK=0
          smoke rest-api 4000 /health || ROLLBACK_OK=0
          smoke hocuspocus-server 4001 /health || ROLLBACK_OK=0
          smoke hocuspocus-worker 4002 /health || ROLLBACK_OK=0
          smoke admin-dashboard 3100 /api/health || ROLLBACK_OK=0

          echo ""
          echo "📊 Post-rollback state:"
          docker ps --format "table {{.Names}}\t{{.Status}}" | head -15

          if [ "${ROLLBACK_OK}" -ne 1 ]; then
            echo "::error::Rollback to ${PREVIOUS_TAG} did not pass smoke tests. Manual intervention required."
            exit 1
          fi

          echo "✅ Rollback to ${PREVIOUS_TAG} verified healthy"

      - name: 📣 Notify deploy failure
        if: failure()
        env:
          ROLLBACK_OUTCOME: ${{ steps.rollback.outcome }}
        run: |
          # Credentials live in the observability env file on this host. Never fatal.
          OBS_ENV=/opt/projects/prod.docs.plus/.env.observability
          if [ "${DEPLOY_STARTED:-}" != "true" ]; then
            TEXT="⚠️ docs.plus deploy of ${DEPLOY_TAG} aborted before touching prod (env guard or build); production unchanged"
          elif [ "${ROLLBACK_OUTCOME}" = "success" ] && [ -n "${PREVIOUS_TAG:-}" ]; then
            TEXT="🚨 docs.plus deploy of ${DEPLOY_TAG} FAILED — rolled back to ${PREVIOUS_TAG}"
          else
            TEXT="🚨 docs.plus deploy of ${DEPLOY_TAG} FAILED and rollback FAILED — manual intervention required"
          fi
          bash scripts/ci/notify-telegram.sh "${OBS_ENV}" "${TEXT}"

  # ===========================================================================
  # STAGE 4 — OBSERVABILITY DEPLOY (ordered after app deploy)
  # ===========================================================================
  observability-deploy:
    name: 🔭 Deploy Observability
    needs: [triage]
    # Independent monitoring infra — deploys whenever `observability` is in the set.
    # Deliberately does NOT `needs: deploy`: GitHub does not reliably run a
    # reusable-workflow call job under `always()` when a needed job (the app deploy)
    # is skipped, so depending on `deploy` made `(build): observability` silently
    # skip. The single prod runner serializes app vs observability deploys anyway,
    # and monitoring should stay up regardless of an app-deploy outcome.
    if: |
      github.event_name == 'push' &&
      needs.triage.outputs.has_observability == 'true'
    uses: ./.github/workflows/observability.docs.plus.yml
    with:
      action: setup

  # ===========================================================================
  # STAGE 5 — UPTIME KUMA DEPLOY (trailing, ordered last)
  # ===========================================================================
  uptime-kuma-deploy:
    name: 🔔 Deploy Uptime Kuma
    runs-on: prod.docs.plus
    timeout-minutes: 10
    # Job-level group serializes a SECOND workflow run's uptime-kuma deploy behind the
    # first (cross-run). It does NOT shield this job from the workflow-level `…-quality`
    # group cancelling the parent run — that residual is the pre-existing, accepted
    # blue-green exposure (Global Constraints "preserve, do not regress"); this plan does
    # not widen it. Matches the app deploy's own `…-deploy` group semantics.
    concurrency:
      group: uptime-kuma-deploy
      cancel-in-progress: false
    needs: [triage, deploy, observability-deploy]
    # Always LAST. Fail-fast: any app domain in the set requires deploy success, and
    # any observability in the set requires observability success (a skip from an
    # upstream failure is not success). Absent domains skip and don't block.
    if: |
      always() &&
      github.event_name == 'push' &&
      needs.triage.outputs.has_uptime_kuma == 'true' &&
      (needs.triage.outputs.app_deploy != 'true' || needs.deploy.result == 'success') &&
      (needs.triage.outputs.has_observability != 'true' || needs.observability-deploy.result == 'success')
    permissions: { contents: read }
    steps:
      - name: 🚀 Deploy
        run: |
          UPTIME_KUMA_IMAGE='louislam/uptime-kuma:1@sha256:bb1bcecbc3e3ffb1cb0f8fc5f9c3cdaa78c1dfb56d98d64e06da13ebfc6dba0d'
          docker network create docsplus-network 2>/dev/null || true
          docker stop uptime-kuma 2>/dev/null || true
          docker rm uptime-kuma 2>/dev/null || true
          docker run -d --name uptime-kuma --network docsplus-network --restart unless-stopped \
            -v uptime-kuma-data:/app/data \
            --label "traefik.enable=true" \
            --label "traefik.http.routers.uptime.rule=Host(\`status.docs.plus\`)" \
            --label "traefik.http.routers.uptime.entrypoints=websecure" \
            --label "traefik.http.routers.uptime.tls.certresolver=letsencrypt" \
            --label "traefik.http.services.uptime.loadbalancer.server.port=3001" \
            "${UPTIME_KUMA_IMAGE}"
          # Health poll replaces blind sleep 15 (D6 cheap win).
          for i in $(seq 1 20); do
            if docker exec uptime-kuma wget -qO- http://localhost:3001 >/dev/null 2>&1; then echo "✅ uptime-kuma healthy"; break; fi
            [ "$i" -eq 20 ] && echo "⚠️ uptime-kuma health timeout (continuing)"; sleep 2
          done