Skip to content

(build): front

(build): front #625

# =============================================================================
# Production CI/CD Pipeline — Quality Gates + Blue-Green Deployment
# =============================================================================
#
# Pipeline:
# 1. Quality Gates (parallel): lint, typecheck, security
# 2. Build Verification: extension + Next.js builds (smoke test)
# 3. Deploy Gate: parses commit message with a strict regex (anti-footgun)
# 4. Deploy: docker compose build on the prod host → blue-green rollout
#
# Triggers:
# - push to main: full pipeline. Deploy chain gated by triage outputs
# (`app_deploy`, `has_observability`, `has_uptime_kuma`). Ordered:
# app → observability → uptime-kuma (fail-fast).
# - pull_request to main: quality gates + build verification only.
# - schedule (Sunday 00:00 UTC): security audit + all five extension suites.
# - workflow_dispatch: manual run with `force_deploy` / `skip_quality_gates`.
#
# Architectural notes (decided 2026-05, see commit history):
# - Images are built on the prod self-hosted runner (`prod.docs.plus`).
# Disk pressure is mitigated by a pre-build disk guard, not by pushing
# to a registry. If pressure resurfaces, revisit M3 (ghcr.io push).
# - Rollback uses an on-disk tag stash on the prod host:
# /opt/projects/prod.docs.plus/.deploy/last-good-tag.
# - All third-party actions (workflow + composite) are pinned to commit
# SHA. Renovate/Dependabot should bump them; never use floating tags.
# =============================================================================
name: CI/CD Production
on:
push:
branches: [main]
pull_request:
branches: [main]
schedule:
# Weekly security scan + unfiltered extension suites (Sunday 00:00 UTC)
- cron: '0 0 * * 0'
workflow_dispatch:
inputs:
skip_quality_gates:
description: 'Skip quality gates (emergency deploy)'
required: false
default: false
type: boolean
force_deploy:
description: 'Force deployment (bypass commit-message gate)'
required: false
default: false
type: boolean
dry_run_subject:
description: 'Simulated commit subject to observe trigger gating (no deploy on dispatch)'
required: false
default: ''
type: string
# Two concurrency groups:
# - quality-gates can be cancelled freely (cheap to redo)
# - deploy MUST finish or rollback (mid-deploy SIGTERM corrupts blue-green state)
concurrency:
group: ${{ github.workflow }}-${{ github.ref }}-quality
cancel-in-progress: true
# Default to bash so set -e/-o pipefail behavior is consistent across runners.
defaults:
run:
shell: bash
# Workflow-level least privilege; per-job overrides where needed.
permissions:
contents: read
env:
ENV_SOURCE: /opt/projects/prod.docs.plus/.env
ENV_FILE: .env.production
COMPOSE_FILE: docker-compose.prod.yml
DEPLOY_TAG: ${{ github.sha }}
# Where the prod host stashes the last successfully deployed SHA for rollback.
DEPLOY_STATE_DIR: /opt/projects/prod.docs.plus/.deploy
LAST_GOOD_TAG_FILE: /opt/projects/prod.docs.plus/.deploy/last-good-tag
jobs:
# ===========================================================================
# STAGE −1 — TRIAGE (runs parse-build-trigger.sh; exposes skip_app_ci output)
# ===========================================================================
# Skipped on `schedule` (weekly security scan has no commit message to parse).
# All app-CI entry jobs gate on `needs.triage.outputs.skip_app_ci != 'true'`,
# replacing the six per-job `startsWith(…, '(build): observability')` guards.
# ===========================================================================
triage:
name: 🧭 Triage build trigger
runs-on: ubuntu-latest
timeout-minutes: 2
if: github.event_name != 'schedule'
permissions:
contents: read
outputs:
triggered: ${{ steps.t.outputs.triggered }}
domains: ${{ steps.t.outputs.domains }}
has_back: ${{ steps.t.outputs.has_back }}
has_front: ${{ steps.t.outputs.has_front }}
has_observability: ${{ steps.t.outputs.has_observability }}
has_uptime_kuma: ${{ steps.t.outputs.has_uptime_kuma }}
app_deploy: ${{ steps.t.outputs.app_deploy }}
skip_app_ci: ${{ steps.t.outputs.skip_app_ci }}
steps:
- uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1
- name: 🧭 Parse trigger
id: t
env:
COMMIT_MSG: ${{ github.event.head_commit.message || inputs.dry_run_subject }}
run: bash .github/scripts/parse-build-trigger.sh
# ===========================================================================
# STAGE 0 — CHANGE DETECTION (cheap; gates the expensive extension suite)
# ===========================================================================
# The clean-room extension suite (~14 min) is the pipeline's long pole and is
# a hard deploy gate. It only needs to run when something that affects an
# extension build/test actually changed. This job emits a boolean the
# extension-tests job keys off; lint/typecheck/security stay always-on
# because they are fast and repo-global (typecheck still catches extension
# TYPE regressions on every push even when the Cypress suite is skipped).
# ===========================================================================
changes:
name: 🔎 Detect Changes
runs-on: ubuntu-latest
timeout-minutes: 5
needs: [triage]
# Skip on `(build): observability` infra-only commits (skip_app_ci=true from
# triage). The weekly cron DOES run this — it is the only unfiltered
# full-matrix run, so a suite nobody touches still gets exercised. `triage`
# is skipped on schedule (it parses a commit message), hence `!cancelled()`
# and the empty-string outputs it leaves behind.
if: |
!cancelled() &&
needs.triage.result != 'failure' &&
needs.triage.outputs.skip_app_ci != 'true'
permissions:
contents: read
outputs:
changed_extensions: ${{ steps.assemble.outputs.list }}
steps:
- uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1
- id: filter
uses: dorny/paths-filter@ceb8a2b8f2d89434be7ff52d3de7ec3738c5cc9d # v4.0.3
with:
filters: .github/filters/extensions.yaml
- id: assemble
env:
# Fallback (NEW.md §7.4): no reliable diff base -> treat all five as changed.
# Covers manual dispatch AND a first push / history rewrite (before == all-zeros SHA).
# `schedule` is deliberate, not a fallback: the weekly cron is the only
# automatic run that is not path-filtered, so it exercises all five.
FORCE_ALL: ${{ github.event_name == 'workflow_dispatch' || github.event_name == 'schedule' || github.event.before == '0000000000000000000000000000000000000000' }}
R_INDENT: ${{ steps.filter.outputs.extension-indent }}
R_HYPERLINK: ${{ steps.filter.outputs.extension-hyperlink }}
R_HMM: ${{ steps.filter.outputs.extension-hypermultimedia }}
R_INLINE: ${{ steps.filter.outputs.extension-inline-code }}
R_PLACEHOLDER: ${{ steps.filter.outputs.extension-placeholder }}
run: |
set -euo pipefail
all='["extension-indent","extension-hyperlink","extension-hypermultimedia","extension-inline-code","extension-placeholder"]'
if [ "${FORCE_ALL}" = "true" ]; then echo "list=${all}" >> "$GITHUB_OUTPUT"; exit 0; fi
sel=()
[ "${R_INDENT}" = "true" ] && sel+=('"extension-indent"')
[ "${R_HYPERLINK}" = "true" ] && sel+=('"extension-hyperlink"')
[ "${R_HMM}" = "true" ] && sel+=('"extension-hypermultimedia"')
[ "${R_INLINE}" = "true" ] && sel+=('"extension-inline-code"')
[ "${R_PLACEHOLDER}" = "true" ] && sel+=('"extension-placeholder"')
if [ "${#sel[@]}" -eq 0 ]; then echo "list=[]" >> "$GITHUB_OUTPUT"; else
IFS=,; echo "list=[${sel[*]}]" >> "$GITHUB_OUTPUT"; fi
# ===========================================================================
# STAGE 1 — QUALITY GATES (parallel, fast feedback)
# ===========================================================================
lint:
needs: [triage]
# Skip on infra-only commits (skip_app_ci=true) or schedule.
if: |
github.event_name != 'schedule' &&
needs.triage.outputs.skip_app_ci != 'true'
uses: ./.github/workflows/quality-lint.yml
typecheck:
name: 📝 Type Check
runs-on: ubuntu-latest
timeout-minutes: 15
needs: [triage]
# Skip on infra-only commits (skip_app_ci=true) or schedule.
if: |
github.event_name != 'schedule' &&
needs.triage.outputs.skip_app_ci != 'true'
permissions:
contents: read
steps:
- name: 📦 Checkout
uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1
- name: 🥟 Setup Environment
uses: ./.github/actions/setup-bun
- name: 🔧 Build Extensions (required for types)
uses: ./.github/actions/build-extensions
- name: 📝 Type Check All
run: bun run typecheck
security:
name: 🔒 Security Audit
runs-on: ubuntu-latest
timeout-minutes: 10
needs: [triage]
# Keeps its schedule escape (weekly scan) even when triage is skipped.
if: |
always() &&
(github.event_name == 'schedule' || needs.triage.outputs.skip_app_ci != 'true')
permissions:
contents: read
steps:
- name: 📦 Checkout
uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1
- name: 🥟 Setup Environment
uses: ./.github/actions/setup-bun
with:
ignore-scripts: 'true'
- name: 🔍 Bun Audit
run: |
set -o pipefail
echo "🔍 Checking for known vulnerabilities..."
# `bun audit`, NOT `bun pm audit` — the latter is not a command, so it
# printed `bun pm` help, the JSON capture failed, and the fallback `{}`
# made this gate report zero for every run it ever made.
bun audit 2>&1 | tee audit-results.txt || true
bun audit --json > audit-results.json 2>/dev/null || true
# The gate fails when the report is missing or reshaped, so a future
# Bun change breaks the build instead of silently passing it.
bun .github/scripts/audit-gate.ts audit-results.json .github/audit-allowlist.json
- name: 📤 Upload Audit Results
if: always()
uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1
with:
name: security-audit-${{ github.sha }}
path: |
audit-results.txt
audit-results.json
retention-days: 30
back-validation:
name: 🔧 Backend Validation
needs: [triage]
# `app_deploy` is the value the deploy job gates on, and the deploy always
# runs the Prisma migration and rolls all three backend services. Gating on
# `has_back` alone let a `(build): front` push deploy backend code no test
# had ever touched. `has_back` stays so `(build): back no-deploy` still runs.
if: needs.triage.outputs.has_back == 'true' || needs.triage.outputs.app_deploy == 'true'
uses: ./.github/workflows/backend-ci.yml
extension-tests:
name: 🧪 Ext (${{ matrix.ext }})
needs: [triage, changes, back-validation]
# Run for a `front` trigger OR a normal app-CI commit; only changed extensions.
if: |
always() &&
needs.triage.outputs.skip_app_ci != 'true' &&
needs.back-validation.result != 'failure' &&
needs.changes.outputs.changed_extensions != '[]' &&
(needs.triage.outputs.has_front == 'true' || needs.triage.outputs.triggered != 'true' || github.event_name == 'pull_request')
strategy:
fail-fast: false
matrix:
# `|| '[]'` guards the runtime throw if `changes` is skipped and the output is '' (actionlint won't catch this).
ext: ${{ fromJson(needs.changes.outputs.changed_extensions || '[]') }}
runs-on: ubuntu-latest
timeout-minutes: 25
permissions:
contents: read
steps:
- uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1
- uses: ./.github/actions/setup-bun
- uses: ./.github/actions/build-extensions
with:
extensions: ${{ matrix.ext }}
- uses: ./.github/actions/setup-cypress
- name: 🧪 Clean-room suite (${{ matrix.ext }})
env:
EXTENSION_DIST_READY: '1'
EXT_ONLY: ${{ matrix.ext }}
run: bash scripts/run-tests.sh --extensions
- name: ✈️ Preflight (${{ matrix.ext }})
env:
EXT_ONLY: ${{ matrix.ext }}
run: bash scripts/extension-preflight.sh
webapp-unit-tests:
name: 🧪 Webapp Unit Tests
runs-on: ubuntu-latest
timeout-minutes: 10
needs: [triage, back-validation]
if: |
always() &&
github.event_name != 'schedule' &&
needs.triage.outputs.skip_app_ci != 'true' &&
needs.back-validation.result != 'failure' &&
(needs.triage.outputs.has_front == 'true' || needs.triage.outputs.triggered != 'true' || github.event_name == 'pull_request')
permissions:
contents: read
steps:
- name: 📦 Checkout
uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1
- name: 🥟 Setup Environment
uses: ./.github/actions/setup-bun
# Webapp tests import built extension dist (e.g. @docs.plus/extension-hyperlink
# via useHyperlinkEditorForm); build them first or the suite fails to resolve.
- name: 🔧 Build Extensions (required for webapp imports)
uses: ./.github/actions/build-extensions
- name: 🧪 Jest (webapp)
# Some webapp modules construct a Supabase client at import time (utils
# barrel → ensureAnonymousSession), which throws without these. Dummy
# fallbacks mirror the build job; tests never make real Supabase calls.
env:
NEXT_PUBLIC_SUPABASE_URL: ${{ secrets.NEXT_PUBLIC_SUPABASE_URL || 'http://localhost:54321' }}
NEXT_PUBLIC_SUPABASE_ANON_KEY: ${{ secrets.NEXT_PUBLIC_SUPABASE_ANON_KEY || 'dummy-key' }}
run: bun run --filter @docs.plus/webapp test
# ===========================================================================
# STAGE 2 — BUILD VERIFICATION (smoke test, no artifacts produced)
# ===========================================================================
# Note: this job intentionally does NOT build Docker images. The deploy job
# rebuilds them on the prod host anyway (decided 2026-05); duplicating the
# docker build here would just slow the pipeline without sharing cache. The
# webapp/admin Next.js compile here catches type/build regressions early.
# ===========================================================================
build:
name: 🏗️ Build Verification
runs-on: ubuntu-latest
timeout-minutes: 35
needs:
[
triage,
changes,
lint,
typecheck,
security,
extension-tests,
webapp-unit-tests,
back-validation
]
if: |
always() &&
github.event_name != 'schedule' &&
needs.triage.outputs.skip_app_ci != 'true' &&
needs.changes.result == 'success' &&
(needs.lint.result == 'success' || (github.event_name == 'workflow_dispatch' && inputs.skip_quality_gates)) &&
(needs.typecheck.result == 'success' || (github.event_name == 'workflow_dispatch' && inputs.skip_quality_gates)) &&
(needs.security.result == 'success' || (github.event_name == 'workflow_dispatch' && inputs.skip_quality_gates)) &&
(needs.back-validation.result == 'success' || needs.back-validation.result == 'skipped' || (github.event_name == 'workflow_dispatch' && inputs.skip_quality_gates)) &&
(needs.extension-tests.result == 'success' || needs.extension-tests.result == 'skipped' || (github.event_name == 'workflow_dispatch' && inputs.skip_quality_gates)) &&
(needs.webapp-unit-tests.result == 'success' || needs.webapp-unit-tests.result == 'skipped' || (github.event_name == 'workflow_dispatch' && inputs.skip_quality_gates))
permissions:
contents: read
steps:
- name: 📦 Checkout
uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1
- name: 🥟 Setup Environment
uses: ./.github/actions/setup-bun
- name: 🔧 Build TipTap Extensions
uses: ./.github/actions/build-extensions
- name: 🏗️ Build Webapp
run: bun run --filter @docs.plus/webapp build:ci
env:
NEXT_PUBLIC_SUPABASE_URL: ${{ secrets.NEXT_PUBLIC_SUPABASE_URL || 'http://localhost:54321' }}
NEXT_PUBLIC_SUPABASE_ANON_KEY: ${{ secrets.NEXT_PUBLIC_SUPABASE_ANON_KEY || 'dummy-key' }}
- name: 🏗️ Build Admin Dashboard
run: bun run --filter @docs.plus/admin-dashboard build:ci
env:
NEXT_PUBLIC_SUPABASE_URL: ${{ secrets.NEXT_PUBLIC_SUPABASE_URL || 'http://localhost:54321' }}
NEXT_PUBLIC_SUPABASE_ANON_KEY: ${{ secrets.NEXT_PUBLIC_SUPABASE_ANON_KEY || 'dummy-key' }}
NEXT_PUBLIC_API_URL: ${{ secrets.NEXT_PUBLIC_API_URL || 'http://localhost:3003' }}
NEXT_PUBLIC_APP_URL: ${{ secrets.NEXT_PUBLIC_APP_URL || 'http://localhost:3000' }}
# ===========================================================================
# STAGE 3 — PRODUCTION DEPLOYMENT (deploy gate inlined — no cross-job output)
# ===========================================================================
# The deploy decision lives directly on this job's `if`. `app_deploy` already
# encodes `app_pipeline && !no_deploy`, so `(build): back front no-deploy` →
# app_deploy=false → no deploy. `always()` lets the `if` evaluate even when an
# upstream is skipped; `needs.build.result == 'success'` is the real fail-fast
# gate. (Inlined from a former deploy-gate job whose constant output did not
# propagate reliably through `needs.deploy-gate.outputs` under `always()`.)
# The job-level `…-deploy` group only serializes a SECOND run's deploy behind
# this one. It does NOT exempt the job from the workflow-level `…-quality`
# group, so a later push still cancels an in-flight roll — the pre-existing,
# accepted blue-green exposure, same residual as `uptime-kuma-deploy`.
# ===========================================================================
deploy:
name: 🚀 Deploy Production
runs-on: prod.docs.plus
# 45, not 30: replicas are now retired serially and the worker's stop grace
# is 150s, so a five-service roll plus a five-service rollback no longer
# fits 30m. A mid-deploy SIGTERM here is worse than a slow deploy.
timeout-minutes: 45
needs: [triage, build]
concurrency:
group: ${{ github.workflow }}-deploy
cancel-in-progress: false
if: |
always() &&
needs.build.result == 'success' &&
((github.event_name == 'push' && needs.triage.outputs.app_deploy == 'true') ||
(github.event_name == 'workflow_dispatch' && inputs.force_deploy))
environment:
name: production
url: https://docs.plus
permissions:
contents: read
steps:
- name: 📦 Checkout Code
uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1
with:
fetch-depth: 1
- name: 🔐 Prepare Environment
run: |
# deploy_service is written to disk rather than defined inline because
# the rollback step runs in its own shell and must take the identical
# route — a second copy of this function is a drift waiting to drain
# the fleet. Written first so a later failure here can still source it.
cat > "${RUNNER_TEMP}/deploy-service.sh" <<'DEPLOY_SVC'
# Rolls one compose service onto the new image without ever dropping
# below its current backend count. Returns non-zero (and retires
# nothing) if the new replicas do not come up healthy.
deploy_service() {
local SERVICE="$1" TARGET="$2"
local -a COMPOSE=(docker compose -f "${COMPOSE_FILE}" --env-file "${ENV_FILE}")
local -a OLD_IDS=() NEW_IDS=()
local id name
# Project-scoped on purpose: these ids are stopped and removed below,
# and `docker ps --filter com.docker.compose.service=` also matches an
# identically-named service in any other compose project on the host.
mapfile -t OLD_IDS < <("${COMPOSE[@]}" ps -q "${SERVICE}")
local SCALE_UP=$(( ${#OLD_IDS[@]} + TARGET ))
# Clamp to one spare generation. A rollback over a half-failed forward
# deploy sees TARGET stale replicas already running and would otherwise
# ask for 3x the footprint on a host that is already unhealthy. If the
# clamp bites, no new container appears and the NEW_IDS check below
# fails loudly instead of the host running out of memory quietly.
local MAX_SCALE=$(( TARGET * 2 ))
if [ "${SCALE_UP}" -gt "${MAX_SCALE}" ]; then
echo "::warning::${SERVICE} already runs ${#OLD_IDS[@]} replicas (target ${TARGET}); capping scale at ${MAX_SCALE}. Retire strays before redeploying."
SCALE_UP=${MAX_SCALE}
fi
echo ""
echo "📦 Deploying ${SERVICE} (running: ${#OLD_IDS[@]}, target: ${TARGET})..."
# --no-recreate is load-bearing. DEPLOY_TAG changes the service config
# hash, so a plain `up --scale` recreates the replicas that are
# currently serving and takes the fleet to zero before the new ones bind.
echo "⬆️ Starting ${TARGET} new replicas alongside the old (scale ${SCALE_UP})..."
"${COMPOSE[@]}" up -d --no-deps --no-recreate --scale "${SERVICE}=${SCALE_UP}" "${SERVICE}"
for id in $("${COMPOSE[@]}" ps -q "${SERVICE}"); do
case " ${OLD_IDS[*]} " in
*" ${id} "*) ;;
*) NEW_IDS+=("${id}") ;;
esac
done
if [ "${#NEW_IDS[@]}" -ne "${TARGET}" ]; then
echo "::error::Expected ${TARGET} new ${SERVICE} containers, found ${#NEW_IDS[@]}. Retiring nothing."
"${COMPOSE[@]}" ps "${SERVICE}"
return 1
fi
# Health is checked per new container id. A count would pass on the
# first tick — the old replicas are already healthy and, unlike
# before, they stay healthy for the whole wait.
echo "⏳ Waiting for ${#NEW_IDS[@]} new ${SERVICE} replicas to report healthy..."
local READY=0 i
for i in {1..60}; do
READY=1
for id in "${NEW_IDS[@]}"; do
[ "$(docker inspect -f '{{.State.Health.Status}}' "${id}" 2>/dev/null)" = healthy ] || READY=0
done
[ "${READY}" -eq 1 ] && break
[ $((i % 10)) -eq 0 ] && echo " ... attempt ${i}/60"
sleep 2
done
if [ "${READY}" -ne 1 ]; then
echo "::error::New ${SERVICE} replicas never reported healthy after 120s. Old replicas left running — no traffic was drained."
"${COMPOSE[@]}" ps "${SERVICE}"
return 1
fi
echo "✅ ${#NEW_IDS[@]} new ${SERVICE} replicas healthy"
# One at a time, so the service never loses more than one backend.
# A bare `docker stop` honours the container's StopTimeout, which
# compose sets from stop_grace_period — 150s for the worker (its
# jobs need it), 30s elsewhere. A flat --time would SIGKILL the worker.
for id in "${OLD_IDS[@]}"; do
name=$(docker inspect -f '{{.Name}}' "${id}" 2>/dev/null || echo "${id}")
echo "🛑 Retiring ${name}..."
docker stop "${id}" >/dev/null || true
docker rm "${id}" >/dev/null || true
sleep 2
done
}
DEPLOY_SVC
# Compose `--env-file` is the single source of truth. We do NOT also
# `set -a; source` it elsewhere — that path was double-loading and
# leaking vars to subshells unintentionally.
cp "${ENV_SOURCE}" "${ENV_FILE}"
echo "DEPLOY_TAG=${DEPLOY_TAG}" >> "${ENV_FILE}"
# Error capture must never silently regress: both DSNs feed GlitchTip.
# Require a real first character — dotenv strips quotes, so KEY="" or
# whitespace-only values are empty at runtime.
for k in GLITCHTIP_DSN NEXT_PUBLIC_GLITCHTIP_DSN; do
if ! grep -Eq "^${k}=[\"']?[^\"'[:space:]]" "${ENV_FILE}"; then
echo "::error::${k} is missing or empty in ${ENV_SOURCE}. Set it to the GlitchTip project DSN (https://glitchtip.docs.plus) before deploying."
exit 1
fi
done
# Stash the previous successful tag (if any) for the rollback step.
mkdir -p "${DEPLOY_STATE_DIR}"
if [ -f "${LAST_GOOD_TAG_FILE}" ]; then
PREVIOUS_TAG=$(cat "${LAST_GOOD_TAG_FILE}")
echo "PREVIOUS_TAG=${PREVIOUS_TAG}" >> "$GITHUB_ENV"
echo "ℹ️ Previous good tag: ${PREVIOUS_TAG}"
else
echo "PREVIOUS_TAG=" >> "$GITHUB_ENV"
echo "ℹ️ No previous good tag stashed (first deploy or fresh state dir)"
fi
echo "✅ Environment ready"
- name: 💾 Pre-deploy disk guard
run: |
echo "📊 Disk before prune:"
df -h / | tail -1
# Free space proactively. Without this, --no-cache builds can fill
# the root volume between deploys and silently OOM/ENOSPC the build
# step (job ends in <2min with no error). Runs before build, not after.
docker image prune -af --filter "until=24h" 2>/dev/null || true
docker builder prune -af --filter "until=24h" 2>/dev/null || true
# Hard guard: refuse to build when <10 GB free. Fail loud here
# rather than fail silently mid-build.
AVAIL_KB=$(df --output=avail / | tail -1)
AVAIL_GB=$((AVAIL_KB / 1024 / 1024))
echo "📊 Disk after prune: ${AVAIL_GB} GB free"
if [ "${AVAIL_GB}" -lt 10 ]; then
echo "::error::Less than 10 GB free on /. Aborting deploy. SSH to host and run 'docker system prune -af --volumes'."
df -h /
docker system df
exit 1
fi
- name: 📂 Verify build context (monorepo root)
run: |
if [ ! -d packages/email-templates ]; then
echo "::error::packages/email-templates missing. Build context must be repo root (context: .). Check checkout includes the workspace."
exit 1
fi
if ! grep -q 'email-templates' apps/hocuspocus.server/docker/Dockerfile.bun; then
echo "::error::apps/hocuspocus.server/docker/Dockerfile.bun must COPY packages/email-templates."
exit 1
fi
if ! grep -q 'email-templates' apps/webapp/docker/Dockerfile.bun; then
echo "::error::apps/webapp/docker/Dockerfile.bun must COPY packages/email-templates."
exit 1
fi
echo "✅ Build context OK (repo root, email-templates present)"
- name: 🏗️ Build Docker Images
env:
DOCKER_BUILDKIT: '1'
COMPOSE_DOCKER_CLI_BUILD: '1'
run: |
echo "🔨 Building images with tag: ${DEPLOY_TAG}"
# hocuspocus-server and hocuspocus-worker share `docsplus-hocuspocus`;
# building both via compose with --no-cache duplicates context transfer
# and ties up the bake plan. Build via hocuspocus-server only; the
# worker reuses the resulting tag at `up` time.
#
# --no-cache: required as long as the prod entrypoint script changes
# are layered late in the Dockerfile and we don't yet have stable
# layer ordering. If/when entrypoint COPY moves to the last layer,
# we can drop --no-cache and gain ~5 min per deploy.
docker compose -f "${COMPOSE_FILE}" --env-file "${ENV_FILE}" \
build --no-cache rest-api hocuspocus-server
docker compose -f "${COMPOSE_FILE}" --env-file "${ENV_FILE}" \
build --parallel webapp admin-dashboard
echo "✅ Images built"
- name: 🔧 Ensure Infrastructure
run: |
# First step that mutates the prod host — everything before (env guard,
# disk guard, image build) leaves production untouched, and the failure
# notification keys off this marker to avoid a false rollback page.
echo "DEPLOY_STARTED=true" >> "$GITHUB_ENV"
echo "🔧 Ensuring infrastructure..."
docker network create docsplus-network 2>/dev/null || true
# No --no-recreate: neither service interpolates DEPLOY_TAG, so compose
# recreates them only when their compose config actually changed.
# --no-recreate made every edge/queue config change inert after boot.
docker compose -f "${COMPOSE_FILE}" --env-file "${ENV_FILE}" \
up -d traefik redis
# Force-start Traefik if somehow not running
if ! docker ps --filter "name=traefik" --filter "status=running" --format '{{.Names}}' | grep -q traefik; then
echo "⚠️ Traefik not running, starting..."
docker compose -f "${COMPOSE_FILE}" --env-file "${ENV_FILE}" up -d traefik
sleep 15
fi
# Wait for healthy
echo "⏳ Waiting for Traefik..."
for i in {1..30}; do
if docker ps --filter "name=traefik" --filter "health=healthy" --format '{{.Names}}' | grep -q traefik; then
echo "✅ Traefik healthy"
break
fi
[ "${i}" -eq 30 ] && echo "⚠️ Traefik health timeout, continuing..."
sleep 2
done
- name: 🗄️ Migrate database (one-shot gate)
run: |
# Single prod migration runner: fail-fast BEFORE any app replica scales
# up, so a bad migration aborts the deploy and the old version keeps
# serving. App replicas have RUN_MIGRATIONS=0 — this is the only path.
# `run --rm` runs once and propagates the migrate exit code.
echo "🗄️ Running database migrations (one-shot, fail-fast)..."
docker compose -f "${COMPOSE_FILE}" --env-file "${ENV_FILE}" \
run --rm --no-deps migrate
echo "✅ Migrations applied"
- name: 🚀 Deploy Services (rolling replace)
run: |
echo "🚀 Starting rolling deployment..."
source "${RUNNER_TEMP}/deploy-service.sh"
# Callees before callers, so a consumer always understands the newest
# shape before a producer emits it. Workers before servers for the job
# shape (compose declares 2 worker replicas — a scale of 1 silently
# halved the fleet on every deploy); hocuspocus-server before rest-api
# because REST forwards content applies to its internal :4003 endpoint.
deploy_service "hocuspocus-worker" 2
deploy_service "webapp" 2
deploy_service "hocuspocus-server" 2
deploy_service "rest-api" 2
deploy_service "admin-dashboard" 1
echo ""
echo "✅ All services deployed"
- name: 🩺 Verify Deployment
run: |
echo "🩺 Verifying deployment..."
sleep 10
# Infrastructure check
echo "📊 Infrastructure:"
for svc in traefik docsplus-redis; do
if docker ps --filter "name=${svc}" --filter "status=running" --format '{{.Names}}' | grep -q "${svc}"; then
echo " ✅ ${svc}: running"
else
echo " ❌ ${svc}: NOT running"
docker logs "${svc}" --tail 30 2>/dev/null || true
exit 1
fi
done
# Service running + healthy check
echo "📊 Services:"
for svc in webapp rest-api hocuspocus-server hocuspocus-worker admin-dashboard; do
RUNNING=$(docker ps --filter "label=com.docker.compose.service=${svc}" --filter "status=running" --format "{{.Names}}" | wc -l)
HEALTHY=$(docker ps --filter "label=com.docker.compose.service=${svc}" --filter "health=healthy" --format "{{.Names}}" | wc -l)
if [ "${RUNNING}" -gt 0 ]; then
echo " ✅ ${svc}: ${RUNNING} running, ${HEALTHY} healthy"
else
echo " ❌ ${svc}: NOT running"
exit 1
fi
done
# Internal smoke test — hit container health endpoints via the
# docker network, NOT via the public DNS+TLS stack. A transient
# ACME / Let's Encrypt hiccup must not trigger a false-fail rollback.
echo ""
echo "🔍 Internal smoke tests..."
smoke() {
local SVC="$1" PORT="$2" PATH_="$3"
if docker compose -f "${COMPOSE_FILE}" --env-file "${ENV_FILE}" exec -T "${SVC}" \
bun -e "fetch('http://localhost:${PORT}${PATH_}').then(r => r.ok ? process.exit(0) : process.exit(1)).catch(() => process.exit(1))"; then
echo " ✅ ${SVC} internal health"
else
echo " ❌ ${SVC} internal health"
return 1
fi
}
smoke webapp 3000 /api/health
smoke rest-api 4000 /health
smoke hocuspocus-server 4001 /health
smoke hocuspocus-worker 4002 /health
smoke admin-dashboard 3100 /api/health
# Public-URL probe is now informational only — does NOT fail the deploy.
# Real public-availability monitoring belongs in uptime-kuma, not here.
echo ""
echo "🌐 Public URL probe (informational):"
PUBLIC_CODE=$(curl -sf -o /dev/null -w "%{http_code}" --max-time 10 https://docs.plus/ 2>/dev/null || echo "000")
echo " https://docs.plus/ → ${PUBLIC_CODE}"
API_CODE=$(curl -sf -o /dev/null -w "%{http_code}" --max-time 10 https://prodback.docs.plus/api/health 2>/dev/null || echo "000")
echo " https://prodback.docs.plus/api/health → ${API_CODE}"
echo ""
echo "✅ Deployment verified"
- name: 📁 Sync compose files for break-glass
if: success()
run: |
# Replaces the previous "Sync Production Directory" step which re-`up`'d
# services from a different cwd — that re-up broke the blue-green
# guarantee. Now we only COPY the active compose + env files to a
# stable path so a human SSH'd in can run, e.g.:
# cd /opt/projects/prod.docs.plus/.deploy/current
# docker compose -f docker-compose.prod.yml --env-file .env.production ps
# without having to know the runner's _work directory.
mkdir -p "${DEPLOY_STATE_DIR}/current"
cp "${COMPOSE_FILE}" "${DEPLOY_STATE_DIR}/current/${COMPOSE_FILE}"
cp "${ENV_FILE}" "${DEPLOY_STATE_DIR}/current/${ENV_FILE}"
echo "✅ Synced compose+env to ${DEPLOY_STATE_DIR}/current/"
- name: 💾 Stash this tag as last-good
# Only on success — failure path is handled by the rollback step.
if: success()
run: |
# Persist current tag for the next deploy's rollback target.
mkdir -p "${DEPLOY_STATE_DIR}"
# Keep the previous one as last-good-tag.previous for one-step-back debugging.
if [ -f "${LAST_GOOD_TAG_FILE}" ]; then
cp "${LAST_GOOD_TAG_FILE}" "${LAST_GOOD_TAG_FILE}.previous"
fi
echo "${DEPLOY_TAG}" > "${LAST_GOOD_TAG_FILE}"
echo "✅ Stashed last-good-tag = ${DEPLOY_TAG}"
- name: 🧹 Cleanup
if: success()
continue-on-error: true # cleanup failure shouldn't fail an otherwise green deploy
run: |
docker image prune -f
docker image prune -f --filter "until=24h" 2>/dev/null || true
echo "✅ Cleanup complete"
- name: 📊 Summary
if: success()
run: |
echo "======================================"
echo "✅ DEPLOYMENT SUCCESSFUL"
echo "======================================"
echo "Tag: ${DEPLOY_TAG}"
echo "Previous tag: ${PREVIOUS_TAG:-<none>}"
echo ""
echo "Services:"
docker ps --format "table {{.Names}}\t{{.Status}}\t{{.Ports}}" | grep -E "(traefik|docsplus|webapp|rest-api|hocuspocus)" | head -15
echo ""
echo "URLs:"
echo " - https://docs.plus"
echo " - https://prodback.docs.plus"
echo "======================================"
- name: 📌 Grafana deploy annotation
if: success()
run: |
# Grafana publishes no host port (traefik-only); reach it via its
# docker-network IP from the host. Never fails the deploy.
OBS_ENV=/opt/projects/prod.docs.plus/.env.observability
GRAFANA_PASS=$(grep '^GRAFANA_ADMIN_PASSWORD=' "${OBS_ENV}" 2>/dev/null | cut -d= -f2- || true)
GRAFANA_IP=$(docker inspect -f '{{range .NetworkSettings.Networks}}{{.IPAddress}}{{end}}' docsplus-grafana 2>/dev/null || true)
if [ -z "${GRAFANA_PASS}" ] || [ -z "${GRAFANA_IP}" ]; then
echo "::warning::Grafana unreachable; skipping deploy annotation"; exit 0
fi
curl -sf --max-time 10 -u "admin:${GRAFANA_PASS}" \
-H 'Content-Type: application/json' \
-d "{\"text\":\"deploy ${DEPLOY_TAG}\",\"tags\":[\"deploy\"]}" \
"http://${GRAFANA_IP}:3000/api/annotations" >/dev/null \
&& echo "✅ Deploy annotation posted" \
|| echo "::warning::Grafana deploy annotation failed (non-fatal)"
- name: 🚨 Rollback on Failure
id: rollback
if: failure()
run: |
echo "⚠️ Deployment failed — attempting rollback..."
if [ -z "${PREVIOUS_TAG:-}" ]; then
echo "::warning::No PREVIOUS_TAG stashed — cannot auto-rollback."
echo "📊 Current state:"
docker ps --format "table {{.Names}}\t{{.Status}}" | head -15
exit 0
fi
echo "↩️ Rolling back to: ${PREVIOUS_TAG}"
# Multi-image precondition (A2): ALL service images for the previous
# tag must still exist locally. The cleanup step honors --filter
# until=24h, so within a 24h window this works reliably; outside
# that window we fail loudly rather than partially-rollback into a
# mixed-version cluster.
MISSING=()
for img in docsplus-webapp docsplus-rest-api docsplus-hocuspocus docsplus-admin; do
if ! docker image inspect "${img}:${PREVIOUS_TAG}" >/dev/null 2>&1; then
MISSING+=("${img}:${PREVIOUS_TAG}")
fi
done
if [ "${#MISSING[@]}" -gt 0 ]; then
echo "::error::Cannot auto-rollback. Missing images for previous tag:"
for img in "${MISSING[@]}"; do
echo " - ${img}"
done
echo "Manual recovery: bring traffic back via the existing healthy containers."
docker compose -f "${COMPOSE_FILE}" --env-file "${ENV_FILE}" up -d --no-recreate 2>/dev/null || true
exit 1
fi
# Override DEPLOY_TAG in the env file and re-deploy with previous images.
sed -i.bak "s|^DEPLOY_TAG=.*|DEPLOY_TAG=${PREVIOUS_TAG}|" "${ENV_FILE}"
rm -f "${ENV_FILE}.bak"
# Identical route to the forward deploy, by construction — a
# --force-recreate here restarts every service at once and takes the
# fleet to zero, which is the outage this path exists to end.
if [ ! -f "${RUNNER_TEMP}/deploy-service.sh" ]; then
echo "::error::deploy-service.sh missing — the run failed before environment prep."
exit 1
fi
source "${RUNNER_TEMP}/deploy-service.sh"
# Keep going past a failed service so the rest still get their old
# image back; the smoke tests below decide the step's exit code.
ROLLBACK_OK=1
deploy_service "hocuspocus-worker" 2 || ROLLBACK_OK=0
deploy_service "webapp" 2 || ROLLBACK_OK=0
deploy_service "hocuspocus-server" 2 || ROLLBACK_OK=0
deploy_service "rest-api" 2 || ROLLBACK_OK=0
deploy_service "admin-dashboard" 1 || ROLLBACK_OK=0
# Post-rollback verification (A3). Give containers a moment to bind
# ports + pass first healthcheck, then run the same internal smoke
# set the forward path runs. If rollback itself can't come healthy,
# we want the workflow to fail RED so the on-call sees it instead
# of a misleading "rollback complete" green check.
echo ""
echo "⏳ Waiting 30s for rolled-back containers to settle..."
sleep 30
smoke() {
local SVC="$1" PORT="$2" PATH_="$3"
if docker compose -f "${COMPOSE_FILE}" --env-file "${ENV_FILE}" exec -T "${SVC}" \
bun -e "fetch('http://localhost:${PORT}${PATH_}').then(r => r.ok ? process.exit(0) : process.exit(1)).catch(() => process.exit(1))"; then
echo " ✅ ${SVC} internal health (post-rollback)"
return 0
else
echo " ❌ ${SVC} internal health (post-rollback)"
return 1
fi
}
smoke webapp 3000 /api/health || ROLLBACK_OK=0
smoke rest-api 4000 /health || ROLLBACK_OK=0
smoke hocuspocus-server 4001 /health || ROLLBACK_OK=0
smoke hocuspocus-worker 4002 /health || ROLLBACK_OK=0
smoke admin-dashboard 3100 /api/health || ROLLBACK_OK=0
echo ""
echo "📊 Post-rollback state:"
docker ps --format "table {{.Names}}\t{{.Status}}" | head -15
if [ "${ROLLBACK_OK}" -ne 1 ]; then
echo "::error::Rollback to ${PREVIOUS_TAG} did not pass smoke tests. Manual intervention required."
exit 1
fi
echo "✅ Rollback to ${PREVIOUS_TAG} verified healthy"
- name: 📣 Notify deploy failure
if: failure()
env:
ROLLBACK_OUTCOME: ${{ steps.rollback.outcome }}
run: |
# Credentials live in the observability env file on this host. Never fatal.
OBS_ENV=/opt/projects/prod.docs.plus/.env.observability
if [ "${DEPLOY_STARTED:-}" != "true" ]; then
TEXT="⚠️ docs.plus deploy of ${DEPLOY_TAG} aborted before touching prod (env guard or build); production unchanged"
elif [ "${ROLLBACK_OUTCOME}" = "success" ] && [ -n "${PREVIOUS_TAG:-}" ]; then
TEXT="🚨 docs.plus deploy of ${DEPLOY_TAG} FAILED — rolled back to ${PREVIOUS_TAG}"
else
TEXT="🚨 docs.plus deploy of ${DEPLOY_TAG} FAILED and rollback FAILED — manual intervention required"
fi
bash scripts/ci/notify-telegram.sh "${OBS_ENV}" "${TEXT}"
# ===========================================================================
# STAGE 4 — OBSERVABILITY DEPLOY (ordered after app deploy)
# ===========================================================================
observability-deploy:
name: 🔭 Deploy Observability
needs: [triage]
# Independent monitoring infra — deploys whenever `observability` is in the set.
# Deliberately does NOT `needs: deploy`: GitHub does not reliably run a
# reusable-workflow call job under `always()` when a needed job (the app deploy)
# is skipped, so depending on `deploy` made `(build): observability` silently
# skip. The single prod runner serializes app vs observability deploys anyway,
# and monitoring should stay up regardless of an app-deploy outcome.
if: |
github.event_name == 'push' &&
needs.triage.outputs.has_observability == 'true'
uses: ./.github/workflows/observability.docs.plus.yml
with:
action: setup
# ===========================================================================
# STAGE 5 — UPTIME KUMA DEPLOY (trailing, ordered last)
# ===========================================================================
uptime-kuma-deploy:
name: 🔔 Deploy Uptime Kuma
runs-on: prod.docs.plus
timeout-minutes: 10
# Job-level group serializes a SECOND workflow run's uptime-kuma deploy behind the
# first (cross-run). It does NOT shield this job from the workflow-level `…-quality`
# group cancelling the parent run — that residual is the pre-existing, accepted
# blue-green exposure (Global Constraints "preserve, do not regress"); this plan does
# not widen it. Matches the app deploy's own `…-deploy` group semantics.
concurrency:
group: uptime-kuma-deploy
cancel-in-progress: false
needs: [triage, deploy, observability-deploy]
# Always LAST. Fail-fast: any app domain in the set requires deploy success, and
# any observability in the set requires observability success (a skip from an
# upstream failure is not success). Absent domains skip and don't block.
if: |
always() &&
github.event_name == 'push' &&
needs.triage.outputs.has_uptime_kuma == 'true' &&
(needs.triage.outputs.app_deploy != 'true' || needs.deploy.result == 'success') &&
(needs.triage.outputs.has_observability != 'true' || needs.observability-deploy.result == 'success')
permissions: { contents: read }
steps:
- name: 🚀 Deploy
run: |
UPTIME_KUMA_IMAGE='louislam/uptime-kuma:1@sha256:bb1bcecbc3e3ffb1cb0f8fc5f9c3cdaa78c1dfb56d98d64e06da13ebfc6dba0d'
docker network create docsplus-network 2>/dev/null || true
docker stop uptime-kuma 2>/dev/null || true
docker rm uptime-kuma 2>/dev/null || true
docker run -d --name uptime-kuma --network docsplus-network --restart unless-stopped \
-v uptime-kuma-data:/app/data \
--label "traefik.enable=true" \
--label "traefik.http.routers.uptime.rule=Host(\`status.docs.plus\`)" \
--label "traefik.http.routers.uptime.entrypoints=websecure" \
--label "traefik.http.routers.uptime.tls.certresolver=letsencrypt" \
--label "traefik.http.services.uptime.loadbalancer.server.port=3001" \
"${UPTIME_KUMA_IMAGE}"
# Health poll replaces blind sleep 15 (D6 cheap win).
for i in $(seq 1 20); do
if docker exec uptime-kuma wget -qO- http://localhost:3001 >/dev/null 2>&1; then echo "✅ uptime-kuma healthy"; break; fi
[ "$i" -eq 20 ] && echo "⚠️ uptime-kuma health timeout (continuing)"; sleep 2
done