Prod DB backups: scripted + lifecycle + restore drill (#115)

borisolver · web-flow · commit 9e73b5ebd331 · 2026-02-11T08:34:03.000+01:00
* Prod DB backups: script + lifecycle + restore drill tooling

* Fix DB backup/restore scripts to avoid host process-list secret exposure

* Harden backup/restore docs to avoid passwords in process list
diff --git a/DEPLOYMENT.md b/DEPLOYMENT.md
@@ -151,6 +151,9 @@ PASS=$(grep MYSQL_APP_PASSWORD /home/deployer/.env | cut -d= -f2)
 
 **Never commit .env files or print passwords in logs.**
 
+Also avoid passing passwords on the command line (they can show up in `ps` output). Prefer piping
+secrets via stdin into a short-lived file *inside* the container when you need to run `mysql`.
+
 ## Quick Health Check
 
 ```bash
@@ -161,6 +164,39 @@ sudo docker ps --format "table {{.Names}}\t{{.Status}}\t{{.Ports}}"
 curl -s https://live.cleanapp.io/api/v3/reports/health | jq
 
 # Check database connections
-sudo docker exec cleanapp_db mysql -u server -p"$PASS" cleanapp \
-  -e "SELECT COUNT(*) as connections FROM information_schema.processlist;"
+PASS="$(gcloud secrets versions access latest --secret=MYSQL_APP_PASSWORD_PROD)"
+printf '%s' "$PASS" | sudo docker exec -i cleanapp_db sh -lc '
+  set -euo pipefail
+  pwfile="$(mktemp)"
+  cat >"$pwfile"
+  chmod 600 "$pwfile"
+  MYSQL_PWD="$(cat "$pwfile")" mysql -u server cleanapp -e "SELECT COUNT(*) as connections FROM information_schema.processlist;"
+  rm -f "$pwfile"
+'
+```
+
+## Database Backups (Prod)
+
+CleanApp stores full MySQL backups in GCS:
+- Prod bucket: `gs://cleanapp_mysql_backup_prod`
+- Current object key (versioned): `gs://cleanapp_mysql_backup_prod/current/cleanapp_all.sql.gz`
+- Weekly pins (kept ~30 weeks): `gs://cleanapp_mysql_backup_prod/weekly/<ISO_WEEK>/cleanapp_all.sql.gz`
+
+### Restore (One-Liner)
+
+Restore the current backup into the running prod DB container:
+
+```bash
+PASS="$(gcloud secrets versions access latest --secret=MYSQL_ROOT_PASSWORD_PROD)"
+printf '%s' "$PASS" | sudo docker exec -i cleanapp_db sh -lc 'cat > /tmp/.restore_pw && chmod 600 /tmp/.restore_pw' && \
+  gsutil cat gs://cleanapp_mysql_backup_prod/current/cleanapp_all.sql.gz | gunzip -c | sudo docker exec -i cleanapp_db sh -lc 'MYSQL_PWD="$(cat /tmp/.restore_pw)" exec mysql -uroot' && \
+  sudo docker exec cleanapp_db sh -lc 'rm -f /tmp/.restore_pw'
+```
+
+### Restore Drill (Recommended)
+
+To prove backups are restorable, run a restore drill into a scratch MySQL container on the prod VM:
+
+```bash
+HOST=deployer@34.122.15.16 ./platform_blueprint/ops/db_backup/restore_drill_prod_vm.sh
 ```
diff --git a/platform_blueprint/ops/db_backup/backup.sh b/platform_blueprint/ops/db_backup/backup.sh
@@ -0,0 +1,134 @@
+#!/usr/bin/env bash
+# CleanApp MySQL full backup -> GCS (prod/dev)
+# - Streams mysqldump -> gzip -> gsutil (no large local temp files)
+# - Writes metadata JSON alongside backup
+# - Weekly pin (Sundays UTC): copies current object to weekly/<ISO_WEEK>/
+set -euo pipefail
+
+ENV=""
+while [[ $# -gt 0 ]]; do
+  case "$1" in
+    -e|--env)
+      ENV="$2"; shift 2;;
+    *)
+      echo "Usage: $0 -e <dev|prod>" >&2
+      exit 2;;
+  esac
+done
+
+if [[ -z "${ENV}" ]]; then
+  echo "Usage: $0 -e <dev|prod>" >&2
+  exit 2
+fi
+
+case "${ENV}" in
+  dev|prod) ;;
+  *) echo "Invalid env: ${ENV} (expected dev|prod)" >&2; exit 2;;
+esac
+
+log() { echo "$(date -u +%Y-%m-%dT%H:%M:%SZ) $*"; }
+need_cmd() { command -v "$1" >/dev/null 2>&1 || { log "ERROR missing command: $1" >&2; exit 1; }; }
+
+need_cmd gcloud
+need_cmd gsutil
+need_cmd gzip
+
+if ! sudo -n true 2>/dev/null; then
+  log "ERROR sudo requires a password; cannot run docker exec" >&2
+  exit 1
+fi
+
+SECRET_SUFFIX="$(echo "${ENV}" | tr '[:lower:]' '[:upper:]')"
+BUCKET="gs://cleanapp_mysql_backup_${ENV}"
+CURRENT_KEY="${BUCKET}/current/cleanapp_all.sql.gz"
+CURRENT_META_KEY="${BUCKET}/current/cleanapp_all.metadata.json"
+
+log "INFO backup start env=${ENV} bucket=${BUCKET}"
+
+MYSQL_ROOT_PASSWORD="$(gcloud secrets versions access latest --secret="MYSQL_ROOT_PASSWORD_${SECRET_SUFFIX}" 2>/dev/null)" || {
+  log "ERROR failed to read MySQL root password from Secret Manager" >&2
+  exit 1
+}
+
+if ! sudo docker ps --format '{{.Names}}' | grep -qx cleanapp_db; then
+  log "ERROR cleanapp_db container not running" >&2
+  exit 1
+fi
+
+# Never pass secrets via `docker exec -e ...` (those end up visible in host `ps` output).
+# Instead, write the secret into a short-lived file inside the container and reference it
+# from inside the container process.
+pwfile="/tmp/cleanapp_mysql_backup_pw.$$.$RANDOM"
+cleanup_pwfile() {
+  sudo docker exec cleanapp_db sh -lc "rm -f '${pwfile}'" >/dev/null 2>&1 || true
+}
+trap cleanup_pwfile EXIT
+
+printf '%s' "${MYSQL_ROOT_PASSWORD}" | sudo docker exec -i cleanapp_db sh -lc \
+  "cat > '${pwfile}' && chmod 600 '${pwfile}'" >/dev/null
+
+if command -v pigz >/dev/null 2>&1; then
+  COMPRESS=(pigz -1)
+else
+  COMPRESS=(gzip -1)
+fi
+
+started_ts="$(date -u +%Y-%m-%dT%H:%M:%SZ)"
+started_epoch="$(date +%s)"
+
+log "INFO mysqldump stream start"
+sudo docker exec -i cleanapp_db sh -lc \
+  "MYSQL_PWD=\"\$(cat '${pwfile}')\" exec mysqldump -uroot \
+    --all-databases \
+    --single-transaction \
+    --quick \
+    --lock-tables=false \
+    --routines --events --triggers \
+    --hex-blob \
+    --set-gtid-purged=OFF" \
+  | "${COMPRESS[@]}" \
+  | gsutil -q -o GSUtil:parallel_composite_upload_threshold=150M cp - "${CURRENT_KEY}"
+
+finished_epoch="$(date +%s)"
+finished_ts="$(date -u +%Y-%m-%dT%H:%M:%SZ)"
+duration_s=$((finished_epoch - started_epoch))
+
+size_bytes="$(gsutil ls -l "${CURRENT_KEY}" | awk 'NR==1{print $1}')"
+size_bytes="${size_bytes:-0}"
+
+log "INFO capturing row counts"
+reports_count="$(sudo docker exec -i cleanapp_db sh -lc "MYSQL_PWD=\"\$(cat '${pwfile}')\" mysql -uroot -N -e \"SELECT COUNT(*) FROM cleanapp.reports\" 2>/dev/null" | tr -d '\r' | tail -n 1 || true)"
+analysis_count="$(sudo docker exec -i cleanapp_db sh -lc "MYSQL_PWD=\"\$(cat '${pwfile}')\" mysql -uroot -N -e \"SELECT COUNT(*) FROM cleanapp.report_analysis\" 2>/dev/null" | tr -d '\r' | tail -n 1 || true)"
+reports_count="${reports_count:-0}"
+analysis_count="${analysis_count:-0}"
+counts_json="{\"reports\":${reports_count},\"report_analysis\":${analysis_count}}"
+
+meta_tmp="/tmp/cleanapp_all.metadata.$$.$RANDOM.json"
+cat >"${meta_tmp}" <<META
+{
+  "env": "${ENV}",
+  "object": "${CURRENT_KEY}",
+  "started_utc": "${started_ts}",
+  "finished_utc": "${finished_ts}",
+  "duration_seconds": ${duration_s},
+  "size_bytes": ${size_bytes},
+  "row_counts": ${counts_json}
+}
+META
+
+gsutil -q cp "${meta_tmp}" "${CURRENT_META_KEY}"
+rm -f "${meta_tmp}" || true
+
+log "INFO backup uploaded object=${CURRENT_KEY} size_bytes=${size_bytes} duration_s=${duration_s}"
+
+if [[ "$(date -u +%u)" == "7" ]]; then
+  week="$(date -u +%G-W%V)"
+  weekly_key="${BUCKET}/weekly/${week}/cleanapp_all.sql.gz"
+  weekly_meta_key="${BUCKET}/weekly/${week}/cleanapp_all.metadata.json"
+  log "INFO weekly pin start week=${week}"
+  gsutil -q cp "${CURRENT_KEY}" "${weekly_key}"
+  gsutil -q cp "${CURRENT_META_KEY}" "${weekly_meta_key}"
+  log "INFO weekly pin done weekly_object=${weekly_key}"
+fi
+
+log "INFO backup complete env=${ENV}"
diff --git a/platform_blueprint/ops/db_backup/install_prod_backup.sh b/platform_blueprint/ops/db_backup/install_prod_backup.sh
@@ -0,0 +1,56 @@
+#!/usr/bin/env bash
+# Installs the prod DB backup script + cron schedule on the prod VM.
+#
+# Also sets bucket lifecycle:
+# - keep 7 noncurrent versions under current/ (daily versions)
+# - keep weekly/ objects for 210 days (~30 weeks)
+set -euo pipefail
+
+HOST="${HOST:-deployer@34.122.15.16}"
+ENV_NAME="${ENV_NAME:-prod}"
+
+if [[ "${ENV_NAME}" != "prod" && "${ENV_NAME}" != "dev" ]]; then
+  echo "ENV_NAME must be prod|dev" >&2
+  exit 2
+fi
+
+SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
+
+tmp_lifecycle="$(mktemp)"
+trap 'rm -f "$tmp_lifecycle"' EXIT
+
+cat >"$tmp_lifecycle" <<EOF
+{
+  "rule": [
+    {
+      "action": {"type": "Delete"},
+      "condition": {
+        "isLive": false,
+        "numNewerVersions": 7,
+        "matchesPrefix": ["current/"]
+      }
+    },
+    {
+      "action": {"type": "Delete"},
+      "condition": {
+        "age": 210,
+        "matchesPrefix": ["weekly/"]
+      }
+    }
+  ]
+}
+EOF
+
+echo "== install backup script on VM =="
+ssh "$HOST" "set -euo pipefail; mkdir -p /home/deployer/backups; true"
+scp "$SCRIPT_DIR/backup.sh" "$HOST:/home/deployer/backup.sh"
+ssh "$HOST" "set -euo pipefail; chmod +x /home/deployer/backup.sh"
+
+echo "== ensure cron (daily 03:30 UTC) =="
+ssh "$HOST" "set -euo pipefail; (crontab -l 2>/dev/null | grep -v '/home/deployer/backup.sh' || true) > /tmp/cron.new; echo '30 3 * * * /home/deployer/backup.sh -e ${ENV_NAME} >> /home/deployer/backups/backup.log 2>&1' >> /tmp/cron.new; crontab /tmp/cron.new; rm -f /tmp/cron.new; crontab -l"
+
+echo "== set bucket lifecycle =="
+bucket="gs://cleanapp_mysql_backup_${ENV_NAME}"
+gsutil lifecycle set "$tmp_lifecycle" "$bucket"
+gsutil lifecycle get "$bucket"
+
diff --git a/platform_blueprint/ops/db_backup/restore_drill_prod_vm.sh b/platform_blueprint/ops/db_backup/restore_drill_prod_vm.sh
@@ -0,0 +1,110 @@
+#!/usr/bin/env bash
+# Restore drill: restore the latest backup object into a scratch MySQL container and
+# compare row counts against the backup metadata.
+#
+# WARNING: This can take a long time and consume significant disk IO/space.
+set -euo pipefail
+
+HOST="${HOST:-deployer@34.122.15.16}"
+ENV_NAME="${ENV_NAME:-prod}"
+
+if [[ "${ENV_NAME}" != "prod" && "${ENV_NAME}" != "dev" ]]; then
+  echo "ENV_NAME must be prod|dev" >&2
+  exit 2
+fi
+
+bucket="gs://cleanapp_mysql_backup_${ENV_NAME}"
+obj_sql="${bucket}/current/cleanapp_all.sql.gz"
+obj_meta="${bucket}/current/cleanapp_all.metadata.json"
+
+ssh "$HOST" "ENV_NAME='${ENV_NAME}' OBJ_SQL='${obj_sql}' OBJ_META='${obj_meta}' bash -s" <<'REMOTE'
+set -euo pipefail
+
+need() { command -v "$1" >/dev/null 2>&1 || { echo "missing $1" >&2; exit 1; }; }
+need gsutil
+need sudo
+need docker
+need python3
+
+if ! sudo -n true 2>/dev/null; then
+  echo "sudo requires password on VM; cannot run restore drill" >&2
+  exit 1
+fi
+
+env_name="${ENV_NAME}"
+obj_sql="${OBJ_SQL}"
+obj_meta="${OBJ_META}"
+
+echo "== restore drill: env=${env_name} obj=${obj_sql} =="
+
+if ! gsutil -q stat "${obj_sql}"; then
+  echo "backup object missing: ${obj_sql}" >&2
+  exit 2
+fi
+if ! gsutil -q stat "${obj_meta}"; then
+  echo "metadata object missing: ${obj_meta}" >&2
+  exit 2
+fi
+
+expected="$(gsutil cat "${obj_meta}")"
+exp_reports="$(python3 -c 'import json,sys; d=json.load(sys.stdin); print(d.get("row_counts",{}).get("reports",0))' <<<"$expected")"
+exp_analysis="$(python3 -c 'import json,sys; d=json.load(sys.stdin); print(d.get("row_counts",{}).get("report_analysis",0))' <<<"$expected")"
+echo "expected counts: reports=${exp_reports} report_analysis=${exp_analysis}"
+
+ts="$(date -u +%Y%m%dT%H%M%SZ)"
+name="cleanapp_db_restore_drill_${ts}"
+vol="eko_mysql_restore_drill_${ts}"
+
+root_pw="$(python3 -c 'import secrets; print(secrets.token_hex(16))')"
+envfile="/tmp/${name}.env"
+
+echo "== create scratch mysql container =="
+sudo docker volume create "${vol}" >/dev/null
+umask 077
+printf 'MYSQL_ROOT_PASSWORD=%s\n' "${root_pw}" >"${envfile}"
+
+sudo docker run -d --name "${name}" --env-file "${envfile}" \
+  -p 127.0.0.1:3307:3306 \
+  -v "${vol}":/var/lib/mysql \
+  mysql:8.0 \
+  --default-authentication-plugin=mysql_native_password \
+  --innodb_flush_log_at_trx_commit=2 \
+  --sync_binlog=0 >/dev/null
+
+cleanup() {
+  echo "== cleanup =="
+  sudo docker rm -f "${name}" >/dev/null 2>&1 || true
+  sudo docker volume rm "${vol}" >/dev/null 2>&1 || true
+  rm -f "${envfile}" >/dev/null 2>&1 || true
+}
+trap cleanup EXIT
+
+echo "== wait for mysql ready =="
+for i in $(seq 1 120); do
+  # Use container env var to avoid putting secrets in host-visible process args.
+  if sudo docker exec "${name}" sh -lc 'mysql -uroot -p"$MYSQL_ROOT_PASSWORD" -e "SELECT 1" >/dev/null 2>&1'; then
+    break
+  fi
+  sleep 2
+  if [[ "$i" -eq 120 ]]; then
+    echo "mysql did not become ready" >&2
+    exit 1
+  fi
+done
+
+echo "== stream restore (this can take a long time) =="
+gsutil cat "${obj_sql}" | gunzip -c | sudo docker exec -i "${name}" sh -lc 'mysql -uroot -p"$MYSQL_ROOT_PASSWORD"'
+
+echo "== verify counts =="
+got_reports="$(sudo docker exec "${name}" sh -lc 'mysql -uroot -p"$MYSQL_ROOT_PASSWORD" -N -e \"SELECT COUNT(*) FROM cleanapp.reports\" 2>/dev/null' | tr -d '\r' | tail -n 1)"
+got_analysis="$(sudo docker exec "${name}" sh -lc 'mysql -uroot -p"$MYSQL_ROOT_PASSWORD" -N -e \"SELECT COUNT(*) FROM cleanapp.report_analysis\" 2>/dev/null' | tr -d '\r' | tail -n 1)"
+
+echo "restored counts: reports=${got_reports} report_analysis=${got_analysis}"
+
+if [[ "${got_reports}" != "${exp_reports}" || "${got_analysis}" != "${exp_analysis}" ]]; then
+  echo "ERROR restore drill mismatch vs metadata" >&2
+  exit 3
+fi
+
+echo "OK restore drill: counts match metadata"
+REMOTE
