Merge pull request #209 from github/mckern/minio

minio backup support

Author: mckern

bin/ghe-backup

@@ -190,6 +190,11 @@ if ghe-ssh "$GHE_HOSTNAME" -- 'ghe-config --true app.actions.enabled'; then
   ghe-backup-actions 1>&3 || failures="$failures actions"
 fi
 
+if ghe-ssh "$GHE_HOSTNAME" -- 'ghe-config --true app.minio.enabled'; then
+  echo "Backing up Minio data ..."
+  ghe-backup-minio 1>&3 || failures="$failures minio"
+fi
+
 commands=("
 echo \"Backing up Redis database ...\"
 ghe-backup-redis > redis.rdb || printf %s \"redis \" >> \"$failures_file\"")

bin/ghe-restore

@@ -336,6 +336,11 @@ if ghe-ssh "$GHE_HOSTNAME" -- 'ghe-config --true app.actions.enabled'; then
   echo "           See https://docs.github.com/en/actions/hosting-your-own-runners/adding-self-hosted-runners for more details on how to reconfigure self-hosted Actions runners."
 fi
 
+if ghe-ssh "$GHE_HOSTNAME" -- 'ghe-config --true app.minio.enabled'; then
+  echo "Restoring MinIO data ..."
+  ghe-restore-minio "$GHE_HOSTNAME" 1>&3
+fi
+
 commands=("
 echo \"Restoring Redis database ...\"
 ghe-ssh \"$GHE_HOSTNAME\" -- 'ghe-import-redis' < \"$GHE_RESTORE_SNAPSHOT_PATH/redis.rdb\" 1>&3")

share/github-backup-utils/ghe-backup-minio

@@ -0,0 +1,56 @@
+#!/usr/bin/env bash
+#/ Usage: ghe-backup-minio
+#/ Take an online, incremental snapshot of all minio data
+#/
+#/ Note: This command typically isn't called directly. It's invoked by
+#/ ghe-backup.
+set -e
+
+# Bring in the backup configuration
+# shellcheck source=share/github-backup-utils/ghe-backup-config
+. "$(dirname "${BASH_SOURCE[0]}")/ghe-backup-config"
+
+bm_start "$(basename "${0}")"
+
+# Set up remote host and root backup snapshot directory based on config
+port="$(ssh_port_part "${GHE_HOSTNAME}")"
+host="$(ssh_host_part "${GHE_HOSTNAME}")"
+backup_dir="${GHE_SNAPSHOT_DIR}/minio"
+
+# Verify rsync is available.
+if ! command -v rsync 1> /dev/null 2>&1; then
+  echo "Error: rsync not found." 1>&2
+  exit 1
+fi
+
+# Perform a host-check and establish GHE_REMOTE_XXX variables.
+ghe_remote_version_required "${host}"
+
+# Make sure root backup dir exists if this is the first run
+mkdir -p "${backup_dir}"
+
+# If we have a previous increment and it is not empty, avoid transferring existing files via rsync's
+# --link-dest support. This also decreases physical space usage considerably.
+# Hilariously, this HAS to stay unquoted when you call `rsync` further
+# down because when the shell interpolates this out, `rsync` will throw
+# an absolute fit if this variable is quoted. Surprise!
+if [[ -d "${GHE_DATA_DIR}/current/minio" ]] &&
+  [[ "$(ls -A "${GHE_DATA_DIR}/current/minio")" ]]; then
+    link_dest="--link-dest=${GHE_DATA_DIR}/current/minio"
+fi
+
+# Transfer all minio data from the user data directory using rsync.
+ghe_verbose "* Transferring minio files from ${host} ..."
+
+ghe-rsync \
+  --archive \
+  --verbose \
+  --compress \
+  --rsh="ghe-ssh -p ${port}" \
+  --rsync-path='sudo -u minio rsync' \
+  --exclude=".minio.sys" \
+  ${link_dest} \
+  "${host}:${GHE_REMOTE_DATA_USER_DIR}/minio/" \
+  "${GHE_SNAPSHOT_DIR}/minio" 1>&3
+
+bm_end "$(basename "${0}")"

share/github-backup-utils/ghe-restore-minio

@@ -0,0 +1,57 @@
+#!/usr/bin/env bash
+#/ Usage: ghe-restore-minio <host>
+#/ Restore additional minio files from an rsync snapshot.
+#/
+#/ Note: This script typically isn't called directly. It's invoked by the
+#/ ghe-restore command.
+set -e
+
+# Bring in the backup configuration
+# shellcheck source=share/github-backup-utils/ghe-backup-config
+. "$(dirname "${BASH_SOURCE[0]}")/ghe-backup-config"
+
+# Show usage and bail with no arguments
+[[ -z ${*} ]] && print_usage
+
+bm_start "$(basename "${0}")"
+
+# Grab host arg
+GHE_HOSTNAME="${1}"
+
+# The snapshot to restore should be set by the ghe-restore command but this lets
+# us run this script directly.
+: "${GHE_RESTORE_SNAPSHOT:=current}"
+
+# Path to snapshot dir we're restoring from
+GHE_RESTORE_SNAPSHOT_PATH="${GHE_DATA_DIR}/${GHE_RESTORE_SNAPSHOT}"
+
+port="$(ssh_port_part "${GHE_HOSTNAME}")"
+host="$(ssh_host_part "${GHE_HOSTNAME}")"
+
+# No need to restore anything, early exit
+if [ ! -d "${GHE_RESTORE_SNAPSHOT_PATH}/minio" ]; then
+  echo "Warning: minio backup missing. Skipping ..."
+  exit 0
+fi
+
+# Perform a host-check and establish GHE_REMOTE_XXX variables.
+ghe_remote_version_required "${host}"
+
+# Transfer all minio data from the snapshot to the user data directory using rsync.
+ghe_verbose "* Transferring minio files to ${host} ..."
+
+ghe-ssh -p "${port}" "${host}" -- sudo mkdir -p "${GHE_REMOTE_DATA_USER_DIR}/minio"
+ghe-ssh -p "${port}" "${host}" -- sudo chown -R minio:minio "${GHE_REMOTE_DATA_USER_DIR}/minio"
+
+ghe-rsync \
+  --verbose \
+  --archive \
+  --hard-links \
+  --relative \
+  --delete \
+  --rsh="ghe-ssh -p ${port}" \
+  --rsync-path='sudo -u minio rsync' \
+  "${GHE_RESTORE_SNAPSHOT_PATH}/minio/./" \
+  "${host}:${GHE_REMOTE_DATA_USER_DIR}/minio/" 1>&3
+
+bm_end "$(basename "${0}")"

test/test-ghe-backup.sh

@@ -356,15 +356,16 @@ begin_test "ghe-backup has default cadence configured"
 (
   set -e
   enable_actions
-  
+
   [ -n "$GHE_MSSQL_BACKUP_CADENCE" ]
 )
 end_test
 
 # Override backup cadence for testing purposes
 GHE_MSSQL_BACKUP_CADENCE=10,5,1
 export GHE_MSSQL_BACKUP_CADENCE
-setup_actions_test_data $GHE_REMOTE_DATA_USER_DIR
+setup_actions_test_data "$GHE_REMOTE_DATA_USER_DIR"
+setup_minio_test_data "$GHE_REMOTE_DATA_USER_DIR"
 
 begin_test "ghe-backup takes full backup on first run"
 (
@@ -373,6 +374,7 @@ begin_test "ghe-backup takes full backup on first run"
   # setup_mssql_backup_file uses "current"
   set -e
   enable_actions
+  enable_minio
 
   rm -rf "$GHE_REMOTE_DATA_USER_DIR"/mssql/backups/*
   rm -rf "$GHE_DATA_DIR"/current/mssql/*
@@ -386,6 +388,7 @@ begin_test "ghe-backup takes full backup upon expiration"
 (
   set -e
   enable_actions
+  enable_minio
   export REMOTE_DBS="full_mssql"
 
   setup_mssql_backup_file "full_mssql" 11 "bak"
@@ -400,6 +403,7 @@ begin_test "ghe-backup takes diff backup upon expiration"
 (
   set -e
   enable_actions
+  enable_minio
   export REMOTE_DBS="full_mssql"
 
   setup_mssql_backup_file "full_mssql" 7 "bak"

test/testlib.sh

@@ -319,6 +319,8 @@ setup_test_data () {
     echo "rsync" > "$loc/strategy"
     echo "$GHE_REMOTE_VERSION" >  "$loc/version"
   fi
+
+  setup_minio_test_data "$GHE_DATA_DIR"
 }
 
 setup_actions_test_data() {
@@ -350,6 +352,23 @@ cleanup_actions_test_data() {
   rm -rf "$loc/actions"
 }
 
+setup_minio_test_data() {
+  local loc=$1
+
+  mkdir -p "$loc/minio/"
+  cd "$loc/minio/"
+  bucket="packages"
+
+  mkdir -p "$bucket"
+  echo "an example blob" "$bucket/91dfa09f-1801-4e00-95ee-6b763d7da3e2"
+}
+
+cleanup_minio_test_data() {
+  local loc=$1
+
+  rm -rf "$loc/minio"
+}
+
 # A unified method to check everything backed up or restored during testing.
 # Everything tested here should pass regardless of whether we're testing a backup
 # or a restore.
@@ -374,6 +393,11 @@ verify_common_data() {
     diff -ru "$GHE_REMOTE_DATA_USER_DIR/mssql/backups" "$GHE_DATA_DIR/current/mssql"
   fi
 
+  if is_minio_enabled; then
+    # verify minio object storge backups were transferred
+    diff -ru "$GHE_REMOTE_DATA_USER_DIR/minio" "$GHE_DATA_DIR/minio"
+  fi
+
   # tests that differ for cluster and single node backups and restores
   if [ "$(cat $GHE_DATA_DIR/current/strategy)" = "rsync" ]; then
     # verify the UUID was transferred
@@ -528,6 +552,14 @@ is_actions_enabled() {
   ghe-ssh "$GHE_HOSTNAME" -- 'ghe-config --true app.actions.enabled'
 }
 
+enable_minio() {
+  ghe-ssh "$GHE_HOSTNAME" -- 'ghe-config app.minio.enabled true'
+}
+
+is_minio_enabled() {
+  ghe-ssh "$GHE_HOSTNAME" -- 'ghe-config --true app.minio.enabled'
+}
+
 setup_moreutils_parallel() {
   # CI servers may have moreutils parallel and GNU parallel installed.
   # We need moreutils parallel