Merge pull request #641 from github/release-2.22.0

Update backup-utils for changes in 2.22

Author: jianghao0718

backup.config-example

@@ -53,6 +53,15 @@ GHE_NUM_SNAPSHOTS=10
 # WARNING: do not enable this, only useful for debugging/development
 #GHE_BACKUP_FSCK=no
 
+# Cadence of MSSQL backups
+# <full>,<differential>,<transactionlog> all in minutes
+# e.g.
+#   - Full backup every week (10080 minutes)
+#   - Differential backup every day (1440 minutes)
+#   - Transactionlog backup every 15 minutes
+#
+GHE_MSSQL_BACKUP_CADENCE=10080,1440,15
+
 # If set to 'yes', ghe-backup jobs will run in parallel. Defaults to 'no'.
 #
 # WARNING: this feature is in beta.
@@ -77,3 +86,11 @@ GHE_NUM_SNAPSHOTS=10
 #
 # WARNING: this feature is in beta.
 #GHE_PARALLEL_MAX_LOAD=50
+
+# When running an external mysql database, run this script to trigger a MySQL backup
+# rather than attempting to backup via backup-utils directly.
+#EXTERNAL_DATABASE_BACKUP_SCRIPT="/bin/false"
+
+# When running an external mysql database, run this script to trigger a MySQL restore
+# rather than attempting to backup via backup-utils directly.
+#EXTERNAL_DATABASE_RESTORE_SCRIPT="/bin/false"

bin/ghe-backup

@@ -180,13 +180,16 @@ ghe-ssh "$GHE_HOSTNAME" -- 'ghe-export-ssh-host-keys' > ssh-host-keys.tar ||
 failures="$failures ssh-host-keys"
 bm_end "ghe-export-ssh-host-keys"
 
-if is_binary_backup_feature_on; then
-  echo "Backing up MySQL database using binary backup strategy ..."
-else
-  echo "Backing up MySQL database using logical backup strategy ..."
-fi
 ghe-backup-mysql || failures="$failures mysql"
 
+if ghe-ssh "$GHE_HOSTNAME" -- 'ghe-config --true app.actions.enabled'; then
+  echo "Backing up MSSQL databases ..."
+  ghe-backup-mssql 1>&3 || failures="$failures mssql"
+
+  echo "Backing up Actions data ..."
+  ghe-backup-actions 1>&3 || failures="$failures actions"
+fi
+
 commands=("
 echo \"Backing up Redis database ...\"
 ghe-backup-redis > redis.rdb || printf %s \"redis \" >> \"$failures_file\"")

bin/ghe-restore

@@ -1,5 +1,5 @@
 #!/usr/bin/env bash
-#/ Usage: ghe-restore [-fchv] [--version] [-s <snapshot-id>] [<host>]
+#/ Usage: ghe-restore [-cfhv] [--version] [--skip-mysql] [-s <snapshot-id>] [<host>]
 #/
 #/ Restores a GitHub instance from local backup snapshots.
 #/
@@ -9,40 +9,54 @@
 #/ <https://enterprise.github.com/help/articles/adding-an-ssh-key-for-shell-access>
 #/
 #/ OPTIONS:
-#/   -f | --force      Don't prompt for confirmation before restoring.
-#/   -c | --config     Restore appliance settings and license in addition to
-#/                     datastores. Settings are not restored by default to
-#/                     prevent overwriting different configuration on the
-#/                     restore host.
-#/   -v | --verbose    Enable verbose output.
-#/   -h | --help       Show this message.
-#/        --version    Display version information and exit.
-#/   -s <snapshot-id>  Restore from the snapshot with the given id. Available
-#/                     snapshots may be listed under the data directory.
-#/   <host>            The <host> is the hostname or IP of the GitHub Enterprise
-#/                     instance. The <host> may be omitted when the
-#/                     GHE_RESTORE_HOST config variable is set in backup.config.
-#/                     When a <host> argument is provided, it always overrides
-#/                     the configured restore host.
+#/   -c | --config       Restore appliance settings and license in addition to
+#/                       datastores. Settings are not restored by default to
+#/                       prevent overwriting different configuration on the
+#/                       restore host.
+#/   -f | --force        Don't prompt for confirmation before restoring.
+#/   -h | --help         Show this message.
+#/   -v | --verbose      Enable verbose output.
+#/        --skip-mysql   Skip MySQL restore steps. Only applicable to external databases.
+#/        --version      Display version information and exit.
+#/
+#/   -s <snapshot-id>    Restore from the snapshot with the given id. Available
+#/                       snapshots may be listed under the data directory.
+#/
+#/   <host>              The <host> is the hostname or IP of the GitHub Enterprise
+#/                       instance. The <host> may be omitted when the
+#/                       GHE_RESTORE_HOST config variable is set in backup.config.
+#/                       When a <host> argument is provided, it always overrides
+#/                       the configured restore host.
 #/
 
 set -e
 
 # Parse arguments
-restore_settings=false
-force=false
+: ${RESTORE_SETTINGS:=false}
+export RESTORE_SETTINGS
+
+: ${FORCE:=false}
+export FORCE
+
+: ${SKIP_MYSQL:=false}
+export SKIP_MYSQL
+
 while true; do
   case "$1" in
+    --skip-mysql)
+      SKIP_MYSQL=true
+      shift
+      ;;
     -f|--force)
-      force=true
+      FORCE=true
       shift
       ;;
     -s)
       snapshot_id="$(basename "$2")"
       shift 2
       ;;
     -c|--config)
-      restore_settings=true
+      RESTORE_SETTINGS=true
       shift
       ;;
     -h|--help)
@@ -116,10 +130,10 @@ ghe_remote_version_required "$GHE_HOSTNAME"
 
 # Figure out if this instance has been configured or is entirely new.
 instance_configured=false
-if ghe-ssh "$GHE_HOSTNAME" -- "[ -f '$GHE_REMOTE_ROOT_DIR/etc/github/configured' ]"; then
+if is_instance_configured; then
   instance_configured=true
 else
-  restore_settings=true
+  RESTORE_SETTINGS=true
 fi
 
 # Figure out if we're restoring into cluster
@@ -137,6 +151,11 @@ if ! $CLUSTER && [ "$GHE_BACKUP_STRATEGY" = "cluster" ]; then
   exit 1
 fi
 
+# Ensure target appliance and restore snapshot are a compatible combination with respect to BYODB
+if ! ghe-restore-external-database-compatibility-check; then
+  exit 1
+fi
+
 # Figure out if this appliance is in a replication pair
 if ghe-ssh "$GHE_HOSTNAME" -- \
   "[ -f '$GHE_REMOTE_ROOT_DIR/etc/github/repl-state' ]"; then
@@ -149,28 +168,30 @@ fi
 # important data on the destination appliance that cannot be recovered. This is
 # mostly to prevent accidents where the backup host is given to restore instead
 # of a separate restore host since they're used in such close proximity.
-if $instance_configured && ! $force; then
+if $instance_configured && ! $FORCE; then
   echo
   echo "WARNING: All data on GitHub Enterprise appliance $hostname ($GHE_REMOTE_VERSION)"
   echo "         will be overwritten with data from snapshot ${GHE_RESTORE_SNAPSHOT}."
-  echo "Please verify that this is the correct restore host before continuing."
-  printf "Type 'yes' to continue: "
+  echo
 
-  while read -r response; do
-    case $response in
-      yes|Yes|YES)
-        break
-        ;;
-      '')
-        printf "Type 'yes' to continue: "
-        ;;
-      *)
-        echo "Restore aborted." 1>&2
-        exit 1
-        ;;
-    esac
-  done
+  if is_external_database_snapshot && $RESTORE_SETTINGS; then
+    echo "WARNING: This operation will also restore the external MySQL connection configuration,"
+    echo "         which may be dangerous if the GHES appliance the snapshot was taken from is still online."
+    echo
+  fi
+
+  prompt_for_confirmation "Please verify that this is the correct restore host before continuing."
+fi
+
+# Prompt to verify that restoring BYODB snapshot to unconfigured instance
+# will result in BYODB connection information being restored as well.
+if is_external_database_snapshot && ! $instance_configured && ! $FORCE; then
   echo
+  echo "WARNING: This operation will also restore the external MySQL connection configuration,"
+  echo "         which may be dangerous if the GHES appliance the snapshot was taken from is still online."
+  echo
+
+  prompt_for_confirmation "Please confirm this before continuing."
 fi
 
 # Log restore start message locally and in /var/log/syslog on remote instance
@@ -207,6 +228,14 @@ if $instance_configured; then
   fi
 fi
 
+# Make sure the GitHub appliance has Actions enabled if the snapshot contains Actions data.
+if [ -d "$GHE_RESTORE_SNAPSHOT_PATH/mssql" ] || [ -d "$GHE_RESTORE_SNAPSHOT_PATH/actions" ]; then
+  if ! ghe-ssh "$GHE_HOSTNAME" -- 'ghe-config --true app.actions.enabled'; then
+    echo "Error: $GHE_HOSTNAME must have GitHub Actions enabled before restoring since the snapshot contains Actions data. Aborting." 1>&2
+    exit 1
+  fi
+fi
+
 # Create benchmark file
 bm_init > /dev/null
 
@@ -236,7 +265,7 @@ fi
 
 # Restore settings and license if restoring to an unconfigured appliance or when
 # specified manually.
-if $restore_settings; then
+if $RESTORE_SETTINGS; then
   ghe-restore-settings "$GHE_HOSTNAME"
 fi
 
@@ -257,20 +286,39 @@ if [ -s "$GHE_RESTORE_SNAPSHOT_PATH/uuid" ] && ! $CLUSTER; then
   ghe-ssh "$GHE_HOSTNAME" -- "sudo rm -rf /data/user/consul/raft"
   fi
 
-if is_binary_backup_feature_on; then
-  appliance_strategy="binary"
+if is_external_database_snapshot; then
+   appliance_strategy="external"
+   backup_snapshot_strategy="external"
 else
-  appliance_strategy="logical"
+  if is_binary_backup_feature_on; then
+    appliance_strategy="binary"
+  else
+    appliance_strategy="logical"
+  fi
+
+  if is_binary_backup "$GHE_DATA_DIR/$GHE_RESTORE_SNAPSHOT"; then
+    backup_snapshot_strategy="binary"
+  else
+    backup_snapshot_strategy="logical"
+  fi
 fi
 
-if is_binary_backup "$GHE_DATA_DIR/$GHE_RESTORE_SNAPSHOT"; then
-  backup_snapshot_strategy="binary"
+if is_external_database_target_or_snapshot && $SKIP_MYSQL; then
+  echo "Skipping MySQL restore."
 else
-  backup_snapshot_strategy="logical"
+  echo "Restoring MySQL database from ${backup_snapshot_strategy} backup snapshot on an appliance configured for ${appliance_strategy} backups ..."
+  ghe-restore-mysql "$GHE_HOSTNAME" 1>&3
 fi
 
-echo "Restoring MySQL database from ${backup_snapshot_strategy} backup snapshot on an appliance configured for ${appliance_strategy} backups ..."
-ghe-restore-mysql "$GHE_HOSTNAME" 1>&3
+if ghe-ssh "$GHE_HOSTNAME" -- 'ghe-config --true app.actions.enabled'; then
+  echo "Restoring MSSQL databases ..."
+  ghe-restore-mssql "$GHE_HOSTNAME" 1>&3
+
+  echo "Restoring Actions data ..."
+  ghe-restore-actions "$GHE_HOSTNAME" 1>&3
+  echo "* WARNING: Every self-hosted Actions runner that communicates with the restored GHES server must be restarted or reconfigured in order to continue functioning."
+  echo "           See https://docs.github.com/en/actions/hosting-your-own-runners/adding-self-hosted-runners for more details on how to reconfigure self-hosted Actions runners."
+fi
 
 commands=("
 echo \"Restoring Redis database ...\"

docs/backup-snapshot-file-structure.md

@@ -32,6 +32,7 @@ most recent successful snapshot:
           |- enterprise.ghl
           |- es-scan-complete
           |- manage-password
+          |- mssql
           |- mysql.sql.gz
           |- redis.rdb
           |- settings.json
@@ -44,3 +45,20 @@ most recent successful snapshot:
 
 Note: the `GHE_DATA_DIR` variable set in `backup.config` can be used to change
 the disk location where snapshots are written.
+
+## MS SQL Server backup structure
+Actions service uses MS SQL Server as backend data store. Each snapshot includes a suite of backup files for MS SQL Server database(s).
+
+To save time in backup, a three-level backup strategy is implemented. Based on the `GHE_MSSQL_BACKUP_CADENCE` setting, at each snapshot, either a (**F**)ull backup, a (**D**)ifferential or a (**T**)ransaction log backup is taken.
+
+As a result, a suite always contains following for each database: a full backup, possibly a differential backup and at least one transaction log backup. Their relationship with timeline is demonstrated below:
+```
+M---8:00--16:00---T---8:00--16:00---W... (timeline)
+
+F-----------------F-----------------F... (full backup)
+#-----D-----D-----#-----D-----D-----#... (differential backup)
+T--T--T--T--T--T--T--T--T--T--T--T--T... (transaction log backup)
+```
+To save disk space, at each snapshot, hard links are created to point to previous backup files. Only newly-created backup files are transferred from appliance to backup host. When a new full/differential backup is created, they become the new source for hard links and new base line for transaction log backups, for subsequent snapshots.
+
+During restore, a suite of backup files are restored in the sequence of full -> differential -> chronological transaction log.

docs/usage.md

@@ -16,6 +16,8 @@ You can supply your own configuration file or use the example configuration file
 
 An example configuration file with documentation on possible settings can found in [backup.config-example](../backup.config-example).
 
+There are a number of command line options that can also be passed to the `ghe-restore` command. Of particular note, if you use an external MySQL service but are restoring from a snapshot prior to enabling this, or vice versa, you must migrate the MySQL data outside of the context of backup-utils first, then pass the `--skip-mysql` flag to `ghe-restore`.
+
 ## Example backup and restore usage
 
 The following assumes that `GHE_HOSTNAME` is set to "github.example.com" in

share/github-backup-utils/ghe-backup-actions

@@ -0,0 +1,50 @@
+#!/usr/bin/env bash
+#/ Usage: ghe-backup-actions
+#/ Take an online, incremental snapshot of all Actions data (excluding
+#/ what is stored in MSSQL)
+#/
+#/ Note: This command typically isn't called directly. It's invoked by
+#/ ghe-backup.
+set -e
+
+# Bring in the backup configuration
+# shellcheck source=share/github-backup-utils/ghe-backup-config
+. "$( dirname "${BASH_SOURCE[0]}" )/ghe-backup-config"
+
+bm_start "$(basename $0)"
+
+# Set up remote host and root backup snapshot directory based on config
+port=$(ssh_port_part "$GHE_HOSTNAME")
+host=$(ssh_host_part "$GHE_HOSTNAME")
+backup_dir="$GHE_SNAPSHOT_DIR/actions"
+
+# Verify rsync is available.
+if ! rsync --version 1>/dev/null 2>&1; then
+  echo "Error: rsync not found." 1>&2
+  exit 1
+fi
+
+# Perform a host-check and establish GHE_REMOTE_XXX variables.
+ghe_remote_version_required "$host"
+
+# Make sure root backup dir exists if this is the first run
+mkdir -p "$backup_dir"
+
+# If we have a previous increment and it is not empty, avoid transferring existing files via rsync's
+# --link-dest support. This also decreases physical space usage considerably.
+if [ -d "$GHE_DATA_DIR/current/actions" ] && [ "$(ls -A $GHE_DATA_DIR/current/actions)" ]; then
+  link_dest="--link-dest=$GHE_DATA_DIR/current/actions"
+fi
+
+# Transfer all Actions data from the user data directory using rsync.
+ghe_verbose "* Transferring Actions files from $host ..."
+
+ghe-rsync -avz \
+  -e "ghe-ssh -p $port" \
+  --rsync-path='sudo -u actions rsync' \
+  --exclude "mutexes" --exclude "dumps" \
+  $link_dest \
+  "$host:$GHE_REMOTE_DATA_USER_DIR/actions/" \
+  "$GHE_SNAPSHOT_DIR/actions" 1>&3
+
+bm_end "$(basename $0)"