Merge branch 'master' into djdefi-expand-matrix

Author: djdefi

.github/workflows/lint.yml

@@ -1,7 +1,7 @@
 name: Lint Code Base
 
 on:
-  push:
+  pull_request:
     branches-ignore:
       - 'master'
 

.github/workflows/main.yml

@@ -1,6 +1,6 @@
 name: Test and build
 
-on: [push]
+on: [pull_request]
 
 jobs:
   build:
@@ -14,7 +14,7 @@ jobs:
       run: |
         sudo apt-get update -y
         sudo apt-get install -y devscripts debhelper moreutils fakeroot jq pigz
-        wget "https://storage.googleapis.com/shellcheck/shellcheck-v0.7.0.linux.x86_64.tar.xz"
+        wget "https://github.com/koalaman/shellcheck/releases/download/v0.7.0/shellcheck-v0.7.0.linux.x86_64.tar.xz"
         tar --xz -xvf "shellcheck-v0.7.0.linux.x86_64.tar.xz"
         sudo cp shellcheck-v0.7.0/shellcheck /usr/bin/shellcheck
       if: matrix.os != 'macos-latest'

Dockerfile.alpine

@@ -0,0 +1,22 @@
+FROM alpine:latest
+
+RUN apk --update --no-cache add \
+    tar \
+    rsync \
+    ca-certificates \
+    openssh \
+    git \
+    bash \
+    gawk \
+    procps
+
+WORKDIR /backup-utils
+ADD https://github.com/github/backup-utils/archive/stable.tar.gz /
+RUN tar xzvf /stable.tar.gz --strip-components=1 -C /backup-utils && \
+    mv /usr/bin/gawk /usr/bin/awk && \
+    rm -r /stable.tar.gz
+
+RUN chmod +x /backup-utils/share/github-backup-utils/ghe-docker-init
+
+ENTRYPOINT ["/backup-utils/share/github-backup-utils/ghe-docker-init"]
+CMD ["ghe-host-check"]

RELEASING.md

@@ -8,6 +8,8 @@ There is no need to align Backup Utilities patch releases with GitHub Enterprise
 
 When making a `.0` release, you will need to specify the minimum supported version of GitHub Enterprise Server that that release supports.
 
+Only repo administrator is allowed to run the release script, otherwise it will fail.
+
 ## Pre-release Actions
 
 Prior to making a release,

backup.config-example

@@ -53,6 +53,15 @@ GHE_NUM_SNAPSHOTS=10
 # WARNING: do not enable this, only useful for debugging/development
 #GHE_BACKUP_FSCK=no
 
+# Cadence of MSSQL backups
+# <full>,<differential>,<transactionlog> all in minutes
+# e.g.
+#   - Full backup every week (10080 minutes)
+#   - Differential backup every day (1440 minutes)
+#   - Transactionlog backup every 15 minutes
+#
+GHE_MSSQL_BACKUP_CADENCE=10080,1440,15
+
 # If set to 'yes', ghe-backup jobs will run in parallel. Defaults to 'no'.
 #
 # WARNING: this feature is in beta.

bin/ghe-backup

@@ -182,6 +182,14 @@ bm_end "ghe-export-ssh-host-keys"
 
 ghe-backup-mysql || failures="$failures mysql"
 
+if ghe-ssh "$GHE_HOSTNAME" -- 'ghe-config --true app.actions.enabled'; then
+  echo "Backing up MSSQL databases ..."
+  ghe-backup-mssql 1>&3 || failures="$failures mssql"
+
+  echo "Backing up Actions data ..."
+  ghe-backup-actions 1>&3 || failures="$failures actions"
+fi
+
 commands=("
 echo \"Backing up Redis database ...\"
 ghe-backup-redis > redis.rdb || printf %s \"redis \" >> \"$failures_file\"")

bin/ghe-host-check

@@ -113,7 +113,7 @@ fi
 
 # backup-utils 2.13 onwards limits support to the current and previous two releases
 # of GitHub Enterprise Server.
-supported_minimum_version="2.18.0"
+supported_minimum_version="2.19.0"
 
 if [ "$(version $version)" -ge "$(version $supported_minimum_version)" ]; then
   supported=1

bin/ghe-restore

@@ -1,5 +1,5 @@
 #!/usr/bin/env bash
-#/ Usage: ghe-restore [-fchv] [--version] [-s <snapshot-id>] [<host>]
+#/ Usage: ghe-restore [-cfhv] [--version] [--skip-mysql] [-s <snapshot-id>] [<host>]
 #/
 #/ Restores a GitHub instance from local backup snapshots.
 #/
@@ -9,40 +9,54 @@
 #/ <https://enterprise.github.com/help/articles/adding-an-ssh-key-for-shell-access>
 #/
 #/ OPTIONS:
-#/   -f | --force      Don't prompt for confirmation before restoring.
-#/   -c | --config     Restore appliance settings and license in addition to
-#/                     datastores. Settings are not restored by default to
-#/                     prevent overwriting different configuration on the
-#/                     restore host.
-#/   -v | --verbose    Enable verbose output.
-#/   -h | --help       Show this message.
-#/        --version    Display version information and exit.
-#/   -s <snapshot-id>  Restore from the snapshot with the given id. Available
-#/                     snapshots may be listed under the data directory.
-#/   <host>            The <host> is the hostname or IP of the GitHub Enterprise
-#/                     instance. The <host> may be omitted when the
-#/                     GHE_RESTORE_HOST config variable is set in backup.config.
-#/                     When a <host> argument is provided, it always overrides
-#/                     the configured restore host.
+#/   -c | --config       Restore appliance settings and license in addition to
+#/                       datastores. Settings are not restored by default to
+#/                       prevent overwriting different configuration on the
+#/                       restore host.
+#/   -f | --force        Don't prompt for confirmation before restoring.
+#/   -h | --help         Show this message.
+#/   -v | --verbose      Enable verbose output.
+#/        --skip-mysql   Skip MySQL restore steps. Only applicable to external databases.
+#/        --version      Display version information and exit.
+#/
+#/   -s <snapshot-id>    Restore from the snapshot with the given id. Available
+#/                       snapshots may be listed under the data directory.
+#/
+#/   <host>              The <host> is the hostname or IP of the GitHub Enterprise
+#/                       instance. The <host> may be omitted when the
+#/                       GHE_RESTORE_HOST config variable is set in backup.config.
+#/                       When a <host> argument is provided, it always overrides
+#/                       the configured restore host.
 #/
 
 set -e
 
 # Parse arguments
-restore_settings=false
-force=false
+: ${RESTORE_SETTINGS:=false}
+export RESTORE_SETTINGS
+
+: ${FORCE:=false}
+export FORCE
+
+: ${SKIP_MYSQL:=false}
+export SKIP_MYSQL
+
 while true; do
   case "$1" in
+    --skip-mysql)
+      SKIP_MYSQL=true
+      shift
+      ;;
     -f|--force)
-      force=true
+      FORCE=true
       shift
       ;;
     -s)
       snapshot_id="$(basename "$2")"
       shift 2
       ;;
     -c|--config)
-      restore_settings=true
+      RESTORE_SETTINGS=true
       shift
       ;;
     -h|--help)
@@ -116,10 +130,10 @@ ghe_remote_version_required "$GHE_HOSTNAME"
 
 # Figure out if this instance has been configured or is entirely new.
 instance_configured=false
-if ghe-ssh "$GHE_HOSTNAME" -- "[ -f '$GHE_REMOTE_ROOT_DIR/etc/github/configured' ]"; then
+if is_instance_configured; then
   instance_configured=true
 else
-  restore_settings=true
+  RESTORE_SETTINGS=true
 fi
 
 # Figure out if we're restoring into cluster
@@ -137,6 +151,11 @@ if ! $CLUSTER && [ "$GHE_BACKUP_STRATEGY" = "cluster" ]; then
   exit 1
 fi
 
+# Ensure target appliance and restore snapshot are a compatible combination with respect to BYODB
+if ! ghe-restore-external-database-compatibility-check; then
+  exit 1
+fi
+
 # Figure out if this appliance is in a replication pair
 if ghe-ssh "$GHE_HOSTNAME" -- \
   "[ -f '$GHE_REMOTE_ROOT_DIR/etc/github/repl-state' ]"; then
@@ -149,31 +168,35 @@ fi
 # important data on the destination appliance that cannot be recovered. This is
 # mostly to prevent accidents where the backup host is given to restore instead
 # of a separate restore host since they're used in such close proximity.
-if $instance_configured && ! $force; then
+if $instance_configured && ! $FORCE; then
   echo
   echo "WARNING: All data on GitHub Enterprise appliance $hostname ($GHE_REMOTE_VERSION)"
   echo "         will be overwritten with data from snapshot ${GHE_RESTORE_SNAPSHOT}."
-  echo "Please verify that this is the correct restore host before continuing."
-  printf "Type 'yes' to continue: "
+  echo
 
-  while read -r response; do
-    case $response in
-      yes|Yes|YES)
-        break
-        ;;
-      '')
-        printf "Type 'yes' to continue: "
-        ;;
-      *)
-        echo "Restore aborted." 1>&2
-        exit 1
-        ;;
-    esac
-  done
+  if is_external_database_snapshot && $RESTORE_SETTINGS; then
+    echo "WARNING: This operation will also restore the external MySQL connection configuration,"
+    echo "         which may be dangerous if the GHES appliance the snapshot was taken from is still online."
+    echo
+  fi
+
+  prompt_for_confirmation "Please verify that this is the correct restore host before continuing."
+fi
+
+# Prompt to verify that restoring BYODB snapshot to unconfigured instance
+# will result in BYODB connection information being restored as well.
+if is_external_database_snapshot && ! $instance_configured && ! $FORCE; then
+  echo
+  echo "WARNING: This operation will also restore the external MySQL connection configuration,"
+  echo "         which may be dangerous if the GHES appliance the snapshot was taken from is still online."
   echo
+
+  prompt_for_confirmation "Please confirm this before continuing."
 fi
 
 # Log restore start message locally and in /var/log/syslog on remote instance
+START_TIME=$(date +%s)
+echo 'Start time:' $START_TIME
 echo "Starting restore of $GHE_HOSTNAME with backup-utils v$BACKUP_UTILS_VERSION from snapshot $GHE_RESTORE_SNAPSHOT"
 ghe_remote_logger "Starting restore from $(hostname) with backup-utils v$BACKUP_UTILS_VERSION / snapshot $GHE_RESTORE_SNAPSHOT ..."
 
@@ -205,6 +228,14 @@ if $instance_configured; then
   fi
 fi
 
+# Make sure the GitHub appliance has Actions enabled if the snapshot contains Actions data.
+if [ -d "$GHE_RESTORE_SNAPSHOT_PATH/mssql" ] || [ -d "$GHE_RESTORE_SNAPSHOT_PATH/actions" ]; then
+  if ! ghe-ssh "$GHE_HOSTNAME" -- 'ghe-config --true app.actions.enabled'; then
+    echo "Error: $GHE_HOSTNAME must have GitHub Actions enabled before restoring since the snapshot contains Actions data. Aborting." 1>&2
+    exit 1
+  fi
+fi
+
 # Create benchmark file
 bm_init > /dev/null
 
@@ -234,7 +265,7 @@ fi
 
 # Restore settings and license if restoring to an unconfigured appliance or when
 # specified manually.
-if $restore_settings; then
+if $RESTORE_SETTINGS; then
   ghe-restore-settings "$GHE_HOSTNAME"
 fi
 
@@ -255,10 +286,10 @@ if [ -s "$GHE_RESTORE_SNAPSHOT_PATH/uuid" ] && ! $CLUSTER; then
   ghe-ssh "$GHE_HOSTNAME" -- "sudo rm -rf /data/user/consul/raft"
   fi
 
-if is_service_external 'mysql' "$GHE_RESTORE_SNAPSHOT_PATH/settings.json"; then
+if is_external_database_snapshot; then
    appliance_strategy="external"
    backup_snapshot_strategy="external"
-else 
+else
   if is_binary_backup_feature_on; then
     appliance_strategy="binary"
   else
@@ -272,8 +303,22 @@ else
   fi
 fi
 
-echo "Restoring MySQL database from ${backup_snapshot_strategy} backup snapshot on an appliance configured for ${appliance_strategy} backups ..."
-ghe-restore-mysql "$GHE_HOSTNAME" 1>&3
+if is_external_database_target_or_snapshot && $SKIP_MYSQL; then
+  echo "Skipping MySQL restore."
+else
+  echo "Restoring MySQL database from ${backup_snapshot_strategy} backup snapshot on an appliance configured for ${appliance_strategy} backups ..."
+  ghe-restore-mysql "$GHE_HOSTNAME" 1>&3
+fi
+
+if ghe-ssh "$GHE_HOSTNAME" -- 'ghe-config --true app.actions.enabled'; then
+  echo "Restoring MSSQL databases ..."
+  ghe-restore-mssql "$GHE_HOSTNAME" 1>&3
+
+  echo "Restoring Actions data ..."
+  ghe-restore-actions "$GHE_HOSTNAME" 1>&3
+  echo "* WARNING: Every self-hosted Actions runner that communicates with the restored GHES server must be restarted or reconfigured in order to continue functioning."
+  echo "           See https://docs.github.com/en/actions/hosting-your-own-runners/adding-self-hosted-runners for more details on how to reconfigure self-hosted Actions runners."
+fi
 
 commands=("
 echo \"Restoring Redis database ...\"
@@ -405,8 +450,13 @@ else
   ghe-ssh "$GHE_HOSTNAME" -- /bin/sh 1>&3
 fi
 
+END_TIME=$(date +%s)
+echo 'End time:' $END_TIME
+echo 'Runtime:' $(($END_TIME - $START_TIME)) 'seconds'
+
 echo "Restore of $GHE_HOSTNAME from snapshot $GHE_RESTORE_SNAPSHOT finished."
 
 if ! $instance_configured; then
   echo "To complete the restore process, please visit https://$hostname/setup/settings to review and save the appliance configuration."
 fi
+

debian/changelog

@@ -1,3 +1,15 @@
+github-backup-utils (2.21.0) UNRELEASED; urgency=medium
+
+  * Introduce option to skip restoring of audit logs #596
+  * Beta: Execute ghe-backup tasks in parallel #597
+  * Beta: Execute ghe-restore tasks in parallel #601
+  * Run repositories restore in parallel #603
+  * Fix mismatched `bm_start` and `bm_end` commands #607
+  * remove rsync restore method used by GHES versions prior to 2.13 #608
+  * Output MySQL backup strategy for clarity during backup and restore #610
+
+ -- Hao Jiang <[email protected]>  Tue, 09 Jun 2020 17:59:06 +0000
+
 github-backup-utils (2.19.5) UNRELEASED; urgency=medium
 
   * In legacy mode we should use ghe-import-mysql #581

docs/backup-snapshot-file-structure.md

@@ -32,6 +32,7 @@ most recent successful snapshot:
           |- enterprise.ghl
           |- es-scan-complete
           |- manage-password
+          |- mssql
           |- mysql.sql.gz
           |- redis.rdb
           |- settings.json
@@ -44,3 +45,20 @@ most recent successful snapshot:
 
 Note: the `GHE_DATA_DIR` variable set in `backup.config` can be used to change
 the disk location where snapshots are written.
+
+## MS SQL Server backup structure
+Actions service uses MS SQL Server as backend data store. Each snapshot includes a suite of backup files for MS SQL Server database(s).
+
+To save time in backup, a three-level backup strategy is implemented. Based on the `GHE_MSSQL_BACKUP_CADENCE` setting, at each snapshot, either a (**F**)ull backup, a (**D**)ifferential or a (**T**)ransaction log backup is taken.
+
+As a result, a suite always contains following for each database: a full backup, possibly a differential backup and at least one transaction log backup. Their relationship with timeline is demonstrated below:
+```
+M---8:00--16:00---T---8:00--16:00---W... (timeline)
+
+F-----------------F-----------------F... (full backup)
+#-----D-----D-----#-----D-----D-----#... (differential backup)
+T--T--T--T--T--T--T--T--T--T--T--T--T... (transaction log backup)
+```
+To save disk space, at each snapshot, hard links are created to point to previous backup files. Only newly-created backup files are transferred from appliance to backup host. When a new full/differential backup is created, they become the new source for hard links and new base line for transaction log backups, for subsequent snapshots.
+
+During restore, a suite of backup files are restored in the sequence of full -> differential -> chronological transaction log.