Merge branch 'enterprise-3.9-release' into enterprise-3.9-backport-351-backup-restore-secret-scanning-encryption-keys

Author: chuckp22

Dockerfile

@@ -26,6 +26,7 @@ RUN apt-get update && apt-get install --no-install-recommends -y \
     libssl-dev \
     git \
     jq \
+    bc \
     curl \
     tar \
     gzip \
@@ -54,8 +55,9 @@ RUN apt-get update && apt-get install --no-install-recommends -y \
     git \
     openssh-client \
     jq \
+    bc \
     moreutils \
-    gawk \ 
+    gawk \
     ca-certificates \
     xxhash \
     && rm -rf /var/lib/apt/lists/*

bin/ghe-backup

@@ -46,12 +46,36 @@ export CALLING_SCRIPT="ghe-backup"
 
 # Setup progress tracking
 init-progress
+export PROGRESS_TOTAL=14 # Minimum number of steps in backup is 14
+echo "$PROGRESS_TOTAL" > /tmp/backup-utils-progress-total
 export PROGRESS_TYPE="Backup"
 echo "$PROGRESS_TYPE" > /tmp/backup-utils-progress-type
 export PROGRESS=0 # Used to track progress of backup
 echo "$PROGRESS" > /tmp/backup-utils-progress
-export PROGRESS_TOTAL=18 # Maximum number of steps in backup
 
+OPTIONAL_STEPS=0
+# Backup actions+mssql
+if ghe-ssh "$GHE_HOSTNAME" -- 'ghe-config --true app.actions.enabled'; then
+  OPTIONAL_STEPS=$((OPTIONAL_STEPS + 2))
+fi
+
+# Backup fsck
+if [ "$GHE_BACKUP_FSCK" = "yes" ]; then
+  OPTIONAL_STEPS=$((OPTIONAL_STEPS + 1))
+fi
+
+# Backup minio
+if ghe-ssh "$GHE_HOSTNAME" -- 'ghe-config --true app.minio.enabled'; then
+  OPTIONAL_STEPS=$((OPTIONAL_STEPS + 1))
+fi
+
+# Backup pages
+if [ "$GHE_BACKUP_PAGES" != "no" ]; then
+  OPTIONAL_STEPS=$((OPTIONAL_STEPS + 1))
+fi
+
+PROGRESS_TOTAL=$((OPTIONAL_STEPS + PROGRESS_TOTAL)) # Minimum number of steps in backup is 14
+echo "$PROGRESS_TOTAL" > /tmp/backup-utils-progress-total
 # Check to make sure moreutils parallel is installed and working properly
 ghe_parallel_check
 
@@ -269,6 +293,7 @@ echo \"$cmd_title\"
 ghe-backup-git-hooks || printf %s \"git-hooks \" >> \"$failures_file\"")
 
 if [ "$GHE_BACKUP_STRATEGY" = "rsync" ]; then
+  increment-progress-total-count 1
   cmd_title=$(log_info "Backing up Elasticsearch indices ...")
   commands+=("
   echo \"$cmd_title\"
@@ -303,6 +328,8 @@ if [ -z "$failures" ]; then
   ln -s "$GHE_SNAPSHOT_TIMESTAMP" "../current"
 
   ghe-prune-snapshots
+else 
+  log_info "Skipping pruning snapshots, since some backups failed..."
 fi
 
 END_TIME=$(date +%s)

bin/ghe-host-check

@@ -131,7 +131,7 @@ fi
 
 # backup-utils 2.13 onwards limits support to the current and previous two releases
 # of GitHub Enterprise Server.
-supported_minimum_version="3.6.0"
+supported_minimum_version="3.7.0"
 
 if [ "$(version $version)" -ge "$(version $supported_minimum_version)" ]; then
   supported=1

bin/ghe-restore

@@ -275,17 +275,14 @@ fi
 # taking into account the options passed to the script and the appliance configuration
 # calculate restore steps
 OPTIONAL_STEPS=0
-# Cluster restores add an additional step
-if $CLUSTER ; then
-  OPTIONAL_STEPS=$((OPTIONAL_STEPS + 1))
-fi
+
 # Restoring UUID
 if [ -s "$GHE_RESTORE_SNAPSHOT_PATH/uuid" ] && ! $CLUSTER; then
   OPTIONAL_STEPS=$((OPTIONAL_STEPS + 1))
 fi
-# Restoring Actions
+# Restoring Actions + MSSQL
 if ghe-ssh "$GHE_HOSTNAME" -- 'ghe-config --true app.actions.enabled'; then
-  OPTIONAL_STEPS=$((OPTIONAL_STEPS + 1))
+  OPTIONAL_STEPS=$((OPTIONAL_STEPS + 2))
 fi
 # Restoring minio
 if ghe-ssh "$GHE_HOSTNAME" -- 'ghe-config --true app.minio.enabled'; then
@@ -305,10 +302,16 @@ fi
 if ! $CLUSTER && $instance_configured; then
   OPTIONAL_STEPS=$((OPTIONAL_STEPS + 1))
 fi
-# Maximum restore steps
-export PROGRESS_TOTAL=$((OPTIONAL_STEPS + 6))
+# Restoring settings + restore-chat-integration + restore-packages
+if $RESTORE_SETTINGS; then
+  OPTIONAL_STEPS=$((OPTIONAL_STEPS + 3))
+fi
+
+# Minimum number of steps is 7
+export PROGRESS_TOTAL=$((OPTIONAL_STEPS + 7))
 
 init-progress
+echo "$PROGRESS_TOTAL" > /tmp/backup-utils-progress-total
 export PROGRESS_TYPE="Restore"
 echo "$PROGRESS_TYPE" > /tmp/backup-utils-progress-type
 export PROGRESS=0 # Used to track progress of restore
@@ -490,6 +493,7 @@ if is_external_database_target_or_snapshot && $SKIP_MYSQL; then
   log_info "Skipping MySQL restore."
 else
   log_info "Restoring MySQL database from ${backup_snapshot_strategy} backup snapshot on an appliance configured for ${appliance_strategy} backups ..."
+  increment-progress-total-count 2
   ghe-restore-mysql "$GHE_HOSTNAME" 1>&3
 fi
 
@@ -618,10 +622,10 @@ if $CLUSTER; then
     ghe-ssh "$GHE_HOSTNAME" -- "ghe-cluster-each -- /usr/local/share/enterprise/ghe-nomad-cleanup" 1>&3 2>&3
   fi
   ghe-ssh "$GHE_HOSTNAME" -- "ghe-cluster-config-apply" 1>&3 2>&3
-  bm_end "configure_cluster"
+  bm_end "$(basename $0) - configure cluster"
 elif $instance_configured; then
   log_info "Configuring appliance ..."
-  bm_start "configure_appliance"
+  bm_start "$(basename $0) - configure appliance"
   if [ "$GHE_VERSION_MAJOR" -eq "3" ]; then
     ghe-ssh "$GHE_HOSTNAME" -- "ghe-nomad-cleanup" 1>&3 2>&3
   elif [ "$GHE_VERSION_MAJOR" -eq "2" ] && [ "$GHE_VERSION_MINOR" -eq "22" ]; then

share/github-backup-utils/ghe-backup-config

@@ -712,6 +712,12 @@ init-progress() {
   rm -f /tmp/backup-utils-progress*
 }
 
+#increase total count of progress
+increment-progress-total-count() {
+  ((PROGRESS_TOTAL += $1))
+  echo "$PROGRESS_TOTAL" > /tmp/backup-utils-progress-total
+}
+
 
 
 

share/github-backup-utils/ghe-backup-mssql

@@ -47,8 +47,8 @@ if [ -z "$GHE_MSSQL_PRIMARY_HOST" ]; then
 fi
 
 tempdir=$(mktemp -d -t backup-utils-backup-XXXXXX)
-ssh_config_file_opt=
-opts=
+ssh_config_file_opt=()
+opts=()
 
 isHA="$(ghe-ssh "$GHE_HOSTNAME" -- "ghe-config cluster.ha" || true)"
 

share/github-backup-utils/ghe-backup-pages

@@ -63,6 +63,7 @@ if [ -d "$GHE_DATA_DIR/current/pages" ] && [ "$(ls -A $GHE_DATA_DIR/current/page
   link_dest="--link-dest=../../current/pages"
 fi
 
+count=0
 for hostname in $hostnames; do
   bm_start "$(basename $0) - $hostname"
   echo 1>&3
@@ -82,6 +83,7 @@ for hostname in $hostnames; do
     "$GHE_SNAPSHOT_DIR/pages" 1>&3
   log_rsync "END: pages rsync" 1>&3
   bm_end "$(basename $0) - $hostname"
+  count=$((count + 1))
 done
-
+increment-progress-total-count $count
 bm_end "$(basename $0)"

share/github-backup-utils/ghe-backup-repositories

@@ -144,6 +144,8 @@ bm_end "$(basename $0) - Processing routes"
 if [ -z "$(find "$tempdir" -maxdepth 1 -name '*.rsync')" ]; then
   log_warn "no routes found, skipping repositories backup ..."
   exit 0
+else
+  increment-progress-total-count 3
 fi
 
 # Transfer repository data from a GitHub instance to the current snapshot
@@ -377,7 +379,7 @@ if [ -z "$GHE_SKIP_ROUTE_VERIFICATION" ]; then
   (cd $backup_dir/ && find * -mindepth 5 -maxdepth 6 -type d -name \*.git | fix_paths_for_ghe_version | uniq | sort | uniq) > $tempdir/destination_routes
 
   git --no-pager diff --unified=0 --no-prefix -- $tempdir/source_routes $tempdir/destination_routes || echo "Warning: One or more repository networks and/or gists were not found on the source appliance. Please contact GitHub Enterprise Support for assistance."
-
+  increment-progress-total-count 1
   bm_end "$(basename $0) - Verifying Routes"
 fi
 

share/github-backup-utils/ghe-backup-settings

@@ -79,14 +79,11 @@ backup-secret "password pepper" "password-pepper" "secrets.github.user-password-
 backup-secret "kredz.credz HMAC key" "kredz-credz-hmac" "secrets.kredz.credz-hmac-secret"
 backup-secret "kredz.varz HMAC key" "kredz-varz-hmac" "secrets.kredz.varz-hmac-secret"
 
-# backup encryption keying material for GHES 3.7.0 onwards
+# backup encryption keying material and create backup value current encryption for GHES 3.7.0 onwards
+# this is for forwards compatibility with GHES 3.8.0 onwards
 if [ "$(version $GHE_REMOTE_VERSION)" -ge "$(version 3.7.0)" ]; then
   backup-secret "encrypted column encryption keying material" "encrypted-column-encryption-keying-material" "secrets.github.encrypted-column-keying-material"
-fi
-
-# backup current encryption key for GHES 3.8.0 onwards
-if [ "$(version $GHE_REMOTE_VERSION)" -ge "$(version 3.8.0)" ]; then
-  backup-secret "encrypted column current encryption key" "encrypted-column-current-encryption-key" "secrets.github.encrypted-column-current-encryption-key"
+  cat "$GHE_SNAPSHOT_DIR/encrypted-column-encryption-keying-material" | sed 's:.*;::' > "$GHE_SNAPSHOT_DIR/encrypted-column-current-encryption-key"
 fi
 
 backup-secret "secret scanning encrypted secrets current storage key" "secret-scanning-encrypted-secrets-current-storage-key" "secrets.secret-scanning.encrypted-secrets-current-storage-key"

share/github-backup-utils/ghe-backup-storage

@@ -113,6 +113,8 @@ bm_end "$(basename $0) - Processing routes"
 if [ -z "$(find "$tempdir" -maxdepth 1 -name '*.rsync')" ]; then
   log_warn "no routes found, skipping storage backup ..."
   exit 0
+else
+  increment-progress-total-count 2
 fi
 
 # rsync all the storage objects
@@ -149,6 +151,7 @@ if [ -z "$GHE_SKIP_ROUTE_VERIFICATION" ]; then
 
   git --no-pager diff --unified=0 --no-prefix -- $tempdir/source_routes $tempdir/destination_routes || echo "Warning: One or more storage objects were not found on the source appliance. Please contact GitHub Enterprise Support for assistance."
 
+  increment-progress-total-count 1
   bm_end "$(basename $0) - Verifying Routes"
 fi