Merge branch 'master' into djj/typo-fix

Author: davidjarzebowski

.github/workflows/main.yml

@@ -17,9 +17,9 @@ jobs:
       run: |
         sudo apt-get update -y
         sudo apt-get install -y devscripts debhelper moreutils fakeroot jq pigz help2man
-        wget "https://github.com/koalaman/shellcheck/releases/download/latest/shellcheck-latest.linux.x86_64.tar.xz"
-        tar --xz -xvf "shellcheck-latest.linux.x86_64.tar.xz"
-        sudo cp shellcheck-latest/shellcheck /usr/bin/shellcheck
+        wget "https://github.com/koalaman/shellcheck/releases/download/stable/shellcheck-stable.linux.x86_64.tar.xz"
+        tar --xz -xvf "shellcheck-stable.linux.x86_64.tar.xz"
+        sudo cp shellcheck-stable/shellcheck /usr/bin/shellcheck
       if: matrix.os != 'macos-latest'
     - name: Install Dependencies (macOS)
       run: |

bin/ghe-backup

@@ -54,38 +54,7 @@ export CALLING_SCRIPT="ghe-backup"
 # shellcheck source=share/github-backup-utils/ghe-backup-config
 . "$( dirname "${BASH_SOURCE[0]}" )/../share/github-backup-utils/ghe-backup-config"
 
-# Setup progress tracking
-init-progress
-export PROGRESS_TOTAL=14 # Minimum number of steps in backup is 14
-echo "$PROGRESS_TOTAL" > /tmp/backup-utils-progress-total
-export PROGRESS_TYPE="Backup"
-echo "$PROGRESS_TYPE" > /tmp/backup-utils-progress-type
-export PROGRESS=0 # Used to track progress of backup
-echo "$PROGRESS" > /tmp/backup-utils-progress
-
-OPTIONAL_STEPS=0
-# Backup actions+mssql
-if ghe-ssh "$GHE_HOSTNAME" -- 'ghe-config --true app.actions.enabled'; then
-  OPTIONAL_STEPS=$((OPTIONAL_STEPS + 2))
-fi
-
-# Backup fsck
-if [ "$GHE_BACKUP_FSCK" = "yes" ]; then
-  OPTIONAL_STEPS=$((OPTIONAL_STEPS + 1))
-fi
-
-# Backup minio
-if ghe-ssh "$GHE_HOSTNAME" -- 'ghe-config --true app.minio.enabled'; then
-  OPTIONAL_STEPS=$((OPTIONAL_STEPS + 1))
-fi
 
-# Backup pages
-if [ "$GHE_BACKUP_PAGES" != "no" ]; then
-  OPTIONAL_STEPS=$((OPTIONAL_STEPS + 1))
-fi
-
-PROGRESS_TOTAL=$((OPTIONAL_STEPS + PROGRESS_TOTAL)) # Minimum number of steps in backup is 14
-echo "$PROGRESS_TOTAL" > /tmp/backup-utils-progress-total
 # Check to make sure moreutils parallel is installed and working properly
 ghe_parallel_check
 
@@ -186,9 +155,44 @@ fi
 # Perform a host connection check and establish the remote appliance version.
 # The version is available in the GHE_REMOTE_VERSION variable and also written
 # to a version file in the snapshot directory itself.
+# ghe_remote_version_required should be run before any other instances of ghe-ssh
+# to ensure that there are no problems with host key verification.
 ghe_remote_version_required
 echo "$GHE_REMOTE_VERSION" > version
 
+# Setup progress tracking
+init-progress
+export PROGRESS_TOTAL=14 # Minimum number of steps in backup is 14
+echo "$PROGRESS_TOTAL" > /tmp/backup-utils-progress-total
+export PROGRESS_TYPE="Backup"
+echo "$PROGRESS_TYPE" > /tmp/backup-utils-progress-type
+export PROGRESS=0 # Used to track progress of backup
+echo "$PROGRESS" > /tmp/backup-utils-progress
+
+OPTIONAL_STEPS=0
+# Backup actions+mssql
+if ghe-ssh "$GHE_HOSTNAME" -- 'ghe-config --true app.actions.enabled'; then
+  OPTIONAL_STEPS=$((OPTIONAL_STEPS + 2))
+fi
+
+# Backup fsck
+if [ "$GHE_BACKUP_FSCK" = "yes" ]; then
+  OPTIONAL_STEPS=$((OPTIONAL_STEPS + 1))
+fi
+
+# Backup minio
+if ghe-ssh "$GHE_HOSTNAME" -- 'ghe-config --true app.minio.enabled'; then
+  OPTIONAL_STEPS=$((OPTIONAL_STEPS + 1))
+fi
+
+# Backup pages
+if [ "$GHE_BACKUP_PAGES" != "no" ]; then
+  OPTIONAL_STEPS=$((OPTIONAL_STEPS + 1))
+fi
+
+PROGRESS_TOTAL=$((OPTIONAL_STEPS + PROGRESS_TOTAL)) # Minimum number of steps in backup is 14
+echo "$PROGRESS_TOTAL" > /tmp/backup-utils-progress-total
+
 # check that incremental settings are valid if set
 is_inc=$(is_incremental_backup_feature_on)
 

bin/ghe-host-check

@@ -91,13 +91,19 @@ if ghe-ssh "$host" -- \
   CLUSTER=true
 fi
 
-# ensure all nodes in the cluster are running the same version
+# ensure all nodes in the cluster are online/reachable and running the same version
 if "$CLUSTER"; then
+  online_status=$(ghe-ssh "$host" ghe-cluster-host-check)
+  if [ "$online_status" != "Cluster is ready to configure." ]; then
+    echo "Error: Not all nodes are online! Please ensure cluster is in a healthy state before using backup-utils." 1>&2
+    exit 1
+  fi
+
   node_version_list=$(ghe-ssh "$host" ghe-cluster-each -- ghe-version)
   distinct_versions=$(echo "$node_version_list" | awk '{split($0, a, ":"); print a[2]}' | awk '{print $4}' | uniq | wc -l)
   if [ "$distinct_versions" -ne 1 ]; then
-    echo "$node_version_list" 1>&2
-    echo "Error: Not all nodes are running the same version! Please ensure all nodes are running the same version before using backup-utils." 1>&3
+    echo "Version mismatch: $node_version_list" 1>&2
+    echo "Error: Not all nodes are running the same version! Please ensure all nodes are running the same version before using backup-utils." 1>&2
     exit 1
   fi
 fi
@@ -184,15 +190,17 @@ SKIP_MSG
   echo "   - Recommended Disk requirement is $recommended_disk_req MB" 1>&2
   echo "" 1>&2
 
-  printf '### Data Transfer Sizes 
+  printf '### Estimated Data Transfer Sizes
+
  - repositories: %d MB
  - pages: %d MB
  - elasticsearch: %d MB
  - storage: %d MB
  - minio: %d MB
  - mysql: %d MB
  - actions: %d MB
- - mssql: %d MB\n' \
+ - mssql: %d MB
+\n' \
   "$repos_disk_size" "$pages_disk_size" "$es_disk_size" "$stor_disk_size" "$minio_disk_size" "$mysql_disk_size" "$actions_disk_size" "$mssql_disk_size" 1>&2
 
   if [[ $((available_space / (1024 * 1024))) -lt $min_disk_req ]]; then

bin/ghe-restore

@@ -469,13 +469,13 @@ fi
 
 # Always restore column encryption keys
 if [ "$(version $GHE_REMOTE_VERSION)" -ge "$(version 3.7.0)" ]; then
-  log_info "Always restore encrypted column encryption keys on GHES verions 3.7.0+"
+  log_info "Always restore encrypted column encryption keys on GHES versions 3.7.0+"
 fi
 ghe-restore-column-encryption-keys "$GHE_HOSTNAME"
 
 # Always restore secret scanning encryption keys
 if [ "$(version $GHE_REMOTE_VERSION)" -ge "$(version 3.8.0)" ]; then
-  log_info "Always restore secret scanning encryption keys on GHES verions 3.8.0+"
+  log_info "Always restore secret scanning encryption keys on GHES versions 3.8.0+"
   increment-progress-total-count 1
   ghe-restore-secret-scanning-encryption-keys "$GHE_HOSTNAME"
 fi

script/release

@@ -34,6 +34,9 @@ DEB_PKG_NAME = 'github-backup-utils'
 GH_BASE_BRANCH = ENV['GH_BASE_BRANCH'] || 'master' # TODO: should we even allow a default or require all params get set explicitly?
 GH_STABLE_BRANCH = ""
 
+# If PUBLISH is false, we leave the release in a draft state to be manually published later through the UI
+PUBLISH = ENV['PUBLISH'] == 'true' || false
+
 CHANGELOG_TMPL = '''<%= package_name %> (<%= package_version %>) UNRELEASED; urgency=medium
 
 <%- changes.each do |ch| -%>
@@ -480,15 +483,21 @@ if $PROGRAM_NAME == __FILE__
     attach_assets_to_release res['upload_url'], res['id'], ["#{base_dir}/dist/#{DEB_PKG_NAME}-v#{version}.tar.gz"]
     attach_assets_to_release res['upload_url'], res['id'], ["#{base_dir}/dist/#{DEB_PKG_NAME}_#{version}_all.deb"]
 
-    puts 'Publishing release...'
-    publish_release res['id']
+    if PUBLISH
+      puts 'Publishing release...'
+      publish_release res['id']
+    end
 
     puts 'Cleaning up...'
     clean_up version
 
     puts "Updating #{GH_STABLE_BRANCH} branch..."
     update_stable_branch
 
+    if !PUBLISH
+      puts 'Release left in a "Draft" state. Go to the https://github.com/github/backup-utils/releases and publish when ready.'
+    end
+
     puts 'Released!'
   rescue RuntimeError => e
     $stderr.puts "Error: #{e}"

share/github-backup-utils/ghe-backup-mssql

@@ -335,9 +335,12 @@ if [ -n "$backup_type" ]; then
   elif [ "$backup_type" = "transaction" ]; then
     backup_command='ghe-export-mssql -t'
   fi
-  
+
+  backup_failed=
+
   bm_start "$(basename "$0")"
-  ghe_ssh_mssql  -- "$backup_command"
+  # record if generating the backup failed, this will allow us to collect any backups that may have been produced, even if they are not complete they are better than nothing
+  ghe_ssh_mssql -- "$backup_command" || backup_failed='true'
   bm_end "$(basename "$0")"
 
   # Configure the backup cadence on the appliance, which is used for diagnostics
@@ -352,4 +355,9 @@ if [ -n "$backup_type" ]; then
     ghe_verbose "Transferring to backup host $b"
     ghe_ssh_mssql "sudo cat $appliance_dir/$b" > "$backup_dir"/"$b"
   done
+
+  if [ -n "$backup_failed" ]; then
+    log_error 'ghe-export-mssql failed to backup at least one mssql database' 1>&2
+    exit 1
+  fi
 fi

test/bin/ghe-cluster-host-check

@@ -0,0 +1,5 @@
+#!/usr/bin/env bash
+# Usage: ghe-cluster-host-check
+# Emulates a cluster reachability check
+set -e
+echo "Cluster is ready to configure."

test/test-shellcheck.sh

@@ -11,8 +11,8 @@ begin_test "shellcheck: reports no errors or warnings"
   set -e
   # We manually install the latest Shellcheck on Linux builds as other options
   # are too old.
-  if [ -x "$BASE_PATH/shellcheck-latest/shellcheck" ]; then
-    shellcheck() { "$BASE_PATH/shellcheck-latest/shellcheck" "$@"; }
+  if [ -x "$BASE_PATH/shellcheck-stable/shellcheck" ]; then
+    shellcheck() { "$BASE_PATH/shellcheck-stable/shellcheck" "$@"; }
   fi
 
   if ! type shellcheck 1>/dev/null 2>&1; then