Merge remote-tracking branch 'private/enterprise-3.9-release' into 3.9.2-patch

Author: dooleydevin

Dockerfile

@@ -26,6 +26,7 @@ RUN apt-get update && apt-get install --no-install-recommends -y \
     libssl-dev \
     git \
     jq \
+    bc \
     curl \
     tar \
     gzip \
@@ -54,8 +55,9 @@ RUN apt-get update && apt-get install --no-install-recommends -y \
     git \
     openssh-client \
     jq \
+    bc \
     moreutils \
-    gawk \ 
+    gawk \
     ca-certificates \
     xxhash \
     && rm -rf /var/lib/apt/lists/*

backup.config-example

@@ -16,6 +16,10 @@ GHE_DATA_DIR="data"
 # be available for the past N days ...
 GHE_NUM_SNAPSHOTS=10
 
+# If GHE_SKIP_CHECKS is set to true (or if --skip-checks is used with ghe-backup) then ghe-host-check
+# disk space validation and software version checks on the backup-host will be disabled.
+#GHE_SKIP_CHECKS=false
+
 # The hostname of the GitHub appliance to restore. If you've set up a separate
 # GitHub appliance to act as a standby for recovery, specify its IP or hostname
 # here. The host to restore to may also be specified directly when running

bin/ghe-backup

@@ -5,9 +5,10 @@
 #/ the MySQL database, instance settings, GitHub Pages data, etc.
 #/
 #/ OPTIONS:
-#/   -v | --verbose    Enable verbose output.
-#/   -h | --help       Show this message.
-#/        --version    Display version information.
+#/   -v | --verbose       Enable verbose output.
+#/   -h | --help          Show this message.
+#/        --version       Display version information.
+#/        --skip-checks   Skip storage/sw version checks
 #/
 
 set -e
@@ -27,6 +28,10 @@ while true; do
       export GHE_VERBOSE=true
       shift
       ;;
+    --skip-checks)
+      export GHE_SKIP_CHECKS=true
+      shift
+      ;;
     -*)
       echo "Error: invalid argument: '$1'" 1>&2
       exit 1
@@ -46,12 +51,36 @@ export CALLING_SCRIPT="ghe-backup"
 
 # Setup progress tracking
 init-progress
+export PROGRESS_TOTAL=14 # Minimum number of steps in backup is 14
+echo "$PROGRESS_TOTAL" > /tmp/backup-utils-progress-total
 export PROGRESS_TYPE="Backup"
 echo "$PROGRESS_TYPE" > /tmp/backup-utils-progress-type
 export PROGRESS=0 # Used to track progress of backup
 echo "$PROGRESS" > /tmp/backup-utils-progress
-export PROGRESS_TOTAL=18 # Maximum number of steps in backup
 
+OPTIONAL_STEPS=0
+# Backup actions+mssql
+if ghe-ssh "$GHE_HOSTNAME" -- 'ghe-config --true app.actions.enabled'; then
+  OPTIONAL_STEPS=$((OPTIONAL_STEPS + 2))
+fi
+
+# Backup fsck
+if [ "$GHE_BACKUP_FSCK" = "yes" ]; then
+  OPTIONAL_STEPS=$((OPTIONAL_STEPS + 1))
+fi
+
+# Backup minio
+if ghe-ssh "$GHE_HOSTNAME" -- 'ghe-config --true app.minio.enabled'; then
+  OPTIONAL_STEPS=$((OPTIONAL_STEPS + 1))
+fi
+
+# Backup pages
+if [ "$GHE_BACKUP_PAGES" != "no" ]; then
+  OPTIONAL_STEPS=$((OPTIONAL_STEPS + 1))
+fi
+
+PROGRESS_TOTAL=$((OPTIONAL_STEPS + PROGRESS_TOTAL)) # Minimum number of steps in backup is 14
+echo "$PROGRESS_TOTAL" > /tmp/backup-utils-progress-total
 # Check to make sure moreutils parallel is installed and working properly
 ghe_parallel_check
 
@@ -269,6 +298,7 @@ echo \"$cmd_title\"
 ghe-backup-git-hooks || printf %s \"git-hooks \" >> \"$failures_file\"")
 
 if [ "$GHE_BACKUP_STRATEGY" = "rsync" ]; then
+  increment-progress-total-count 1
   cmd_title=$(log_info "Backing up Elasticsearch indices ...")
   commands+=("
   echo \"$cmd_title\"
@@ -303,6 +333,8 @@ if [ -z "$failures" ]; then
   ln -s "$GHE_SNAPSHOT_TIMESTAMP" "../current"
 
   ghe-prune-snapshots
+else 
+  log_info "Skipping pruning snapshots, since some backups failed..."
 fi
 
 END_TIME=$(date +%s)

bin/ghe-host-check

@@ -144,7 +144,13 @@ if [ -z "$supported" ]; then
   exit 1
 fi
 
-if [[ "$CALLING_SCRIPT" == "ghe-backup" ]]; then
+if [[ "$CALLING_SCRIPT" == "ghe-backup" && "$GHE_SKIP_CHECKS" != "true" ]]; then
+  cat << SKIP_MSG 1>&2
+**You can disable the following storage & version checks by running ghe-backup with option "--skip-checks"
+OR updating GHE_SKIP_CHECKS to 'true' in your backup.config file.
+
+SKIP_MSG
+
   # Bring in the requirements file
   min_rsync=""
   min_openssh=""
@@ -184,6 +190,7 @@ minio: $minio_disk_size MB
 mysql: $mysql_disk_size MB
 actions: $actions_disk_size MB
 mssql: $mssql_disk_size MB
+
 DATA_TRANSFER_SIZE
 
   if [[ $((available_space / (1024 * 1024))) -lt $min_disk_req ]]; then

bin/ghe-restore

@@ -275,17 +275,14 @@ fi
 # taking into account the options passed to the script and the appliance configuration
 # calculate restore steps
 OPTIONAL_STEPS=0
-# Cluster restores add an additional step
-if $CLUSTER ; then
-  OPTIONAL_STEPS=$((OPTIONAL_STEPS + 1))
-fi
+
 # Restoring UUID
 if [ -s "$GHE_RESTORE_SNAPSHOT_PATH/uuid" ] && ! $CLUSTER; then
   OPTIONAL_STEPS=$((OPTIONAL_STEPS + 1))
 fi
-# Restoring Actions
+# Restoring Actions + MSSQL
 if ghe-ssh "$GHE_HOSTNAME" -- 'ghe-config --true app.actions.enabled'; then
-  OPTIONAL_STEPS=$((OPTIONAL_STEPS + 1))
+  OPTIONAL_STEPS=$((OPTIONAL_STEPS + 2))
 fi
 # Restoring minio
 if ghe-ssh "$GHE_HOSTNAME" -- 'ghe-config --true app.minio.enabled'; then
@@ -305,10 +302,16 @@ fi
 if ! $CLUSTER && $instance_configured; then
   OPTIONAL_STEPS=$((OPTIONAL_STEPS + 1))
 fi
-# Maximum restore steps
-export PROGRESS_TOTAL=$((OPTIONAL_STEPS + 6))
+# Restoring settings + restore-chat-integration + restore-packages
+if $RESTORE_SETTINGS; then
+  OPTIONAL_STEPS=$((OPTIONAL_STEPS + 3))
+fi
+
+# Minimum number of steps is 7
+export PROGRESS_TOTAL=$((OPTIONAL_STEPS + 7))
 
 init-progress
+echo "$PROGRESS_TOTAL" > /tmp/backup-utils-progress-total
 export PROGRESS_TYPE="Restore"
 echo "$PROGRESS_TYPE" > /tmp/backup-utils-progress-type
 export PROGRESS=0 # Used to track progress of restore
@@ -443,6 +446,12 @@ if [ "$(version $GHE_REMOTE_VERSION)" -ge "$(version 3.7.0)" ]; then
 fi
 ghe-restore-column-encryption-keys "$GHE_HOSTNAME"
 
+# Always restore secret scanning encryption keys
+if [ "$(version $GHE_REMOTE_VERSION)" -ge "$(version 3.8.0)" ]; then
+  log_info "Always restore secret scanning encryption keys on GHES verions 3.8.0+"
+  ghe-restore-secret-scanning-encryption-keys "$GHE_HOSTNAME"
+fi
+
 # Make sure mysql and elasticsearch are prep'd and running before restoring.
 # These services will not have been started on appliances that have not been
 # configured yet.
@@ -484,6 +493,7 @@ if is_external_database_target_or_snapshot && $SKIP_MYSQL; then
   log_info "Skipping MySQL restore."
 else
   log_info "Restoring MySQL database from ${backup_snapshot_strategy} backup snapshot on an appliance configured for ${appliance_strategy} backups ..."
+  increment-progress-total-count 2
   ghe-restore-mysql "$GHE_HOSTNAME" 1>&3
 fi
 

docs/usage.md

@@ -73,13 +73,18 @@ appliance at IP "5.5.5.5":
     Restore of 5.5.5.5:122 from snapshot 20180326T020444 finished.
     To complete the restore process, please visit https://5.5.5.5/setup/settings to review and save the appliance configuration.
 
-A different backup snapshot may be selected by passing the `-s` argument and the
-datestamp-named directory from the backup location.
+A different backup snapshot may be selected by passing the `-s` argument to `ghe-restore` and specifying the
+datestamp-named directory from the backup location as the value.
 
 The `ghe-backup` and `ghe-restore` commands also have a verbose output mode
 (`-v`) that lists files as they're being transferred. It's often useful to
 enable when output is logged to a file.
 
+Every time you execute `ghe-backup` we verify the storage and software setup of the host 
+you [installed][1] Backup Utilities on, to make sure our [requirements][2] for the host are 
+met. You can disable this check using the `--skip-checks` argument or by 
+adding `GHE_SKIP_CHECKS=true` to your configuration file.
+
 ### Restoring settings, TLS certificate, and license 
 
 When restoring to a new GitHub Enterprise Server instance, settings, certificate, and
@@ -107,3 +112,4 @@ GitHub Actions enabled, the following steps are required:
 Please refer to [GHES Documentation](https://docs.github.com/en/enterprise-server/admin/github-actions/advanced-configuration-and-troubleshooting/backing-up-and-restoring-github-enterprise-server-with-github-actions-enabled) for more details.
 
 [1]: https://github.com/github/backup-utils/blob/master/docs/getting-started.md
+[2]: requirements.md

script/release

@@ -34,6 +34,9 @@ DEB_PKG_NAME = 'github-backup-utils'
 GH_BASE_BRANCH = ENV['GH_BASE_BRANCH'] || 'master' # TODO: should we even allow a default or require all params get set explicitly?
 GH_STABLE_BRANCH = ""
 
+# If PUBLISH is false, we leave the release in a draft state to be manually published later through the UI
+PUBLISH = ENV['PUBLISH'] == 'true' || false
+
 CHANGELOG_TMPL = '''<%= package_name %> (<%= package_version %>) UNRELEASED; urgency=medium
 
 <%- changes.each do |ch| -%>
@@ -480,15 +483,21 @@ if $PROGRAM_NAME == __FILE__
     attach_assets_to_release res['upload_url'], res['id'], ["#{base_dir}/dist/#{DEB_PKG_NAME}-v#{version}.tar.gz"]
     attach_assets_to_release res['upload_url'], res['id'], ["#{base_dir}/dist/#{DEB_PKG_NAME}_#{version}_all.deb"]
 
-    puts 'Publishing release...'
-    publish_release res['id']
+    if PUBLISH
+      puts 'Publishing release...'
+      publish_release res['id']
+    end
 
     puts 'Cleaning up...'
     clean_up version
 
     puts "Updating #{GH_STABLE_BRANCH} branch..."
     update_stable_branch
 
+    if !PUBLISH
+      puts 'Release left in a "Draft" state. Go to the https://github.com/github/backup-utils/releases and publish when ready.'
+    end
+
     puts 'Released!'
   rescue RuntimeError => e
     $stderr.puts "Error: #{e}"

share/github-backup-utils/ghe-backup-config

@@ -35,7 +35,7 @@ if [ -n "$GHE_SHOW_VERSION" ]; then
 fi
 
 # Check for "--help|-h" in args or GHE_SHOW_HELP=true and show usage
-# shellcheck disable=SC2120 # the script name is always referenced
+# shellcheck disable=SC2120 # Our arguments are optional and not meant to be the owning script's
 print_usage() {
   grep '^#/' <"$0" | cut -c 4-
   exit "${1:-1}"
@@ -51,10 +51,6 @@ else
   done
 fi
 
-# Add the bin and share/github-backup-utils dirs to PATH
-PATH="$GHE_BACKUP_ROOT/bin:$GHE_BACKUP_ROOT/share/github-backup-utils:$PATH"
-# shellcheck source=share/github-backup-utils/bm.sh
-. "$GHE_BACKUP_ROOT/share/github-backup-utils/bm.sh"
 # Save off GHE_HOSTNAME from the environment since we want it to override the
 # backup.config value when set.
 GHE_HOSTNAME_PRESERVE="$GHE_HOSTNAME"
@@ -150,44 +146,12 @@ log_ssh(){
   log_level "ssh" "$1"
 }
 
-# Assume this script lives in share/github-backup-utils/ when setting the root
-GHE_BACKUP_ROOT="$(cd "$(dirname "${BASH_SOURCE[0]}")/../.." && pwd)"
-
-# Get the version from the version file.
-BACKUP_UTILS_VERSION="$(cat "$GHE_BACKUP_ROOT/share/github-backup-utils/version")"
-
-# If a version check was requested, show the current version and exit
-if [ -n "$GHE_SHOW_VERSION" ]; then
-  echo "GitHub backup-utils v$BACKUP_UTILS_VERSION"
-  exit 0
-fi
-
-# Check for "--help|-h" in args or GHE_SHOW_HELP=true and show usage
-# shellcheck disable=SC2120 # Our arguments are optional and not meant to be the owning script's
-print_usage() {
-  grep '^#/' <"$0" | cut -c 4-
-  exit "${1:-1}"
-}
-
-if [ -n "$GHE_SHOW_HELP" ]; then
-  print_usage
-else
-  for a in "$@"; do
-    if [ "$a" = "--help" ] || [ "$a" = "-h" ]; then
-      print_usage
-    fi
-  done
-fi
-
 # Add the bin and share/github-backup-utils dirs to PATH
 PATH="$GHE_BACKUP_ROOT/bin:$GHE_BACKUP_ROOT/share/github-backup-utils:$PATH"
 # shellcheck source=share/github-backup-utils/bm.sh
 . "$GHE_BACKUP_ROOT/share/github-backup-utils/bm.sh"
 # shellcheck source=share/github-backup-utils/track-progress
 . "$GHE_BACKUP_ROOT/share/github-backup-utils/track-progress"
-# Save off GHE_HOSTNAME from the environment since we want it to override the
-# backup.config value when set.
-GHE_HOSTNAME_PRESERVE="$GHE_HOSTNAME"
 
 # Source in the backup config file from the copy specified in the environment
 # first and then fall back to the backup-utils root, home directory and system.
@@ -712,6 +676,12 @@ init-progress() {
   rm -f /tmp/backup-utils-progress*
 }
 
+#increase total count of progress
+increment-progress-total-count() {
+  ((PROGRESS_TOTAL += $1))
+  echo "$PROGRESS_TOTAL" > /tmp/backup-utils-progress-total
+}
+
 
 
 

share/github-backup-utils/ghe-backup-es-rsync

@@ -4,6 +4,7 @@
 #/
 #/ Note: This command typically isn't called directly. It's invoked by
 #/ ghe-backup when the rsync strategy is used.
+# shellcheck disable=SC2086
 set -e
 
 # Bring in the backup configuration
@@ -54,15 +55,15 @@ log_rsync "END elasticsearch rsync" 1>&3
 # Set up a trap to re-enable flushing on exit and remove temp file
 cleanup () {
   ghe_verbose "* Enabling ES index flushing ..."
-  echo '{"index":{"translog.disable_flush":false}}' |
+  echo '{"index":{"translog.flush_threshold_size":"512MB"}}' |
   ghe-ssh "$host" -- curl -s -XPUT "localhost:9200/_settings" -d @- >/dev/null
 }
 trap 'cleanup' EXIT
 trap 'exit $?' INT # ^C always terminate
 
 # Disable ES flushing and force a flush right now
 ghe_verbose "* Disabling ES index flushing ..."
-echo '{"index":{"translog.disable_flush":true}}' |
+echo '{"index":{"translog.flush_threshold_size":"1PB"}}' |
 ghe-ssh "$host" -- curl -s -XPUT  "localhost:9200/_settings" -d @- >/dev/null
 ghe-ssh "$host" -- curl -s -XPOST "localhost:9200/_flush" >/dev/null
 

share/github-backup-utils/ghe-backup-pages

@@ -63,6 +63,7 @@ if [ -d "$GHE_DATA_DIR/current/pages" ] && [ "$(ls -A $GHE_DATA_DIR/current/page
   link_dest="--link-dest=../../current/pages"
 fi
 
+count=0
 for hostname in $hostnames; do
   bm_start "$(basename $0) - $hostname"
   echo 1>&3
@@ -82,6 +83,7 @@ for hostname in $hostnames; do
     "$GHE_SNAPSHOT_DIR/pages" 1>&3
   log_rsync "END: pages rsync" 1>&3
   bm_end "$(basename $0) - $hostname"
+  count=$((count + 1))
 done
-
+increment-progress-total-count $count
 bm_end "$(basename $0)"