:Merge remote-tracking branch 'private/enterprise-3.9-release' into 3.9.0-ga-release

Author: JoeFranks1993

bin/ghe-backup

@@ -37,11 +37,21 @@ while true; do
   esac
 done
 
+
 export CALLING_SCRIPT="ghe-backup"
+
 # Bring in the backup configuration
 # shellcheck source=share/github-backup-utils/ghe-backup-config
 . "$( dirname "${BASH_SOURCE[0]}" )/../share/github-backup-utils/ghe-backup-config"
 
+# Setup progress tracking
+init-progress
+export PROGRESS_TYPE="Backup"
+echo "$PROGRESS_TYPE" > /tmp/backup-utils-progress-type
+export PROGRESS=0 # Used to track progress of backup
+echo "$PROGRESS" > /tmp/backup-utils-progress
+export PROGRESS_TOTAL=18 # Maximum number of steps in backup
+
 # Check to make sure moreutils parallel is installed and working properly
 ghe_parallel_check
 
@@ -121,7 +131,7 @@ ghe_restore_check
 # Check to see if there is a running backup
 if [ -h ../in-progress ]; then
 
-  log_error "Error: detected a backup already in progress from a previous version of ghe-backup. \nIf there is no backup in progress anymore, please remove \nthe $GHE_DATA_DIR/in-progress file." 1>&2
+  log_error "Detected a backup already in progress from a previous version of ghe-backup. \nIf there is no backup in progress anymore, please remove \nthe $GHE_DATA_DIR/in-progress file." >&2
   exit 1
 fi
 
@@ -306,6 +316,7 @@ else
   steps="${failures// /, }"
   ghe_remote_logger "Completed backup from $(hostname) / snapshot $GHE_SNAPSHOT_TIMESTAMP with failures: ${steps}."
   log_error "Error: Snapshot incomplete. Some steps failed: ${steps}. "
+  ghe_backup_finished
   exit 1
 fi
 

bin/ghe-backup-progress

@@ -0,0 +1,56 @@
+#!/usr/bin/env bash
+#/ Usage: ghe-backup-progress [--once]
+#/ Tracks the completed steps of a backup or restore operation.
+#/
+#/ By default the progress is printed every continuously or until a key is pressed.
+#/ Use the --once option to print the current progress once and exit.
+#/
+#/ Options:
+#/  --once  Don't loop, just print the current progress once.
+#
+set -e
+
+while true; do
+    case "$1" in
+        -o|--once)
+            ONCE=1
+            shift
+            ;;
+        -h|--help)
+            export GHE_SHOW_HELP=true
+            shift
+            ;;
+        -*)
+            echo "Unknown option: $1" >&2
+            exit 1
+            ;;
+        *)
+            break
+            ;;
+    esac
+done
+
+check_for_progress_file() {
+    if [ ! -f /tmp/backup-utils-progress-info ]; then
+        echo "No progress file found. Has a backup or restore been started?"
+        exit 1
+    fi
+}
+
+if [ -n "$ONCE" ]; then
+    check_for_progress_file
+    cat /tmp/backup-utils-progress-info
+else
+    check_for_progress_file
+    clear
+    cat /tmp/backup-utils-progress-info
+    while true; do
+    if read -r -t 1 -n 1; then
+	clear
+        exit ;
+    else
+	clear
+	cat /tmp/backup-utils-progress-info
+    fi
+    done
+fi

bin/ghe-host-check

@@ -186,7 +186,7 @@ actions: $actions_disk_size MB
 mssql: $mssql_disk_size MB
 DATA_TRANSFER_SIZE
 
-  if [[ $available_space -lt $min_disk_req ]]; then
+  if [[ $((available_space / (1024 * 1024))) -lt $min_disk_req ]]; then
     echo "There is not enough disk space for the backup. Please allocate more space and continue." 1>&2
     exit 1
   fi

bin/ghe-restore

@@ -86,6 +86,8 @@ while true; do
   esac
 done
 
+
+
 start_cron () {
   log_info "Starting cron ..."
   if $CLUSTER; then
@@ -164,6 +166,10 @@ cleanup_cluster_nodes() {
 # shellcheck source=share/github-backup-utils/ghe-backup-config
 . "$( dirname "${BASH_SOURCE[0]}" )/../share/github-backup-utils/ghe-backup-config"
 
+
+
+
+
 # Check to make sure moreutils parallel is installed and working properly
 ghe_parallel_check
 
@@ -265,6 +271,48 @@ if is_external_database_snapshot && ! $instance_configured && ! $FORCE; then
 
   prompt_for_confirmation "Please confirm this before continuing."
 fi
+# Calculate the actual amounts of steps in the restore process
+# taking into account the options passed to the script and the appliance configuration
+# calculate restore steps
+OPTIONAL_STEPS=0
+# Cluster restores add an additional step
+if $CLUSTER ; then
+  OPTIONAL_STEPS=$((OPTIONAL_STEPS + 1))
+fi
+# Restoring UUID
+if [ -s "$GHE_RESTORE_SNAPSHOT_PATH/uuid" ] && ! $CLUSTER; then
+  OPTIONAL_STEPS=$((OPTIONAL_STEPS + 1))
+fi
+# Restoring Actions
+if ghe-ssh "$GHE_HOSTNAME" -- 'ghe-config --true app.actions.enabled'; then
+  OPTIONAL_STEPS=$((OPTIONAL_STEPS + 1))
+fi
+# Restoring minio
+if ghe-ssh "$GHE_HOSTNAME" -- 'ghe-config --true app.minio.enabled'; then
+  OPTIONAL_STEPS=$((OPTIONAL_STEPS + 1))
+fi
+# Restoring Elasticsearch
+if ! $CLUSTER && [ -d "$GHE_RESTORE_SNAPSHOT_PATH/elasticsearch" ]; then
+  OPTIONAL_STEPS=$((OPTIONAL_STEPS + 1))
+fi
+# Restoring audit log
+if $CLUSTER || [ "$(version "$GHE_REMOTE_VERSION")" -ge "$(version 2.12.9)" ]; then
+  if [[ "$GHE_RESTORE_SKIP_AUDIT_LOG" != "yes" ]]; then
+    OPTIONAL_STEPS=$((OPTIONAL_STEPS + 1))
+  fi
+fi
+# Replica cleanup
+if ! $CLUSTER && $instance_configured; then
+  OPTIONAL_STEPS=$((OPTIONAL_STEPS + 1))
+fi
+# Maximum restore steps
+export PROGRESS_TOTAL=$((OPTIONAL_STEPS + 6))
+
+init-progress
+export PROGRESS_TYPE="Restore"
+echo "$PROGRESS_TYPE" > /tmp/backup-utils-progress-type
+export PROGRESS=0 # Used to track progress of restore
+echo "$PROGRESS" > /tmp/backup-utils-progress
 
 # Log restore start message locally and in /var/log/syslog on remote instance
 START_TIME=$(date +%s)
@@ -343,10 +391,11 @@ log_warn "Warning: storing backup-utils version remotely failed."
 # Stop cron and timerd, as scheduled jobs may disrupt the restore process.
 log_info "Stopping cron and github-timerd ..."
 if $CLUSTER; then
+  bm_start "$(basename $0) - Stopping cron and github-timerd on cluster"
   if ! ghe-ssh "$GHE_HOSTNAME" -- "ghe-cluster-each -- sudo service cron stop"; then
     log_warn "Failed to stop cron on one or more nodes" 1>&3
   fi
-
+  bm_end "$(basename $0) - Stopping cron and github-timerd on cluster"
   if [ "$GHE_VERSION_MAJOR" -eq "3" ]; then
     if ghe-ssh "$GHE_HOSTNAME" -- "systemctl -q is-active nomad && nomad job status --short github-timerd &>/dev/null"; then
       if ! ghe-ssh "$GHE_HOSTNAME" -- "sudo nomad stop github-timerd 1>/dev/null"; then
@@ -358,11 +407,14 @@ if $CLUSTER; then
       log_warn "Failed to stop github-timerd on one or more nodes" 1>&3
     fi
   fi
+ 
 else
+  bm_start "$(basename $0) - Stopping cron and github-timerd"
+  echo "$(basename $0) - Stopping cron and github-timerd"
   if ! ghe-ssh "$GHE_HOSTNAME" -- "sudo service cron stop"; then
     log_warn "Failed to stop cron" 1>&3
   fi
-
+  bm_end "$(basename $0) - Stopping cron and github-timerd"
   if [ "$GHE_VERSION_MAJOR" -eq "3" ]; then
     if ghe-ssh "$GHE_HOSTNAME" -- "systemctl -q is-active nomad && nomad job status --short github-timerd &>/dev/null"; then
       if ! ghe-ssh "$GHE_HOSTNAME" -- "sudo nomad stop github-timerd 1>/dev/null"; then
@@ -374,9 +426,11 @@ else
       log_warn "Failed to stop github-timerd" 1>&3
     fi
   fi
+  
 fi
 CRON_RUNNING=false
 
+
 # Restore settings and license if restoring to an unconfigured appliance or when
 # specified manually.
 if $RESTORE_SETTINGS; then
@@ -394,11 +448,15 @@ fi
 # Restore UUID if present and not restoring to cluster.
 if [ -s "$GHE_RESTORE_SNAPSHOT_PATH/uuid" ] && ! $CLUSTER; then
   log_info "Restoring UUID ..."
+
+  bm_start "$(basename $0) - Restore UUID"
   ghe-ssh "$GHE_HOSTNAME" -- "sudo sponge '$GHE_REMOTE_DATA_USER_DIR/common/uuid' 2>/dev/null" <"$GHE_RESTORE_SNAPSHOT_PATH/uuid"
   ghe-ssh "$GHE_HOSTNAME" -- "sudo systemctl stop consul" || true
   ghe-ssh "$GHE_HOSTNAME" -- "sudo rm -rf /data/user/consul/raft"
+  bm_end "$(basename $0) - Restore UUID"
 fi
 
+
 if is_external_database_snapshot; then
    appliance_strategy="external"
    backup_snapshot_strategy="external"
@@ -448,7 +506,7 @@ fi
 cmd_title=$(log_info "Restoring Redis database ...")
 commands=("
 echo \"$cmd_title\"
-ghe-ssh \"$GHE_HOSTNAME\" -- 'ghe-import-redis' < \"$GHE_RESTORE_SNAPSHOT_PATH/redis.rdb\" 1>&3")
+ghe-restore-redis \"$GHE_HOSTNAME\" \"$GHE_RESTORE_SNAPSHOT_PATH\"")
 
 cmd_title=$(log_info "Restoring Git Repositories ...")
 commands+=("
@@ -470,7 +528,7 @@ fi
 cmd_title=$(log_info "Restoring SSH authorized keys ...")
 commands+=("
 echo \"$cmd_title\"
-ghe-ssh \"$GHE_HOSTNAME\" -- 'ghe-import-authorized-keys' < \"$GHE_RESTORE_SNAPSHOT_PATH/authorized-keys.json\" 1>&3")
+ghe-restore-ssh-keys \"$GHE_HOSTNAME\" \"$GHE_RESTORE_SNAPSHOT_PATH\"")
 
 cmd_title=$(log_info "Restoring storage data ...")
 commands+=("
@@ -517,14 +575,17 @@ if [ "$GHE_PARALLEL_ENABLED" = "yes" ]; then
 else
   log_info "Restoring data serially ..." 1>&3
   for c in "${commands[@]}"; do
+    . "$( dirname "${BASH_SOURCE[0]}" )/../share/github-backup-utils/bm.sh"
     eval "$c"
   done
 fi
 
 # Restart an already running memcached to reset the cache after restore
-log_info "Restarting memcached ..." 1>&3
+log_info  "Restarting memcached ..." 1>&3
+bm_start "$(basename $0) - Restarting memcached"
 echo "sudo restart -q memcached 2>/dev/null || true" |
   ghe-ssh "$GHE_HOSTNAME" -- /bin/sh
+bm_end "$(basename $0) - Restarting memcached"
 
 # Prevent GitHub Connect jobs running before we've had a chance to reset
 # the configuration by setting the last run date to now.
@@ -538,20 +599,24 @@ fi
 # config run to perform data migrations.
 if $CLUSTER; then
   log_info "Configuring cluster ..."
+  bm_start "$(basename $0) - configure cluster"
   if [ "$GHE_VERSION_MAJOR" -eq "3" ]; then
     ghe-ssh "$GHE_HOSTNAME" -- "ghe-cluster-nomad-cleanup" 1>&3 2>&3
   elif [ "$GHE_VERSION_MAJOR" -eq "2" ] && [ "$GHE_VERSION_MINOR" -eq "22" ]; then
     ghe-ssh "$GHE_HOSTNAME" -- "ghe-cluster-each -- /usr/local/share/enterprise/ghe-nomad-cleanup" 1>&3 2>&3
   fi
   ghe-ssh "$GHE_HOSTNAME" -- "ghe-cluster-config-apply" 1>&3 2>&3
+  bm_end "configure_cluster"
 elif $instance_configured; then
   log_info "Configuring appliance ..."
+  bm_start "configure_appliance"
   if [ "$GHE_VERSION_MAJOR" -eq "3" ]; then
     ghe-ssh "$GHE_HOSTNAME" -- "ghe-nomad-cleanup" 1>&3 2>&3
   elif [ "$GHE_VERSION_MAJOR" -eq "2" ] && [ "$GHE_VERSION_MINOR" -eq "22" ]; then
     ghe-ssh "$GHE_HOSTNAME" -- "/usr/local/share/enterprise/ghe-nomad-cleanup" 1>&3 2>&3
   fi
   ghe-ssh "$GHE_HOSTNAME" -- "ghe-config-apply" 1>&3 2>&3
+  bm_end "$(basename $0) - configure appliance"
 fi
 
 # Clear GitHub Connect settings stored in the restored database.
@@ -569,6 +634,7 @@ CRON_RUNNING=true
 # Clean up all stale replicas on configured instances.
 if ! $CLUSTER && $instance_configured; then
   log_info "Cleaning up replicas..." 1>&3
+  bm_start "$(basename $0) - Cleanup replicas"
   restored_uuid=$(cat "$GHE_RESTORE_SNAPSHOT_PATH/uuid")
   other_nodes=$(echo "
     set -o pipefail; \
@@ -584,6 +650,7 @@ if ! $CLUSTER && $instance_configured; then
       echo "set -o pipefail; $(typeset -f cleanup_cluster_nodes); cleanup_cluster_nodes $uuid" | ghe-ssh "$GHE_HOSTNAME" 1>&3
     done
   fi
+  bm_end "$(basename $0) - Cleanup replicas"
 fi
 
 # Update the remote status to "complete". This has to happen before importing

script/release

@@ -31,7 +31,8 @@ GH_REPO = ENV['GH_REPO'] || 'backup-utils'
 GH_OWNER = ENV['GH_OWNER'] || 'github'
 GH_AUTHOR = ENV['GH_AUTHOR']
 DEB_PKG_NAME = 'github-backup-utils'
-GH_BASE_BRANCH = ENV['GH_BASE_BRANCH'] || 'master'
+GH_BASE_BRANCH = ENV['GH_BASE_BRANCH'] || 'master' # TODO: should we even allow a default or require all params get set explicitly?
+GH_STABLE_BRANCH = ""
 
 CHANGELOG_TMPL = '''<%= package_name %> (<%= package_version %>) UNRELEASED; urgency=medium
 
@@ -137,7 +138,8 @@ def beautify_changes(changes)
 end
 
 def changelog
-  changes = `git log --pretty=oneline origin/stable...origin/#{GH_BASE_BRANCH} --reverse --grep "Merge pull request" | sort -t\# -k2`.lines.map(&:strip)
+  puts "building changelog by comparing origin/#{GH_STABLE_BRANCH}...origin/#{GH_BASE_BRANCH}"
+  changes = `git log --pretty=oneline origin/#{GH_STABLE_BRANCH}...origin/#{GH_BASE_BRANCH} --reverse --grep "Merge pull request" | sort -t\# -k2`.lines.map(&:strip)
   raise 'Building the changelog failed' if $CHILD_STATUS != 0
 
   changes
@@ -228,12 +230,12 @@ def push_release_branch(version)
 end
 
 def update_stable_branch
-  `git checkout --quiet stable`
+  `git checkout --quiet #{GH_STABLE_BRANCH}`
   unless (out = `git merge --quiet --ff-only origin/#{GH_BASE_BRANCH}`)
-    warn "Merging #{GH_BASE_BRANCH} into stable failed:\n\n#{out}"
+    warn "Merging #{GH_BASE_BRANCH} into #{GH_STABLE_BRANCH} failed:\n\n#{out}"
   end
-  unless (out = `git push --quiet origin stable`)
-    warn "Failed pushing the stable branch:\n\n#{out}"
+  unless (out = `git push --quiet origin #{GH_STABLE_BRANCH}`)
+    warn "Failed pushing the #{GH_STABLE_BRANCH} branch:\n\n#{out}"
   end
 end
 
@@ -333,9 +335,38 @@ def clean_up(version)
   `git branch --quiet -D tmp-packaging >/dev/null 2>&1`
 end
 
+def is_base_branch_valid?(branch)
+  if branch == "master" || branch.match(/^\d+\.\d+-main$/)
+    return true
+  else
+    return false
+  end
+end
+
+def get_stable_branch_name(branch)
+  ## derive the proper stable branch. if the base branch is "master" the stable branch is just "stable"
+  ## if the base branch is a release branch, the stable branch will be "x.y-stable"
+  result = ""
+  if branch == "master"
+    result = "stable"
+  else
+    result = branch.gsub(/-main$/, "-stable")
+  end
+
+  result
+end
+
 #### All the action starts ####
 if $PROGRAM_NAME == __FILE__
   begin
+    ## validate base branch. this must either be "master" or a release branch which will match the pattern "x.y-main"
+    raise "The branch #{GH_BASE_BRANCH} is not valid for releasing backup-utils. branch name must be master or match x.y-main" if !is_base_branch_valid?(GH_BASE_BRANCH)
+
+    GH_STABLE_BRANCH = get_stable_branch_name(GH_BASE_BRANCH)
+
+    puts "base branch = " + GH_BASE_BRANCH
+    puts "stable branch = " + GH_STABLE_BRANCH
+
     args = ARGV.dup
     dry_run = false
     skip_version_bump_check = false
@@ -455,7 +486,7 @@ if $PROGRAM_NAME == __FILE__
     puts 'Cleaning up...'
     clean_up version
 
-    puts 'Updating stable branch...'
+    puts "Updating #{GH_STABLE_BRANCH} branch..."
     update_stable_branch
 
     puts 'Released!'