Merge pull request #280 from github/progress-indicator

Progress Indicator

Author: gamefiend

bin/ghe-backup

@@ -37,11 +37,21 @@ while true; do
   esac
 done
 
+
 export CALLING_SCRIPT="ghe-backup"
+
 # Bring in the backup configuration
 # shellcheck source=share/github-backup-utils/ghe-backup-config
 . "$( dirname "${BASH_SOURCE[0]}" )/../share/github-backup-utils/ghe-backup-config"
 
+# Setup progress tracking
+init-progress
+export PROGRESS_TYPE="Backup"
+echo "$PROGRESS_TYPE" > /tmp/backup-utils-progress-type
+export PROGRESS=0 # Used to track progress of backup
+echo "$PROGRESS" > /tmp/backup-utils-progress
+export PROGRESS_TOTAL=18 # Maximum number of steps in backup
+
 # Check to make sure moreutils parallel is installed and working properly
 ghe_parallel_check
 
@@ -121,7 +131,7 @@ ghe_restore_check
 # Check to see if there is a running backup
 if [ -h ../in-progress ]; then
 
-  log_error "Error: detected a backup already in progress from a previous version of ghe-backup. \nIf there is no backup in progress anymore, please remove \nthe $GHE_DATA_DIR/in-progress file." 1>&2
+  log_error "Detected a backup already in progress from a previous version of ghe-backup. \nIf there is no backup in progress anymore, please remove \nthe $GHE_DATA_DIR/in-progress file." >&2
   exit 1
 fi
 
@@ -306,6 +316,7 @@ else
   steps="${failures// /, }"
   ghe_remote_logger "Completed backup from $(hostname) / snapshot $GHE_SNAPSHOT_TIMESTAMP with failures: ${steps}."
   log_error "Error: Snapshot incomplete. Some steps failed: ${steps}. "
+  ghe_backup_finished
   exit 1
 fi
 

bin/ghe-backup-progress

@@ -0,0 +1,56 @@
+#!/usr/bin/env bash
+#/ Usage: ghe-backup-progress [--once]
+#/ Tracks the completed steps of a backup or restore operation.
+#/
+#/ By default the progress is printed every continuously or until a key is pressed.
+#/ Use the --once option to print the current progress once and exit.
+#/
+#/ Options:
+#/  --once  Don't loop, just print the current progress once.
+#
+set -e
+
+while true; do
+    case "$1" in
+        -o|--once)
+            ONCE=1
+            shift
+            ;;
+        -h|--help)
+            export GHE_SHOW_HELP=true
+            shift
+            ;;
+        -*)
+            echo "Unknown option: $1" >&2
+            exit 1
+            ;;
+        *)
+            break
+            ;;
+    esac
+done
+
+check_for_progress_file() {
+    if [ ! -f /tmp/backup-utils-progress-info ]; then
+        echo "No progress file found. Has a backup or restore been started?"
+        exit 1
+    fi
+}
+
+if [ -n "$ONCE" ]; then
+    check_for_progress_file
+    cat /tmp/backup-utils-progress-info
+else
+    check_for_progress_file
+    clear
+    cat /tmp/backup-utils-progress-info
+    while true; do
+    if read -r -t 1 -n 1; then
+	clear
+        exit ;
+    else
+	clear
+	cat /tmp/backup-utils-progress-info
+    fi
+    done
+fi

bin/ghe-restore

@@ -86,6 +86,8 @@ while true; do
   esac
 done
 
+
+
 start_cron () {
   log_info "Starting cron ..."
   if $CLUSTER; then
@@ -164,6 +166,10 @@ cleanup_cluster_nodes() {
 # shellcheck source=share/github-backup-utils/ghe-backup-config
 . "$( dirname "${BASH_SOURCE[0]}" )/../share/github-backup-utils/ghe-backup-config"
 
+
+
+
+
 # Check to make sure moreutils parallel is installed and working properly
 ghe_parallel_check
 
@@ -265,6 +271,48 @@ if is_external_database_snapshot && ! $instance_configured && ! $FORCE; then
 
   prompt_for_confirmation "Please confirm this before continuing."
 fi
+# Calculate the actual amounts of steps in the restore process
+# taking into account the options passed to the script and the appliance configuration
+# calculate restore steps
+OPTIONAL_STEPS=0
+# Cluster restores add an additional step
+if $CLUSTER ; then
+  OPTIONAL_STEPS=$((OPTIONAL_STEPS + 1))
+fi
+# Restoring UUID
+if [ -s "$GHE_RESTORE_SNAPSHOT_PATH/uuid" ] && ! $CLUSTER; then
+  OPTIONAL_STEPS=$((OPTIONAL_STEPS + 1))
+fi
+# Restoring Actions
+if ghe-ssh "$GHE_HOSTNAME" -- 'ghe-config --true app.actions.enabled'; then
+  OPTIONAL_STEPS=$((OPTIONAL_STEPS + 1))
+fi
+# Restoring minio
+if ghe-ssh "$GHE_HOSTNAME" -- 'ghe-config --true app.minio.enabled'; then
+  OPTIONAL_STEPS=$((OPTIONAL_STEPS + 1))
+fi
+# Restoring Elasticsearch
+if ! $CLUSTER && [ -d "$GHE_RESTORE_SNAPSHOT_PATH/elasticsearch" ]; then
+  OPTIONAL_STEPS=$((OPTIONAL_STEPS + 1))
+fi
+# Restoring audit log
+if $CLUSTER || [ "$(version "$GHE_REMOTE_VERSION")" -ge "$(version 2.12.9)" ]; then
+  if [[ "$GHE_RESTORE_SKIP_AUDIT_LOG" != "yes" ]]; then
+    OPTIONAL_STEPS=$((OPTIONAL_STEPS + 1))
+  fi
+fi
+# Replica cleanup
+if ! $CLUSTER && $instance_configured; then
+  OPTIONAL_STEPS=$((OPTIONAL_STEPS + 1))
+fi
+# Maximum restore steps
+export PROGRESS_TOTAL=$((OPTIONAL_STEPS + 6))
+
+init-progress
+export PROGRESS_TYPE="Restore"
+echo "$PROGRESS_TYPE" > /tmp/backup-utils-progress-type
+export PROGRESS=0 # Used to track progress of restore
+echo "$PROGRESS" > /tmp/backup-utils-progress
 
 # Log restore start message locally and in /var/log/syslog on remote instance
 START_TIME=$(date +%s)
@@ -343,10 +391,11 @@ log_warn "Warning: storing backup-utils version remotely failed."
 # Stop cron and timerd, as scheduled jobs may disrupt the restore process.
 log_info "Stopping cron and github-timerd ..."
 if $CLUSTER; then
+  bm_start "$(basename $0) - Stopping cron and github-timerd on cluster"
   if ! ghe-ssh "$GHE_HOSTNAME" -- "ghe-cluster-each -- sudo service cron stop"; then
     log_warn "Failed to stop cron on one or more nodes" 1>&3
   fi
-
+  bm_end "$(basename $0) - Stopping cron and github-timerd on cluster"
   if [ "$GHE_VERSION_MAJOR" -eq "3" ]; then
     if ghe-ssh "$GHE_HOSTNAME" -- "systemctl -q is-active nomad && nomad job status --short github-timerd &>/dev/null"; then
       if ! ghe-ssh "$GHE_HOSTNAME" -- "sudo nomad stop github-timerd 1>/dev/null"; then
@@ -358,11 +407,14 @@ if $CLUSTER; then
       log_warn "Failed to stop github-timerd on one or more nodes" 1>&3
     fi
   fi
+ 
 else
+  bm_start "$(basename $0) - Stopping cron and github-timerd"
+  echo "$(basename $0) - Stopping cron and github-timerd"
   if ! ghe-ssh "$GHE_HOSTNAME" -- "sudo service cron stop"; then
     log_warn "Failed to stop cron" 1>&3
   fi
-
+  bm_end "$(basename $0) - Stopping cron and github-timerd"
   if [ "$GHE_VERSION_MAJOR" -eq "3" ]; then
     if ghe-ssh "$GHE_HOSTNAME" -- "systemctl -q is-active nomad && nomad job status --short github-timerd &>/dev/null"; then
       if ! ghe-ssh "$GHE_HOSTNAME" -- "sudo nomad stop github-timerd 1>/dev/null"; then
@@ -374,9 +426,11 @@ else
       log_warn "Failed to stop github-timerd" 1>&3
     fi
   fi
+  
 fi
 CRON_RUNNING=false
 
+
 # Restore settings and license if restoring to an unconfigured appliance or when
 # specified manually.
 if $RESTORE_SETTINGS; then
@@ -394,11 +448,15 @@ fi
 # Restore UUID if present and not restoring to cluster.
 if [ -s "$GHE_RESTORE_SNAPSHOT_PATH/uuid" ] && ! $CLUSTER; then
   log_info "Restoring UUID ..."
+
+  bm_start "$(basename $0) - Restore UUID"
   ghe-ssh "$GHE_HOSTNAME" -- "sudo sponge '$GHE_REMOTE_DATA_USER_DIR/common/uuid' 2>/dev/null" <"$GHE_RESTORE_SNAPSHOT_PATH/uuid"
   ghe-ssh "$GHE_HOSTNAME" -- "sudo systemctl stop consul" || true
   ghe-ssh "$GHE_HOSTNAME" -- "sudo rm -rf /data/user/consul/raft"
+  bm_end "$(basename $0) - Restore UUID"
 fi
 
+
 if is_external_database_snapshot; then
    appliance_strategy="external"
    backup_snapshot_strategy="external"
@@ -448,7 +506,7 @@ fi
 cmd_title=$(log_info "Restoring Redis database ...")
 commands=("
 echo \"$cmd_title\"
-ghe-ssh \"$GHE_HOSTNAME\" -- 'ghe-import-redis' < \"$GHE_RESTORE_SNAPSHOT_PATH/redis.rdb\" 1>&3")
+ghe-restore-redis \"$GHE_HOSTNAME\" \"$GHE_RESTORE_SNAPSHOT_PATH\"")
 
 cmd_title=$(log_info "Restoring Git Repositories ...")
 commands+=("
@@ -470,7 +528,7 @@ fi
 cmd_title=$(log_info "Restoring SSH authorized keys ...")
 commands+=("
 echo \"$cmd_title\"
-ghe-ssh \"$GHE_HOSTNAME\" -- 'ghe-import-authorized-keys' < \"$GHE_RESTORE_SNAPSHOT_PATH/authorized-keys.json\" 1>&3")
+ghe-restore-ssh-keys \"$GHE_HOSTNAME\" \"$GHE_RESTORE_SNAPSHOT_PATH\"")
 
 cmd_title=$(log_info "Restoring storage data ...")
 commands+=("
@@ -517,14 +575,17 @@ if [ "$GHE_PARALLEL_ENABLED" = "yes" ]; then
 else
   log_info "Restoring data serially ..." 1>&3
   for c in "${commands[@]}"; do
+    . "$( dirname "${BASH_SOURCE[0]}" )/../share/github-backup-utils/bm.sh"
     eval "$c"
   done
 fi
 
 # Restart an already running memcached to reset the cache after restore
-log_info "Restarting memcached ..." 1>&3
+log_info  "Restarting memcached ..." 1>&3
+bm_start "$(basename $0) - Restarting memcached"
 echo "sudo restart -q memcached 2>/dev/null || true" |
   ghe-ssh "$GHE_HOSTNAME" -- /bin/sh
+bm_end "$(basename $0) - Restarting memcached"
 
 # Prevent GitHub Connect jobs running before we've had a chance to reset
 # the configuration by setting the last run date to now.
@@ -538,20 +599,24 @@ fi
 # config run to perform data migrations.
 if $CLUSTER; then
   log_info "Configuring cluster ..."
+  bm_start "$(basename $0) - configure cluster"
   if [ "$GHE_VERSION_MAJOR" -eq "3" ]; then
     ghe-ssh "$GHE_HOSTNAME" -- "ghe-cluster-nomad-cleanup" 1>&3 2>&3
   elif [ "$GHE_VERSION_MAJOR" -eq "2" ] && [ "$GHE_VERSION_MINOR" -eq "22" ]; then
     ghe-ssh "$GHE_HOSTNAME" -- "ghe-cluster-each -- /usr/local/share/enterprise/ghe-nomad-cleanup" 1>&3 2>&3
   fi
   ghe-ssh "$GHE_HOSTNAME" -- "ghe-cluster-config-apply" 1>&3 2>&3
+  bm_end "configure_cluster"
 elif $instance_configured; then
   log_info "Configuring appliance ..."
+  bm_start "configure_appliance"
   if [ "$GHE_VERSION_MAJOR" -eq "3" ]; then
     ghe-ssh "$GHE_HOSTNAME" -- "ghe-nomad-cleanup" 1>&3 2>&3
   elif [ "$GHE_VERSION_MAJOR" -eq "2" ] && [ "$GHE_VERSION_MINOR" -eq "22" ]; then
     ghe-ssh "$GHE_HOSTNAME" -- "/usr/local/share/enterprise/ghe-nomad-cleanup" 1>&3 2>&3
   fi
   ghe-ssh "$GHE_HOSTNAME" -- "ghe-config-apply" 1>&3 2>&3
+  bm_end "$(basename $0) - configure appliance"
 fi
 
 # Clear GitHub Connect settings stored in the restored database.
@@ -569,6 +634,7 @@ CRON_RUNNING=true
 # Clean up all stale replicas on configured instances.
 if ! $CLUSTER && $instance_configured; then
   log_info "Cleaning up replicas..." 1>&3
+  bm_start "$(basename $0) - Cleanup replicas"
   restored_uuid=$(cat "$GHE_RESTORE_SNAPSHOT_PATH/uuid")
   other_nodes=$(echo "
     set -o pipefail; \
@@ -584,6 +650,7 @@ if ! $CLUSTER && $instance_configured; then
       echo "set -o pipefail; $(typeset -f cleanup_cluster_nodes); cleanup_cluster_nodes $uuid" | ghe-ssh "$GHE_HOSTNAME" 1>&3
     done
   fi
+  bm_end "$(basename $0) - Cleanup replicas"
 fi
 
 # Update the remote status to "complete". This has to happen before importing

share/github-backup-utils/bm.sh

@@ -49,6 +49,7 @@ bm_end() {
   local tend tstart total
   tend=$(date +%s)
   tstart=$(eval "echo \$$(bm_desc_to_varname "$@")_start")
+
   total=$(($tend - $tstart))
 
   echo "$1 took ${total}s" >> $BM_FILE_PATH
@@ -57,4 +58,6 @@ bm_end() {
   if [ -n "$GHE_DEBUG" ]; then
     echo "Debug: $1 took ${total}s (bm_end)"
   fi
+  # track progress
+  progress "$1 took ${total}s"
 }