Merge branch 'master' into progress-indicator

Author: gamefiend

bin/ghe-backup

@@ -88,7 +88,7 @@ if ! output=$(rsync -a src/ dest1 2>&1 && rsync -av src/ --link-dest=../dest1 de
   exit 1
 fi
 
-if [ "$(ls -il dest1/testfile | awk '{ print $1 }')" != "$(ls -il dest2/testfile | awk '{ print $1 }')" ]; then
+if [ "$(stat -c %i dest1/testfile)" != "$(stat -c %i dest2/testfile)" ]; then
   log_error "Error: the filesystem containing $GHE_DATA_DIR does not support hard links.\n Backup Utilities use hard links to store backup data efficiently." 1>&2
   exit 1
 fi
@@ -108,13 +108,13 @@ cleanup () {
     progress=$(cat ../in-progress)
     snapshot=$(echo "$progress" | cut -d ' ' -f 1)
     pid=$(echo "$progress" | cut -d ' ' -f 2)
-    if [ "$snapshot" = "$GHE_SNAPSHOT_TIMESTAMP" ] && [ "$$" = $pid ]; then
+    if [ "$snapshot" = "$GHE_SNAPSHOT_TIMESTAMP" ] && [ "$$" = "$pid" ]; then
       unlink ../in-progress
     fi
   fi
 
   rm -rf "$failures_file"
-  rm -f ${GHE_DATA_DIR}/in-progress-backup
+  rm -f "${GHE_DATA_DIR}/in-progress-backup"
 
   # Cleanup SSH multiplexing
   ghe-ssh --clean
@@ -150,11 +150,11 @@ if [ -f ../in-progress ]; then
 fi
 
 echo "$GHE_SNAPSHOT_TIMESTAMP $$" > ../in-progress
-echo "$GHE_SNAPSHOT_TIMESTAMP $$" > ${GHE_DATA_DIR}/in-progress-backup
+echo "$GHE_SNAPSHOT_TIMESTAMP $$" > "${GHE_DATA_DIR}/in-progress-backup"
 
+START_TIME=$(date +%s)
 log_info "Starting backup of $GHE_HOSTNAME with backup-utils v$BACKUP_UTILS_VERSION in snapshot $GHE_SNAPSHOT_TIMESTAMP"
 
-
 # Perform a host connection check and establish the remote appliance version.
 # The version is available in the GHE_REMOTE_VERSION variable and also written
 # to a version file in the snapshot directory itself.
@@ -176,9 +176,6 @@ echo "$GHE_BACKUP_STRATEGY" > strategy
 # Create benchmark file
 bm_init > /dev/null
 
-START_TIME=$(date +%s)
-log_info "Starting backup of $GHE_HOSTNAME with backup-utils v$BACKUP_UTILS_VERSION"
-
 ghe-backup-store-version  ||
 log_warn "Warning: storing backup-utils version remotely failed."
 
@@ -250,7 +247,7 @@ if [ "$GHE_BACKUP_STRATEGY" = "rsync" ]; then
 fi
 
 if [ "$GHE_PARALLEL_ENABLED" = "yes" ]; then
-  $GHE_PARALLEL_COMMAND $GHE_PARALLEL_COMMAND_OPTIONS -- "${commands[@]}"
+  "$GHE_PARALLEL_COMMAND" "${GHE_PARALLEL_COMMAND_OPTIONS[@]}" -- "${commands[@]}"
 else
   for c in "${commands[@]}"; do
     eval "$c"
@@ -264,7 +261,7 @@ fi
 # git fsck repositories after the backup
 if [ "$GHE_BACKUP_FSCK" = "yes" ]; then
   log_info "Running git fsck on repositories ..."
-  ghe-backup-fsck $GHE_SNAPSHOT_DIR || failures="$failures fsck"
+  ghe-backup-fsck "$GHE_SNAPSHOT_DIR" || failures="$failures fsck"
 fi
 
 # If everything was successful, mark the snapshot as complete, update the
@@ -279,13 +276,15 @@ if [ -z "$failures" ]; then
   ghe-prune-snapshots
 fi
 
+END_TIME=$(date +%s)
+log_info "Runtime: $((END_TIME - START_TIME)) seconds"
 log_info "Completed backup of $GHE_HOSTNAME in snapshot $GHE_SNAPSHOT_TIMESTAMP at $(date +"%H:%M:%S")"
 
 # Exit non-zero and list the steps that failed.
 if [ -z "$failures" ]; then
   ghe_remote_logger "Completed backup from $(hostname) / snapshot $GHE_SNAPSHOT_TIMESTAMP successfully."
 else
-  steps="$(echo $failures | sed 's/ /, /g')"
+  steps="${failures// /, }"
   ghe_remote_logger "Completed backup from $(hostname) / snapshot $GHE_SNAPSHOT_TIMESTAMP with failures: ${steps}."
   log_error "Error: Snapshot incomplete. Some steps failed: ${steps}. "
   ghe_backup_finished
@@ -296,11 +295,7 @@ fi
 log_info "Checking for leaked ssh keys ..."
 ghe-detect-leaked-ssh-keys -s "$GHE_SNAPSHOT_DIR" || true
 
-END_TIME=$(date +%s)
-log_info "Runtime: $((${END_TIME} - ${START_TIME})) seconds"
 log_info "Backup of $GHE_HOSTNAME finished."
-# Make sure we exit zero after the conditional
-true
 
 # Remove in-progress file
 ghe_backup_finished

bin/ghe-host-check

@@ -159,7 +159,7 @@ if [[ "$CALLING_SCRIPT" == "ghe-backup" ]]; then
   #Display dir requirements for repositories and mysql
   echo "Checking host for sufficient space for a backup..." 1>&2
   available_space=$(df -B 1k $GHE_DATA_DIR | awk 'END{printf "%.0f", $4 * 1024}') 
-  printf "Available space: %d MB. We recommend allocating at least 5x the amount of storage allocated to the primary GitHub appliance for historical snapshots and growth over time.\n" "$((available_space / 1024 ** 2))" 1>&2
+  echo "We recommend allocating at least 5x the amount of storage allocated to the primary GitHub appliance for historical snapshots and growth over time." 1>&2
 
   repos_disk_size=$(transfer_size repositories /tmp)
   pages_disk_size=$(transfer_size pages /tmp)
@@ -170,17 +170,29 @@ if [[ "$CALLING_SCRIPT" == "ghe-backup" ]]; then
   actions_disk_size=$(transfer_size actions /tmp)
   mssql_disk_size=$(transfer_size mssql /tmp)
 
-  #min_disk_req=$(( $(echo "$repos_disk_size" | awk '{printf "%.0f", $1}') + $(echo "$pages_disk_size" | awk '{printf "%.0f", $1}') + $(echo "$es_disk_size" | awk '{printf "%.0f", $1}') + $(echo "$stor_disk_size" | awk '{printf "%.0f", $1}') + $(echo "$minio_disk_size" | awk '{printf "%.0f", $1}') + $(echo "$mysql_disk_size" | awk '{printf "%.0f", $1/2}') + $(echo "$actions_disk_size" | awk '{printf "%.0f", $1}') + $(echo "$mssql_disk_size" | awk '{printf "%.0f", $1}') ))
-  min_disk_req=$(( $(echo "$repos_disk_size") + $(echo "$pages_disk_size") + $(echo "$es_disk_size") + $(echo "$stor_disk_size") + $(echo "$minio_disk_size") + $(echo "$mysql_disk_size") + $(echo "$actions_disk_size") + $(echo "$mssql_disk_size") ))
-echo -e "### Data Transfer Sizes \nrepositories: $repos_disk_size \npages: $pages_disk_size \nelasticsearch: $es_disk_size \nstorage: $stor_disk_size \nminio: $minio_disk_size \nmysql: $mysql_disk_size \nactions: $actions_disk_size \nmssql: $mssql_disk_size" 1>&2
-  printf "min_disk_required for this backup is at least %d MB\n" "$((min_disk_req / 1024 ** 2))" 1>&2
+  min_disk_req=$((repos_disk_size + pages_disk_size + es_disk_size + stor_disk_size + minio_disk_size + mysql_disk_size + actions_disk_size + mssql_disk_size))
+  echo "Available space: $((available_space / (1024 ** 2))) MB" 1>&2
+  echo -e "Min Disk required for this backup is at least $min_disk_req MB\n" 1>&2
+
+cat <<DATA_TRANSFER_SIZE 1>&2
+### Data Transfer Sizes 
+repositories: $repos_disk_size MB
+pages: $pages_disk_size MB
+elasticsearch: $es_disk_size MB
+storage: $stor_disk_size MB
+minio: $minio_disk_size MB
+mysql: $mysql_disk_size MB
+actions: $actions_disk_size MB
+mssql: $mssql_disk_size MB
+DATA_TRANSFER_SIZE
 
   if [[ $available_space -lt $min_disk_req ]]; then
     echo "There is not enough disk space for the backup. Please allocate more space and continue." 1>&2
     exit 1
   fi
 
   #Check rsync, openssh & jq versions
+  echo "### Software versions" 1>&2
   rsync_version=$(rsync --version | grep 'version' | awk '{print $3}')
   if awk "BEGIN {exit !($rsync_version < $min_rsync)}" &> /dev/null; then
     echo "rsync version $rsync_version in backup-host does not meet minimum requirements." 1>&2