Updated hostcheck to use rsync

Author: chuckp22

bin/ghe-host-check

@@ -151,42 +151,54 @@ min_jq=""
 # shellcheck source=share/github-backup-utils/requirements.txt
 . "$(dirname "${BASH_SOURCE[0]}")/../share/github-backup-utils/requirements.txt"
 
+#source disk size file
+. "$(dirname "${BASH_SOURCE[0]}")/../share/github-backup-utils/ghe-rsync-size.sh"
+
 #Display dir requirements for repositories and mysql
-backup_dir=${GHE_DATA_DIR}
-available_space=$(df $backup_dir | awk 'END{print $4}')
+available_space=$(df $GHE_DATA_DIR | awk 'END{print $4}') 
+echo "Available space: $available_space" 1>&2
+
 repo_disk_size=$(ghe-ssh "$host" sudo du -sh /data/user/repositories | awk '{print $1}')
-#mysql_disk_size=$(ghe-ssh "$host" sudo du -sh /data/user/mysql | awk '{print $1}')
 # Convert the data size to kilobytes
 case "$repo_disk_size" in
   *T) repo_disk_size=$(( $(echo "$repo_disk_size" | sed 's/T/000000000/') )) ;;
   *G) repo_disk_size=$(( $(echo "$repo_disk_size" | sed 's/G/000000/') )) ;;
   *M) repo_disk_size=$(( $(echo "$repo_disk_size" | sed 's/M/000/') )) ;;
   *K) repo_disk_size=$(( $(echo "$repo_disk_size" | sed 's/K//') )) ;;
 esac
+repos_disk_size=$(transfer_size repositories /tmp)
+pages_disk_size=$(transfer_size pages /tmp)
+es_disk_size=$(transfer_size elasticsearch /tmp)
+stor_disk_size=$(transfer_size storage /tmp)
+minio_disk_size=$(transfer_size minio /tmp)
+min_disk_req=$(( $(echo "$repos_disk_size" | awk '{print $1}') + $(echo "$pages_disk_size" | awk '{print $1}') + $(echo "$es_disk_size" | awk '{print $1}') + $(echo "$stor_disk_size" | awk '{print $1}') + $(echo "$minio_disk_size" | awk '{print $1}') ))
+echo "repositories = $repos_disk_size, pages: $pages_disk_size, elasticsearch: $es_disk_size, storage: $stor_disk_size, minio: $minio_disk_size" 1>&2
+echo "min_disk_required for backup = $min_disk_req bytes" 1>&2
+
 if [[ $available_space -lt $repo_disk_size ]]; then
-  echo "There is not enough disk space for the backup"
+  echo "There is not enough disk space for the backup" 1>&2
   exit 1
 fi
 
 #Check rsync, openssh & jq versions
 rsync_version=$(rsync --version | grep 'version' | awk '{print $3}')
 if awk "BEGIN {exit !($rsync_version < $min_rsync)}" &> /dev/null; then
-  echo "rsync version $rsync_version in backup-host does not meet minimum requirements."
-  echo "Please make sure you have the minimum required version of rsync: $min_rsync installed"
+  echo "rsync version $rsync_version in backup-host does not meet minimum requirements." 1>&2
+  echo "Please make sure you have the minimum required version of rsync: $min_rsync installed" 1>&2
   exit 1
 fi
 
 ssh_version=$(ssh -V 2>&1 | awk '{print $1}'|grep -oPm 1 '[\d\.]+' |head -1)
 if awk "BEGIN {exit !($ssh_version < $min_openssh)}" &> /dev/null; then
-  echo "openSSH version $ssh_version in backup-host does not meet minimum requirements."
-  echo "Please make sure the minimum required version of openSSH: $min_openssh is installed"
+  echo "openSSH version $ssh_version in backup-host does not meet minimum requirements." 1>&2
+  echo "Please make sure the minimum required version of openSSH: $min_openssh is installed" 1>&2
   exit 1
 fi
 
 jq_version=$(jq --version |awk -F\- '{print $2}')
 if awk "BEGIN {exit !($jq_version < $min_jq)}" &> /dev/null; then
-  echo "jq version $jq_version in backup-host does not meet minimum requirements."
-  echo "Please make sure you have the minimum required version of jq: $min_jq installed"
+  echo "jq version $jq_version in backup-host does not meet minimum requirements." 1>&2
+  echo "Please make sure you have the minimum required version of jq: $min_jq installed" 1>&2
   exit 1
 fi
 

share/github-backup-utils/ghe-rsync-size.sh

@@ -0,0 +1,63 @@
+#!/usr/bin/env bash
+# get-rsync-size.sh Get the total size of dir-files to be transfered using rsync --link-dest
+#
+# Example:
+#   transfer_size repositories /dest_dir
+#
+# Sample output:
+#   Total transferred file size: 80 bytes
+
+# Bring in the backup configuration
+# shellcheck source=share/github-backup-utils/ghe-backup-config
+. "$(dirname "${BASH_SOURCE[0]}")/ghe-backup-config"
+
+# Location of last good backup for rsync --link-dest
+backup_current="$GHE_DATA_DIR/current/"
+
+# If we have a previous increment, avoid using those unchanged files using --link-dest support.
+if [ -d "$backup_current" ]; then
+  link_dest="--link-dest=${GHE_DATA_DIR}/current"
+fi
+
+transfer_size()
+{
+  local backup_data=$1
+  local dir=/data/user/$1/
+  local dest_dir=$2/
+
+  # Define user for rsync-path
+  case "$backup_data" in
+  "repositories" | "pages")
+    user="git"
+    ;;
+  "storage")
+    user="alambic"
+    ;;
+  "elasticsearch")
+    user="elasticsearch"
+    ;;
+  "mysql")
+    user="mysql"
+    ;;
+  "mssql")
+    user="mssql"
+    ;;
+  "minio")
+    user="minio"
+    ;;
+  *)
+    echo "Unknown user: $backup_data"
+    exit 1
+    ;;
+  esac
+
+#  local total_file_size=$(ghe-ssh "$GHE_HOSTNAME" sudo rsync -arvn $link_dest/$1 --stats "$dir" "$dest_dir" | grep "Total transferred file size" | sed 's/.*size: //; s/,//')
+  local total_file_size=$(ghe-rsync -arn --stats \
+      -e "ssh -q $GHE_EXTRA_SSH_OPTS -p 122 -l admin" \
+      --rsync-path="sudo -u $user rsync" \
+      $link_dest/$1 \
+      --ignore-missing-args \
+      "$GHE_HOSTNAME:$GHE_REMOTE_DATA_USER_DIR/user/$1/" \
+      "$dest_dir" | grep "Total transferred file size" | sed 's/.*size: //; s/,//g')
+  echo "$total_file_size" 
+}