Merge pull request #247 from github/sridharaviash/cluster-speed-imporvements

Adding a script to speed up cluster backups on enterprise

Author: rubiojr

bin/ghe-backup

@@ -166,8 +166,18 @@ ghe-backup-es-hookshot ||
 failures="$failures hookshot"
 
 echo "Backing up Git repositories ..."
-ghe-backup-repositories-${GHE_BACKUP_STRATEGY} ||
-failures="$failures repositories"
+if [ "$GHE_BACKUP_STRATEGY" = "cluster" ]; then
+  if ghe-ssh "$GHE_HOSTNAME" test -f /data/github/current/bin/dgit-cluster-backup-routes ; then
+    echo "* Using calculated routes method..."
+    ghe-backup-repositories-cluster-ng || failures="$failure repositories"
+  else
+    echo "* Using legacy method. A faster backup method is available on enterprise 2.7 and up."
+    ghe-backup-repositories-cluster || failures="$failures repositories"
+  fi
+else
+  ghe-backup-repositories-${GHE_BACKUP_STRATEGY} ||
+  failures="$failures repositories"
+fi
 
 echo "Backing up GitHub Pages ..."
 ghe-backup-pages-${GHE_BACKUP_STRATEGY} ||

share/github-backup-utils/ghe-backup-repositories-cluster

@@ -51,8 +51,8 @@ backup_current="$GHE_DATA_DIR/current/repositories"
 
 # Verify rsync is available.
 if ! rsync --version 1>/dev/null 2>&1; then
-    echo "Error: rsync not found." 1>&2
-    exit 1
+  echo "Error: rsync not found." 1>&2
+  exit 1
 fi
 
 # Perform a host-check and establish GHE_REMOTE_XXX variables.
@@ -108,28 +108,27 @@ done
 # If we have a previous increment, avoid transferring existing files via rsync's
 # --link-dest support. This also decreases physical space usage considerably.
 if [ -d "$backup_current" ]; then
-    link_dest="--link-dest=../../current/repositories"
+  link_dest="--link-dest=../../current/repositories"
 fi
 
 # Transfer repository data from a GitHub instance to the current snapshot
 # directory, using a previous snapshot to avoid transferring files that have
 # already been transferred. A set of rsync filter rules are provided on stdin
 # for each invocation.
 rsync_repository_data () {
-    port=$(ssh_port_part "$1")
-    host=$(ssh_host_part "$1")
-
-    shift
-    ghe-rsync -av \
-        -e "ssh -q $opts -p $port -F $config_file -l $user" \
-        $link_dest "$@" \
-        --rsync-path='sudo -u git rsync' \
-        --include-from=- --exclude=\* \
-        "$host:$GHE_REMOTE_DATA_USER_DIR/repositories/" \
-        "$backup_dir" 1>&3
+  port=$(ssh_port_part "$1")
+  host=$(ssh_host_part "$1")
+
+  shift
+  ghe-rsync -av \
+    -e "ssh -q $opts -p $port -F $config_file -l $user" \
+    $link_dest "$@" \
+    --rsync-path='sudo -u git rsync' \
+    --include-from=- --exclude=\* \
+    "$host:$GHE_REMOTE_DATA_USER_DIR/repositories/" \
+    "$backup_dir" 1>&3
 }
 
-
 for hostname in $hostnames; do
   bm_start "$(basename $0) - $hostname"
   echo 1>&3
@@ -199,11 +198,11 @@ RULES
 + /*/nw/??/??/??/*/*.git/packed-refs
 RULES
 
-# Sync loose refs and reflogs. This must be performed before object data is
-# transferred to ensure that all referenced objects are included.
-echo 1>&3
-echo "* Transferring refs and reflogs ..."  1>&3
-rsync_repository_data $hostname:122 -z <<RULES
+  # Sync loose refs and reflogs. This must be performed before object data is
+  # transferred to ensure that all referenced objects are included.
+  echo 1>&3
+  echo "* Transferring refs and reflogs ..."  1>&3
+  rsync_repository_data $hostname:122 -z <<RULES
 - /__*__/
 - /info/