Merge pull request #369 from github/lildude/optimise-cluster-pages-restore

Optimise route generation and finalisation during cluster restores of pages

Author: lildude

bin/ghe-restore

@@ -332,7 +332,12 @@ fi
 
 if $cluster; then
   echo "Restoring GitHub Pages into DPages..."
-  ghe-restore-pages-dpages "$GHE_HOSTNAME" 1>&3
+  if ghe-ssh "$GHE_HOSTNAME" test -f /data/github/current/script/dpages-cluster-restore-routes; then
+    ghe_verbose "* Using ghe-restore-pages-dpages-ng to restore"
+    ghe-restore-pages-dpages-ng "$GHE_HOSTNAME" 1>&3
+  else
+    ghe-restore-pages-dpages "$GHE_HOSTNAME" 1>&3
+  fi
 else
   echo "Restoring GitHub Pages ..."
   ghe-restore-pages-${GHE_BACKUP_STRATEGY} "$GHE_HOSTNAME" 1>&3

share/github-backup-utils/ghe-restore-pages-dpages-ng

@@ -0,0 +1,129 @@
+#!/usr/bin/env bash
+#/ Usage: ghe-restore-pages-dpages-ng <host>
+#/ Restore repositories fron an rsync snapshot of all Git repository data to a GitHub cluster.
+#/
+#/ Note: This script typically isn't called directly. It's invoked by the
+#/ ghe-restore command when restoring into a cluster.
+set -e
+
+# Bring in the backup configuration
+. $( dirname "${BASH_SOURCE[0]}" )/ghe-backup-config
+
+# Show usage and bail with no arguments
+[ -z "$*" ] && print_usage
+
+bm_start "$(basename $0)"
+
+# Grab host arg
+GHE_HOSTNAME="$1"
+
+# The snapshot to restore should be set by the ghe-restore command but this lets
+# us run this script directly.
+: ${GHE_RESTORE_SNAPSHOT:=current}
+
+# Find the pages to restore
+pages_paths=$(cd $GHE_DATA_DIR/$GHE_RESTORE_SNAPSHOT/ && find pages -mindepth 5 -maxdepth 5 | cut -d / -f2-)
+
+# No need to restore anything, early exit
+if [ -z "$pages_paths" ]; then
+  echo "Warning: Pages backup missing. Skipping ..."
+  exit 0
+fi
+
+# Perform a host-check and establish GHE_REMOTE_XXX variables.
+ghe_remote_version_required "$GHE_HOSTNAME"
+
+# Split host:port into parts
+port=$(ssh_port_part "$GHE_HOSTNAME")
+host=$(ssh_host_part "$GHE_HOSTNAME")
+
+# Add user / -l option
+user="${host%@*}"
+[ "$user" = "$host" ] && user="admin"
+
+hostnames=$(ghe-cluster-nodes "$GHE_HOSTNAME" "pages-server")
+
+tempdir=$(mktemp -d -t backup-utils-restore-XXXXXX)
+ghe-ssh "$GHE_HOSTNAME" -- mkdir -p $tempdir
+ssh_config_file=$(mktemp -t cluster-backup-restore-XXXXXX)
+ghe-ssh-config "$GHE_HOSTNAME" "$hostnames" > "$ssh_config_file"
+opts="$GHE_EXTRA_SSH_OPTS -o UserKnownHostsFile=/dev/null -o StrictHostKeyChecking=no -o PasswordAuthentication=no"
+tmp_list=$tempdir/tmp_list
+routes_list=$tempdir/routes_list
+
+cleanup() {
+  rm -rf $tempdir
+  ghe-ssh "$GHE_HOSTNAME" -- rm -rf $tempdir
+  true
+}
+
+trap 'cleanup' EXIT
+
+# Build a list of pages paths to send to the server to calculate
+# the restore routes, something like:
+#
+# 5/d3/d9/44/10
+# 0/02/e7/4f/27
+# 4/c1/6a/53/31
+# 3/34/17/3c/30
+# 6/6e/a9/ab/29
+# ...
+#
+# One pages path per line.
+bm_start "$(basename $0) - Building pages list"
+OLDIFS=$IFS; IFS=$'\n'
+for path in $pages_paths; do
+   ghe_verbose "Adding path $path to the list of pages to send"
+   echo $path
+done > $tmp_list
+IFS=$OLDIFS
+bm_end "$(basename $0) - Building pages list"
+
+bm_start "$(basename $0) - Transferring pages list"
+cat $tmp_list | ghe-ssh "$GHE_HOSTNAME" -- sponge $tmp_list
+bm_end "$(basename $0) - Transferring pages list"
+
+# The server returns a list of routes:
+#
+# 5/d3/d9/44/10 pages-server-1 pages-server-2 pages-server-3
+# 0/02/e7/4f/27 pages-server-1 pages-server-3 pages-server-4
+# 4/c1/6a/53/31 pages-server-2 pages-server-3 pages-server-4
+# 3/34/17/3c/30 pages-server-4 pages-server-2 pages-server-1
+# 6/6e/a9/ab/29 pages-server-3 pages-server-2 pages-server-1
+# ...
+#
+# One route per line.
+bm_start "$(basename $0) - Generating routes"
+echo "cat $tmp_list | github-env ./bin/dpages-cluster-restore-routes > $routes_list" | ghe-ssh "$GHE_HOSTNAME" -- /bin/bash
+bm_end "$(basename $0) - Generating routes"
+
+bm_start "$(basename $0) - Transferring routes"
+ghe-ssh "$GHE_HOSTNAME" -- cat $routes_list > $routes_list
+bm_end "$(basename $0) - Transferring routes"
+
+bm_start "$(basename $0) - Processing routes"
+cat $routes_list | awk -v tempdir="$tempdir" '{ for(i=2;i<=NF;i++){ print $1 > (tempdir"/"$i".rsync") }}'
+bm_end "$(basename $0) - Processing routes"
+
+bm_start "$(basename $0) - Restoring pages"
+for file_list in $tempdir/*.rsync; do
+  server=$(basename $file_list .rsync)
+  ghe_verbose "* Transferring Pages to $server"
+  ghe-rsync -avrHR --delete \
+    -e "ssh -q $opts -p $port -F $ssh_config_file -l $user" \
+    --rsync-path="sudo -u git rsync" \
+    --files-from=$file_list \
+    "$GHE_DATA_DIR/$GHE_RESTORE_SNAPSHOT/pages/./" \
+    "$server:$GHE_REMOTE_DATA_USER_DIR/pages/" 1>&3
+  done
+bm_end "$(basename $0) - Restoring pages"
+
+bm_start "$(basename $0) - Finalizing routes"
+ghe-ssh "$GHE_HOSTNAME" -- /bin/bash >&3 <<EOF
+  split -l 1000 -d $routes_list $tempdir/chunk
+  chunks=\$(find $tempdir/ -name chunk\*)
+  parallel -i /bin/sh -c "cat {} | github-env ./bin/dpages-cluster-restore-finalize" -- \$chunks
+EOF
+bm_end "$(basename $0) - Finalizing routes"
+
+bm_end "$(basename $0)"