Merge pull request #201 from github/backup-utils-2.6

Bump version: 2.6.0

Author: rubiojr

bin/ghe-backup

@@ -173,6 +173,10 @@ if [ "$GHE_VERSION_MAJOR" -ge 2 ]; then
         echo "Backing up asset attachments ..."
         ghe-backup-alambic-cluster ||
         failures="$failures alambic_assets"
+
+        echo "Backing up custom Git hooks ..."
+        ghe-backup-git-hooks-cluster ||
+        failures="$failures alambic_assets"
     else
         echo "Backing up asset attachments ..."
         ghe-backup-userdata alambic_assets ||
@@ -185,6 +189,10 @@ if [ "$GHE_VERSION_MAJOR" -ge 2 ]; then
         echo "Backing up hook deliveries ..."
         ghe-backup-userdata hookshot ||
         failures="$failures hookshot"
+
+        echo "Backing up custom Git hooks ..."
+        ghe-backup-userdata git-hooks ||
+        failures="$failures git-hooks"
     fi
 fi
 

bin/ghe-restore

@@ -241,6 +241,7 @@ ghe-ssh "$GHE_HOSTNAME" -- 'ghe-import-authorized-keys' < "$GHE_RESTORE_SNAPSHOT
 if $cluster; then
   echo "Restoring storage data ..."
   ghe-restore-alambic-cluster "$GHE_HOSTNAME" 1>&3
+  ghe-restore-git-hooks-cluster "$GHE_HOSTNAME" 1>&3
 elif [ "$GHE_VERSION_MAJOR" -ge 2 ]; then
   echo "Restoring asset attachments ..."
   ghe-restore-userdata alambic_assets "$GHE_HOSTNAME" 1>&3
@@ -250,6 +251,9 @@ elif [ "$GHE_VERSION_MAJOR" -ge 2 ]; then
 
   echo "Restoring hook deliveries ..."
   ghe-restore-userdata hookshot "$GHE_HOSTNAME" 1>&3
+
+  echo "Restoring custom Git hooks ..."
+  ghe-restore-userdata git-hooks "$GHE_HOSTNAME" 1>&3
 fi
 
 if $cluster; then

debian/changelog

@@ -1,3 +1,12 @@
+github-backup-utils (2.6.0) UNRELEASED; urgency=medium
+
+  * Adds support for GitHub Enterprise 2.6
+  * Adds an extra supported location for the backup configuration #197
+  * New config option to check for corrupted repositories after the backup #195
+  * General improvements and bug fixes
+
+ -- Sergio Rubio <[email protected]>  Tue, 26 Apr 2016 18:03:01 +0200
+
 github-backup-utils (2.5.2) UNRELEASED; urgency=medium
 
   * New configuration variable: GHE_CREATE_DATA_DIR #186

share/github-backup-utils/ghe-backup-git-hooks-cluster

@@ -0,0 +1,88 @@
+#!/usr/bin/env bash
+#/ Usage: ghe-backup-git-hooks-cluster
+#/ Take an online, incremental snapshot of custom Git hooks configuration.
+#/
+#/ Note: This command typically isn't called directly. It's invoked by
+#/ ghe-backup when the cluster strategy is used.
+set -e
+
+# Bring in the backup configuration
+cd $(dirname "$0")/../..
+. share/github-backup-utils/ghe-backup-config
+
+# Split host:port into parts
+port=$(ssh_port_part "$GHE_HOSTNAME")
+host=$(ssh_host_part "$GHE_HOSTNAME")
+
+# Add user / -l option
+user="${host%@*}"
+[ "$user" = "$host" ] && user="admin"
+
+backup_dir="$GHE_SNAPSHOT_DIR/git-hooks"
+
+# Location of last good backup for rsync --link-dest
+backup_current="$GHE_DATA_DIR/current/git-hooks"
+
+# Verify rsync is available.
+if ! rsync --version 1>/dev/null 2>&1; then
+    echo "Error: rsync not found." 1>&2
+    exit 1
+fi
+
+# Perform a host-check and establish GHE_REMOTE_XXX variables.
+ghe_remote_version_required "$host"
+
+# Generate SSH config for forwarding
+
+config=""
+
+# git server hostnames
+hostnames=$(ghe_cluster_online_nodes "git-server")
+for hostname in $hostnames; do
+  config="$config
+Host $hostname
+  ProxyCommand ssh -q $GHE_EXTRA_SSH_OPTS -p $port $user@$host nc.openbsd %h %p
+  StrictHostKeyChecking=no
+"
+done
+
+config_file=$(mktemp -t cluster-backup-restore-XXXXXX)
+echo "$config" > "$config_file"
+
+opts="$GHE_EXTRA_SSH_OPTS -o UserKnownHostsFile=/dev/null -o StrictHostKeyChecking=no -o PasswordAuthentication=no"
+
+# Removes the remote sync-in-progress file on exit, re-enabling GC operations
+# on the remote instance.
+cleanup() {
+  rm -f $config_file
+}
+trap 'cleanup' EXIT
+trap 'exit $?' INT # ^C always terminate
+
+# If we have a previous increment, avoid transferring existing files via rsync's
+# --link-dest support. This also decreases physical space usage considerably.
+if [ -d "$backup_current" ]; then
+    link_dest="--link-dest=../../current/git-hooks"
+fi
+
+# Transfer Git hooks data from a GitHub instance to the current snapshot
+# directory, using a previous snapshot to avoid transferring files that have
+# already been transferred. A set of rsync filter rules are provided on stdin
+# for each invocation.
+rsync_git_hooks_data () {
+    port=$(ssh_port_part "$1")
+    host=$(ssh_host_part "$1")
+
+    shift
+    ghe-rsync -a \
+        -e "ssh -q $opts -p $port -F $config_file -l $user" $link_dest \
+        --rsync-path='sudo -u git rsync' \
+        "$host:$GHE_REMOTE_DATA_USER_DIR/git-hooks/" \
+        "$backup_dir" 1>&3
+}
+
+for hostname in $hostnames; do
+  ghe-ssh -F $config_file "$hostname:122" -- "[ -d '$GHE_REMOTE_DATA_USER_DIR/git-hooks' ]" || exit 0
+  # We can stop after the first successful sync since hooks are the same everywhere
+  rsync_git_hooks_data $hostname:122 && break
+done

share/github-backup-utils/ghe-restore-git-hooks-cluster

@@ -0,0 +1,77 @@
+#!/usr/bin/env bash
+#/ Usage: ghe-restore-git-hooks-cluster <host>
+#/ Restore custom Git hooks data from an rsync snapshot
+#/
+#/ Note: This command typically isn't called directly. It's invoked by
+#/ ghe-restore when the cluster strategy is used.
+set -e
+
+# Bring in the backup configuration
+cd $(dirname "$0")/../..
+. share/github-backup-utils/ghe-backup-config
+
+# Show usage and bail with no arguments
+[ -z "$*" ] && print_usage
+
+# Grab host arg
+GHE_HOSTNAME="$1"
+
+# Perform a host-check and establish GHE_REMOTE_XXX variables.
+ghe_remote_version_required "$GHE_HOSTNAME"
+
+# The snapshot to restore should be set by the ghe-restore command but this lets
+# us run this script directly.
+: ${GHE_RESTORE_SNAPSHOT:=current}
+
+# Generate SSH config for forwarding
+config=""
+
+# Split host:port into parts
+port=$(ssh_port_part "$GHE_HOSTNAME")
+host=$(ssh_host_part "$GHE_HOSTNAME")
+
+# Add user / -l option
+user="${host%@*}"
+[ "$user" = "$host" ] && user="admin"
+
+hostnames=$(ghe_cluster_online_nodes "git-server")
+for hostname in $hostnames; do
+  config="$config
+Host $hostname
+  ServerAliveInterval 60
+  ProxyCommand ssh -q $GHE_EXTRA_SSH_OPTS -p $port $user@$host nc.openbsd %h %p
+  StrictHostKeyChecking=no
+"
+done
+
+config_file=$(mktemp -t cluster-backup-restore-XXXXXX)
+echo "$config" > "$config_file"
+
+opts="$GHE_EXTRA_SSH_OPTS -o UserKnownHostsFile=/dev/null -o StrictHostKeyChecking=no -o PasswordAuthentication=no"
+
+cleanup() {
+  for pid in $(jobs -p); do
+    kill -KILL $pid > /dev/null 2>&1 || true
+  done
+  rm -f $config_file
+}
+trap 'cleanup' INT TERM EXIT
+
+if [ -d "$GHE_DATA_DIR/$GHE_RESTORE_SNAPSHOT/git-hooks" ]; then
+  for hostname in $hostnames; do
+    ghe-rsync -aHR --delete \
+    -e "ssh -q $opts -p $port -F $config_file -l $user" \
+    --rsync-path="sudo -u git rsync" \
+    "$GHE_DATA_DIR/$GHE_RESTORE_SNAPSHOT/git-hooks" \
+    "$route:$GHE_REMOTE_DATA_USER_DIR/git-hooks" &
+  done
+
+  for pid in $(jobs -p); do
+    wait $pid
+    ret_code=$?
+    if [ "$ret_code" != "0" ]; then
+      echo "$pid exited $ret_code"
+      exit $ret_code
+    fi
+  done
+fi

share/github-backup-utils/ghe-restore-repositories-dgit

@@ -1,6 +1,6 @@
 #!/usr/bin/env bash
 #/ Usage: ghe-restore-repositories-dgit <host>
-#/ Restore repositories fron an rsync snapshot of all Git repository data to a GitHub cluster.
+#/ Restore repositories from an rsync snapshot of all Git repository data to a GitHub cluster.
 #/
 #/ Note: This script typically isn't called directly. It's invoked by the
 #/ ghe-restore command when restoring into a cluster.

share/github-backup-utils/version

@@ -1 +1 @@
-2.5.2
+2.6.0

test/test-ghe-backup.sh

@@ -23,6 +23,11 @@ if [ "$GHE_VERSION_MAJOR" -ge 2 ]; then
     cd "$GHE_REMOTE_DATA_USER_DIR/hookshot"
     mkdir -p repository-123 repository-456
     touch repository-123/test.bpack repository-456/test.bpack
+
+    mkdir -p "$GHE_REMOTE_DATA_USER_DIR/git-hooks"
+    cd "$GHE_REMOTE_DATA_USER_DIR/git-hooks"
+    mkdir -p repository-123 repository-456
+    touch repository-123/script.sh repository-456/foo.sh
 fi
 
 # Create some fake alambic data in the remote data directory
@@ -122,6 +127,9 @@ begin_test "ghe-backup first snapshot"
         # verify all hookshot user data was transferred
         diff -ru "$GHE_REMOTE_DATA_USER_DIR/hookshot" "$GHE_DATA_DIR/current/hookshot"
 
+        # verify all git hooks data was transferred
+        diff -ru "$GHE_REMOTE_DATA_USER_DIR/git-hooks" "$GHE_DATA_DIR/current/git-hooks"
+
         # verify all alambic assets user data was transferred
         diff -ru "$GHE_REMOTE_DATA_USER_DIR/alambic_assets" "$GHE_DATA_DIR/current/alambic_assets"
     fi
@@ -197,6 +205,9 @@ begin_test "ghe-backup subsequent snapshot"
         # verify all hookshot user data was transferred
         diff -ru "$GHE_REMOTE_DATA_USER_DIR/hookshot" "$GHE_DATA_DIR/current/hookshot"
 
+        # verify all git hooks data was transferred
+        diff -ru "$GHE_REMOTE_DATA_USER_DIR/git-hooks" "$GHE_DATA_DIR/current/git-hooks"
+
         # verify all alambic assets user data was transferred
         diff -ru "$GHE_REMOTE_DATA_USER_DIR/alambic_assets" "$GHE_DATA_DIR/current/alambic_assets"
     fi
@@ -282,6 +293,19 @@ begin_test "ghe-backup empty hookshot directory"
 )
 end_test
 
+begin_test "ghe-backup empty git-hooks directory"
+(
+  set -e
+
+  rm -rf $GHE_REMOTE_DATA_USER_DIR/git-hooks/repository-*
+  rm -rf $GHE_DATA_DIR/current/git-hooks/repository-*
+  ghe-backup
+
+  # Check that the "--link-dest arg does not exist" message hasn't occurred.
+  [ ! "$(grep "[l]ink-dest arg does not exist" $TRASHDIR/out)" ]
+)
+end_test
+
 begin_test "ghe-backup fsck"
 (
   set -e

test/test-ghe-restore.sh

@@ -25,6 +25,11 @@ if [ "$GHE_VERSION_MAJOR" -ge 2 ]; then
     cd "$GHE_DATA_DIR/1/hookshot"
     mkdir -p repository-123 repository-456
     touch repository-123/test.bpack repository-456/test.bpack
+
+    mkdir -p "$GHE_DATA_DIR/1/git-hooks"
+    cd "$GHE_DATA_DIR/1/git-hooks"
+    mkdir -p repository-123 repository-456
+    touch repository-123/script.sh repository-456/foo.sh
 fi
 
 # Create some fake alambic data in the remote data directory
@@ -122,6 +127,9 @@ begin_test "ghe-restore into configured vm"
         # verify all hookshot user data was transferred
         diff -ru "$GHE_DATA_DIR/current/hookshot" "$GHE_REMOTE_DATA_USER_DIR/hookshot"
 
+        # verify all git hooks data was transferred
+        diff -ru "$GHE_DATA_DIR/current/git-hooks" "$GHE_REMOTE_DATA_USER_DIR/git-hooks"
+
         # verify all alambic assets user data was transferred
         diff -ru "$GHE_DATA_DIR/current/alambic_assets" "$GHE_REMOTE_DATA_USER_DIR/alambic_assets"
     fi
@@ -240,6 +248,9 @@ begin_test "ghe-restore -c into unconfigured vm"
         # verify all hookshot user data was transferred
         diff -ru "$GHE_DATA_DIR/current/hookshot" "$GHE_REMOTE_DATA_USER_DIR/hookshot"
 
+        # verify all git hooks data was transferred
+        diff -ru "$GHE_DATA_DIR/current/git-hooks" "$GHE_REMOTE_DATA_USER_DIR/git-hooks"
+
         # verify all alambic assets user data was transferred
         diff -ru "$GHE_DATA_DIR/current/alambic_assets" "$GHE_REMOTE_DATA_USER_DIR/alambic_assets"
     fi
@@ -299,6 +310,9 @@ begin_test "ghe-restore into unconfigured vm"
         # verify all hookshot user data was transferred
         diff -ru "$GHE_DATA_DIR/current/hookshot" "$GHE_REMOTE_DATA_USER_DIR/hookshot"
 
+        # verify all git hooks data was transferred
+        diff -ru "$GHE_DATA_DIR/current/git-hooks" "$GHE_REMOTE_DATA_USER_DIR/git-hooks"
+
         # verify all alambic assets user data was transferred
         diff -ru "$GHE_DATA_DIR/current/alambic_assets" "$GHE_REMOTE_DATA_USER_DIR/alambic_assets"
 
@@ -348,6 +362,9 @@ begin_test "ghe-restore with host arg"
         # verify all hookshot user data was transferred
         diff -ru "$GHE_DATA_DIR/current/hookshot" "$GHE_REMOTE_DATA_USER_DIR/hookshot"
 
+        # verify all git hooks data was transferred
+        diff -ru "$GHE_DATA_DIR/current/git-hooks" "$GHE_REMOTE_DATA_USER_DIR/git-hooks"
+
         # verify all alambic assets user data was transferred
         diff -ru "$GHE_DATA_DIR/current/alambic_assets" "$GHE_REMOTE_DATA_USER_DIR/alambic_assets"
     fi