Merge branch 'master' into snh/es-cluster-restore

Author: snh

bin/ghe-backup

@@ -69,7 +69,7 @@ if [ -f ../in-progress ]; then
   progress=$(cat ../in-progress)
   snapshot=$(echo "$progress" | cut -d ' ' -f 1)
   pid=$(echo "$progress" | cut -d ' ' -f 2)
-  if ! ps -p $pid -o command= | grep ghe-backup; then
+  if ! ps -p $pid | grep ghe-backup; then
     # We can safely remove in-progress, ghe-prune-snapshots
     # will clean up the failed backup.
     unlink ../in-progress

bin/ghe-restore

@@ -109,7 +109,7 @@ if $cluster; then
   snapshot_instance_version=$(cat $GHE_RESTORE_SNAPSHOT_PATH/version)
   if ! echo $snapshot_instance_version | \
     grep -Eq "v2\.[5-9]|v2\.[1-9][0-9]|v[3-9]|v[1-9][0-9]"; then
-    echo "Error: Snapshot must be from GitHub Enterprise v2.5.0 or above to be restored"
+    echo "Error: Snapshot must be from GitHub Enterprise v2.5.0 or above to be restored" >&2
     echo "       into a cluster (detected $snapshot_instance_version).  Aborting." >&2
     exit 1
   fi
@@ -123,6 +123,21 @@ if ghe-ssh "$GHE_HOSTNAME" -- \
   exit 1
 fi
 
+# Only allow restores of 2.9 and 2.10 snapshots that have run the audit log migration to 2.11 and above
+if ! $force; then
+  snapshot_instance_version=$(cat $GHE_RESTORE_SNAPSHOT_PATH/version)
+  snapshot_version_major=$(echo "${snapshot_instance_version#v}" | cut -f 1 -d .)
+  snapshot_version_minor=$(echo "$snapshot_instance_version"     | cut -f 2 -d .)
+  if ! test -f $GHE_RESTORE_SNAPSHOT_PATH/es-scan-complete && \
+     [ "$snapshot_version_major" -eq 2 ] && [ "$snapshot_version_minor" -lt 11 ] && \
+     [ "$GHE_VERSION_MAJOR" -eq 2 ] && [ "$GHE_VERSION_MINOR" -ge 11 ]; then
+        echo "Error: Snapshot must be from GitHub Enterprise v2.9 or v2.10 after running the" >&2
+        echo "       audit log migration, or from v2.11.0 or above." >&2
+        echo "Please see https://git.io/v5rCE for the audit log migration procedure." >&2
+        exit 1
+  fi
+fi
+
 # Prompt to verify the restore host given is correct. Restoring overwrites
 # important data on the destination appliance that cannot be recovered. This is
 # mostly to prevent accidents where the backup host is given to restore instead
@@ -339,6 +354,11 @@ else
   ghe-restore-es-${GHE_BACKUP_STRATEGY} "$GHE_HOSTNAME" 1>&3
 fi
 
+# Restore the audit log migration sentinel file, if it exists in the snapshot
+if test -f $GHE_RESTORE_SNAPSHOT_PATH/es-scan-complete; then
+  ghe-ssh "$GHE_HOSTNAME" -- "sudo touch $GHE_REMOTE_DATA_USER_DIR/common/es-scan-complete"
+fi
+
 # Restart an already running memcached to reset the cache after restore
 if [ "$GHE_VERSION_MAJOR" -ge 2 ]; then
     echo "Restarting memcached ..." 1>&3

debian/changelog

@@ -1,3 +1,12 @@
+github-backup-utils (2.11.0) UNRELEASED; urgency=medium
+
+  * Use calculated routes when backing up storage data from a cluster #318
+  * Add SSH multiplexing support #321
+  * Optimise route generation and finalisation during cluster restores #322
+  * Prefer the SSH port specified on the command line #324
+
+ -- Colin Seymour <[email protected]>  Wed, 13 Sep 2017 16:31:20 +0000
+
 github-backup-utils (2.10.0) UNRELEASED; urgency=medium
 
   * Include the user data directory in the benchmark name #311

script/cibuild

@@ -11,6 +11,7 @@ REMOTE_VERSIONS="
   2.0.0
   2.2.0
   2.5.0
+  2.11.0
 "
 
 # Enable verbose logging of ssh commands

share/github-backup-utils/ghe-backup-es-audit-log

@@ -31,6 +31,13 @@ if ! indices=$(ghe-ssh "$host" "curl -s \"localhost:$es_port/_cat/indices/audit_
   exit 1
 fi
 
+# Determine if the audit log migration has occurred or is needed.
+if echo 'set -o pipefail; ! test -e /data/user/common/es-scan-complete && test -f /usr/local/share/enterprise/run-audit-log-transitions.sh' | ghe-ssh "$host" /bin/bash; then
+  if echo 'set -o pipefail; echo n | /usr/local/share/enterprise/run-audit-log-transitions.sh > /dev/null 2>&1 && touch /data/user/common/es-scan-complete' | ghe-ssh "$host" /bin/bash; then
+    touch $GHE_SNAPSHOT_DIR/es-scan-complete
+  fi
+fi
+
 current_index=audit_log-$(ghe-ssh "$host" 'date +"%Y-%m"')
 
 for index in $indices; do

share/github-backup-utils/ghe-backup-es-rsync

@@ -96,4 +96,9 @@ ghe-rsync -avz \
     "$(ssh_host_part "$host"):$GHE_REMOTE_DATA_USER_DIR/elasticsearch/" \
     "$GHE_SNAPSHOT_DIR/elasticsearch" 1>&3
 
+# "Backup" audit log migration sentinel file
+if ghe-ssh "$host" -- "test -f $GHE_REMOTE_DATA_USER_DIR/common/es-scan-complete"; then
+  touch $GHE_SNAPSHOT_DIR/es-scan-complete
+fi
+
 bm_end "$(basename $0)"

share/github-backup-utils/ghe-ssh

@@ -58,7 +58,7 @@ if echo "$*" | grep "[|;]" >/dev/null || [ $(echo "$*" | wc -l) -gt 1 ]; then
 fi
 
 if [ -z "$GHE_DISABLE_SSH_MUX" ]; then
-  controlpath="$TMPDIR/.ghe-sshmux-$(echo -n "$user@$host:$port" | sha256sum | cut -c 1-8)"
+  controlpath="$TMPDIR/.ghe-sshmux-$(echo -n "$user@$host:$port" | git hash-object --stdin | cut -c 1-8)"
   opts="-o ControlMaster=auto -o ControlPath=\"$controlpath\" -o ControlPersist=10m -o ServerAliveInterval=10 $opts"
   # Workaround for https://bugzilla.mindrot.org/show_bug.cgi?id=1988
   [ -S $controlpath ] || ssh -f -p $port $opts -o BatchMode=yes "$host" -- /bin/true 1>/dev/null 2>&1 || true

share/github-backup-utils/ghe-ssh-config

@@ -25,7 +25,7 @@ proxy_user="${proxy_host%@*}"
 
 opts="$GHE_EXTRA_SSH_OPTS"
 
-[ -z "$GHE_DISABLE_SSH_MUX" ] && opts="-o ControlMaster=auto -o ControlPath=\"$TMPDIR/.ghe-sshmux-$(echo -n "$proxy_user@$proxy_host:$proxy_port" | sha256sum | cut -c 1-8)\" -o ControlPersist=10m -o ServerAliveInterval=10 $opts"
+[ -z "$GHE_DISABLE_SSH_MUX" ] && opts="-o ControlMaster=auto -o ControlPath=\"$TMPDIR/.ghe-sshmux-$(echo -n "$proxy_user@$proxy_host:$proxy_port" | git hash-object --stdin | cut -c 1-8)\" -o ControlPersist=10m -o ServerAliveInterval=10 $opts"
 
 for host in $hosts; do
   cat <<EOF

share/github-backup-utils/version

@@ -1 +1 @@
-2.10.0
+2.11.0

test/test-ghe-backup.sh

@@ -49,6 +49,9 @@ if [ "$GHE_VERSION_MAJOR" -ge 2 ]; then
 
     # Create a fake UUID
     echo "fake uuid" > "$GHE_REMOTE_DATA_USER_DIR/common/uuid"
+
+    # Create fake audit log migration sentinel file
+    touch "$GHE_REMOTE_DATA_USER_DIR/common/es-scan-complete"
 fi
 
 # Create some fake elasticsearch data in the remote data directory
@@ -154,6 +157,9 @@ begin_test "ghe-backup first snapshot"
 
         # check that ca certificates were backed up
         [ "$(cat "$GHE_DATA_DIR/current/ssl-ca-certificates.tar")" = "fake ghe-export-ssl-ca-certificates data" ]
+
+        # verify the audit log migration sentinel file has been created
+        [ -f "$GHE_DATA_DIR/current/es-scan-complete" ]
     fi
 
     # verify that ghe-backup wrote its version information to the host
@@ -241,6 +247,9 @@ begin_test "ghe-backup subsequent snapshot"
 
         # check that ca certificates were backed up
         [ "$(cat "$GHE_DATA_DIR/current/ssl-ca-certificates.tar")" = "fake ghe-export-ssl-ca-certificates data" ]
+
+        # verify the audit log migration sentinel file has been created
+        [ -f "$GHE_DATA_DIR/current/es-scan-complete" ]
     fi
 )
 end_test
@@ -344,6 +353,9 @@ begin_test "ghe-backup with relative data dir path"
 
         # check that ca certificates were backed up
         [ "$(cat "$GHE_DATA_DIR/current/ssl-ca-certificates.tar")" = "fake ghe-export-ssl-ca-certificates data" ]
+
+        # verify the audit log migration sentinel file has been created
+        [ -f "$GHE_DATA_DIR/current/es-scan-complete" ]
     fi
 
     # verify that ghe-backup wrote its version information to the host