Merge remote-tracking branch 'public/master'

Author: dbussink

.travis.yml

@@ -7,6 +7,7 @@ matrix:
         - brew install gnu-tar
         - brew install moreutils
         - brew install shellcheck
+        - brew install jq
       script: make test
     - os: linux
       dist: trusty
@@ -21,4 +22,5 @@ matrix:
             - debhelper
             - moreutils
             - fakeroot
+            - jq
       script: debuild -uc -us

bin/ghe-backup

@@ -119,12 +119,14 @@ if [ -f ../in-progress ]; then
   progress=$(cat ../in-progress)
   snapshot=$(echo "$progress" | cut -d ' ' -f 1)
   pid=$(echo "$progress" | cut -d ' ' -f 2)
-  if ! ps -p $pid | grep ghe-backup; then
+  if ! ps -p "$pid" >/dev/null 2>&1; then
     # We can safely remove in-progress, ghe-prune-snapshots
     # will clean up the failed backup.
     unlink ../in-progress
   else
-    echo "Error: backup process $pid of $GHE_HOSTNAME already in progress in snapshot $snapshot. Aborting." 1>&2
+    echo "Error: A backup of $GHE_HOSTNAME may still be running on PID $pid." 1>&2
+    echo "If PID $pid is not a process related to the backup utilities, please remove" 1>&2
+    echo "the $GHE_DATA_DIR/in-progress file and try again." 1>&2
     exit 1
   fi
 fi

bin/ghe-host-check

@@ -91,6 +91,17 @@ if [ -z "$version" ]; then
   exit 2
 fi
 
+# Block restoring snapshots to older releases of GitHub Enterprise
+if [ -n "$GHE_RESTORE_SNAPSHOT_PATH" ]; then
+  snapshot_version=$(cat $GHE_RESTORE_SNAPSHOT_PATH/version)
+  # shellcheck disable=SC2046 # Word splitting is required to populate the variables
+  read -r snapshot_version_major snapshot_version_minor _ <<<$(ghe_parse_version $snapshot_version)
+  if [ "$(version $GHE_REMOTE_VERSION)" -lt "$(version $snapshot_version_major.$snapshot_version_minor.0)" ]; then
+    echo "Error: Snapshot can not be restored to an older release of GitHub Enterprise." >&2
+    exit 1
+  fi
+fi
+
 if [ -z "$GHE_ALLOW_REPLICA_BACKUP" ]; then
   if [ "$(ghe-ssh $host -- cat $GHE_REMOTE_ROOT_DIR/etc/github/repl-state 2>/dev/null || true)" = "replica" ]; then
     echo "Error: high availability replica detected." 1>&2

bin/ghe-restore

@@ -258,12 +258,12 @@ if [ -s "$GHE_RESTORE_SNAPSHOT_PATH/uuid" ] && ! $CLUSTER; then
   echo "Restoring UUID ..."
   cat "$GHE_RESTORE_SNAPSHOT_PATH/uuid" |
   ghe-ssh "$GHE_HOSTNAME" -- "sudo sponge '$GHE_REMOTE_DATA_USER_DIR/common/uuid' 2>/dev/null"
-fi
+  ghe-ssh "$GHE_HOSTNAME" -- "sudo systemctl stop consul" || true
+  ghe-ssh "$GHE_HOSTNAME" -- "sudo rm -rf /data/user/consul/raft"
+  fi
 
 echo "Restoring MySQL database ..."
-bm_start "ghe-import-mysql"
-gzip -dc "$GHE_RESTORE_SNAPSHOT_PATH/mysql.sql.gz" | ghe-ssh "$GHE_HOSTNAME" -- 'ghe-import-mysql'
-bm_end "ghe-import-mysql"
+ghe-restore-mysql "$GHE_HOSTNAME" 1>&3
 
 echo "Restoring Redis database ..."
 bm_start "ghe-import-redis"
@@ -273,17 +273,21 @@ bm_end "ghe-import-redis"
 # Unified and enhanced restore method to 2.13.0 and newer
 if $CLUSTER || [ "$(version $GHE_REMOTE_VERSION)" -ge "$(version 2.13.0)" ]; then
   echo "Restoring Git repositories ..."
-  ghe-restore-repositories "$GHE_HOSTNAME" 1>&3
+  ghe-restore-repositories "$GHE_HOSTNAME"
 
   echo "Restoring Gists ..."
-  ghe-restore-repositories-gist "$GHE_HOSTNAME" 1>&3
+  ghe-restore-repositories-gist "$GHE_HOSTNAME"
+
+  echo "Restoring GitHub Pages ..."
+  ghe-restore-pages "$GHE_HOSTNAME" 1>&3
 else
   echo "Restoring Git repositories and Gists ..."
   ghe-restore-repositories-rsync "$GHE_HOSTNAME" 1>&3
+
+  echo "Restoring GitHub Pages ..."
+  ghe-restore-pages-rsync "$GHE_HOSTNAME" 1>&3
 fi
 
-echo "Restoring GitHub Pages ..."
-ghe-restore-pages "$GHE_HOSTNAME" 1>&3
 
 echo "Restoring SSH authorized keys ..."
 ghe-ssh "$GHE_HOSTNAME" -- 'ghe-import-authorized-keys' < "$GHE_RESTORE_SNAPSHOT_PATH/authorized-keys.json" 1>&3
@@ -341,6 +345,23 @@ else
   fi
 fi
 
+# Clean up all stale replicas on configured instances.
+if ! $CLUSTER && $instance_configured; then
+  restored_uuid=$(cat $GHE_RESTORE_SNAPSHOT_PATH/uuid)
+  other_nodes=$(echo "
+    set -o pipefail; \
+    ghe-spokes server show --json \
+    | jq -r '.[] | select(.host | contains(\"git-server\")).host' \
+    | sed 's/^git-server-//g' \
+    | ( grep -F -x -v \"$restored_uuid\" || true )" \
+  | ghe-ssh "$GHE_HOSTNAME" -- /bin/bash)
+  if [ -n "$other_nodes" ]; then
+    echo "Cleaning up stale nodes ..."
+    for uuid in $other_nodes; do
+      ghe-ssh "$GHE_HOSTNAME" -- "/usr/local/share/enterprise/ghe-cluster-cleanup-node $uuid" 1>&3
+    done
+  fi
+fi
 
 # Update the remote status to "complete". This has to happen before importing
 # ssh host keys because subsequent commands will fail due to the host key

debian/changelog

@@ -1,3 +1,42 @@
+github-backup-utils (2.14.3) UNRELEASED; urgency=medium
+
+  * Improve multi-platform detection of simultaneous ghe-backup runs #435
+
+ -- Colin Seymour <[email protected]>  Tue, 11 Sep 2018 17:03:42 +0000
+
+github-backup-utils (2.14.2) UNRELEASED; urgency=medium
+
+  * Capture and display repository/gist warnings during cluster restore #423
+  * Use remote tempdir when finalizing Pages routes #424
+  * Use old rsync restore method for pages prior to 2.13 #426
+
+ -- Colin Seymour <[email protected]>  Tue, 21 Aug 2018 13:57:20 +0000
+
+github-backup-utils (2.14.1) UNRELEASED; urgency=medium
+
+  * Don't fail a backup if the Management Console password isn't set #416
+  * Fix permissions issues when repeat restoring to configured cluster instance #417
+  * Add missing dependencies to debian packaging #418
+  * Prevent restoring snapshots to older releases #420
+
+ -- Colin Seymour <[email protected]>  Tue, 07 Aug 2018 16:00:36 +0000
+
+github-backup-utils (2.14.0) UNRELEASED; urgency=medium
+
+  * Disable pager and context when running git-diff #411
+  * Optimise hookshot and audit log backups and restores and MySQL restores #413
+
+ -- Colin Seymour <[email protected]>  Thu, 12 Jul 2018 15:11:11 +0000
+
+github-backup-utils (2.13.2) UNRELEASED; urgency=medium
+
+  * Treat missing repository networks, gists, and storage objects as a non-critical error  #386
+  * Clean up stale HA nodes on restore #396
+  * Cleanup all SSH muxes in a non blocking way #402
+  * Raise an error if the current symlink doesn't exist when attempting to restore it #404
+
+ -- Colin Seymour <[email protected]>  Fri, 22 Jun 2018 10:08:22 +0000
+
 github-backup-utils (2.13.1) UNRELEASED; urgency=medium
 
   * Retry with the admin ssh port on network unreachable too. #377

debian/control

@@ -3,11 +3,11 @@ Maintainer: Twan Wolthof <[email protected]>
 Section: misc
 Priority: optional
 Standards-Version: 3.9.2
-Build-Depends: debhelper (>= 9), git, devscripts
+Build-Depends: debhelper (>= 9), git, devscripts, moreutils, jq
 
 Package: github-backup-utils
 Architecture: any
-Depends: ${misc:Depends}, rsync (>= 2.6.4)
+Depends: ${misc:Depends}, rsync (>= 2.6.4), moreutils, jq
 Description: Backup and recovery utilities for GitHub Enterprise
  The backup utilities implement a number of advanced capabilities for backup
  hosts, built on top of the backup and restore features already included in

docs/getting-started.md

@@ -12,10 +12,13 @@
     backup and restore GitHub Enterprise 2.10 and earlier.
 
  2. Copy the [`backup.config-example`][3] file to `backup.config` and modify as
-    necessary. The `GHE_HOSTNAME` value must be set to the GitHub Enterprise
+    necessary. The `GHE_HOSTNAME` value must be set to the primary GitHub Enterprise
     host name. Additional options are available and documented in the
     configuration file but none are required for basic backup functionality.
 
+    As the data on a High Availability replica may be in a transient state at the time of backup,
+    Backup Utilities should not be used to backup data from a High Availability replica.
+
     * Backup Utilities will attempt to load the backup configuration from the following
       locations, in this order:
 

script/cibuild

@@ -7,8 +7,8 @@ set -e
 # conditional logic based on the remote version. Running the suite against
 # different major versions ensures we're covering these conditional paths.
 REMOTE_VERSIONS="
-  2.11.0
-  2.13.0
+  2.12.0
+  2.14.0
 "
 
 # Enable verbose logging of ssh commands

script/release

@@ -205,6 +205,10 @@ def bump_version(new_version, min_version = nil, path = 'share/github-backup-uti
     content = File.read('bin/ghe-host-check')
     new_content = content.gsub(/supported_minimum_version="[0-9]\.[0-9]+\.0"/, "supported_minimum_version=\"#{min_version}\"")
     File.open('bin/ghe-host-check', 'w') {|file| file.puts new_content }
+
+    content = File.read('test/testlib.sh')
+    new_content = content.gsub(/GHE_TEST_REMOTE_VERSION:=[0-9]\.[0-9]+\.0/,"GHE_TEST_REMOTE_VERSION:=#{new_version}")
+    File.open('test/testlib.sh', 'w') {|file| file.puts new_content }
   end
 end
 
@@ -384,6 +388,7 @@ if $PROGRAM_NAME == __FILE__
       puts `git diff --color`
       `git checkout -- share/github-backup-utils/version`
       `git checkout -- bin/ghe-host-check`
+      `git checkout -- test/testlib.sh`
       exit
     end
 
@@ -402,7 +407,8 @@ if $PROGRAM_NAME == __FILE__
 
     puts 'Updating changelog...'
     update_changelog release_changes, DEB_PKG_NAME, version
-    release_body = "Includes general improvements, bug fixes and support for GitHub Enterprise v#{version}"
+    release_body = "Includes general improvements & bug fixes"
+    release_body += " and support for GitHub Enterprise v#{version}" unless min_version.nil?
     release_changes.each do |c|
       release_body += "\n* #{c}"
     end

share/github-backup-utils/ghe-backup-config

@@ -80,7 +80,7 @@ fi
 # otherwise, redirect it to /dev/null. Write verbose output to fd 3.
 if [ -n "$GHE_VERBOSE" ]; then
   if [ -n "$GHE_VERBOSE_LOG" ]; then
-    exec 3<> "$GHE_VERBOSE_LOG"
+    exec 3>> "$GHE_VERBOSE_LOG"
   else
     exec 3>&1
   fi