Merge remote-tracking branch 'public/master' into merge-public-into-private

Author: mckern

Dockerfile.alpine

@@ -8,7 +8,8 @@ RUN apk --update --no-cache add \
     git \
     bash \
     gawk \
-    procps
+    procps \
+    coreutils
 
 WORKDIR /backup-utils
 ADD https://github.com/github/backup-utils/archive/stable.tar.gz /

RELEASING.md

@@ -14,11 +14,12 @@ Only repo administrator is allowed to run the release script, otherwise it will
 
 Prior to making a release,
 
+1. Sync any changes that have been merged to backup-utils-private into this repository.
 1. Go through the list of open pull requests and merge any that are ready for merging.
-2. Go through the list of closed pull requests since the last release and ensure those that should be included in the release notes:
+1. Go through the list of closed pull requests since the last release and ensure those that should be included in the release notes:
   - have a "bug", "enhancement" or "feature" label,
   - have a title that clearly describes the changes in that pull request. Reword if necessary.
-3. Perform a dry run (add `--dry-run` to one of the commands below) and verify the version strings are going to be changed and verify the release notes.
+1. Perform a dry run (add `--dry-run` to one of the commands below) and verify the version strings are going to be changed and verify the release notes.
 
 ## Automatic Process from chatops (internal to GitHub only)
 
@@ -62,3 +63,4 @@ Immediately after making a release using one of the methods above, verify the re
 - release has the notes you expect to see,
 - asset download links for the latest release at https://github.com/github/backup-utils/releases all download the correct version of Backup Utilities,
 - the stable branch is inline with master - https://github.com/github/backup-utils/compare/stable...master.
+- sync this repository to backup-utils-private

bin/ghe-host-check

@@ -15,27 +15,27 @@ set -e
 
 while true; do
   case "$1" in
-    -h|--help)
-      export GHE_SHOW_HELP=true
-      shift
-      ;;
-    --version)
-      export GHE_SHOW_VERSION=true
-      shift
-      ;;
-    -*)
-      echo "Error: invalid argument: '$1'" 1>&2
-      exit 1
-      ;;
-    *)
-      break
-      ;;
+  -h | --help)
+    export GHE_SHOW_HELP=true
+    shift
+    ;;
+  --version)
+    export GHE_SHOW_VERSION=true
+    shift
+    ;;
+  -*)
+    echo "Error: invalid argument: '$1'" 1>&2
+    exit 1
+    ;;
+  *)
+    break
+    ;;
   esac
 done
 
 # Bring in the backup configuration
 # shellcheck source=share/github-backup-utils/ghe-backup-config
-. "$( dirname "${BASH_SOURCE[0]}" )/../share/github-backup-utils/ghe-backup-config"
+. "$(dirname "${BASH_SOURCE[0]}")/../share/github-backup-utils/ghe-backup-config"
 
 # Use the host provided on the command line if provided, or fallback on the
 # $GHE_HOSTNAME configured in backup.config when not present.
@@ -59,31 +59,48 @@ set -e
 
 if [ $rc -ne 0 ]; then
   case $rc in
-    255)
-      if echo "$output" | grep -i "port 22: Network is unreachable\|port 22: connection refused\|port 22: no route to host\|ssh_exchange_identification: Connection closed by remote host\|Connection timed out during banner exchange\|port 22: Connection timed out" >/dev/null; then
-        exec "$(basename $0)" "$hostname:122"
-      fi
-
+  255)
+    if echo "$output" | grep -i "port 22: Network is unreachable\|port 22: connection refused\|port 22: no route to host\|ssh_exchange_identification: Connection closed by remote host\|Connection timed out during banner exchange\|port 22: Connection timed out" >/dev/null; then
+      exec "$(basename $0)" "$hostname:122"
+    fi
+
+    echo "$output" 1>&2
+    echo "Error: ssh connection with '$host' failed" 1>&2
+    echo "Note that your SSH key needs to be setup on $host as described in:" 1>&2
+    echo "* https://enterprise.github.com/help/articles/adding-an-ssh-key-for-shell-access" 1>&2
+    ;;
+  101)
+    echo "Error: couldn't read GitHub Enterprise Server fingerprint on '$host' or this isn't a GitHub appliance." 1>&2
+    ;;
+  1)
+    if [ "${port:-22}" -eq 22 ] && echo "$output" | grep "use port 122" >/dev/null; then
+      exec "$(basename $0)" "$hostname:122"
+    else
       echo "$output" 1>&2
-      echo "Error: ssh connection with '$host' failed" 1>&2
-      echo "Note that your SSH key needs to be setup on $host as described in:" 1>&2
-      echo "* https://enterprise.github.com/help/articles/adding-an-ssh-key-for-shell-access" 1>&2
-      ;;
-    101)
-      echo "Error: couldn't read GitHub Enterprise Server fingerprint on '$host' or this isn't a GitHub appliance." 1>&2
-      ;;
-    1)
-      if [ "${port:-22}" -eq 22 ] && echo "$output" | grep "use port 122" >/dev/null; then
-        exec "$(basename $0)" "$hostname:122"
-      else
-        echo "$output" 1>&2
-      fi
-      ;;
+    fi
+    ;;
 
   esac
   exit $rc
 fi
 
+CLUSTER=false
+if ghe-ssh "$host" -- \
+  "[ -f '$GHE_REMOTE_ROOT_DIR/etc/github/cluster' ]"; then
+  CLUSTER=true
+fi
+
+# ensure all nodes in the cluster are running the same version
+if "$CLUSTER"; then
+  node_version_list=$(ghe-ssh "$host" ghe-cluster-each -- ghe-version)
+  distinct_versions=$(echo "$node_version_list" | awk '{split($0, a, ":"); print a[2]}' | awk '{print $4}' | uniq | wc -l)
+  if [ "$distinct_versions" -ne 1 ]; then
+    echo "$node_version_list" 1>&2
+    echo "Error: Not all nodes are running the same version! Please ensure all nodes are running the same version before using backup-utils." 1>&2
+    exit 1
+  fi
+fi
+
 version=$(echo "$output" | grep "GitHub Enterprise" | awk '{print $NF}')
 
 if [ -z "$version" ]; then
@@ -113,7 +130,7 @@ fi
 
 # backup-utils 2.13 onwards limits support to the current and previous two releases
 # of GitHub Enterprise Server.
-supported_minimum_version="2.19.0"
+supported_minimum_version="2.21.0"
 
 if [ "$(version $version)" -ge "$(version $supported_minimum_version)" ]; then
   supported=1

bin/ghe-restore

@@ -249,16 +249,32 @@ if $CLUSTER; then
     ghe_verbose "* Warning: Failed to stop cron on one or more nodes"
   fi
 
-  if ! ghe-ssh "$GHE_HOSTNAME" -- "ghe-cluster-each -- sudo service github-timerd stop"; then
-    ghe_verbose "* Warning: Failed to stop github-timerd on one or more nodes"
+  if [ "$GHE_VERSION_MAJOR" -eq "3" ]; then
+    if ghe-ssh "$GHE_HOSTNAME" -- "systemctl -q is-active nomad && nomad job status --short github-timerd &>/dev/null"; then
+      if ! ghe-ssh "$GHE_HOSTNAME" -- "sudo nomad stop github-timerd 1>/dev/null"; then
+        ghe_verbose "* Warning: Failed to stop github-timerd on one or more nodes"
+      fi
+    fi
+  else
+    if ! ghe-ssh "$GHE_HOSTNAME" -- "ghe-cluster-each -- sudo service github-timerd stop"; then
+      ghe_verbose "* Warning: Failed to stop github-timerd on one or more nodes"
+    fi
   fi
 else
   if ! ghe-ssh "$GHE_HOSTNAME" -- "sudo service cron stop"; then
     ghe_verbose "* Warning: Failed to stop cron"
   fi
 
-  if ! ghe-ssh "$GHE_HOSTNAME" -- "sudo service github-timerd stop"; then
-    ghe_verbose "* Warning: Failed to stop github-timerd"
+  if [ "$GHE_VERSION_MAJOR" -eq "3" ]; then
+    if ghe-ssh "$GHE_HOSTNAME" -- "systemctl -q is-active nomad && nomad job status --short github-timerd &>/dev/null"; then
+      if ! ghe-ssh "$GHE_HOSTNAME" -- "sudo nomad stop github-timerd 1>/dev/null"; then
+        ghe_verbose "* Warning: Failed to stop github-timerd"
+      fi
+    fi
+  else
+    if ! ghe-ssh "$GHE_HOSTNAME" -- "sudo service github-timerd stop"; then
+      ghe_verbose "* Warning: Failed to stop github-timerd"
+    fi
   fi
 fi
 
@@ -392,9 +408,19 @@ echo "sudo restart -q memcached 2>/dev/null || true" |
 # config run to perform data migrations.
 if $CLUSTER; then
   echo "Configuring cluster ..."
+  if [ "$GHE_VERSION_MAJOR" -eq "3" ]; then
+    ghe-ssh "$GHE_HOSTNAME" -- "ghe-cluster-nomad-cleanup" 1>&3 2>&3
+  elif [ "$GHE_VERSION_MAJOR" -eq "2" ] && [ "$GHE_VERSION_MINOR" -eq "22" ]; then
+    ghe-ssh "$GHE_HOSTNAME" -- "ghe-cluster-each -- /usr/local/share/enterprise/ghe-nomad-cleanup" 1>&3 2>&3
+  fi
   ghe-ssh "$GHE_HOSTNAME" -- "ghe-cluster-config-apply" 1>&3 2>&3
 elif $instance_configured; then
   echo "Configuring appliance ..."
+  if [ "$GHE_VERSION_MAJOR" -eq "3" ]; then
+    ghe-ssh "$GHE_HOSTNAME" -- "ghe-nomad-cleanup" 1>&3 2>&3
+  elif [ "$GHE_VERSION_MAJOR" -eq "2" ] && [ "$GHE_VERSION_MINOR" -eq "22" ]; then
+    ghe-ssh "$GHE_HOSTNAME" -- "/usr/local/share/enterprise/ghe-nomad-cleanup" 1>&3 2>&3
+  fi
   ghe-ssh "$GHE_HOSTNAME" -- "ghe-config-apply" 1>&3 2>&3
 fi
 

debian/changelog

@@ -1,3 +1,33 @@
+github-backup-utils (3.0.0) UNRELEASED; urgency=medium
+
+  * Fix restoring the password pepper for already configured instances #683
+
+ -- Michael Dang <[email protected]>  Tue, 16 Feb 2021 22:32:25 +0000
+
+github-backup-utils (3.0.0.rc1) UNRELEASED; urgency=medium
+
+  * Cleanup nomad container when restore for 2.22 #670
+  * Use ghe-cluster-nomad-cleanup for cluster mode #663
+  * Only run ghe-nomad-cleanup in 3.0 #662
+  * Revert backup-utils gitHub env and a few more fixes #661
+  * Note how to test filesystem symlink / hardlink support #660
+  * stop github-timerd based on its running environment #659
+  * Backup and restore password pepper #656
+  * github-env -> github-env-dispatch #654
+  * Rename redis-cli to ghe-redis-cli #639
+
+ -- Michael Dang <[email protected]>  Thu, 14 Jan 2021 21:17:53 +0000
+
+github-backup-utils (2.22.0) UNRELEASED; urgency=medium
+
+  * Added basic timing around the ghe-restore process #625
+  * Improve python3 & finding moreutils parallel  #627
+  * Turn off POSIX for ghe-backup-config #632
+  * Add parallelized restore capability to ghe-restore-storage #635
+  * Update backup-utils for new features in 2.22 #641
+
+ -- [email protected]  Wed, 23 Sep 2020 15:48:54 +0000
+
 github-backup-utils (2.21.0) UNRELEASED; urgency=medium
 
   * Introduce option to skip restoring of audit logs #596

docs/requirements.md

@@ -27,6 +27,15 @@ Backup Utilities use [hard links][5] to store data efficiently, and the
 repositories on GitHub Enterprise Server use [symbolic links][6] so the backup snapshots
 must be written to a filesystem with support for symbolic and hard links.
 
+To check if your filesystem supports creating hardlinks of symbolic links, you can run the following within your backup destination directory:
+
+```bash
+touch file
+ln -s file symlink
+ln symlink hardlink
+ls -la
+```
+
 Using a [case sensitive][7] file system is also required to avoid conflicts.
 
 ## GitHub Enterprise Server version requirements
@@ -52,6 +61,8 @@ snapshot of GitHub Enterprise Server 2.11, the target GitHub Enterprise Server a
 be running GitHub Enterprise Server 2.12.x or 2.13.x. You can't restore a snapshot from
 2.10 to 2.13, because that's three releases ahead.
 
+**Note**: You _cannot_ restore a backup created from a newer verison of GitHub Enterprise Server to an older version. For example, an attempt to restore a snapshot of GitHub Enterprise Server 2.21 to a GitHub Enterprise Server 2.20 environment will fail with an error of `Error: Snapshot can not be restored to an older release of GitHub Enterprise Server.`.
+
 [1]: https://www.gnu.org/software/bash/
 [2]: https://git-scm.com/
 [3]: https://www.openssh.com/
@@ -61,4 +72,4 @@ be running GitHub Enterprise Server 2.12.x or 2.13.x. You can't restore a snapsh
 [7]: https://en.wikipedia.org/wiki/Case_sensitivity
 [8]: https://help.github.com/enterprise/admin/guides/installation/upgrade-requirements/
 [9]: https://joeyh.name/code/moreutils
-[10]: https://www.gnu.org/software/gawk
+[10]: https://www.gnu.org/software/gawk

script/release

@@ -1,7 +1,7 @@
 #!/usr/bin/env ruby
 # frozen_string_literal: true
 
-#/ Usage: release [--dry-run] <version> [min_version]
+#/ Usage: release [--dry-run] [--skip-version-bump-check] <version> [min_version]
 #/
 #/ Publish a backup-utils release:
 #/ * Updates the package changelog
@@ -196,7 +196,7 @@ end
 
 def bump_version(new_version, min_version = nil, path = 'share/github-backup-utils/version')
   current_version = Gem::Version.new(File.read(path).strip.chomp)
-  if Gem::Version.new(new_version) < current_version
+  if !@skip_version_bump_check && (Gem::Version.new(new_version) < current_version)
     raise "New version should be newer than #{current_version}"
   end
   File.open("#{path}.new", 'w') { |f| f.puts new_version }
@@ -338,6 +338,7 @@ if $PROGRAM_NAME == __FILE__
   begin
     args = ARGV.dup
     dry_run = false
+    skip_version_bump_check = false
     if args.include?('--dry-run')
       dry_run = true
       args.delete '--dry-run'
@@ -348,7 +349,12 @@ if $PROGRAM_NAME == __FILE__
       args.delete '--no-warn'
     end
 
-    raise 'Usage: release [--dry-run] <version> [min_version]' if args.empty?
+    if args.include?('--skip-version-bump-check')
+      @skip_version_bump_check = true
+      args.delete '--skip-version-bump-check'
+    end
+
+    raise 'Usage: release [--dry-run] [--skip-version-bump-check] <version> [min_version]' if args.empty?
 
     begin
       version = Gem::Version.new(args[0])

share/github-backup-utils/ghe-backup-settings

@@ -40,6 +40,7 @@ backup-secret() {
 }
 
 backup-secret "management console password" "manage-password" "secrets.manage"
+backup-secret "password pepper" "password-pepper" "secrets.github.user-password-secrets"
 
 # Backup external MySQL password if running external MySQL DB.
 if is_service_external 'mysql'; then

share/github-backup-utils/ghe-restore-mysql

@@ -27,7 +27,7 @@ ghe_remote_version_required "$GHE_HOSTNAME"
 export GHE_RESTORE_SNAPSHOT
 
 # The directory holding the snapshot to restore
-snapshot_dir="$GHE_DATA_DIR/$GHE_RESTORE_SNAPSHOT"
+GHE_RESTORE_SNAPSHOT_PATH="$GHE_DATA_DIR/$GHE_RESTORE_SNAPSHOT"
 
 if is_external_database_snapshot; then
   if [ -n "$EXTERNAL_DATABASE_RESTORE_SCRIPT" ]; then
@@ -40,7 +40,7 @@ if is_external_database_snapshot; then
     bm_end "$(basename $0)"
     exit 0
   else
-    if is_binary_backup "$snapshot_dir"; then
+    if is_binary_backup "$GHE_RESTORE_SNAPSHOT_PATH"; then
       echo "Error: Restore of a binary backup to appliance with an external database configured is not supported."
       echo "Please provide a custom external database restore script with EXTERNAL_DATABASE_RESTORE_SCRIPT"
       exit 1
@@ -63,18 +63,22 @@ if is_external_database_snapshot; then
 fi
 
 if is_binary_backup_feature_on; then
+  # Always restore the password pepper here since it is tied to the MySQL data.
+  restore-secret "password pepper" "password-pepper" "secrets.github.user-password-secrets"
   # Feature "mysql.backup.binary" is on, which means new backup scripts are available
-  if is_binary_backup "$snapshot_dir"; then
+  if is_binary_backup "$GHE_RESTORE_SNAPSHOT_PATH"; then
     ghe-restore-mysql-binary $GHE_HOSTNAME
   else
     ghe-restore-mysql-logical $GHE_HOSTNAME
   fi
 else
   # We do not allow to restore binary backup without "mysql.backup.binary" set
-  if is_binary_backup "$snapshot_dir"; then
+  if is_binary_backup "$GHE_RESTORE_SNAPSHOT_PATH"; then
     echo "To restore from a binary backup, you have to set ghe-config \"mysql.backup.binary\" to true" >&2
     exit 2
   else
+    # Always restore the password pepper here since it is tied to the MySQL data.
+    restore-secret "password pepper" "password-pepper" "secrets.github.user-password-secrets"
     if is_default_external_database_snapshot; then
       ghe-restore-mysql-logical $GHE_HOSTNAME
     else

share/github-backup-utils/ghe-restore-repositories

@@ -120,7 +120,7 @@ cat $tmp_list | ghe_debug
 bm_end "$(basename $0) - Transferring network list"
 
 bm_start "$(basename $0) - Generating routes"
-echo "cat $remote_tmp_list | github-env ./bin/dgit-cluster-restore-routes > $remote_routes_list" | ghe-ssh "$GHE_HOSTNAME" -- /bin/bash
+echo "cat $remote_tmp_list | github-env ./bin/dgit-cluster-restore-routes | grep 'git-server-' > $remote_routes_list" | ghe-ssh "$GHE_HOSTNAME" -- /bin/bash
 ghe-ssh "$GHE_HOSTNAME" -- cat $remote_routes_list | ghe_debug
 bm_end "$(basename $0) - Generating routes"
 
@@ -144,7 +144,7 @@ fi
 # One rsync invocation per server available.
 bm_start "$(basename $0) - Restoring repository networks"
 rsync_commands=()
-for file_list in $tempdir/*.rsync; do
+for file_list in $tempdir/git-server-*.rsync; do
   if $CLUSTER; then
     server=$(basename $file_list .rsync)
   else