Merge remote-tracking branch 'public/master' into sync-public-to-private

Author: Steve Culver

.dockerignore

@@ -1,2 +1,4 @@
-*
-!share/github-backup-utils/ghe-docker-init
+.git/
+.github/
+.gitattributes
+.gitignore

.github/workflows/main.yml

@@ -6,22 +6,21 @@ jobs:
   build:
     strategy:
       matrix:
-        os: ['ubuntu-20.04', 'ubuntu-18.04', 'ubuntu-16.04', 'macos-latest']
+        os: ['ubuntu-20.04', 'ubuntu-18.04', 'macos-latest']
       fail-fast: false
     runs-on: ${{ matrix.os }}
     steps:
     - name: Install Dependencies (Linux)
       run: |
         sudo apt-get update -y
-        sudo apt-get install -y devscripts debhelper moreutils fakeroot jq pigz
+        sudo apt-get install -y devscripts debhelper moreutils fakeroot jq pigz help2man
         wget "https://github.com/koalaman/shellcheck/releases/download/latest/shellcheck-latest.linux.x86_64.tar.xz"
         tar --xz -xvf "shellcheck-latest.linux.x86_64.tar.xz"
         sudo cp shellcheck-latest/shellcheck /usr/bin/shellcheck
       if: matrix.os != 'macos-latest'
     - name: Install Dependencies (macOS)
       run: |
         brew install gnu-tar shellcheck jq pigz coreutils gnu-sed gnu-getopt
-        brew unlink parallel
         brew install moreutils gawk
       if: matrix.os == 'macos-latest'
     - name: Get Sources

Dockerfile

@@ -11,10 +11,8 @@ RUN apt-get -q -y update && \
     gawk \
     && rm -rf /var/lib/apt/lists/*
 
+COPY ./ /backup-utils/
 WORKDIR /backup-utils
-ADD https://github.com/github/backup-utils/archive/stable.tar.gz /
-RUN tar xzvf /stable.tar.gz --strip-components=1 -C /backup-utils && \
-    rm -r /stable.tar.gz
 
 RUN chmod +x /backup-utils/share/github-backup-utils/ghe-docker-init
 

Dockerfile.alpine

@@ -11,11 +11,8 @@ RUN apk --update --no-cache add \
     procps \
     coreutils
 
+COPY ./ /backup-utils/
 WORKDIR /backup-utils
-ADD https://github.com/github/backup-utils/archive/stable.tar.gz /
-RUN tar xzvf /stable.tar.gz --strip-components=1 -C /backup-utils && \
-    mv /usr/bin/gawk /usr/bin/awk && \
-    rm -r /stable.tar.gz
 
 RUN chmod +x /backup-utils/share/github-backup-utils/ghe-docker-init
 

README.md

@@ -50,6 +50,6 @@ to your specific GitHub Enterprise Server setup or would like assistance with
 backup site setup or recovery, please contact our [Enterprise support team][3]
 instead.
 
-[1]: https://enterprise.github.com
+[1]: https://github.com/enterprise
 [2]: docs/requirements.md#github-enterprise-version-requirements
-[3]: https://enterprise.github.com/support/
+[3]: https://support.github.com/

bin/ghe-host-check

@@ -53,6 +53,7 @@ port=$(ssh_port_part "$host")
 hostname=$(ssh_host_part "$host")
 
 set +e
+# ghe-negotiate-version verifies if the target is a Github Enterprise Server instance
 output=$(echo "ghe-negotiate-version backup-utils $BACKUP_UTILS_VERSION" | ghe-ssh -o BatchMode=no $options $host -- /bin/sh 2>&1)
 rc=$?
 set -e
@@ -130,7 +131,7 @@ fi
 
 # backup-utils 2.13 onwards limits support to the current and previous two releases
 # of GitHub Enterprise Server.
-supported_minimum_version="2.22.0"
+supported_minimum_version="3.2.0"
 
 if [ "$(version $version)" -ge "$(version $supported_minimum_version)" ]; then
   supported=1

bin/ghe-restore

@@ -91,10 +91,43 @@ cleanup () {
     update_restore_status "$1"
   fi
 
+  if ghe-ssh "$GHE_HOSTNAME" -- 'ghe-config --true app.actions.enabled'; then
+    echo "Restarting Actions after restore ..."
+    # In GHES 3.3+, ghe-actions-start no longer has a -f (force) flag. In GHES 3.2 and below, we must provide the
+    # force flag to make sure it can start in maintenance mode. Use it conditionally based on whether it exists
+    # in the --help output
+    if ghe-ssh "$GHE_HOSTNAME" -- 'ghe-actions-start --help' | grep -q force ; then
+      ghe-ssh "$GHE_HOSTNAME" -- 'ghe-actions-start -f' 1>&3
+    else
+      ghe-ssh "$GHE_HOSTNAME" -- 'ghe-actions-start' 1>&3
+    fi
+  fi
+
   # Cleanup SSH multiplexing
   ghe-ssh --clean
 }
 
+# This function's type definition is being passed to a remote host via `ghe-ssh` but is not used locally.
+# shellcheck disable=SC2034
+cleanup_cluster_nodes() {
+  uuid="$1"
+  if [ -z "$uuid" ]; then
+    echo "Node UUID required."
+    exit 2
+  fi
+
+  ghe-spokes server evacuate git-server-$uuid 'Removing replica'
+  ghe-spokes server destroy git-server-$uuid
+
+  ghe-storage destroy-host storage-server-$uuid --force
+
+  ghe-dpages offline pages-server-$uuid
+  ghe-dpages remove pages-server-$uuid
+
+  ghe-redis-cli del resque:queue:maint_git-server-$uuid
+  ghe-redis-cli srem resque:queues maint_git-server-$uuid
+}
+
 # Bring in the backup configuration
 # shellcheck source=share/github-backup-utils/ghe-backup-config
 . "$( dirname "${BASH_SOURCE[0]}" )/../share/github-backup-utils/ghe-backup-config"
@@ -231,7 +264,11 @@ fi
 # Make sure the GitHub appliance has Actions enabled if the snapshot contains Actions data.
 if [ -d "$GHE_RESTORE_SNAPSHOT_PATH/mssql" ] || [ -d "$GHE_RESTORE_SNAPSHOT_PATH/actions" ]; then
   if ! ghe-ssh "$GHE_HOSTNAME" -- 'ghe-config --true app.actions.enabled'; then
+    # Get GHES release version in major.minor format
+    RELEASE_VERSION=$(ghe-ssh "$GHE_HOSTNAME" -- 'ghe-config --get core.package-version' | cut -d '.' -f 1,2)
+
     echo "Error: $GHE_HOSTNAME must have GitHub Actions enabled before restoring since the snapshot contains Actions data. Aborting." 1>&2
+    echo "Setup details for enabling Actions can be found here: https://docs.github.com/en/enterprise-server@$RELEASE_VERSION/admin/github-actions/advanced-configuration-and-troubleshooting/backing-up-and-restoring-github-enterprise-server-with-github-actions-enabled" 1>&2
     exit 1
   fi
 fi
@@ -327,6 +364,9 @@ else
 fi
 
 if ghe-ssh "$GHE_HOSTNAME" -- 'ghe-config --true app.actions.enabled'; then
+  echo "Stopping Actions before restoring databases ..."
+  ghe-ssh "$GHE_HOSTNAME" -- 'ghe-actions-stop' 1>&3
+
   echo "Restoring MSSQL databases ..."
   ghe-restore-mssql "$GHE_HOSTNAME" 1>&3
 
@@ -426,6 +466,13 @@ elif $instance_configured; then
   ghe-ssh "$GHE_HOSTNAME" -- "ghe-config-apply" 1>&3 2>&3
 fi
 
+# Clear GitHub Connect settings stored in the restored database.
+# This needs to happen after `ghe-config-apply` to ensure all migrations have run.
+if ! $RESTORE_SETTINGS; then
+  echo "if [ -f /usr/local/share/enterprise/ghe-reset-gh-connect ]; then /usr/local/share/enterprise/ghe-reset-gh-connect -y; fi" |
+  ghe-ssh "$GHE_HOSTNAME" -- /bin/sh 1>&3
+fi
+
 # Start cron. Timerd will start automatically as part of the config run.
 echo "Starting cron ..."
 if $CLUSTER; then
@@ -451,7 +498,8 @@ if ! $CLUSTER && $instance_configured; then
   if [ -n "$other_nodes" ]; then
     echo "Cleaning up stale nodes ..."
     for uuid in $other_nodes; do
-      ghe-ssh "$GHE_HOSTNAME" -- "/usr/local/share/enterprise/ghe-cluster-cleanup-node $uuid" 1>&3
+      # shellcheck disable=SC2034
+      echo "set -o pipefail; $(typeset -f cleanup_cluster_nodes); cleanup_cluster_nodes $uuid" | ghe-ssh "$GHE_HOSTNAME" 1>&3
     done
   fi
 fi

debian/changelog

@@ -1,12 +1,128 @@
+github-backup-utils (3.4.0) UNRELEASED; urgency=medium
+
+  * Add anchor to usage doc for settings restore #865
+
+ -- Steve Culver <[email protected]>  Tue, 15 Feb 2022 19:25:09 +0000
+
+github-backup-utils (3.3.1) UNRELEASED; urgency=medium
+
+  * Fix compat issue with ghe-actions-start during maintenance mode #836
+
+ -- [email protected]  Tue, 21 Dec 2021 23:38:01 +0000
+
+github-backup-utils (3.3.0) UNRELEASED; urgency=medium
+
+
+ -- [email protected]  Wed, 08 Dec 2021 03:12:53 +0000
+
+github-backup-utils (3.3.0) UNRELEASED; urgency=medium
+
+
+ -- [email protected]  Wed, 08 Dec 2021 03:10:21 +0000
+
+github-backup-utils (3.3.0) UNRELEASED; urgency=medium
+
+
+ -- [email protected]  Wed, 08 Dec 2021 02:56:47 +0000
+
+github-backup-utils (3.3.0) UNRELEASED; urgency=medium
+
+
+ -- [email protected]  Wed, 08 Dec 2021 02:51:06 +0000
+
+github-backup-utils (3.3.0) UNRELEASED; urgency=medium
+
+
+ -- [email protected]  Wed, 08 Dec 2021 02:02:25 +0000
+
+github-backup-utils (3.3.0) UNRELEASED; urgency=medium
+
+
+ -- [email protected]  Wed, 08 Dec 2021 01:56:26 +0000
+
+github-backup-utils (3.3.0) UNRELEASED; urgency=medium
+
+
+ -- [email protected]  Wed, 08 Dec 2021 01:25:12 +0000
+
+github-backup-utils (3.3.0.rc1) UNRELEASED; urgency=medium
+
+
+ -- Nick Iodice <[email protected]>  Tue, 09 Nov 2021 19:56:08 +0000
+
+github-backup-utils (3.2.0) UNRELEASED; urgency=medium
+
+
+ -- Brett Westover <[email protected]>  Tue, 28 Sep 2021 16:50:00 +0000
+
+github-backup-utils (3.2.0.rc3) UNRELEASED; urgency=medium
+
+  * Move GitHub Connect reset to after ghe-config-apply #783
+
+ -- Brett Westover <[email protected]>  Fri, 17 Sep 2021 00:44:59 +0000
+
+github-backup-utils (3.2.0.rc1) UNRELEASED; urgency=medium
+
+  * Leaked host key check: Avoid false positives from FIPS mode #748
+  * Always restore user-password-secrets #762
+  * Make some of the actions setting best effort #767
+  * Remove reference to `ghe-cluster-cleanup-nodes` #768
+  * Stop/Restart Actions in ghe-restore #769
+  * Clear GitHub Connect settings when not restoring settings #770
+  * GitHub Connect Reset Issue #776
+
+ -- Brett Westover <[email protected]>  Fri, 10 Sep 2021 20:10:07 +0000
+
+github-backup-utils (3.2.0) UNRELEASED; urgency=medium
+
+  * Leaked host key check: Avoid false positives from FIPS mode #748
+  * Always restore user-password-secrets #762
+  * Make some of the actions setting best effort #767
+  * Remove reference to `ghe-cluster-cleanup-nodes` #768
+  * Stop/Restart Actions in ghe-restore #769
+  * Clear GitHub Connect settings when not restoring settings #770
+
+ -- Brett Westover <[email protected]>  Thu, 09 Sep 2021 16:42:24 +0000
+
+github-backup-utils (3.1.0) UNRELEASED; urgency=medium
+
+
+ -- Zachary Mark <[email protected]>  Thu, 03 Jun 2021 16:55:16 +0000
+
 github-backup-utils (3.1.0~rc1) UNRELEASED; urgency=medium
 
+  [ Zachary Mark ]
   * Update repository backups to use ghe-gc-* for lock file management #188
   * A faster way to restore storage blobs (clusters) #212
   * Bug fix: Be more specific in restore routes globbing #715
   * fix(docker): add coreutils to get flags for date #717
   * Add backup cadence variable to the appliance #719
   * Fix is_default_external_database_snapshot function #720
 
+  [ Hideki Yamane ]
+  * debian/rules
+    - Drop unnecessary build-indep:
+    - Add manpages generation.
+  * debian/changelog
+    - Fix some lintian warnings for old entries.
+  * debian/source/format
+    - Upgrade to newer source format 3.0 (native).
+  * debian/copyright
+    - Format it as Machine-readable debian/copyright format 1.0
+    - Add entries Twan Wolthof <[email protected]> and me for debian/* since
+      enough code was written by non-github.com employees, at least.
+  * debian/control
+    - Update descriptions in debian/control
+    - Fix "Architecture: all", instead of "any".
+    - Declare its Homepage as https://github.com/github/backup-utils
+    - Add Vcs-* metadata field.
+    - Specify "Rules-Requires-Root: no"
+    - Drop unnecessary "Build-Depends: devscripts"
+    - Set Standards-Version: 4.5.1
+    - Update Maintainer as Zachary Mark <[email protected]>
+  * debian/{clean,manpages}
+    - Add files to support manpages handles.
+
  -- Zachary Mark <[email protected]>  Thu, 06 May 2021 17:11:18 +0000
 
 github-backup-utils (3.0.0) UNRELEASED; urgency=medium
@@ -37,7 +153,7 @@ github-backup-utils (2.22.0) UNRELEASED; urgency=medium
   * Add parallelized restore capability to ghe-restore-storage #635
   * Update backup-utils for new features in 2.22 #641
 
- -- [email protected]  Wed, 23 Sep 2020 15:48:54 +0000
+ -- Hao Jiang <[email protected]>  Wed, 23 Sep 2020 15:48:54 +0000
 
 github-backup-utils (2.21.0) UNRELEASED; urgency=medium
 
@@ -135,6 +251,7 @@ github-backup-utils (2.16.1) UNRELEASED; urgency=medium
 
 github-backup-utils (2.16.0) UNRELEASED; urgency=medium
 
+  * (There was no descriptions)
 
  -- Colin Seymour <[email protected]>  Tue, 22 Jan 2019 20:25:34 +0000
 
@@ -146,11 +263,13 @@ github-backup-utils (2.15.1) UNRELEASED; urgency=medium
 
 github-backup-utils (2.15.0) UNRELEASED; urgency=medium
 
+  * (There was no descriptions)
 
  -- Colin Seymour <[email protected]>  Tue, 16 Oct 2018 16:40:03 +0000
 
 github-backup-utils (2.15.0) UNRELEASED; urgency=medium
 
+  * (There was no descriptions)
 
  -- Colin Seymour <[email protected]>  Tue, 16 Oct 2018 16:07:36 +0000
 

debian/clean

@@ -0,0 +1 @@
+debian/*.1

debian/control

@@ -1,33 +1,32 @@
 Source: github-backup-utils
-Maintainer: Twan Wolthof <xeago@spotify.com>
+Maintainer: Zachary Mark <zachary-mark@github.com>
 Section: misc
 Priority: optional
-Standards-Version: 3.9.2
-Build-Depends: debhelper (>= 9), git, devscripts, moreutils, jq, rsync (>= 2.6.4)
+Standards-Version: 4.5.1
+Build-Depends: debhelper (>= 9), git, moreutils, jq, rsync (>= 2.6.4), help2man,
+Homepage: https://github.com/github/backup-utils
+Vcs-Git: https://github.com/github/backup-utils.git
+Vcs-Browser: https://github.com/github/backup-utils
+Rules-Requires-Root: no
 
 Package: github-backup-utils
-Architecture: any
+Architecture: all
 Depends: ${misc:Depends}, rsync (>= 2.6.4), moreutils, jq, git
 Description: Backup and recovery utilities for GitHub Enterprise Server
  The backup utilities implement a number of advanced capabilities for backup
  hosts, built on top of the backup and restore features already included in
  GitHub Enterprise Server.
  .
  These advanced features include:
- .
- Complete GitHub Enterprise Server backup and recovery system via two simple
- utilities:
- `ghe-backup` and `ghe-restore`.
- Online backups. The GitHub appliance need not be put in maintenance mode for
- the duration of the backup run.
- Incremental backup of Git repository data. Only changes since the last
- snapshot are transferred, leading to faster backup runs and lower network
- bandwidth and machine utilization.
- Efficient snapshot storage. Only data added since the previous snapshot
- consumes new space on the backup host.
- Multiple backup snapshots with configurable retention periods.
- Backup commands run under the lowest CPU/IO priority on the GitHub appliance,
- reducing performance impact while backups are in progress.
- Runs under most Linux/Unix environments.
- MIT licensed, open source software maintained by GitHub, Inc.
-
+  - Complete GitHub Enterprise Server backup and recovery system via two simple
+    utilities: `ghe-backup` and `ghe-restore`.
+  - Online backups. The GitHub appliance need not be put in maintenance mode for
+    the duration of the backup run.
+  - Incremental backup of Git repository data. Only changes since the last
+    snapshot are transferred, leading to faster backup runs and lower network
+    bandwidth and machine utilization.
+  - Efficient snapshot storage. Only data added since the previous snapshot
+    consumes new space on the backup host.
+  - Multiple backup snapshots with configurable retention periods.
+  - Backup runs under the lowest CPU/IO priority on the GitHub appliance,
+    reducing performance impact while backups are in progress.