Merge pull request #217 from github/sync-public-into-private

zachary-mark · web-flow · commit 3fe4656d98f8 · 2021-05-25T17:12:24.000-05:00
Sync backup-utils public into private
diff --git a/.github/workflows/main.yml b/.github/workflows/main.yml
@@ -14,9 +14,9 @@ jobs:
       run: |
         sudo apt-get update -y
         sudo apt-get install -y devscripts debhelper moreutils fakeroot jq pigz
-        wget "https://github.com/koalaman/shellcheck/releases/download/v0.7.0/shellcheck-v0.7.0.linux.x86_64.tar.xz"
-        tar --xz -xvf "shellcheck-v0.7.0.linux.x86_64.tar.xz"
-        sudo cp shellcheck-v0.7.0/shellcheck /usr/bin/shellcheck
+        wget "https://github.com/koalaman/shellcheck/releases/download/latest/shellcheck-latest.linux.x86_64.tar.xz"
+        tar --xz -xvf "shellcheck-latest.linux.x86_64.tar.xz"
+        sudo cp shellcheck-latest/shellcheck /usr/bin/shellcheck
       if: matrix.os != 'macos-latest'
     - name: Install Dependencies (macOS)
       run: |
diff --git a/README.md b/README.md
@@ -3,7 +3,7 @@
 This repository includes backup and recovery utilities for
 [GitHub Enterprise Server][1].
 
-**UPDATE**: The new parallel backup and restore beta feature will require [GNU awk](https://www.gnu.org/software/gawk) and [moreutils](https://joeyh.name/code/moreutils) to be installed.
+**UPDATE**: The new parallel backup and restore beta feature will require [GNU awk](https://www.gnu.org/software/gawk) and [moreutils](https://joeyh.name/code/moreutils) to be installed. Note that on some distributions/platforms, the `moreutils-parallel` package is separate from `moreutils` and must be installed on its own.
 
 **Note**: the [GitHub Enterprise Server version requirements][2] have
 changed starting with Backup Utilities v2.13.0, released on 27 March 2018.
diff --git a/bin/ghe-backup b/bin/ghe-backup
@@ -203,10 +203,6 @@ commands+=("
 echo \"Backing up audit log ...\"
 ghe-backup-es-audit-log || printf %s \"audit-log \" >> \"$failures_file\"")
 
-commands+=("
-echo \"Backing up hookshot logs ...\"
-ghe-backup-es-hookshot || printf %s \"hookshot \" >> \"$failures_file\"")
-
 commands+=("
 echo \"Backing up Git repositories ...\"
 ghe-backup-repositories || printf %s \"repositories \" >> \"$failures_file\"")
diff --git a/bin/ghe-host-check b/bin/ghe-host-check
@@ -130,7 +130,7 @@ fi
 
 # backup-utils 2.13 onwards limits support to the current and previous two releases
 # of GitHub Enterprise Server.
-supported_minimum_version="2.21.0"
+supported_minimum_version="2.22.0"
 
 if [ "$(version $version)" -ge "$(version $supported_minimum_version)" ]; then
   supported=1
diff --git a/bin/ghe-restore b/bin/ghe-restore
@@ -380,7 +380,7 @@ if test -f $GHE_RESTORE_SNAPSHOT_PATH/es-scan-complete; then
   ghe-ssh "$GHE_HOSTNAME" -- "sudo touch $GHE_REMOTE_DATA_USER_DIR/common/es-scan-complete"
 fi
 
-# Restore exported audit and hookshot logs to 2.12.9 and newer single nodes and
+# Restore exported audit logs to 2.12.9 and newer single nodes and
 # all releases of cluster
 if $CLUSTER || [ "$(version $GHE_REMOTE_VERSION)" -ge "$(version 2.12.9)" ]; then
   if [[ "$GHE_RESTORE_SKIP_AUDIT_LOGS" = "yes" ]]; then
@@ -391,9 +391,6 @@ if $CLUSTER || [ "$(version $GHE_REMOTE_VERSION)" -ge "$(version 2.12.9)" ]; the
     ghe-restore-es-audit-log \"$GHE_HOSTNAME\" 1>&3")
   fi
 
-  commands+=("
-  echo \"Restoring hookshot logs ...\"
-  ghe-restore-es-hookshot \"$GHE_HOSTNAME\" 1>&3")
 fi
 
 if [ "$GHE_PARALLEL_ENABLED" = "yes" ]; then
diff --git a/debian/changelog b/debian/changelog
@@ -1,3 +1,14 @@
+github-backup-utils (3.1.0~rc1) UNRELEASED; urgency=medium
+
+  * Update repository backups to use ghe-gc-* for lock file management #188
+  * A faster way to restore storage blobs (clusters) #212
+  * Bug fix: Be more specific in restore routes globbing #715
+  * fix(docker): add coreutils to get flags for date #717
+  * Add backup cadence variable to the appliance #719
+  * Fix is_default_external_database_snapshot function #720
+
+ -- Zachary Mark <zachary-mark@github.com>  Thu, 06 May 2021 17:11:18 +0000
+
 github-backup-utils (3.0.0) UNRELEASED; urgency=medium
 
   * Fix restoring the password pepper for already configured instances #683
diff --git a/share/github-backup-utils/ghe-backup-es-hookshot b/share/github-backup-utils/ghe-backup-es-hookshot
diff --git a/share/github-backup-utils/ghe-restore-es-hookshot b/share/github-backup-utils/ghe-restore-es-hookshot
diff --git a/share/github-backup-utils/version b/share/github-backup-utils/version
@@ -1 +1 @@
-3.0.0
+3.1.0.rc1
diff --git a/test/bin/curl b/test/bin/curl
@@ -3,7 +3,7 @@
 # Fake curl command stub for tests.
 set -e
 
-# Return empty list of indexes for ghe-backup-es-audit-log and ghe-backup-es-hookshot
+# Return empty list of indexes for ghe-backup-es-audit-log
 if echo "$@" | grep -q '_cat/indices/'; then
   exit 0
 # Exit with non-zero code to mimic a non-existant index
diff --git a/test/test-ghe-backup.sh b/test/test-ghe-backup.sh
@@ -132,19 +132,6 @@ begin_test "ghe-backup without password pepper"
 )
 end_test
 
-begin_test "ghe-backup empty hookshot directory"
-(
-  set -e
-
-  rm -rf $GHE_REMOTE_DATA_USER_DIR/hookshot/repository-*
-  rm -rf $GHE_DATA_DIR/current/hookshot/repository-*
-  ghe-backup
-
-  # Check that the "--link-dest arg does not exist" message hasn't occurred.
-  [ ! "$(grep "[l]ink-dest arg does not exist" $TRASHDIR/out)" ]
-)
-end_test
-
 begin_test "ghe-backup empty git-hooks directory"
 (
   set -e
diff --git a/test/test-ghe-host-check.sh b/test/test-ghe-host-check.sh
@@ -58,7 +58,9 @@ begin_test "ghe-host-check detects unsupported GitHub Enterprise Server versions
   ! GHE_TEST_REMOTE_VERSION=11.340.36 ghe-host-check
   # hardcode until https://github.com/github/backup-utils/issues/675 is resolved
   ! GHE_TEST_REMOTE_VERSION=2.20.0 ghe-host-check
-  GHE_TEST_REMOTE_VERSION=2.21.0 ghe-host-check
+  ! GHE_TEST_REMOTE_VERSION=2.21.0 ghe-host-check
+  GHE_TEST_REMOTE_VERSION=2.22.0 ghe-host-check
+  GHE_TEST_REMOTE_VERSION=3.0.0 ghe-host-check
   GHE_TEST_REMOTE_VERSION=$BACKUP_UTILS_VERSION ghe-host-check
   GHE_TEST_REMOTE_VERSION=$BACKUP_UTILS_VERSION ghe-host-check
   GHE_TEST_REMOTE_VERSION=$bu_version_major.$bu_version_minor.999 ghe-host-check
diff --git a/test/test-shellcheck.sh b/test/test-shellcheck.sh
@@ -9,10 +9,10 @@ BASE_PATH=$(cd "$(dirname "$0")/../" && pwd)
 begin_test "shellcheck: reports no errors or warnings"
 (
   set -e
-  # We manually install Shellcheck 0.7.0 on Linux builds as other options
+  # We manually install the latest Shellcheck on Linux builds as other options
   # are too old.
-  if [ -x "$BASE_PATH/shellcheck-v0.7.0/shellcheck" ]; then
-    shellcheck() { "$BASE_PATH/shellcheck-v0.7.0/shellcheck" "$@"; }
+  if [ -x "$BASE_PATH/shellcheck-latest/shellcheck" ]; then
+    shellcheck() { "$BASE_PATH/shellcheck-latest/shellcheck" "$@"; }
   fi
 
   if ! type shellcheck 1>/dev/null 2>&1; then
diff --git a/test/testlib.sh b/test/testlib.sh
@@ -42,7 +42,7 @@ export GHE_BACKUP_CONFIG GHE_DATA_DIR GHE_REMOTE_DATA_DIR GHE_REMOTE_ROOT_DIR
 
 # The default remote appliance version. This may be set in the environment prior
 # to invoking tests to emulate a different remote vm version.
-: ${GHE_TEST_REMOTE_VERSION:=3.0.0.rc1}
+: ${GHE_TEST_REMOTE_VERSION:=3.1.0.rc1}
 export GHE_TEST_REMOTE_VERSION
 
 # Source in the backup config and set GHE_REMOTE_XXX variables based on the
@@ -146,7 +146,7 @@ report_failure () {
   printf "test: %-73s $msg\\n" "$desc ..."
   (
     sed 's/^/    /' <"$TRASHDIR/out" |
-    grep -a -v -e '^\+ end_test' -e '^+ set +x' <"$TRASHDIR/out" |
+    grep -a -v -e '^\+ end_test' -e '^+ set +x' - "$TRASHDIR/out" |
     sed 's/[+] test_status=/test failed. last command exited with /' |
     sed 's/^/    /'
   ) 1>&2
@@ -261,11 +261,6 @@ setup_test_data () {
   echo "fake audit log this yr this mth" | gzip > audit_log-1-$this_yr-$this_mth-1.gz
   echo "1" > audit_log-1-$this_yr-$this_mth-1.size
 
-  # Create hookshot logs
-  mkdir -p "$loc/hookshot/"
-  cd "$loc/hookshot/"
-  echo "fake hookshot log" | gzip > hookshot-logs-2018-03-05.gz
-
   # Create some test repositories in the remote repositories dir
   mkdir -p "$loc/repositories/info"
   mkdir -p "$TRASHDIR/hooks"
@@ -497,7 +492,6 @@ verify_all_restored_data() {
   else
     grep -q "fake audit log last yr last mth" "$TRASHDIR/restore-out"
     grep -q "fake audit log this yr this mth" "$TRASHDIR/restore-out"
-    grep -q "fake hookshot log" "$TRASHDIR/restore-out"
   fi
 
   # verify settings import was *not* run due to instance already being
