Merge branch 'master' into stable

Author: cainejette

RELEASING.md

@@ -26,7 +26,8 @@ Coming :soon:
 
 1. Install the Debian `devscripts` package:
   `sudo apt-get install devscripts`
-2. Run...
+2. Generate a PAT through github.com with access to the `github/backup-utils` repository. This will be used for the `GH_RELEASE_TOKEN` environment variable in the next step.
+3. Run...
   - Feature release:
   `GH_AUTHOR="Bob Smith <[email protected]>" GH_RELEASE_TOKEN=your-amazing-secure-token script/release 2.13.0 2.11.0`
   - Patch release:

bin/ghe-host-check

@@ -113,7 +113,7 @@ fi
 
 # backup-utils 2.13 onwards limits support to the current and previous two releases
 # of GitHub Enterprise Server.
-supported_minimum_version="2.17.0"
+supported_minimum_version="2.18.0"
 
 if [ "$(version $version)" -ge "$(version $supported_minimum_version)" ]; then
   supported=1

debian/changelog

@@ -4,6 +4,13 @@ github-backup-utils (2.19.4) UNRELEASED; urgency=medium
 
  -- Hao Jiang <[email protected]>  Tue, 18 Feb 2020 17:54:31 +0000
 
+github-backup-utils (2.20.0) UNRELEASED; urgency=medium
+
+  * Fix `ghe-backup-repositories` performance for large instances #541
+  * Fix two issues with binary backup (slow gzip compression and support for restore from instances other than SQL master) #551
+
+ -- Alejandro Figueroa <[email protected]>  Tue, 11 Feb 2020 20:47:17 +0000
+
 github-backup-utils (2.19.3) UNRELEASED; urgency=medium
 
   * Fix two issues with binary backup (slow gzip compression and support for restore from instances other than SQL master) #551
@@ -241,7 +248,7 @@ github-backup-utils (2.7.1) UNRELEASED; urgency=medium
 
 github-backup-utils (2.7.0) UNRELEASED; urgency=medium
 
-  * GitHub Enterprise 2.7.0 support 
+  * GitHub Enterprise 2.7.0 support
 
  -- Sergio Rubio <[email protected]>  Wed, 03 Aug 2016 20:25:31 +0200
 

script/cibuild

@@ -6,7 +6,7 @@ set -e
 # place with this version at the beginning of each test and many commands have
 # conditional logic based on the remote version. Running the suite against
 # different major versions ensures we're covering these conditional paths.
-REMOTE_VERSIONS="2.17.0 2.19.0"
+REMOTE_VERSIONS="2.18.0 2.20.0"
 
 # Enable verbose logging of ssh commands
 export GHE_VERBOSE_SSH=true

script/release

@@ -31,6 +31,7 @@ GH_REPO = ENV['GH_REPO'] || 'backup-utils'
 GH_OWNER = ENV['GH_OWNER'] || 'github'
 GH_AUTHOR = ENV['GH_AUTHOR']
 DEB_PKG_NAME = 'github-backup-utils'
+GH_BASE_BRANCH = ENV['GH_BASE_BRANCH'] || 'master'
 
 CHANGELOG_TMPL = '''<%= package_name %> (<%= package_version %>) UNRELEASED; urgency=medium
 
@@ -136,7 +137,7 @@ def beautify_changes(changes)
 end
 
 def changelog
-  changes = `git log --pretty=oneline origin/stable...origin/master --reverse --grep "Merge pull request" | sort -t\# -k2`.lines.map(&:strip)
+  changes = `git log --pretty=oneline origin/stable...origin/#{GH_BASE_BRANCH} --reverse --grep "Merge pull request" | sort -t\# -k2`.lines.map(&:strip)
   raise 'Building the changelog failed' if $CHILD_STATUS != 0
 
   changes
@@ -232,8 +233,8 @@ end
 
 def update_stable_branch
   `git checkout --quiet stable`
-  unless (out = `git merge --quiet --ff-only origin/master`)
-    warn "Merging master into stable failed:\n\n#{out}"
+  unless (out = `git merge --quiet --ff-only origin/#{GH_BASE_BRANCH}`)
+    warn "Merging #{GH_BASE_BRANCH} into stable failed:\n\n#{out}"
   end
   unless (out = `git push --quiet origin stable`)
     warn "Failed pushing the stable branch:\n\n#{out}"
@@ -245,7 +246,7 @@ def create_release_pr(version, release_body)
     'title': "Bump version: #{version}",
     'body': release_body,
     'head': "release-#{version}",
-    'base': 'master'
+    'base': GH_BASE_BRANCH
   }.to_json
   res = post("/repos/#{GH_OWNER}/#{GH_REPO}/pulls", body)
   raise "Creating release PR failed (#{res.code})" unless res.is_a? Net::HTTPSuccess
@@ -328,9 +329,9 @@ def attach_assets_to_release(upload_url, release_id, files)
 end
 
 def clean_up(version)
-  `git checkout --quiet master`
+  `git checkout --quiet #{GH_BASE_BRANCH}`
   `git fetch --quiet origin --prune`
-  `git pull --quiet origin master --prune`
+  `git pull --quiet origin #{GH_BASE_BRANCH} --prune`
   `git branch --quiet -D release-#{version} >/dev/null 2>&1`
   `git push --quiet origin :release-#{version} >/dev/null 2>&1`
   `git branch --quiet -D tmp-packging >/dev/null 2>&1`
@@ -382,6 +383,7 @@ if $PROGRAM_NAME == __FILE__
       puts "Repo: #{GH_REPO}"
       puts "Author: #{GH_AUTHOR}"
       puts "Token: #{ENV['GH_RELEASE_TOKEN'] && 'set' || 'unset'}"
+      puts "Base branch: #{GH_BASE_BRANCH}"
       puts 'Changelog:'
       if release_changes.empty?
         puts ' => No new bug fixes, enhancements or features.'
@@ -429,7 +431,7 @@ if $PROGRAM_NAME == __FILE__
 
     puts 'Creating release...'
     release_title = "GitHub Enterprise Server Backup Utilities v#{version}"
-    res = create_release "v#{version}", 'master', release_title, release_body, true
+    res = create_release "v#{version}", GH_BASE_BRANCH, release_title, release_body, true
 
     # Tidy up before building tarball and deb pkg
     clean_up version

share/github-backup-utils/ghe-backup-audit-log

@@ -29,6 +29,11 @@ is_skip_truncate_enabled(){
   ghe-ssh "$host" test -e "$GHE_REMOTE_DATA_USER_DIR/common/audit-log-import/skip_truncate"
 }
 
+# Check whether the audit_entries table is deleted
+audit_entries_deleted(){
+   [ -z "$(ghe-ssh "$host" -- "/usr/local/share/enterprise/github-mysql 'SHOW TABLES LIKE \"audit_entries\"'")" ]
+}
+
 is_binary_backup(){
   ghe-ssh "$host" ghe-config --true "mysql.backup.binary"
 }
@@ -38,7 +43,10 @@ backup_mysql(){
     ghe_verbose "Skip backup audit_entries for Mysql since it is using binary backup"
     return
   fi
-  if is_skip_truncate_enabled; then
+
+  if audit_entries_deleted; then
+    ghe_verbose "audit_entries table does not exist"
+  elif is_skip_truncate_enabled; then
     # As skip_truncate exists, we need to also backup the audit entries
     # in MySQL because Elasticsearch may not be fully synced.
     "${base_path}/ghe-backup-mysql-audit-log"

share/github-backup-utils/ghe-restore-audit-log

@@ -46,8 +46,9 @@ is_mysql_supported(){
      'test \"\$ENTERPRISE_AUDIT_LOG_MYSQL_LOGGER_ENABLED\" = \"1\""' | ghe-ssh "$GHE_HOSTNAME" -- /bin/bash
 }
 
+# Check if the backup is binary by looking up the sentinel file 
 is_binary_backup(){
-  ghe-ssh "$GHE_HOSTNAME" ghe-config --true "mysql.backup.binary"
+  test -f "$GHE_DATA_DIR/$GHE_RESTORE_SNAPSHOT"/mysql-binary-backup-sentinel 
 }
 
 # Helper function to set remote flags in `/data/user/common/audit-log-import`
@@ -90,6 +91,11 @@ restore_mysql(){
 
   ghe_verbose "Restoring MySQL audit logs ..."
 
+  if ! mysql_table_schema_available; then
+    ghe_verbose "schema.gz does not exist"
+    return
+  fi
+
   "${base_path}/ghe-restore-mysql-audit-log" "$GHE_HOSTNAME" "$only_schema"
 }
 

share/github-backup-utils/version

@@ -1 +1 @@
-2.19.4
+2.20.0

test/bin/dgit-cluster-backup-routes

@@ -1,11 +1,31 @@
 #!/usr/bin/env bash
 # Usage: dgit-cluster-backup-routes
-# Emulates the remote GitHub enterprise-configure command. Tests use this
-# to assert that the command was executed.
+# Emulates the remote GitHub dgit-cluster-backup-routes command. Tests use this
+# to assert that all repos and gists get backed up.
 set -e
-cat <<EOF
-0/nw/01/aa/3f/1234 git-server-fake-uuid
-1/nw/23/bb/4c/2345 git-server-fake-uuid1
-0/01/aa/3f/gist git-server-fake-uuid2
-1/23/bb/4c/gist git-server-fake-uuid
-EOF
+
+version() {
+  echo "${@#v}" | awk -F. '{ printf("%d%03d%03d%03d\n", $1,$2,$3,$4); }';
+}
+
+if [ -z "$GHE_REMOTE_VERSION" ]; then
+  echo GHE_REMOTE_VERSION must be set for this script to work.
+  exit 1
+fi
+
+# The list of gists returned by the source changed in 2.16.23, 2.17.14,
+# 2.18.8, and 2.19.3.  We need to account for this difference here.
+if [[ "$GHE_REMOTE_VERSION" =~ 2.16. && "$(version $GHE_REMOTE_VERSION)" -ge "$(version 2.16.23)" ]] || \
+   [[ "$GHE_REMOTE_VERSION" =~ 2.17. && "$(version $GHE_REMOTE_VERSION)" -ge "$(version 2.17.14)" ]] || \
+   [[ "$GHE_REMOTE_VERSION" =~ 2.18. && "$(version $GHE_REMOTE_VERSION)" -ge "$(version 2.18.8)" ]] || \
+   [[ "$(version $GHE_REMOTE_VERSION)" -ge "$(version 2.19.3)" ]]; then
+    echo 0/nw/01/aa/3f/1234 git-server-fake-uuid
+    echo 1/nw/23/bb/4c/2345 git-server-fake-uuid1
+    echo 0/01/aa/3f/gist/93069ad4c391b6203f183e147d52a97a.git git-server-fake-uuid2
+    echo 1/23/bb/4c/gist/1234.git git-server-fake-uuid
+else
+    echo 0/nw/01/aa/3f/1234 git-server-fake-uuid
+    echo 1/nw/23/bb/4c/2345 git-server-fake-uuid1
+    echo 0/01/aa/3f/gist git-server-fake-uuid2
+    echo 1/23/bb/4c/gist git-server-fake-uuid
+fi

test/testlib.sh

@@ -42,7 +42,7 @@ export GHE_BACKUP_CONFIG GHE_DATA_DIR GHE_REMOTE_DATA_DIR GHE_REMOTE_ROOT_DIR
 
 # The default remote appliance version. This may be set in the environment prior
 # to invoking tests to emulate a different remote vm version.
-: ${GHE_TEST_REMOTE_VERSION:=2.19.0}
+: ${GHE_TEST_REMOTE_VERSION:=2.20.0}
 export GHE_TEST_REMOTE_VERSION
 
 # Source in the backup config and set GHE_REMOTE_XXX variables based on the