Merge pull request #543 from github/merge-stable-to-master

Merge stable to master

Author: cainejette

.travis.yml

@@ -9,6 +9,7 @@ matrix:
         - brew install moreutils
         - brew install shellcheck
         - brew install jq
+        - brew install coreutils
         - brew install pigz
       script: make test
     - os: linux
@@ -25,5 +26,6 @@ matrix:
             - moreutils
             - fakeroot
             - jq
+            - coreutils
             - pigz
       script: debuild -uc -us

debian/changelog

@@ -5,6 +5,12 @@ github-backup-utils (2.20.0) UNRELEASED; urgency=medium
 
  -- Alejandro Figueroa <[email protected]>  Tue, 11 Feb 2020 20:47:17 +0000
 
+github-backup-utils (2.19.2) UNRELEASED; urgency=medium
+
+  * Fix `ghe-backup-repositories` performance for large instances #541
+
+ -- Evgenii Khramkov <[email protected]>  Fri, 31 Jan 2020 19:13:46 +0000
+
 github-backup-utils (2.19.1) UNRELEASED; urgency=medium
 
   * Cater for more explicit gist paths used in routes file #524

share/github-backup-utils/ghe-backup-config

@@ -304,3 +304,29 @@ ghe_debug() {
 version() {
   echo "${@#v}" | awk -F. '{ printf("%d%03d%03d%03d\n", $1,$2,$3,$4); }';
 }
+
+# The list of gists returned by the source changed in 2.16.23, 2.17.14,
+# 2.18.8, and 2.19.3.  We need to account for this difference here.
+# In older versions, all paths need to be truncated with `dirname`.
+# In newer versions, gist paths are unmodified, and only other repo types
+# are truncated with `dirname`.
+fix_paths_for_ghe_version() {
+  if [[ "$GHE_REMOTE_VERSION" =~ 2.16. && "$(version $GHE_REMOTE_VERSION)" -ge "$(version 2.16.23)" ]] || \
+     [[ "$GHE_REMOTE_VERSION" =~ 2.17. && "$(version $GHE_REMOTE_VERSION)" -ge "$(version 2.17.14)" ]] || \
+     [[ "$GHE_REMOTE_VERSION" =~ 2.18. && "$(version $GHE_REMOTE_VERSION)" -ge "$(version 2.18.8)" ]] || \
+     [[ "$(version $GHE_REMOTE_VERSION)" -ge "$(version 2.19.3)" ]]; then
+    GIST_FILTER="-e /gist/b"
+  else
+    unset GIST_FILTER
+  fi
+
+  # This sed expression is equivalent to running `dirname` on each line,
+  # but without all the fork+exec overhead of calling `dirname` that many
+  # times:
+  #   1. strip off trailing slashes
+  #   2. if the result has no slashes in it, the dirname is "."
+  #   3. truncate from the final slash (if any) to the end
+  # If the GIST_FILTER was set above (because we're on a modern version of
+  # GHES), then don't modify lines with "gist" in them.
+  sed $GIST_FILTER -e 's/\/$//; s/^[^\/]*$/./; s/\/[^\/]*$//'
+}

share/github-backup-utils/ghe-backup-repositories

@@ -366,25 +366,8 @@ bm_end "$(basename $0) - Special Data Directories Sync"
 
 if [ -z "$GHE_SKIP_ROUTE_VERIFICATION" ]; then
   bm_start "$(basename $0) - Verifying Routes"
-
-  # The list of gists returned by the source changed in 2.16.23, 2.17.14, 2.18.8 & 2.19.3
-  # so we need to account for this difference here.
-  parse_paths() {
-    while read -r line; do
-      if [[ "$GHE_REMOTE_VERSION" =~ 2.16 && "$(version $GHE_REMOTE_VERSION)" -ge "$(version 2.16.23)" ]] || \
-         [[ "$GHE_REMOTE_VERSION" =~ 2.17 && "$(version $GHE_REMOTE_VERSION)" -ge "$(version 2.17.14)" ]] || \
-         [[ "$GHE_REMOTE_VERSION" =~ 2.18 && "$(version $GHE_REMOTE_VERSION)" -ge "$(version 2.18.8)" ]] || \
-         [[ "$GHE_REMOTE_VERSION" =~ 2.19 && "$(version $GHE_REMOTE_VERSION)" -ge "$(version 2.19.3)" ]] && \
-         (echo "$line" | grep -q "gist"); then
-        echo "$line"
-      else
-        dirname "$line"
-      fi
-    done
-  }
-
-  cat $tempdir/*.rsync | sort | uniq > $tempdir/source_routes
-  (cd $backup_dir/ && find * -mindepth 5 -maxdepth 6 -type d -name \*.git | parse_paths | sort | uniq) > $tempdir/destination_routes
+  cat $tempdir/*.rsync | uniq | sort | uniq > $tempdir/source_routes
+  (cd $backup_dir/ && find * -mindepth 5 -maxdepth 6 -type d -name \*.git | fix_paths_for_ghe_version | uniq | sort | uniq) > $tempdir/destination_routes
 
   git --no-pager diff --unified=0 --no-prefix -- $tempdir/source_routes $tempdir/destination_routes || echo "Warning: One or more repository networks and/or gists were not found on the source appliance. Please contact GitHub Enterprise Support for assistance."
 

share/github-backup-utils/ghe-backup-storage

@@ -141,8 +141,8 @@ bm_end "$(basename $0) - Storage object sync"
 if [ -z "$GHE_SKIP_ROUTE_VERIFICATION" ]; then
   bm_start "$(basename $0) - Verifying Routes"
 
-  cat $tempdir/*.rsync | sort | uniq > $tempdir/source_routes
-  (cd $backup_dir/ && find * -mindepth 3 -maxdepth 3 -type f -print | sort | uniq) > $tempdir/destination_routes
+  cat $tempdir/*.rsync | uniq | sort | uniq > $tempdir/source_routes
+  (cd $backup_dir/ && find * -mindepth 3 -maxdepth 3 -type f -print | uniq | sort | uniq) > $tempdir/destination_routes
 
   git --no-pager diff --unified=0 --no-prefix -- $tempdir/source_routes $tempdir/destination_routes || echo "Warning: One or more storage objects were not found on the source appliance. Please contact GitHub Enterprise Support for assistance."
 

test/bin/dgit-cluster-backup-routes

@@ -1,11 +1,31 @@
 #!/usr/bin/env bash
 # Usage: dgit-cluster-backup-routes
-# Emulates the remote GitHub enterprise-configure command. Tests use this
-# to assert that the command was executed.
+# Emulates the remote GitHub dgit-cluster-backup-routes command. Tests use this
+# to assert that all repos and gists get backed up.
 set -e
-cat <<EOF
-0/nw/01/aa/3f/1234 git-server-fake-uuid
-1/nw/23/bb/4c/2345 git-server-fake-uuid1
-0/01/aa/3f/gist git-server-fake-uuid2
-1/23/bb/4c/gist git-server-fake-uuid
-EOF
+
+version() {
+  echo "${@#v}" | awk -F. '{ printf("%d%03d%03d%03d\n", $1,$2,$3,$4); }';
+}
+
+if [ -z "$GHE_REMOTE_VERSION" ]; then
+  echo GHE_REMOTE_VERSION must be set for this script to work.
+  exit 1
+fi
+
+# The list of gists returned by the source changed in 2.16.23, 2.17.14,
+# 2.18.8, and 2.19.3.  We need to account for this difference here.
+if [[ "$GHE_REMOTE_VERSION" =~ 2.16. && "$(version $GHE_REMOTE_VERSION)" -ge "$(version 2.16.23)" ]] || \
+   [[ "$GHE_REMOTE_VERSION" =~ 2.17. && "$(version $GHE_REMOTE_VERSION)" -ge "$(version 2.17.14)" ]] || \
+   [[ "$GHE_REMOTE_VERSION" =~ 2.18. && "$(version $GHE_REMOTE_VERSION)" -ge "$(version 2.18.8)" ]] || \
+   [[ "$(version $GHE_REMOTE_VERSION)" -ge "$(version 2.19.3)" ]]; then
+    echo 0/nw/01/aa/3f/1234 git-server-fake-uuid
+    echo 1/nw/23/bb/4c/2345 git-server-fake-uuid1
+    echo 0/01/aa/3f/gist/93069ad4c391b6203f183e147d52a97a.git git-server-fake-uuid2
+    echo 1/23/bb/4c/gist/1234.git git-server-fake-uuid
+else
+    echo 0/nw/01/aa/3f/1234 git-server-fake-uuid
+    echo 1/nw/23/bb/4c/2345 git-server-fake-uuid1
+    echo 0/01/aa/3f/gist git-server-fake-uuid2
+    echo 1/23/bb/4c/gist git-server-fake-uuid
+fi

test/test-ghe-backup.sh

@@ -1,9 +1,10 @@
 #!/usr/bin/env bash
 # ghe-backup command tests
 
+TESTS_DIR="$PWD/$(dirname "$0")"
 # Bring in testlib
 # shellcheck source=test/testlib.sh
-. "$(dirname "$0")/testlib.sh"
+. "$TESTS_DIR/testlib.sh"
 
 # Create the backup data dir and fake remote repositories dirs
 mkdir -p "$GHE_DATA_DIR" "$GHE_REMOTE_DATA_USER_DIR"
@@ -344,3 +345,68 @@ begin_test "ghe-backup missing directories or files on source appliance"
     verify_all_backedup_data
 )
 end_test
+
+# acceptance criteria is less then 2 seconds for 100,000 lines
+begin_test "ghe-backup fix_paths_for_ghe_version performance tests - gists"
+(
+    set -e
+    timeout 2 bash -c "
+        source '$TESTS_DIR/../share/github-backup-utils/ghe-backup-config'
+        GHE_REMOTE_VERSION=2.16.23
+        seq 1 100000 | sed -e 's/$/ gist/' | fix_paths_for_ghe_version | grep -c gist
+    "
+)
+end_test
+
+# acceptance criteria is less then 2 seconds for 100,000 lines
+begin_test "ghe-backup fix_paths_for_ghe_version performance tests - wikis"
+(
+    set -e
+    timeout 2 bash -c "
+        source '$TESTS_DIR/../share/github-backup-utils/ghe-backup-config'
+        GHE_REMOTE_VERSION=2.16.23
+        seq 1 100000 | sed -e 's/$/ wiki/' | fix_paths_for_ghe_version | grep -c '^\.$'
+    "
+)
+end_test
+
+# check fix_paths_for_ghe_version version thresholds
+begin_test "ghe-backup fix_paths_for_ghe_version newer/older"
+(
+    set -e
+
+    # modern versions keep foo/gist as foo/gist
+    for ver in 2.16.23 v2.16.23 v2.17.14 v2.18.8 v2.19.3 v2.20.0 v3.0.0; do
+        echo "## $ver, not gist"
+        [ "$(bash -c "
+            source '$TESTS_DIR/../share/github-backup-utils/ghe-backup-config'
+            GHE_REMOTE_VERSION=$ver
+            echo foo/bar | fix_paths_for_ghe_version
+        ")" == "foo" ]
+
+        echo "## $ver, gist"
+        [ "$(bash -c "
+            source '$TESTS_DIR/../share/github-backup-utils/ghe-backup-config'
+            GHE_REMOTE_VERSION=$ver
+            echo foo/gist | fix_paths_for_ghe_version
+        ")" == "foo/gist" ]
+    done
+
+    # old versions change foo/gist to foo
+    for ver in 1.0.0 bob a.b.c "" 1.2.16 2.0.0 v2.0.0 v2.15.123 v2.16.22 v2.17.13 v2.18.7 v2.19.2; do
+        echo "## $ver, not gist"
+        [ "$(bash -c "
+            source '$TESTS_DIR/../share/github-backup-utils/ghe-backup-config'
+            GHE_REMOTE_VERSION=$ver
+            echo foo/bar | fix_paths_for_ghe_version
+        ")" == "foo" ]
+
+        echo "## $ver, gist"
+        [ "$(bash -c "
+            source '$TESTS_DIR/../share/github-backup-utils/ghe-backup-config'
+            GHE_REMOTE_VERSION=$ver
+            echo foo/gist | fix_paths_for_ghe_version
+        ")" == "foo" ]
+    done
+)
+end_test