Merge branch 'master' into use-ghe-redis-cli

Author: jlisam

Dockerfile.alpine

@@ -0,0 +1,22 @@
+FROM alpine:latest
+
+RUN apk --update --no-cache add \
+    tar \
+    rsync \
+    ca-certificates \
+    openssh \
+    git \
+    bash \
+    gawk \
+    procps
+
+WORKDIR /backup-utils
+ADD https://github.com/github/backup-utils/archive/stable.tar.gz /
+RUN tar xzvf /stable.tar.gz --strip-components=1 -C /backup-utils && \
+    mv /usr/bin/gawk /usr/bin/awk && \
+    rm -r /stable.tar.gz
+
+RUN chmod +x /backup-utils/share/github-backup-utils/ghe-docker-init
+
+ENTRYPOINT ["/backup-utils/share/github-backup-utils/ghe-docker-init"]
+CMD ["ghe-host-check"]

share/github-backup-utils/ghe-backup-config

@@ -72,10 +72,20 @@ ghe_parallel_check() {
   fi
 
   # Some machines may have both moreutils parallel and GNU parallel installed.
+  # Check some variants to find it
   GHE_PARALLEL_COMMAND="parallel"
-  if [ -x "/usr/bin/parallel.moreutils" ]; then
-    GHE_PARALLEL_COMMAND="/usr/bin/parallel.moreutils"
-  fi
+  local x
+  for x in \
+      /usr/bin/parallel.moreutils \
+      /usr/bin/parallel_moreutils \
+      /usr/bin/moreutils.parallel \
+      /usr/bin/moreutils_parallel \
+      ; do
+        if [ -x "${x}" ]; then
+            GHE_PARALLEL_COMMAND="${x}"
+            break
+        fi
+  done
 
   # Check that the GHE_PARALLEL_COMMAND is pointing to moreutils parallel
   if ! $GHE_PARALLEL_COMMAND -h | grep -q "parallel \[OPTIONS\] command -- arguments"; then

share/github-backup-utils/ghe-restore-storage

@@ -11,6 +11,9 @@ set -e
 # shellcheck source=share/github-backup-utils/ghe-backup-config
 . "$( dirname "${BASH_SOURCE[0]}" )/ghe-backup-config"
 
+# Check to make sure moreutils parallel is installed and working properly
+ghe_parallel_check
+
 # Show usage and bail with no arguments
 [ -z "$*" ] && print_usage
 
@@ -128,17 +131,32 @@ for file_list in $tempdir/*.rsync; do
   else
     server=$host
   fi
+
   storage_user=$(ghe-ssh $ssh_config_file_opt $server:$port -- stat -c %U /data/user/storage || echo git)
 
-  ghe_verbose "* Transferring data to $server ..."
+  rsync_commands+=("
+  if [ -n \"$GHE_VERBOSE\" ]; then
+    echo \"* Transferring data to $server ...\" 1>&3
+  fi
+
   ghe-rsync -arvHR --delete \
-    -e "ssh -q $opts -p $port $ssh_config_file_opt -l $user" \
-    --rsync-path="sudo -u $storage_user rsync" \
+    -e \"ssh -q $opts -p $port $ssh_config_file_opt -l $user\" \
+    --rsync-path=\"sudo -u $storage_user rsync\" \
     --files-from=$file_list \
     --size-only \
-    "$GHE_DATA_DIR/$GHE_RESTORE_SNAPSHOT/storage/./" \
-    "$server:$GHE_REMOTE_DATA_USER_DIR/storage/" 1>&3
+    \"$GHE_DATA_DIR/$GHE_RESTORE_SNAPSHOT/storage/./\" \
+    \"$server:$GHE_REMOTE_DATA_USER_DIR/storage/\" 1>&3
+    ")
 done
+
+if [ "$GHE_PARALLEL_ENABLED" = "yes" ]; then
+  $GHE_PARALLEL_COMMAND $GHE_PARALLEL_RSYNC_COMMAND_OPTIONS -- "${rsync_commands[@]}"
+else
+  for c in "${rsync_commands[@]}"; do
+    eval "$c"
+  done
+fi
+
 bm_end "$(basename $0) - Restoring objects"
 
 if $CLUSTER; then

test/bin/python

@@ -17,8 +17,12 @@ cat >/dev/null
 
 # verify the python compiles at least. if this fails then the python code passed
 # to -c failed basic syntax checks.
+# Compiler package ws removed in Py3
+# see https://www.python.org/dev/peps/pep-3108/#id53
+# The closest replacement is the ast package
+# https://docs.python.org/3/library/ast.html
 echo "$2" |
-/usr/bin/python2.7 -c "import sys; __import__('compiler').parse(sys.stdin.read())"
+/usr/bin/python3 -c "import sys; __import__('ast').parse(sys.stdin.read())"
 
 # pretend we found zero processes.
 echo 0

test/test-ghe-restore.sh

@@ -539,6 +539,7 @@ begin_test "ghe-restore cluster"
 (
   set -e
   rm -rf "$GHE_REMOTE_ROOT_DIR"
+  setup_moreutils_parallel
   setup_remote_metadata
   setup_remote_cluster
   echo "cluster" > "$GHE_DATA_DIR/current/strategy"
@@ -550,21 +551,14 @@ begin_test "ghe-restore cluster"
   GHE_RESTORE_HOST=127.0.0.1
   export GHE_RESTORE_HOST
 
-  # CI servers may have moreutils parallel and GNU parallel installed. We need moreutils parallel.
-  if [ -x "/usr/bin/parallel.moreutils" ]; then
-    ln -sf /usr/bin/parallel.moreutils "$ROOTDIR/test/bin/parallel"
-  fi
-
   # run ghe-restore and write output to file for asserting against
   if ! ghe-restore -v -f > "$TRASHDIR/restore-out" 2>&1; then
       cat "$TRASHDIR/restore-out"
       : ghe-restore should have exited successfully
       false
   fi
 
-  if [ -h "$ROOTDIR/test/bin/parallel" ]; then
-    unlink "$ROOTDIR/test/bin/parallel"
-  fi
+  cleanup_moreutils_parallel
 
   # for debugging
   cat "$TRASHDIR/restore-out"
@@ -609,6 +603,7 @@ begin_test "ghe-restore missing directories or files from source snapshot displa
     # Tests the scenario where something exists in the database, but not on disk.
     set -e
     rm -rf "$GHE_REMOTE_ROOT_DIR"
+    setup_moreutils_parallel
     setup_remote_metadata
     setup_remote_cluster
     echo "cluster" > "$GHE_DATA_DIR/current/strategy"
@@ -620,11 +615,6 @@ begin_test "ghe-restore missing directories or files from source snapshot displa
     GHE_RESTORE_HOST=127.0.0.1
     export GHE_RESTORE_HOST
 
-    # CI servers may have moreutils parallel and GNU parallel installed. We need moreutils parallel.
-    if [ -x "/usr/bin/parallel.moreutils" ]; then
-      ln -sf /usr/bin/parallel.moreutils "$ROOTDIR/test/bin/parallel"
-    fi
-
     # Tell dgit-cluster-restore-finalize and gist-cluster-restore-finalize to return warnings
     export GHE_DGIT_CLUSTER_RESTORE_FINALIZE_WARNING=1
     export GHE_GIST_CLUSTER_RESTORE_FINALIZE_WARNING=1
@@ -636,9 +626,7 @@ begin_test "ghe-restore missing directories or files from source snapshot displa
         false
     fi
 
-    if [ -h "$ROOTDIR/test/bin/parallel" ]; then
-      unlink "$ROOTDIR/test/bin/parallel"
-    fi
+    cleanup_moreutils_parallel
 
     # for debugging
     cat "$TRASHDIR/restore-out"

test/testlib.sh

@@ -519,3 +519,26 @@ enable_actions() {
 is_actions_enabled() {
   ghe-ssh "$GHE_HOSTNAME" -- 'ghe-config --true app.actions.enabled'
 }
+
+setup_moreutils_parallel() {
+  # CI servers may have moreutils parallel and GNU parallel installed.
+  # We need moreutils parallel
+  local x
+  for x in \
+      /usr/bin/parallel.moreutils \
+      /usr/bin/parallel_moreutils \
+      /usr/bin/moreutils.parallel \
+      /usr/bin/moreutils_parallel \
+      ; do
+        if [ -x "${x}" ]; then
+            ln -sf "${x}" "$ROOTDIR/test/bin/parallel"
+            break
+        fi
+  done
+}
+
+cleanup_moreutils_parallel() {
+  if [ -h "$ROOTDIR/test/bin/parallel" ]; then
+    unlink "$ROOTDIR/test/bin/parallel"
+  fi
+}