Merge pull request #627 from robbat2/gentoo

Improve Python3 & Finding Moreutils Parallel

Author: ryansimmen

share/github-backup-utils/ghe-backup-config

@@ -72,10 +72,20 @@ ghe_parallel_check() {
   fi
 
   # Some machines may have both moreutils parallel and GNU parallel installed.
+  # Check some variants to find it
   GHE_PARALLEL_COMMAND="parallel"
-  if [ -x "/usr/bin/parallel.moreutils" ]; then
-    GHE_PARALLEL_COMMAND="/usr/bin/parallel.moreutils"
-  fi
+  local x
+  for x in \
+      /usr/bin/parallel.moreutils \
+      /usr/bin/parallel_moreutils \
+      /usr/bin/moreutils.parallel \
+      /usr/bin/moreutils_parallel \
+      ; do
+        if [ -x "${x}" ]; then
+            GHE_PARALLEL_COMMAND="${x}"
+            break
+        fi
+  done
 
   # Check that the GHE_PARALLEL_COMMAND is pointing to moreutils parallel
   if ! $GHE_PARALLEL_COMMAND -h | grep -q "parallel \[OPTIONS\] command -- arguments"; then

test/bin/python

@@ -17,8 +17,12 @@ cat >/dev/null
 
 # verify the python compiles at least. if this fails then the python code passed
 # to -c failed basic syntax checks.
+# Compiler package ws removed in Py3
+# see https://www.python.org/dev/peps/pep-3108/#id53
+# The closest replacement is the ast package
+# https://docs.python.org/3/library/ast.html
 echo "$2" |
-/usr/bin/python2.7 -c "import sys; __import__('compiler').parse(sys.stdin.read())"
+/usr/bin/python3 -c "import sys; __import__('ast').parse(sys.stdin.read())"
 
 # pretend we found zero processes.
 echo 0

test/test-ghe-restore.sh

@@ -391,6 +391,7 @@ begin_test "ghe-restore cluster"
 (
   set -e
   rm -rf "$GHE_REMOTE_ROOT_DIR"
+  setup_moreutils_parallel
   setup_remote_metadata
   setup_remote_cluster
   echo "cluster" > "$GHE_DATA_DIR/current/strategy"
@@ -402,21 +403,14 @@ begin_test "ghe-restore cluster"
   GHE_RESTORE_HOST=127.0.0.1
   export GHE_RESTORE_HOST
 
-  # CI servers may have moreutils parallel and GNU parallel installed. We need moreutils parallel.
-  if [ -x "/usr/bin/parallel.moreutils" ]; then
-    ln -sf /usr/bin/parallel.moreutils "$ROOTDIR/test/bin/parallel"
-  fi
-
   # run ghe-restore and write output to file for asserting against
   if ! ghe-restore -v -f > "$TRASHDIR/restore-out" 2>&1; then
       cat "$TRASHDIR/restore-out"
       : ghe-restore should have exited successfully
       false
   fi
 
-  if [ -h "$ROOTDIR/test/bin/parallel" ]; then
-    unlink "$ROOTDIR/test/bin/parallel"
-  fi
+  cleanup_moreutils_parallel
 
   # for debugging
   cat "$TRASHDIR/restore-out"
@@ -461,6 +455,7 @@ begin_test "ghe-restore missing directories or files from source snapshot displa
     # Tests the scenario where something exists in the database, but not on disk.
     set -e
     rm -rf "$GHE_REMOTE_ROOT_DIR"
+    setup_moreutils_parallel
     setup_remote_metadata
     setup_remote_cluster
     echo "cluster" > "$GHE_DATA_DIR/current/strategy"
@@ -472,11 +467,6 @@ begin_test "ghe-restore missing directories or files from source snapshot displa
     GHE_RESTORE_HOST=127.0.0.1
     export GHE_RESTORE_HOST
 
-    # CI servers may have moreutils parallel and GNU parallel installed. We need moreutils parallel.
-    if [ -x "/usr/bin/parallel.moreutils" ]; then
-      ln -sf /usr/bin/parallel.moreutils "$ROOTDIR/test/bin/parallel"
-    fi
-
     # Tell dgit-cluster-restore-finalize and gist-cluster-restore-finalize to return warnings
     export GHE_DGIT_CLUSTER_RESTORE_FINALIZE_WARNING=1
     export GHE_GIST_CLUSTER_RESTORE_FINALIZE_WARNING=1
@@ -488,9 +478,7 @@ begin_test "ghe-restore missing directories or files from source snapshot displa
         false
     fi
 
-    if [ -h "$ROOTDIR/test/bin/parallel" ]; then
-      unlink "$ROOTDIR/test/bin/parallel"
-    fi
+    cleanup_moreutils_parallel
 
     # for debugging
     cat "$TRASHDIR/restore-out"

test/testlib.sh

@@ -431,3 +431,26 @@ verify_all_restored_data() {
   # verify common data
   verify_common_data
 }
+
+setup_moreutils_parallel() {
+  # CI servers may have moreutils parallel and GNU parallel installed.
+  # We need moreutils parallel
+  local x
+  for x in \
+      /usr/bin/parallel.moreutils \
+      /usr/bin/parallel_moreutils \
+      /usr/bin/moreutils.parallel \
+      /usr/bin/moreutils_parallel \
+      ; do
+        if [ -x "${x}" ]; then
+            ln -sf "${x}" "$ROOTDIR/test/bin/parallel"
+            break
+        fi
+  done
+}
+
+cleanup_moreutils_parallel() {
+  if [ -h "$ROOTDIR/test/bin/parallel" ]; then
+    unlink "$ROOTDIR/test/bin/parallel"
+  fi
+}