Merge pull request #630 from github/hao/fix-paralle-cmd

Update parallel command to adapt to changes in 3.11

Author: jianghao0718

.github/workflows/integration-tests.yml

@@ -37,6 +37,7 @@ jobs:
           janky-token: '${{ secrets.API_AUTH_TOKEN }}'
           job-name: '${{ matrix.jankyJobName }}'
           branch-name: '${{ env.SOURCE_BRANCH }}'
+          force : 'false'
           # enterprise2 target branch is same as target branch for PR (either master or enterprise-[0-9]*.[0-9]*-release)
           envVars: "JANKY_ENV_BACKUP_UTILS_BRANCH=${{ env.SOURCE_BRANCH }},JANKY_ENV_ENTERPRISE2_BRANCH=${{ env.TARGET_BRANCH }}"
 
@@ -63,5 +64,6 @@ jobs:
           janky-token: '${{ secrets.API_AUTH_TOKEN }}'
           job-name: '${{ matrix.jankyJobName }}'
           branch-name: '${{ env.SOURCE_BRANCH }}'
+          force : 'false'
           # enterprise2 target branch is same as target branch for PR (either master or enterprise-[0-9]*.[0-9]*-release)
           envVars: "JANKY_ENV_BACKUP_UTILS_BRANCH=${{ env.SOURCE_BRANCH }},JANKY_ENV_ENTERPRISE2_BRANCH=${{ env.TARGET_BRANCH }}"

share/github-backup-utils/ghe-backup-config

@@ -462,6 +462,24 @@ ghe_parse_remote_version() {
   export GHE_VERSION_MAJOR GHE_VERSION_MINOR GHE_VERSION_PATCH
 }
 
+# In 3.11 we started to install 2 different version parallel(s)
+# moreutils parallel and GNU parallel. When gnu parallel is installed,
+# it renames moreutils parallel to parallel.moreutils
+# set $PARALLEL_CMD envvar to be used in place of parallel commands
+ghe_remote_parallel() {
+  if [ -z "$GHE_REMOTE_VERSION" ]; then
+    echo "Error: ghe_remote_version_required needs to be invoked before ghe_remote_parallel" 1>&2
+    exit 1
+  fi
+
+  if [ "$GHE_VERSION_MINOR" -lt 11 ]; then
+    PARALLEL_CMD="parallel"
+  else
+    PARALLEL_CMD="parallel.moreutils"
+  fi
+  export PARALLEL_CMD
+}
+
 # Parses the <host> part out of a "<host>:<port>" or just "<host>" string.
 # This is used primarily to break hostspecs with non-standard ports down for
 # rsync commands.

share/github-backup-utils/ghe-restore-pages

@@ -35,6 +35,7 @@ fi
 
 # Perform a host-check and establish GHE_REMOTE_XXX variables.
 ghe_remote_version_required "$GHE_HOSTNAME"
+ghe_remote_parallel
 
 # Split host:port into parts
 port=$(ssh_port_part "$GHE_HOSTNAME")
@@ -156,7 +157,7 @@ if $CLUSTER; then
   ghe-ssh "$GHE_HOSTNAME" -- /bin/bash >&3 <<EOF
     split -l 1000 $remote_routes_list $remote_tempdir/chunk
     chunks=\$(find $remote_tempdir/ -name chunk\*)
-    parallel -i /bin/sh -c "cat {} | github-env ./bin/dpages-cluster-restore-finalize" -- \$chunks
+    $PARALLEL_CMD -i /bin/sh -c "cat {} | github-env ./bin/dpages-cluster-restore-finalize" -- \$chunks
 EOF
   increment-progress-total-count 1
   bm_end "$(basename $0) - Finalizing routes"

share/github-backup-utils/ghe-restore-repositories

@@ -36,6 +36,7 @@ fi
 
 # Perform a host-check and establish GHE_REMOTE_XXX variables.
 ghe_remote_version_required "$GHE_HOSTNAME"
+ghe_remote_parallel
 
 # Generate SSH config for forwarding
 # Split host:port into parts
@@ -210,7 +211,7 @@ if $CLUSTER; then
   ghe-ssh "$GHE_HOSTNAME" -- /bin/bash >&3 <<EOF
     split -l 1000 $remote_to_restore $remote_tempdir/chunk
     chunks=\$(find $remote_tempdir/ -name chunk\*)
-    parallel -i /bin/sh -c "cat {} | github-env ./bin/dgit-cluster-restore-finalize 2>>$remote_warnings" -- \$chunks
+    $PARALLEL_CMD -i /bin/sh -c "cat {} | github-env ./bin/dgit-cluster-restore-finalize 2>>$remote_warnings" -- \$chunks
 EOF
   increment-progress-total-count 1
   bm_end "$(basename $0) - Finalizing routes"

share/github-backup-utils/ghe-restore-repositories-gist

@@ -35,6 +35,7 @@ fi
 
 # Perform a host-check and establish GHE_REMOTE_XXX variables.
 ghe_remote_version_required "$GHE_HOSTNAME"
+ghe_remote_parallel
 
 # Generate SSH config for forwarding
 # Split host:port into parts
@@ -159,7 +160,7 @@ if $CLUSTER; then
   ghe-ssh "$GHE_HOSTNAME" -- /bin/bash >&3 <<EOF
     split -l 1000 $remote_to_restore $remote_tempdir/chunk
     chunks=\$(find $remote_tempdir/ -name chunk\*)
-    parallel -i /bin/sh -c "cat {} | github-env ./bin/gist-cluster-restore-finalize 2>>$remote_warnings" -- \$chunks
+    $PARALLEL_CMD -i /bin/sh -c "cat {} | github-env ./bin/gist-cluster-restore-finalize 2>>$remote_warnings" -- \$chunks
 EOF
   increment-progress-total-count 1
   bm_end "$(basename $0) - Finalizing routes"

share/github-backup-utils/ghe-restore-storage

@@ -39,6 +39,7 @@ fi
 
 # Perform a host-check and establish GHE_REMOTE_XXX variables.
 ghe_remote_version_required "$GHE_HOSTNAME"
+ghe_remote_parallel
 
 # Split host:port into parts
 port=$(ssh_port_part "$GHE_HOSTNAME")
@@ -171,7 +172,7 @@ if $CLUSTER; then
   ghe-ssh "$GHE_HOSTNAME" -- /bin/bash >&3 <<EOF
     split -l 1000 $remote_routes_list $remote_tempdir/chunk
     chunks=\$(find $remote_tempdir/ -name chunk\*)
-    parallel -i /bin/sh -c "cat {} | github-env ./bin/storage-cluster-restore-finalize" -- \$chunks
+    $PARALLEL_CMD -i /bin/sh -c "cat {} | github-env ./bin/storage-cluster-restore-finalize" -- \$chunks
 EOF
   increment-progress-total-count 1
   bm_end "$(basename $0) - Finalizing routes"