Merge branch 'master' into Luke-J-Reid-patch-1

Author: Luke-J-Reid

.github/workflows/integration-tests.yml

@@ -37,6 +37,7 @@ jobs:
           janky-token: '${{ secrets.API_AUTH_TOKEN }}'
           job-name: '${{ matrix.jankyJobName }}'
           branch-name: '${{ env.SOURCE_BRANCH }}'
+          force : 'false'
           # enterprise2 target branch is same as target branch for PR (either master or enterprise-[0-9]*.[0-9]*-release)
           envVars: "JANKY_ENV_BACKUP_UTILS_BRANCH=${{ env.SOURCE_BRANCH }},JANKY_ENV_ENTERPRISE2_BRANCH=${{ env.TARGET_BRANCH }}"
 
@@ -63,5 +64,6 @@ jobs:
           janky-token: '${{ secrets.API_AUTH_TOKEN }}'
           job-name: '${{ matrix.jankyJobName }}'
           branch-name: '${{ env.SOURCE_BRANCH }}'
+          force : 'false'
           # enterprise2 target branch is same as target branch for PR (either master or enterprise-[0-9]*.[0-9]*-release)
           envVars: "JANKY_ENV_BACKUP_UTILS_BRANCH=${{ env.SOURCE_BRANCH }},JANKY_ENV_ENTERPRISE2_BRANCH=${{ env.TARGET_BRANCH }}"

bin/ghe-backup

@@ -118,6 +118,7 @@ cleanup () {
 
   rm -rf "$failures_file"
   rm -f "${GHE_DATA_DIR}/in-progress-backup"
+  rm -rf /tmp/backup-utils-progress/*
 
   # Cleanup SSH multiplexing
   ghe-ssh --clean

bin/ghe-restore

@@ -139,6 +139,9 @@ cleanup () {
     log_error "Failed to remove in-progress file" 1>&3
   fi
 
+  # Remove progress files
+  rm -rf /tmp/backup-utils-progress/*
+
   bm_end "$(basename $0)"
 }
 

share/github-backup-utils/ghe-backup-config

@@ -462,6 +462,24 @@ ghe_parse_remote_version() {
   export GHE_VERSION_MAJOR GHE_VERSION_MINOR GHE_VERSION_PATCH
 }
 
+# In 3.11 we started to install 2 different version parallel(s)
+# moreutils parallel and GNU parallel. When gnu parallel is installed,
+# it renames moreutils parallel to parallel.moreutils
+# set $PARALLEL_CMD envvar to be used in place of parallel commands
+ghe_remote_parallel() {
+  if [ -z "$GHE_REMOTE_VERSION" ]; then
+    echo "Error: ghe_remote_version_required needs to be invoked before ghe_remote_parallel" 1>&2
+    exit 1
+  fi
+
+  if [ "$GHE_VERSION_MINOR" -lt 11 ]; then
+    PARALLEL_CMD="parallel"
+  else
+    PARALLEL_CMD="parallel.moreutils"
+  fi
+  export PARALLEL_CMD
+}
+
 # Parses the <host> part out of a "<host>:<port>" or just "<host>" string.
 # This is used primarily to break hostspecs with non-standard ports down for
 # rsync commands.

share/github-backup-utils/ghe-restore-pages

@@ -35,6 +35,7 @@ fi
 
 # Perform a host-check and establish GHE_REMOTE_XXX variables.
 ghe_remote_version_required "$GHE_HOSTNAME"
+ghe_remote_parallel
 
 # Split host:port into parts
 port=$(ssh_port_part "$GHE_HOSTNAME")
@@ -156,7 +157,7 @@ if $CLUSTER; then
   ghe-ssh "$GHE_HOSTNAME" -- /bin/bash >&3 <<EOF
     split -l 1000 $remote_routes_list $remote_tempdir/chunk
     chunks=\$(find $remote_tempdir/ -name chunk\*)
-    parallel -i /bin/sh -c "cat {} | github-env ./bin/dpages-cluster-restore-finalize" -- \$chunks
+    $PARALLEL_CMD -i /bin/sh -c "cat {} | github-env ./bin/dpages-cluster-restore-finalize" -- \$chunks
 EOF
   increment-progress-total-count 1
   bm_end "$(basename $0) - Finalizing routes"

share/github-backup-utils/ghe-restore-repositories

@@ -36,6 +36,7 @@ fi
 
 # Perform a host-check and establish GHE_REMOTE_XXX variables.
 ghe_remote_version_required "$GHE_HOSTNAME"
+ghe_remote_parallel
 
 # Generate SSH config for forwarding
 # Split host:port into parts
@@ -210,7 +211,7 @@ if $CLUSTER; then
   ghe-ssh "$GHE_HOSTNAME" -- /bin/bash >&3 <<EOF
     split -l 1000 $remote_to_restore $remote_tempdir/chunk
     chunks=\$(find $remote_tempdir/ -name chunk\*)
-    parallel -i /bin/sh -c "cat {} | github-env ./bin/dgit-cluster-restore-finalize 2>>$remote_warnings" -- \$chunks
+    $PARALLEL_CMD -i /bin/sh -c "cat {} | github-env ./bin/dgit-cluster-restore-finalize 2>>$remote_warnings" -- \$chunks
 EOF
   increment-progress-total-count 1
   bm_end "$(basename $0) - Finalizing routes"

share/github-backup-utils/ghe-restore-repositories-gist

@@ -35,6 +35,7 @@ fi
 
 # Perform a host-check and establish GHE_REMOTE_XXX variables.
 ghe_remote_version_required "$GHE_HOSTNAME"
+ghe_remote_parallel
 
 # Generate SSH config for forwarding
 # Split host:port into parts
@@ -159,7 +160,7 @@ if $CLUSTER; then
   ghe-ssh "$GHE_HOSTNAME" -- /bin/bash >&3 <<EOF
     split -l 1000 $remote_to_restore $remote_tempdir/chunk
     chunks=\$(find $remote_tempdir/ -name chunk\*)
-    parallel -i /bin/sh -c "cat {} | github-env ./bin/gist-cluster-restore-finalize 2>>$remote_warnings" -- \$chunks
+    $PARALLEL_CMD -i /bin/sh -c "cat {} | github-env ./bin/gist-cluster-restore-finalize 2>>$remote_warnings" -- \$chunks
 EOF
   increment-progress-total-count 1
   bm_end "$(basename $0) - Finalizing routes"

share/github-backup-utils/ghe-restore-storage

@@ -39,6 +39,7 @@ fi
 
 # Perform a host-check and establish GHE_REMOTE_XXX variables.
 ghe_remote_version_required "$GHE_HOSTNAME"
+ghe_remote_parallel
 
 # Split host:port into parts
 port=$(ssh_port_part "$GHE_HOSTNAME")
@@ -171,7 +172,7 @@ if $CLUSTER; then
   ghe-ssh "$GHE_HOSTNAME" -- /bin/bash >&3 <<EOF
     split -l 1000 $remote_routes_list $remote_tempdir/chunk
     chunks=\$(find $remote_tempdir/ -name chunk\*)
-    parallel -i /bin/sh -c "cat {} | github-env ./bin/storage-cluster-restore-finalize" -- \$chunks
+    $PARALLEL_CMD -i /bin/sh -c "cat {} | github-env ./bin/storage-cluster-restore-finalize" -- \$chunks
 EOF
   increment-progress-total-count 1
   bm_end "$(basename $0) - Finalizing routes"

share/github-backup-utils/track-progress

@@ -2,11 +2,17 @@
 #/ track-progress: track progress of backup or restore tasks
 
 progress(){
-    
+  ## Those progress files should be created by init_progress function
+  ## If they are not present (e.g., individual script is being invoked directly), 
+  ## we will not track progress
+  if [ -f "/tmp/backup-utils-progress/progress" ] && 
+    [ -f "/tmp/backup-utils-progress/total" ] &&
+    [ -f "/tmp/backup-utils-progress/type" ]; then
     PROGRESS=$(cat /tmp/backup-utils-progress/progress)
     PROGRESS_TOTAL=$(cat /tmp/backup-utils-progress/total)
     PROGRESS_TYPE=$(cat /tmp/backup-utils-progress/type)
     PROGRESS_PERCENT=$( echo "scale = 2; ($PROGRESS / $PROGRESS_TOTAL) * 100" | bc)
     echo $((PROGRESS + 1)) > /tmp/backup-utils-progress/progress
     echo "${PROGRESS_TYPE} progress: $PROGRESS_PERCENT % ($PROGRESS / $PROGRESS_TOTAL ) $1 " > /tmp/backup-utils-progress/info
+  fi
 }