Merge branch 'master' into separate-pruning-snapshots

Author: chuckp22

.github/workflows/docker-image.yml

@@ -14,12 +14,15 @@ jobs:
 
     steps:
     - uses: actions/checkout@v3
-    - name: Build the Debian Docker image
+    - name: Build the Ubuntu Docker image
       run: docker build . --file Dockerfile --tag backup-utils:${GITHUB_RUN_ID}
     - name: Build the Alpine Docker image
       run: docker build . --file Dockerfile.alpine --tag backup-utils-alpine:${GITHUB_RUN_ID}
-    - name: Run tests in Debian Docker image
-      run: docker run backup-utils:${GITHUB_RUN_ID} ghe-backup --version
+    - name: Run tests in Ubuntu Docker image
+      run: |
+        docker run backup-utils:${GITHUB_RUN_ID} ghe-backup --version
+        docker run backup-utils:${GITHUB_RUN_ID} rsync --version
     - name: Run tests in Alpine Docker image
-      run: docker run backup-utils-alpine:${GITHUB_RUN_ID} ghe-backup --version
-      
+      run: |
+        docker run backup-utils-alpine:${GITHUB_RUN_ID} ghe-backup --version
+        docker run backup-utils-alpine:${GITHUB_RUN_ID} rsync --version

.github/workflows/lint.yml

@@ -1,8 +1,6 @@
 name: Lint Code Base
 
 on:
-  push:
-    branches-ignore: [master]
   pull_request:
     branches: [master]
 

.github/workflows/main.yml

@@ -2,7 +2,6 @@ name: Test and build
 
 on: [pull_request]
 
-
 jobs:
   build:
     strategy:

.github/workflows/rsync-docker-bump.yml

@@ -0,0 +1,34 @@
+name: Update Rsync Tag in Dockerfile
+
+on:
+  schedule:
+    - cron: '0 0 * * *' # Runs daily at 00:00
+
+jobs:
+  update-rsync-tag:
+    runs-on: ubuntu-latest
+
+    steps:
+      - name: Checkout repository
+        uses: actions/checkout@v3
+
+      - name: Get latest rsync tag
+        id: latest_tag
+        run: |
+          curl --silent "https://api.github.com/repos/WayneD/rsync/tags" | jq -r '.[0].name' | xargs -I {} echo "::set-output name=latest_tag::{}"
+
+      - name: Update Dockerfile with latest tag
+        run: |
+          sed -i -E "s/RSYNC_TAG=[0-9\.]+/RSYNC_TAG=${{ steps.latest_tag.outputs.latest_tag }}/g" Dockerfile
+
+      - name: Create Pull Request for tag update
+        uses: peter-evans/create-pull-request@v3
+        with:
+          token: ${{ secrets.GITHUB_TOKEN }}
+          commit-message: "Update rsync tag in Dockerfile"
+          title: "Update rsync tag in Dockerfile"
+          body: "This PR updates the rsync tag in the Dockerfile to the latest tagged version."
+          branch: "update-rsync-tag"
+          base: "master"
+          path: "."
+          labels: "automated-update,rsync"

Dockerfile

@@ -1,18 +1,73 @@
-FROM ubuntu:focal
+# Multi stage build for backup-utils
+# Build layer is for compiling rsync from source
+# Runtime layer is for running backup-utils
+# https://docs.docker.com/develop/develop-images/multistage-build/
+# https://docs.docker.com/engine/userguide/eng-image/multistage-build/
 
-RUN apt-get -q -y update && \
-    apt-get install -y --no-install-recommends \
-    tar \
+# Build layer
+FROM ubuntu:focal AS build
+
+# Install build dependencies
+RUN apt-get update && apt-get install --no-install-recommends -y \
+    gcc \
+    g++ \
+    gawk \
+    autoconf \
+    make \
+    automake \
+    python3-cmarkgfm \
+    acl \
+    libacl1-dev \
+    attr \
+    libattr1-dev \
+    libxxhash-dev \
+    libzstd-dev \
+    liblz4-dev \
+    libssl-dev \
+    git \
     jq \
-    rsync \
+    bc \
+    curl \
+    tar \
+    gzip \
     ca-certificates \
-    ssh \
+    && rm -rf /var/lib/apt/lists/*
+
+# Download rsync source from https://github.com/WayneD/rsync/archive/refs/tags/[TAG].tar.gz pinned to specified tag
+ARG RSYNC_TAG=v3.2.7
+RUN curl https://github.com/WayneD/rsync/archive/refs/tags/${RSYNC_TAG}.tar.gz -L -o ${RSYNC_TAG}.tar.gz
+RUN mkdir -p /rsync-${RSYNC_TAG}&& tar -xzf ${RSYNC_TAG}.tar.gz -C /rsync-${RSYNC_TAG} --strip-components=1 && ls -la
+# Change to the working directory of the rsync source
+WORKDIR /rsync-${RSYNC_TAG}
+RUN ls -la && ./configure
+RUN make
+RUN make install
+
+# Reset working directory
+WORKDIR /
+
+# Runtime layer
+FROM ubuntu:focal AS runtime
+
+# Install runtime dependencies -  bash, git, OpenSSH 5.6 or newer, and jq v1.5 or newer.
+RUN apt-get update && apt-get install --no-install-recommends -y \
+    bash \
     git \
+    openssh-client \
+    jq \
+    bc \
     moreutils \
     gawk \
+    ca-certificates \
+    xxhash \
     && rm -rf /var/lib/apt/lists/*
 
+# Copy rsync from build layer
+COPY --from=build /usr/local/bin/rsync /usr/local/bin/rsync
+
+# Copy backup-utils from repository into /backup-utils
 COPY ./ /backup-utils/
+
 WORKDIR /backup-utils
 
 RUN chmod +x /backup-utils/share/github-backup-utils/ghe-docker-init

Makefile

@@ -1,6 +1,7 @@
 SHELL = /bin/sh
 
 test: info
+	@echo Running tests
 	@script/cibuild --no-package
 
 info:

backup.config-example

@@ -46,6 +46,9 @@ GHE_NUM_SNAPSHOTS=10
 #
 #GHE_EXTRA_RSYNC_OPTS=""
 
+# If enabled and set to 'no', rsync warning message during backups will be suppressed.
+#RSYNC_WARNING=no
+
 
 # If set to 'yes', logging output will be colorized.
 #

bin/ghe-backup

@@ -37,11 +37,45 @@ while true; do
   esac
 done
 
+
 export CALLING_SCRIPT="ghe-backup"
+
 # Bring in the backup configuration
 # shellcheck source=share/github-backup-utils/ghe-backup-config
 . "$( dirname "${BASH_SOURCE[0]}" )/../share/github-backup-utils/ghe-backup-config"
 
+# Setup progress tracking
+init-progress
+export PROGRESS_TOTAL=14 # Minimum number of steps in backup is 14
+echo "$PROGRESS_TOTAL" > /tmp/backup-utils-progress-total
+export PROGRESS_TYPE="Backup"
+echo "$PROGRESS_TYPE" > /tmp/backup-utils-progress-type
+export PROGRESS=0 # Used to track progress of backup
+echo "$PROGRESS" > /tmp/backup-utils-progress
+
+OPTIONAL_STEPS=0
+# Backup actions+mssql
+if ghe-ssh "$GHE_HOSTNAME" -- 'ghe-config --true app.actions.enabled'; then
+  OPTIONAL_STEPS=$((OPTIONAL_STEPS + 2))
+fi
+
+# Backup fsck
+if [ "$GHE_BACKUP_FSCK" = "yes" ]; then
+  OPTIONAL_STEPS=$((OPTIONAL_STEPS + 1))
+fi
+
+# Backup minio
+if ghe-ssh "$GHE_HOSTNAME" -- 'ghe-config --true app.minio.enabled'; then
+  OPTIONAL_STEPS=$((OPTIONAL_STEPS + 1))
+fi
+
+# Backup pages
+if [ "$GHE_BACKUP_PAGES" != "no" ]; then
+  OPTIONAL_STEPS=$((OPTIONAL_STEPS + 1))
+fi
+
+PROGRESS_TOTAL=$((OPTIONAL_STEPS + PROGRESS_TOTAL)) # Minimum number of steps in backup is 14
+echo "$PROGRESS_TOTAL" > /tmp/backup-utils-progress-total
 # Check to make sure moreutils parallel is installed and working properly
 ghe_parallel_check
 
@@ -121,7 +155,7 @@ ghe_restore_check
 # Check to see if there is a running backup
 if [ -h ../in-progress ]; then
 
-  log_error "Error: detected a backup already in progress from a previous version of ghe-backup. \nIf there is no backup in progress anymore, please remove \nthe $GHE_DATA_DIR/in-progress file." 1>&2
+  log_error "Detected a backup already in progress from a previous version of ghe-backup. \nIf there is no backup in progress anymore, please remove \nthe $GHE_DATA_DIR/in-progress file." >&2
   exit 1
 fi
 
@@ -259,6 +293,7 @@ echo \"$cmd_title\"
 ghe-backup-git-hooks || printf %s \"git-hooks \" >> \"$failures_file\"")
 
 if [ "$GHE_BACKUP_STRATEGY" = "rsync" ]; then
+  increment-progress-total-count 1
   cmd_title=$(log_info "Backing up Elasticsearch indices ...")
   commands+=("
   echo \"$cmd_title\"
@@ -295,8 +330,10 @@ if [ -z "$failures" ]; then
   if [[ $GHE_PRUNING_SCHEDULED != "yes" ]]; then
     ghe-prune-snapshots
   else
-    echo "Expired and incomplete snapshots to be pruned separately"
+    log_info "Expired and incomplete snapshots to be pruned separately"
   fi
+else 
+  log_info "Skipping pruning snapshots, since some backups failed..."
 fi
 
 END_TIME=$(date +%s)
@@ -310,6 +347,7 @@ else
   steps="${failures// /, }"
   ghe_remote_logger "Completed backup from $(hostname) / snapshot $GHE_SNAPSHOT_TIMESTAMP with failures: ${steps}."
   log_error "Error: Snapshot incomplete. Some steps failed: ${steps}. "
+  ghe_backup_finished
   exit 1
 fi
 

bin/ghe-backup-progress

@@ -0,0 +1,56 @@
+#!/usr/bin/env bash
+#/ Usage: ghe-backup-progress [--once]
+#/ Tracks the completed steps of a backup or restore operation.
+#/
+#/ By default the progress is printed every continuously or until a key is pressed.
+#/ Use the --once option to print the current progress once and exit.
+#/
+#/ Options:
+#/  --once  Don't loop, just print the current progress once.
+#
+set -e
+
+while true; do
+    case "$1" in
+        -o|--once)
+            ONCE=1
+            shift
+            ;;
+        -h|--help)
+            export GHE_SHOW_HELP=true
+            shift
+            ;;
+        -*)
+            echo "Unknown option: $1" >&2
+            exit 1
+            ;;
+        *)
+            break
+            ;;
+    esac
+done
+
+check_for_progress_file() {
+    if [ ! -f /tmp/backup-utils-progress-info ]; then
+        echo "No progress file found. Has a backup or restore been started?"
+        exit 1
+    fi
+}
+
+if [ -n "$ONCE" ]; then
+    check_for_progress_file
+    cat /tmp/backup-utils-progress-info
+else
+    check_for_progress_file
+    clear
+    cat /tmp/backup-utils-progress-info
+    while true; do
+    if read -r -t 1 -n 1; then
+	clear
+        exit ;
+    else
+	clear
+	cat /tmp/backup-utils-progress-info
+    fi
+    done
+fi

bin/ghe-host-check

@@ -131,7 +131,7 @@ fi
 
 # backup-utils 2.13 onwards limits support to the current and previous two releases
 # of GitHub Enterprise Server.
-supported_minimum_version="3.6.0"
+supported_minimum_version="3.7.0"
 
 if [ "$(version $version)" -ge "$(version $supported_minimum_version)" ]; then
   supported=1
@@ -186,7 +186,7 @@ actions: $actions_disk_size MB
 mssql: $mssql_disk_size MB
 DATA_TRANSFER_SIZE
 
-  if [[ $available_space -lt $min_disk_req ]]; then
+  if [[ $((available_space / (1024 * 1024))) -lt $min_disk_req ]]; then
     echo "There is not enough disk space for the backup. Please allocate more space and continue." 1>&2
     exit 1
   fi
@@ -213,9 +213,17 @@ DATA_TRANSFER_SIZE
     echo "rsync version $rsync_version in backup-host does not meet minimum requirements." 1>&2
     echo "Please make sure you have the minimum required version of rsync: $min_rsync installed" 1>&2
     exit 1
-  else
-    echo "rsync ${rsync_version} >= required ($min_rsync)" 1>&2
+  elif [[ $rsync_version < 3.2.5 ]] && [[ $RSYNC_WARNING != "no" ]]; then
+  cat << WARN_MSG 1>&2
+**WARNING:** rsync version $rsync_version on backup host is less than 3.2.5, which could result in performance degradation.
+
+For more details, please read documentation at https://github.com/github/backup-utils/blob/master/docs/requirements.md#april-2023-update-of-rsync-requirements
+
+You can disable this warning by changing RSYNC_WARNING to 'no' in your backup.config file.
+   
+WARN_MSG
   fi
+  echo "rsync ${rsync_version} >= required ($min_rsync)" 1>&2
 
   ssh_version=$(ssh -V 2>&1 | awk '{print $1}'|grep -oPm 1 '[\d\.]+' |head -1 | tr -cd '[:digit:].\n')
   if awk "BEGIN {exit !($ssh_version < $min_openssh)}" &> /dev/null; then