Merge pull request #1091 from github/3.10.0-release

3.10.0 RC release

Author: bonsohi

.github/workflows/lint.yml

@@ -1,8 +1,6 @@
 name: Lint Code Base
 
 on:
-  push:
-    branches-ignore: [master]
   pull_request:
     branches: [master]
 

.github/workflows/main.yml

@@ -2,7 +2,6 @@ name: Test and build
 
 on: [pull_request]
 
-
 jobs:
   build:
     strategy:

.gitignore

@@ -2,3 +2,5 @@
 /data
 /dist
 .DS_Store
+dash
+parallel

Dockerfile

@@ -26,6 +26,7 @@ RUN apt-get update && apt-get install --no-install-recommends -y \
     libssl-dev \
     git \
     jq \
+    bc \
     curl \
     tar \
     gzip \
@@ -54,8 +55,9 @@ RUN apt-get update && apt-get install --no-install-recommends -y \
     git \
     openssh-client \
     jq \
+    bc \
     moreutils \
-    gawk \ 
+    gawk \
     ca-certificates \
     xxhash \
     && rm -rf /var/lib/apt/lists/*

Makefile

@@ -1,6 +1,7 @@
 SHELL = /bin/sh
 
 test: info
+	@echo Running tests
 	@script/cibuild --no-package
 
 info:

backup.config-example

@@ -16,6 +16,22 @@ GHE_DATA_DIR="data"
 # be available for the past N days ...
 GHE_NUM_SNAPSHOTS=10
 
+# Pruning snapshots can be scheduled outside of the backup process.
+# If set to 'yes', snapshots will not be pruned by ghe-backup.
+# Instead, ghe-pruning-snapshots will need to be invoked separately via cron
+#GHE_PRUNING_SCHEDULED=yes
+
+# If --incremental is used to generate incremental MySQL backups with ghe-backup,
+# then you need to specify how many cycles of full and incremental backups will be 
+# performed before the next full backup is created.
+# For example, if `GHE_INCREMENTAL_BACKUP_MAX` is set to 14, backup-utils will 
+# run 1 full backup and then 13 incremental backups before performing another full backup on the next cycle.
+#GHE_INCREMENTAL_BACKUP_MAX=14
+
+# If GHE_SKIP_CHECKS is set to true (or if --skip-checks is used with ghe-backup) then ghe-host-check
+# disk space validation and software version checks on the backup-host will be disabled.
+#GHE_SKIP_CHECKS=false
+
 # The hostname of the GitHub appliance to restore. If you've set up a separate
 # GitHub appliance to act as a standby for recovery, specify its IP or hostname
 # here. The host to restore to may also be specified directly when running

bin/ghe-backup

@@ -5,9 +5,11 @@
 #/ the MySQL database, instance settings, GitHub Pages data, etc.
 #/
 #/ OPTIONS:
-#/   -v | --verbose    Enable verbose output.
-#/   -h | --help       Show this message.
-#/        --version    Display version information.
+#/   -v | --verbose       Enable verbose output.
+#/   -h | --help          Show this message.
+#/        --version       Display version information.
+#/   -i | --incremental   Incremental backup
+#/        --skip-checks   Skip storage/sw version checks
 #/
 
 set -e
@@ -27,6 +29,14 @@ while true; do
       export GHE_VERBOSE=true
       shift
       ;;
+    -i|--incremental)
+      export GHE_INCREMENTAL=true
+      shift
+      ;;
+    --skip-checks)
+      export GHE_SKIP_CHECKS=true
+      shift
+      ;;
     -*)
       echo "Error: invalid argument: '$1'" 1>&2
       exit 1
@@ -37,11 +47,45 @@ while true; do
   esac
 done
 
+
 export CALLING_SCRIPT="ghe-backup"
+
 # Bring in the backup configuration
 # shellcheck source=share/github-backup-utils/ghe-backup-config
 . "$( dirname "${BASH_SOURCE[0]}" )/../share/github-backup-utils/ghe-backup-config"
 
+# Setup progress tracking
+init-progress
+export PROGRESS_TOTAL=14 # Minimum number of steps in backup is 14
+echo "$PROGRESS_TOTAL" > /tmp/backup-utils-progress-total
+export PROGRESS_TYPE="Backup"
+echo "$PROGRESS_TYPE" > /tmp/backup-utils-progress-type
+export PROGRESS=0 # Used to track progress of backup
+echo "$PROGRESS" > /tmp/backup-utils-progress
+
+OPTIONAL_STEPS=0
+# Backup actions+mssql
+if ghe-ssh "$GHE_HOSTNAME" -- 'ghe-config --true app.actions.enabled'; then
+  OPTIONAL_STEPS=$((OPTIONAL_STEPS + 2))
+fi
+
+# Backup fsck
+if [ "$GHE_BACKUP_FSCK" = "yes" ]; then
+  OPTIONAL_STEPS=$((OPTIONAL_STEPS + 1))
+fi
+
+# Backup minio
+if ghe-ssh "$GHE_HOSTNAME" -- 'ghe-config --true app.minio.enabled'; then
+  OPTIONAL_STEPS=$((OPTIONAL_STEPS + 1))
+fi
+
+# Backup pages
+if [ "$GHE_BACKUP_PAGES" != "no" ]; then
+  OPTIONAL_STEPS=$((OPTIONAL_STEPS + 1))
+fi
+
+PROGRESS_TOTAL=$((OPTIONAL_STEPS + PROGRESS_TOTAL)) # Minimum number of steps in backup is 14
+echo "$PROGRESS_TOTAL" > /tmp/backup-utils-progress-total
 # Check to make sure moreutils parallel is installed and working properly
 ghe_parallel_check
 
@@ -121,7 +165,7 @@ ghe_restore_check
 # Check to see if there is a running backup
 if [ -h ../in-progress ]; then
 
-  log_error "Error: detected a backup already in progress from a previous version of ghe-backup. \nIf there is no backup in progress anymore, please remove \nthe $GHE_DATA_DIR/in-progress file." 1>&2
+  log_error "Detected a backup already in progress from a previous version of ghe-backup. \nIf there is no backup in progress anymore, please remove \nthe $GHE_DATA_DIR/in-progress file." >&2
   exit 1
 fi
 
@@ -139,17 +183,39 @@ if [ -f ../in-progress ]; then
   fi
 fi
 
+# Perform a host connection check and establish the remote appliance version.
+# The version is available in the GHE_REMOTE_VERSION variable and also written
+# to a version file in the snapshot directory itself.
+ghe_remote_version_required
+echo "$GHE_REMOTE_VERSION" > version
+
+# check that incremental settings are valid if set
+is_inc=$(is_incremental_backup_feature_on)
+
+if [ "$is_inc" = true ]; then
+if [ "$GHE_VERSION_MAJOR" -lt 3 ]; then
+  log_error "Can only perform incremental backups on enterprise version 3.10 or higher"
+  exit 1
+fi
+if [ "$GHE_VERSION_MINOR" -lt 10 ]; then
+  log_error "Can only perform incremental backups on enterprise version 3.10 or higher"
+  exit 1
+fi
+
+  incremental_backup_check
+  # If everything is ok, check if we have hit GHE_MAX_INCREMENTAL_BACKUPS, performing pruning actions if necessary
+  check_for_incremental_max_backups
+  # initialize incremental backup if it hasn't been done yet
+  incremental_backup_init
+fi
+
 echo "$GHE_SNAPSHOT_TIMESTAMP $$" > ../in-progress
 echo "$GHE_SNAPSHOT_TIMESTAMP $$" > "${GHE_DATA_DIR}/in-progress-backup"
 
 START_TIME=$(date +%s)
 log_info "Starting backup of $GHE_HOSTNAME with backup-utils v$BACKUP_UTILS_VERSION in snapshot $GHE_SNAPSHOT_TIMESTAMP"
 
-# Perform a host connection check and establish the remote appliance version.
-# The version is available in the GHE_REMOTE_VERSION variable and also written
-# to a version file in the snapshot directory itself.
-ghe_remote_version_required
-echo "$GHE_REMOTE_VERSION" > version
+
 
 if [ -n "$GHE_ALLOW_REPLICA_BACKUP" ]; then
   echo "Warning: backing up a high availability replica may result in inconsistent or unreliable backups."
@@ -259,6 +325,7 @@ echo \"$cmd_title\"
 ghe-backup-git-hooks || printf %s \"git-hooks \" >> \"$failures_file\"")
 
 if [ "$GHE_BACKUP_STRATEGY" = "rsync" ]; then
+  increment-progress-total-count 1
   cmd_title=$(log_info "Backing up Elasticsearch indices ...")
   commands+=("
   echo \"$cmd_title\"
@@ -292,7 +359,13 @@ if [ -z "$failures" ]; then
   rm -f "../current"
   ln -s "$GHE_SNAPSHOT_TIMESTAMP" "../current"
 
-  ghe-prune-snapshots
+  if [[ $GHE_PRUNING_SCHEDULED != "yes" ]]; then
+    ghe-prune-snapshots
+  else
+    log_info "Expired and incomplete snapshots to be pruned separately"
+  fi
+else 
+  log_info "Skipping pruning snapshots, since some backups failed..."
 fi
 
 END_TIME=$(date +%s)
@@ -306,6 +379,7 @@ else
   steps="${failures// /, }"
   ghe_remote_logger "Completed backup from $(hostname) / snapshot $GHE_SNAPSHOT_TIMESTAMP with failures: ${steps}."
   log_error "Error: Snapshot incomplete. Some steps failed: ${steps}. "
+  ghe_backup_finished
   exit 1
 fi
 

bin/ghe-backup-progress

@@ -0,0 +1,56 @@
+#!/usr/bin/env bash
+#/ Usage: ghe-backup-progress [--once]
+#/ Tracks the completed steps of a backup or restore operation.
+#/
+#/ By default the progress is printed every continuously or until a key is pressed.
+#/ Use the --once option to print the current progress once and exit.
+#/
+#/ Options:
+#/  --once  Don't loop, just print the current progress once.
+#
+set -e
+
+while true; do
+    case "$1" in
+        -o|--once)
+            ONCE=1
+            shift
+            ;;
+        -h|--help)
+            export GHE_SHOW_HELP=true
+            shift
+            ;;
+        -*)
+            echo "Unknown option: $1" >&2
+            exit 1
+            ;;
+        *)
+            break
+            ;;
+    esac
+done
+
+check_for_progress_file() {
+    if [ ! -f /tmp/backup-utils-progress-info ]; then
+        echo "No progress file found. Has a backup or restore been started?"
+        exit 1
+    fi
+}
+
+if [ -n "$ONCE" ]; then
+    check_for_progress_file
+    cat /tmp/backup-utils-progress-info
+else
+    check_for_progress_file
+    clear
+    cat /tmp/backup-utils-progress-info
+    while true; do
+    if read -r -t 1 -n 1; then
+	clear
+        exit ;
+    else
+	clear
+	cat /tmp/backup-utils-progress-info
+    fi
+    done
+fi