Merge branch 'master' into xn4p4lm-host-check-formatting-update

Author: xN4P4LM

.gitignore

@@ -1,3 +1,5 @@
 /backup.config
 /data
 /dist
+dash
+parallel

backup.config-example

@@ -16,6 +16,14 @@ GHE_DATA_DIR="data"
 # be available for the past N days ...
 GHE_NUM_SNAPSHOTS=10
 
+# Pruning snapshots can be scheduled outside of the backup process. If set to 'yes'
+# ghe-pruning-snapshots will need to be invoked separately via cron
+#GHE_PRUNING_SCHEDULED=yes
+
+# If GHE_SKIP_CHECKS is set to true (or if --skip-checks is used with ghe-backup) then ghe-host-check
+# disk space validation and software version checks on the backup-host will be disabled.
+#GHE_SKIP_CHECKS=false
+
 # The hostname of the GitHub appliance to restore. If you've set up a separate
 # GitHub appliance to act as a standby for recovery, specify its IP or hostname
 # here. The host to restore to may also be specified directly when running

bin/ghe-backup

@@ -5,9 +5,11 @@
 #/ the MySQL database, instance settings, GitHub Pages data, etc.
 #/
 #/ OPTIONS:
-#/   -v | --verbose    Enable verbose output.
-#/   -h | --help       Show this message.
-#/        --version    Display version information.
+#/   -v | --verbose       Enable verbose output.
+#/   -h | --help          Show this message.
+#/        --version       Display version information.
+#/   -i | --incremental   Incremental backup
+#/        --skip-checks   Skip storage/sw version checks
 #/
 
 set -e
@@ -27,6 +29,14 @@ while true; do
       export GHE_VERBOSE=true
       shift
       ;;
+    -i|--incremental)
+      export GHE_INCREMENTAL=true
+      shift
+      ;;
+    --skip-checks)
+      export GHE_SKIP_CHECKS=true
+      shift
+      ;;
     -*)
       echo "Error: invalid argument: '$1'" 1>&2
       exit 1
@@ -46,12 +56,36 @@ export CALLING_SCRIPT="ghe-backup"
 
 # Setup progress tracking
 init-progress
+export PROGRESS_TOTAL=14 # Minimum number of steps in backup is 14
+echo "$PROGRESS_TOTAL" > /tmp/backup-utils-progress-total
 export PROGRESS_TYPE="Backup"
 echo "$PROGRESS_TYPE" > /tmp/backup-utils-progress-type
 export PROGRESS=0 # Used to track progress of backup
 echo "$PROGRESS" > /tmp/backup-utils-progress
-export PROGRESS_TOTAL=18 # Maximum number of steps in backup
 
+OPTIONAL_STEPS=0
+# Backup actions+mssql
+if ghe-ssh "$GHE_HOSTNAME" -- 'ghe-config --true app.actions.enabled'; then
+  OPTIONAL_STEPS=$((OPTIONAL_STEPS + 2))
+fi
+
+# Backup fsck
+if [ "$GHE_BACKUP_FSCK" = "yes" ]; then
+  OPTIONAL_STEPS=$((OPTIONAL_STEPS + 1))
+fi
+
+# Backup minio
+if ghe-ssh "$GHE_HOSTNAME" -- 'ghe-config --true app.minio.enabled'; then
+  OPTIONAL_STEPS=$((OPTIONAL_STEPS + 1))
+fi
+
+# Backup pages
+if [ "$GHE_BACKUP_PAGES" != "no" ]; then
+  OPTIONAL_STEPS=$((OPTIONAL_STEPS + 1))
+fi
+
+PROGRESS_TOTAL=$((OPTIONAL_STEPS + PROGRESS_TOTAL)) # Minimum number of steps in backup is 14
+echo "$PROGRESS_TOTAL" > /tmp/backup-utils-progress-total
 # Check to make sure moreutils parallel is installed and working properly
 ghe_parallel_check
 
@@ -149,17 +183,39 @@ if [ -f ../in-progress ]; then
   fi
 fi
 
+# Perform a host connection check and establish the remote appliance version.
+# The version is available in the GHE_REMOTE_VERSION variable and also written
+# to a version file in the snapshot directory itself.
+ghe_remote_version_required
+echo "$GHE_REMOTE_VERSION" > version
+
+# check that incremental settings are valid if set
+is_inc=$(is_incremental_backup_feature_on)
+
+if [ "$is_inc" = true ]; then
+if [ "$GHE_VERSION_MAJOR" -lt 3 ]; then
+  log_error "Can only perform incremental backups on enterprise version 3.10 or higher"
+  exit 1
+fi
+if [ "$GHE_VERSION_MINOR" -lt 10 ]; then
+  log_error "Can only perform incremental backups on enterprise version 3.10 or higher"
+  exit 1
+fi
+
+  incremental_backup_check
+  # If everything is ok, check if we have hit GHE_MAX_INCREMENTAL_BACKUPS, performing pruning actions if necessary
+  check_for_incremental_max_backups
+  # initialize incremental backup if it hasn't been done yet
+  incremental_backup_init
+fi
+
 echo "$GHE_SNAPSHOT_TIMESTAMP $$" > ../in-progress
 echo "$GHE_SNAPSHOT_TIMESTAMP $$" > "${GHE_DATA_DIR}/in-progress-backup"
 
 START_TIME=$(date +%s)
 log_info "Starting backup of $GHE_HOSTNAME with backup-utils v$BACKUP_UTILS_VERSION in snapshot $GHE_SNAPSHOT_TIMESTAMP"
 
-# Perform a host connection check and establish the remote appliance version.
-# The version is available in the GHE_REMOTE_VERSION variable and also written
-# to a version file in the snapshot directory itself.
-ghe_remote_version_required
-echo "$GHE_REMOTE_VERSION" > version
+
 
 if [ -n "$GHE_ALLOW_REPLICA_BACKUP" ]; then
   echo "Warning: backing up a high availability replica may result in inconsistent or unreliable backups."
@@ -269,6 +325,7 @@ echo \"$cmd_title\"
 ghe-backup-git-hooks || printf %s \"git-hooks \" >> \"$failures_file\"")
 
 if [ "$GHE_BACKUP_STRATEGY" = "rsync" ]; then
+  increment-progress-total-count 1
   cmd_title=$(log_info "Backing up Elasticsearch indices ...")
   commands+=("
   echo \"$cmd_title\"
@@ -302,7 +359,11 @@ if [ -z "$failures" ]; then
   rm -f "../current"
   ln -s "$GHE_SNAPSHOT_TIMESTAMP" "../current"
 
-  ghe-prune-snapshots
+  if [[ $GHE_PRUNING_SCHEDULED != "yes" ]]; then
+    ghe-prune-snapshots
+  else
+    log_info "Expired and incomplete snapshots to be pruned separately"
+  fi
 else 
   log_info "Skipping pruning snapshots, since some backups failed..."
 fi

bin/ghe-host-check

@@ -144,7 +144,13 @@ if [ -z "$supported" ]; then
   exit 1
 fi
 
-if [[ "$CALLING_SCRIPT" == "ghe-backup" ]]; then
+if [[ "$CALLING_SCRIPT" == "ghe-backup" && "$GHE_SKIP_CHECKS" != "true" ]]; then
+  cat << SKIP_MSG 1>&2
+**You can disable the following storage & version checks by running ghe-backup with option "--skip-checks"
+OR updating GHE_SKIP_CHECKS to 'true' in your backup.config file.
+
+SKIP_MSG
+
   # Bring in the requirements file
   min_rsync=""
   min_openssh=""
@@ -178,18 +184,16 @@ if [[ "$CALLING_SCRIPT" == "ghe-backup" ]]; then
   echo "   - Recommended Disk requirement is $recommended_disk_req MB" 1>&2
   echo "" 1>&2
 
-  cat <<DATA_TRANSFER_SIZE 1>&2
-### Data Transfer Sizes 
- - repositories: $repos_disk_size MB
- - pages: $pages_disk_size MB
- - elasticsearch: $es_disk_size MB
- - storage: $stor_disk_size MB
- - minio: $minio_disk_size MB
- - mysql: $mysql_disk_size MB
- - actions: $actions_disk_size MB
- - mssql: $mssql_disk_size MB
-DATA_TRANSFER_SIZE
-  echo "" 1>&2
+  printf '### Data Transfer Sizes 
+ - repositories: %d MB
+ - pages: %d MB
+ - elasticsearch: %d MB
+ - storage: %d MB
+ - minio: %d MB
+ - mysql: %d MB
+ - actions: %d MB
+ - mssql: %d MB\n' \
+  "$repos_disk_size" "$pages_disk_size" "$es_disk_size" "$stor_disk_size" "$minio_disk_size" "$mysql_disk_size" "$actions_disk_size" "$mssql_disk_size" 1>&2
 
   if [[ $((available_space / (1024 * 1024))) -lt $min_disk_req ]]; then
     echo "There is not enough disk space for the backup. Please allocate more space and continue." 1>&2

bin/ghe-restore

@@ -71,6 +71,10 @@ while true; do
       export GHE_VERBOSE=true
       shift
       ;;
+     -i|--incremental)
+      export GHE_INCREMENTAL=true
+      shift
+      ;;
     -*)
       echo "Error: invalid argument: '$1'" 1>&2
       exit 1
@@ -275,17 +279,14 @@ fi
 # taking into account the options passed to the script and the appliance configuration
 # calculate restore steps
 OPTIONAL_STEPS=0
-# Cluster restores add an additional step
-if $CLUSTER ; then
-  OPTIONAL_STEPS=$((OPTIONAL_STEPS + 1))
-fi
+
 # Restoring UUID
 if [ -s "$GHE_RESTORE_SNAPSHOT_PATH/uuid" ] && ! $CLUSTER; then
   OPTIONAL_STEPS=$((OPTIONAL_STEPS + 1))
 fi
-# Restoring Actions
+# Restoring Actions + MSSQL
 if ghe-ssh "$GHE_HOSTNAME" -- 'ghe-config --true app.actions.enabled'; then
-  OPTIONAL_STEPS=$((OPTIONAL_STEPS + 1))
+  OPTIONAL_STEPS=$((OPTIONAL_STEPS + 2))
 fi
 # Restoring minio
 if ghe-ssh "$GHE_HOSTNAME" -- 'ghe-config --true app.minio.enabled'; then
@@ -305,10 +306,16 @@ fi
 if ! $CLUSTER && $instance_configured; then
   OPTIONAL_STEPS=$((OPTIONAL_STEPS + 1))
 fi
-# Maximum restore steps
-export PROGRESS_TOTAL=$((OPTIONAL_STEPS + 6))
+# Restoring settings + restore-chat-integration + restore-packages
+if $RESTORE_SETTINGS; then
+  OPTIONAL_STEPS=$((OPTIONAL_STEPS + 3))
+fi
+
+# Minimum number of steps is 7
+export PROGRESS_TOTAL=$((OPTIONAL_STEPS + 7))
 
 init-progress
+echo "$PROGRESS_TOTAL" > /tmp/backup-utils-progress-total
 export PROGRESS_TYPE="Restore"
 echo "$PROGRESS_TYPE" > /tmp/backup-utils-progress-type
 export PROGRESS=0 # Used to track progress of restore
@@ -318,6 +325,29 @@ echo "$PROGRESS" > /tmp/backup-utils-progress
 START_TIME=$(date +%s)
 log_info "Starting restore of $GHE_HOSTNAME with backup-utils v$BACKUP_UTILS_VERSION from snapshot $GHE_RESTORE_SNAPSHOT"
 
+if [ "$GHE_INCREMENTAL" ]; then
+  if [ "$GHE_VERSION_MAJOR" -lt 3 ]; then
+  log_error "Can only perform incremental restores on enterprise version 3.10 or higher"
+  exit 1
+fi
+if [ "$GHE_VERSION_MINOR" -lt 10 ]; then
+  log_error "Can only perform incremental restores on enterprise version 3.10 or higher"
+  exit 1
+fi
+  log_info "Incremental restore from snapshot $GHE_RESTORE_SNAPSHOT"
+  # If we see 'inc_previous' prepended to the snapshot name, then
+  # we set $INC_FULL_BACKUP and $INC_SNAPSHOT_DATA to $INC_PREVIOUS_FULL_BACKUP and
+  # $INC_PREVIOUS_SNAPSHOT_DATA respectively. Otherwise, leave them at default setting
+  # so that incremental restore is from current cycle
+  if [[ "$GHE_RESTORE_SNAPSHOT" =~ ^inc_previous ]]; then
+    INC_FULL_BACKUP=$INC_PREVIOUS_FULL_BACKUP
+    INC_SNAPSHOT_DATA=$INC_PREVIOUS_SNAPSHOT_DATA
+    log_info "Incremental restore from previous cycle snapshot. Using $INC_FULL_BACKUP"
+    log_info "Incremental restore from previous cycle snapshot. Using $INC_SNAPSHOT_DATA"
+  fi 
+  log_info "Validating snapshot $GHE_RESTORE_SNAPSHOT"
+  validate_inc_snapshot_data "$GHE_RESTORE_SNAPSHOT"
+fi
 ghe_remote_logger "Starting restore from $(hostname) with backup-utils v$BACKUP_UTILS_VERSION / snapshot $GHE_RESTORE_SNAPSHOT ..."
 # Create an in-progress-restore file to prevent simultaneous backup or restore runs
 echo "${START_TIME} $$" > "${GHE_DATA_DIR}/in-progress-restore"
@@ -446,6 +476,7 @@ ghe-restore-column-encryption-keys "$GHE_HOSTNAME"
 # Always restore secret scanning encryption keys
 if [ "$(version $GHE_REMOTE_VERSION)" -ge "$(version 3.8.0)" ]; then
   log_info "Always restore secret scanning encryption keys on GHES verions 3.8.0+"
+  increment-progress-total-count 1
   ghe-restore-secret-scanning-encryption-keys "$GHE_HOSTNAME"
 fi
 
@@ -490,9 +521,12 @@ if is_external_database_target_or_snapshot && $SKIP_MYSQL; then
   log_info "Skipping MySQL restore."
 else
   log_info "Restoring MySQL database from ${backup_snapshot_strategy} backup snapshot on an appliance configured for ${appliance_strategy} backups ..."
+  increment-progress-total-count 2
   ghe-restore-mysql "$GHE_HOSTNAME" 1>&3
 fi
 
+
+
 if ghe-ssh "$GHE_HOSTNAME" -- 'ghe-config --true app.actions.enabled'; then
   log_info "Stopping Actions before restoring databases ..."
   # We mark Actions as stopped even if the `ghe-actions-stop`
@@ -596,7 +630,7 @@ fi
 log_info  "Restarting memcached ..." 1>&3
 bm_start "$(basename $0) - Restarting memcached"
 echo "sudo restart -q memcached 2>/dev/null || true" |
-  ghe-ssh "$GHE_HOSTNAME" -- /bin/sh
+ghe-ssh "$GHE_HOSTNAME" -- /bin/sh
 bm_end "$(basename $0) - Restarting memcached"
 
 # Prevent GitHub Connect jobs running before we've had a chance to reset

docs/README.md

@@ -6,7 +6,7 @@
   - **[GitHub Enterprise Server version requirements](requirements.md#github-enterprise-version-requirements)**
 - **[Getting started](getting-started.md)**
 - **[Using the backup and restore commands](usage.md)**
-- **[Scheduling backups](scheduling-backups.md)**
+- **[Scheduling backups & snapshot pruning](scheduling-backups.md)**
 - **[Backup snapshot file structure](backup-snapshot-file-structure.md)**
 - **[How does Backup Utilities differ from a High Availability replica?](faq.md)**
 - **[Docker](docker.md)**

docs/scheduling-backups.md

@@ -1,10 +1,9 @@
-# Scheduling backups
+# Scheduling backups & snapshot pruning
 
 Regular backups should be scheduled using `cron(8)` or similar command
 scheduling service on the backup host. The backup frequency will dictate the
 worst case [recovery point objective (RPO)][1] in your backup plan. We recommend
 hourly backups at the least.
-
 ## Example scheduling usage
 
 The following examples assume the Backup Utilities are installed under
@@ -19,16 +18,34 @@ storage.
 
 To schedule hourly backup snapshots with verbose informational output written to
 a log file and errors generating an email:
+```
+[email protected]
 
-    [email protected]
-
-    0 * * * * /opt/backup-utils/bin/ghe-backup -v 1>>/opt/backup-utils/backup.log 2>&1
+0 * * * * /opt/backup-utils/bin/ghe-backup -v 1>>/opt/backup-utils/backup.log 2>&1
+```
 
 To schedule nightly backup snapshots instead, use:
 
-    [email protected]
+```
+[email protected]
+
+0 0 * * * /opt/backup-utils/bin/ghe-backup -v 1>>/opt/backup-utils/backup.log 2>&1
+```
+
+### Example snapshot pruning 
+
+By default all expired and incomplete snapshots are deleted at the end of the main
+backup process `ghe-backup`. If pruning these snapshots takes a long time you can
+choose to disable the pruning process from the backup run and schedule it separately.
+This can be achieved by enabling the `GHE_PRUNING_SCHEDULED` option in `backup.config`.
+Please note that this option is only avilable for `backup-utils` >= v3.10.0, if this option is enabled you will need to schedule the pruning script `ghe-prune-snapshots` using `cron` or a similar command scheduling service on the backup host.
+
+To schedule daily snapshot pruning, use:
 
-    0 0 * * * /opt/backup-utils/bin/ghe-backup -v 1>>/opt/backup-utils/backup.log 2>&1
+```
+[email protected]
 
+0 3 * * * /opt/backup-utils/share/github-backup-utils/ghe-prune-snapshots 1>>/opt/backup-utils/prune-snapshots.log 2>&1
+```
 
 [1]: https://en.wikipedia.org/wiki/Recovery_point_objective

docs/usage.md

@@ -73,13 +73,18 @@ appliance at IP "5.5.5.5":
     Restore of 5.5.5.5:122 from snapshot 20180326T020444 finished.
     To complete the restore process, please visit https://5.5.5.5/setup/settings to review and save the appliance configuration.
 
-A different backup snapshot may be selected by passing the `-s` argument and the
-datestamp-named directory from the backup location.
+A different backup snapshot may be selected by passing the `-s` argument to `ghe-restore` and specifying the
+datestamp-named directory from the backup location as the value.
 
 The `ghe-backup` and `ghe-restore` commands also have a verbose output mode
 (`-v`) that lists files as they're being transferred. It's often useful to
 enable when output is logged to a file.
 
+Every time you execute `ghe-backup` we verify the storage and software setup of the host 
+you [installed][1] Backup Utilities on, to make sure our [requirements][2] for the host are 
+met. You can disable this check using the `--skip-checks` argument or by 
+adding `GHE_SKIP_CHECKS=true` to your configuration file.
+
 ### Restoring settings, TLS certificate, and license 
 
 When restoring to a new GitHub Enterprise Server instance, settings, certificate, and
@@ -107,3 +112,4 @@ GitHub Actions enabled, the following steps are required:
 Please refer to [GHES Documentation](https://docs.github.com/en/enterprise-server/admin/github-actions/advanced-configuration-and-troubleshooting/backing-up-and-restoring-github-enterprise-server-with-github-actions-enabled) for more details.
 
 [1]: https://github.com/github/backup-utils/blob/master/docs/getting-started.md
+[2]: requirements.md