Merge remote-tracking branch 'private/master' into sync-private-to-public

Author: wirecat

.github/workflows/docker-image.yml

@@ -15,11 +15,11 @@ jobs:
     steps:
     - uses: actions/checkout@v3
     - name: Build the Debian Docker image
-      run: docker build . --file Dockerfile --tag backup-utils-debian:${GITHUB_RUN_ID}
+      run: docker build . --file Dockerfile --tag backup-utils:${GITHUB_RUN_ID}
     - name: Build the Alpine Docker image
       run: docker build . --file Dockerfile.alpine --tag backup-utils-alpine:${GITHUB_RUN_ID}
     - name: Run tests in Debian Docker image
-      run: docker run backup-utils-debian:${GITHUB_RUN_ID} ghe-backup --version
+      run: docker run backup-utils:${GITHUB_RUN_ID} ghe-backup --version
     - name: Run tests in Alpine Docker image
       run: docker run backup-utils-alpine:${GITHUB_RUN_ID} ghe-backup --version
 

.github/workflows/main.yml

@@ -2,11 +2,15 @@ name: Test and build
 
 on: [pull_request]
 
+
 jobs:
   build:
     strategy:
       matrix:
-        os: ['ubuntu-22.04', 'ubuntu-20.04', 'ubuntu-18.04', 'macos-latest']
+        # macos-latest references are kept here for historical purposes. removed macos-latest from the 
+        #matrix as it is not a typical case for users and causes a lot of friction with other linux-based 
+        # installs. Recommend developing on codespaces or using an ubuntu container. 
+        os: ['ubuntu-22.04', 'ubuntu-20.04', 'ubuntu-18.04']
       fail-fast: false
     runs-on: ${{ matrix.os }}
     steps:

.github/workflows/stale-support-escalation.yml

@@ -0,0 +1,27 @@
+
+name: 'Close stale support escalation issues'
+on:
+  push:
+    branches: 
+      - master
+  schedule:
+    - cron: '30 1 * * *' # Run each day at 1:30 UTC
+
+permissions:
+  issues: write
+  
+jobs:
+  stale:
+    if: github.repository == 'github/ghes' || github.repository == 'github/enterprise2' || github.repository == 'github/backup-utils-private'
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/stale@v7
+        with:
+          repo-token: ${{ secrets.GITHUB_TOKEN }}
+          stale-issue-message: 'This support escalation issue is stale because it has been open 30 days with no activity. To make it never stale, add a label never-stale.'
+          close-issue-message: 'This support escalation issue is closed because it has been open 45 days with no activity. To make it never stale, add a label never-stale.'
+          days-before-stale: 30
+          days-before-close: 45
+          only-labels: 'support-escalation'
+          stale-issue-label: 'stale'
+          exempt-issue-labels: 'never-stale,P1,P2,P3'

.github/workflows/stale.yml

@@ -0,0 +1,25 @@
+
+name: Mark stale issues and pull requests
+
+on:
+  schedule:
+  - cron: "0 0 * * *"
+
+jobs:
+  stale:
+
+    runs-on: ubuntu-latest
+
+    steps:
+    - uses: actions/stale@v7
+      with:
+        repo-token: ${{ secrets.GITHUB_TOKEN }}
+        stale-issue-message: "👋 This issue has been marked as stale because it has been open with no activity. You can: comment on the issue or remove the stale label to hold stale off for a while, add the `Keep` label to hold stale off permanently, or do nothing. If you do nothing this issue will be closed eventually by the stale bot."
+        stale-issue-label: "Stale"
+        exempt-issue-labels: "Keep"
+        stale-pr-message: "👋 This pull request has been marked as stale because it has been open with no activity. You can: comment on the issue or remove the stale label to hold stale off for a while, add the `Keep` label to hold stale off permanently, or do nothing. If you do nothing this pull request will be closed eventually by the stale bot."
+        stale-pr-label: "Stale"
+        exempt-pr-labels: "Keep, epic, initiative, GHAE"
+        days-before-stale: 30 # 1 month, which accounts for "now" and "next", but anything beyond is "never"
+        days-before-close: 5
+        ascending: true

Dockerfile

@@ -1,4 +1,4 @@
-FROM debian:buster-slim
+FROM ubuntu:focal
 
 RUN apt-get -q -y update && \
     apt-get install -y --no-install-recommends \

RELEASING.md

@@ -15,11 +15,24 @@ Only repo administrator is allowed to run the release script, otherwise it will
 Prior to making a release,
 
 1. Sync any changes that have been merged to backup-utils-private into this repository.
-1. Go through the list of open pull requests and merge any that are ready for merging.
-1. Go through the list of closed pull requests since the last release and ensure those that should be included in the release notes:
-  - have a "bug", "enhancement" or "feature" label,
-  - have a title that clearly describes the changes in that pull request. Reword if necessary.
-1. Perform a dry run (add `--dry-run` to one of the commands below) and verify the version strings are going to be changed and verify the release notes.
+
+  One possible way to accomplish this is to add the other repository as a remote and merge the changes from the default branch of that remote.
+  ```
+  git clone [email protected]:github/backup-utils
+  cd backup-utils
+  git checkout master
+  git checkout -b sync-private-to-public
+  git remote add private <private-repo>
+  git fetch private
+  git merge private/master
+  git push origin HEAD
+  ```
+  Then open a pull request on this repository with the changes.
+2. Go through the list of open pull requests and merge any that are ready for merging.
+3. Go through the list of closed pull requests since the last release and ensure those that should be included in the release notes:
+    - have a "bug", "enhancement" or "feature" label,
+    - have a title that clearly describes the changes in that pull request. Reword if necessary.
+4. Perform a dry run (add `--dry-run` to one of the commands below) and verify the version strings are going to be changed and verify the release notes.
 
 ## Automatic Process from chatops (internal to GitHub only)
 

bin/ghe-backup

@@ -44,6 +44,7 @@ done
 # Check to make sure moreutils parallel is installed and working properly
 ghe_parallel_check
 
+
 # Used to record failed backup steps
 failures=
 failures_file="$(mktemp -t backup-utils-backup-failures-XXXXXX)"
@@ -63,7 +64,7 @@ touch "incomplete"
 # Exit early if the snapshot filesystem doesn't support hard links, symlinks and
 # if rsync doesn't support hardlinking of dangling symlinks
 trap 'rm -rf src dest1 dest2' EXIT
-mkdir src
+mkdir -p src
 touch src/testfile
 if ! ln -s /data/does/not/exist/hooks/ src/ >/dev/null 2>&1; then
   echo "Error: the filesystem containing $GHE_DATA_DIR does not support symbolic links." 1>&2
@@ -105,6 +106,7 @@ cleanup () {
   fi
 
   rm -rf "$failures_file"
+  rm -f ${GHE_DATA_DIR}/in-progress-backup
 
   # Cleanup SSH multiplexing
   ghe-ssh --clean
@@ -114,10 +116,15 @@ cleanup () {
 trap 'cleanup' EXIT
 trap 'exit $?' INT # ^C always terminate
 
+
+# Check to see if there is a running restore
+ghe_restore_check
+
+# Check to see if there is a running backup
 if [ -h ../in-progress ]; then
   echo "Error: detected a backup already in progress from a previous version of ghe-backup." 1>&2
   echo "If there is no backup in progress anymore, please remove" 1>&2
-  echo "the $GHE_DATA_DIR/in-progress file." 1>&2
+  echo "the $GHE_DATA_DIR/in-progress file and try again." 1>&2
   exit 1
 fi
 
@@ -131,13 +138,15 @@ if [ -f ../in-progress ]; then
     unlink ../in-progress
   else
     echo "Error: A backup of $GHE_HOSTNAME may still be running on PID $pid." 1>&2
-    echo "If PID $pid is not a process related to the backup utilities, please remove" 1>&2
-    echo "the $GHE_DATA_DIR/in-progress file and try again." 1>&2
+    echo 1>&2
+    echo "  If PID $pid is not a process related to the backup utilities, please remove" 1>&2
+    echo "  the $GHE_DATA_DIR/in-progress file and try again." 1>&2
     exit 1
   fi
 fi
 
 echo "$GHE_SNAPSHOT_TIMESTAMP $$" > ../in-progress
+echo "$GHE_SNAPSHOT_TIMESTAMP $$" > ${GHE_DATA_DIR}/in-progress-backup
 
 START_TIME=$(date +%s)
 echo 'Start time:' $START_TIME
@@ -278,3 +287,6 @@ ghe-detect-leaked-ssh-keys -s "$GHE_SNAPSHOT_DIR" || true
 
 # Make sure we exit zero after the conditional
 true
+
+# Remove in-progress file
+ghe_backup_finished

bin/ghe-restore

@@ -122,6 +122,8 @@ cleanup () {
 
   # Cleanup SSH multiplexing
   ghe-ssh --clean
+  # Remove in-progress file
+  rm -f ${GHE_DATA_DIR}/in-progress-restore
 }
 
 # This function's type definition is being passed to a remote host via `ghe-ssh` but is not used locally.
@@ -152,6 +154,9 @@ cleanup_cluster_nodes() {
 # Check to make sure moreutils parallel is installed and working properly
 ghe_parallel_check
 
+# Check to make sure another restore process is not running
+ghe_restore_check
+
 # Grab the host arg
 GHE_HOSTNAME="${GHE_RESTORE_HOST_OPT:-$GHE_RESTORE_HOST}"
 
@@ -170,6 +175,9 @@ GHE_RESTORE_SNAPSHOT_PATH="$(ghe-restore-snapshot-path "$snapshot_id")"
 GHE_RESTORE_SNAPSHOT=$(basename "$GHE_RESTORE_SNAPSHOT_PATH")
 export GHE_RESTORE_SNAPSHOT
 
+# Check to make sure backup is not running
+ghe_backup_check
+
 # Detect if the backup we are restoring has a leaked ssh key
 echo "Checking for leaked keys in the backup snapshot that is being restored ..."
 ghe-detect-leaked-ssh-keys -s "$GHE_RESTORE_SNAPSHOT_PATH" || true
@@ -252,6 +260,8 @@ START_TIME=$(date +%s)
 echo 'Start time:' $START_TIME
 echo "Starting restore of $GHE_HOSTNAME with backup-utils v$BACKUP_UTILS_VERSION from snapshot $GHE_RESTORE_SNAPSHOT"
 ghe_remote_logger "Starting restore from $(hostname) with backup-utils v$BACKUP_UTILS_VERSION / snapshot $GHE_RESTORE_SNAPSHOT ..."
+# Create an in-progress-restore file to prevent simultaneous backup or restore runs
+echo "${START_TIME} $$" > ${GHE_DATA_DIR}/in-progress-restore
 
 # Keep other processes on the VM or cluster in the loop about the restore status.
 #
@@ -579,6 +589,7 @@ echo 'End time:' $END_TIME
 echo 'Runtime:' $(($END_TIME - $START_TIME)) 'seconds'
 
 echo "Restore of $GHE_HOSTNAME from snapshot $GHE_RESTORE_SNAPSHOT finished."
+ghe_restore_finished
 
 if ! $instance_configured; then
   echo "To complete the restore process, please visit https://$hostname/setup/settings to review and save the appliance configuration."

ownership.yaml

@@ -0,0 +1,49 @@
+---
+version: 1
+ownership:
+- name: backup-utils-private
+  long_name: backup-utils-private
+  description: backup-utils-private is the private fork of backup-utils
+  kind: code
+  repo: https://github.com/github/backup-utils-private
+  qos: critical
+  team: github/ghes-lifecycle
+  maintainer: shcorbett
+  exec_sponsor: scottdensmore
+  tier: 3
+  product_manager: davidjarzebowski
+  sev1:
+    pagerduty: https://github.pagerduty.com/escalation_policies#PBQWK20
+    tta: 30 minutes
+  sev2:
+    issue: https://github.com/github/ghes/issues/new?labels=ghes-lifecycle
+    tta: 1 business day
+  sev3:
+    slack: ghes-dev
+    tta: 1 week
+  support_squad:
+    slack: support-squad-infrastructure
+    issue: https://github.com/github/support-squad-infrastructure/issues
+- name: backup-utils
+  long_name: backup-utils
+  description: backup-utils is backup and restore tooling for Github enterprise server
+  kind: code
+  repo: https://github.com/github/backup-utils
+  qos: critical
+  team: github/ghes-lifecycle
+  maintainer: shcorbett
+  exec_sponsor: scottdensmore
+  tier: 3
+  product_manager: davidjarzebowski
+  sev1:
+    pagerduty: https://github.pagerduty.com/escalation_policies#PBQWK20
+    tta: 30 minutes
+  sev2:
+    issue: https://github.com/github/ghes/issues/new?labels=ghes-lifecycle
+    tta: 1 business day
+  sev3:
+    slack: ghes-dev
+    tta: 1 week
+  support_squad:
+    slack: support-squad-infrastructure
+    issue: https://github.com/github/support-squad-infrastructure/issues

share/github-backup-utils/ghe-backup-config

@@ -66,6 +66,60 @@ for f in "$GHE_BACKUP_CONFIG" "$GHE_BACKUP_ROOT/backup.config" \
   fi
 done
 
+GHE_RESTORE_IN_PROGRESS=$(readlink -fm "${GHE_DATA_DIR}/in-progress-restore")
+GHE_BACKUP_IN_PROGRESS=$(readlink -fm "${GHE_DATA_DIR}/in-progress-backup")
+
+export GHE_RESTORE_IN_PROGRESS
+export GHE_BACKUP_IN_PROGRESS
+
+ghe_restore_check() {
+  if [ -h $GHE_RESTORE_IN_PROGRESS ]; then
+    echo "  Error: detected a restore already in progress from a previous version of ghe-restore." 1>&2
+    echo "  If there is no restore in progress anymore, please remove" 1>&2
+    echo "  the $GHE_RESTORE_IN_PROGRESS file and try again." 1>&2
+    exit 1
+  fi
+
+  if [ -f $GHE_RESTORE_IN_PROGRESS ]; then
+    progress=$(cat $GHE_RESTORE_IN_PROGRESS)
+    pid=$(echo "$progress" | cut -d ' ' -f 2)
+    echo "  Error: A restore of $GHE_HOSTNAME may still be running on PID $pid." 1>&2
+    echo "  If PID $pid is not a process related to the restore utilities, please remove" 1>&2
+    echo "  the $GHE_RESTORE_IN_PROGRESS file and try again." 1>&2
+    exit 1
+  fi
+}
+
+ghe_backup_check() {
+  if [ -h $GHE_BACKUP_IN_PROGRESS ]; then
+    echo "  Error: detected a backup already in progress from a previous version of ghe-backup." 1>&2
+    echo "  If there is no backup in progress anymore, please remove" 1>&2
+    echo "  the $GHE_DATA_DIR/$GHE_BACKUP_IN_PROGRESS file and try again." 1>&2
+    exit 1
+  fi
+
+  if [ -f $GHE_BACKUP_IN_PROGRESS ]; then
+    progress=$(cat $GHE_BACKUP_IN_PROGRESS)
+    pid=$(echo "$progress" | cut -d ' ' -f 2)
+    echo "  Error: A backup of $GHE_HOSTNAME may still be running on PID $pid." 1>&2
+    echo "  If PID $pid is not a process related to the backup utilities, please remove" 1>&2
+    echo "  the $GHE_BACKUP_IN_PROGRESS file and try again." 1>&2
+    exit 1
+  fi
+}
+
+ghe_restore_finished() {
+  if [ -f $GHE_RESTORE_IN_PROGRESS ]; then
+    rm -f $GHE_RESTORE_IN_PROGRESS
+  fi
+}
+
+ghe_backup_finished() {
+  if [ -f $GHE_BACKUP_IN_PROGRESS ]; then
+    rm -f $GHE_BACKUP_IN_PROGRESS
+  fi
+}
+
 ghe_parallel_check() {
   if [ "$GHE_PARALLEL_ENABLED" != "yes" ]; then
     return 0