Skip to content

Commit 29b75d9

Browse files
bonsohibonsohi
authored
Merge pull request #1111 from github/3.10.0-release
3.10.0 GA release
2 parents 63438af + af4c609 commit 29b75d9

File tree

10 files changed

+100
-87
lines changed

10 files changed

+100
-87
lines changed

‎.github/workflows/main.yml

Lines changed: 3 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -17,9 +17,9 @@ jobs:
1717
run: |
1818
sudo apt-get update -y
1919
sudo apt-get install -y devscripts debhelper moreutils fakeroot jq pigz help2man
20-
wget "https://github.com/koalaman/shellcheck/releases/download/latest/shellcheck-latest.linux.x86_64.tar.xz"
21-
tar --xz -xvf "shellcheck-latest.linux.x86_64.tar.xz"
22-
sudo cp shellcheck-latest/shellcheck /usr/bin/shellcheck
20+
wget "https://github.com/koalaman/shellcheck/releases/download/stable/shellcheck-stable.linux.x86_64.tar.xz"
21+
tar --xz -xvf "shellcheck-stable.linux.x86_64.tar.xz"
22+
sudo cp shellcheck-stable/shellcheck /usr/bin/shellcheck
2323
if: matrix.os != 'macos-latest'
2424
- name: Install Dependencies (macOS)
2525
run: |

‎bin/ghe-backup

Lines changed: 35 additions & 31 deletions
Original file line numberDiff line numberDiff line change
@@ -54,38 +54,7 @@ export CALLING_SCRIPT="ghe-backup"
5454
# shellcheck source=share/github-backup-utils/ghe-backup-config
5555
. "$( dirname "${BASH_SOURCE[0]}" )/../share/github-backup-utils/ghe-backup-config"
5656

57-
# Setup progress tracking
58-
init-progress
59-
export PROGRESS_TOTAL=14 # Minimum number of steps in backup is 14
60-
echo "$PROGRESS_TOTAL" > /tmp/backup-utils-progress-total
61-
export PROGRESS_TYPE="Backup"
62-
echo "$PROGRESS_TYPE" > /tmp/backup-utils-progress-type
63-
export PROGRESS=0 # Used to track progress of backup
64-
echo "$PROGRESS" > /tmp/backup-utils-progress
65-
66-
OPTIONAL_STEPS=0
67-
# Backup actions+mssql
68-
if ghe-ssh "$GHE_HOSTNAME" -- 'ghe-config --true app.actions.enabled'; then
69-
OPTIONAL_STEPS=$((OPTIONAL_STEPS + 2))
70-
fi
71-
72-
# Backup fsck
73-
if [ "$GHE_BACKUP_FSCK" = "yes" ]; then
74-
OPTIONAL_STEPS=$((OPTIONAL_STEPS + 1))
75-
fi
76-
77-
# Backup minio
78-
if ghe-ssh "$GHE_HOSTNAME" -- 'ghe-config --true app.minio.enabled'; then
79-
OPTIONAL_STEPS=$((OPTIONAL_STEPS + 1))
80-
fi
8157

82-
# Backup pages
83-
if [ "$GHE_BACKUP_PAGES" != "no" ]; then
84-
OPTIONAL_STEPS=$((OPTIONAL_STEPS + 1))
85-
fi
86-
87-
PROGRESS_TOTAL=$((OPTIONAL_STEPS + PROGRESS_TOTAL)) # Minimum number of steps in backup is 14
88-
echo "$PROGRESS_TOTAL" > /tmp/backup-utils-progress-total
8958
# Check to make sure moreutils parallel is installed and working properly
9059
ghe_parallel_check
9160

@@ -186,9 +155,44 @@ fi
186155
# Perform a host connection check and establish the remote appliance version.
187156
# The version is available in the GHE_REMOTE_VERSION variable and also written
188157
# to a version file in the snapshot directory itself.
158+
# ghe_remote_version_required should be run before any other instances of ghe-ssh
159+
# to ensure that there are no problems with host key verification.
189160
ghe_remote_version_required
190161
echo "$GHE_REMOTE_VERSION" > version
191162

163+
# Setup progress tracking
164+
init-progress
165+
export PROGRESS_TOTAL=14 # Minimum number of steps in backup is 14
166+
echo "$PROGRESS_TOTAL" > /tmp/backup-utils-progress-total
167+
export PROGRESS_TYPE="Backup"
168+
echo "$PROGRESS_TYPE" > /tmp/backup-utils-progress-type
169+
export PROGRESS=0 # Used to track progress of backup
170+
echo "$PROGRESS" > /tmp/backup-utils-progress
171+
172+
OPTIONAL_STEPS=0
173+
# Backup actions+mssql
174+
if ghe-ssh "$GHE_HOSTNAME" -- 'ghe-config --true app.actions.enabled'; then
175+
OPTIONAL_STEPS=$((OPTIONAL_STEPS + 2))
176+
fi
177+
178+
# Backup fsck
179+
if [ "$GHE_BACKUP_FSCK" = "yes" ]; then
180+
OPTIONAL_STEPS=$((OPTIONAL_STEPS + 1))
181+
fi
182+
183+
# Backup minio
184+
if ghe-ssh "$GHE_HOSTNAME" -- 'ghe-config --true app.minio.enabled'; then
185+
OPTIONAL_STEPS=$((OPTIONAL_STEPS + 1))
186+
fi
187+
188+
# Backup pages
189+
if [ "$GHE_BACKUP_PAGES" != "no" ]; then
190+
OPTIONAL_STEPS=$((OPTIONAL_STEPS + 1))
191+
fi
192+
193+
PROGRESS_TOTAL=$((OPTIONAL_STEPS + PROGRESS_TOTAL)) # Minimum number of steps in backup is 14
194+
echo "$PROGRESS_TOTAL" > /tmp/backup-utils-progress-total
195+
192196
# check that incremental settings are valid if set
193197
is_inc=$(is_incremental_backup_feature_on)
194198

‎bin/ghe-host-check

Lines changed: 13 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -91,13 +91,19 @@ if ghe-ssh "$host" -- \
9191
CLUSTER=true
9292
fi
9393

94-
# ensure all nodes in the cluster are running the same version
94+
# ensure all nodes in the cluster are online/reachable and running the same version
9595
if "$CLUSTER"; then
96+
online_status=$(ghe-ssh "$host" ghe-cluster-host-check)
97+
if [ "$online_status" != "Cluster is ready to configure." ]; then
98+
echo "Error: Not all nodes are online! Please ensure cluster is in a healthy state before using backup-utils." 1>&2
99+
exit 1
100+
fi
101+
96102
node_version_list=$(ghe-ssh "$host" ghe-cluster-each -- ghe-version)
97103
distinct_versions=$(echo "$node_version_list" | awk '{split($0, a, ":"); print a[2]}' | awk '{print $4}' | uniq | wc -l)
98104
if [ "$distinct_versions" -ne 1 ]; then
99-
echo "$node_version_list" 1>&2
100-
echo "Error: Not all nodes are running the same version! Please ensure all nodes are running the same version before using backup-utils." 1>&3
105+
echo "Version mismatch: $node_version_list" 1>&2
106+
echo "Error: Not all nodes are running the same version! Please ensure all nodes are running the same version before using backup-utils." 1>&2
101107
exit 1
102108
fi
103109
fi
@@ -184,15 +190,17 @@ SKIP_MSG
184190
echo " - Recommended Disk requirement is $recommended_disk_req MB" 1>&2
185191
echo "" 1>&2
186192

187-
printf '### Data Transfer Sizes
193+
printf '### Estimated Data Transfer Sizes
194+
188195
- repositories: %d MB
189196
- pages: %d MB
190197
- elasticsearch: %d MB
191198
- storage: %d MB
192199
- minio: %d MB
193200
- mysql: %d MB
194201
- actions: %d MB
195-
- mssql: %d MB\n' \
202+
- mssql: %d MB
203+
\n' \
196204
"$repos_disk_size" "$pages_disk_size" "$es_disk_size" "$stor_disk_size" "$minio_disk_size" "$mysql_disk_size" "$actions_disk_size" "$mssql_disk_size" 1>&2
197205

198206
if [[ $((available_space / (1024 * 1024))) -lt $min_disk_req ]]; then

‎docs/requirements.md

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -5,7 +5,7 @@ storage and must have network connectivity with the GitHub Enterprise Server app
55

66
## Backup host requirements
77

8-
Backup host software requirements are modest: Linux or other modern Unix operating system (Ubuntu is highly recommended) with [bash][1], [git][2], [OpenSSH][3] 5.6 or newer, [rsync][4] v2.6.4 or newer* (see [below](april-2023-update-of-rsync-requirements) for exceptions), [jq][11] v1.5 or newer, and [bc][12] v1.07 or newer.
8+
Backup host software requirements are modest: Linux or other modern Unix operating system (Ubuntu is highly recommended) with [bash][1], [git][2], [OpenSSH][3] 5.6 or newer, [rsync][4] v2.6.4 or newer* (see [below](#april-2023-update-of-rsync-requirements) for exceptions), [jq][11] v1.5 or newer, and [bc][12] v1.07 or newer.
99

1010
Ubuntu is the operating system we use to test `backup-utils` and it’s what we recommend you use too. You are welcome to use a different operating system, and we'll do our best to help you if you run into issues. But we can't guarantee that we'll be able to resolve issues that are specific to that operating system.
1111

‎share/github-backup-utils/ghe-backup-config

Lines changed: 1 addition & 37 deletions
Original file line numberDiff line numberDiff line change
@@ -35,7 +35,7 @@ if [ -n "$GHE_SHOW_VERSION" ]; then
3535
fi
3636

3737
# Check for "--help|-h" in args or GHE_SHOW_HELP=true and show usage
38-
# shellcheck disable=SC2120 # the script name is always referenced
38+
# shellcheck disable=SC2120 # Our arguments are optional and not meant to be the owning script's
3939
print_usage() {
4040
grep '^#/' <"$0" | cut -c 4-
4141
exit "${1:-1}"
@@ -51,10 +51,6 @@ else
5151
done
5252
fi
5353

54-
# Add the bin and share/github-backup-utils dirs to PATH
55-
PATH="$GHE_BACKUP_ROOT/bin:$GHE_BACKUP_ROOT/share/github-backup-utils:$PATH"
56-
# shellcheck source=share/github-backup-utils/bm.sh
57-
. "$GHE_BACKUP_ROOT/share/github-backup-utils/bm.sh"
5854
# Save off GHE_HOSTNAME from the environment since we want it to override the
5955
# backup.config value when set.
6056
GHE_HOSTNAME_PRESERVE="$GHE_HOSTNAME"
@@ -150,35 +146,6 @@ log_ssh(){
150146
log_level "ssh" "$1"
151147
}
152148

153-
# Assume this script lives in share/github-backup-utils/ when setting the root
154-
GHE_BACKUP_ROOT="$(cd "$(dirname "${BASH_SOURCE[0]}")/../.." && pwd)"
155-
156-
# Get the version from the version file.
157-
BACKUP_UTILS_VERSION="$(cat "$GHE_BACKUP_ROOT/share/github-backup-utils/version")"
158-
159-
# If a version check was requested, show the current version and exit
160-
if [ -n "$GHE_SHOW_VERSION" ]; then
161-
echo "GitHub backup-utils v$BACKUP_UTILS_VERSION"
162-
exit 0
163-
fi
164-
165-
# Check for "--help|-h" in args or GHE_SHOW_HELP=true and show usage
166-
# shellcheck disable=SC2120 # Our arguments are optional and not meant to be the owning script's
167-
print_usage() {
168-
grep '^#/' <"$0" | cut -c 4-
169-
exit "${1:-1}"
170-
}
171-
172-
if [ -n "$GHE_SHOW_HELP" ]; then
173-
print_usage
174-
else
175-
for a in "$@"; do
176-
if [ "$a" = "--help" ] || [ "$a" = "-h" ]; then
177-
print_usage
178-
fi
179-
done
180-
fi
181-
182149
# Add the bin and share/github-backup-utils dirs to PATH
183150
PATH="$GHE_BACKUP_ROOT/bin:$GHE_BACKUP_ROOT/share/github-backup-utils:$PATH"
184151
# shellcheck source=share/github-backup-utils/bm.sh
@@ -187,9 +154,6 @@ PATH="$GHE_BACKUP_ROOT/bin:$GHE_BACKUP_ROOT/share/github-backup-utils:$PATH"
187154
. "$GHE_BACKUP_ROOT/share/github-backup-utils/ghe-incremental-backup-restore"
188155
# shellcheck source=share/github-backup-utils/track-progress
189156
. "$GHE_BACKUP_ROOT/share/github-backup-utils/track-progress"
190-
# Save off GHE_HOSTNAME from the environment since we want it to override the
191-
# backup.config value when set.
192-
GHE_HOSTNAME_PRESERVE="$GHE_HOSTNAME"
193157

194158

195159
ghe_restore_check() {

‎share/github-backup-utils/ghe-backup-settings

Lines changed: 7 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -86,10 +86,13 @@ if [ "$(version $GHE_REMOTE_VERSION)" -ge "$(version 3.7.0)" ]; then
8686
cat "$GHE_SNAPSHOT_DIR/encrypted-column-encryption-keying-material" | sed 's:.*;::' > "$GHE_SNAPSHOT_DIR/encrypted-column-current-encryption-key"
8787
fi
8888

89-
backup-secret "secret scanning encrypted secrets current storage key" "secret-scanning-encrypted-secrets-current-storage-key" "secrets.secret-scanning.encrypted-secrets-current-storage-key"
90-
backup-secret "secret scanning encrypted secrets delimited storage keys" "secret-scanning-encrypted-secrets-delimited-storage-keys" "secrets.secret-scanning.encrypted-secrets-delimited-storage-keys"
91-
backup-secret "secret scanning encrypted secrets current shared transit key" "secret-scanning-encrypted-secrets-current-shared-transit-key" "secrets.secret-scanning.encrypted-secrets-current-shared-transit-key"
92-
backup-secret "secret scanning encrypted secrets delimited shared transit keys" "secret-scanning-encrypted-secrets-delimited-shared-transit-keys" "secrets.secret-scanning.encrypted-secrets-delimited-shared-transit-keys"
89+
# secret scanning encrypted secrets keys were added in GHES 3.8.0
90+
if [ "$(version $GHE_REMOTE_VERSION)" -ge "$(version 3.8.0)" ]; then
91+
backup-secret "secret scanning encrypted secrets current storage key" "secret-scanning-encrypted-secrets-current-storage-key" "secrets.secret-scanning.encrypted-secrets-current-storage-key"
92+
backup-secret "secret scanning encrypted secrets delimited storage keys" "secret-scanning-encrypted-secrets-delimited-storage-keys" "secrets.secret-scanning.encrypted-secrets-delimited-storage-keys"
93+
backup-secret "secret scanning encrypted secrets current shared transit key" "secret-scanning-encrypted-secrets-current-shared-transit-key" "secrets.secret-scanning.encrypted-secrets-current-shared-transit-key"
94+
backup-secret "secret scanning encrypted secrets delimited shared transit keys" "secret-scanning-encrypted-secrets-delimited-shared-transit-keys" "secrets.secret-scanning.encrypted-secrets-delimited-shared-transit-keys"
95+
fi
9396

9497
# Backup argon secrets for multiuser from ghes version 3.8 onwards
9598
if [[ "$(version $GHE_REMOTE_VERSION)" -ge "$(version 3.8.0)" && "$(version $GHE_REMOTE_VERSION)" -lt "$(version 3.8.2)" ]]; then

‎share/github-backup-utils/track-progress

Lines changed: 0 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -1,6 +1,5 @@
11
#!/usr/bin/env bash
22
#/ track-progress: track progress of backup or restore tasks
3-
set -e
43

54
# Current version is working solely with backups
65
progress(){

‎test/bin/ghe-cluster-host-check

Lines changed: 5 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,5 @@
1+
#!/usr/bin/env bash
2+
# Usage: ghe-cluster-host-check
3+
# Emulates a cluster reachability check
4+
set -e
5+
echo "Cluster is ready to configure."

‎test/test-ghe-backup.sh

Lines changed: 32 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -772,7 +772,7 @@ begin_test "ghe-backup takes backup of encrypted column encryption keying materi
772772
)
773773
end_test
774774

775-
begin_test "ghe-backup takes backup of secret scanning encrypted secrets encryption keys"
775+
begin_test "ghe-backup does not take backups of secret scanning encrypted secrets encryption keys on versions below 3.8.0"
776776
(
777777
set -e
778778

@@ -787,7 +787,37 @@ begin_test "ghe-backup takes backup of secret scanning encrypted secrets encrypt
787787
ghe-ssh "$GHE_HOSTNAME" -- ghe-config "$secret" "foo"
788788
done
789789

790-
ghe-backup
790+
GHE_REMOTE_VERSION=3.7.0 ghe-backup -v | grep -q "secret scanning encrypted secrets" && exit 1
791+
792+
required_files=(
793+
"secret-scanning-encrypted-secrets-current-storage-key"
794+
"secret-scanning-encrypted-secrets-delimited-storage-keys"
795+
"secret-scanning-encrypted-secrets-current-shared-transit-key"
796+
"secret-scanning-encrypted-secrets-delimited-shared-transit-keys"
797+
)
798+
799+
for file in "${required_files[@]}"; do
800+
[ "$(cat "$GHE_DATA_DIR/current/$file")" = "" ]
801+
done
802+
)
803+
end_test
804+
805+
begin_test "ghe-backup takes backup of secret scanning encrypted secrets encryption keys on versions 3.8.0+"
806+
(
807+
set -e
808+
809+
required_secrets=(
810+
"secrets.secret-scanning.encrypted-secrets-current-storage-key"
811+
"secrets.secret-scanning.encrypted-secrets-delimited-storage-keys"
812+
"secrets.secret-scanning.encrypted-secrets-current-shared-transit-key"
813+
"secrets.secret-scanning.encrypted-secrets-delimited-shared-transit-keys"
814+
)
815+
816+
for secret in "${required_secrets[@]}"; do
817+
ghe-ssh "$GHE_HOSTNAME" -- ghe-config "$secret" "foo"
818+
done
819+
820+
GHE_REMOTE_VERSION=3.8.0 ghe-backup
791821

792822
required_files=(
793823
"secret-scanning-encrypted-secrets-current-storage-key"

‎test/test-shellcheck.sh

Lines changed: 3 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -11,8 +11,8 @@ begin_test "shellcheck: reports no errors or warnings"
1111
set -e
1212
# We manually install the latest Shellcheck on Linux builds as other options
1313
# are too old.
14-
if [ -x "$BASE_PATH/shellcheck-latest/shellcheck" ]; then
15-
shellcheck() { "$BASE_PATH/shellcheck-latest/shellcheck" "$@"; }
14+
if [ -x "$BASE_PATH/shellcheck-stable/shellcheck" ]; then
15+
shellcheck() { "$BASE_PATH/shellcheck-stable/shellcheck" "$@"; }
1616
fi
1717

1818
if ! type shellcheck 1>/dev/null 2>&1; then
@@ -49,7 +49,7 @@ begin_test "shellopts: set -e set on all scripts"
4949
# Check all executable scripts checked into the repo, except bm.sh, ghe-backup-config, ghe-rsync and the dummy test scripts
5050
set +x
5151
cd $BASE_PATH
52-
git ls-tree -r HEAD | grep -Ev 'bm.sh|ghe-backup-config|ghe-rsync|test/bin' | grep -E '^1007|.*\..*sh$' | awk '{print $4}' | while read -r script; do
52+
git ls-tree -r HEAD | grep -Ev 'bm.sh|ghe-backup-config|ghe-rsync|track-progress|test/bin' | grep -E '^1007|.*\..*sh$' | awk '{print $4}' | while read -r script; do
5353
if head -n1 "$script" | grep -E -w "sh|bash" >/dev/null 2>&1; then
5454
grep -q "set -e" $script || echo $script >> $results || true
5555
fi

0 commit comments

Comments
 (0)