Skip to content

Commit 354ec00

Browse files
dooleydevindooleydevin
committed
Merge remote-tracking branch 'private/enterprise-3.8-release' into 3.8.2-patch
2 parents 4d60e24 + d8c939a commit 354ec00

11 files changed

+200
-15
lines changed

‎backup.config-example

Lines changed: 3 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -95,3 +95,6 @@ GHE_NUM_SNAPSHOTS=10
9595
# When running an external mysql database, run this script to trigger a MySQL restore
9696
# rather than attempting to backup via backup-utils directly.
9797
#EXTERNAL_DATABASE_RESTORE_SCRIPT="/bin/false"
98+
99+
# If set to 'yes', Pages data will be included in backup and restore. Defaults to 'yes'
100+
#GHE_BACKUP_PAGES=no

‎bin/ghe-backup

Lines changed: 11 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -240,10 +240,15 @@ commands+=("
240240
echo \"$cmd_title\"
241241
ghe-backup-repositories || printf %s \"repositories \" >> \"$failures_file\"")
242242

243-
cmd_title=$(log_info "Backing up GitHub Pages artifacts ...")
244-
commands+=("
245-
echo \"$cmd_title\"
246-
ghe-backup-pages || printf %s \"pages \" >> \"$failures_file\"")
243+
# Pages backups are skipped only if GHE_BACKUP_PAGES is explicitly set to 'no' to guarantee backward compatibility.
244+
# If a customer upgrades backup-utils but keeps the config file from a previous version, Pages backups still work as expected.
245+
246+
if [ "$GHE_BACKUP_PAGES" != "no" ]; then
247+
cmd_title=$(log_info "Backing up GitHub Pages artifacts ...")
248+
commands+=("
249+
echo \"$cmd_title\"
250+
ghe-backup-pages || printf %s \"pages \" >> \"$failures_file\"")
251+
fi
247252

248253
cmd_title=$(log_info "Backing up storage data ...")
249254
commands+=("
@@ -290,6 +295,8 @@ if [ -z "$failures" ]; then
290295
ln -s "$GHE_SNAPSHOT_TIMESTAMP" "../current"
291296

292297
ghe-prune-snapshots
298+
else
299+
log_info "Skipping pruning snapshots, since some backups failed..."
293300
fi
294301

295302
log_info "Completed backup of $GHE_HOSTNAME in snapshot $GHE_SNAPSHOT_TIMESTAMP at $(date +"%H:%M:%S")"

‎bin/ghe-restore

Lines changed: 12 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -386,6 +386,12 @@ if [ "$(version $GHE_REMOTE_VERSION)" -ge "$(version 3.7.0)" ]; then
386386
fi
387387
ghe-restore-column-encryption-keys "$GHE_HOSTNAME"
388388

389+
# Always restore secret scanning encryption keys
390+
if [ "$(version $GHE_REMOTE_VERSION)" -ge "$(version 3.8.0)" ]; then
391+
log_info "Always restore secret scanning encryption keys on GHES verions 3.8.0+"
392+
ghe-restore-secret-scanning-encryption-keys "$GHE_HOSTNAME"
393+
fi
394+
389395
# Make sure mysql and elasticsearch are prep'd and running before restoring.
390396
# These services will not have been started on appliances that have not been
391397
# configured yet.
@@ -463,10 +469,12 @@ commands+=("
463469
echo \"$cmd_title\"
464470
ghe-restore-repositories-gist \"$GHE_HOSTNAME\"")
465471

466-
cmd_title=$(log_info "Restoring Pages ...")
467-
commands+=("
468-
echo \"$cmd_title\"
469-
ghe-restore-pages \"$GHE_HOSTNAME\" 1>&3")
472+
if [ "$GHE_BACKUP_PAGES" != "no" ]; then
473+
cmd_title=$(log_info "Restoring Pages ...")
474+
commands+=("
475+
echo \"$cmd_title\"
476+
ghe-restore-pages \"$GHE_HOSTNAME\" 1>&3")
477+
fi
470478

471479
cmd_title=$(log_info "Restoring SSH authorized keys ...")
472480
commands+=("

‎script/release

Lines changed: 11 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -34,6 +34,9 @@ DEB_PKG_NAME = 'github-backup-utils'
3434
GH_BASE_BRANCH = ENV['GH_BASE_BRANCH'] || 'master' # TODO: should we even allow a default or require all params get set explicitly?
3535
GH_STABLE_BRANCH = ""
3636

37+
# If PUBLISH is false, we leave the release in a draft state to be manually published later through the UI
38+
PUBLISH = ENV['PUBLISH'] == 'true' || false
39+
3740
CHANGELOG_TMPL = '''<%= package_name %> (<%= package_version %>) UNRELEASED; urgency=medium
3841
3942
<%- changes.each do |ch| -%>
@@ -480,15 +483,21 @@ if $PROGRAM_NAME == __FILE__
480483
attach_assets_to_release res['upload_url'], res['id'], ["#{base_dir}/dist/#{DEB_PKG_NAME}-v#{version}.tar.gz"]
481484
attach_assets_to_release res['upload_url'], res['id'], ["#{base_dir}/dist/#{DEB_PKG_NAME}_#{version}_all.deb"]
482485

483-
puts 'Publishing release...'
484-
publish_release res['id']
486+
if PUBLISH
487+
puts 'Publishing release...'
488+
publish_release res['id']
489+
end
485490

486491
puts 'Cleaning up...'
487492
clean_up version
488493

489494
puts "Updating #{GH_STABLE_BRANCH} branch..."
490495
update_stable_branch
491496

497+
if !PUBLISH
498+
puts 'Release left in a "Draft" state. Go to the https://github.com/github/backup-utils/releases and publish when ready.'
499+
end
500+
492501
puts 'Released!'
493502
rescue RuntimeError => e
494503
$stderr.puts "Error: #{e}"

‎share/github-backup-utils/ghe-backup-es-rsync

Lines changed: 3 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -4,6 +4,7 @@
44
#/
55
#/ Note: This command typically isn't called directly. It's invoked by
66
#/ ghe-backup when the rsync strategy is used.
7+
# shellcheck disable=SC2086
78
set -e
89

910
# Bring in the backup configuration
@@ -54,15 +55,15 @@ log_rsync "END elasticsearch rsync" 1>&3
5455
# Set up a trap to re-enable flushing on exit and remove temp file
5556
cleanup () {
5657
ghe_verbose "* Enabling ES index flushing ..."
57-
echo '{"index":{"translog.disable_flush":false}}' |
58+
echo '{"index":{"translog.flush_threshold_size":"512MB"}}' |
5859
ghe-ssh "$host" -- curl -s -XPUT "localhost:9200/_settings" -d @- >/dev/null
5960
}
6061
trap 'cleanup' EXIT
6162
trap 'exit $?' INT # ^C always terminate
6263

6364
# Disable ES flushing and force a flush right now
6465
ghe_verbose "* Disabling ES index flushing ..."
65-
echo '{"index":{"translog.disable_flush":true}}' |
66+
echo '{"index":{"translog.flush_threshold_size":"1PB"}}' |
6667
ghe-ssh "$host" -- curl -s -XPUT "localhost:9200/_settings" -d @- >/dev/null
6768
ghe-ssh "$host" -- curl -s -XPOST "localhost:9200/_flush" >/dev/null
6869

‎share/github-backup-utils/ghe-backup-settings

Lines changed: 6 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -86,8 +86,13 @@ if [ "$(version $GHE_REMOTE_VERSION)" -ge "$(version 3.7.0)" ]; then
8686
cat "$GHE_SNAPSHOT_DIR/encrypted-column-encryption-keying-material" | sed 's:.*;::' > "$GHE_SNAPSHOT_DIR/encrypted-column-current-encryption-key"
8787
fi
8888

89+
backup-secret "secret scanning encrypted secrets current storage key" "secret-scanning-encrypted-secrets-current-storage-key" "secrets.secret-scanning.encrypted-secrets-current-storage-key"
90+
backup-secret "secret scanning encrypted secrets delimited storage keys" "secret-scanning-encrypted-secrets-delimited-storage-keys" "secrets.secret-scanning.encrypted-secrets-delimited-storage-keys"
91+
backup-secret "secret scanning encrypted secrets current shared transit key" "secret-scanning-encrypted-secrets-current-shared-transit-key" "secrets.secret-scanning.encrypted-secrets-current-shared-transit-key"
92+
backup-secret "secret scanning encrypted secrets delimited shared transit keys" "secret-scanning-encrypted-secrets-delimited-shared-transit-keys" "secrets.secret-scanning.encrypted-secrets-delimited-shared-transit-keys"
93+
8994
# Backup argon secrets for multiuser from ghes version 3.8 onwards
90-
if ! [ "$(version $GHE_REMOTE_VERSION)" -lt "$(version 3.8.0)" ]; then
95+
if [ "$(version $GHE_REMOTE_VERSION)" -ge "$(version 3.8.0)" ]; then
9196
backup-secret "management console argon2 secret" "manage-argon-secret" "secrets.manage-auth.argon-secret"
9297
fi
9398

‎share/github-backup-utils/ghe-restore-secret-scanning-encryption-keys

Lines changed: 39 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,39 @@
1+
#!/usr/bin/env bash
2+
#/ Usage: ghe-restore-secret-scanning-encryption-keys <host>
3+
#/ Restore the secret scanning encryption keys from a snapshot to the given <host>.
4+
#/ This script will be run automatically by `ghe-restore`
5+
set -e
6+
7+
# Bring in the backup configuration
8+
# shellcheck source=share/github-backup-utils/ghe-backup-config
9+
. "$(dirname "${BASH_SOURCE[0]}")/ghe-backup-config"
10+
11+
# Show usage and bail with no arguments
12+
[ -z "$*" ] && print_usage
13+
14+
bm_start "$(basename $0)"
15+
16+
# Grab host arg
17+
GHE_HOSTNAME="$1"
18+
19+
# Perform a host-check and establish GHE_REMOTE_XXX variables.
20+
ghe_remote_version_required "$GHE_HOSTNAME"
21+
22+
# The snapshot to restore should be set by the ghe-restore command but this lets
23+
# us run this script directly.
24+
: ${GHE_RESTORE_SNAPSHOT:=current}
25+
26+
# Path to snapshot dir we're restoring from
27+
: ${GHE_RESTORE_SNAPSHOT_PATH:="$GHE_DATA_DIR/current"}
28+
29+
# Restore secret scanning encrypted secrets storage keys if present
30+
log_info "Restoring secret scanning encrypted secrets storage keys"
31+
restore-secret "secret scanning encrypted secrets current storage key" "secret-scanning-encrypted-secrets-current-storage-key" "secrets.secret-scanning.encrypted-secrets-current-storage-key"
32+
restore-secret "secret scanning encrypted secrets delimited storage keys" "secret-scanning-encrypted-secrets-delimited-storage-keys" "secrets.secret-scanning.encrypted-secrets-delimited-storage-keys"
33+
34+
# Restore secret scanning encrypted secrets transit keys if present
35+
log_info "Restoring secret scanning encrypted secrets transit keys"
36+
restore-secret "secret scanning encrypted secrets current shared transit key" "secret-scanning-encrypted-secrets-current-shared-transit-key" "secrets.secret-scanning.encrypted-secrets-current-shared-transit-key"
37+
restore-secret "secret scanning encrypted secrets delimited shared transit keys" "secret-scanning-encrypted-secrets-delimited-shared-transit-keys" "secrets.secret-scanning.encrypted-secrets-delimited-shared-transit-keys"
38+
39+
bm_end "$(basename $0)"

‎share/github-backup-utils/ghe-restore-settings

Lines changed: 6 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -53,6 +53,12 @@ restore-secret "kredz.credz HMAC key" "kredz-credz-hmac" "secrets.kredz.credz-hm
5353
# Restore kredz.varz HMAC key if present.
5454
restore-secret "kredz.varz HMAC key" "kredz-varz-hmac" "secrets.kredz.varz-hmac-secret"
5555

56+
# Restore encrypted column encryption keying material if present
57+
restore-secret "encrypted column encryption keying material" "encrypted-column-encryption-keying-material" "secrets.github.encrypted-column-keying-material"
58+
59+
# Restore encrypted column current encryption key if present
60+
restore-secret "encrypted column current encryption key" "encrypted-column-current-encryption-key" "secrets.github.encrypted-column-current-encryption-key"
61+
5662
# Restore SAML keys if present.
5763
if [ -f "$GHE_RESTORE_SNAPSHOT_PATH/saml-keys.tar" ]; then
5864
log_info "Restoring SAML keys ..."

‎test/test-ghe-backup.sh

Lines changed: 43 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -138,7 +138,13 @@ begin_test "ghe-backup management console does not backup argon secret"
138138
(
139139
set -e
140140

141-
GHE_REMOTE_VERSION=3.7.0 ghe-backup -v | grep -q "management console argon2 secret not set" && exit 1
141+
GHE_REMOTE_VERSION=2.1.10 ghe-backup -v | grep -q "management console argon2 secret not set" && exit 1
142+
[ ! -f "$GHE_DATA_DIR/current/manage-argon-secret" ]
143+
144+
GHE_REMOTE_VERSION=3.6.1 ghe-backup -v | grep -q "management console argon2 secret not set" && exit 1
145+
[ ! -f "$GHE_DATA_DIR/current/manage-argon-secret" ]
146+
147+
GHE_REMOTE_VERSION=3.7.10 ghe-backup -v | grep -q "management console argon2 secret not set" && exit 1
142148
[ ! -f "$GHE_DATA_DIR/current/manage-argon-secret" ]
143149
)
144150
end_test
@@ -152,6 +158,12 @@ begin_test "ghe-backup management console backs up argon secret"
152158
GHE_REMOTE_VERSION=3.8.0 ghe-backup
153159

154160
[ "$(cat "$GHE_DATA_DIR/current/manage-argon-secret")" = "fake pw" ]
161+
162+
rm -rf "$GHE_DATA_DIR/current"
163+
164+
GHE_REMOTE_VERSION=4.1.0 ghe-backup
165+
166+
[ "$(cat "$GHE_DATA_DIR/current/manage-argon-secret")" = "fake pw" ]
155167
)
156168
end_test
157169

@@ -686,6 +698,36 @@ begin_test "ghe-backup takes backup of encrypted column encryption keying materi
686698
)
687699
end_test
688700

701+
begin_test "ghe-backup takes backup of secret scanning encrypted secrets encryption keys"
702+
(
703+
set -e
704+
705+
required_secrets=(
706+
"secrets.secret-scanning.encrypted-secrets-current-storage-key"
707+
"secrets.secret-scanning.encrypted-secrets-delimited-storage-keys"
708+
"secrets.secret-scanning.encrypted-secrets-current-shared-transit-key"
709+
"secrets.secret-scanning.encrypted-secrets-delimited-shared-transit-keys"
710+
)
711+
712+
for secret in "${required_secrets[@]}"; do
713+
ghe-ssh "$GHE_HOSTNAME" -- ghe-config "$secret" "foo"
714+
done
715+
716+
ghe-backup
717+
718+
required_files=(
719+
"secret-scanning-encrypted-secrets-current-storage-key"
720+
"secret-scanning-encrypted-secrets-delimited-storage-keys"
721+
"secret-scanning-encrypted-secrets-current-shared-transit-key"
722+
"secret-scanning-encrypted-secrets-delimited-shared-transit-keys"
723+
)
724+
725+
for file in "${required_files[@]}"; do
726+
[ "$(cat "$GHE_DATA_DIR/current/$file")" = "foo" ]
727+
done
728+
)
729+
end_test
730+
689731
begin_test "ghe-backup takes backup of Actions settings"
690732
(
691733
set -e

‎test/test-ghe-restore.sh

Lines changed: 65 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -389,6 +389,71 @@ begin_test "ghe-restore with encrypted column current encryption key for version
389389
)
390390
end_test
391391

392+
begin_test "ghe-restore with secret scanning encrypted secrets encryption keys for versions below 3.8.0"
393+
(
394+
set -e
395+
rm -rf "$GHE_REMOTE_ROOT_DIR"
396+
setup_remote_metadata
397+
398+
required_files=(
399+
"secret-scanning-encrypted-secrets-current-storage-key"
400+
"secret-scanning-encrypted-secrets-delimited-storage-keys"
401+
"secret-scanning-encrypted-secrets-current-shared-transit-key"
402+
"secret-scanning-encrypted-secrets-delimited-shared-transit-keys"
403+
)
404+
405+
for file in "${required_files[@]}"; do
406+
echo "foo" >"$GHE_DATA_DIR/current/$file"
407+
done
408+
409+
GHE_REMOTE_VERSION=3.7.0 ghe-restore -v -f localhost
410+
411+
required_secrets=(
412+
"secrets.secret-scanning.encrypted-secrets-current-storage-key"
413+
"secrets.secret-scanning.encrypted-secrets-delimited-storage-keys"
414+
"secrets.secret-scanning.encrypted-secrets-current-shared-transit-key"
415+
"secrets.secret-scanning.encrypted-secrets-delimited-shared-transit-keys"
416+
)
417+
418+
for secret in "${required_secrets[@]}"; do
419+
[ "$(ghe-ssh "$GHE_HOSTNAME" -- ghe-config "$secret")" = "" ] # expecting these to not be set for versions below 3.8.0
420+
done
421+
)
422+
end_test
423+
424+
425+
begin_test "ghe-restore with secret scanning encrypted secrets encryption keys for versions 3.8.0+"
426+
(
427+
set -e
428+
rm -rf "$GHE_REMOTE_ROOT_DIR"
429+
setup_remote_metadata
430+
431+
required_files=(
432+
"secret-scanning-encrypted-secrets-current-storage-key"
433+
"secret-scanning-encrypted-secrets-delimited-storage-keys"
434+
"secret-scanning-encrypted-secrets-current-shared-transit-key"
435+
"secret-scanning-encrypted-secrets-delimited-shared-transit-keys"
436+
)
437+
438+
for file in "${required_files[@]}"; do
439+
echo "foo" >"$GHE_DATA_DIR/current/$file"
440+
done
441+
442+
GHE_REMOTE_VERSION=3.8.0 ghe-restore -v -f localhost
443+
444+
required_secrets=(
445+
"secrets.secret-scanning.encrypted-secrets-current-storage-key"
446+
"secrets.secret-scanning.encrypted-secrets-delimited-storage-keys"
447+
"secrets.secret-scanning.encrypted-secrets-current-shared-transit-key"
448+
"secrets.secret-scanning.encrypted-secrets-delimited-shared-transit-keys"
449+
)
450+
451+
for secret in "${required_secrets[@]}"; do
452+
[ "$(ghe-ssh "$GHE_HOSTNAME" -- ghe-config "$secret")" = "foo" ] # expecting this to have been restored successfully for versions 3.8.0+
453+
done
454+
)
455+
end_test
456+
392457
# Setup Actions data for the subsequent tests
393458
setup_actions_test_data "$GHE_DATA_DIR/1"
394459

0 commit comments

Comments
 (0)