Skip to content

Commit ed76a61

Browse files
davidjarzebowskidavidjarzebowski
authored
Merge branch 'master' into djj-remove-git-clone-option
2 parents ad39e62 + 5f40dc1 commit ed76a61

11 files changed

+160
-7
lines changed

‎bin/ghe-host-check

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -131,7 +131,7 @@ fi
131131

132132
# backup-utils 2.13 onwards limits support to the current and previous two releases
133133
# of GitHub Enterprise Server.
134-
supported_minimum_version="3.6.0"
134+
supported_minimum_version="3.7.0"
135135

136136
if [ "$(version $version)" -ge "$(version $supported_minimum_version)" ]; then
137137
supported=1

‎bin/ghe-restore

Lines changed: 6 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -443,6 +443,12 @@ if [ "$(version $GHE_REMOTE_VERSION)" -ge "$(version 3.7.0)" ]; then
443443
fi
444444
ghe-restore-column-encryption-keys "$GHE_HOSTNAME"
445445

446+
# Always restore secret scanning encryption keys
447+
if [ "$(version $GHE_REMOTE_VERSION)" -ge "$(version 3.8.0)" ]; then
448+
log_info "Always restore secret scanning encryption keys on GHES verions 3.8.0+"
449+
ghe-restore-secret-scanning-encryption-keys "$GHE_HOSTNAME"
450+
fi
451+
446452
# Make sure mysql and elasticsearch are prep'd and running before restoring.
447453
# These services will not have been started on appliances that have not been
448454
# configured yet.

‎docs/requirements.md

Lines changed: 2 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -5,7 +5,7 @@ storage and must have network connectivity with the GitHub Enterprise Server app
55

66
## Backup host requirements
77

8-
Backup host software requirements are modest: Linux or other modern Unix operating system (Ubuntu is highly recommended) with [bash][1], [git][2], [OpenSSH][3] 5.6 or newer, [rsync][4] v2.6.4 or newer* (see [below](april-2023-update-of-rsync-requirements) for exceptions), and [jq][11] v1.5 or newer. See below for an update on rsync.
8+
Backup host software requirements are modest: Linux or other modern Unix operating system (Ubuntu is highly recommended) with [bash][1], [git][2], [OpenSSH][3] 5.6 or newer, [rsync][4] v2.6.4 or newer* (see [below](april-2023-update-of-rsync-requirements) for exceptions), [jq][11] v1.5 or newer, and [bc][12] v1.07 or newer.
99

1010
The parallel backup and restore feature will require [GNU awk][10] and [moreutils][9] to be installed.
1111

@@ -99,3 +99,4 @@ Due to how some components of Backup Utilities (e.g. MSSQL) take incremental bac
9999
[9]: https://joeyh.name/code/moreutils
100100
[10]: https://www.gnu.org/software/gawk
101101
[11]: https://stedolan.github.io/jq/
102+
[12]: https://www.gnu.org/software/bc/

‎share/github-backup-utils/ghe-backup-es-rsync

Lines changed: 3 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -4,6 +4,7 @@
44
#/
55
#/ Note: This command typically isn't called directly. It's invoked by
66
#/ ghe-backup when the rsync strategy is used.
7+
# shellcheck disable=SC2086
78
set -e
89

910
# Bring in the backup configuration
@@ -54,15 +55,15 @@ log_rsync "END elasticsearch rsync" 1>&3
5455
# Set up a trap to re-enable flushing on exit and remove temp file
5556
cleanup () {
5657
ghe_verbose "* Enabling ES index flushing ..."
57-
echo '{"index":{"translog.disable_flush":false}}' |
58+
echo '{"index":{"translog.flush_threshold_size":"512MB"}}' |
5859
ghe-ssh "$host" -- curl -s -XPUT "localhost:9200/_settings" -d @- >/dev/null
5960
}
6061
trap 'cleanup' EXIT
6162
trap 'exit $?' INT # ^C always terminate
6263

6364
# Disable ES flushing and force a flush right now
6465
ghe_verbose "* Disabling ES index flushing ..."
65-
echo '{"index":{"translog.disable_flush":true}}' |
66+
echo '{"index":{"translog.flush_threshold_size":"1PB"}}' |
6667
ghe-ssh "$host" -- curl -s -XPUT "localhost:9200/_settings" -d @- >/dev/null
6768
ghe-ssh "$host" -- curl -s -XPOST "localhost:9200/_flush" >/dev/null
6869

‎share/github-backup-utils/ghe-backup-mssql

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -47,8 +47,8 @@ if [ -z "$GHE_MSSQL_PRIMARY_HOST" ]; then
4747
fi
4848

4949
tempdir=$(mktemp -d -t backup-utils-backup-XXXXXX)
50-
ssh_config_file_opt=
51-
opts=
50+
ssh_config_file_opt=()
51+
opts=()
5252

5353
isHA="$(ghe-ssh "$GHE_HOSTNAME" -- "ghe-config cluster.ha" || true)"
5454

‎share/github-backup-utils/ghe-backup-settings

Lines changed: 5 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -89,6 +89,11 @@ if [ "$(version $GHE_REMOTE_VERSION)" -ge "$(version 3.8.0)" ]; then
8989
backup-secret "encrypted column current encryption key" "encrypted-column-current-encryption-key" "secrets.github.encrypted-column-current-encryption-key"
9090
fi
9191

92+
backup-secret "secret scanning encrypted secrets current storage key" "secret-scanning-encrypted-secrets-current-storage-key" "secrets.secret-scanning.encrypted-secrets-current-storage-key"
93+
backup-secret "secret scanning encrypted secrets delimited storage keys" "secret-scanning-encrypted-secrets-delimited-storage-keys" "secrets.secret-scanning.encrypted-secrets-delimited-storage-keys"
94+
backup-secret "secret scanning encrypted secrets current shared transit key" "secret-scanning-encrypted-secrets-current-shared-transit-key" "secrets.secret-scanning.encrypted-secrets-current-shared-transit-key"
95+
backup-secret "secret scanning encrypted secrets delimited shared transit keys" "secret-scanning-encrypted-secrets-delimited-shared-transit-keys" "secrets.secret-scanning.encrypted-secrets-delimited-shared-transit-keys"
96+
9297
# Backup argon secrets for multiuser from ghes version 3.8 onwards
9398
if [[ "$(version $GHE_REMOTE_VERSION)" -ge "$(version 3.8.0)" && "$(version $GHE_REMOTE_VERSION)" -lt "$(version 3.8.2)" ]]; then
9499
backup-secret "management console argon2 secret" "manage-argon-secret" "secrets.manage-auth.argon-secret"

‎share/github-backup-utils/ghe-restore-secret-scanning-encryption-keys

Lines changed: 39 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,39 @@
1+
#!/usr/bin/env bash
2+
#/ Usage: ghe-restore-secret-scanning-encryption-keys <host>
3+
#/ Restore the secret scanning encryption keys from a snapshot to the given <host>.
4+
#/ This script will be run automatically by `ghe-restore`
5+
set -e
6+
7+
# Bring in the backup configuration
8+
# shellcheck source=share/github-backup-utils/ghe-backup-config
9+
. "$(dirname "${BASH_SOURCE[0]}")/ghe-backup-config"
10+
11+
# Show usage and bail with no arguments
12+
[ -z "$*" ] && print_usage
13+
14+
bm_start "$(basename $0)"
15+
16+
# Grab host arg
17+
GHE_HOSTNAME="$1"
18+
19+
# Perform a host-check and establish GHE_REMOTE_XXX variables.
20+
ghe_remote_version_required "$GHE_HOSTNAME"
21+
22+
# The snapshot to restore should be set by the ghe-restore command but this lets
23+
# us run this script directly.
24+
: ${GHE_RESTORE_SNAPSHOT:=current}
25+
26+
# Path to snapshot dir we're restoring from
27+
: ${GHE_RESTORE_SNAPSHOT_PATH:="$GHE_DATA_DIR/current"}
28+
29+
# Restore secret scanning encrypted secrets storage keys if present
30+
log_info "Restoring secret scanning encrypted secrets storage keys"
31+
restore-secret "secret scanning encrypted secrets current storage key" "secret-scanning-encrypted-secrets-current-storage-key" "secrets.secret-scanning.encrypted-secrets-current-storage-key"
32+
restore-secret "secret scanning encrypted secrets delimited storage keys" "secret-scanning-encrypted-secrets-delimited-storage-keys" "secrets.secret-scanning.encrypted-secrets-delimited-storage-keys"
33+
34+
# Restore secret scanning encrypted secrets transit keys if present
35+
log_info "Restoring secret scanning encrypted secrets transit keys"
36+
restore-secret "secret scanning encrypted secrets current shared transit key" "secret-scanning-encrypted-secrets-current-shared-transit-key" "secrets.secret-scanning.encrypted-secrets-current-shared-transit-key"
37+
restore-secret "secret scanning encrypted secrets delimited shared transit keys" "secret-scanning-encrypted-secrets-delimited-shared-transit-keys" "secrets.secret-scanning.encrypted-secrets-delimited-shared-transit-keys"
38+
39+
bm_end "$(basename $0)"

‎share/github-backup-utils/ghe-restore-settings

Lines changed: 6 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -56,6 +56,12 @@ restore-secret "kredz.credz HMAC key" "kredz-credz-hmac" "secrets.kredz.credz-hm
5656
# Restore kredz.varz HMAC key if present.
5757
restore-secret "kredz.varz HMAC key" "kredz-varz-hmac" "secrets.kredz.varz-hmac-secret"
5858

59+
# Restore encrypted column encryption keying material if present
60+
restore-secret "encrypted column encryption keying material" "encrypted-column-encryption-keying-material" "secrets.github.encrypted-column-keying-material"
61+
62+
# Restore encrypted column current encryption key if present
63+
restore-secret "encrypted column current encryption key" "encrypted-column-current-encryption-key" "secrets.github.encrypted-column-current-encryption-key"
64+
5965
# Restore SAML keys if present.
6066
if [ -f "$GHE_RESTORE_SNAPSHOT_PATH/saml-keys.tar" ]; then
6167
log_info "Restoring SAML keys ..."

‎share/github-backup-utils/version

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -1 +1 @@
1-
3.8.0
1+
3.9.0

‎test/test-ghe-backup.sh

Lines changed: 30 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -663,6 +663,36 @@ begin_test "ghe-backup takes backup of encrypted column current encryption key f
663663
)
664664
end_test
665665

666+
begin_test "ghe-backup takes backup of secret scanning encrypted secrets encryption keys"
667+
(
668+
set -e
669+
670+
required_secrets=(
671+
"secrets.secret-scanning.encrypted-secrets-current-storage-key"
672+
"secrets.secret-scanning.encrypted-secrets-delimited-storage-keys"
673+
"secrets.secret-scanning.encrypted-secrets-current-shared-transit-key"
674+
"secrets.secret-scanning.encrypted-secrets-delimited-shared-transit-keys"
675+
)
676+
677+
for secret in "${required_secrets[@]}"; do
678+
ghe-ssh "$GHE_HOSTNAME" -- ghe-config "$secret" "foo"
679+
done
680+
681+
ghe-backup
682+
683+
required_files=(
684+
"secret-scanning-encrypted-secrets-current-storage-key"
685+
"secret-scanning-encrypted-secrets-delimited-storage-keys"
686+
"secret-scanning-encrypted-secrets-current-shared-transit-key"
687+
"secret-scanning-encrypted-secrets-delimited-shared-transit-keys"
688+
)
689+
690+
for file in "${required_files[@]}"; do
691+
[ "$(cat "$GHE_DATA_DIR/current/$file")" = "foo" ]
692+
done
693+
)
694+
end_test
695+
666696
begin_test "ghe-backup takes backup of Actions settings"
667697
(
668698
set -e

0 commit comments

Comments
 (0)