Skip to content

Commit 7a80973

Browse files
brandonemlawbrandonemlaw
authored
Merge branch 'master' into brandonemlaw-secret-scanning-backup-content-encryption-keys-simple
2 parents f08984b + adde328 commit 7a80973

File tree

5 files changed

+155
-5
lines changed

5 files changed

+155
-5
lines changed

‎share/github-backup-utils/ghe-backup-config

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -217,8 +217,10 @@ ghe_parallel_check() {
217217
GHE_PARALLEL_COMMAND="parallel"
218218
local x
219219
for x in \
220+
/usr/bin/parallel-moreutils \
220221
/usr/bin/parallel.moreutils \
221222
/usr/bin/parallel_moreutils \
223+
/usr/bin/moreutils-parallel \
222224
/usr/bin/moreutils.parallel \
223225
/usr/bin/moreutils_parallel \
224226
; do

‎share/github-backup-utils/ghe-rsync-size

100755100644
Lines changed: 24 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -21,6 +21,7 @@ fi
2121

2222
transfer_size()
2323
{
24+
local host=$GHE_HOSTNAME
2425
local backup_data=$1
2526
if [[ "$1" == "mssql" ]]; then
2627
data_user_dir="/data/user/$1/backups"
@@ -58,20 +59,41 @@ transfer_size()
5859
;;
5960
esac
6061

62+
# Check if instance is cluster and fetch appropriate primary host for the different components
63+
if "$CLUSTER"; then
64+
cluster_nodes_output=$(ghe-ssh "$host" "ghe-cluster-nodes -i")
65+
case $1 in
66+
elasticsearch | storage | pages | actions | mssql)
67+
cluster_host=$(ghe-ssh "$host" "ghe-cluster-nodes -r $backup_data" | head -1)
68+
;;
69+
mysql)
70+
cluster_host=$(ghe-ssh "$host" "ghe-config cluster.mysql-master")
71+
;;
72+
repositories)
73+
cluster_host=$(ghe-ssh "$host" "ghe-cluster-nodes -r git" | head -1)
74+
;;
75+
*)
76+
exit 0
77+
;;
78+
esac
79+
host=$(echo "$cluster_nodes_output" | grep "$cluster_host" | awk '{print $2}' | head -1)
80+
fi
81+
82+
# Get file transfer size estimates
6183
if [ -d "${GHE_DATA_DIR}/current/$1" ]; then
6284
total_file_size=$(ghe-rsync -arn --stats \
6385
-e "ssh -q $GHE_EXTRA_SSH_OPTS -p 122 -l admin" \
6486
--rsync-path="sudo -u $user rsync" \
6587
"$link_dest"/"$1" \
6688
--ignore-missing-args \
67-
"$GHE_HOSTNAME:$data_user_dir/" \
89+
"$host:$data_user_dir/" \
6890
"$dest_dir/" | grep "Total transferred file size" | sed 's/.*size: //; s/,//g')
6991
else
7092
total_file_size=$(ghe-rsync -arn --stats \
7193
-e "ssh -q $GHE_EXTRA_SSH_OPTS -p 122 -l admin" \
7294
--rsync-path="sudo -u $user rsync" \
7395
--ignore-missing-args \
74-
"$GHE_HOSTNAME:$data_user_dir/" \
96+
"$host:$data_user_dir/" \
7597
"$dest_dir/" | grep "Total transferred file size" | sed 's/.*size: //; s/,//g')
7698
fi
7799

‎test/test-ghe-host-check.sh

Lines changed: 11 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -56,9 +56,14 @@ begin_test "ghe-host-check detects unsupported GitHub Enterprise Server versions
5656
read -r bu_version_major bu_version_minor _ <<<$(ghe_parse_version $BACKUP_UTILS_VERSION)
5757
bu_major_minor="$bu_version_major.$bu_version_minor"
5858
releases=$(/usr/bin/curl -s https://github-enterprise.s3.amazonaws.com/release/latest.json)
59-
supported=$(echo $releases | jq -r 'select(."'${bu_major_minor}'")')
59+
latest_value=$(echo "$releases" | jq -r '.latest')
60+
latest_major_version=$(echo $latest_value | cut -d "." -f 1-2)
61+
# Replace "latest" with the derived major version in the releases string
62+
releases_with_replacement=$(echo "$releases" | sed 's/"latest"/"'"$latest_major_version"'"/g')
63+
# Use the modified releases string as needed
64+
supported=$(echo "$releases_with_replacement" | jq -r 'select(."'${bu_major_minor}'")')
6065
# shellcheck disable=SC2207 # Command required as alternatives fail
61-
keys=($(echo $releases | jq -r 'keys[]'))
66+
keys=($(echo "$releases_with_replacement" | jq -r 'keys[]'))
6267

6368
if [ -z "$supported" ]
6469
then
@@ -78,7 +83,10 @@ begin_test "ghe-host-check detects unsupported GitHub Enterprise Server versions
7883
ix=$(( $ix + 1 ))
7984
done
8085
GHE_TEST_REMOTE_VERSION="${keys[$ix]}.0" ghe-host-check
81-
GHE_TEST_REMOTE_VERSION="${keys[$(( $ix - 1 ))]}.0" ghe-host-check
86+
# sometimes when the latest.json is updated during a release this test gets broken.
87+
if [ "${keys[$(( $ix - 1 ))]}" != "latest" ]; then
88+
GHE_TEST_REMOTE_VERSION="${keys[$(( $ix - 1 ))]}.0" ghe-host-check
89+
fi
8290
GHE_TEST_REMOTE_VERSION="${keys[$(( $ix - 2 ))]}.0" ghe-host-check
8391

8492
fi

‎test/test-ghe-incremental-restore.sh

Lines changed: 82 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,82 @@
1+
#!/usr/bin/env bash
2+
# ghe-restore command tests
3+
4+
# Bring in testlib
5+
# shellcheck source=test/testlib.sh
6+
. "$(dirname "$0")/testlib.sh"
7+
8+
setup_incremental_restore_data
9+
setup_actions_enabled_settings_for_restore true
10+
11+
# Make the current symlink
12+
ln -s 1 "$GHE_DATA_DIR/current"
13+
begin_test "ghe_restore -i doesn't run on unsupported versions"
14+
(
15+
set -e
16+
GHE_RESTORE_HOST=127.0.0.1
17+
export GHE_RESTORE_HOST
18+
19+
# restore should fail on versions older than 3.10
20+
! GHE_TEST_REMOTE_VERSION=3.9.0 ghe-restore -i -v
21+
! GHE_TEST_REMOTE_VERSION=3.7.0 ghe-restore -i -v
22+
! GHE_TEST_REMOTE_VERSION=3.1.0 ghe-restore -i -v
23+
)
24+
end_test
25+
26+
begin_test "ghe-restore -i into configured vm from full backup"
27+
(
28+
set -e
29+
rm -rf "$GHE_REMOTE_ROOT_DIR"
30+
setup_remote_metadata
31+
32+
# set as configured, enable maintenance mode and create required directories
33+
setup_maintenance_mode "configured"
34+
35+
# set restore host environ var
36+
GHE_RESTORE_HOST=127.0.0.1
37+
export GHE_RESTORE_HOST
38+
# run ghe-restore and write output to file for asserting against
39+
if ! GHE_TEST_REMOTE_VERSION=3.10.0 GHE_DEBUG=1 ghe-restore -i -v -f > "$TRASHDIR/restore-out" 2>&1; then
40+
output_debug_logs_and_fail_test
41+
fi
42+
43+
44+
# verify connect to right host
45+
grep -q "Connect 127.0.0.1:122 OK" "$TRASHDIR/restore-out"
46+
47+
# verify stale servers were cleared
48+
grep -q "Cleaning up stale nodes ..." "$TRASHDIR/restore-out"
49+
50+
# Verify all the data we've restored is as expected
51+
verify_all_restored_data
52+
)
53+
end_test
54+
55+
begin_test "ghe-restore -i fails when the lsn information for the listed files is out of order"
56+
(
57+
set -e
58+
rm -rf "$GHE_REMOTE_ROOT_DIR"
59+
setup_remote_metadata
60+
61+
# set as configured, enable maintenance mode and create required directories
62+
setup_maintenance_mode "configured"
63+
64+
# set restore host environ var
65+
GHE_RESTORE_HOST=127.0.0.1
66+
export GHE_RESTORE_HOST
67+
68+
inc_1="$GHE_DATA_DIR/2"
69+
inc_2="$GHE_DATA_DIR/3"
70+
71+
# screw up the order of the LSNs in xtrabackup_checkpoints
72+
setup_incremental_lsn $inc_1 100 200 incremental
73+
setup_incremental_lsn $inc_2 50 50 incremental
74+
# run ghe-restore and write output to file for asserting against
75+
# we expect failure and need the right output.
76+
if GHE_DEBUG=1 ghe-restore -i -v -f > "$TRASHDIR/restore-out" 2>&1; then
77+
true
78+
fi
79+
)
80+
end_test
81+
82+

‎test/testlib.sh

100644100755
Lines changed: 36 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -327,6 +327,40 @@ setup_test_data () {
327327
setup_minio_test_data "$GHE_DATA_DIR"
328328
}
329329

330+
# Sets up test data for testing incremental restores.
331+
setup_incremental_restore_data() {
332+
local full="$GHE_DATA_DIR/1"
333+
local inc_1="$GHE_DATA_DIR/2"
334+
local inc_2="$GHE_DATA_DIR/3"
335+
# Run the setup_test_data function to create three directories: 1 for full backup and two incremental.
336+
# we can use these directories for different types of tests
337+
setup_test_data "$full"
338+
setup_test_data "$inc_1"
339+
setup_test_data "$inc_2"
340+
# Setup the metadata files that track which files are used to track full and incremental files
341+
echo "$full" >> "$GHE_DATA_DIR/inc_full_backup"
342+
echo -e "$inc_1\n$inc_2" >> "$GHE_DATA_DIR/inc_snapshot_data"
343+
# Configure lsn data in xtrabackup_checkpoints for the full backup and the incremental backup
344+
setup_incremental_lsn $full 1 100 full
345+
setup_incremental_lsn $inc_1 101 200 incremental
346+
setup_incremental_lsn $inc_2 201 300 incremental
347+
}
348+
349+
setup_incremental_lsn() {
350+
local loc=$1
351+
local start=$2
352+
local end=$3
353+
local type=$4
354+
355+
cat <<LSN >> "$loc/xtrabackup_checkpoints"
356+
backup_type = $type
357+
from_lsn = $start
358+
to_lsn = $end
359+
last_lsn = $end
360+
flushed_lsn = $end
361+
LSN
362+
}
363+
330364
setup_incremental_backup_config() {
331365
ghe-ssh "$GHE_HOSTNAME" -- 'mkdir -p /tmp/lsndir'
332366
ghe-ssh "$GHE_HOSTNAME" -- 'echo "fake xtrabackup checkpoint" > /tmp/lsndir/xtrabackup_checkpoints'
@@ -600,8 +634,10 @@ setup_moreutils_parallel() {
600634
# We need moreutils parallel
601635
local x
602636
for x in \
637+
/usr/bin/parallel-moreutils \
603638
/usr/bin/parallel.moreutils \
604639
/usr/bin/parallel_moreutils \
640+
/usr/bin/moreutils-parallel \
605641
/usr/bin/moreutils.parallel \
606642
/usr/bin/moreutils_parallel \
607643
; do

0 commit comments

Comments
 (0)