Skip to content

Commit 96d0baa

Browse files
FelixRottlerFelixRottler
authored
Merge pull request #261 from github/felixrottler/use-mssql-master-host
use mssql master hostname for mssql backup/restore
2 parents a6501f4 + 0447b39 commit 96d0baa

File tree

2 files changed

+104
-42
lines changed

2 files changed

+104
-42
lines changed

‎share/github-backup-utils/ghe-backup-mssql

Lines changed: 63 additions & 32 deletions
Original file line numberDiff line numberDiff line change
@@ -22,14 +22,45 @@ tran_expire=
2222
# Check if the export tool is available in this version
2323
export_tool_available() {
2424
if [ -z "$GHE_TEST_REMOTE_VERSION" ]; then
25-
ghe-ssh "$GHE_HOSTNAME" "test -e /usr/local/bin/ghe-export-mssql"
25+
ghe_ssh_mssql "test -e /usr/local/bin/ghe-export-mssql"
2626
else
2727
# Always return available for test
2828
return 0
2929
fi
3030
}
3131

32-
if ! export_tool_available; then
32+
ghe_ssh_mssql() {
33+
ghe-ssh $opts $ssh_config_file_opt "$GHE_MSSQL_PRIMARY_HOST" "$@"
34+
}
35+
36+
cleanup() {
37+
rm -rf $tempdir
38+
}
39+
trap 'cleanup' EXIT INT
40+
41+
# use the mssql primary host if GHES cluster configuration contains a mssql-master or use the ghe server if the mssql-master is not available.
42+
GHE_MSSQL_PRIMARY_NODE="$(ghe-ssh "$GHE_HOSTNAME" -- "ghe-config cluster.mssql-master" || true)"
43+
GHE_MSSQL_PRIMARY_HOST="$(ghe-ssh "$GHE_HOSTNAME" -- "ghe-config cluster.$GHE_MSSQL_PRIMARY_NODE.hostname" || true)"
44+
45+
if [ -z "$GHE_MSSQL_PRIMARY_HOST" ]; then
46+
GHE_MSSQL_PRIMARY_HOST="$GHE_HOSTNAME"
47+
fi
48+
49+
tempdir=$(mktemp -d -t backup-utils-backup-XXXXXX)
50+
ssh_config_file_opt=
51+
opts=
52+
53+
isHA="$(ghe-ssh "$GHE_HOSTNAME" -- "ghe-config cluster.ha" || true)"
54+
55+
# get server hostnames under cluster and HA
56+
if [ "$GHE_BACKUP_STRATEGY" = "cluster" ] || [ "$isHA" = "true" ] ; then
57+
ssh_config_file="$tempdir/ssh_config"
58+
ssh_config_file_opt="-F $ssh_config_file"
59+
opts="-o UserKnownHostsFile=/dev/null -o StrictHostKeyChecking=no -o PasswordAuthentication=no"
60+
ghe-ssh-config "$GHE_HOSTNAME" "$GHE_MSSQL_PRIMARY_HOST" > "$ssh_config_file"
61+
fi
62+
63+
if ! export_tool_available ; then
3364
ghe_verbose "ghe-export-mssql is not available"
3465
exit
3566
fi
@@ -49,11 +80,11 @@ find_timestamp() {
4980
filename="${1##*/}"
5081
IFS='@' read -ra parts <<< "$filename"
5182
datetime_part=${parts[1]:0:15}
52-
echo $datetime_part
83+
echo "$datetime_part"
5384
}
5485

5586
actions_dbs() {
56-
all_dbs=$(echo 'set -o pipefail; ghe-mssql-console -y -n -q "SET NOCOUNT ON; SELECT name FROM sys.databases"' | ghe-ssh "$GHE_HOSTNAME" /bin/bash)
87+
all_dbs=$(echo 'set -o pipefail; ghe-mssql-console -y -n -q "SET NOCOUNT ON; SELECT name FROM sys.databases"' | ghe_ssh_mssql /bin/bash)
5788
for db in $all_dbs; do
5889
if [[ ! "$db" =~ ^(master|tempdb|model|msdb)$ ]] && [[ "$db" =~ ^[a-zA-Z0-9_-]+$ ]]; then
5990
echo "$db"
@@ -90,15 +121,15 @@ ensure_same_dbs() {
90121
}
91122

92123
run_query() {
93-
echo "set -o pipefail; ghe-mssql-console -y -n -q \"SET NOCOUNT ON; $1\"" | ghe-ssh "$GHE_HOSTNAME" /bin/bash | sed 's/^[[:space:]]*//;s/[[:space:]]*$//'
124+
echo "set -o pipefail; ghe-mssql-console -y -n -q \"SET NOCOUNT ON; $1\"" | ghe_ssh_mssql /bin/bash | sed 's/^[[:space:]]*//;s/[[:space:]]*$//'
94125
}
95126

96127
get_latest_backup_file() {
97128
backups_dir=$1
98129
db=$2
99130
ext=$3
100131

101-
latest_full_backup=$(find "$backups_dir" -type f -name "$db*.$ext" | egrep '[0-9]{8}T[0-9]{6}' | sort | tail -n 1)
132+
latest_full_backup=$(find "$backups_dir" -type f -name "$db*.$ext" | grep -E '[0-9]{8}T[0-9]{6}' | sort | tail -n 1)
102133
latest_full_backup_file="${latest_full_backup##*/}"
103134
echo "$latest_full_backup_file"
104135
}
@@ -146,8 +177,8 @@ get_next_diff_backup_base_lsn() {
146177

147178
last_mssql=$GHE_DATA_DIR/current/mssql
148179

149-
if [ ! -d $last_mssql ] \
150-
|| [ -z "$(find $last_mssql -type f -name '*.bak' | head -n 1)" ]; then
180+
if [ ! -d "$last_mssql" ] \
181+
|| [ -z "$(find "$last_mssql" -type f -name '*.bak' | head -n 1)" ]; then
151182
ghe_verbose "Taking first full backup"
152183
backup_type="full"
153184
else
@@ -159,34 +190,34 @@ else
159190
current=$(date -u +%Y%m%d%H%M%S)
160191

161192
full=$(find "$last_mssql" -type f -name "*.bak" | head -n 1)
162-
full=$(find_timestamp $full)
163-
full_expire=$(add_minute $full ${cadence[0]})
193+
full=$(find_timestamp "$full")
194+
full_expire=$(add_minute "$full" "${cadence[0]}")
164195
full_expire="${full_expire//T}"
165196

166197
diff=$(find "$last_mssql" -type f -name "*.diff" | head -n 1)
167198
if [ -f "$diff" ]; then
168-
diff=$(find_timestamp $diff)
169-
diff_expire=$(add_minute $diff ${cadence[1]})
199+
diff=$(find_timestamp "$diff")
200+
diff_expire=$(add_minute "$diff" "${cadence[1]}")
170201
diff_expire="${diff_expire//T}"
171202
else
172-
diff_expire=$(add_minute $full ${cadence[1]})
203+
diff_expire=$(add_minute "$full" "${cadence[1]}")
173204
diff_expire="${diff_expire//T}"
174205
fi
175206

176-
tran=$(find "$last_mssql" -type f -name "*.log" | egrep '[0-9]{8}T[0-9]{6}' | sort | tail -1)
177-
tran=$(find_timestamp $tran)
178-
tran_expire=$(add_minute $tran ${cadence[2]})
207+
tran=$(find "$last_mssql" -type f -name "*.log" | grep -E '[0-9]{8}T[0-9]{6}' | sort | tail -1)
208+
tran=$(find_timestamp "$tran")
209+
tran_expire=$(add_minute "$tran" "${cadence[2]}")
179210
tran_expire="${tran_expire//T}"
180211

181212
ghe_verbose "current $current, full expire $full_expire, \
182213
diff expire $diff_expire, tran expire $tran_expire"
183214

184215
# Determine the type of backup to take based on expiry time
185-
if [ $current -gt $full_expire ]; then
216+
if [ "$current" -gt "$full_expire" ]; then
186217
backup_type='full'
187-
elif [ $current -gt $diff_expire ]; then
218+
elif [ "$current" -gt "$diff_expire" ]; then
188219
backup_type='diff'
189-
elif [ $current -gt $tran_expire ]; then
220+
elif [ "$current" -gt "$tran_expire" ]; then
190221
backup_type='transaction'
191222
fi
192223

@@ -264,8 +295,8 @@ fi
264295
mkdir -p "$backup_dir"
265296

266297
# Use hard links to "copy" over previous applicable backups to the new snapshot folder to save disk space and time
267-
if [ -d $last_mssql ]; then
268-
for p in $last_mssql/*
298+
if [ -d "$last_mssql" ]; then
299+
for p in "$last_mssql"/*
269300
do
270301
[[ -e "$p" ]] || break
271302

@@ -274,23 +305,23 @@ if [ -d $last_mssql ]; then
274305
transfer=
275306

276307
# Copy full backups unless we're taking a new full backup
277-
if [ $extension = "bak" ] && [ "$backup_type" != 'full' ]; then
308+
if [ "$extension" = "bak" ] && [ "$backup_type" != 'full' ]; then
278309
transfer=1
279310
fi
280311

281312
# Copy diff backups unless we're taking a new full or diff backup
282-
if [ $extension = "diff" ] && [ "$backup_type" != 'full' ] && [ "$backup_type" != 'diff' ]; then
313+
if [ "$extension" = "diff" ] && [ "$backup_type" != 'full' ] && [ "$backup_type" != 'diff' ]; then
283314
transfer=1
284315
fi
285316

286317
# Copy transaction log backups unless we're taking a new full or diff backup
287-
if [ $extension = "log" ] && [ "$backup_type" != 'full' ] && [ "$backup_type" != 'diff' ]; then
318+
if [ "$extension" = "log" ] && [ "$backup_type" != 'full' ] && [ "$backup_type" != 'diff' ]; then
288319
transfer=1
289320
fi
290321

291322
if [ -n "$transfer" ]; then
292323
ghe_verbose "Creating hard link to $filename"
293-
ln $last_mssql/$filename $backup_dir/$filename
324+
ln "$last_mssql"/"$filename" "$backup_dir"/"$filename"
294325
fi
295326
done
296327
fi
@@ -304,21 +335,21 @@ if [ -n "$backup_type" ]; then
304335
elif [ "$backup_type" = "transaction" ]; then
305336
backup_command='ghe-export-mssql -t'
306337
fi
307-
308-
bm_start "$(basename $0)"
309-
ghe-ssh "$GHE_HOSTNAME" -- "$backup_command" || failures="$failures mssql"
310-
bm_end "$(basename $0)"
338+
339+
bm_start "$(basename "$0")"
340+
ghe_ssh_mssql -- "$backup_command" || failures="$failures mssql"
341+
bm_end "$(basename "$0")"
311342

312343
# Configure the backup cadence on the appliance, which is used for diagnostics
313-
ghe-ssh "$GHE_HOSTNAME" "ghe-config mssql.backup.cadence $GHE_MSSQL_BACKUP_CADENCE"
344+
ghe_ssh_mssql "ghe-config mssql.backup.cadence $GHE_MSSQL_BACKUP_CADENCE"
314345

315346
# Transfer backup files from appliance to backup host
316347
appliance_dir="$GHE_REMOTE_DATA_DIR/user/mssql/backups"
317348
backups=$(echo "set -o pipefail; if sudo test -d \"$appliance_dir\"; then \
318-
sudo ls \"$appliance_dir\"; fi" | ghe-ssh "$GHE_HOSTNAME" /bin/bash)
349+
sudo ls \"$appliance_dir\"; fi" | ghe_ssh_mssql /bin/bash)
319350
for b in $backups
320351
do
321352
ghe_verbose "Transferring to backup host $b"
322-
ghe-ssh "$GHE_HOSTNAME" "sudo cat $appliance_dir/$b" > $backup_dir/$b
353+
ghe_ssh_mssql "sudo cat $appliance_dir/$b" > "$backup_dir"/"$b"
323354
done
324355
fi

‎share/github-backup-utils/ghe-restore-mssql

Lines changed: 41 additions & 10 deletions
Original file line numberDiff line numberDiff line change
@@ -15,20 +15,51 @@ set -e
1515
# Check if the import tool is available in this version
1616
import_tool_available() {
1717
if [ -z "$GHE_TEST_REMOTE_VERSION" ]; then
18-
ghe-ssh "$GHE_HOSTNAME" "test -e /usr/local/bin/ghe-import-mssql"
18+
ghe_ssh_mssql "test -e /usr/local/bin/ghe-import-mssql"
1919
else
20-
ghe-ssh "$GHE_HOSTNAME" "type ghe-import-mssql"
20+
ghe_ssh_mssql "type ghe-import-mssql"
2121
fi
2222
}
2323

24+
ghe_ssh_mssql() {
25+
ghe-ssh $opts $ssh_config_file_opt "$GHE_MSSQL_PRIMARY_HOST" "$@"
26+
}
27+
28+
cleanup() {
29+
rm -rf $tempdir
30+
}
31+
trap 'cleanup' EXIT INT
32+
33+
# Grab host arg
34+
GHE_HOSTNAME="$1"
35+
36+
# use the mssql primary host if GHES cluster configuration contains a mssql-master or use the ghe server if the mssql-master is not available.
37+
GHE_MSSQL_PRIMARY_NODE="$(ghe-ssh "$GHE_HOSTNAME" -- "ghe-config cluster.mssql-master" || true)"
38+
GHE_MSSQL_PRIMARY_HOST="$(ghe-ssh "$GHE_HOSTNAME" -- "ghe-config cluster.$GHE_MSSQL_PRIMARY_NODE.hostname" || true)"
39+
40+
if [ -z "$GHE_MSSQL_PRIMARY_HOST" ]; then
41+
GHE_MSSQL_PRIMARY_HOST="$GHE_HOSTNAME"
42+
fi
43+
44+
tempdir=$(mktemp -d -t backup-utils-backup-XXXXXX)
45+
ssh_config_file_opt=
46+
opts=
47+
48+
isHA="$(ghe-ssh "$GHE_HOSTNAME" -- "ghe-config cluster.ha" || true)"
49+
50+
# get server hostnames under cluster and HA
51+
if [ "$GHE_BACKUP_STRATEGY" = "cluster" ] || [ "$isHA" = "true" ] ; then
52+
ssh_config_file="$tempdir/ssh_config"
53+
ssh_config_file_opt="-F $ssh_config_file"
54+
opts="-o UserKnownHostsFile=/dev/null -o StrictHostKeyChecking=no -o PasswordAuthentication=no"
55+
ghe-ssh-config "$GHE_HOSTNAME" "$GHE_MSSQL_PRIMARY_HOST" > "$ssh_config_file"
56+
fi
57+
2458
if ! import_tool_available; then
2559
ghe_verbose "ghe-import-mssql is not available"
2660
exit
2761
fi
2862

29-
# Grab host arg
30-
GHE_HOSTNAME="$1"
31-
3263
# Perform a host-check and establish the remote version in GHE_REMOTE_VERSION.
3364
ghe_remote_version_required "$GHE_HOSTNAME"
3465

@@ -41,20 +72,20 @@ snapshot_dir_mssql="$GHE_DATA_DIR/$GHE_RESTORE_SNAPSHOT/mssql"
4172

4273
# Transfer backup files from appliance to backup host
4374
appliance_dir="$GHE_REMOTE_DATA_USER_DIR/mssql/backups"
44-
echo "set -o pipefail; sudo rm -rf $appliance_dir; sudo mkdir -p $appliance_dir" | ghe-ssh $GHE_HOSTNAME /bin/bash
45-
for b in $snapshot_dir_mssql/*
75+
echo "set -o pipefail; sudo rm -rf $appliance_dir; sudo mkdir -p $appliance_dir" | ghe_ssh_mssql /bin/bash
76+
for b in "$snapshot_dir_mssql"/*
4677
do
4778
[[ -e "$b" ]] || break
4879

4980
filename="${b##*/}"
5081
ghe_verbose "Transferring $filename to appliance host"
51-
cat $snapshot_dir_mssql/$filename | ghe-ssh $GHE_HOSTNAME "sudo tee -a $appliance_dir/$filename >/dev/null 2>&1"
82+
cat $snapshot_dir_mssql/$filename | ghe_ssh_mssql "sudo tee -a $appliance_dir/$filename >/dev/null 2>&1"
5283
done
5384

5485
# Change owner to mssql:mssql to ready for restore
55-
ghe-ssh $GHE_HOSTNAME "sudo chown -R mssql:mssql $appliance_dir"
86+
ghe_ssh_mssql "sudo chown -R mssql:mssql $appliance_dir"
5687

5788
# Invoke restore command
5889
bm_start "$(basename $0)"
59-
ghe-ssh "$GHE_HOSTNAME" -- "ghe-import-mssql" < "/dev/null" 1>&3
90+
ghe_ssh_mssql -- "ghe-import-mssql" < "/dev/null" 1>&3
6091
bm_end "$(basename $0)"

0 commit comments

Comments
 (0)