Skip to content

Commit 6e2184a

Browse files
chuckp22chuckp22
authored
Merge pull request #269 from github/preflight-checks
Checks for backup host reqs
2 parents 5ad5f10 + 4c5179a commit 6e2184a

File tree

4 files changed

+144
-0
lines changed

4 files changed

+144
-0
lines changed

‎bin/ghe-backup

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -37,6 +37,7 @@ while true; do
3737
esac
3838
done
3939

40+
export CALLING_SCRIPT="ghe-backup"
4041
# Bring in the backup configuration
4142
# shellcheck source=share/github-backup-utils/ghe-backup-config
4243
. "$( dirname "${BASH_SOURCE[0]}" )/../share/github-backup-utils/ghe-backup-config"

‎bin/ghe-host-check

Lines changed: 65 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -144,4 +144,69 @@ if [ -z "$supported" ]; then
144144
exit 1
145145
fi
146146

147+
if [[ "$CALLING_SCRIPT" == "ghe-backup" ]]; then
148+
# Bring in the requirements file
149+
min_rsync=""
150+
min_openssh=""
151+
min_jq=""
152+
# shellcheck source=share/github-backup-utils/requirements.txt
153+
. "$(dirname "${BASH_SOURCE[0]}")/../share/github-backup-utils/requirements.txt"
154+
155+
#source disk size file
156+
# shellcheck source=share/github-backup-utils/ghe-rsync-size.sh
157+
. "$(dirname "${BASH_SOURCE[0]}")/../share/github-backup-utils/ghe-rsync-size.sh"
158+
159+
#Display dir requirements for repositories and mysql
160+
echo "Checking host for sufficient space for a backup..." 1>&2
161+
available_space=$(df -B 1k $GHE_DATA_DIR | awk 'END{printf "%.0f", $4 * 1024}')
162+
printf "Available space: %d MB. We recommend allocating at least 5x the amount of storage allocated to the primary GitHub appliance for historical snapshots and growth over time.\n" "$((available_space / 1024 ** 2))" 1>&2
163+
164+
repos_disk_size=$(transfer_size repositories /tmp)
165+
pages_disk_size=$(transfer_size pages /tmp)
166+
es_disk_size=$(transfer_size elasticsearch /tmp)
167+
stor_disk_size=$(transfer_size storage /tmp)
168+
minio_disk_size=$(transfer_size minio /tmp)
169+
mysql_disk_size=$(transfer_size mysql /tmp)
170+
actions_disk_size=$(transfer_size actions /tmp)
171+
mssql_disk_size=$(transfer_size mssql /tmp)
172+
173+
#min_disk_req=$(( $(echo "$repos_disk_size" | awk '{printf "%.0f", $1}') + $(echo "$pages_disk_size" | awk '{printf "%.0f", $1}') + $(echo "$es_disk_size" | awk '{printf "%.0f", $1}') + $(echo "$stor_disk_size" | awk '{printf "%.0f", $1}') + $(echo "$minio_disk_size" | awk '{printf "%.0f", $1}') + $(echo "$mysql_disk_size" | awk '{printf "%.0f", $1/2}') + $(echo "$actions_disk_size" | awk '{printf "%.0f", $1}') + $(echo "$mssql_disk_size" | awk '{printf "%.0f", $1}') ))
174+
min_disk_req=$(( $(echo "$repos_disk_size") + $(echo "$pages_disk_size") + $(echo "$es_disk_size") + $(echo "$stor_disk_size") + $(echo "$minio_disk_size") + $(echo "$mysql_disk_size") + $(echo "$actions_disk_size") + $(echo "$mssql_disk_size") ))
175+
echo -e "### Data Transfer Sizes \nrepositories: $repos_disk_size \npages: $pages_disk_size \nelasticsearch: $es_disk_size \nstorage: $stor_disk_size \nminio: $minio_disk_size \nmysql: $mysql_disk_size \nactions: $actions_disk_size \nmssql: $mssql_disk_size" 1>&2
176+
printf "min_disk_required for this backup is at least %d MB\n" "$((min_disk_req / 1024 ** 2))" 1>&2
177+
178+
if [[ $available_space -lt $min_disk_req ]]; then
179+
echo "There is not enough disk space for the backup. Please allocate more space and continue." 1>&2
180+
exit 1
181+
fi
182+
183+
#Check rsync, openssh & jq versions
184+
rsync_version=$(rsync --version | grep 'version' | awk '{print $3}')
185+
if awk "BEGIN {exit !($rsync_version < $min_rsync)}" &> /dev/null; then
186+
echo "rsync version $rsync_version in backup-host does not meet minimum requirements." 1>&2
187+
echo "Please make sure you have the minimum required version of rsync: $min_rsync installed" 1>&2
188+
exit 1
189+
else
190+
echo "rsync ${rsync_version} >= required ($min_rsync)" 1>&2
191+
fi
192+
193+
ssh_version=$(ssh -V 2>&1 | awk '{print $1}'|grep -oPm 1 '[\d\.]+' |head -1)
194+
if awk "BEGIN {exit !($ssh_version < $min_openssh)}" &> /dev/null; then
195+
echo "openSSH version $ssh_version in backup-host does not meet minimum requirements." 1>&2
196+
echo "Please make sure the minimum required version of openSSH: $min_openssh is installed" 1>&2
197+
exit 1
198+
else
199+
echo "openSSH ${ssh_version} >= required ($min_openssh)" 1>&2
200+
fi
201+
202+
jq_version=$(jq --version |awk -F\- '{print $2}')
203+
if awk "BEGIN {exit !($jq_version < $min_jq)}" &> /dev/null; then
204+
echo "jq version $jq_version in backup-host does not meet minimum requirements." 1>&2
205+
echo "Please make sure you have the minimum required version of jq: $min_jq installed" 1>&2
206+
exit 1
207+
else
208+
echo "jq ${jq_version} >= required ($min_jq)" 1>&2
209+
fi
210+
fi
211+
147212
echo "Connect $hostname:$port OK (v$version)"

‎share/github-backup-utils/ghe-rsync-size.sh

Lines changed: 75 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,75 @@
1+
#!/usr/bin/env bash
2+
# get-rsync-size.sh Get the total size of dir-files to be transfered using rsync --link-dest
3+
#
4+
# Example:
5+
# transfer_size repositories /dest_dir
6+
#
7+
# Sample output:
8+
# Total transferred file size: 80 bytes
9+
10+
# Bring in the backup configuration
11+
# shellcheck source=share/github-backup-utils/ghe-backup-config
12+
. "$(dirname "${BASH_SOURCE[0]}")/ghe-backup-config"
13+
14+
# Location of last good backup for rsync --link-dest
15+
backup_current="$GHE_DATA_DIR/current/"
16+
17+
# If we have a previous increment, avoid using those unchanged files using --link-dest support.
18+
if [ -d "$backup_current" ]; then
19+
link_dest="--link-dest=${GHE_DATA_DIR}/current"
20+
fi
21+
22+
transfer_size()
23+
{
24+
local backup_data=$1
25+
if [[ "$1" == "mssql" ]]; then
26+
data_user_dir="/data/user/$1/backups"
27+
else
28+
data_user_dir="/data/user/$1"
29+
fi
30+
local dest_dir=$2
31+
32+
# Define user for rsync-path
33+
case "$backup_data" in
34+
"repositories" | "pages")
35+
user="git"
36+
;;
37+
"storage")
38+
user="alambic"
39+
;;
40+
"elasticsearch")
41+
user="elasticsearch"
42+
;;
43+
"mysql")
44+
user="mysql"
45+
;;
46+
"mssql")
47+
user="mssql"
48+
;;
49+
"actions")
50+
user="actions"
51+
;;
52+
"minio")
53+
user="minio"
54+
;;
55+
*)
56+
echo "Unknown user: $backup_data"
57+
exit 1
58+
;;
59+
esac
60+
61+
total_file_size=$(ghe-rsync -arn --stats \
62+
-e "ssh -q $GHE_EXTRA_SSH_OPTS -p 122 -l admin" \
63+
--rsync-path="sudo -u $user rsync" \
64+
$link_dest/$1 \
65+
--ignore-missing-args \
66+
"$GHE_HOSTNAME:$data_user_dir/" \
67+
"$dest_dir/" | grep "Total transferred file size" | sed 's/.*size: //; s/,//g')
68+
69+
# Reduce mysql size as only the compressed file is transferred
70+
if [[ "$1" == "mysql" ]]; then
71+
echo "$total_file_size" | awk '{printf "%.0f\n", $1/2}'
72+
else
73+
echo "$total_file_size" | awk '{printf "%.0f\n", $1}'
74+
fi
75+
}

‎share/github-backup-utils/requirements.txt

Lines changed: 3 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,3 @@
1+
min_rsync=2.6.4
2+
min_openssh=5.6
3+
min_jq=1.5

0 commit comments

Comments
 (0)