Skip to content

Commit 2153c50

Browse files
kgrzebienkgrzebien
authored
Merge branch 'master' into users/kgrzebie/actions-migrate-4293
2 parents fbd4bec + f26ee12 commit 2153c50

File tree

5 files changed

+223
-20
lines changed

5 files changed

+223
-20
lines changed

‎debian/changelog

Lines changed: 35 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -1,3 +1,38 @@
1+
github-backup-utils (3.3.0) UNRELEASED; urgency=medium
2+
3+
4+
-- [email protected] Wed, 08 Dec 2021 03:12:53 +0000
5+
6+
github-backup-utils (3.3.0) UNRELEASED; urgency=medium
7+
8+
9+
-- [email protected] Wed, 08 Dec 2021 03:10:21 +0000
10+
11+
github-backup-utils (3.3.0) UNRELEASED; urgency=medium
12+
13+
14+
-- [email protected] Wed, 08 Dec 2021 02:56:47 +0000
15+
16+
github-backup-utils (3.3.0) UNRELEASED; urgency=medium
17+
18+
19+
-- [email protected] Wed, 08 Dec 2021 02:51:06 +0000
20+
21+
github-backup-utils (3.3.0) UNRELEASED; urgency=medium
22+
23+
24+
-- [email protected] Wed, 08 Dec 2021 02:02:25 +0000
25+
26+
github-backup-utils (3.3.0) UNRELEASED; urgency=medium
27+
28+
29+
-- [email protected] Wed, 08 Dec 2021 01:56:26 +0000
30+
31+
github-backup-utils (3.3.0) UNRELEASED; urgency=medium
32+
33+
34+
-- [email protected] Wed, 08 Dec 2021 01:25:12 +0000
35+
136
github-backup-utils (3.3.0.rc1) UNRELEASED; urgency=medium
237

338

‎share/github-backup-utils/ghe-ssh-config

Lines changed: 60 additions & 11 deletions
Original file line numberDiff line numberDiff line change
@@ -1,7 +1,8 @@
11
#!/usr/bin/env bash
2-
#/ Usage: ghe-ssh-config <proxy_host> [<host>...]
2+
#/ Usage: ghe-ssh-config <ghe_host> [<host>...]
33
#/
4-
#/ Returns a SSH configuration file.
4+
#/ Returns a SSH configuration file which configures the connections either through proxy
5+
#/ using <ghe_host> or connect directly by fetching the IP to list of <host> by <ghe_host>
56
#/
67
#/ Note: This script typically isn't called directly. It's invoked by the
78
#/ ghe-[backup|restore]-* commands.
@@ -18,20 +19,68 @@ GHE_HOSTNAME="$1"
1819
shift
1920

2021
hosts="$*"
21-
22-
proxy_host=$(ssh_host_part "$GHE_HOSTNAME")
23-
proxy_port=$(ssh_port_part "$GHE_HOSTNAME")
24-
proxy_user="${proxy_host%@*}"
25-
[ "$proxy_user" = "$proxy_host" ] && proxy_user="admin"
26-
22+
ghe_host=$(ssh_host_part "$GHE_HOSTNAME")
23+
ghe_port=$(ssh_port_part "$GHE_HOSTNAME")
24+
ghe_user="${ghe_host%@*}"
25+
[ "$ghe_user" = "$ghe_host" ] && ghe_user="admin"
2726
opts="$GHE_EXTRA_SSH_OPTS"
27+
# In case we are connecting to node without <role>-server-<uuid> format, revert back to proxy mode
28+
[ -z "$GHE_DISABLE_SSH_MUX" ] && opts="-o ControlMaster=auto -o ControlPath=\"$TMPDIR/.ghe-sshmux-$(echo -n "$ghe_user@$ghe_host:$ghe_port" | git hash-object --stdin | cut -c 1-8)\" -o ControlPersist=10m -o ServerAliveInterval=10 $opts"
2829

29-
[ -z "$GHE_DISABLE_SSH_MUX" ] && opts="-o ControlMaster=auto -o ControlPath=\"$TMPDIR/.ghe-sshmux-$(echo -n "$proxy_user@$proxy_host:$proxy_port" | git hash-object --stdin | cut -c 1-8)\" -o ControlPersist=10m -o ServerAliveInterval=10 $opts"
30+
# Allow GIT_CONFIG to be specified manually for CI.
31+
if [ -z "$GIT_CONFIG" ]; then
32+
# If an individual backup step is being run directly, or this is a restore
33+
# then ghe-backup-settings won't have ran, which transfers cluster.conf.
34+
if ! $GHE_RESTORE_SNAPSHOT_PATH && [ -f "$GHE_SNAPSHOT_DIR/cluster.conf" ]; then
35+
GIT_CONFIG="$GHE_SNAPSHOT_DIR/cluster.conf"
36+
else
37+
cluster_config_file="$(mktemp -t ".ghe-cluster-conf-XXXXXX")"
38+
ghe-ssh "$GHE_HOSTNAME" -- "sudo cat $GHE_REMOTE_CLUSTER_CONF_FILE 2>/dev/null" > "$cluster_config_file"
39+
GIT_CONFIG="$cluster_config_file"
40+
fi
41+
fi
42+
export GIT_CONFIG_NOSYSTEM=1 GIT_CONFIG
3043

3144
for host in $hosts; do
32-
cat <<EOF
45+
# Determine if a <role>-server-<uuid> host has been specified, and if so
46+
# generate the relevant SSH configuration.
47+
if [[ "$host" =~ [A-Za-z]+-server-[A-Za-z0-9]{8}-[A-Za-z0-9]{4}-[A-Za-z0-9]{4}-[A-Za-z0-9]{4}-[A-Za-z0-9]{12} ]]; then
48+
for cluster_host in $(git config --get-regexp cluster.*.hostname | cut -d ' ' -f2); do
49+
uuid=$(git config cluster.$cluster_host.uuid)
50+
if [[ "$host" =~ [A-Za-z]+-server-$uuid ]]; then
51+
if [ -n "$(git config cluster.$cluster_host.ipv6)" ]; then
52+
ip=$(git config "cluster.$cluster_host.ipv6")
53+
elif [ -n "$(git config cluster.$cluster_host.ipv4)" ]; then
54+
ip=$(git config "cluster.$cluster_host.ipv4")
55+
fi
56+
57+
if [ -z "$temp_ssh_config_file" ]; then
58+
temp_ssh_config_file="$(mktemp -t ".hostfile-XXXXXX")"
59+
echo "Host *
60+
User $ghe_user
61+
Port $ghe_port
62+
BatchMode yes" >> "$temp_ssh_config_file"
63+
fi
64+
65+
echo "Host git-server-$uuid pages-server-$uuid storage-server-$uuid
66+
HostName $ip
67+
Port $ghe_port
68+
StrictHostKeyChecking no" >> "$temp_ssh_config_file"
69+
# If proxy mode is set
70+
if [ -n "$GHE_SSH_PROXY" ]; then
71+
echo "ProxyCommand ssh -q $opts -p $ghe_port $ghe_user@$ghe_host nc.openbsd %h %p" >> "$temp_ssh_config_file"
72+
fi
73+
fi
74+
done
75+
else
76+
cat <<EOF
3377
Host $host
34-
ProxyCommand ssh -q $opts -p $proxy_port $proxy_user@$proxy_host nc.openbsd %h %p
78+
ProxyCommand ssh -q $opts -p $ghe_port $ghe_user@$ghe_host nc.openbsd %h %p
3579
StrictHostKeyChecking=no
3680
EOF
81+
fi
3782
done
83+
84+
if [ -n "$temp_ssh_config_file" ]; then
85+
cat "$temp_ssh_config_file"
86+
fi

‎share/github-backup-utils/version

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -1 +1 @@
1-
3.3.0.rc1
1+
3.3.0

‎test/cluster.conf

Lines changed: 65 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,65 @@
1+
[cluster]
2+
mysql-master = data1
3+
redis-master = data1
4+
primary-datacenter = default
5+
[cluster "app1"]
6+
hostname = app1
7+
ipv4 = 172.31.17.242
8+
consul-datacenter = default
9+
web-server = true
10+
job-server = true
11+
vpn = 169.254.179.1
12+
uuid = 1915bffc-4be0-11ec-9121-020e101d0b31
13+
[cluster "app2"]
14+
hostname = app2
15+
ipv4 = 172.31.16.237
16+
consul-datacenter = default
17+
web-server = true
18+
job-server = true
19+
vpn = 169.254.179.2
20+
uuid = 118ab3aa-4be0-11ec-b260-02a9f5631633
21+
[cluster "data1"]
22+
hostname = data1
23+
ipv4 = 172.31.22.90
24+
consul-server = true
25+
consul-datacenter = default
26+
git-server = true
27+
pages-server = true
28+
mysql-server = true
29+
elasticsearch-server = true
30+
redis-server = true
31+
memcache-server = true
32+
metrics-server = true
33+
storage-server = true
34+
vpn = 169.254.179.3
35+
uuid = 1451687c-4be0-11ec-8684-02c387bd966b
36+
[cluster "data2"]
37+
hostname = data2
38+
ipv4 = 172.31.26.173
39+
consul-server = true
40+
consul-datacenter = default
41+
git-server = true
42+
pages-server = true
43+
mysql-server = true
44+
elasticsearch-server = true
45+
redis-server = true
46+
memcache-server = true
47+
metrics-server = true
48+
storage-server = true
49+
vpn = 169.254.179.4
50+
uuid = 16089d52-4be0-11ec-b892-026c4c5e5bb1
51+
[cluster "data3"]
52+
hostname = data3
53+
ipv4 = 172.31.27.252
54+
consul-server = true
55+
consul-datacenter = default
56+
git-server = true
57+
pages-server = true
58+
mysql-server = true
59+
elasticsearch-server = true
60+
redis-server = true
61+
memcache-server = true
62+
metrics-server = true
63+
storage-server = true
64+
vpn = 169.254.179.5
65+
uuid = 1b355f04-4be0-11ec-8a88-02c296aec13d

‎test/test-ghe-ssh-config.sh

Lines changed: 62 additions & 8 deletions
Original file line numberDiff line numberDiff line change
@@ -5,16 +5,66 @@
55
# shellcheck source=test/testlib.sh
66
. "$(dirname "$0")/testlib.sh"
77

8-
begin_test "ghe-ssh-config returns config for multiple nodes"
8+
export CLUSTER_CONF="$ROOTDIR/test/cluster.conf"
9+
10+
begin_test "ghe-ssh-config returns config for git-server nodes"
11+
(
12+
set -e
13+
14+
output=$(GIT_CONFIG=$CLUSTER_CONF ghe-ssh-config host1 git-server-1451687c-4be0-11ec-8684-02c387bd966b git-server-16089d52-4be0-11ec-b892-026c4c5e5bb1)
15+
# Confirm we don't have a host1 entry as this is the proxy host
16+
[ "$(echo "$output" | grep -c "^Host host1")" -eq 0 ]
17+
# Confirm we have git-server-<uuid> entries
18+
echo "$output" | grep -Eq "^Host git-server-[A-Za-z0-9]{8}-[A-Za-z0-9]{4}-[A-Za-z0-9]{4}-[A-Za-z0-9]{4}-[A-Za-z0-9]{12}"
19+
echo "$output" | grep -Eq "pages-server-[A-Za-z0-9]{8}-[A-Za-z0-9]{4}-[A-Za-z0-9]{4}-[A-Za-z0-9]{4}-[A-Za-z0-9]{12}"
20+
echo "$output" | grep -Eq "storage-server-[A-Za-z0-9]{8}-[A-Za-z0-9]{4}-[A-Za-z0-9]{4}-[A-Za-z0-9]{4}-[A-Za-z0-9]{12}"
21+
[ "$(echo "$output" | grep -Ec "^Host git-server-[A-Za-z0-9]{8}-[A-Za-z0-9]{4}-[A-Za-z0-9]{4}-[A-Za-z0-9]{4}-[A-Za-z0-9]{12}")" -eq 2 ]
22+
# Confirm the git-server entries has right IP
23+
echo "$output" | grep -q "HostName 172.31.22.90"
24+
echo "$output" | grep -q "HostName 172.31.26.173"
25+
# Confirm No proxy enabled
26+
[ "$(echo "$output" | grep -c "ProxyCommand")" -eq 0 ]
27+
)
28+
end_test
29+
30+
begin_test "ghe-ssh-config returns config for git-server nodes with GHE_SSH_PROXY=1"
31+
(
32+
set -e
33+
34+
output=$(GIT_CONFIG=$CLUSTER_CONF GHE_SSH_PROXY=1 ghe-ssh-config host1 git-server-1451687c-4be0-11ec-8684-02c387bd966b git-server-16089d52-4be0-11ec-b892-026c4c5e5bb1)
35+
# Confirm we don't have a host1 entry as this is the proxy host
36+
[ "$(echo "$output" | grep -c "^Host host1")" -eq 0 ]
37+
# Confirm we have git-server-<uuid> entries
38+
echo "$output" | grep -Eq "^Host git-server-[A-Za-z0-9]{8}-[A-Za-z0-9]{4}-[A-Za-z0-9]{4}-[A-Za-z0-9]{4}-[A-Za-z0-9]{12}"
39+
echo "$output" | grep -Eq "pages-server-[A-Za-z0-9]{8}-[A-Za-z0-9]{4}-[A-Za-z0-9]{4}-[A-Za-z0-9]{4}-[A-Za-z0-9]{12}"
40+
echo "$output" | grep -Eq "storage-server-[A-Za-z0-9]{8}-[A-Za-z0-9]{4}-[A-Za-z0-9]{4}-[A-Za-z0-9]{4}-[A-Za-z0-9]{12}"
41+
[ "$(echo "$output" | grep -Ec "^Host git-server-[A-Za-z0-9]{8}-[A-Za-z0-9]{4}-[A-Za-z0-9]{4}-[A-Za-z0-9]{4}-[A-Za-z0-9]{12}")" -eq 2 ]
42+
# Confirm the git-server entries has right IP
43+
echo "$output" | grep -q "HostName 172.31.22.90"
44+
echo "$output" | grep -q "HostName 172.31.26.173"
45+
# Confirm proxy enabled
46+
[ "$(echo "$output" | grep -c "ProxyCommand")" -eq 2 ]
47+
48+
# Confirm ControlPath returns correct hash for admin@host1:122
49+
echo "$output" | grep -q "admin@host1 nc.openbsd"
50+
# Confirm multiplexing enabled
51+
echo "$output" | grep -q "ControlMaster=auto"
52+
# Confirm ControlPath returns correct hash for admin@host1:122
53+
echo "$output" | grep -q ".ghe-sshmux-7cb77002"
54+
)
55+
end_test
56+
57+
58+
begin_test "ghe-ssh-config returns config for non-server-uuid nodes"
959
(
1060
set -e
1161

12-
output=$(ghe-ssh-config host1 git-server1 git-server2)
62+
output=$(GIT_CONFIG=$CLUSTER_CONF ghe-ssh-config host1 mysql-node1 mysql-node2)
1363
# Confirm we don't have a host1 entry as this is the proxy host
1464
echo "$output" | grep -Evq "^Host host1"
1565
# Confirm we have a host2 and host3 entry
16-
echo "$output" | grep -Eq "^Host git-server[12]"
17-
[ "$(echo "$output" | grep -E "^Host git-server[12]" | wc -l)" -eq 2 ]
66+
echo "$output" | grep -Eq "^Host mysql-node[12]"
67+
[ "$(echo "$output" | grep -c "^Host mysql-node[12]")" -eq 2 ]
1868
# Confirm the host2 and host3 entries proxy though host1
1969
echo "$output" | grep -q "admin@host1 nc.openbsd"
2070
# Confirm multiplexing enabled
@@ -28,22 +78,26 @@ begin_test "ghe-ssh-config multiplexing disabled"
2878
(
2979
set -e
3080

31-
output=$(GHE_DISABLE_SSH_MUX=1 ghe-ssh-config host1 git-server1)
81+
output=$(GIT_CONFIG=$CLUSTER_CONF GHE_DISABLE_SSH_MUX=1 ghe-ssh-config host1 git-server1)
3282
echo "$output" | grep -vq "ControlMaster=auto"
3383

34-
output=$(GHE_DISABLE_SSH_MUX=1 ghe-ssh-config host1 git-server1 git-server2)
84+
output=$(GIT_CONFIG=$CLUSTER_CONF GHE_DISABLE_SSH_MUX=1 ghe-ssh-config host1 git-server1 git-server2)
3585
echo "$output" | grep -vq "ControlMaster=auto"
86+
87+
# Confirm multiplexing disabled
88+
[ "$(echo "$output" | grep -c "ControlMaster=auto")" -eq 0 ]
89+
[ "$(echo "$output" | grep -c ".ghe-sshmux-7cb77002")" -eq 0 ]
3690
)
3791
end_test
3892

3993
begin_test "ghe-ssh-config with extra SSH opts"
4094
(
4195
set -e
4296

43-
output=$(GHE_EXTRA_SSH_OPTS="-o foo=bar" ghe-ssh-config host1 git-server1)
97+
output=$(GIT_CONFIG=$CLUSTER_CONF GHE_EXTRA_SSH_OPTS="-o foo=bar" ghe-ssh-config host1 git-server1)
4498
echo "$output" | grep -q "foo=bar"
4599

46-
output=$(GHE_EXTRA_SSH_OPTS="-o foo=bar" ghe-ssh-config host1 git-server1 git-server2)
100+
output=$(GIT_CONFIG=$CLUSTER_CONF GHE_EXTRA_SSH_OPTS="-o foo=bar" ghe-ssh-config host1 git-server1 git-server2)
47101
echo "$output" | grep -q "foo=bar"
48102
)
49103
end_test

0 commit comments

Comments
 (0)