Skip to content

Commit 5901c75

Browse files
cybertroncybertron
authored
Support running with external loadbalancer (#1825)
1 parent 965ea4b commit 5901c75

File tree

7 files changed

+92
-3
lines changed

7 files changed

+92
-3
lines changed

‎06_create_cluster.sh‎

Lines changed: 4 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -29,6 +29,10 @@ if [[ ! -z "$INSTALLER_PROXY" ]]; then
2929
fi
3030
fi
3131

32+
if [ -n "$EXTERNAL_LOADBALANCER" ]; then
33+
./external_loadbalancer.sh &
34+
fi
35+
3236
# Call openshift-installer to deploy the bootstrap node and masters
3337
create_cluster ${OCP_DIR}
3438

‎common.sh‎

Lines changed: 7 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -385,6 +385,13 @@ export TEST_CUSTOM_MAO=${TEST_CUSTOM_MAO:-false}
385385
# (Currently this just expects a non-empty value, the IP is fixed to .9)
386386
export ENABLE_BOOTSTRAP_STATIC_IP=${ENABLE_BOOTSTRAP_STATIC_IP:-}
387387

388+
export EXTERNAL_LOADBALANCER=${EXTERNAL_LOADBALANCER:-}
389+
390+
if [ -n "$EXTERNAL_LOADBALANCER" -a -z "$ENABLE_BOOTSTRAP_STATIC_IP" ]; then
391+
error "EXTERNAL_LOADBALANCER requires ENABLE_BOOTSTRAP_STATIC_IP to be set as well"
392+
exit 1
393+
fi
394+
388395
# TODO(bnemec): Once https://github.com/ansible/ansible/pull/75537 merges this
389396
# can be removed.
390397
ALMA_PYTHON_OVERRIDE=

‎config_example.sh‎

Lines changed: 10 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -427,6 +427,16 @@ set -x
427427
# When set to any value this will cause dev-scripts to include duplicate nics
428428
# on the primary network. This is intended for testing bonded network configs
429429
# and may not work without a bond config.
430+
# export BOND_PRIMARY_INTERFACE=1
431+
432+
# EXTERNAL_LOADBALANCER -
433+
# When set to any value this will cause dev-scripts to configure an haproxy
434+
# loadbalancer on the host and configure the cluster to use it instead of the
435+
# internal loadbalancer.
436+
# Because of the way the loadbalancer config is written, this only works when
437+
# using single stack (either ipv4 or ipv6) and a static bootstrap IP (see the
438+
# ENABLE_BOOTSTRAP_STATIC_IP option above).
439+
# export EXTERNAL_LOADBALANCER=1
430440

431441
################################################################################
432442
## VM Settings

‎external_loadbalancer.sh‎

Lines changed: 43 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -9,6 +9,7 @@ source utils.sh
99

1010
sudo firewall-cmd --zone=libvirt --add-port=6443/tcp
1111
sudo firewall-cmd --zone=libvirt --add-port=8080/tcp
12+
sudo firewall-cmd --zone=libvirt --add-port=22623/tcp
1213

1314
haproxy_config="${WORKING_DIR}/haproxy.cfg"
1415
echo $haproxy_config
@@ -21,13 +22,15 @@ then
2122
master2=$(nth_ip $EXTERNAL_SUBNET_V6 22)
2223
worker0=$(nth_ip $EXTERNAL_SUBNET_V6 23)
2324
worker1=$(nth_ip $EXTERNAL_SUBNET_V6 24)
25+
bootstrap=$(nth_ip $EXTERNAL_SUBNET_V6 9)
2426
else
2527

2628
master0=$(nth_ip $EXTERNAL_SUBNET_V4 20)
2729
master1=$(nth_ip $EXTERNAL_SUBNET_V4 21)
2830
master2=$(nth_ip $EXTERNAL_SUBNET_V4 22)
2931
worker0=$(nth_ip $EXTERNAL_SUBNET_V4 23)
3032
worker1=$(nth_ip $EXTERNAL_SUBNET_V4 24)
33+
bootstrap=$(nth_ip $EXTERNAL_SUBNET_V4 9)
3134
fi
3235

3336
cat << EOF > "$haproxy_config"
@@ -43,13 +46,23 @@ frontend main
4346
frontend ingress
4447
bind :::8080 v4v6
4548
default_backend ingress
49+
frontend https
50+
bind :::443 v4v6
51+
default_backend https
52+
frontend mcs
53+
bind :::22623 v4v6
54+
default_backend mcs
55+
frontend ironic
56+
bind :::6385 v4v6
57+
default_backend ironic
4658
backend api
4759
option httpchk GET /readyz HTTP/1.0
4860
option log-health-checks
4961
balance roundrobin
5062
server master-0 ${master0}:6443 check check-ssl inter 1s fall 2 rise 3 verify none
5163
server master-1 ${master1}:6443 check check-ssl inter 1s fall 2 rise 3 verify none
5264
server master-2 ${master2}:6443 check check-ssl inter 1s fall 2 rise 3 verify none
65+
server bootstrap ${bootstrap}:6443 check check-ssl inter 1s fall 2 rise 3 verify none
5366
backend ingress
5467
option httpchk GET /healthz/ready HTTP/1.0
5568
option log-health-checks
@@ -59,6 +72,36 @@ backend ingress
5972
server master-2 ${master2}:80 check check-ssl port 1936 inter 1s fall 2 rise 3 verify none
6073
server w-0 ${worker0}:80 check check-ssl port 1936 inter 1s fall 2 rise 3 verify none
6174
server w-1 ${worker1}:80 check check-ssl port 1936 inter 1s fall 2 rise 3 verify none
75+
backend https
76+
option httpchk GET /healthz/ready HTTP/1.0
77+
option log-health-checks
78+
balance roundrobin
79+
server master-0 ${master0}:443 check check-ssl port 1936 inter 1s fall 2 rise 3 verify none
80+
server master-1 ${master1}:443 check check-ssl port 1936 inter 1s fall 2 rise 3 verify none
81+
server master-2 ${master2}:443 check check-ssl port 1936 inter 1s fall 2 rise 3 verify none
82+
server w-0 ${worker0}:443 check check-ssl port 1936 inter 1s fall 2 rise 3 verify none
83+
server w-1 ${worker1}:443 check check-ssl port 1936 inter 1s fall 2 rise 3 verify none
84+
server bootstrap ${bootstrap}:443 check check-ssl port 1936 inter 1s fall 2 rise 3 verify none
85+
backend mcs
86+
option httpchk GET /config/master HTTP/1.0
87+
option log-health-checks
88+
balance roundrobin
89+
server master-0 ${master0}:22623 check check-ssl inter 1s fall 2 rise 3 verify none
90+
server master-1 ${master1}:22623 check check-ssl inter 1s fall 2 rise 3 verify none
91+
server master-2 ${master2}:22623 check check-ssl inter 1s fall 2 rise 3 verify none
92+
server w-0 ${worker0}:22623 check check-ssl inter 1s fall 2 rise 3 verify none
93+
server w-1 ${worker1}:22623 check check-ssl inter 1s fall 2 rise 3 verify none
94+
server bootstrap ${bootstrap}:22623 check check-ssl inter 1s fall 2 rise 3 verify none
95+
backend ironic
96+
option httpchk GET /v1 HTTP/1.0
97+
option log-health-checks
98+
balance roundrobin
99+
server master-0 ${master0}:6385 check check-ssl inter 30s fall 2 rise 3 verify none
100+
server master-1 ${master1}:6385 check check-ssl inter 30s fall 2 rise 3 verify none
101+
server master-2 ${master2}:6385 check check-ssl inter 30s fall 2 rise 3 verify none
102+
server w-0 ${worker0}:6385 check check-ssl inter 1s fall 2 rise 3 verify none
103+
server w-1 ${worker1}:6385 check check-ssl inter 1s fall 2 rise 3 verify none
104+
server bootstrap ${bootstrap}:6385 check check-ssl inter 30s fall 2 rise 3 verify none
62105
EOF
63106

64107
sudo podman run -d --net host -v "${WORKING_DIR}":/etc/haproxy/:z --entrypoint bash --name extlb quay.io/openshift/origin-haproxy-router -c 'haproxy -f /etc/haproxy/haproxy.cfg'

‎network.sh‎

Lines changed: 10 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -253,12 +253,20 @@ function get_vips() {
253253
#
254254
if [[ -n "${EXTERNAL_SUBNET_V4}" ]]; then
255255
API_VIPS_V4=$(dig +noall +answer "api.${CLUSTER_DOMAIN}" @$(network_ip ${BAREMETAL_NETWORK_NAME}) | awk '{print $NF}')
256-
INGRESS_VIPS_V4=$(nth_ip $EXTERNAL_SUBNET_V4 4)
256+
if [ -z "$EXTERNAL_LOADBALANCER" ]; then
257+
INGRESS_VIPS_V4=$(nth_ip $EXTERNAL_SUBNET_V4 4)
258+
else
259+
INGRESS_VIPS_V4=$(nth_ip $EXTERNAL_SUBNET_V4 1)
260+
fi
257261
fi
258262

259263
if [[ -n "${EXTERNAL_SUBNET_V6}" ]]; then
260264
API_VIPS_V6=$(dig -t AAAA +noall +answer "api.${CLUSTER_DOMAIN}" @$(network_ip ${BAREMETAL_NETWORK_NAME}) | awk '{print $NF}')
261-
INGRESS_VIPS_V6=$(nth_ip $EXTERNAL_SUBNET_V6 4)
265+
if [ -z "$EXTERNAL_LOADBALANCER" ]; then
266+
INGRESS_VIPS_V6=$(nth_ip $EXTERNAL_SUBNET_V6 4)
267+
else
268+
INGRESS_VIPS_V6=$(nth_ip $EXTERNAL_SUBNET_V6 1)
269+
fi
262270
fi
263271

264272
if [[ "$IP_STACK" == "v4" || "$IP_STACK" == "v4v6" ]]; then

‎ocp_install_env.sh‎

Lines changed: 10 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -191,6 +191,15 @@ function setVIPs() {
191191
esac
192192
}
193193

194+
function loadbalancer_type() {
195+
if [ -n "$EXTERNAL_LOADBALANCER" ]; then
196+
cat <<EOF
197+
loadBalancer:
198+
type: UserManaged
199+
EOF
200+
fi
201+
}
202+
194203
function featureSet() {
195204
if [[ -n "$FEATURE_SET" ]]; then
196205
cat <<EOF
@@ -395,6 +404,7 @@ $(cluster_os_image)
395404
$(setVIPs apivips)
396405
$(setVIPs ingressvips)
397406
$(dnsvip)
407+
$(loadbalancer_type)
398408
hosts:
399409
EOF
400410

‎vm_setup_vars.yml‎

Lines changed: 8 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -59,6 +59,13 @@ dns_extrahosts:
5959
hostnames:
6060
- "virthost"
6161

62+
dns_externalhosts:
63+
- ip: "{{ baremetal_network_cidr | nthhost(1) }}"
64+
hostnames:
65+
- "virthost"
66+
- "api"
67+
- "api-int"
68+
6269
network_config_folder: "{{ lookup('env', 'NETWORK_CONFIG_FOLDER') | default(false) }}"
6370
hosts_config: "{{ lookup('template', network_config_folder + '/hosts.yaml', errors='ignore') | default('[]', true) | from_yaml }}"
6471
dns_customhosts: "{{ [] if not network_config_folder else hosts_config }}"
@@ -88,7 +95,7 @@ external_network:
8895
- 65535
8996
domain: "{{ cluster_domain }}"
9097
dns:
91-
hosts: "{{ dns_extrahosts + dns_customhosts + dns_dualstackhost if lookup('env', 'EXTERNAL_SUBNET_V6') else dns_extrahosts + dns_customhosts }}"
98+
hosts: "{{ dns_externalhosts + dns_customhosts if lookup('env', 'EXTERNAL_LOADBALANCER') else dns_extrahosts + dns_customhosts + dns_dualstackhost if lookup('env', 'EXTERNAL_SUBNET_V6') else dns_extrahosts + dns_customhosts }}"
9299
forwarders:
93100
- domain: "apps.{{ cluster_domain }}"
94101
addr: "127.0.0.1"

0 commit comments

Comments
 (0)