improve bootstrap logic to wait for at least 2 apiserver instances

Author: tkashem

data/data/bootstrap/files/usr/local/bin/report-progress.sh

@@ -3,6 +3,9 @@
 # shellcheck disable=SC1091  # using path on bootstrap machine
 . /usr/local/bin/bootstrap-service-record.sh
 
+# shellcheck disable=SC1091  # using path on bootstrap machine
+. /usr/local/bin/wait-for-ha-api.sh
+
 KUBECONFIG="${1}"
 
 wait_for_existence() {
@@ -17,6 +20,9 @@ echo "Waiting for bootstrap to complete..."
 wait_for_existence /opt/openshift/.bootkube.done
 record_service_stage_success
 
+## wait for API to be available
+wait_for_ha_api
+
 record_service_stage_start "report-bootstrap-complete"
 echo "Reporting install progress..."
 while ! oc --kubeconfig="$KUBECONFIG" create -f - <<-EOF

data/data/bootstrap/files/usr/local/bin/wait-for-ha-api.sh

@@ -0,0 +1,41 @@
+#!/usr/bin/env bash
+
+wait_for_ha_api() {
+    if [ "$BOOTSTRAP_INPLACE" = true ]
+    then
+	return 0
+    fi
+
+    echo "Waiting for at least 2 available IP addresses for the default/kubernetes service"
+    while ! is_api_available
+    do
+	sleep 5
+    done
+}
+
+##
+## for HA cluster, we mark the bootstrap process as complete when there
+## are at least two IP addresses available to the endpoints
+## of the default/kubernetes service object.
+## TODO: move this to kas operator as a subcommand of the render command
+is_api_available() {
+    output=$(oc --kubeconfig="$KUBECONFIG" get endpoints kubernetes --namespace=default -o jsonpath='{range @.subsets[*]}{range @.addresses[*]}{.ip}{" "}' 2>&1 )
+    # shellcheck disable=SC2124
+    status=$?
+    if [[ $status -ne 0 ]]
+    then
+	echo "The following error happened while retrieving the default/kubernetes endpoint object"
+	echo "$output"
+	return 1
+    fi
+    
+    echo "Got the following addresses for the default/kubernetes endpoint object: $output"
+    count=$(echo "$output" | wc -w)
+    if [[ ! $count -gt 1 ]]
+    then
+	return 1
+    fi
+    
+    echo "Got at least 2 available addresses for the default/kubernetes service"
+    return 0
+}

data/data/bootstrap/gcp/files/usr/local/bin/report-progress.sh

@@ -1,5 +1,8 @@
 #!/usr/bin/env bash
 
+# shellcheck disable=SC1091  # using path on bootstrap machine
+. /usr/local/bin/wait-for-ha-api.sh
+
 KUBECONFIG="${1}"
 
 wait_for_existance() {
@@ -18,6 +21,9 @@ systemctl stop gcp-routes.service
 echo "Waiting for bootstrap to complete..."
 wait_for_existance /opt/openshift/.bootkube.done
 
+## wait for API to be available
+wait_for_ha_api
+
 echo "Reporting install progress..."
 while ! oc --kubeconfig="$KUBECONFIG" create -f - <<-EOF
 	apiVersion: v1