Include bootstrap IP in etcd endpoints list

During bootstrap kube-apiserver points to master etcd instances.
These instances are being added to etcd-servers list one by one,
to ensure that if any master can't reach some other etcds this revision
 won't block the entire rollout.
 However, at some point the masters may end up with just one etcd
 (localhost and its IP), which makes it non-HA and apiserver may
 lose etcd connection if this etcd is being reconfigured.

This change updates the list to include bootstrap etcd so that
any master would always have an etcd instance to connect to.

Author: Vadim Rutkovsky

pkg/operator/configobserver/etcd/observe_etcd_endpoints.go

@@ -18,7 +18,8 @@ const (
 	etcdEndpointName      = "etcd-endpoints"
 
 	// EtcdEndpointName points to the old host-etcd-2 endpoint applicable for clusters prior to 4.5
-	EtcdEndpointName = "host-etcd-2"
+	EtcdEndpointName         = "host-etcd-2"
+	BootstrapIPAnnotationKey = "alpha.installer.openshift.io/etcd-bootstrap"
 )
 
 var (
@@ -94,12 +95,19 @@ func innerObserveStorageURLs(fallbackObserver fallBackObserverFn, alwaysAppendLo
 		return previouslyObservedConfig, append(errs, err)
 	}
 
-	// note: etcd bootstrap should never be added to the in-cluster kube-apiserver
-	// this can result in some early pods crashlooping, but ensures that we never contact the bootstrap machine from
-	// the in-cluster kube-apiserver so we can safely teardown out of order.
-
+	allEtcdEnpoints := []string{}
 	for k := range etcdEndpoints.Data {
-		address := etcdEndpoints.Data[k]
+		allEtcdEnpoints = append(allEtcdEnpoints, etcdEndpoints.Data[k])
+	}
+
+	// include etcd bootstrap IP in the list
+	// this ensures that all masters use it and their local etcd can be torn down / updated
+	// and this won't block revision rollout
+	if bootstrapIP, found := etcdEndpoints.GetAnnotations()[BootstrapIPAnnotationKey]; found {
+		allEtcdEnpoints = append(allEtcdEnpoints, bootstrapIP)
+	}
+
+	for _, address := range allEtcdEnpoints {
 		ip := net.ParseIP(address)
 		if ip == nil {
 			ipErr := fmt.Errorf("configmaps/%s in the %s namespace: %v is not a valid IP address", etcdEndpointName, EtcdEndpointNamespace, address)

pkg/operator/configobserver/etcd/observe_etcd_endpoints_test.go

@@ -184,13 +184,13 @@ func TestInnerObserveStorageURLs(t *testing.T) {
 			expectErrors:      true,
 		},
 		{
-			name:             "IgnoreBootstrap",
+			name:             "BootstrapIncluded",
 			currentConfigFor: observedConfigFor(withStorageURLFor("https://previous.url:2379")),
 			endpoint: endpoints(
 				withBootstrap("10.0.0.2"),
 				withAddress("10.0.0.1"),
 			),
-			expectedConfigFor: observedConfigFor(withStorageURLFor("https://10.0.0.1:2379")),
+			expectedConfigFor: observedConfigFor(withStorageURLFor("https://10.0.0.1:2379"), withStorageURLFor("https://10.0.0.2:2379")),
 		},
 	}
 	for _, tt := range tests {