OCPBUGS-2183: Updating quorum detection logic to absolve TNF of quorum loss reports.

- Added a new utility function to detect when external etcd cluster have completed transition
- Updated the quorum check in the membership controller prevent CEO being marked as unavailable
  when you would have quorum loss but can't lose quorum because you've completed the
  transition to ExternalEtcd.
- Added new ExternalEtcd lifecycle test to track the flow of an ExternalEtcd transition.
- Generated basic validation tests to ensure that we only set CEO to unavailable
  when you lose quorum on a cluster that doesn't automatically recover quorum
- Updated tests to use utils-based initializers by default

Author: jaypoulz

pkg/operator/bootstrapteardown/bootstrap_teardown_controller_test.go

@@ -371,7 +371,7 @@ func TestRemoveBootstrap(t *testing.T) {
 			fakeNamespaceLister := corev1listers.NewNamespaceLister(indexer)
 			fakeConfigmapLister := corev1listers.NewConfigMapLister(indexer)
 			fakeInfraLister := configv1listers.NewInfrastructureLister(indexer)
-			fakeStaticPodClient := v1helpers.NewFakeStaticPodOperatorClient(&operatorv1.StaticPodOperatorSpec{}, &operatorv1.StaticPodOperatorStatus{}, nil, nil)
+			fakeStaticPodClient := v1helpers.NewFakeStaticPodOperatorClient(&operatorv1.StaticPodOperatorSpec{}, u.StaticPodOperatorStatus(), nil, nil)
 			fakeEtcdClient, err := etcdcli.NewFakeEtcdClient([]*etcdserverpb.Member{u.FakeEtcdBootstrapMember(1)})
 			require.NoError(t, err)
 			fakeKubeClient := fake.NewClientset([]runtime.Object{}...)

pkg/operator/ceohelpers/common_test.go

@@ -9,6 +9,7 @@ import (
 	"k8s.io/apimachinery/pkg/runtime"
 
 	operatorv1 "github.com/openshift/api/operator/v1"
+	u "github.com/openshift/cluster-etcd-operator/pkg/testutils"
 	"github.com/openshift/library-go/pkg/operator/v1helpers"
 )
 
@@ -57,7 +58,7 @@ func TestReadDesiredControlPlaneReplicaCount(t *testing.T) {
 	for _, scenario := range scenarios {
 		t.Run(scenario.name, func(t *testing.T) {
 			// test data
-			fakeOperatorClient := v1helpers.NewFakeStaticPodOperatorClient(&scenario.operatorSpec, &operatorv1.StaticPodOperatorStatus{}, nil, nil)
+			fakeOperatorClient := v1helpers.NewFakeStaticPodOperatorClient(&scenario.operatorSpec, u.StaticPodOperatorStatus(), nil, nil)
 
 			// act
 			actualReplicaCount, err := ReadDesiredControlPlaneReplicasCount(fakeOperatorClient)

pkg/operator/ceohelpers/external_etcd_status.go

@@ -11,9 +11,10 @@ import (
 )
 
 type ExternalEtcdClusterStatus struct {
-	IsExternalEtcdCluster    bool
-	IsEtcdRunningInCluster   bool
-	IsReadyForEtcdTransition bool
+	IsExternalEtcdCluster              bool
+	IsEtcdRunningInCluster             bool
+	IsReadyForEtcdTransition           bool
+	HasExternalEtcdCompletedTransition bool
 }
 
 // IsExternalEtcdCluster determines if the cluster is configured for external etcd
@@ -36,13 +37,18 @@ func IsExternalEtcdCluster(ctx context.Context, infraLister configv1listers.Infr
 // IsReadyForEtcdTransition checks if the cluster is ready for etcd transition
 // by examining the operator status for the ExternalEtcdReadyForTransition condition.
 // This condition is set when the TNF setup is ready to take over the etcd container.
-func IsReadyForEtcdTransition(ctx context.Context, operatorClient v1helpers.StaticPodOperatorClient) (bool, error) {
+func IsReadyForEtcdTransition(operatorClient v1helpers.StaticPodOperatorClient) (bool, error) {
 	_, opStatus, _, err := operatorClient.GetStaticPodOperatorState()
 	if err != nil {
 		klog.Errorf("failed to get static pod operator state: %v", err)
 		return false, err
 	}
 
+	if opStatus == nil {
+		klog.V(2).Info("static pod operator status not yet populated; ready for etcd transition unknown")
+		return false, nil
+	}
+
 	readyForEtcdTransition := v1helpers.IsOperatorConditionTrue(opStatus.Conditions, etcd.OperatorConditionExternalEtcdReadyForTransition)
 	if readyForEtcdTransition {
 		klog.V(4).Infof("ready for etcd transition")
@@ -60,6 +66,11 @@ func IsEtcdRunningInCluster(ctx context.Context, operatorClient v1helpers.Static
 		return false, err
 	}
 
+	if opStatus == nil {
+		klog.V(2).Info("static pod operator status not yet populated; bootstrap completion unknown")
+		return false, nil
+	}
+
 	etcdRunningInCluster := v1helpers.IsOperatorConditionTrue(opStatus.Conditions, etcd.OperatorConditionEtcdRunningInCluster)
 	if etcdRunningInCluster {
 		klog.V(4).Infof("bootstrap completed, etcd running in cluster")
@@ -68,16 +79,39 @@ func IsEtcdRunningInCluster(ctx context.Context, operatorClient v1helpers.Static
 	return etcdRunningInCluster, nil
 }
 
+// HasExternalEtcdCompletedTransition checks if the transition to external etcd process is completed
+// by examining the operator status for the HasExternalEtcdCompletedTransition condition.
+func HasExternalEtcdCompletedTransition(ctx context.Context, operatorClient v1helpers.StaticPodOperatorClient) (bool, error) {
+	_, opStatus, _, err := operatorClient.GetStaticPodOperatorState()
+	if err != nil {
+		klog.Errorf("failed to get static pod operator state: %v", err)
+		return false, err
+	}
+
+	if opStatus == nil {
+		klog.V(2).Info("static pod operator status not yet populated; transition completion unknown")
+		return false, nil
+	}
+
+	hasExternalEtcdCompletedTransition := v1helpers.IsOperatorConditionTrue(opStatus.Conditions, etcd.OperatorConditionExternalEtcdHasCompletedTransition)
+	if hasExternalEtcdCompletedTransition {
+		klog.V(4).Infof("etcd has transitioned to running externally")
+	}
+
+	return hasExternalEtcdCompletedTransition, nil
+}
+
 // GetExternalEtcdClusterStatus provides a comprehensive status check for external etcd clusters.
 // It returns the external etcd status, bootstrap completion status, and readiness for transition.
 func GetExternalEtcdClusterStatus(ctx context.Context,
 	operatorClient v1helpers.StaticPodOperatorClient,
 	infraLister configv1listers.InfrastructureLister) (externalEtcdStatus ExternalEtcdClusterStatus, err error) {
 
 	externalEtcdStatus = ExternalEtcdClusterStatus{
-		IsExternalEtcdCluster:    false,
-		IsEtcdRunningInCluster:   false,
-		IsReadyForEtcdTransition: false,
+		IsExternalEtcdCluster:              false,
+		IsEtcdRunningInCluster:             false,
+		IsReadyForEtcdTransition:           false,
+		HasExternalEtcdCompletedTransition: false,
 	}
 
 	// Check if this is an external etcd cluster
@@ -98,18 +132,29 @@ func GetExternalEtcdClusterStatus(ctx context.Context,
 		return externalEtcdStatus, err
 	}
 
+	if opStatus == nil {
+		klog.V(2).Info("static pod operator status not yet populated; external etcd cluster status unknown")
+		return externalEtcdStatus, nil
+	}
+
 	// Check bootstrap completion
 	externalEtcdStatus.IsEtcdRunningInCluster = v1helpers.IsOperatorConditionTrue(opStatus.Conditions, etcd.OperatorConditionEtcdRunningInCluster)
 
 	// Check readiness for transition
 	externalEtcdStatus.IsReadyForEtcdTransition = v1helpers.IsOperatorConditionTrue(opStatus.Conditions, etcd.OperatorConditionExternalEtcdReadyForTransition)
 
+	// Check if etcd has completed transition to running externally
+	externalEtcdStatus.HasExternalEtcdCompletedTransition = v1helpers.IsOperatorConditionTrue(opStatus.Conditions, etcd.OperatorConditionExternalEtcdHasCompletedTransition)
+
 	if externalEtcdStatus.IsEtcdRunningInCluster {
 		klog.V(4).Infof("bootstrap completed, etcd running in cluster")
 	}
 	if externalEtcdStatus.IsReadyForEtcdTransition {
 		klog.V(4).Infof("ready for etcd transition")
 	}
+	if externalEtcdStatus.HasExternalEtcdCompletedTransition {
+		klog.V(4).Infof("etcd has transitioned to running externally")
+	}
 
 	return externalEtcdStatus, nil
 }