Use StatefulSet status instead of List Pod by selector labels (#228)

* use statefulset status instead of list pod by labels

* fix missing "ydb.tech/remote-cluster" label

* bump helm chart version

Author: kobzonega

deploy/ydb-operator/Chart.yaml

@@ -15,10 +15,10 @@ type: application
 # This is the chart version. This version number should be incremented each time you make changes
 # to the chart and its templates, including the app version.
 # Versions are expected to follow Semantic Versioning (https://semver.org/)
-version: 0.5.17
+version: 0.5.18
 
 # This is the version number of the application being deployed. This version number should be
 # incremented each time you make changes to the application. Versions are not expected to
 # follow Semantic Versioning. They should reflect the version the application is using.
 # It is recommended to use it with quotes.
-appVersion: "0.5.17"
+appVersion: "0.5.18"

internal/controllers/database/sync.go

@@ -20,7 +20,6 @@ import (
 
 	"github.com/ydb-platform/ydb-kubernetes-operator/api/v1alpha1"
 	. "github.com/ydb-platform/ydb-kubernetes-operator/internal/controllers/constants" //nolint:revive,stylecheck
-	"github.com/ydb-platform/ydb-kubernetes-operator/internal/labels"
 	"github.com/ydb-platform/ydb-kubernetes-operator/internal/resources"
 )
 
@@ -304,11 +303,11 @@ func (r *Reconciler) waitForStatefulSetToScale(
 		return Continue, ctrl.Result{Requeue: false}, nil
 	}
 
-	found := &appsv1.StatefulSet{}
+	foundStatefulSet := &appsv1.StatefulSet{}
 	err := r.Get(ctx, types.NamespacedName{
 		Name:      database.Name,
 		Namespace: database.Namespace,
-	}, found)
+	}, foundStatefulSet)
 	if err != nil {
 		if apierrors.IsNotFound(err) {
 			r.Recorder.Event(
@@ -328,42 +327,18 @@ func (r *Reconciler) waitForStatefulSetToScale(
 		return Stop, ctrl.Result{RequeueAfter: DefaultRequeueDelay}, err
 	}
 
-	podList := &corev1.PodList{}
-	opts := []client.ListOption{
-		client.InNamespace(database.Namespace),
-		client.MatchingLabels{labels.StatefulsetComponent: database.Name},
-	}
-
-	err = r.List(ctx, podList, opts...)
-	if err != nil {
-		r.Recorder.Event(
-			database,
-			corev1.EventTypeWarning,
-			"ControllerError",
-			fmt.Sprintf("Failed to list StatefulSet pods: %s", err),
-		)
-		return Stop, ctrl.Result{RequeueAfter: DefaultRequeueDelay}, err
-	}
-
-	runningPods := 0
-	for _, e := range podList.Items {
-		if resources.PodIsReady(e) {
-			runningPods++
-		}
-	}
-
-	if runningPods != int(database.Spec.Nodes) {
+	if foundStatefulSet.Status.ReadyReplicas != database.Spec.Nodes {
 		r.Recorder.Event(
 			database,
 			corev1.EventTypeNormal,
 			string(DatabaseProvisioning),
-			fmt.Sprintf("Waiting for number of running pods to match expected: %d != %d", runningPods, database.Spec.Nodes),
+			fmt.Sprintf("Waiting for number of running pods to match expected: %d != %d", foundStatefulSet.Status.ReadyReplicas, database.Spec.Nodes),
 		)
 		meta.SetStatusCondition(&database.Status.Conditions, metav1.Condition{
 			Type:    DatabaseProvisionedCondition,
 			Status:  metav1.ConditionFalse,
 			Reason:  ReasonInProgress,
-			Message: fmt.Sprintf("Number of running pods does not match expected: %d != %d", runningPods, database.Spec.Nodes),
+			Message: fmt.Sprintf("Number of running pods does not match expected: %d != %d", foundStatefulSet.Status.ReadyReplicas, database.Spec.Nodes),
 		})
 		return r.updateStatus(ctx, database, DefaultRequeueDelay)
 	}

internal/controllers/databasenodeset/sync.go

@@ -14,12 +14,10 @@ import (
 	"k8s.io/apimachinery/pkg/runtime"
 	"k8s.io/apimachinery/pkg/types"
 	ctrl "sigs.k8s.io/controller-runtime"
-	"sigs.k8s.io/controller-runtime/pkg/client"
 	"sigs.k8s.io/controller-runtime/pkg/controller/controllerutil"
 
 	"github.com/ydb-platform/ydb-kubernetes-operator/api/v1alpha1"
 	. "github.com/ydb-platform/ydb-kubernetes-operator/internal/controllers/constants" //nolint:revive,stylecheck
-	"github.com/ydb-platform/ydb-kubernetes-operator/internal/labels"
 	"github.com/ydb-platform/ydb-kubernetes-operator/internal/resources"
 )
 
@@ -204,11 +202,11 @@ func (r *Reconciler) waitForStatefulSetToScale(
 		return r.updateStatus(ctx, databaseNodeSet, StatusUpdateRequeueDelay)
 	}
 
-	found := &appsv1.StatefulSet{}
+	foundStatefulSet := &appsv1.StatefulSet{}
 	err := r.Get(ctx, types.NamespacedName{
 		Name:      databaseNodeSet.Name,
 		Namespace: databaseNodeSet.Namespace,
-	}, found)
+	}, foundStatefulSet)
 	if err != nil {
 		if apierrors.IsNotFound(err) {
 			r.Recorder.Event(
@@ -223,51 +221,23 @@ func (r *Reconciler) waitForStatefulSetToScale(
 			databaseNodeSet,
 			corev1.EventTypeWarning,
 			"ControllerError",
-			fmt.Sprintf("Failed to get StatefulSets: %s", err),
+			fmt.Sprintf("Failed to get StatefulSet: %s", err),
 		)
 		return Stop, ctrl.Result{RequeueAfter: DefaultRequeueDelay}, err
 	}
 
-	matchingLabels := client.MatchingLabels{}
-	for k, v := range databaseNodeSet.Labels {
-		matchingLabels[k] = v
-	}
-
-	podList := &corev1.PodList{}
-	opts := []client.ListOption{
-		client.InNamespace(databaseNodeSet.Namespace),
-		client.MatchingLabels{labels.StatefulsetComponent: databaseNodeSet.Name},
-	}
-	err = r.List(ctx, podList, opts...)
-	if err != nil {
-		r.Recorder.Event(
-			databaseNodeSet,
-			corev1.EventTypeWarning,
-			"ControllerError",
-			fmt.Sprintf("Failed to list StatefulSet pods: %s", err),
-		)
-		return Stop, ctrl.Result{RequeueAfter: DefaultRequeueDelay}, err
-	}
-
-	runningPods := 0
-	for _, e := range podList.Items {
-		if resources.PodIsReady(e) {
-			runningPods++
-		}
-	}
-
-	if runningPods != int(databaseNodeSet.Spec.Nodes) {
+	if foundStatefulSet.Status.ReadyReplicas != databaseNodeSet.Spec.Nodes {
 		r.Recorder.Event(
 			databaseNodeSet,
 			corev1.EventTypeNormal,
 			string(DatabaseNodeSetProvisioning),
-			fmt.Sprintf("Waiting for number of running nodes to match expected: %d != %d", runningPods, databaseNodeSet.Spec.Nodes),
+			fmt.Sprintf("Waiting for number of running nodes to match expected: %d != %d", foundStatefulSet.Status.ReadyReplicas, databaseNodeSet.Spec.Nodes),
 		)
 		meta.SetStatusCondition(&databaseNodeSet.Status.Conditions, metav1.Condition{
 			Type:    NodeSetProvisionedCondition,
 			Status:  metav1.ConditionFalse,
 			Reason:  ReasonInProgress,
-			Message: fmt.Sprintf("Number of running nodes does not match expected: %d != %d", runningPods, databaseNodeSet.Spec.Nodes),
+			Message: fmt.Sprintf("Number of running nodes does not match expected: %d != %d", foundStatefulSet.Status.ReadyReplicas, databaseNodeSet.Spec.Nodes),
 		})
 		return r.updateStatus(ctx, databaseNodeSet, DefaultRequeueDelay)
 	}

internal/controllers/storage/sync.go

@@ -21,7 +21,6 @@ import (
 	"github.com/ydb-platform/ydb-kubernetes-operator/api/v1alpha1"
 	. "github.com/ydb-platform/ydb-kubernetes-operator/internal/controllers/constants" //nolint:revive,stylecheck
 	"github.com/ydb-platform/ydb-kubernetes-operator/internal/healthcheck"
-	"github.com/ydb-platform/ydb-kubernetes-operator/internal/labels"
 	"github.com/ydb-platform/ydb-kubernetes-operator/internal/resources"
 )
 
@@ -124,11 +123,11 @@ func (r *Reconciler) waitForStatefulSetToScale(
 		return r.updateStatus(ctx, storage, StatusUpdateRequeueDelay)
 	}
 
-	found := &appsv1.StatefulSet{}
+	foundStatefulSet := &appsv1.StatefulSet{}
 	err := r.Get(ctx, types.NamespacedName{
 		Name:      storage.Name,
 		Namespace: storage.Namespace,
-	}, found)
+	}, foundStatefulSet)
 	if err != nil {
 		if apierrors.IsNotFound(err) {
 			r.Recorder.Event(
@@ -148,42 +147,18 @@ func (r *Reconciler) waitForStatefulSetToScale(
 		return Stop, ctrl.Result{RequeueAfter: DefaultRequeueDelay}, err
 	}
 
-	podList := &corev1.PodList{}
-	opts := []client.ListOption{
-		client.InNamespace(storage.Namespace),
-		client.MatchingLabels{labels.StatefulsetComponent: storage.Name},
-	}
-
-	err = r.List(ctx, podList, opts...)
-	if err != nil {
-		r.Recorder.Event(
-			storage,
-			corev1.EventTypeWarning,
-			"ControllerError",
-			fmt.Sprintf("Failed to list StatefulSet pods: %s", err),
-		)
-		return Stop, ctrl.Result{RequeueAfter: DefaultRequeueDelay}, err
-	}
-
-	runningPods := 0
-	for _, e := range podList.Items {
-		if resources.PodIsReady(e) {
-			runningPods++
-		}
-	}
-
-	if runningPods != int(storage.Spec.Nodes) {
+	if foundStatefulSet.Status.ReadyReplicas != storage.Spec.Nodes {
 		r.Recorder.Event(
 			storage,
 			corev1.EventTypeNormal,
 			string(StorageProvisioning),
-			fmt.Sprintf("Waiting for number of running nodes to match expected: %d != %d", runningPods, storage.Spec.Nodes),
+			fmt.Sprintf("Waiting for number of running nodes to match expected: %d != %d", foundStatefulSet.Status.ReadyReplicas, storage.Spec.Nodes),
 		)
 		meta.SetStatusCondition(&storage.Status.Conditions, metav1.Condition{
 			Type:    StorageProvisionedCondition,
 			Status:  metav1.ConditionFalse,
 			Reason:  ReasonInProgress,
-			Message: fmt.Sprintf("Number of running nodes does not match expected: %d != %d", runningPods, storage.Spec.Nodes),
+			Message: fmt.Sprintf("Number of running nodes does not match expected: %d != %d", foundStatefulSet.Status.ReadyReplicas, storage.Spec.Nodes),
 		})
 		return r.updateStatus(ctx, storage, DefaultRequeueDelay)
 	}

internal/controllers/storagenodeset/sync.go

@@ -14,12 +14,10 @@ import (
 	"k8s.io/apimachinery/pkg/runtime"
 	"k8s.io/apimachinery/pkg/types"
 	ctrl "sigs.k8s.io/controller-runtime"
-	"sigs.k8s.io/controller-runtime/pkg/client"
 	"sigs.k8s.io/controller-runtime/pkg/controller/controllerutil"
 
 	"github.com/ydb-platform/ydb-kubernetes-operator/api/v1alpha1"
 	. "github.com/ydb-platform/ydb-kubernetes-operator/internal/controllers/constants" //nolint:revive,stylecheck
-	"github.com/ydb-platform/ydb-kubernetes-operator/internal/labels"
 	"github.com/ydb-platform/ydb-kubernetes-operator/internal/resources"
 )
 
@@ -228,42 +226,18 @@ func (r *Reconciler) waitForStatefulSetToScale(
 		return Stop, ctrl.Result{RequeueAfter: DefaultRequeueDelay}, err
 	}
 
-	podList := &corev1.PodList{}
-	opts := []client.ListOption{
-		client.InNamespace(storageNodeSet.Namespace),
-		client.MatchingLabels{labels.StatefulsetComponent: storageNodeSet.Name},
-	}
-
-	err = r.List(ctx, podList, opts...)
-	if err != nil {
-		r.Recorder.Event(
-			storageNodeSet,
-			corev1.EventTypeWarning,
-			"ControllerError",
-			fmt.Sprintf("Failed to list StatefulSet pods: %s", err),
-		)
-		return Stop, ctrl.Result{RequeueAfter: DefaultRequeueDelay}, err
-	}
-
-	runningPods := 0
-	for _, e := range podList.Items {
-		if resources.PodIsReady(e) {
-			runningPods++
-		}
-	}
-
-	if runningPods != int(storageNodeSet.Spec.Nodes) {
+	if foundStatefulSet.Status.ReadyReplicas != storageNodeSet.Spec.Nodes {
 		r.Recorder.Event(
 			storageNodeSet,
 			corev1.EventTypeNormal,
 			string(StorageNodeSetProvisioning),
-			fmt.Sprintf("Waiting for number of running nodes to match expected: %d != %d", runningPods, storageNodeSet.Spec.Nodes),
+			fmt.Sprintf("Waiting for number of running nodes to match expected: %d != %d", foundStatefulSet.Status.ReadyReplicas, storageNodeSet.Spec.Nodes),
 		)
 		meta.SetStatusCondition(&storageNodeSet.Status.Conditions, metav1.Condition{
 			Type:    NodeSetProvisionedCondition,
 			Status:  metav1.ConditionFalse,
 			Reason:  ReasonInProgress,
-			Message: fmt.Sprintf("Number of running nodes does not match expected: %d != %d", runningPods, storageNodeSet.Spec.Nodes),
+			Message: fmt.Sprintf("Number of running nodes does not match expected: %d != %d", foundStatefulSet.Status.ReadyReplicas, storageNodeSet.Spec.Nodes),
 		})
 		return r.updateStatus(ctx, storageNodeSet, DefaultRequeueDelay)
 	}

internal/resources/database.go

@@ -200,6 +200,9 @@ func (b *DatabaseBuilder) getNodeSetBuilders(databaseLabels labels.Labels) []Res
 		nodeSetLabels := databaseLabels.Copy()
 		nodeSetLabels.Merge(nodeSetSpecInline.Labels)
 		nodeSetLabels.Merge(map[string]string{labels.DatabaseNodeSetComponent: nodeSetSpecInline.Name})
+		if nodeSetSpecInline.Remote != nil {
+			nodeSetLabels.Merge(map[string]string{labels.RemoteClusterKey: nodeSetSpecInline.Remote.Cluster})
+		}
 
 		nodeSetAnnotations := CopyDict(b.Annotations)
 		if nodeSetSpecInline.Annotations != nil {
@@ -210,9 +213,6 @@ func (b *DatabaseBuilder) getNodeSetBuilders(databaseLabels labels.Labels) []Res
 
 		databaseNodeSetSpec := b.recastDatabaseNodeSetSpecInline(nodeSetSpecInline.DeepCopy())
 		if nodeSetSpecInline.Remote != nil {
-			nodeSetLabels = nodeSetLabels.Merge(map[string]string{
-				labels.RemoteClusterKey: nodeSetSpecInline.Remote.Cluster,
-			})
 			nodeSetBuilders = append(
 				nodeSetBuilders,
 				&RemoteDatabaseNodeSetBuilder{

internal/resources/databasenodeset.go

@@ -58,20 +58,26 @@ func (b *DatabaseNodeSetResource) GetResourceBuilders(restConfig *rest.Config) [
 	ydbCr := api.RecastDatabaseNodeSet(b.Unwrap())
 	databaseLabels := labels.DatabaseLabels(ydbCr)
 
+	statefulSetName := b.Name
 	statefulSetLabels := databaseLabels.Copy()
-	statefulSetLabels.Merge(map[string]string{labels.DatabaseNodeSetComponent: b.Labels[labels.DatabaseNodeSetComponent]})
-	statefulSetLabels.Merge(map[string]string{labels.StatefulsetComponent: b.Name})
+	statefulSetLabels.Merge(map[string]string{labels.StatefulsetComponent: statefulSetName})
+
+	databaseNodeSetName := b.Labels[labels.DatabaseNodeSetComponent]
+	statefulSetLabels.Merge(map[string]string{labels.DatabaseNodeSetComponent: databaseNodeSetName})
+	if remoteCluster, exist := b.Labels[labels.RemoteClusterKey]; exist {
+		statefulSetLabels.Merge(map[string]string{labels.RemoteClusterKey: remoteCluster})
+	}
 
 	statefulSetAnnotations := CopyDict(b.Spec.AdditionalAnnotations)
 	statefulSetAnnotations[annotations.ConfigurationChecksum] = GetConfigurationChecksum(b.Spec.Configuration)
 
 	var resourceBuilders []ResourceBuilder
 	resourceBuilders = append(resourceBuilders,
 		&DatabaseStatefulSetBuilder{
-			Database:   api.RecastDatabaseNodeSet(b.DatabaseNodeSet),
+			Database:   ydbCr,
 			RestConfig: restConfig,
 
-			Name:        b.Name,
+			Name:        statefulSetName,
 			Labels:      statefulSetLabels,
 			Annotations: statefulSetAnnotations,
 		},

internal/resources/storage.go

@@ -165,6 +165,9 @@ func (b *StorageClusterBuilder) getNodeSetBuilders(storageLabels labels.Labels)
 		nodeSetLabels := storageLabels.Copy()
 		nodeSetLabels.Merge(nodeSetSpecInline.Labels)
 		nodeSetLabels.Merge(map[string]string{labels.StorageNodeSetComponent: nodeSetSpecInline.Name})
+		if nodeSetSpecInline.Remote != nil {
+			nodeSetLabels.Merge(map[string]string{labels.RemoteClusterKey: nodeSetSpecInline.Remote.Cluster})
+		}
 
 		nodeSetAnnotations := CopyDict(b.Annotations)
 		if nodeSetSpecInline.Annotations != nil {
@@ -175,9 +178,6 @@ func (b *StorageClusterBuilder) getNodeSetBuilders(storageLabels labels.Labels)
 
 		storageNodeSetSpec := b.recastStorageNodeSetSpecInline(nodeSetSpecInline.DeepCopy())
 		if nodeSetSpecInline.Remote != nil {
-			nodeSetLabels = nodeSetLabels.Merge(map[string]string{
-				labels.RemoteClusterKey: nodeSetSpecInline.Remote.Cluster,
-			})
 			nodeSetBuilders = append(
 				nodeSetBuilders,
 				&RemoteStorageNodeSetBuilder{

internal/resources/storagenodeset.go

@@ -58,9 +58,15 @@ func (b *StorageNodeSetResource) GetResourceBuilders(restConfig *rest.Config) []
 	ydbCr := api.RecastStorageNodeSet(b.Unwrap())
 	storageLabels := labels.StorageLabels(ydbCr)
 
+	statefulSetName := b.Name
 	statefulSetLabels := storageLabels.Copy()
-	statefulSetLabels.Merge(map[string]string{labels.StorageNodeSetComponent: b.Labels[labels.DatabaseNodeSetComponent]})
-	statefulSetLabels.Merge(map[string]string{labels.StatefulsetComponent: b.Name})
+	statefulSetLabels.Merge(map[string]string{labels.StatefulsetComponent: statefulSetName})
+
+	storageNodeSetName := b.Labels[labels.StorageNodeSetComponent]
+	statefulSetLabels.Merge(map[string]string{labels.StorageNodeSetComponent: storageNodeSetName})
+	if remoteCluster, exist := b.Labels[labels.RemoteClusterKey]; exist {
+		statefulSetLabels.Merge(map[string]string{labels.RemoteClusterKey: remoteCluster})
+	}
 
 	statefulSetAnnotations := CopyDict(b.Spec.AdditionalAnnotations)
 	statefulSetAnnotations[annotations.ConfigurationChecksum] = GetConfigurationChecksum(b.Spec.Configuration)
@@ -69,10 +75,10 @@ func (b *StorageNodeSetResource) GetResourceBuilders(restConfig *rest.Config) []
 	resourceBuilders = append(
 		resourceBuilders,
 		&StorageStatefulSetBuilder{
-			Storage:    api.RecastStorageNodeSet(b.StorageNodeSet),
+			Storage:    ydbCr,
 			RestConfig: restConfig,
 
-			Name:        b.Name,
+			Name:        statefulSetName,
 			Labels:      statefulSetLabels,
 			Annotations: statefulSetAnnotations,
 		},