Use statefulset.IsReady() to validate status

Changes to use the common IsReady() func to validate a deployment
is fully up as requested and e.g. no update rollout in progress.
During a minor update this has seen to already report/mark the
condition ready, even the deployment is still in progress, or
the replacement pod failed.

Jira: OSPRH-14472

Depends-On: openstack-k8s-operators/lib-common#616

Signed-off-by: Martin Schuppert <[email protected]>

Author: stuggi

controllers/ovndbcluster_controller.go

@@ -603,7 +603,10 @@ func (r *OVNDBClusterReconciler) reconcileNormal(ctx context.Context, instance *
 		return ctrlResult, nil
 	}
 
-	instance.Status.ReadyCount = sfset.GetStatefulSet().Status.ReadyReplicas
+	deploy := sfset.GetStatefulSet()
+	if deploy.Generation == deploy.Status.ObservedGeneration {
+		instance.Status.ReadyCount = deploy.Status.ReadyReplicas
+	}
 
 	// verify if network attachment matches expectations
 	networkReady, networkAttachmentStatus, err := nad.VerifyNetworkStatusFromAnnotation(ctx, helper, networkAttachments, serviceLabels, instance.Status.ReadyCount)
@@ -662,7 +665,7 @@ func (r *OVNDBClusterReconciler) reconcileNormal(ctx context.Context, instance *
 		return ctrl.Result{}, err
 	}
 
-	if instance.Status.ReadyCount > 0 && len(svcList.Items) > 0 {
+	if statefulset.IsReady(deploy) && len(svcList.Items) > 0 {
 		instance.Status.Conditions.MarkTrue(condition.DeploymentReadyCondition, condition.DeploymentReadyMessage)
 		instance.Status.Conditions.MarkTrue(condition.ExposeServiceReadyCondition, condition.ExposeServiceReadyMessage)
 		internalDbAddress := []string{}
@@ -700,6 +703,15 @@ func (r *OVNDBClusterReconciler) reconcileNormal(ctx context.Context, instance *
 			}
 		}
 
+	} else {
+		Log.Info("Deployment is not ready")
+		instance.Status.Conditions.Set(condition.FalseCondition(
+			condition.DeploymentReadyCondition,
+			condition.RequestedReason,
+			condition.SeverityInfo,
+			condition.DeploymentReadyRunningMessage))
+		// It is OK to return success as we are watching for StatefulSet changes
+		return ctrl.Result{}, nil
 	}
 	Log.Info("Reconciled Service successfully")
 	return ctrl.Result{}, nil

tests/functional/ovndbcluster_controller_test.go

@@ -305,7 +305,7 @@ var _ = Describe("OVNDBCluster controller", func() {
 	})
 
 	When("OVNDBClusters are created with networkAttachments", func() {
-		It("does not break if pods are not created yet", func() {
+		It("does not break if pods get scaled", func() {
 			// Create OVNDBCluster with 1 replica
 			spec := GetDefaultOVNDBClusterSpec()
 			spec.NetworkAttachment = "internalapi"
@@ -321,12 +321,13 @@ var _ = Describe("OVNDBCluster controller", func() {
 				g.Expect(k8sClient.Update(ctx, c)).Should(Succeed())
 			}).Should(Succeed())
 
-			//Check that error occurs
-			Eventually(func(g Gomega) {
-				conditions := GetOVNDBCluster(dbs[0]).Status.Conditions
-				cond := conditions.Get(condition.ExposeServiceReadyCondition)
-				g.Expect(cond.Status).To(Equal(corev1.ConditionFalse))
-			}).Should(Succeed())
+			// check that cluster reaches ready
+			th.ExpectCondition(
+				dbs[0],
+				ConditionGetterFunc(OVNDBClusterConditionGetter),
+				condition.ReadyCondition,
+				corev1.ConditionTrue,
+			)
 
 			// Decrease replicas back to 1
 			Eventually(func(g Gomega) {
@@ -335,12 +336,13 @@ var _ = Describe("OVNDBCluster controller", func() {
 				g.Expect(k8sClient.Update(ctx, c)).Should(Succeed())
 			}).Should(Succeed())
 
-			//Check that error doesn't happen and instance is ready
-			Eventually(func(g Gomega) {
-				conditions := GetOVNDBCluster(dbs[0]).Status.Conditions
-				cond := conditions.Get(condition.DeploymentReadyCondition)
-				g.Expect(cond.Status).To(Equal(corev1.ConditionTrue))
-			}).Should(Succeed())
+			// check that cluster reaches ready
+			th.ExpectCondition(
+				dbs[0],
+				ConditionGetterFunc(OVNDBClusterConditionGetter),
+				condition.ReadyCondition,
+				corev1.ConditionTrue,
+			)
 
 		})