Merge pull request #12613 from sbueringer/pr-fix-autoscaler-test-flake

🐛 Fix autoscaler e2e test flake

Author: k8s-ci-robot

test/e2e/autoscaler.go

@@ -203,7 +203,7 @@ func AutoscalerSpec(ctx context.Context, inputGetter func() AutoscalerSpecInput)
 		By("Checking the MachineDeployment is scaled up")
 		mdScaledUpReplicas := mdOriginalReplicas + 1
 		framework.AssertMachineDeploymentReplicas(ctx, framework.AssertMachineDeploymentReplicasInput{
-			Getter:                   input.BootstrapClusterProxy.GetClient(),
+			GetLister:                input.BootstrapClusterProxy.GetClient(),
 			MachineDeployment:        clusterResources.MachineDeployments[0],
 			Replicas:                 mdScaledUpReplicas,
 			WaitForMachineDeployment: input.E2EConfig.GetIntervals(specName, "wait-autoscaler"),
@@ -240,7 +240,7 @@ func AutoscalerSpec(ctx context.Context, inputGetter func() AutoscalerSpecInput)
 		// Since we scaled up the MachineDeployment manually and the workload has not changed auto scaler
 		// should detect that there are unneeded nodes and scale down the MachineDeployment.
 		framework.AssertMachineDeploymentReplicas(ctx, framework.AssertMachineDeploymentReplicasInput{
-			Getter:                   input.BootstrapClusterProxy.GetClient(),
+			GetLister:                input.BootstrapClusterProxy.GetClient(),
 			MachineDeployment:        clusterResources.MachineDeployments[0],
 			Replicas:                 mdScaledUpReplicas,
 			WaitForMachineDeployment: input.E2EConfig.GetIntervals(specName, "wait-controllers"),
@@ -257,21 +257,21 @@ func AutoscalerSpec(ctx context.Context, inputGetter func() AutoscalerSpecInput)
 				WaitForAnnotationsToBeAdded: input.E2EConfig.GetIntervals(specName, "wait-autoscaler"),
 			})
 
-			By("Scaling the MachineDeployment scale up deployment to zero")
+			By("Scaling the MachineDeployment scale up deployment to 0")
 			framework.ScaleScaleUpDeploymentAndWait(ctx, framework.ScaleScaleUpDeploymentAndWaitInput{
 				ClusterProxy: workloadClusterProxy,
 				Replicas:     0,
 			}, input.E2EConfig.GetIntervals(specName, "wait-autoscaler")...)
 
 			By("Checking the MachineDeployment finished scaling down to zero")
 			framework.AssertMachineDeploymentReplicas(ctx, framework.AssertMachineDeploymentReplicasInput{
-				Getter:                   input.BootstrapClusterProxy.GetClient(),
+				GetLister:                input.BootstrapClusterProxy.GetClient(),
 				MachineDeployment:        clusterResources.MachineDeployments[0],
 				Replicas:                 0,
 				WaitForMachineDeployment: input.E2EConfig.GetIntervals(specName, "wait-controllers"),
 			})
 
-			By("Scaling the MachineDeployment scale up deployment to 1")
+			Byf("Scaling the MachineDeployment scale up deployment to %d", mpOriginalReplicas+1)
 			framework.ScaleScaleUpDeploymentAndWait(ctx, framework.ScaleScaleUpDeploymentAndWaitInput{
 				ClusterProxy: workloadClusterProxy,
 				// We need to sum up the expected number of MachineDeployment replicas and the current
@@ -281,7 +281,7 @@ func AutoscalerSpec(ctx context.Context, inputGetter func() AutoscalerSpecInput)
 
 			By("Checking the MachineDeployment finished scaling up")
 			framework.AssertMachineDeploymentReplicas(ctx, framework.AssertMachineDeploymentReplicasInput{
-				Getter:                   input.BootstrapClusterProxy.GetClient(),
+				GetLister:                input.BootstrapClusterProxy.GetClient(),
 				MachineDeployment:        clusterResources.MachineDeployments[0],
 				Replicas:                 1,
 				WaitForMachineDeployment: input.E2EConfig.GetIntervals(specName, "wait-controllers"),

test/framework/autoscaler_helpers.go

@@ -221,7 +221,7 @@ func AddScaleUpDeploymentAndWait(ctx context.Context, input AddScaleUpDeployment
 		},
 	}
 
-	By("Create scale up deployment")
+	byf("Create scale up deployment (%d replicas)", replicas)
 	Expect(input.ClusterProxy.GetClient().Create(ctx, scaleUpDeployment)).To(Succeed(), "failed to create the scale up pod")
 
 	By("Wait for the scale up deployment to become ready (this implies machines to be created)")

test/framework/machinedeployment_helpers.go

@@ -777,15 +777,15 @@ func ScaleAndWaitMachineDeploymentTopologyV1Beta1(ctx context.Context, input Sca
 }
 
 type AssertMachineDeploymentReplicasInput struct {
-	Getter                   Getter
+	GetLister                GetLister
 	MachineDeployment        *clusterv1.MachineDeployment
 	Replicas                 int32
 	WaitForMachineDeployment []interface{}
 }
 
 func AssertMachineDeploymentReplicas(ctx context.Context, input AssertMachineDeploymentReplicasInput) {
 	Expect(ctx).NotTo(BeNil(), "ctx is required for AssertMachineDeploymentReplicas")
-	Expect(input.Getter).ToNot(BeNil(), "Invalid argument. input.Getter can't be nil when calling AssertMachineDeploymentReplicas")
+	Expect(input.GetLister).ToNot(BeNil(), "Invalid argument. input.GetLister can't be nil when calling AssertMachineDeploymentReplicas")
 	Expect(input.MachineDeployment).ToNot(BeNil(), "Invalid argument. input.MachineDeployment can't be nil when calling AssertMachineDeploymentReplicas")
 
 	Eventually(func(g Gomega) {
@@ -795,9 +795,23 @@ func AssertMachineDeploymentReplicas(ctx context.Context, input AssertMachineDep
 			Namespace: input.MachineDeployment.Namespace,
 			Name:      input.MachineDeployment.Name,
 		}
-		g.Expect(input.Getter.Get(ctx, key, md)).To(Succeed(), fmt.Sprintf("failed to get MachineDeployment %s", klog.KObj(input.MachineDeployment)))
+		g.Expect(input.GetLister.Get(ctx, key, md)).To(Succeed(), fmt.Sprintf("failed to get MachineDeployment %s", klog.KObj(input.MachineDeployment)))
 		g.Expect(md.Spec.Replicas).Should(Not(BeNil()), fmt.Sprintf("MachineDeployment %s replicas should not be nil", klog.KObj(md)))
 		g.Expect(*md.Spec.Replicas).Should(Equal(input.Replicas), fmt.Sprintf("MachineDeployment %s replicas should match expected replicas", klog.KObj(md)))
 		g.Expect(ptr.Deref(md.Status.Replicas, 0)).Should(Equal(input.Replicas), fmt.Sprintf("MachineDeployment %s status.replicas should match expected replicas", klog.KObj(md)))
+
+		// Check all replicas have a NodeRef (i.e. they have a corresponding Node).
+		selectorMap, err := metav1.LabelSelectorAsMap(&input.MachineDeployment.Spec.Selector)
+		g.Expect(err).ToNot(HaveOccurred())
+		machines := &clusterv1.MachineList{}
+		err = input.GetLister.List(ctx, machines, client.InNamespace(input.MachineDeployment.Namespace), client.MatchingLabels(selectorMap))
+		g.Expect(err).ToNot(HaveOccurred())
+		count := 0
+		for _, machine := range machines.Items {
+			if machine.Status.NodeRef.IsDefined() {
+				count++
+			}
+		}
+		g.Expect(count).To(Equal(int(input.Replicas)))
 	}, input.WaitForMachineDeployment...).Should(Succeed())
 }