add e2e tests for UnhealthyPodEvictionPolicy

Author: atiratree

test/e2e/apps/disruption.go

@@ -38,16 +38,19 @@ import (
 	"k8s.io/apimachinery/pkg/types"
 	"k8s.io/apimachinery/pkg/util/intstr"
 	"k8s.io/apimachinery/pkg/util/json"
+	"k8s.io/apimachinery/pkg/util/sets"
 	"k8s.io/apimachinery/pkg/util/wait"
 	"k8s.io/client-go/dynamic"
 	"k8s.io/client-go/kubernetes"
 	clientscheme "k8s.io/client-go/kubernetes/scheme"
 	"k8s.io/client-go/util/retry"
 	podutil "k8s.io/kubernetes/pkg/api/v1/pod"
 	"k8s.io/kubernetes/test/e2e/framework"
+	e2epod "k8s.io/kubernetes/test/e2e/framework/pod"
 	e2eskipper "k8s.io/kubernetes/test/e2e/framework/skipper"
 	imageutils "k8s.io/kubernetes/test/utils/image"
 	admissionapi "k8s.io/pod-security-admission/api"
+	"k8s.io/utils/ptr"
 )
 
 // schedulingTimeout is longer specifically because sometimes we need to wait
@@ -301,7 +304,7 @@ var _ = SIGDescribe("DisruptionController", func() {
 			}
 
 			if c.maxUnavailable.String() != "" {
-				createPDBMaxUnavailableOrDie(ctx, cs, ns, defaultName, c.maxUnavailable)
+				createPDBMaxUnavailableOrDie(ctx, cs, ns, defaultName, c.maxUnavailable, nil)
 			}
 
 			// Locate a running pod.
@@ -375,7 +378,7 @@ var _ = SIGDescribe("DisruptionController", func() {
 
 		ginkgo.By("Trying to evict the same pod we tried earlier which should now be evictable")
 		waitForPodsOrDie(ctx, cs, ns, 3)
-		waitForPdbToObserveHealthyPods(ctx, cs, ns, 3)
+		waitForPdbToObserveHealthyPods(ctx, cs, ns, 3, 2)
 		err = cs.CoreV1().Pods(ns).EvictV1(ctx, e)
 		framework.ExpectNoError(err) // the eviction is now allowed
 
@@ -414,6 +417,112 @@ var _ = SIGDescribe("DisruptionController", func() {
 		framework.ExpectNoError(err) // the eviction is now allowed
 	})
 
+	unhealthyPodEvictionPolicyCases := []struct {
+		name                          string
+		unhealthyPodEvictionPolicy    *policyv1.UnhealthyPodEvictionPolicyType
+		podsShouldBecomeReadyFirst    bool
+		expectedSuccesfulPodEvictions int
+	}{
+		{
+			name:                          "should evict ready pods with Default UnhealthyPodEvictionPolicy",
+			unhealthyPodEvictionPolicy:    nil,
+			podsShouldBecomeReadyFirst:    true,
+			expectedSuccesfulPodEvictions: 1,
+		},
+		{
+			name:                          "should evict ready pods with IfHealthyBudget UnhealthyPodEvictionPolicy",
+			unhealthyPodEvictionPolicy:    ptr.To(policyv1.IfHealthyBudget),
+			podsShouldBecomeReadyFirst:    true,
+			expectedSuccesfulPodEvictions: 1,
+		},
+		{
+			name:                          "should evict ready pods with AlwaysAllow UnhealthyPodEvictionPolicy",
+			unhealthyPodEvictionPolicy:    ptr.To(policyv1.AlwaysAllow),
+			podsShouldBecomeReadyFirst:    true,
+			expectedSuccesfulPodEvictions: 1,
+		},
+		{
+			name:                          "should not evict unready pods with Default UnhealthyPodEvictionPolicy",
+			unhealthyPodEvictionPolicy:    nil,
+			podsShouldBecomeReadyFirst:    false,
+			expectedSuccesfulPodEvictions: 0,
+		},
+		{
+			name:                          "should not evict unready pods with IfHealthyBudget UnhealthyPodEvictionPolicy",
+			unhealthyPodEvictionPolicy:    ptr.To(policyv1.IfHealthyBudget),
+			podsShouldBecomeReadyFirst:    false,
+			expectedSuccesfulPodEvictions: 0,
+		},
+		{
+			name:                          "should evict unready pods with AlwaysAllow UnhealthyPodEvictionPolicy",
+			unhealthyPodEvictionPolicy:    ptr.To(policyv1.AlwaysAllow),
+			podsShouldBecomeReadyFirst:    false,
+			expectedSuccesfulPodEvictions: 3,
+		},
+	}
+	for i := range unhealthyPodEvictionPolicyCases {
+		tc := unhealthyPodEvictionPolicyCases[i]
+
+		framework.It(tc.name, func(ctx context.Context) {
+			ginkgo.By("Creating a pdb")
+			createPDBMaxUnavailableOrDie(ctx, cs, ns, defaultName, intstr.FromInt32(1), tc.unhealthyPodEvictionPolicy)
+
+			ginkgo.By("Creating a replica set")
+			rsName := "test-rs-with-delayed-ready"
+			replicas := int32(3)
+			rs := newRS(rsName, replicas, defaultLabels, WebserverImageName, WebserverImage, nil)
+			rs.Labels["name"] = rsName
+			initialDelaySeconds := framework.PodStartTimeout.Seconds() + 30
+			if tc.podsShouldBecomeReadyFirst {
+				initialDelaySeconds = 0
+			}
+			rs.Spec.Template.Spec.Containers[0].ReadinessProbe = &v1.Probe{
+				ProbeHandler: v1.ProbeHandler{
+					HTTPGet: &v1.HTTPGetAction{
+						Path: "/index.html",
+						Port: intstr.IntOrString{IntVal: 80},
+					},
+				},
+				InitialDelaySeconds: int32(initialDelaySeconds),
+			}
+			_, err := cs.AppsV1().ReplicaSets(ns).Create(ctx, rs, metav1.CreateOptions{})
+			framework.ExpectNoError(err, "Creating replica set %q in namespace %q", rs.Name, ns)
+
+			if tc.podsShouldBecomeReadyFirst {
+				ginkgo.By("Wait for pods to be running and ready")
+				waitForPodsOrDie(ctx, cs, ns, int(replicas))
+				waitForPdbToObserveHealthyPods(ctx, cs, ns, replicas, replicas-1)
+			} else {
+				ginkgo.By("Wait for pods to be running and not ready")
+				err := e2epod.VerifyPodsRunning(ctx, cs, ns, rsName, false, replicas)
+				framework.ExpectNoError(err)
+				waitForPdbToObserveHealthyPods(ctx, cs, ns, 0, replicas-1)
+			}
+
+			ginkgo.By("Try to evict all pods guarded by a PDB")
+			podList, err := cs.CoreV1().Pods(ns).List(ctx, metav1.ListOptions{})
+			framework.ExpectNoError(err)
+
+			evictedPods := sets.New[string]()
+			for _, pod := range podList.Items {
+				e := &policyv1.Eviction{
+					ObjectMeta: metav1.ObjectMeta{
+						Name:      pod.Name,
+						Namespace: ns,
+					},
+				}
+				err = cs.CoreV1().Pods(ns).EvictV1(ctx, e)
+				if err == nil {
+					evictedPods.Insert(pod.Name)
+				}
+			}
+			gomega.Expect(evictedPods).Should(gomega.HaveLen(tc.expectedSuccesfulPodEvictions))
+			_, err = e2epod.WaitForPods(ctx, cs, ns, metav1.ListOptions{}, e2epod.Range{NoneMatching: true}, framework.PodDeleteTimeout, "evicted pods should be deleted", func(pod *v1.Pod) bool {
+				return evictedPods.Has(pod.Name) && pod.DeletionTimestamp == nil
+			})
+			framework.ExpectNoError(err)
+		})
+	}
 })
 
 func createPDBMinAvailableOrDie(ctx context.Context, cs kubernetes.Interface, ns string, name string, minAvailable intstr.IntOrString, labels map[string]string) {
@@ -433,15 +542,16 @@ func createPDBMinAvailableOrDie(ctx context.Context, cs kubernetes.Interface, ns
 	waitForPdbToBeProcessed(ctx, cs, ns, name)
 }
 
-func createPDBMaxUnavailableOrDie(ctx context.Context, cs kubernetes.Interface, ns string, name string, maxUnavailable intstr.IntOrString) {
+func createPDBMaxUnavailableOrDie(ctx context.Context, cs kubernetes.Interface, ns string, name string, maxUnavailable intstr.IntOrString, unhealthyPodEvictionPolicy *policyv1.UnhealthyPodEvictionPolicyType) {
 	pdb := policyv1.PodDisruptionBudget{
 		ObjectMeta: metav1.ObjectMeta{
 			Name:      name,
 			Namespace: ns,
 		},
 		Spec: policyv1.PodDisruptionBudgetSpec{
-			Selector:       &metav1.LabelSelector{MatchLabels: map[string]string{"foo": "bar"}},
-			MaxUnavailable: &maxUnavailable,
+			Selector:                   &metav1.LabelSelector{MatchLabels: map[string]string{"foo": "bar"}},
+			MaxUnavailable:             &maxUnavailable,
+			UnhealthyPodEvictionPolicy: unhealthyPodEvictionPolicy,
 		},
 	}
 	_, err := cs.PolicyV1().PodDisruptionBudgets(ns).Create(ctx, &pdb, metav1.CreateOptions{})
@@ -670,7 +780,7 @@ func waitForPdbToBeDeleted(ctx context.Context, cs kubernetes.Interface, ns stri
 	framework.ExpectNoError(err, "Waiting for the pdb to be deleted in namespace %s", ns)
 }
 
-func waitForPdbToObserveHealthyPods(ctx context.Context, cs kubernetes.Interface, ns string, healthyCount int32) {
+func waitForPdbToObserveHealthyPods(ctx context.Context, cs kubernetes.Interface, ns string, healthyCount, desiredHealthy int32) {
 	ginkgo.By("Waiting for the pdb to observed all healthy pods")
 	err := wait.PollUntilContextTimeout(ctx, framework.Poll, wait.ForeverTestTimeout, true, func(ctx context.Context) (bool, error) {
 		pdb, err := cs.PolicyV1().PodDisruptionBudgets(ns).Get(ctx, "foo", metav1.GetOptions{})
@@ -680,6 +790,9 @@ func waitForPdbToObserveHealthyPods(ctx context.Context, cs kubernetes.Interface
 		if pdb.Status.CurrentHealthy != healthyCount {
 			return false, nil
 		}
+		if pdb.Status.DesiredHealthy != desiredHealthy {
+			return false, nil
+		}
 		return true, nil
 	})
 	framework.ExpectNoError(err, "Waiting for the pdb in namespace %s to observed %d healthy pods", ns, healthyCount)