Merge pull request #9085 from zalando-incubator/karpenter-tests-2

Extend test case to ensure that node pools can scale-out after cluster update

Author: linki

cluster/manifests/e2e-resources/pool-reserve.yaml

@@ -22,6 +22,16 @@ spec:
     spec:
       nodeSelector:
         node.kubernetes.io/node-pool: "{{$pool}}"
+      affinity:
+        podAntiAffinity:
+          requiredDuringSchedulingIgnoredDuringExecution:
+          - labelSelector:
+              matchExpressions:
+              - key: pool
+                operator: In
+                values:
+                - "{{$pool}}"
+            topologyKey: "kubernetes.io/hostname"
       tolerations:
       {{ if eq $pool "worker-node-tests" }}
         - effect: NoSchedule

test/e2e/infra.go

@@ -11,6 +11,7 @@ import (
 	admissionapi "k8s.io/pod-security-admission/api"
 
 	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
+	applyconfigurationsautoscalingv1 "k8s.io/client-go/applyconfigurations/autoscaling/v1"
 	"k8s.io/client-go/kubernetes"
 	"k8s.io/kubernetes/test/e2e/framework"
 	"k8s.io/kubernetes/test/e2e/framework/deployment"
@@ -35,7 +36,34 @@ var _ = describe("Infrastructure tests", func() {
 
 	It("All node pools should be able to run pods [Zalando]", func() {
 		// When modifying this list, don't forget to modify cluster/manifests/e2e-resources/pool-reserve.yaml
-		for _, pool := range []string{"default-worker-splitaz", "worker-combined", "worker-limit-az", "worker-instance-storage"} {
+		nodePools := []string{
+			"default-worker-splitaz",
+			"worker-combined",
+			"worker-limit-az",
+			"worker-instance-storage",
+			"worker-node-tests",
+			"worker-karpenter",
+			"worker-arm64",
+		}
+
+		for _, pool := range nodePools {
+			deploy, err := cs.AppsV1().Deployments("default").Get(context.Background(), fmt.Sprintf("pool-reserve-%s", pool), metav1.GetOptions{})
+			framework.ExpectNoError(err)
+
+			err = deployment.WaitForDeploymentComplete(cs, deploy)
+			framework.ExpectNoError(err)
+
+			// Scale out deployment to one more replica. In combination with Pod-Anti-Affinity, this should require one more node.
+			_, err = cs.AppsV1().Deployments("default").ApplyScale(
+				context.Background(),
+				fmt.Sprintf("pool-reserve-%s", pool),
+				applyconfigurationsautoscalingv1.Scale().WithSpec(applyconfigurationsautoscalingv1.ScaleSpec().WithReplicas(2)),
+				metav1.ApplyOptions{FieldManager: "e2e.test", Force: true},
+			)
+			framework.ExpectNoError(err)
+		}
+
+		for _, pool := range nodePools {
 			deploy, err := cs.AppsV1().Deployments("default").Get(context.Background(), fmt.Sprintf("pool-reserve-%s", pool), metav1.GetOptions{})
 			framework.ExpectNoError(err)