Merge pull request #7965 from zalando-incubator/kube-1.31

Update to Kubernetes 1.31

Author: mikkeloscar

cluster/config-defaults.yaml

@@ -609,7 +609,7 @@ teapot_admission_controller_namespace_delete_protection_enabled: "false"
 teapot_admission_controller_resolve_vanity_images: "true"
 
 {{if eq .Cluster.Environment "e2e"}}
-teapot_admission_controller_ignore_namespaces: "^kube-system|((downward-api|kubectl|projected|statefulset|pod-network|scope-selectors|resourcequota|limitrange|sysctl|node-tests|e2e-kubelet-etc-hosts|csiinlinevolumes|dns)-.*)$"
+teapot_admission_controller_ignore_namespaces: "^kube-system|((downward-api|kubectl|projected|statefulset|pod-network|scope-selectors|resourcequota|limitrange|sysctl|node-tests|e2e-kubelet-etc-hosts|csiinlinevolumes|job|dns)-.*)$"
 teapot_admission_controller_crd_ensure_no_resources_on_delete: "false"
 {{else}}
 teapot_admission_controller_ignore_namespaces: "^kube-system$"
@@ -727,6 +727,9 @@ tracing_coredns_local_zone_traces_endpoint: ""
 kuberuntu_image_v1_30_jammy_amd64: {{ amiID "zalando-ubuntu-jammy-22.04-kubernetes-production-v1.30.2-amd64-master-341" "861068367966" }}
 kuberuntu_image_v1_30_jammy_arm64: {{ amiID "zalando-ubuntu-jammy-22.04-kubernetes-production-v1.30.2-arm64-master-341" "861068367966" }}
 
+kuberuntu_image_v1_31_jammy_amd64: {{ amiID "zalando-ubuntu-jammy-22.04-kubernetes-production-v1.31.0-amd64-master-347" "861068367966" }}
+kuberuntu_image_v1_31_jammy_arm64: {{ amiID "zalando-ubuntu-jammy-22.04-kubernetes-production-v1.31.0-arm64-master-347" "861068367966" }}
+
 # Which distro from the previous config items should be used. Valid options are only `jammy` for now. Can be set for each node pool.
 kuberuntu_distro_master: "jammy"
 kuberuntu_distro_worker: "jammy"
@@ -904,8 +907,6 @@ enable_topology_aware_hints: "false"
 
 # Enable FeatureGate HPAScaleToZero
 enable_hpa_scale_to_zero: "true"
-# Enable FeatureGate HPAContainerMetrics
-enable_hpa_container_metrics: "true"
 
 # Enable FeatureGate MaxUnavailableStatefulSet
 max_unavailable_statefulset_enabled: "false"
@@ -1129,6 +1130,3 @@ control_plane_graceful_shutdown: "true"
 #  fs.aio-max-nr = 8388608
 #  fs.inotify.max_user_watches = 100000
 sysctl_settings: ""
-
-# enables/disables the minDomains field for pod topology spread.
-min_domains_in_pod_topology_spread_enabled: "true"

cluster/manifests/aws-cloud-controller-manager/daemonset.yaml

@@ -27,7 +27,7 @@ spec:
         - --cloud-provider=aws
         - --use-service-account-credentials=true
         - --configure-cloud-routes=false
-        image: container-registry.zalando.net/teapot/aws-cloud-controller-manager-internal:v1.30.2-master-126
+        image: container-registry.zalando.net/teapot/aws-cloud-controller-manager-internal:v1.31.0-master-127
         name: aws-cloud-controller-manager
         resources:
           requests:

cluster/node-pools/master-default/stack.yaml

@@ -10,7 +10,7 @@ Mappings:
   Images:
     {{.Cluster.Region}}:
       # Use the node pool's architecture to construct the config item name that we're using to get the AMI name.
-      MachineImage: '{{ index .NodePool.ConfigItems (print "kuberuntu_image_v1_30_" .NodePool.ConfigItems.kuberuntu_distro_master "_" .Values.InstanceInfo.Architecture) }}'
+      MachineImage: '{{ index .NodePool.ConfigItems (print "kuberuntu_image_v1_31_" .NodePool.ConfigItems.kuberuntu_distro_master "_" .Values.InstanceInfo.Architecture) }}'
 
 Resources:
   AutoScalingGroup:

cluster/node-pools/master-default/userdata.yaml

@@ -132,7 +132,6 @@ write_files:
           - --runtime-config=authorization.k8s.io/v1beta1=true,scheduling.k8s.io/v1alpha1=true,admissionregistration.k8s.io/v1beta1=true
           - --authentication-token-webhook-config-file=/etc/kubernetes/config/authn.yaml
           - --authentication-token-webhook-cache-ttl=10s
-          - --cloud-provider=external
           - --authorization-mode=Node,Webhook,RBAC
           - --authorization-webhook-config-file=/etc/kubernetes/config/authz.yaml
           - --authorization-webhook-version=v1
@@ -143,7 +142,7 @@ write_files:
           - "--oidc-username-prefix=okta:"
           - --oidc-groups-claim=groups
           - "--oidc-groups-prefix=okta:"
-          - --feature-gates=HPAScaleToZero={{ .Cluster.ConfigItems.enable_hpa_scale_to_zero }},HPAContainerMetrics={{ .Cluster.ConfigItems.enable_hpa_container_metrics }},StatefulSetAutoDeletePVC={{ .Cluster.ConfigItems.enable_statefulset_autodelete_pvc }},TopologyAwareHints={{ .Cluster.ConfigItems.enable_topology_aware_hints }},MinDomainsInPodTopologySpread={{ .Cluster.ConfigItems.min_domains_in_pod_topology_spread_enabled }},MaxUnavailableStatefulSet={{.Cluster.ConfigItems.max_unavailable_statefulset_enabled}},KMSv1=true
+          - --feature-gates=HPAScaleToZero={{ .Cluster.ConfigItems.enable_hpa_scale_to_zero }},StatefulSetAutoDeletePVC={{ .Cluster.ConfigItems.enable_statefulset_autodelete_pvc }},TopologyAwareHints={{ .Cluster.ConfigItems.enable_topology_aware_hints }},MaxUnavailableStatefulSet={{.Cluster.ConfigItems.max_unavailable_statefulset_enabled}},KMSv1=true
           - --service-account-key-file=/etc/kubernetes/ssl/service-account-public-key.pem
           - --service-account-signing-key-file=/etc/kubernetes/ssl/service-account-private-key.pem
           - --service-account-issuer={{ .Cluster.APIServerURL }}
@@ -154,6 +153,8 @@ write_files:
           - --audit-log-path=/var/log/kube-audit.log
           - --audit-log-maxage=0
           - --audit-log-maxbackup=0
+          # we need this since the /logs/kube-audit.log endpoint is off by default since v1.31
+          - --enable-logs-handler
           {{ else }}
           - --audit-webhook-config-file=/etc/kubernetes/config/audit.yaml
           - --audit-webhook-mode=batch
@@ -670,7 +671,6 @@ write_files:
           args:
           - --config=/etc/kubernetes/config/scheduler-config.yaml
           - --leader-elect=true
-          - --feature-gates=MinDomainsInPodTopologySpread={{ .Cluster.ConfigItems.min_domains_in_pod_topology_spread_enabled }}
           - --profiling={{ .Cluster.ConfigItems.enable_control_plane_profiling }}
           resources:
             requests:

cluster/node-pools/worker-combined/stack.yaml

@@ -10,7 +10,7 @@ Mappings:
   Images:
     {{.Cluster.Region}}:
       # Use the node pool's architecture to construct the config item name that we're using to get the AMI name.
-      MachineImage: '{{ index .NodePool.ConfigItems (print "kuberuntu_image_v1_30_" .NodePool.ConfigItems.kuberuntu_distro_worker "_" .Values.InstanceInfo.Architecture) }}'
+      MachineImage: '{{ index .NodePool.ConfigItems (print "kuberuntu_image_v1_31_" .NodePool.ConfigItems.kuberuntu_distro_worker "_" .Values.InstanceInfo.Architecture) }}'
 
 Resources:
   AutoScalingGroup:

cluster/node-pools/worker-karpenter/provisioners.yaml

@@ -8,8 +8,8 @@ spec:
   amiFamily: Custom
   amiSelectorTerms:
     # Select on any AMI that has any of the following IDs
-    - id: {{ index .NodePool.ConfigItems (print "kuberuntu_image_v1_30_" .NodePool.ConfigItems.kuberuntu_distro_worker "_amd64") }}
-    - id: {{ index .NodePool.ConfigItems (print "kuberuntu_image_v1_30_" .NodePool.ConfigItems.kuberuntu_distro_worker "_arm64") }}
+    - id: {{ index .NodePool.ConfigItems (print "kuberuntu_image_v1_31_" .NodePool.ConfigItems.kuberuntu_distro_worker "_amd64") }}
+    - id: {{ index .NodePool.ConfigItems (print "kuberuntu_image_v1_31_" .NodePool.ConfigItems.kuberuntu_distro_worker "_arm64") }}
   metadataOptions:
     httpEndpoint: enabled
     httpProtocolIPv6: disabled

cluster/node-pools/worker-splitaz/stack.yaml

@@ -10,7 +10,7 @@ Mappings:
   Images:
     {{.Cluster.Region}}:
       # Use the node pool's architecture to construct the config item name that we're using to get the AMI name.
-      MachineImage: '{{ index .NodePool.ConfigItems (print "kuberuntu_image_v1_30_" .NodePool.ConfigItems.kuberuntu_distro_worker "_" .Values.InstanceInfo.Architecture) }}'
+      MachineImage: '{{ index .NodePool.ConfigItems (print "kuberuntu_image_v1_31_" .NodePool.ConfigItems.kuberuntu_distro_worker "_" .Values.InstanceInfo.Architecture) }}'
 
 Resources:
 {{ with $data := . }}

test/e2e/Makefile

@@ -2,7 +2,7 @@
 
 BINARY       ?= kubernetes-on-aws-e2e
 VERSION      ?= $(shell git describe --tags --always --dirty)
-KUBE_VERSION ?= v1.30.2
+KUBE_VERSION ?= v1.31.0
 IMAGE        ?= pierone.stups.zalan.do/teapot/$(BINARY)
 SOURCES      = $(shell find . -name '*.go')
 TAG          ?= $(VERSION)

test/e2e/apiserver.go

@@ -26,6 +26,7 @@ import (
 	. "github.com/onsi/gomega"
 
 	appsv1 "k8s.io/api/apps/v1"
+	batchv1 "k8s.io/api/batch/v1"
 	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
 	"k8s.io/apimachinery/pkg/util/uuid"
 	"k8s.io/client-go/kubernetes"
@@ -34,6 +35,7 @@ import (
 	e2epod "k8s.io/kubernetes/test/e2e/framework/pod"
 	"k8s.io/kubernetes/test/e2e/framework/statefulset"
 	admissionapi "k8s.io/pod-security-admission/api"
+	"k8s.io/utils/ptr"
 )
 
 const (
@@ -472,7 +474,7 @@ var _ = describe("Image Policy Tests (Job)", func() {
 			framework.ExpectNoError(err)
 		}()
 
-		job.WaitForJobComplete(context.TODO(), cs, namespace, jobObj.Name, 1)
+		job.WaitForJobComplete(context.TODO(), cs, namespace, jobObj.Name, ptr.To(batchv1.JobReasonCompletionsReached), 1)
 	})
 
 	It("Should not create Job using non-compliant image [Image-Policy] [Non-Compliant] [Zalando]", func() {
@@ -493,7 +495,7 @@ var _ = describe("Image Policy Tests (Job)", func() {
 			framework.ExpectNoError(err)
 		}()
 
-		job.WaitForJobComplete(context.TODO(), cs, namespace, jobObj.Name, 1)
+		job.WaitForJobComplete(context.TODO(), cs, namespace, jobObj.Name, ptr.To(batchv1.JobReasonCompletionsReached), 1)
 	})
 })
 
@@ -524,7 +526,7 @@ var _ = describe("Image Policy Tests (Job) (when disabled)", func() {
 			framework.ExpectNoError(err)
 		}()
 
-		job.WaitForJobComplete(context.TODO(), cs, namespace, jobObj.Name, 1)
+		job.WaitForJobComplete(context.TODO(), cs, namespace, jobObj.Name, ptr.To(batchv1.JobReasonCompletionsReached), 1)
 	})
 })
 
@@ -558,7 +560,7 @@ var _ = describe("ECR Registry Pull", func() {
 			framework.ExpectNoError(err)
 		}()
 
-		job.WaitForJobComplete(context.TODO(), cs, namespace, jobObj.Name, 1)
+		job.WaitForJobComplete(context.TODO(), cs, namespace, jobObj.Name, ptr.To(batchv1.JobReasonCompletionsReached), 1)
 	})
 
 	It("Should run a Job using a vanity image from the staging registry [ECR] [Zalando]", func() {
@@ -582,6 +584,6 @@ var _ = describe("ECR Registry Pull", func() {
 			framework.ExpectNoError(err)
 		}()
 
-		job.WaitForJobComplete(context.TODO(), cs, namespace, jobObj.Name, 1)
+		job.WaitForJobComplete(context.TODO(), cs, namespace, jobObj.Name, ptr.To(batchv1.JobReasonCompletionsReached), 1)
 	})
 })

test/e2e/audit.go

@@ -51,7 +51,7 @@ var _ = describe("Audit", func() {
 			Spec: apiv1.PodSpec{
 				Containers: []apiv1.Container{{
 					Name:  "pause",
-					Image: "container-registry.zalando.net/teapot/pause:3.4.1-master-18",
+					Image: "container-registry.zalando.net/teapot/pause:3.7-master-21",
 				}},
 			},
 		}