Merge pull request #6065 from zalando-incubator/kube-1.25

Update to Kubernetes v1.25

Author: gargravarr

cluster/config-defaults.yaml

@@ -455,9 +455,6 @@ horizontal_pod_downscale_stabilization: "5m0s"
 autoscaling_v2beta1_enabled: "false"
 autoscaling_v2beta2_enabled: "false"
 
-# enable/disable legacy api versions prior to v1.25
-batch_v1beta1_enabled: "false"
-
 # Vertical pod autoscaler version for controlling roll-out, can be "current" or "legacy"
 # current => v0.11.0-internal.17
 # legacy => v0.6.1-internal.16
@@ -628,9 +625,6 @@ teapot_admission_controller_pod_security_policy_privileged_service_accounts: ""
 {{ end }}
 teapot_admission_controller_pod_security_policy_privileged_allow_privilege_escalation: "false"
 
-# Optionally disable PodSecurityPolicy. Make sure `teapot_admission_controller_pod_security_policy_enabled` is true if this is disabled, otherwise there are no Pod security Policy enforcement in the cluster.
-pod_security_policy_enabled: "false"
-
 # Prevent the use of a particular AZ as much as possible
 blocked_availability_zone: ""
 
@@ -673,6 +667,10 @@ kuberuntu_image_v1_24_focal_amd64: {{ amiID "zalando-ubuntu-focal-20.04-kubernet
 kuberuntu_image_v1_24_focal_arm64: {{ amiID "zalando-ubuntu-focal-20.04-kubernetes-production-v1.24.17-arm64-master-283" "861068367966" }}
 kuberuntu_image_v1_24_jammy_amd64: {{ amiID "zalando-ubuntu-jammy-22.04-kubernetes-production-v1.24.17-amd64-master-297" "861068367966" }}
 kuberuntu_image_v1_24_jammy_arm64: {{ amiID "zalando-ubuntu-jammy-22.04-kubernetes-production-v1.24.17-arm64-master-297" "861068367966" }}
+kuberuntu_image_v1_25_focal_amd64: {{ amiID "zalando-ubuntu-focal-20.04-kubernetes-production-v1.25.16-amd64-master-305" "861068367966" }}
+kuberuntu_image_v1_25_focal_arm64: {{ amiID "zalando-ubuntu-focal-20.04-kubernetes-production-v1.25.16-arm64-master-305" "861068367966" }}
+kuberuntu_image_v1_25_jammy_amd64: {{ amiID "zalando-ubuntu-jammy-22.04-kubernetes-production-v1.25.16-amd64-master-305" "861068367966" }}
+kuberuntu_image_v1_25_jammy_arm64: {{ amiID "zalando-ubuntu-jammy-22.04-kubernetes-production-v1.25.16-arm64-master-305" "861068367966" }}
 
 # Which distro from the previous config items should be used. Valid options are `focal` and `jammy`. Can be set for each node pool.
 {{if eq .Cluster.Environment "test"}}
@@ -837,10 +835,6 @@ enable_hpa_scale_to_zero: "true"
 # Enable FeatureGate HPAContainerMetrics
 enable_hpa_container_metrics: "true"
 
-# Enable FeatureGate EphemeralContainers (Alpha)
-# https://kubernetes.io/docs/tasks/debug-application-cluster/debug-running-pod/
-enable_ephemeral_containers: "false"
-
 # Enable FeatureGate MaxUnavailableStatefulSet
 max_unavailable_statefulset_enabled: "false"
 

cluster/manifests/deletions.yaml

@@ -293,12 +293,6 @@ post_apply:
   kind: HorizontalPodAutoscaler
   namespace: kube-system
 {{ end }}
-{{- if ne .Cluster.ConfigItems.pod_security_policy_enabled "true" }}
-- kind: PodSecurityPolicy
-  name: privileged
-- kind: PodSecurityPolicy
-  name: restricted
-{{- end }}
 {{- if eq .Cluster.ConfigItems.audittrail_nakadi_url "" }}
 - name: audittrail-adapter-nakadi
   kind: PlatformCredentialsSet

cluster/manifests/psp/pod_security_policy.yaml

@@ -10,7 +10,7 @@ Mappings:
   Images:
     eu-central-1:
       # Use the node pool's architecture to construct the config item name that we're using to get the AMI name.
-      MachineImage: '{{ index .NodePool.ConfigItems (print "kuberuntu_image_v1_24_" .NodePool.ConfigItems.kuberuntu_distro_master "_" .Values.InstanceInfo.Architecture) }}'
+      MachineImage: '{{ index .NodePool.ConfigItems (print "kuberuntu_image_v1_25_" .NodePool.ConfigItems.kuberuntu_distro_master "_" .Values.InstanceInfo.Architecture) }}'
 
 Resources:
   AutoScalingGroup:

cluster/node-pools/master-default/stack.yaml

@@ -30,9 +30,6 @@ write_files:
 {{- end }}
       featureGates:
         SizeMemoryBackedVolumes: {{ .Cluster.ConfigItems.enable_size_memory_backed_volumes }}
-{{- if eq .Cluster.ConfigItems.enable_ephemeral_containers "true" }}
-        EphemeralContainers: true
-{{- end }}
       podPidsLimit: {{ .NodePool.ConfigItems.pod_max_pids }}
       maxPods: {{ nodeCIDRMaxPods (parseInt64 .Cluster.ConfigItems.node_cidr_mask_size) 8 }}
 {{- if ne .Cluster.ConfigItems.serialize_image_pulls "true" }}
@@ -120,14 +117,14 @@ write_files:
           - --allow-privileged=true
           - --service-cluster-ip-range=10.5.0.0/16
           - --secure-port=443
-          - --enable-admission-plugins=NamespaceLifecycle,LimitRanger,ServiceAccount,DefaultStorageClass,DefaultTolerationSeconds,ExtendedResourceToleration,MutatingAdmissionWebhook,ValidatingAdmissionWebhook,ResourceQuota,StorageObjectInUseProtection,{{ if eq .Cluster.ConfigItems.pod_security_policy_enabled "true" }}PodSecurityPolicy,{{end}}Priority,NodeRestriction{{if eq .Cluster.ConfigItems.event_rate_limit_enable "true"}},EventRateLimit{{end}}
+          - --enable-admission-plugins=NamespaceLifecycle,LimitRanger,ServiceAccount,DefaultStorageClass,DefaultTolerationSeconds,ExtendedResourceToleration,MutatingAdmissionWebhook,ValidatingAdmissionWebhook,ResourceQuota,StorageObjectInUseProtection,Priority,NodeRestriction{{if eq .Cluster.ConfigItems.event_rate_limit_enable "true"}},EventRateLimit{{end}}
           {{- if eq .Cluster.ConfigItems.event_rate_limit_enable "true"}}
           # This file specifies the EventRateLimit admission plugin's configuration
           - --admission-control-config-file=/etc/kubernetes/config/admission-config.yaml
           {{- end }}
           - --tls-cert-file=/etc/kubernetes/ssl/apiserver.pem
           - --tls-private-key-file=/etc/kubernetes/ssl/apiserver-key.pem
-          - --runtime-config={{ if eq .Cluster.ConfigItems.pod_security_policy_enabled "true" }}policy/v1beta1=true,{{end}}authorization.k8s.io/v1beta1=true,scheduling.k8s.io/v1alpha1=true,admissionregistration.k8s.io/v1beta1=true,autoscaling/v2beta2={{ .Cluster.ConfigItems.autoscaling_v2beta2_enabled }},autoscaling/v2beta1={{ .Cluster.ConfigItems.autoscaling_v2beta1_enabled }},batch/v1beta1={{ .Cluster.ConfigItems.batch_v1beta1_enabled }}
+          - --runtime-config=authorization.k8s.io/v1beta1=true,scheduling.k8s.io/v1alpha1=true,admissionregistration.k8s.io/v1beta1=true,autoscaling/v2beta2={{ .Cluster.ConfigItems.autoscaling_v2beta2_enabled }},autoscaling/v2beta1={{ .Cluster.ConfigItems.autoscaling_v2beta1_enabled }}
           - --authentication-token-webhook-config-file=/etc/kubernetes/config/authn.yaml
           - --authentication-token-webhook-cache-ttl=10s
           - --cloud-provider=aws
@@ -143,7 +140,7 @@ write_files:
           - --oidc-groups-claim=groups
           - "--oidc-groups-prefix=okta:"
 {{- end }}
-          - --feature-gates=HPAScaleToZero={{ .Cluster.ConfigItems.enable_hpa_scale_to_zero }},EphemeralContainers={{ .Cluster.ConfigItems.enable_ephemeral_containers }},HPAContainerMetrics={{ .Cluster.ConfigItems.enable_hpa_container_metrics }},StatefulSetAutoDeletePVC={{ .Cluster.ConfigItems.enable_statefulset_autodelete_pvc }},TopologyAwareHints={{ .Cluster.ConfigItems.enable_topology_aware_hints }},MinDomainsInPodTopologySpread={{ .Cluster.ConfigItems.min_domains_in_pod_topology_spread_enabled }},CronJobTimeZone={{.Cluster.ConfigItems.cronjob_time_zone_enabled}},MaxUnavailableStatefulSet={{.Cluster.ConfigItems.max_unavailable_statefulset_enabled}}
+          - --feature-gates=HPAScaleToZero={{ .Cluster.ConfigItems.enable_hpa_scale_to_zero }},HPAContainerMetrics={{ .Cluster.ConfigItems.enable_hpa_container_metrics }},StatefulSetAutoDeletePVC={{ .Cluster.ConfigItems.enable_statefulset_autodelete_pvc }},TopologyAwareHints={{ .Cluster.ConfigItems.enable_topology_aware_hints }},MinDomainsInPodTopologySpread={{ .Cluster.ConfigItems.min_domains_in_pod_topology_spread_enabled }},CronJobTimeZone={{.Cluster.ConfigItems.cronjob_time_zone_enabled}},MaxUnavailableStatefulSet={{.Cluster.ConfigItems.max_unavailable_statefulset_enabled}}
           - --service-account-key-file=/etc/kubernetes/ssl/service-account-public-key.pem
           - --service-account-signing-key-file=/etc/kubernetes/ssl/service-account-private-key.pem
           - --service-account-issuer={{ .Cluster.APIServerURL }}

cluster/node-pools/master-default/userdata.yaml

@@ -10,7 +10,7 @@ Mappings:
   Images:
     eu-central-1:
       # Use the node pool's architecture to construct the config item name that we're using to get the AMI name.
-      MachineImage: '{{ index .NodePool.ConfigItems (print "kuberuntu_image_v1_24_" .NodePool.ConfigItems.kuberuntu_distro_worker "_" .Values.InstanceInfo.Architecture) }}'
+      MachineImage: '{{ index .NodePool.ConfigItems (print "kuberuntu_image_v1_25_" .NodePool.ConfigItems.kuberuntu_distro_worker "_" .Values.InstanceInfo.Architecture) }}'
 
 Resources:
   AutoScalingGroup:

cluster/node-pools/worker-combined/stack.yaml

@@ -8,8 +8,8 @@ spec:
   amiFamily: Custom
   amiSelectorTerms:
     # Select on any AMI that has any of the following IDs
-    - id: {{ index .NodePool.ConfigItems (print "kuberuntu_image_v1_24_" .NodePool.ConfigItems.kuberuntu_distro_worker "_amd64") }}
-    - id: {{ index .NodePool.ConfigItems (print "kuberuntu_image_v1_24_" .NodePool.ConfigItems.kuberuntu_distro_worker "_arm64") }}
+    - id: {{ index .NodePool.ConfigItems (print "kuberuntu_image_v1_25_" .NodePool.ConfigItems.kuberuntu_distro_worker "_amd64") }}
+    - id: {{ index .NodePool.ConfigItems (print "kuberuntu_image_v1_25_" .NodePool.ConfigItems.kuberuntu_distro_worker "_arm64") }}
   metadataOptions:
     httpEndpoint: enabled
     httpProtocolIPv6: disabled
@@ -208,4 +208,4 @@ spec:
   weight: {{.NodePool.ConfigItems.scaling_priority}}
 #  {{ end}}
 
-#{{ end }}
+#{{ end }}

cluster/node-pools/worker-karpenter/provisioners.yaml

@@ -10,7 +10,7 @@ Mappings:
   Images:
     eu-central-1:
       # Use the node pool's architecture to construct the config item name that we're using to get the AMI name.
-      MachineImage: '{{ index .NodePool.ConfigItems (print "kuberuntu_image_v1_24_" .NodePool.ConfigItems.kuberuntu_distro_worker "_" .Values.InstanceInfo.Architecture) }}'
+      MachineImage: '{{ index .NodePool.ConfigItems (print "kuberuntu_image_v1_25_" .NodePool.ConfigItems.kuberuntu_distro_worker "_" .Values.InstanceInfo.Architecture) }}'
 
 Resources:
 {{ with $data := . }}

cluster/node-pools/worker-splitaz/stack.yaml

@@ -66,9 +66,6 @@ write_files:
       cpuCFSQuota: false
       featureGates:
         SizeMemoryBackedVolumes: {{ .Cluster.ConfigItems.enable_size_memory_backed_volumes }}
-{{- if eq .Cluster.ConfigItems.enable_ephemeral_containers "true" }}
-        EphemeralContainers: true
-{{- end }}
 {{- if eq .NodePool.ConfigItems.exec_probe_timeout_enabled "false" }}
         ExecProbeTimeout: false
 {{- end }}

cluster/node-pools/worker-splitaz/userdata.yaml

@@ -1,7 +1,7 @@
 # builder image
 FROM golang:1.21 as builder
 
-RUN CGO_ENABLED=0 go install github.com/onsi/ginkgo/ginkgo@v1.16.5
+RUN CGO_ENABLED=0 go install github.com/onsi/ginkgo/v2/ginkgo@v2.1.6
 
 # final image
 # TODO get rid of python dependencies