Merge branch 'dev' into container-registry.zalando.net/teapot/aws-cloud-controller-manager-internal

Author: linki

cluster/cluster.yaml

@@ -28,9 +28,6 @@ cluster_autoscaler_max_graceful_termination_sec: "1209600" # 2 weeks
 cluster_autoscaler_max_usnchedulable_pods_considered: "1000"
 
 # karpenter settings
-# DO NOT SET TO FALSE IF THE CLUSTER HAS KARPENTER POOLS OR NODES. REFER TO TEAPOT DOCS  FOR HOW TO ROLLBACK KARPENTER
-# https://teapot.docs.zalando.net/howtos/karpenter-operations/
-karpenter_pools_enabled: "true"
 
 karpenter_controller_cpu: "25m"
 karpenter_controller_memory: "256Mi"
@@ -50,8 +47,15 @@ karpenter_instance_storage_raid0: "true"
 # Can be set cluster wide or per node pool
 karpenter_in_transit_support_required: "false"
 
+# configure whether we allow t instance families for Karpenter nodes
+# t type instances have burstable CPU, which can be undesirable in production
+karpenter_instance_family_t_enabled: "false"
+
+# configure whether spot instances should be enabled in Karpenter's capacity-types
+karpenter_enable_spot: "true"
+
 # ALB config created by kube-aws-ingress-controller
-kube_aws_ingress_controller_ssl_policy: "ELBSecurityPolicy-TLS-1-2-2017-01"
+kube_aws_ingress_controller_ssl_policy: "ELBSecurityPolicy-TLS13-1-2-2021-06"
 kube_aws_ingress_controller_idle_timeout: "1m"
 kube_aws_ingress_controller_deregistration_delay_timeout: "10s"
 # allow using NLBs for ingress
@@ -252,14 +256,6 @@ skipper_pod_deletion_cost_controller_resync_interval: "1h"
 # polarsignals - only enabled for testing teapot
 polarsignals_enabled: "false"
 
-# Emergency Access Service
-# Control whether the emergency access service is enabled or not.
-{{ if and (eq .Cluster.Environment "production") (eq .Cluster.Provider "zalando-aws") }}
-emergency_access_service_enabled: "true"
-{{else}}
-emergency_access_service_enabled: "false"
-{{end}}
-
 # Kube-Metrics-Adapter
 ## Scheduled scaling metrics: ramp up/down over this period of time
 kube_metrics_adapter_default_scaling_window: "10m"
@@ -312,6 +308,12 @@ skipper_serve_method_metric: "false"
 # defines if the http response status code is included in the dimension
 # of the skipper_serve_host_duration_seconds_bucket metric.
 skipper_serve_status_code_metric: "false"
+# skipper_combined_response_metrics sets the flag -combined-response-metrics.
+# It enables reporting combined response time metrics
+skipper_combined_response_metrics: "false"
+# skipper_backend_host_metrics sets the flag -backend-host-metrics.
+# It enables reporting total serve time metrics for backend
+skipper_backend_host_metrics: "false"
 
 # disabled|provisioned|enabled routegroup validation via skipper webhook
 # can be one of disabled|provisioned|enabled
@@ -385,7 +387,7 @@ fabric_gateway_controller_enabled: "true"
 fabric_gateway_controller_cpu: "50m"
 fabric_gateway_controller_memory: "150Mi"
 fabric_gateway_controller_allow_all_filters: "false"
-fabric_gateway_controller_snapshots_history_limit: "3"
+fabric_gateway_controller_snapshots_history_limit: "1"
 fabric_gateway_controller_enable_versioning: "true"
 fabric_gateway_controller_ssl_policy: ""
 fabric_gateway_controller_log_level: "INFO"
@@ -398,12 +400,6 @@ event_rate_limit_enable: "true"
 event_rate_limit_config_qps: "500"
 event_rate_limit_config_burst: "1000"
 
-# cadvisor settings
-cadvisor_cpu: "150m"
-cadvisor_memory: "150Mi"
-cadvisor_profiling_enabled: "false"
-cadvisor_enabled: "false"
-
 # settings for enabling the kubelet-summary-metrics proxy and prometheus metric
 # collection.
 kubelet_summary_metrics_enabled: "true"
@@ -780,18 +776,12 @@ tracing_coredns_local_zone_traces_endpoint: ""
 # AMI id given the image name and the Image AWS account owner.
 #
 # [0]: https://github.com/zalando-incubator/cluster-lifecycle-manager/blob/8a9bd1cb2d094038a9e23e646421f8146b48886a/provisioner/template.go#L116
-kuberuntu_image_v1_31_aws_amd64: {{ amiID "zalando-ubuntu-jammy-22.04-kubernetes-production-v1.31.6-amd64-master-368" "861068367966" }}
-kuberuntu_image_v1_31_aws_arm64: {{ amiID "zalando-ubuntu-jammy-22.04-kubernetes-production-v1.31.6-arm64-master-368" "861068367966" }}
-kuberuntu_image_v1_31_eks_amd64: {{ amiID "zalando-ubuntu-jammy-22.04-kubernetes-production-v1.31.7-amd64-master-371" "861068367966" }}
-kuberuntu_image_v1_31_eks_arm64: {{ amiID "zalando-ubuntu-jammy-22.04-kubernetes-production-v1.31.7-arm64-master-371" "861068367966" }}
+kuberuntu_image_v1_32_new_amd64: {{ amiID "zalando-ubuntu-jammy-22.04-kubernetes-production-v1.32.4-amd64-master-373" "861068367966" }}
+kuberuntu_image_v1_32_new_arm64: {{ amiID "zalando-ubuntu-jammy-22.04-kubernetes-production-v1.32.4-arm64-master-373" "861068367966" }}
 
 # This is used to determine which AMI to use for the cluster or individual node
-# pools. Possible values are 'aws' or 'eks'
-{{if eq .Cluster.Provider "zalando-eks"}}
-kuberuntu_ami_version: "eks"
-{{else}}
-kuberuntu_ami_version: "aws"
-{{end}}
+# pools. Possible values are 'new' or 'old'
+kuberuntu_ami_version: "new"
 
 # Feature toggle for auditing events
 audit_pod_events: "true"
@@ -928,16 +918,9 @@ external_dns_zones_cache_duration: "1h"
 # resource configuration
 external_dns_mem: "4Gi"
 
-# select which cache to use for Cluster DNS: unbound or dnsmasq.
-dns_cache: "unbound"
-
 expirimental_dns_unbound_liveness_probe: "true"
 
 # DNS container resources
-dns_dnsmasq_cpu: "100m"
-dns_dnsmasq_mem: "50Mi"
-dns_dnsmasq_sidecar_cpu: "10m"
-dns_dnsmasq_sidecar_mem: "45Mi"
 dns_unbound_cpu: "100m"
 dns_unbound_mem: "50Mi"
 dns_unbound_exporter_cpu: "10m"
@@ -1104,6 +1087,9 @@ config_provider_service: "false"
 # enable SizeMemoryBackedVolumes feature flag
 enable_size_memory_backed_volumes: "true"
 
+# enable ImageVolume feature flag
+enable_image_volumes: "false"
+
 # enable StatefulSetAutoDeletePVC feature flag
 # https://kubernetes.io/blog/2021/12/16/kubernetes-1-23-statefulset-pvc-auto-deletion/
 enable_statefulset_autodelete_pvc: "true"
@@ -1143,7 +1129,7 @@ enable_statefulset_autodelete_pvc: "true"
 # Source for the template function: sgIngressRanges: https://github.com/zalando-incubator/cluster-lifecycle-manager/blob/42695865a251fef58e22ce612d6549e75fa5d103/provisioner/template.go#L336-L417
 open_sg_ingress_ranges: ""
 
-# Each subdomain can reach a max of 63 bytes on Route53
+# Each DNS label (subdomain) can be 63 octets or less (https://datatracker.ietf.org/doc/html/rfc1035#section-2.3.4)
 # This custom value sets the subdomain max allowed length taking into consideration the 'cname-' prefix added by external-dns
 subdomain_max_length: "57"
 
@@ -1269,7 +1255,7 @@ wiz_sensor_cpu: "200m"
 wiz_sensor_memory: "300Mi"
 wiz_connector_cpu: "50m"
 wiz_connector_memory: "150Mi"
-wiz_priority: "false"
+wiz_priority: "true"
 # Please note when this is set to true it allows the use of the node selector feature
 # to deploy the sensor and connector on specific nodes, by manually setting the node selector label on the nodes.
 # This is useful when you want to deploy the sensor and connector on specific nodes.
@@ -1284,6 +1270,8 @@ eks_zalando_iam_aws_proxy_hpa_max_replicas: "10"
 eks_zalando_iam_aws_proxy_hpa_cpu_target: "80"
 eks_zalando_iam_aws_proxy_hpa_memory_target: "80"
 eks_okta_identity_provider: "true"
+eks_legacy_cluster_local_id: "kube-1"
+eks_oidc_issuer_url: "https://"
 eks_fis_support_enabled: "false"
 eks_fis_namespaces: "default"
 
@@ -1297,3 +1285,7 @@ aws_vpc_cni_custom_networking: "false"
 aws_vpc_cni_enable_network_policy: "false"
 # specify the network policy enforcement mode.
 aws_vpc_cni_network_policy_enforcing_mode: "standard"
+
+# aws-load-balancer-controller resource settings
+aws_load_balancer_controller_cpu: "100m"
+aws_load_balancer_controller_mem_max: "4Gi"

cluster/config-defaults.yaml

@@ -42,7 +42,6 @@ spec:
             cpu: 1m
             memory: 50Mi
       containers:
-{{ if eq .Cluster.ConfigItems.dns_cache "unbound" }}
       - name: unbound
 {{- if eq .Cluster.Provider "zalando-eks" }}
         image: 926694233939.dkr.ecr.eu-central-1.amazonaws.com/production_namespace/teapot/unbound:1.22.0-master-10
@@ -119,98 +118,9 @@ spec:
         - mountPath: /run/unbound
           name: unbound-socket
           readOnly: false
-{{ end }}
-{{ if eq .Cluster.ConfigItems.dns_cache "dnsmasq" }}
-      - name: dnsmasq
-        {{- if eq .Cluster.Provider "zalando-eks" }}
-        image: 926694233939.dkr.ecr.eu-central-1.amazonaws.com/production_namespace/teapot/k8s-dns-dnsmasq-nanny:1.17.4-master-15
-        {{- else }}
-        image: container-registry.zalando.net/teapot/k8s-dns-dnsmasq-nanny:1.17.4-master-15
-        {{- end }}
-        securityContext:
-          privileged: true
-        livenessProbe:
-          httpGet:
-            path: /healthcheck/dnsmasq
-            port: 9054
-            scheme: HTTP
-          initialDelaySeconds: 60
-          timeoutSeconds: 5
-          successThreshold: 1
-          failureThreshold: 5
-        args:
-        - -v=2
-        - -logtostderr
-        - -configDir=/etc/k8s/dns/dnsmasq-nanny
-        - -restartDnsmasq=true
-        - --
-        - --no-resolv
-        - --keep-in-foreground
-        - --log-facility=-
-        - --cache-size=50000
-        - --dns-forward-max=500
-        - --neg-ttl=60
-        # send requests to the last server first, only fallback to the previous ones if it's unreachable
-        - --strict-order
-        - --server=10.5.0.11#53 # TODO: fix this for ipv6
-        - --server={{ if eq .Cluster.ConfigItems.eks_ip_family "ipv4" }}127.0.0.1{{else}}::1{{end}}#9254
-        ports:
-        - containerPort: 53
-          name: dns
-          protocol: UDP
-        - containerPort: 53
-          name: dns-tcp
-          protocol: TCP
-        resources:
-          requests:
-            ephemeral-storage: 256Mi
-          limits:
-            cpu: {{.Cluster.ConfigItems.dns_dnsmasq_cpu}}
-            memory: {{.Cluster.ConfigItems.dns_dnsmasq_mem}}
-        lifecycle:
-          preStop:
-            sleep:
-              seconds: 35
-      - name: sidecar
-        {{- if eq .Cluster.Provider "zalando-eks" }}
-        image: 926694233939.dkr.ecr.eu-central-1.amazonaws.com/production_namespace/teapot/k8s-dns-sidecar:1.17.4-master-15
-        {{- else }}
-        image: container-registry.zalando.net/teapot/k8s-dns-sidecar:1.17.4-master-15
-        {{- end }}
-        securityContext:
-          privileged: true
-        livenessProbe:
-          httpGet:
-            path: /metrics
-            port: 9054
-            scheme: HTTP
-          initialDelaySeconds: 60
-          timeoutSeconds: 5
-          successThreshold: 1
-          failureThreshold: 5
-        args:
-        - --v=2
-        - --logtostderr
-        - --probe=dnsmasq,127.0.0.1:9254,ec2.amazonaws.com,5,A
-        - --prometheus-port=9054
-        ports:
-        - containerPort: 9054
-          name: metrics
-          protocol: TCP
-        resources:
-          requests:
-            ephemeral-storage: 256Mi
-          limits:
-            cpu: {{.Cluster.ConfigItems.dns_dnsmasq_sidecar_cpu}}
-            memory: {{.Cluster.ConfigItems.dns_dnsmasq_sidecar_mem}}
-        lifecycle:
-          preStop:
-            sleep:
-              seconds: 35
-{{ end }}
       - name: coredns
         {{- if eq .Cluster.Provider "zalando-eks" }}
-        image: 926694233939.dkr.ecr.eu-central-1.amazonaws.com/production_namespace/teapot/coredns:1.12.0-master-25
+        image: 926694233939.dkr.ecr.eu-central-1.amazonaws.com/production_namespace/teapot/coredns:1.12.1-master-26
         {{- else }}
         image: container-registry.zalando.net/teapot/coredns:1.12.1-master-26
         {{- end }}
@@ -299,7 +209,5 @@ spec:
             path: Corefile
           - key: unbound.conf
             path: unbound.conf
-{{- if eq .Cluster.ConfigItems.dns_cache "unbound" }}
       - name: unbound-socket
         emptyDir: {}
-{{- end }}

cluster/manifests/01-coredns-local/daemonset-coredns.yaml

@@ -13,6 +13,9 @@ data:
   dns.default.subdomain-max-length: "{{ .Cluster.ConfigItems.subdomain_max_length }}"
 
   generic.prevent-write-operations.enable: "{{ .Cluster.ConfigItems.teapot_admission_controller_prevent_write_operations }}"
+{{- if and (eq .Cluster.Provider "zalando-eks") (eq .Cluster.ConfigItems.eks_ip_family "ipv6") }}
+  generic.inject-albc-defaults.enable: "true"
+{{- end }}
 
   pod.container-resource-control.min-memory-request: "25Mi"
   pod.container-resource-control.default-cpu-request: "{{ .Cluster.ConfigItems.teapot_admission_controller_default_cpu_request }}"
@@ -29,7 +32,7 @@ data:
   pod.service-account-iam.enable: "true"
   pod.service-account-iam.base-aws-account-id: "{{ accountID .Cluster.InfrastructureAccount }}"
 {{- if eq .Cluster.ConfigItems.teapot_admission_controller_inject_aws_waiter "true" }}
-  pod.aws-waiter.image: "926694233939.dkr.ecr.eu-central-1.amazonaws.com/production_namespace/automata/aws-credentials-waiter:master-257"
+  pod.aws-waiter.image: "926694233939.dkr.ecr.eu-central-1.amazonaws.com/production_namespace/automata/aws-credentials-waiter:master-266"
 {{- end }}
   pod.env-inject.enable: "{{ .Cluster.ConfigItems.teapot_admission_controller_inject_environment_variables }}"
   pod.env-inject.variable._PLATFORM_ACCOUNT: "{{ .Cluster.Alias }}"

cluster/manifests/02-admission-control/config.yaml

@@ -33,11 +33,11 @@ spec:
       priorityClassName: system-cluster-critical
       containers:
       - name: admission-controller
-        image: 926694233939.dkr.ecr.eu-central-1.amazonaws.com/production_namespace/teapot/admission-controller:master-252
+        image: 926694233939.dkr.ecr.eu-central-1.amazonaws.com/production_namespace/teapot/admission-controller:master-260
         lifecycle:
           preStop:
-            exec:
-              command: ["/bin/sh", "-c",  "sleep 60"]
+            sleep:
+              seconds: 20
         readinessProbe:
           httpGet:
             scheme: HTTPS

cluster/manifests/02-admission-control/deployment.yaml

@@ -475,6 +475,48 @@ webhooks:
         apiGroups: [""]
         apiVersions: ["v1"]
         resources: ["services"]
+
+  - name: ingress-admitter.teapot.zalan.do
+    clientConfig:
+      {{- if eq .Cluster.Provider "zalando-eks"}}
+      service:
+        name: "admission-controller"
+        namespace: "kube-system"
+        path: "/ingress"
+      {{- else }}
+      url: "https://localhost:8085/ingress"
+      {{- end }}
+      caBundle: "{{ .Cluster.ConfigItems.ca_cert_decompressed }}"
+    admissionReviewVersions: ["v1beta1"]
+    failurePolicy: Fail
+    sideEffects: "NoneOnDryRun"
+    matchPolicy: Equivalent
+    rules:
+      - operations: [ "CREATE", "UPDATE" ]
+        apiGroups: ["networking.k8s.io"]
+        apiVersions: ["v1"]
+        resources: ["ingresses"]
+  - name: routegroup-admitter.teapot.zalan.do
+    clientConfig:
+      {{- if eq .Cluster.Provider "zalando-eks"}}
+      service:
+        name: "admission-controller"
+        namespace: "kube-system"
+        path: "/routegroup"
+      {{- else }}
+      url: "https://localhost:8085/routegroup"
+      {{- end }}
+      caBundle: "{{ .Cluster.ConfigItems.ca_cert_decompressed }}"
+    admissionReviewVersions: ["v1beta1"]
+    failurePolicy: Fail
+    sideEffects: "NoneOnDryRun"
+    matchPolicy: Equivalent
+    rules:
+      - operations: [ "CREATE", "UPDATE" ]
+        apiGroups: ["zalando.org"]
+        apiVersions: ["v1"]
+        resources: ["routegroups"]
+
 {{- if eq .Cluster.ConfigItems.teapot_admission_controller_enable_rolebinding_webhook "true" }}
   - name: rolebinding-admitter.teapot.zalan.do
 {{- if eq .Cluster.Provider "zalando-eks"}}

cluster/manifests/02-admission-control/teapot.yaml

@@ -32,16 +32,16 @@ spec:
       priorityClassName: system-cluster-critical
       containers:
       - name: skipper-admission-webhook
-        image: 926694233939.dkr.ecr.eu-central-1.amazonaws.com/production_namespace/teapot/skipper:v0.22.6
+        image: 926694233939.dkr.ecr.eu-central-1.amazonaws.com/production_namespace/teapot/skipper:v0.22.23
         args:
           - webhook
           - --address=:9085
           - --tls-cert-file=/etc/tls-certs/skipper-validation-webhook.pem
           - --tls-key-file=/etc/tls-certs/skipper-validation-webhook-key.pem
         lifecycle:
           preStop:
-            exec:
-              command: ["/bin/sh", "-c",  " sleep 60"]
+            sleep:
+              seconds: 20
         readinessProbe:
           httpGet:
             scheme: HTTPS

cluster/manifests/02-skipper-validation-webhook/deployment.yaml

@@ -26,9 +26,9 @@ spec:
       containers:
       - name: admission-controller
         {{if eq .Cluster.ConfigItems.vertical_pod_autoscaler_version "current"}}
-        image: container-registry.zalando.net/teapot/vpa-admission-controller:v1.3.0-main-8-custom
+        image: container-registry.zalando.net/teapot/vpa-admission-controller:v1.3.1-main-10-custom
         {{else if eq .Cluster.ConfigItems.vertical_pod_autoscaler_version "legacy"}}
-        image: container-registry.zalando.net/teapot/vpa-admission-controller:v1.2.1-main-6-custom
+        image: container-registry.zalando.net/teapot/vpa-admission-controller:v1.3.0-main-8-custom
         {{end}}
         command:
           - /admission-controller