Merge pull request #6953 from zalando-incubator/dev-to-kube-1.26

dev to kube-1.26

Author: mikkeloscar

cluster/config-defaults.yaml

@@ -417,7 +417,7 @@ metrics_service_mem_max: "4Gi"
 metrics_server_metric_resolution: "15s"
 
 kube_aws_iam_controller_cpu: "5m"
-kube_aws_iam_controller_mem_max: "1Gi"
+kube_aws_iam_controller_mem: "50Mi"
 
 kube_state_metrics_cpu: "100m"
 kube_state_metrics_mem_max: "4Gi"
@@ -729,6 +729,7 @@ allowed_unsafe_sysctls: "net.ipv4.tcp_keepalive_time,net.ipv4.tcp_keepalive_intv
 
 # the maximum amount of memory for EBS CSI controller's sidecars
 ebs_csi_controller_sidecar_memory: "80Mi"
+ebs_csi_controller_sidecar_cpu: "10m"
 
 # pull images in parallel
 serialize_image_pulls: "false"

cluster/manifests/02-kube-aws-iam-controller/deployment.yaml

@@ -34,10 +34,10 @@ spec:
         resources:
           limits:
             cpu: "{{.Cluster.ConfigItems.kube_aws_iam_controller_cpu}}"
-            memory: "{{.Cluster.ConfigItems.kube_aws_iam_controller_mem_max}}"
+            memory: "{{.Cluster.ConfigItems.kube_aws_iam_controller_mem}}"
           requests:
             cpu: "{{.Cluster.ConfigItems.kube_aws_iam_controller_cpu}}"
-            memory: "{{.Cluster.ConfigItems.kube_aws_iam_controller_mem_max}}"
+            memory: "{{.Cluster.ConfigItems.kube_aws_iam_controller_mem}}"
       tolerations:
       - key: node.kubernetes.io/role
         value: master

cluster/manifests/02-kube-aws-iam-controller/vpa.yaml

@@ -17,4 +17,11 @@ spec:
     containerPolicies:
     - containerName: kube-aws-iam-controller
       maxAllowed:
-        memory: {{.Cluster.ConfigItems.kube_aws_iam_controller_mem_max}}
+        {{ range $NodePool := .Cluster.NodePools }}
+        {{ if eq $NodePool.Name "default-master" }}
+        # Scaling is relative to r6g.large (smallest master node)
+        # 0.014 -> 25m CPU, 0.003 -> 50Mi memory
+        cpu: {{ scaleQuantity ( instanceTypeCPUQuantity ( index .InstanceTypes 0 )) 0.014 }}
+        memory: {{ scaleQuantity ( instanceTypeMemoryQuantity ( index .InstanceTypes 0 )) 0.003 }}
+        {{ end }}
+        {{ end }}

cluster/manifests/03-ebs-csi/controller.yaml

@@ -95,10 +95,10 @@ spec:
               value: /var/lib/csi/sockets/pluginproxy/csi.sock
           resources:
             requests:
-              cpu: 10m
+              cpu: {{ .Cluster.ConfigItems.ebs_csi_controller_sidecar_cpu }}
               memory: {{ .Cluster.ConfigItems.ebs_csi_controller_sidecar_memory }}
             limits:
-              cpu: 10m
+              cpu: {{ .Cluster.ConfigItems.ebs_csi_controller_sidecar_cpu }}
               memory: {{ .Cluster.ConfigItems.ebs_csi_controller_sidecar_memory }}
           volumeMounts:
             - name: socket-dir
@@ -117,10 +117,10 @@ spec:
               value: /var/lib/csi/sockets/pluginproxy/csi.sock
           resources:
             requests:
-              cpu: 10m
+              cpu: {{ .Cluster.ConfigItems.ebs_csi_controller_sidecar_cpu }}
               memory: {{ .Cluster.ConfigItems.ebs_csi_controller_sidecar_memory }}
             limits:
-              cpu: 10m
+              cpu: {{ .Cluster.ConfigItems.ebs_csi_controller_sidecar_cpu }}
               memory: {{ .Cluster.ConfigItems.ebs_csi_controller_sidecar_memory }}
           volumeMounts:
             - name: socket-dir
@@ -139,10 +139,10 @@ spec:
               value: /var/lib/csi/sockets/pluginproxy/csi.sock
           resources:
             requests:
-              cpu: 10m
+              cpu: {{ .Cluster.ConfigItems.ebs_csi_controller_sidecar_cpu }}
               memory: {{ .Cluster.ConfigItems.ebs_csi_controller_sidecar_memory }}
             limits:
-              cpu: 10m
+              cpu: {{ .Cluster.ConfigItems.ebs_csi_controller_sidecar_cpu }}
               memory: {{ .Cluster.ConfigItems.ebs_csi_controller_sidecar_memory }}
           volumeMounts:
             - name: socket-dir

cluster/manifests/03-ebs-csi/vpa.yaml

@@ -16,11 +16,22 @@ spec:
   resourcePolicy:
     containerPolicies:
     - containerName: csi-provisioner
+    {{ range $NodePool := .Cluster.NodePools }}
+    {{ if eq $NodePool.Name "default-master" }}
+    # Scaling is relative to r6g.large (smallest master node)
+    # 0.006 -> ~90Mi memory, 0.031 -> ~55m CPU
+    {{ $scaledCPU := scaleQuantity ( instanceTypeCPUQuantity ( index .InstanceTypes 0 )) 0.031 }}
+    {{ $scaledMemory := scaleQuantity ( instanceTypeMemoryQuantity ( index .InstanceTypes 0 )) 0.006 }}
       maxAllowed:
-        memory: {{ .Cluster.ConfigItems.ebs_csi_controller_sidecar_memory }}
+        cpu: {{ $scaledCPU }}
+        memory: {{ $scaledMemory }}
     - containerName: csi-attacher
       maxAllowed:
-        memory: {{ .Cluster.ConfigItems.ebs_csi_controller_sidecar_memory }}
+        cpu: {{ $scaledCPU }}
+        memory: {{ $scaledMemory }}
     - containerName: csi-resizer
       maxAllowed:
-        memory: {{ .Cluster.ConfigItems.ebs_csi_controller_sidecar_memory }}
+        cpu: {{ $scaledCPU }}
+        memory: {{ $scaledMemory }}
+    {{ end }}
+    {{ end }}

cluster/manifests/z-karpenter/vpa.yaml

@@ -19,6 +19,15 @@ spec:
     containerPolicies:
       - containerName: controller
         minAllowed:
-          memory: {{.Cluster.ConfigItems.karpenter_controller_memory}}
-          cpu: {{.Cluster.ConfigItems.karpenter_controller_cpu}}
+          memory: {{ .Cluster.ConfigItems.karpenter_controller_memory }}
+          cpu: {{ .Cluster.ConfigItems.karpenter_controller_cpu }}
+        maxAllowed:
+        {{ range $NodePool := .Cluster.NodePools}}
+        {{ if eq $NodePool.Name "default-master" }}
+          # Scaling is relative to r6g.large (smallest master node)
+          # 0.016 -> ~250Mi memory, 0.027 -> ~50m CPU
+          memory: {{ scaleQuantity (instanceTypeMemoryQuantity (index .InstanceTypes 0)) 0.016 }}
+          cpu: {{ scaleQuantity (instanceTypeCPUQuantity (index .InstanceTypes 0)) 0.027 }}
+        {{ end }}
+        {{ end }}
 {{end}}