Merge branch 'dev' into opa-e2e-tests

nwickramasin · nwickramasin · commit 5f852d6845ee · 2025-01-22T07:53:10.000+01:00
diff --git a/cluster/config-defaults.yaml b/cluster/config-defaults.yaml
@@ -427,6 +427,11 @@ nvidia_dcgm_exporter_enabled: "false"
 nvidia_dcgm_exporter_cpu: "10m"
 nvidia_dcgm_exporter_memory: "200Mi"
 
+# AWS EFA device plugin
+aws_efa_device_plugin_enabled: "false"
+aws_efa_device_plugin_cpu: "10m"
+aws_efa_device_plugin_memory: "20Mi"
+
 # static egress controller settings
 static_egress_controller_enabled: "true"
 
@@ -1147,6 +1152,16 @@ control_plane_graceful_shutdown: "true"
 # For rolling back it needs to be done in multiple stages: active -> serving -> pre -> none
 control_plane_load_balancer_internal: "none"
 
+# Optionally use internal subnets for running the nodes. This can be configured
+# a node pool level to only run a subset of nodes in the internal subnets.
+# If this is true then `associate_public_ip_on_launch` is automatically treated
+# as false.
+internal_node_subnets_enabled: "false"
+
+# Configure whether to associate public ip when launching instances.
+# This is only relevant when `internal_node_subnets_enabled` is false.
+associate_public_ip_on_launch: "true"
+
 # This allows setting custom sysctl settings. The config-item is intended to be
 # used on node-pools rather being set globally.
 #
@@ -1187,6 +1202,13 @@ kube_janitor_enabled: "true"
 teapot_admission_controller_scheduling_controls_enabled: "false"
 teapot_admission_controller_scheduling_controls_default_architecture: "amd64"
 
+# master-node-autoscaler configuration
+
+# Ignore recommendations for downgrading the instance generation of the master nodes
+# This is required to persist the upgrade of instance generation for master nodes
+# in a cluster.
+master_node_autoscaler_instance_generation_downgrade_disabled: "false"
+
 # role-sync-controller configs
 # Enabled by default only on Zalando EKS clusters
 {{ if eq .Cluster.Provider "zalando-eks" }}
diff --git a/cluster/manifests/01-admission-control/config.yaml b/cluster/manifests/01-admission-control/config.yaml
@@ -29,7 +29,7 @@ data:
   pod.service-account-iam.enable: "true"
   pod.service-account-iam.base-aws-account-id: "{{ accountID .Cluster.InfrastructureAccount }}"
 {{- if eq .Cluster.ConfigItems.teapot_admission_controller_inject_aws_waiter "true" }}
-  pod.aws-waiter.image: "926694233939.dkr.ecr.eu-central-1.amazonaws.com/production_namespace/automata/aws-credentials-waiter:master-234"
+  pod.aws-waiter.image: "926694233939.dkr.ecr.eu-central-1.amazonaws.com/production_namespace/automata/aws-credentials-waiter:master-246"
 {{- end }}
   pod.env-inject.enable: "{{ .Cluster.ConfigItems.teapot_admission_controller_inject_environment_variables }}"
   pod.env-inject.variable._PLATFORM_ACCOUNT: "{{ .Cluster.Alias }}"
@@ -107,6 +107,9 @@ data:
   pod.pod-security-policy.privileged-service-accounts.kube-system_efs-provisioner: ""
 {{- if eq .Cluster.ConfigItems.s3_csi_driver "true" }}
   pod.pod-security-policy.privileged-service-accounts.kube-system_s3-csi-driver: ""
+{{- end }}
+{{- if eq .Cluster.ConfigItems.aws_efa_device_plugin_enabled "true" }}
+  pod.pod-security-policy.privileged-service-accounts.kube-system_aws-efa-k8s-device-plugin: ""
 {{- end }}
   pod.pod-security-policy.privileged-service-accounts.visibility_logging-agent: ""
 {{- range $sa := split .Cluster.ConfigItems.teapot_admission_controller_pod_security_policy_privileged_service_accounts "," }}
diff --git a/cluster/manifests/audittrail-adapter/daemonset.yaml b/cluster/manifests/audittrail-adapter/daemonset.yaml
@@ -33,7 +33,7 @@ spec:
       hostNetwork: true
       containers:
       - name: audittrail-adapter
-        image: container-registry.zalando.net/teapot/audittrail-adapter:master-71
+        image: container-registry.zalando.net/teapot/audittrail-adapter:master-72
         env:
           - name: AWS_REGION
             value: "{{ .Cluster.Region }}"
diff --git a/cluster/manifests/aws-cloud-controller-manager/daemonset.yaml b/cluster/manifests/aws-cloud-controller-manager/daemonset.yaml
@@ -27,7 +27,7 @@ spec:
         - --cloud-provider=aws
         - --use-service-account-credentials=true
         - --configure-cloud-routes=false
-        image: container-registry.zalando.net/teapot/aws-cloud-controller-manager-internal:v1.31.4-master-136
+        image: container-registry.zalando.net/teapot/aws-cloud-controller-manager-internal:v1.31.4-master-137
         name: aws-cloud-controller-manager
         resources:
           requests:
diff --git a/cluster/manifests/aws-efa-device-plugin/daemonset.yaml b/cluster/manifests/aws-efa-device-plugin/daemonset.yaml
@@ -0,0 +1,159 @@
+{{ if eq .Cluster.ConfigItems.aws_efa_device_plugin_enabled "true" }}
+# source: https://github.com/aws/eks-charts/blob/master/stable/aws-efa-k8s-device-plugin/templates/daemonset.yaml
+apiVersion: apps/v1
+kind: DaemonSet
+metadata:
+  name: aws-efa-k8s-device-plugin
+  namespace: kube-system
+  labels:
+    application: kubernetes
+    component: aws-efa-k8s-device-plugin
+spec:
+  selector:
+    matchLabels:
+      daemonset:  aws-efa-k8s-device-plugin
+  updateStrategy:
+    type: RollingUpdate
+  template:
+    metadata:
+      labels:
+        daemonset: aws-efa-k8s-device-plugin
+        application: kubernetes
+        component: aws-efa-k8s-device-plugin
+      annotations:
+        logging/destination: "{{.Cluster.ConfigItems.log_destination_infra}}"
+    spec:
+      serviceAccountName: aws-efa-k8s-device-plugin
+      tolerations:
+        - operator: Exists
+          effect: NoExecute
+        - operator: Exists
+          effect: NoSchedule
+      # Mark this pod as a critical add-on; when enabled, the critical add-on
+      # scheduler reserves resources for critical add-on pods so that they can
+      # be rescheduled after a failure.
+      # See https://kubernetes.io/docs/tasks/administer-cluster/guaranteed-scheduling-critical-addon-pods/
+      priorityClassName: "system-node-critical"
+      affinity:
+        nodeAffinity:
+          requiredDuringSchedulingIgnoredDuringExecution:
+            nodeSelectorTerms:
+              - matchExpressions:
+                - key: node.kubernetes.io/instance-type
+                  operator: In
+                  values:
+                    - m5dn.24xlarge
+                    - m5n.24xlarge
+                    - m5zn.12xlarge
+                    - m6a.48xlarge
+                    - m6i.32xlarge
+                    - m6id.32xlarge
+                    - m6idn.32xlarge
+                    - m6in.32xlarge
+                    - m7a.48xlarge
+                    - m7g.16xlarge
+                    - m7gd.16xlarge
+                    - m7i.48xlarge
+                    - c5n.9xlarge
+                    - c5n.18xlarge
+                    - c6a.48xlarge
+                    - c6gn.16xlarge
+                    - c6i.32xlarge
+                    - c6id.32xlarge
+                    - c6in.32xlarge
+                    - c7a.48xlarge
+                    - c7g.16xlarge
+                    - c7gd.16xlarge
+                    - c7gn.16xlarge
+                    - c7i.48xlarge
+                    - r5dn.24xlarge
+                    - r5n.24xlarge
+                    - r6a.48xlarge
+                    - r6i.32xlarge
+                    - r6idn.32xlarge
+                    - r6in.32xlarge
+                    - r6id.32xlarge
+                    - r7a.48xlarge
+                    - r7g.16xlarge
+                    - r7gd.16xlarge
+                    - r7i.48xlarge
+                    - r7iz.32xlarge
+                    - x2idn.32xlarge
+                    - x2iedn.32xlarge
+                    - x2iezn.12xlarge
+                    - i3en.12xlarge
+                    - i3en.24xlarge
+                    - i4g.16xlarge
+                    - i4i.32xlarge
+                    - im4gn.16xlarge
+                    - dl1.24xlarge
+                    - dl2q.24xlarge
+                    - g4dn.8xlarge
+                    - g4dn.12xlarge
+                    - g4dn.16xlarge
+                    - g5.8xlarge
+                    - g5.12xlarge
+                    - g5.16xlarge
+                    - g5.24xlarge
+                    - g5.48xlarge
+                    - g6.8xlarge
+                    - g6.12xlarge
+                    - g6.16xlarge
+                    - g6.24xlarge
+                    - g6.48xlarge
+                    - g6e.8xlarge
+                    - g6e.12xlarge
+                    - g6e.16xlarge
+                    - g6e.24xlarge
+                    - g6e.48xlarge
+                    - gr6.8xlarge
+                    - inf1.24xlarge
+                    - p3dn.24xlarge
+                    - p4d.24xlarge
+                    - p4de.24xlarge
+                    - p5.48xlarge
+                    - p5e.48xlarge
+                    - p5en.48xlarge
+                    - trn1.32xlarge
+                    - trn1n.32xlarge
+                    - trn2.48xlarge
+                    - vt1.24xlarge
+                    - hpc6a.48xlarge
+                    - hpc6id.32xlarge
+                    - hpc7a.12xlarge
+                    - hpc7a.24xlarge
+                    - hpc7a.48xlarge
+                    - hpc7a.96xlarge
+                    - hpc7g.4xlarge
+                    - hpc7g.8xlarge
+                    - hpc7g.16xlarge
+      hostNetwork: true
+      containers:
+        - image: container-registry.zalando.net/teapot/aws-efa-k8s-device-plugin:v0.5.4-main-1
+          name: aws-efa-k8s-device-plugin
+          securityContext:
+            allowPrivilegeEscalation: false
+            capabilities:
+              drop:
+              - ALL
+            runAsNonRoot: false
+          resources:
+            requests:
+              cpu: "{{ .Cluster.ConfigItems.aws_efa_device_plugin_cpu }}"
+              memory: "{{ .Cluster.ConfigItems.aws_efa_device_plugin_memory }}"
+            limits:
+              cpu: "{{ .Cluster.ConfigItems.aws_efa_device_plugin_cpu }}"
+              memory: "{{ .Cluster.ConfigItems.aws_efa_device_plugin_memory }}"
+          volumeMounts:
+            - name: device-plugin
+              mountPath: /var/lib/kubelet/device-plugins
+            - name: infiniband-volume
+              mountPath: /dev/infiniband/
+      volumes:
+        - name: device-plugin
+          hostPath:
+            path: /var/lib/kubelet/device-plugins
+        - name: infiniband-volume
+          hostPath:
+            path: /dev/infiniband/
+{{ end }}
diff --git a/cluster/manifests/aws-efa-device-plugin/rbac.yaml b/cluster/manifests/aws-efa-device-plugin/rbac.yaml
@@ -0,0 +1,10 @@
+{{ if eq .Cluster.ConfigItems.aws_efa_device_plugin_enabled "true" }}
+apiVersion: v1
+kind: ServiceAccount
+metadata:
+  name: aws-efa-k8s-device-plugin
+  namespace: kube-system
+  labels:
+    application: kubernetes
+    component: aws-efa-k8s-device-plugin
+{{ end }}
diff --git a/cluster/manifests/deletions.yaml b/cluster/manifests/deletions.yaml
@@ -339,3 +339,11 @@ post_apply:
 - name: kube-janitor
   kind: ClusterRoleBinding
 {{- end }}
+{{- if ne .Cluster.ConfigItems.aws_efa_device_plugin_enabled "true"}}
+- name: aws-efa-k8s-device-plugin
+  kind: DaemonSet
+  namespace: kube-system
+- name: aws-efa-k8s-device-plugin
+  kind: ServiceAccount
+  namespace: kube-system
+{{- end}}
diff --git a/cluster/manifests/deployment-service/01-config.yaml b/cluster/manifests/deployment-service/01-config.yaml
@@ -5,6 +5,7 @@ metadata:
   namespace: kube-system
 data:
   aws-account-id: "{{accountID .Cluster.InfrastructureAccount}}"
+  aws-account-name: "{{.Cluster.AccountName}}"
   cluster-alias: "{{.Cluster.Alias}}"
   cluster-vpc-id: "{{.Cluster.ConfigItems.vpc_id}}"
   scalyr-team-token: "{{.Cluster.ConfigItems.scalyr_team_token}}"
diff --git a/cluster/manifests/deployment-service/controller-statefulset.yaml b/cluster/manifests/deployment-service/controller-statefulset.yaml
@@ -29,7 +29,7 @@ spec:
       terminationGracePeriodSeconds: 300
       containers:
         - name: "deployment-service-controller"
-          image: "container-registry.zalando.net/teapot/deployment-controller:master-235"
+          image: "container-registry.zalando.net/teapot/deployment-controller:master-236"
           args:
             - "--config-namespace=kube-system"
             - "--decrypt-kms-alias-arn=arn:aws:kms:{{ .Cluster.Region }}:{{ .Cluster.InfrastructureAccount | getAWSAccountID }}:alias/deployment-secret"
diff --git a/cluster/manifests/deployment-service/status-service-deployment.yaml b/cluster/manifests/deployment-service/status-service-deployment.yaml
@@ -1,4 +1,4 @@
-# {{ $image   := "container-registry.zalando.net/teapot/deployment-status-service:master-235" }}
+# {{ $image   := "container-registry.zalando.net/teapot/deployment-status-service:master-236" }}
 # {{ $version := index (split $image ":") 1 }}
 
 apiVersion: apps/v1
diff --git a/cluster/manifests/fabric-gateway/deployment.yaml b/cluster/manifests/fabric-gateway/deployment.yaml
@@ -1,4 +1,4 @@
-# {{ $image := "container-registry.zalando.net/gwproxy/fabric-gateway:master-293" }}
+# {{ $image := "container-registry.zalando.net/gwproxy/fabric-gateway:master-296" }}
 # {{ $version := index (split $image ":") 1 }}
 apiVersion: apps/v1
 kind: Deployment
diff --git a/cluster/manifests/fabric-gateway/fabricgateway_crd.yaml b/cluster/manifests/fabric-gateway/fabricgateway_crd.yaml
@@ -7,6 +7,8 @@ metadata:
 spec:
   group: zalando.org
   names:
+    categories:
+    - all
     kind: FabricGateway
     listKind: FabricGatewayList
     plural: fabricgateways
diff --git a/cluster/manifests/flannel/daemonset.yaml b/cluster/manifests/flannel/daemonset.yaml
@@ -57,7 +57,7 @@ spec:
           failureThreshold: 30
           periodSeconds: 10
       - name: kube-flannel
-        image: container-registry.zalando.net/teapot/flannel:v0.26.2-master-33
+        image: container-registry.zalando.net/teapot/flannel:v0.26.3-master-34
         command:
         - /opt/bin/flanneld
         args:
diff --git a/cluster/manifests/kube-metrics-adapter/deployment.yaml b/cluster/manifests/kube-metrics-adapter/deployment.yaml
@@ -27,7 +27,7 @@ spec:
       serviceAccountName: custom-metrics-apiserver
       containers:
       - name: kube-metrics-adapter
-        image: container-registry.zalando.net/teapot/kube-metrics-adapter:kube-metrics-adapter-0.2.3-43-g26baf4b
+        image: container-registry.zalando.net/teapot/kube-metrics-adapter:kube-metrics-adapter-0.2.3-45-g5800062
         env:
         - name: AWS_REGION
           value: {{ .Cluster.Region }}
diff --git a/cluster/manifests/skipper/deployment.yaml b/cluster/manifests/skipper/deployment.yaml
@@ -1,7 +1,7 @@
 {{/* image-updater-bot detects *image variables so use print to disable it for main image */}}
 
-{{ $main_image := print "container-registry.zalando.net/teapot/skipper-internal:" "v0.21.240-1063" }}
-{{ $canary_image := "container-registry.zalando.net/teapot/skipper-internal:v0.21.240-1063" }}
+{{ $main_image := print "container-registry.zalando.net/teapot/skipper-internal:" "v0.21.247-1070" }}
+{{ $canary_image := "container-registry.zalando.net/teapot/skipper-internal:v0.21.247-1070" }}
 
 
 {{/* Optional canary arguments separated by "[cf724afc]" to allow whitespaces, e.g. "-foo=has a whitespace[cf724afc]-baz=qux" */}}
diff --git a/cluster/node-pools/master-default/stack.yaml b/cluster/node-pools/master-default/stack.yaml
@@ -37,9 +37,13 @@ Resources:
         PropagateAtLaunch: true
         Value: "{{ .NodePool.ConfigItems.pod_max_pids }}"
       VPCZoneIdentifier:
-{{ with $values := .Values }}
-{{ range $az := $values.availability_zones }}
-        - "{{ index $values.subnets $az }}"
+{{ with $data := . }}
+{{ range $az := $data.Values.availability_zones }}
+        # {{ if eq $data.NodePool.ConfigItems.internal_node_subnets_enabled "true" }}
+        - "{{ index $data.Values.internal_node_subnets $az }}"
+        # {{ else }}
+        - "{{ index $data.Values.subnets $az }}"
+        # {{ end }}
 {{ end }}
 {{ end }}
       TargetGroupARNs:
@@ -81,7 +85,9 @@ Resources:
             VolumeType: gp3
         NetworkInterfaces:
         - DeviceIndex: 0
+          # {{ if and (eq .NodePool.ConfigItems.associate_public_ip_on_launch "true") (ne .NodePool.ConfigItems.internal_node_subnets_enabled "true") }}
           AssociatePublicIpAddress: true
+          # {{ end }}
           Groups:
           - !ImportValue '{{ .Cluster.ID }}:master-security-group'
         EbsOptimized: false
diff --git a/cluster/node-pools/master-default/userdata.yaml b/cluster/node-pools/master-default/userdata.yaml
diff --git a/cluster/node-pools/worker-combined/stack.yaml b/cluster/node-pools/worker-combined/stack.yaml
diff --git a/cluster/node-pools/worker-karpenter/provisioners.yaml b/cluster/node-pools/worker-karpenter/provisioners.yaml
diff --git a/cluster/node-pools/worker-splitaz/stack.yaml b/cluster/node-pools/worker-splitaz/stack.yaml
diff --git a/test/e2e/cluster_config.sh b/test/e2e/cluster_config.sh

Original file line number	Diff line number	Diff line change
`@@ -1,4 +1,4 @@`
`1`		`-# {{ $image := "container-registry.zalando.net/teapot/deployment-status-service:master-235" }}`
	`1`	`+# {{ $image := "container-registry.zalando.net/teapot/deployment-status-service:master-236" }}`
`2`	`2`	`# {{ $version := index (split $image ":") 1 }}`
`3`	`3`
`4`	`4`	`apiVersion: apps/v1`