Merge branch 'dev' into eks-custom-networking

Author: linki

cluster/cluster.yaml

@@ -171,15 +171,17 @@ Resources:
         EndpointPublicAccess: true
         EndpointPrivateAccess: true
         # PublicAccessCidrs: [ "1.1.1.2/32" ]
-{{- if eq .Cluster.ConfigItems.eks_control_plane_logging "true" }}
       Logging:
         ClusterLogging:
+{{- if eq .Cluster.ConfigItems.eks_control_plane_logging "true" }}
           EnabledTypes:
             - Type: api
             - Type: audit
             - Type: authenticator
             - Type: controllerManager
             - Type: scheduler
+{{- else }}
+          EnabledTypes: []
 {{- end }}
       # Tags:
       # - Key: "application"

cluster/config-defaults.yaml

@@ -250,6 +250,14 @@ skipper_pod_deletion_cost_controller_resync_interval: "1h"
 # polarsignals - only enabled for testing teapot
 polarsignals_enabled: "false"
 
+# Emergency Access Service
+# Control whether the emergency access service is enabled or not.
+{{ if and (eq .Cluster.Environment "production") (eq .Cluster.Provider "zalando-aws") }}
+emergency_access_service_enabled: "true"
+{{else}}
+emergency_access_service_enabled: "false"
+{{end}}
+
 # Kube-Metrics-Adapter
 ## Scheduled scaling metrics: ramp up/down over this period of time
 kube_metrics_adapter_default_scaling_window: "10m"
@@ -915,8 +923,8 @@ dns_dnsmasq_sidecar_cpu: "10m"
 dns_dnsmasq_sidecar_mem: "45Mi"
 dns_unbound_cpu: "100m"
 dns_unbound_mem: "50Mi"
-dns_unbound_telemetry_cpu: "10m"
-dns_unbound_telemetry_mem: "45Mi"
+dns_unbound_exporter_cpu: "10m"
+dns_unbound_exporter_mem: "45Mi"
 dns_coredns_cpu: "50m"
 dns_coredns_mem: "100Mi"
 
@@ -1251,7 +1259,7 @@ wiz_priority: "false"
 wiz_node_feature_rollout : "false"
 
 # EKS specific configuration
-eks_control_plane_logging: "false"
+eks_control_plane_logging: "true"
 eks_ip_family: "ipv4"
 eks_zalando_iam_aws_proxy_cpu: "100m"
 eks_zalando_iam_aws_proxy_memory: "512Mi"

cluster/manifests/01-coredns-local/configmap-local.yaml

@@ -10,7 +10,11 @@ data:
   unbound.conf: |
     server:
       directory: "/etc/unbound/"
+{{- if and (eq .Cluster.Provider "zalando-eks") (eq .Cluster.ConfigItems.eks_ip_family "ipv6") }}
+      interface: "::0"
+{{- else }}
       interface: 0.0.0.0
+{{- end }}
       interface-automatic: yes
       # Drop user privileges after binding the port.
       username: "_unbound"
@@ -21,24 +25,52 @@ data:
       log-servfail: yes
       # allow query localhost (coredns at 127.0.0.1:9254)
       do-not-query-localhost: no
+{{- if and (eq .Cluster.Provider "zalando-eks") (eq .Cluster.ConfigItems.eks_ip_family "ipv6") }}
+      access-control: ::/0 allow
+{{- else }}
       access-control: 0.0.0.0/0 allow
+{{- end }}
       harden-dnssec-stripped: no
       so-reuseport: yes
       cache-min-ttl: 1
       disable-dnssec-lame-check: yes
       minimal-responses: yes
       extended-statistics: yes
       # support reverse lookup of kubernetes addresses
+{{- if eq .Cluster.Provider "zalando-eks" }}
+      # CoreDNS is authoritative for the reverse lookup ranges. Therefore
+      # disable the default protection in unbound to allow reverse lookup
+      # queries to pass through to CoreDNS
+      # https://github.com/NLnetLabs/unbound/blob/5c84bb573f9728c10bcb3592dbd12be403d362de/doc/example.conf.in#L804-L850
+      local-zone: "d.f.ip6.arpa." nodefault
+      local-zone: "8.e.f.ip6.arpa." nodefault
+      local-zone: "9.e.f.ip6.arpa." nodefault
+      local-zone: "a.e.f.ip6.arpa." nodefault
+      local-zone: "b.e.f.ip6.arpa." nodefault
+      local-zone: "8.b.d.0.1.0.0.2.ip6.arpa." nodefault
+      local-zone: "ip6.arpa." transparent
+      local-zone: "10.in-addr.arpa." nodefault
+      local-zone: "in-addr.arpa." transparent
+{{- else }}
       local-zone: "2.10.in-addr.arpa." transparent
       local-zone: "3.10.in-addr.arpa." transparent
       local-zone: "5.10.in-addr.arpa." transparent
-      # make metrics available for the unbound-telemetry container (127.0.0.1:9054)
+{{- end }}
+      # make metrics available for the unbound_exporter
       remote-control:
         control-enable: yes
-        control-use-cert: no
+        control-interface: /run/unbound/unbound.ctl
     forward-zone:
       name: "."
       forward-addr: 127.0.0.1@9254 # coredns
+{{- if eq .Cluster.Provider "zalando-eks" }}
+    forward-zone:
+      name: "ip6.arpa."
+      forward-addr: 127.0.0.1@9254 # coredns
+    forward-zone:
+      name: "in-addr.arpa."
+      forward-addr: 127.0.0.1@9254 # coredns
+{{- else }}
     forward-zone:
       name: "2.10.in-addr.arpa."
       forward-addr: 127.0.0.1@9254 # coredns
@@ -48,6 +80,7 @@ data:
     forward-zone:
       name: "5.10.in-addr.arpa."
       forward-addr: 127.0.0.1@9254 # coredns
+{{- end }}
   Corefile: |
 {{ if and .Cluster.ConfigItems.custom_dns_zone .Cluster.ConfigItems.custom_dns_zone_nameservers }}
     {{ .Cluster.ConfigItems.custom_dns_zone }}:9254 {

cluster/manifests/01-coredns-local/daemonset-coredns.yaml

@@ -66,10 +66,10 @@ spec:
             command:
             - dig
             - "+short"
-{{- if and (eq .Cluster.Provider "zalando-eks") (eq .Cluster.ConfigItems.eks_ip_family "ipv4") }}
-            - "@127.0.0.1"
+{{- if and (eq .Cluster.Provider "zalando-eks") (eq .Cluster.ConfigItems.eks_ip_family "ipv6") }}
+            - "@::1"
 {{- else }}
-            - "::1"
+            - "@127.0.0.1"
 {{- end }}
             - "kubernetes.default.svc.cluster.local"
           initialDelaySeconds: 60
@@ -91,16 +91,20 @@ spec:
           name: config-volume
           readOnly: true
           subPath: unbound.conf
-      - name: unbound-telemetry
-{{- if eq .Cluster.Provider "zalando-eks" }}
-        image: 926694233939.dkr.ecr.eu-central-1.amazonaws.com/production_namespace/teapot/unbound-telemetry:master-5
-{{- else }}
-        image: container-registry.zalando.net/teapot/unbound-telemetry:master-5
-{{- end }}
+        - mountPath: /run/unbound
+          name: unbound-socket
+          readOnly: false
+      - name: unbound-exporter
+        image: 926694233939.dkr.ecr.eu-central-1.amazonaws.com/production_namespace/teapot/unbound_exporter:v0.4.6-main-1.custom
         args:
-        - tcp
-        # TODO: ipv6
-        - --bind=0.0.0.0:9054
+        - -unbound.ca
+        - ""
+        - -unbound.cert
+        - ""
+        - -unbound.host
+        - "unix:///run/unbound/unbound.ctl"
+        - -web.listen-address
+        - ":9054"
         ports:
         - name: metrics
           containerPort: 9054
@@ -109,8 +113,12 @@ spec:
           requests:
             ephemeral-storage: 256Mi
           limits:
-            cpu: {{.Cluster.ConfigItems.dns_unbound_telemetry_cpu}}
-            memory: {{.Cluster.ConfigItems.dns_unbound_telemetry_mem}}
+            cpu: {{.Cluster.ConfigItems.dns_unbound_exporter_cpu}}
+            memory: {{.Cluster.ConfigItems.dns_unbound_exporter_mem}}
+        volumeMounts:
+        - mountPath: /run/unbound
+          name: unbound-socket
+          readOnly: false
 {{ end }}
 {{ if eq .Cluster.ConfigItems.dns_cache "dnsmasq" }}
       - name: dnsmasq
@@ -183,7 +191,6 @@ spec:
         args:
         - --v=2
         - --logtostderr
-        # TODO: ipv6
         - --probe=dnsmasq,127.0.0.1:9254,ec2.amazonaws.com,5,A
         - --prometheus-port=9054
         ports:
@@ -203,7 +210,7 @@ spec:
 {{ end }}
       - name: coredns
         {{- if eq .Cluster.Provider "zalando-eks" }}
-        image: 926694233939.dkr.ecr.eu-central-1.amazonaws.com/production_namespace/teapot/coredns:1.11.3-master-24
+        image: 926694233939.dkr.ecr.eu-central-1.amazonaws.com/production_namespace/teapot/coredns:1.12.0-master-25
         {{- else }}
         image: container-registry.zalando.net/teapot/coredns:1.12.0-master-25
         {{- end }}
@@ -292,3 +299,7 @@ spec:
             path: Corefile
           - key: unbound.conf
             path: unbound.conf
+{{- if eq .Cluster.ConfigItems.dns_cache "unbound" }}
+      - name: unbound-socket
+        emptyDir: {}
+{{- end }}

cluster/manifests/02-admission-control/config.yaml

@@ -197,7 +197,6 @@ data:
 {{- end}}
 
   node.node-not-ready-taint.enable: "{{ .Cluster.ConfigItems.teapot_admission_controller_node_not_ready_taint }}"
-  node.karpenter-unregistered-taint.enable: "true"
   node.extended-node-restriction.enable: "true"
 
 {{- range $group, $provider := nodeLifeCycleProviderPerNodePoolGroup .Cluster.NodePools }}

cluster/manifests/02-admission-control/deployment.yaml

@@ -33,7 +33,7 @@ spec:
       priorityClassName: system-cluster-critical
       containers:
       - name: admission-controller
-        image: 926694233939.dkr.ecr.eu-central-1.amazonaws.com/production_namespace/teapot/admission-controller:master-247
+        image: 926694233939.dkr.ecr.eu-central-1.amazonaws.com/production_namespace/teapot/admission-controller:master-248
         lifecycle:
           preStop:
             exec:

cluster/manifests/02-admission-control/teapot.yaml

@@ -13,13 +13,21 @@ webhooks:
       matchExpressions:
         - key: component
           operator: NotIn
-          values: ["admission-controller", "coredns", "aws-node"]
+          values:
+            - "admission-controller"
+            - "coredns"
+            - "aws-node"
+            - "external-dns"
+            - "zalando-iam-aws-proxy"
         - key: k8s-app
           operator: NotIn
           values: ["kube-proxy"]
         - key: app.kubernetes.io/name
           operator: NotIn
           values: ["eks-pod-identity-agent"]
+        - key: application
+          operator: NotIn
+          values: ["kube-ingress-aws-controller"]
 {{- end }}
     clientConfig:
       {{- if eq .Cluster.Provider "zalando-eks"}}
@@ -48,13 +56,21 @@ webhooks:
       matchExpressions:
         - key: component
           operator: NotIn
-          values: ["admission-controller", "coredns", "aws-node"]
+          values:
+            - "admission-controller"
+            - "coredns"
+            - "aws-node"
+            - "external-dns"
+            - "zalando-iam-aws-proxy"
         - key: k8s-app
           operator: NotIn
           values: ["kube-proxy"]
         - key: app.kubernetes.io/name
           operator: NotIn
           values: ["eks-pod-identity-agent"]
+        - key: application
+          operator: NotIn
+          values: ["kube-ingress-aws-controller"]
 {{- end }}
     clientConfig:
       {{- if eq .Cluster.Provider "zalando-eks"}}
@@ -552,6 +568,13 @@ webhooks:
         expression: '!(request.userInfo.username in ["system:kube-controller-manager", "system:kube-scheduler", "zalando-iam:zalando:service:k8sapi_credentials-provider"])'
       - name: 'exclude-eks-components'
         expression: '!request.userInfo.username.startsWith("eks:")'
+      - name: 'allow-api-monitoring-controller-access'
+        expression: |
+          !(
+            request.userInfo.username == "system:serviceaccount:api-infrastructure:api-monitoring-controller" &&
+            object.kind == "ConfigMap" &&
+            object.metadata.name == "skipper-default-filters"
+          )
   - name: collaborator-deny-admitter.teapot.zalan.do
     clientConfig:
       {{- if eq .Cluster.Provider "zalando-eks"}}

cluster/manifests/02-skipper-validation-webhook/skipper-webhook.yaml

@@ -13,6 +13,12 @@ webhooks:
         apiGroups: ["zalando.org"]
         apiVersions: ["v1"]
         resources: ["routegroups"]
+    namespaceSelector:
+      matchExpressions:
+        - key: kubernetes.io/metadata.name
+          operator: NotIn
+          values:
+          - kube-system
     clientConfig:
       # {{- if eq .Cluster.Provider "zalando-eks"}}
       service:
@@ -34,6 +40,12 @@ webhooks:
         apiGroups: ["networking.k8s.io"]
         apiVersions: ["v1"]
         resources: ["ingresses"]
+    namespaceSelector:
+      matchExpressions:
+        - key: kubernetes.io/metadata.name
+          operator: NotIn
+          values:
+          - kube-system
     clientConfig:
       # {{- if eq .Cluster.Provider "zalando-eks"}}
       service:

cluster/manifests/03-kube-aws-iam-controller/deployment.yaml

@@ -27,7 +27,7 @@ spec:
       hostNetwork: true
       containers:
       - name: kube-aws-iam-controller
-        image: container-registry.zalando.net/teapot/kube-aws-iam-controller:v0.3.0-66-g9bdbaef
+        image: container-registry.zalando.net/teapot/kube-aws-iam-controller:v0.3.0-68-g08c195b
         env:
         - name: AWS_DEFAULT_REGION
           value: "{{.Cluster.Region}}"

cluster/manifests/04-ebs-csi/controller.yaml

@@ -91,7 +91,7 @@ spec:
             allowPrivilegeEscalation: false
             readOnlyRootFilesystem: true
         - name: csi-provisioner
-          image: container-registry.zalando.net/teapot/external-provisioner:v5.1.0-eks-1-31-10-master-26
+          image: container-registry.zalando.net/teapot/external-provisioner:v5.1.0-eks-1-31-10-master-27
           args:
             - --csi-address=$(ADDRESS)
             - --v=2