Merge pull request #9303 from zalando-incubator/dev-to-kube-1.32

dev to kube-1.32

Author: zaklawrencea

cluster/cluster.yaml

@@ -3222,6 +3222,217 @@ Resources:
           PolicyName: root
       RoleName: "{{.Cluster.LocalID}}-audittrail-adapter"
     Type: 'AWS::IAM::Role'
+# {{- if and (eq .Cluster.Provider "zalando-eks") (eq .Cluster.ConfigItems.eks_ip_family "ipv6") }}
+  AWSLoadBalancerControllerIAMRole:
+    Properties:
+      AssumeRolePolicyDocument: !Sub
+        - |
+          {
+            "Version": "2012-10-17",
+            "Statement": [
+              {
+                "Effect": "Allow",
+                "Principal": {
+                  "Federated": [
+                    "arn:aws:iam::${AWS::AccountId}:oidc-provider/${OIDC}"
+                  ]
+                },
+                "Action": [
+                  "sts:AssumeRoleWithWebIdentity"
+                ],
+                "Condition": {
+                  "StringEquals": {
+                    "${OIDC}:sub": "system:serviceaccount:kube-system:aws-load-balancer-controller"
+                  }
+                }
+              }
+            ]
+          }
+        - OIDC: !Select [1, !Split ["//", !GetAtt EKSCluster.OpenIdConnectIssuerUrl]]
+      Path: /
+      Policies:
+        - PolicyDocument:
+            Version: '2012-10-17'
+            Statement:
+            - Effect: Allow
+              Action:
+              - iam:CreateServiceLinkedRole
+              Resource: "*"
+              Condition:
+                StringEquals:
+                  iam:AWSServiceName: elasticloadbalancing.amazonaws.com
+            - Effect: Allow
+              Action:
+              - ec2:DescribeAccountAttributes
+              - ec2:DescribeAddresses
+              - ec2:DescribeAvailabilityZones
+              - ec2:DescribeInternetGateways
+              - ec2:DescribeVpcs
+              - ec2:DescribeVpcPeeringConnections
+              - ec2:DescribeSubnets
+              - ec2:DescribeSecurityGroups
+              - ec2:DescribeInstances
+              - ec2:DescribeNetworkInterfaces
+              - ec2:DescribeTags
+              - ec2:GetCoipPoolUsage
+              - ec2:DescribeCoipPools
+              - ec2:GetSecurityGroupsForVpc
+              - ec2:DescribeIpamPools
+              - elasticloadbalancing:DescribeLoadBalancers
+              - elasticloadbalancing:DescribeLoadBalancerAttributes
+              - elasticloadbalancing:DescribeListeners
+              - elasticloadbalancing:DescribeListenerCertificates
+              - elasticloadbalancing:DescribeSSLPolicies
+              - elasticloadbalancing:DescribeRules
+              - elasticloadbalancing:DescribeTargetGroups
+              - elasticloadbalancing:DescribeTargetGroupAttributes
+              - elasticloadbalancing:DescribeTargetHealth
+              - elasticloadbalancing:DescribeTags
+              - elasticloadbalancing:DescribeTrustStores
+              - elasticloadbalancing:DescribeListenerAttributes
+              - elasticloadbalancing:DescribeCapacityReservation
+              Resource: "*"
+            - Effect: Allow
+              Action:
+              - cognito-idp:DescribeUserPoolClient
+              - acm:ListCertificates
+              - acm:DescribeCertificate
+              - iam:ListServerCertificates
+              - iam:GetServerCertificate
+              - waf-regional:GetWebACL
+              - waf-regional:GetWebACLForResource
+              - waf-regional:AssociateWebACL
+              - waf-regional:DisassociateWebACL
+              - wafv2:GetWebACL
+              - wafv2:GetWebACLForResource
+              - wafv2:AssociateWebACL
+              - wafv2:DisassociateWebACL
+              - shield:GetSubscriptionState
+              - shield:DescribeProtection
+              - shield:CreateProtection
+              - shield:DeleteProtection
+              Resource: "*"
+            - Effect: Allow
+              Action:
+              - ec2:AuthorizeSecurityGroupIngress
+              - ec2:RevokeSecurityGroupIngress
+              Resource: "*"
+            - Effect: Allow
+              Action:
+              - ec2:CreateSecurityGroup
+              Resource: "*"
+            - Effect: Allow
+              Action:
+              - ec2:CreateTags
+              Resource: arn:aws:ec2:*:*:security-group/*
+              Condition:
+                StringEquals:
+                  ec2:CreateAction: CreateSecurityGroup
+                'Null':
+                  aws:RequestTag/elbv2.k8s.aws/cluster: 'false'
+            - Effect: Allow
+              Action:
+              - ec2:CreateTags
+              - ec2:DeleteTags
+              Resource: arn:aws:ec2:*:*:security-group/*
+              Condition:
+                'Null':
+                  aws:RequestTag/elbv2.k8s.aws/cluster: 'true'
+                  aws:ResourceTag/elbv2.k8s.aws/cluster: 'false'
+            - Effect: Allow
+              Action:
+              - ec2:AuthorizeSecurityGroupIngress
+              - ec2:RevokeSecurityGroupIngress
+              - ec2:DeleteSecurityGroup
+              Resource: "*"
+              Condition:
+                'Null':
+                  aws:ResourceTag/elbv2.k8s.aws/cluster: 'false'
+            - Effect: Allow
+              Action:
+              - elasticloadbalancing:CreateLoadBalancer
+              - elasticloadbalancing:CreateTargetGroup
+              Resource: "*"
+              Condition:
+                'Null':
+                  aws:RequestTag/elbv2.k8s.aws/cluster: 'false'
+            - Effect: Allow
+              Action:
+              - elasticloadbalancing:CreateListener
+              - elasticloadbalancing:DeleteListener
+              - elasticloadbalancing:CreateRule
+              - elasticloadbalancing:DeleteRule
+              Resource: "*"
+            - Effect: Allow
+              Action:
+              - elasticloadbalancing:AddTags
+              - elasticloadbalancing:RemoveTags
+              Resource:
+              - arn:aws:elasticloadbalancing:*:*:targetgroup/*/*
+              - arn:aws:elasticloadbalancing:*:*:loadbalancer/net/*/*
+              - arn:aws:elasticloadbalancing:*:*:loadbalancer/app/*/*
+              Condition:
+                'Null':
+                  aws:RequestTag/elbv2.k8s.aws/cluster: 'true'
+                  aws:ResourceTag/elbv2.k8s.aws/cluster: 'false'
+            - Effect: Allow
+              Action:
+              - elasticloadbalancing:AddTags
+              - elasticloadbalancing:RemoveTags
+              Resource:
+              - arn:aws:elasticloadbalancing:*:*:listener/net/*/*/*
+              - arn:aws:elasticloadbalancing:*:*:listener/app/*/*/*
+              - arn:aws:elasticloadbalancing:*:*:listener-rule/net/*/*/*
+              - arn:aws:elasticloadbalancing:*:*:listener-rule/app/*/*/*
+            - Effect: Allow
+              Action:
+              - elasticloadbalancing:ModifyLoadBalancerAttributes
+              - elasticloadbalancing:SetIpAddressType
+              - elasticloadbalancing:SetSecurityGroups
+              - elasticloadbalancing:SetSubnets
+              - elasticloadbalancing:DeleteLoadBalancer
+              - elasticloadbalancing:ModifyTargetGroup
+              - elasticloadbalancing:ModifyTargetGroupAttributes
+              - elasticloadbalancing:DeleteTargetGroup
+              - elasticloadbalancing:ModifyListenerAttributes
+              - elasticloadbalancing:ModifyCapacityReservation
+              - elasticloadbalancing:ModifyIpPools
+              Resource: "*"
+              Condition:
+                'Null':
+                  aws:ResourceTag/elbv2.k8s.aws/cluster: 'false'
+            - Effect: Allow
+              Action:
+              - elasticloadbalancing:AddTags
+              Resource:
+              - arn:aws:elasticloadbalancing:*:*:targetgroup/*/*
+              - arn:aws:elasticloadbalancing:*:*:loadbalancer/net/*/*
+              - arn:aws:elasticloadbalancing:*:*:loadbalancer/app/*/*
+              Condition:
+                StringEquals:
+                  elasticloadbalancing:CreateAction:
+                  - CreateTargetGroup
+                  - CreateLoadBalancer
+                'Null':
+                  aws:RequestTag/elbv2.k8s.aws/cluster: 'false'
+            - Effect: Allow
+              Action:
+              - elasticloadbalancing:RegisterTargets
+              - elasticloadbalancing:DeregisterTargets
+              Resource: arn:aws:elasticloadbalancing:*:*:targetgroup/*/*
+            - Effect: Allow
+              Action:
+              - elasticloadbalancing:SetWebAcl
+              - elasticloadbalancing:ModifyListener
+              - elasticloadbalancing:AddListenerCertificates
+              - elasticloadbalancing:RemoveListenerCertificates
+              - elasticloadbalancing:ModifyRule
+              - elasticloadbalancing:SetRulePriorities
+              Resource: "*"
+          PolicyName: root
+      RoleName: "aws-load-balancer-controller-{{.Cluster.Name}}"
+    Type: 'AWS::IAM::Role'
+# {{- end }}
   RemoteFilesEncryptionKey:
     Type: "AWS::KMS::Key"
     Properties:

cluster/config-defaults.yaml

@@ -253,10 +253,6 @@ skipper_pod_deletion_cost_controller_resync_interval: "1h"
 # polarsignals - only enabled for testing teapot
 polarsignals_enabled: "false"
 
-# Emergency Access Service
-# Control whether the emergency access service is enabled or not.
-emergency_access_service_enabled: "false"
-
 # Kube-Metrics-Adapter
 ## Scheduled scaling metrics: ramp up/down over this period of time
 kube_metrics_adapter_default_scaling_window: "10m"
@@ -1288,3 +1284,7 @@ aws_vpc_cni_custom_networking: "false"
 aws_vpc_cni_enable_network_policy: "false"
 # specify the network policy enforcement mode.
 aws_vpc_cni_network_policy_enforcing_mode: "standard"
+
+# aws-load-balancer-controller resource settings
+aws_load_balancer_controller_cpu: "100m"
+aws_load_balancer_controller_mem_max: "4Gi"

cluster/manifests/02-admission-control/config.yaml

@@ -13,6 +13,9 @@ data:
   dns.default.subdomain-max-length: "{{ .Cluster.ConfigItems.subdomain_max_length }}"
 
   generic.prevent-write-operations.enable: "{{ .Cluster.ConfigItems.teapot_admission_controller_prevent_write_operations }}"
+{{- if and (eq .Cluster.Provider "zalando-eks") (eq .Cluster.ConfigItems.eks_ip_family "ipv6") }}
+  generic.inject-albc-defaults.enable: "true"
+{{- end }}
 
   pod.container-resource-control.min-memory-request: "25Mi"
   pod.container-resource-control.default-cpu-request: "{{ .Cluster.ConfigItems.teapot_admission_controller_default_cpu_request }}"

cluster/manifests/02-admission-control/deployment.yaml

@@ -33,7 +33,7 @@ spec:
       priorityClassName: system-cluster-critical
       containers:
       - name: admission-controller
-        image: 926694233939.dkr.ecr.eu-central-1.amazonaws.com/production_namespace/teapot/admission-controller:master-252
+        image: 926694233939.dkr.ecr.eu-central-1.amazonaws.com/production_namespace/teapot/admission-controller:master-254
         lifecycle:
           preStop:
             exec:

cluster/manifests/02-skipper-validation-webhook/deployment.yaml

@@ -32,7 +32,7 @@ spec:
       priorityClassName: system-cluster-critical
       containers:
       - name: skipper-admission-webhook
-        image: 926694233939.dkr.ecr.eu-central-1.amazonaws.com/production_namespace/teapot/skipper:v0.22.21
+        image: 926694233939.dkr.ecr.eu-central-1.amazonaws.com/production_namespace/teapot/skipper:v0.22.23
         args:
           - webhook
           - --address=:9085

cluster/manifests/03-kube-aws-iam-controller/deployment.yaml

@@ -27,7 +27,7 @@ spec:
       hostNetwork: true
       containers:
       - name: kube-aws-iam-controller
-        image: container-registry.zalando.net/teapot/kube-aws-iam-controller:v0.3.0-76-g1a28806
+        image: container-registry.zalando.net/teapot/kube-aws-iam-controller:v0.3.0-78-gfb82543
         env:
         - name: AWS_DEFAULT_REGION
           value: "{{.Cluster.Region}}"

cluster/manifests/04-ebs-csi/controller.yaml

@@ -91,7 +91,7 @@ spec:
             allowPrivilegeEscalation: false
             readOnlyRootFilesystem: true
         - name: csi-provisioner
-          image: container-registry.zalando.net/teapot/external-provisioner:v5.1.0-eks-1-31-10-master-28
+          image: container-registry.zalando.net/teapot/external-provisioner:v5.1.0-eks-1-31-10-master-29
           args:
             - --csi-address=$(ADDRESS)
             - --v=2
@@ -116,7 +116,7 @@ spec:
             allowPrivilegeEscalation: false
             readOnlyRootFilesystem: true
         - name: csi-attacher
-          image: container-registry.zalando.net/teapot/external-attacher:v4.7.0-eks-1-31-10-master-28
+          image: container-registry.zalando.net/teapot/external-attacher:v4.7.0-eks-1-31-10-master-29
           args:
             - --csi-address=$(ADDRESS)
             - --v=2

cluster/manifests/04-ebs-csi/node.yaml

@@ -77,7 +77,7 @@ spec:
             privileged: true
             readOnlyRootFilesystem: true
         - name: node-driver-registrar
-          image: container-registry.zalando.net/teapot/node-driver-registrar:v2.12.0-eks-1-31-10-master-28
+          image: container-registry.zalando.net/teapot/node-driver-registrar:v2.12.0-eks-1-31-10-master-29
           args:
             - --csi-address=$(ADDRESS)
             - --kubelet-registration-path=$(DRIVER_REG_SOCK_PATH)
@@ -114,7 +114,7 @@ spec:
             allowPrivilegeEscalation: false
             readOnlyRootFilesystem: true
         - name: liveness-probe
-          image: container-registry.zalando.net/teapot/livenessprobe:v2.14.0-eks-1-31-10-master-27
+          image: container-registry.zalando.net/teapot/livenessprobe:v2.14.0-eks-1-31-10-master-29
           args:
             - --csi-address=/csi/csi.sock
           volumeMounts:

cluster/manifests/aws-load-balancer-controller/deployment.yaml

@@ -0,0 +1,78 @@
+# {{- if and (eq .Cluster.Provider "zalando-eks") (eq .Cluster.ConfigItems.eks_ip_family "ipv6") }}
+apiVersion: apps/v1
+kind: Deployment
+metadata:
+  name: aws-load-balancer-controller
+  namespace: kube-system
+  labels:
+    application: kubernetes
+    component: aws-load-balancer-controller
+spec:
+  replicas: 2
+  selector:
+    matchLabels:
+      deployment: aws-load-balancer-controller
+  template:
+    metadata:
+      labels:
+        application: kubernetes
+        component: aws-load-balancer-controller
+        deployment: aws-load-balancer-controller
+    spec:
+      containers:
+      - args:
+        - "--aws-region={{.Cluster.Region}}"
+        - "--aws-vpc-id={{.Cluster.ConfigItems.vpc_id}}"
+        - "--cluster-name={{.Cluster.Name}}"
+        - --feature-gates=ServiceTypeLoadBalancerOnly=true
+        - --webhook-cert-file=admission-controller.pem
+        - --webhook-key-file=admission-controller-key.pem
+        - --default-load-balancer-scheme=internet-facing
+        - --default-target-type=ip
+        image: container-registry.zalando.net/teapot/aws-load-balancer-controller:v2.12.0-main-2.patched
+        livenessProbe:
+          failureThreshold: 2
+          httpGet:
+            path: /healthz
+            port: 61779
+            scheme: HTTP
+          initialDelaySeconds: 30
+          timeoutSeconds: 10
+        name: controller
+        ports:
+        - containerPort: 9443
+          name: webhook-server
+          protocol: TCP
+        resources:
+          limits:
+            cpu: {{ .Cluster.ConfigItems.aws_load_balancer_controller_cpu }}
+            memory: {{ .Cluster.ConfigItems.aws_load_balancer_controller_mem_max }}
+          requests:
+            cpu: {{ .Cluster.ConfigItems.aws_load_balancer_controller_cpu }}
+            memory: {{ .Cluster.ConfigItems.aws_load_balancer_controller_mem_max }}
+        securityContext:
+          allowPrivilegeEscalation: false
+          readOnlyRootFilesystem: true
+          runAsNonRoot: true
+        volumeMounts:
+        - mountPath: /tmp/k8s-webhook-server/serving-certs
+          name: cert
+          readOnly: true
+      priorityClassName: system-cluster-critical
+      securityContext:
+        fsGroup: 1337
+      serviceAccountName: aws-load-balancer-controller
+      terminationGracePeriodSeconds: 10
+      affinity:
+        podAntiAffinity:
+          requiredDuringSchedulingIgnoredDuringExecution:
+          - topologyKey: topology.kubernetes.io/zone
+            labelSelector:
+              matchLabels:
+                deployment: aws-load-balancer-controller
+      volumes:
+      - name: cert
+        secret:
+          defaultMode: 420
+          secretName: admission-controller-tls-certs
+# {{- end }}