Merge pull request #8510 from zalando-incubator/eks-custom-networking

eks: Optional support for AWS VPC CNI Custom Networking

Author: mikkeloscar

cluster/config-defaults.yaml

@@ -1272,6 +1272,10 @@ eks_fis_namespaces: "default"
 
 # prefix delegation can only be configured for ipv4. For ipv6 it can only be true.
 aws_vpc_cni_prefix_delegation: "false"
+# enable custom networking for the AWS VPC CNI. This assumes that a custom CIDR
+# range is available in the VPC and that there are dedicated subnets for the
+# custom CIDR.
+aws_vpc_cni_custom_networking: "false"
 # enable network policy enforcement in the cluster.
 aws_vpc_cni_enable_network_policy: "false"
 # specify the network policy enforcement mode.

cluster/manifests/01-aws-node/daemonset.yaml

@@ -55,7 +55,11 @@ spec:
         - name: AWS_VPC_ENI_MTU
           value: "9001"
         - name: AWS_VPC_K8S_CNI_CUSTOM_NETWORK_CFG
-          value: "false"
+          value: "{{ .Cluster.ConfigItems.aws_vpc_cni_custom_networking }}"
+        # {{ if eq .Cluster.ConfigItems.aws_vpc_cni_custom_networking "true" }}
+        - name: ENI_CONFIG_LABEL_DEF
+          value: topology.kubernetes.io/zone
+        # {{ end }}
         - name: AWS_VPC_K8S_CNI_EXTERNALSNAT
           value: "false"
         - name: AWS_VPC_K8S_CNI_LOGLEVEL

cluster/manifests/01-aws-node/pod_subnets.yaml

@@ -0,0 +1,24 @@
+# {{ if eq .Cluster.Provider "zalando-eks" }}
+# {{ if eq .Cluster.ConfigItems.aws_vpc_cni_custom_networking "true" }}
+# {{ with $data := . }}
+# {{ with $azCount := len $data.Values.availability_zones }}
+# {{ range $az := $data.Values.availability_zones }}
+# {{ with $azID := azID $az }}
+---
+apiVersion : crd.k8s.amazonaws.com/v1alpha1
+kind : ENIConfig
+metadata:
+  name: "{{$az}}"
+  labels:
+    application: kubernetes
+    component: aws-node
+spec:
+  securityGroups:
+    - {{ $data.Values.ClusterStackOutputs.EKSWorkerSecurityGroup }}
+  subnet: "{{ index $data.Values.pod_subnets $az }}"
+# {{end}}
+# {{end}}
+# {{end}}
+# {{end}}
+# {{end}}
+# {{end}}

cluster/manifests/deletions.yaml

@@ -415,6 +415,14 @@ post_apply:
   kind: ServiceAccount
   namespace: kube-system
 {{- end}}
+{{- if eq .Cluster.Provider "zalando-eks"}}
+{{- if ne .Cluster.ConfigItems.aws_vpc_cni_custom_networking "true"}}
+- kind: ENIConfig
+  labels:
+    application: kubernetes
+    component: aws-node
+{{- end }}
+{{- end }}
 {{- if ne .Cluster.ConfigItems.emergency_access_service_enabled "true" }}
 - name: emergency-access-service
   kind: Deployment