Make it clear that only EKS works and prevent unmanaged clusters from using ipv6

Author: Skarlso

api/v1beta1/awscluster_webhook.go

@@ -56,6 +56,7 @@ func (r *AWSCluster) ValidateCreate() error {
 	allErrs = append(allErrs, r.validateSSHKeyName()...)
 	allErrs = append(allErrs, r.Spec.AdditionalTags.Validate()...)
 	allErrs = append(allErrs, r.Spec.S3Bucket.Validate()...)
+	allErrs = append(allErrs, r.validateNetwork()...)
 
 	return aggregateObjErrors(r.GroupVersionKind().GroupKind(), r.Name, allErrs)
 }
@@ -178,3 +179,11 @@ func (r *AWSCluster) Default() {
 func (r *AWSCluster) validateSSHKeyName() field.ErrorList {
 	return validateSSHKeyName(r.Spec.SSHKeyName)
 }
+
+func (r *AWSCluster) validateNetwork() field.ErrorList {
+	var allErrs field.ErrorList
+	if r.Spec.NetworkSpec.VPC.IsIPv6Enabled() {
+		allErrs = append(allErrs, field.Invalid(field.NewPath("ipv6"), r.Spec.NetworkSpec.VPC.IPv6, "IPv6 cannot be used with unmanaged clusters at this time."))
+	}
+	return allErrs
+}

api/v1beta1/awscluster_webhook_test.go

@@ -222,6 +222,22 @@ func TestAWSCluster_ValidateCreate(t *testing.T) {
 			},
 			wantErr: false,
 		},
+		{
+			name: "rejects ipv6",
+			cluster: &AWSCluster{
+				Spec: AWSClusterSpec{
+					NetworkSpec: NetworkSpec{
+						VPC: VPCSpec{
+							IPv6: &IPv6{
+								CidrBlock: "2001:2345:5678::/64",
+								PoolID:    "pool-id",
+							},
+						},
+					},
+				},
+			},
+			wantErr: true,
+		},
 	}
 	for _, tt := range tests {
 		t.Run(tt.name, func(t *testing.T) {

controlplane/eks/api/v1beta1/awsmanagedcontrolplane_webhook.go

@@ -388,7 +388,7 @@ func (r *AWSManagedControlPlane) validateNetwork() field.ErrorList {
 	var allErrs field.ErrorList
 
 	if r.Spec.NetworkSpec.VPC.IsIPv6Enabled() && r.Spec.NetworkSpec.VPC.IPv6.CidrBlock != "" && r.Spec.NetworkSpec.VPC.IPv6.PoolID == "" {
-		poolField := field.NewPath("spec", "networkSpec", "vpc", "poolId")
+		poolField := field.NewPath("spec", "networkSpec", "vpc", "ipv6", "poolId")
 		allErrs = append(allErrs, field.Invalid(poolField, r.Spec.NetworkSpec.VPC.IPv6.PoolID, "poolId cannot be empty if cidrBlock is set"))
 	}
 

docs/book/src/topics/eks/ipv6-enabled-cluster.md

@@ -83,179 +83,4 @@ address range of `fc00::/7`.
 
 ## Unmanaged Clusters
 
-Now comes the tricky part. If you wish, it's possible to set up IPv6 with unmanaged clusters, however, that requires
-a lot of extra manual steps and some extra configuration settings.
-
-_Note_: I DO NOT recommend doing this in production. AWS provides a more robust and easier approach on doing IPv6.
-This approach is brittle and has many manual steps that need to be performed in order to get things working.
-
-### Extra Config
-
-The extra configs are on the kubeadm side. These are `node-ip` and `bind-address`. These need to be set as follows:
-
-```yaml
-  kubeadmConfigSpec:
-    initConfiguration:
-      nodeRegistration:
-        name: '{{ ds.meta_data.local_hostname }}'
-        kubeletExtraArgs:
-          cloud-provider: aws
-          node-ip: '::'
-    clusterConfiguration:
-      apiServer:
-        extraArgs:
-          cloud-provider: aws
-          bind-address: '::'
-      controllerManager:
-        extraArgs:
-          cloud-provider: aws
-          bind-address: '::'
-      scheduler:
-        extraArgs:
-          bind-address: '::'
-    joinConfiguration:
-      nodeRegistration:
-        name: '{{ ds.meta_data.local_hostname }}'
-        kubeletExtraArgs:
-          node-ip: '::'
-          cloud-provider: aws
-
-```
-
-This will tell kubeadm to bind to a specific address type which should be IPv6.
-
-Next, it's pod CIDRs and service CIDRs. This is a bit more tricky. You need to know your IPv6 CIDR beforehand.
-Having your own IPv6 pool is most of the time, impractical. But there is a way to get you started up quickly
-and with low effort. You can ask CAPA to create the network topology for you with a simple cluster config such
-as this one:
-
-```yaml
----
-apiVersion: cluster.x-k8s.io/v1beta1
-kind: Cluster
-metadata:
-  name: "ipv6-network-topology"
-spec:
-  infrastructureRef:
-    apiVersion: infrastructure.cluster.x-k8s.io/v1beta1
-    kind: AWSCluster
-    name: "ipv6-network-topology"
-  controlPlaneRef:
-    kind: KubeadmControlPlane
-    apiVersion: controlplane.cluster.x-k8s.io/v1beta1
-    name: "ipv6-network-topology-control-plane"
----
-apiVersion: infrastructure.cluster.x-k8s.io/v1beta1
-kind: AWSCluster
-metadata:
-  name: "ipv6-network-topology"
-spec:
-  network:
-    vpc:
-      ipv6: {}
-  region: "eu-central-1"
-```
-
-This will create a VPC with proper load-balancing and elastic ips and everything. This can be fine-tuned as
-desired. This is the bare minimum where CAPA creates the whole topology.
-
-Once this is done, we can acquire the IPv6 CIDR, and we can continue by using the vpc id and subnets in the
-unmanaged setting like this:
-
-```yaml
----
-apiVersion: cluster.x-k8s.io/v1beta1
-kind: Cluster
-metadata:
-  name: "unmanaged-ipv6"
-spec:
-  clusterNetwork:
-    services:
-      cidrBlocks: ["192.168.0.0/16", "2a05:d014:852:f::/112"]
-    pods:
-      cidrBlocks: ["192.168.0.0/16", "2a05:d014:852:f::/56"]
-  infrastructureRef:
-    apiVersion: infrastructure.cluster.x-k8s.io/v1beta1
-    kind: AWSCluster
-    name: "unmanaged-ipv6"
-  controlPlaneRef:
-    kind: KubeadmControlPlane
-    apiVersion: controlplane.cluster.x-k8s.io/v1beta1
-    name: "test-ipv6-unmanaged-2-control-plane"
----
-apiVersion: infrastructure.cluster.x-k8s.io/v1beta1
-kind: AWSCluster
-metadata:
-  name: "unmanaged-ipv6"
-spec:
-  network:
-    subnets:
-    - id: "subnet-0812d970f75867a72"
-    - id: "subnet-0aa534118f62cab86"
-    - id: "subnet-0380be0501f2c8bb0"
-    - id: "subnet-09d083492229f6281"
-    - id: "subnet-0b12a78a3b2cebdec"
-    - id: "subnet-0a0b33746595ecc89"
-    vpc:
-      id: "vpc-024ae81c0ca3b7209"
-      ipv6: {}
-  region: "eu-central-1"
----
-```
-
-### Cilium
-
-Since we are on an unmanaged cluster, we need a cni installed. We'll use Cilium as an example. There are two settings
-that are needed for cilium to work. (3 really, you also have to set up ipv4 cidr to the proper cidr the VPC provides).
-
-These are:
-
-```
---cluster-pool-ipv6-cidr="2a05:d014:852:f02::/112"
---cluster-pool-ipv6-cidr-size="128"
-```
-
-Note that Cilium doesn't allow any CIDR above `112`. So the CIDR you've got from AWS with size `64` needs to be cut down
-to `112`.
-
-Once this is done, we can install Cilium into the workload cluster and restart all Pods so they can acquire IPv6
-addresses.
-
-### Calico
-
-Another approach is to use Calico. Calico has detailed guides on how to set up IPv6 located [here](https://projectcalico.docs.tigera.io/networking/ipv6) and [here](https://projectcalico.docs.tigera.io/networking/ipv6-control-plane).
-
-You can use CAPA to bootstrap Calico in the following way:
-
-- Create a ClusterResourceSet like this:
-```yaml
-apiVersion: addons.cluster.x-k8s.io/v1alpha3
-kind: ClusterResourceSet
-metadata:
- name: crs1
- namespace: default
-spec:
- mode: "ApplyOnce"
- clusterSelector:
-   matchLabels:
-     cni: calico
- resources:
-   - name: db-secret
-     kind: Secret
-   - name: calico-addon
-     kind: ConfigMap
-```
-- Download the latest Calico manifest and set up the required properties for IPv6 as the guides suggest ( you will already 
-  need to have an IPv6 CIDR )
-- Create a config map in the control plane with the following command: `kubectl create configmap calico-addon --from-file=calico.yaml`
-- Tag your cluster with the label `cni: calico` so cluster-api can find it and install the cni addon
-```yaml
-apiVersion: cluster.x-k8s.io/v1beta1
-kind: Cluster
-metadata:
-  name: "test-ipv6-unmanaged-2"
-  labels:
-    cni: calico
-```
-- Apply and monitor
-- Note that only new pods will get an ipv6 address; existing pods will remain using ipv4
+Unmanaged clusters are not supported at this time.

docs/proposal/20220718-ipv6.md

@@ -1,21 +1,21 @@
 ---
-title: Proposal Template
+title: IPv6 for EKS
 authors:
   - @Skarlso
   - @nikimanoledaki
   - @richardcase
 reviewers:
   - "@richardcase"
 creation-date: 2022-04-28
-last-updated: 2022-07-19
+last-updated: 2022-08-23
 status: provisional
 ---
 
-# IPv6 Support in CAPA
+# IPv6 Support in CAPA for EKS
 
 ## Table of Contents
 
-- [IPv6 Support in CAPA](#ipv6-support-in-capa)
+- [IPv6 Support in CAPA](#ipv6-support-in-capa-for-eks)
   - [Table of Contents](#table-of-contents)
   - [Glossary](#glossary)
   - [Summary](#summary)
@@ -26,6 +26,7 @@ status: provisional
   - [Non-Goals/Future Work](#non-goalsfuture-work)
   - [Proposal](#proposal)
     - [Plan](#plan)
+      - [Managed and Unmanaged clusters](#managed-and-unmanaged-clusters)
       - [Additions and Configuration changes](#additions-and-configuration-changes)
       - [Networking and Subnet Splitting strategies](#networking-and-subnet-splitting-strategies)
       - [vpc-cni](#vpc-cni)
@@ -54,9 +55,9 @@ communication between old services is still functioning. Only IPv6 is not suppor
 
 ## Summary
 
-This proposal defines how to implement IPv6 for clusters in CAPA. It defines various validations that need to take place
-in order to properly inform the user when IPv6 can be used. It defines components which need to be created and set up.
-It also details with examples and images how the architecture looks like using IPv6.
+This proposal defines how to implement IPv6 for clusters in CAPA for EKS. It defines various validations that need to
+take place in order to properly inform the user when IPv6 can be used. It defines components which need to be created
+and set up. It also details with examples and images how the architecture looks like using IPv6 in EKS.
 
 ## Motivation
 
@@ -80,7 +81,7 @@ limitations. Now users can run as many pods as their instances CPU and RAM capac
 
 ## Goals
 
-- Create a cluster with IPv6 networking features for new clusters created with k8s v1.21+
+- Create a cluster with IPv6 networking features for new clusters created with k8s v1.21+ on EKS
 - Dual-stack (IPv4+IPv6) VPC, subnets and EC2 instances/nodes
 - Allow users to set their own VPC in config
 - Allow users to create VPC with own IPv6 CIDR
@@ -91,6 +92,7 @@ limitations. Now users can run as many pods as their instances CPU and RAM capac
 ## Non-Goals/Future Work
 
 - IPv6-only VPC
+- Unmanaged clusters
 - Migrate to IPv6 after cluster creation ( means that reconciliation will not update existing cluster to use ipv6 )
 - Make IPv6 the default IP family
 - Support k8s version that are `< 1.21`
@@ -101,17 +103,22 @@ limitations. Now users can run as many pods as their instances CPU and RAM capac
 
 ### Plan
 
-Newly created clusters should be able to support IPv6 based communication throughout the entire cluster and in addition,
-to the outside world via exposed services. The pods should have IPv6 addresses but should be able to contact AWS metadata
-service using IPv4. A mixed communication is preferred as fully IPv6 clusters are not supported yet. Note, AWS does
-provide an IPv6 metadata service under `fd00:ec2::254` well-known address.
+Newly created clusters backed on EKS should be able to support IPv6 based communication throughout the entire cluster
+and in addition, to the outside world via exposed services. The pods should have IPv6 addresses but should be able to
+contact AWS metadata service using IPv4. A mixed communication is preferred as fully IPv6 clusters are not supported yet
+using EKS. Note, AWS does provide an IPv6 metadata service under `fd00:ec2::254` well-known address.
 
-### Managed and Unmanaged clusters
+#### Managed and Unmanaged clusters
 
-For managed clusters the described approach below will work. For unmanaged clusters there is a lot more that needs to be
-done, but all the things that do need to be done are actually manual steps from users. The code will support it both.
+After careful considering and a lot of debugging and back and forth, we decided that unmanaged clusters will not be
+supported at this time. It will come at a later date. The implementation as it stands, allows for unmanaged clusters to
+work with ipv6 ( once the validation is removed from `AWSCluster` ) but the circumstances regarding getting the nodes
+to work and kubeadm to play nicely are difficult to pinpoint.
 
-The details on how to create an IPv6 enabled unmanaged cluster can be found in [ipv6-enabled-cluster.md](../book/src/topics/eks/ipv6-enabled-cluster.md).
+Nevertheless, a sample template can be found under ![template](../../templates/cluster-template-ipv6.yaml). This
+represents a possible combination of configuration objects that kubeadm requires.
+
+A validation is added to prevent unmanaged clusters from being able to use IPv6 specific configurations.
 
 #### Additions and Configuration changes
 
@@ -253,6 +260,7 @@ The following validations need to be applied:
 - Cluster version must be 1.21 or higher
 - Addon version of CNI must be 1.10 or higher in case of IPv6
 - Possibly validate ( if we don't set it automatically ) that the right environment properties are set for vpc-cni
+- Prevent unmanaged clusters from using IPv6 settings
 
 #### Instance Type
 
@@ -326,10 +334,10 @@ connectivity works such as, but not limited to:
 ## User Stories
 
 As a CAPA user:
-- I can create a cluster that is in a new IPv6 & IPv4 dual-stack VPC
+- I can create a cluster using EKS that is in a new IPv6 & IPv4 dual-stack VPC
 - I can create a nodegroup which completely supports IPv6 CIDR
 - I can bring my own IPv6 subnet and create a nodegroup with that
-- I can create infrastructure using an IPv6 & IPv4 dual-stack VPC
+- I can create infrastructure on EKS using an IPv6 & IPv4 dual-stack VPC
 
 ## Security Model
 
@@ -345,7 +353,6 @@ No other alternatives.
 - [x] 04/28/2022: Proposed idea in an issue or [community meeting]
 - [x] 04/28/2022: Compile a Google Doc following the CAEP template (link here)
 - [x] 08/06/2022: Open proposal PR
-- [ ] MM/DD/YYYY: First round of feedback from community
-- [ ] MM/DD/YYYY: Present proposal at a [community meeting]
+- [x] 08/20/2022: First round of feedback from community
 
 <!-- Links -->

templates/cluster-template-ipv6.yaml

@@ -5,8 +5,10 @@ metadata:
   name: "${CLUSTER_NAME}"
 spec:
   clusterNetwork:
+    services:
+      cidrBlocks: ["2a05:d014:2d3:4200::/112"]
     pods:
-      cidrBlocks: ["192.168.0.0/16"]
+      cidrBlocks: ["2a05:d014:2d3:4200::/56"]
   infrastructureRef:
     apiVersion: infrastructure.cluster.x-k8s.io/v1beta1
     kind: AWSCluster
@@ -42,6 +44,7 @@ spec:
         kubeletExtraArgs:
           cloud-provider: aws
           node-ip: '::'
+          cluster-dns: '2a05:d014:2d3:4200::/56'
     clusterConfiguration:
       apiServer:
         extraArgs:
@@ -51,6 +54,7 @@ spec:
         extraArgs:
           cloud-provider: aws
           bind-address: '::'
+          cluster-cidr: "2a05:d014:2d3:4200::/56"
       scheduler:
         extraArgs:
           bind-address: '::'
@@ -59,6 +63,7 @@ spec:
         name: '{{ ds.meta_data.local_hostname }}'
         kubeletExtraArgs:
           node-ip: '::'
+          cluster-dns: "2a05:d014:2d3:4200::/56"
           cloud-provider: aws
   version: "${KUBERNETES_VERSION}"
 ---