Add dualstack e2e test

killianmuldoon · killianmuldoon · commit 68213cff63f1 · 2023-05-15T15:46:30.000+01:00
Signed-off-by: killianmuldoon &lt;kmuldoon@vmware.com&gt;
diff --git a/Makefile b/Makefile
@@ -513,6 +513,8 @@ generate-e2e-templates-main: $(KUSTOMIZE)
 	$(KUSTOMIZE) build $(DOCKER_TEMPLATES)/main/cluster-template-upgrades-runtimesdk --load-restrictor LoadRestrictionsNone > $(DOCKER_TEMPLATES)/main/cluster-template-upgrades-runtimesdk.yaml
 	$(KUSTOMIZE) build $(DOCKER_TEMPLATES)/main/cluster-template-kcp-scale-in --load-restrictor LoadRestrictionsNone > $(DOCKER_TEMPLATES)/main/cluster-template-kcp-scale-in.yaml
 	$(KUSTOMIZE) build $(DOCKER_TEMPLATES)/main/cluster-template-ipv6 --load-restrictor LoadRestrictionsNone > $(DOCKER_TEMPLATES)/main/cluster-template-ipv6.yaml
+	$(KUSTOMIZE) build $(DOCKER_TEMPLATES)/main/cluster-template-topology-dualstack-ipv6-primary --load-restrictor LoadRestrictionsNone > $(DOCKER_TEMPLATES)/main/cluster-template-topology-dualstack-ipv6-primary.yaml
+	$(KUSTOMIZE) build $(DOCKER_TEMPLATES)/main/cluster-template-topology-dualstack-ipv4-primary --load-restrictor LoadRestrictionsNone > $(DOCKER_TEMPLATES)/main/cluster-template-topology-dualstack-ipv4-primary.yaml
 	$(KUSTOMIZE) build $(DOCKER_TEMPLATES)/main/cluster-template-topology-single-node-cluster --load-restrictor LoadRestrictionsNone > $(DOCKER_TEMPLATES)/main/cluster-template-topology-single-node-cluster.yaml
 	$(KUSTOMIZE) build $(DOCKER_TEMPLATES)/main/cluster-template-topology --load-restrictor LoadRestrictionsNone > $(DOCKER_TEMPLATES)/main/cluster-template-topology.yaml
 	$(KUSTOMIZE) build $(DOCKER_TEMPLATES)/main/cluster-template-ignition --load-restrictor LoadRestrictionsNone > $(DOCKER_TEMPLATES)/main/cluster-template-ignition.yaml
diff --git a/bootstrap/kubeadm/internal/controllers/kubeadmconfig_controller.go b/bootstrap/kubeadm/internal/controllers/kubeadmconfig_controller.go
@@ -50,6 +50,7 @@ import (
 	"sigs.k8s.io/cluster-api/controllers/remote"
 	expv1 "sigs.k8s.io/cluster-api/exp/api/v1beta1"
 	"sigs.k8s.io/cluster-api/feature"
+	"sigs.k8s.io/cluster-api/internal/util/taints"
 	"sigs.k8s.io/cluster-api/util"
 	"sigs.k8s.io/cluster-api/util/annotations"
 	"sigs.k8s.io/cluster-api/util/conditions"
@@ -555,7 +556,7 @@ func (r *KubeadmConfigReconciler) joinWorker(ctx context.Context, scope *Scope)
 	// Do not modify the KubeadmConfig in etcd as this is a temporary taint that will be dropped after the node
 	// is initialized by ClusterAPI.
 	joinConfiguration := scope.Config.Spec.JoinConfiguration.DeepCopy()
-	if !hasTaint(joinConfiguration.NodeRegistration.Taints, clusterv1.NodeUninitializedTaint) {
+	if !taints.HasTaint(joinConfiguration.NodeRegistration.Taints, clusterv1.NodeUninitializedTaint) {
 		joinConfiguration.NodeRegistration.Taints = append(joinConfiguration.NodeRegistration.Taints, clusterv1.NodeUninitializedTaint)
 	}
 
@@ -1074,12 +1075,3 @@ func (r *KubeadmConfigReconciler) ensureBootstrapSecretOwnersRef(ctx context.Con
 	}
 	return nil
 }
-
-func hasTaint(taints []corev1.Taint, targetTaint corev1.Taint) bool {
-	for _, taint := range taints {
-		if taint.MatchTaint(&targetTaint) {
-			return true
-		}
-	}
-	return false
-}
diff --git a/internal/util/taints/taints.go b/internal/util/taints/taints.go
@@ -36,3 +36,13 @@ func RemoveNodeTaint(node *corev1.Node, drop corev1.Taint) bool {
 	node.Spec.Taints = taints
 	return droppedTaint
 }
+
+// HasTaint returns true if the targetTaint is in the list of taints.
+func HasTaint(taints []corev1.Taint, targetTaint corev1.Taint) bool {
+	for _, taint := range taints {
+		if taint.MatchTaint(&targetTaint) {
+			return true
+		}
+	}
+	return false
+}
diff --git a/test/e2e/config/docker.yaml b/test/e2e/config/docker.yaml
@@ -273,6 +273,8 @@ providers:
     - sourcePath: "../data/infrastructure-docker/main/cluster-template-upgrades-runtimesdk.yaml"
     - sourcePath: "../data/infrastructure-docker/main/cluster-template-kcp-scale-in.yaml"
     - sourcePath: "../data/infrastructure-docker/main/cluster-template-ipv6.yaml"
+    - sourcePath: "../data/infrastructure-docker/main/cluster-template-topology-dualstack-ipv6-primary.yaml"
+    - sourcePath: "../data/infrastructure-docker/main/cluster-template-topology-dualstack-ipv4-primary.yaml"
     - sourcePath: "../data/infrastructure-docker/main/cluster-template-topology-single-node-cluster.yaml"
     - sourcePath: "../data/infrastructure-docker/main/cluster-template-topology.yaml"
     - sourcePath: "../data/infrastructure-docker/main/cluster-template-ignition.yaml"
@@ -300,7 +302,7 @@ variables:
   ETCD_VERSION_UPGRADE_TO: "3.5.7-0"
   COREDNS_VERSION_UPGRADE_TO: "v1.10.1"
   DOCKER_SERVICE_DOMAIN: "cluster.local"
-  IP_FAMILY: "IPv4"
+  IP_FAMILY: "dual"
   DOCKER_SERVICE_CIDRS: "10.128.0.0/12"
   DOCKER_POD_CIDRS: "192.168.0.0/16"
   DOCKER_SERVICE_IPV6_CIDRS: "fd00:100:64::/108"
diff --git a/test/e2e/data/cni/kindnet/kindnet.yaml b/test/e2e/data/cni/kindnet/kindnet.yaml
@@ -76,8 +76,10 @@ spec:
               valueFrom:
                 fieldRef:
                   fieldPath: status.podIP
+            # We're using the dualstack CIDRs here. The order doesn't matter for kindnet as the loops are run concurrently.
+            # REF: https://github.com/kubernetes-sigs/kind/blob/3dbeb894e3092a336ab4278d3823e73a1d66aff7/images/kindnetd/cmd/kindnetd/main.go#L149-L175
             - name: POD_SUBNET
-              value: '${DOCKER_POD_CIDRS}'
+              value: '${DOCKER_POD_CIDRS},${DOCKER_POD_IPV6_CIDRS}'
           volumeMounts:
             - name: cni-cfg
               mountPath: /etc/cni/net.d
diff --git a/test/e2e/data/infrastructure-docker/main/cluster-template-topology-dualstack-ipv4-primary/cluster.yaml b/test/e2e/data/infrastructure-docker/main/cluster-template-topology-dualstack-ipv4-primary/cluster.yaml
@@ -0,0 +1,22 @@
+---
+apiVersion: cluster.x-k8s.io/v1beta1
+kind: Cluster
+metadata:
+  name: '${CLUSTER_NAME}'
+spec:
+  topology:
+    class: quick-start
+    variables:
+       - name: ipv6Primary
+         value: false
+       - name: externalCloudProvider
+         value: true
+  clusterNetwork:
+    services:
+      cidrBlocks:
+        - '${DOCKER_SERVICE_CIDRS}'
+        - '${DOCKER_SERVICE_IPV6_CIDRS}'
+    pods:
+      cidrBlocks:
+        - '${DOCKER_POD_CIDRS}'
+        - '${DOCKER_POD_IPV6_CIDRS}'
diff --git a/test/e2e/data/infrastructure-docker/main/cluster-template-topology-dualstack-ipv4-primary/kustomization.yaml b/test/e2e/data/infrastructure-docker/main/cluster-template-topology-dualstack-ipv4-primary/kustomization.yaml
@@ -0,0 +1,6 @@
+bases:
+  - ../bases/cluster-with-topology.yaml
+  - ../bases/crs.yaml
+
+patches:
+  - cluster.yaml
diff --git a/test/e2e/data/infrastructure-docker/main/cluster-template-topology-dualstack-ipv6-primary/cluster.yaml b/test/e2e/data/infrastructure-docker/main/cluster-template-topology-dualstack-ipv6-primary/cluster.yaml
@@ -0,0 +1,22 @@
+---
+apiVersion: cluster.x-k8s.io/v1beta1
+kind: Cluster
+metadata:
+  name: '${CLUSTER_NAME}'
+spec:
+  topology:
+    class: quick-start
+    variables:
+       - name: ipv6Primary
+         value: true
+       - name: externalCloudProvider
+         value: true
+  clusterNetwork:
+    services:
+      cidrBlocks:
+        - '${DOCKER_SERVICE_IPV6_CIDRS}'
+        - '${DOCKER_SERVICE_CIDRS}'
+    pods:
+      cidrBlocks:
+        - '${DOCKER_POD_IPV6_CIDRS}'
+        - '${DOCKER_POD_CIDRS}'
diff --git a/test/e2e/data/infrastructure-docker/main/cluster-template-topology-dualstack-ipv6-primary/kustomization.yaml b/test/e2e/data/infrastructure-docker/main/cluster-template-topology-dualstack-ipv6-primary/kustomization.yaml
@@ -0,0 +1,6 @@
+bases:
+  - ../bases/cluster-with-topology.yaml
+  - ../bases/crs.yaml
+
+patches:
+  - cluster.yaml
diff --git a/test/e2e/data/infrastructure-docker/main/clusterclass-quick-start.yaml b/test/e2e/data/infrastructure-docker/main/clusterclass-quick-start.yaml
@@ -100,6 +100,18 @@ spec:
       openAPIV3Schema:
         type: boolean
         default: true
+  - name: externalCloudProvider
+    required: false
+    schema:
+      openAPIV3Schema:
+        type: boolean
+        default: false
+  - name: ipv6Primary
+    required: false
+    schema:
+      openAPIV3Schema:
+        type: boolean
+        default: false
   patches:
   - name: lbImageRepository
     definitions:
@@ -256,6 +268,54 @@ spec:
       - op: add
         path: "/spec/template/spec/kubeadmConfigSpec/joinConfiguration/nodeRegistration/taints"
         value: []
+  - name: controlPlaneExternalCloudProvider
+    enabledIf: "{{ .externalCloudProvider }}"
+    description: "Configures kubelet to run with an external cloud provider for control plane nodes."
+    definitions:
+      - selector:
+          apiVersion: controlplane.cluster.x-k8s.io/v1beta1
+          kind: KubeadmControlPlaneTemplate
+          matchResources:
+            controlPlane: true
+        jsonPatches:
+          - op: add
+            path: "/spec/template/spec/kubeadmConfigSpec/joinConfiguration/nodeRegistration/kubeletExtraArgs"
+            value:
+              cloud-provider: "external"
+          - op: add
+            path: "/spec/template/spec/kubeadmConfigSpec/initConfiguration/nodeRegistration/kubeletExtraArgs"
+            value:
+              cloud-provider: "external"
+  - name: machineDeploymentExternalCloudProvider
+    enabledIf: "{{ .externalCloudProvider }}"
+    description: "Configures kubelet to run with an external cloud provider for machineDeployment nodes."
+    definitions:
+      - selector:
+          apiVersion: bootstrap.cluster.x-k8s.io/v1beta1
+          kind: KubeadmConfigTemplate
+          matchResources:
+            machineDeploymentClass:
+              names:
+                - '*-worker'
+        jsonPatches:
+          - op: add
+            path: "/spec/template/spec/joinConfiguration/nodeRegistration/kubeletExtraArgs"
+            value:
+              cloud-provider: "external"
+  - name: localEndpointIPv6
+    enabledIf: "{{ .ipv6Primary }}"
+    description: "Configures KCP to use IPv6 for its localAPIEndpoint."
+    definitions:
+      - selector:
+          apiVersion: controlplane.cluster.x-k8s.io/v1beta1
+          kind: KubeadmControlPlaneTemplate
+          matchResources:
+            controlPlane: true
+        jsonPatches:
+          - op: add
+            path: "/spec/template/spec/kubeadmConfigSpec/initConfiguration/localAPIEndpoint"
+            value:
+              advertiseAddress: '::'
   - name: podSecurityStandard
     description: "Adds an admission configuration for PodSecurity to the kube-apiserver."
     definitions:
@@ -368,7 +428,7 @@ spec:
             extraArgs: { enable-hostpath-provisioner: 'true' }
           apiServer:
             # host.docker.internal is required by kubetest when running on MacOS because of the way ports are proxied.
-            certSANs: [localhost, 127.0.0.1, 0.0.0.0, host.docker.internal]
+            certSANs: [localhost, host.docker.internal, "::", "::1", "127.0.0.1", "0.0.0.0"]
         initConfiguration:
           nodeRegistration:
             # We have to set the criSocket to containerd as kubeadm defaults to docker runtime if both containerd and docker sockets are found
diff --git a/test/e2e/data/kubetest/conformance-fast.yaml b/test/e2e/data/kubetest/conformance-fast.yaml
@@ -6,3 +6,4 @@ ginkgo.slowSpecThreshold: 120.0
 ginkgo.flakeAttempts: 3
 ginkgo.trace: true
 ginkgo.v: true
+ginkgo.no-color: true
diff --git a/test/e2e/data/kubetest/conformance.yaml b/test/e2e/data/kubetest/conformance.yaml
@@ -6,6 +6,7 @@ ginkgo.slowSpecThreshold: 120.0
 ginkgo.flakeAttempts: 3
 ginkgo.trace: true
 ginkgo.v: true
+ginkgo.no-color: true
 # Use 5m instead of the default 10m to fail faster
 # if kube-system Pods are not coming up.
 system-pods-startup-timeout: 5m
diff --git a/test/e2e/data/kubetest/dualstack.yaml b/test/e2e/data/kubetest/dualstack.yaml
@@ -0,0 +1,12 @@
+ginkgo.focus: \[Feature\:IPv6DualStack\]
+ginkgo.skip: \[Feature\:SCTPConnectivity\]
+disable-log-dump: true
+ginkgo.progress: true
+ginkgo.slowSpecThreshold: 120.0
+ginkgo.flakeAttempts: 3
+ginkgo.trace: true
+ginkgo.v: true
+ginkgo.no-color: true
+# Use 5m instead of the default 10m to fail faster
+# if kube-system Pods are not coming up.
+system-pods-startup-timeout: 5m
diff --git a/test/e2e/quick_start_test.go b/test/e2e/quick_start_test.go
@@ -21,9 +21,11 @@ package e2e
 
 import (
 	. "github.com/onsi/ginkgo/v2"
+	. "github.com/onsi/gomega"
 	"k8s.io/utils/pointer"
 
 	"sigs.k8s.io/cluster-api/test/framework"
+	"sigs.k8s.io/cluster-api/test/framework/kubetest"
 )
 
 var _ = Describe("When following the Cluster API quick-start [PR-Blocking]", func() {
@@ -99,3 +101,53 @@ var _ = Describe("When following the Cluster API quick-start with Ignition", fun
 		}
 	})
 })
+
+var _ = Describe("When following the Cluster API quick-start with dualstack and ipv4 primary [IPv6]", func() {
+	QuickStartSpec(ctx, func() QuickStartSpecInput {
+		return QuickStartSpecInput{
+			E2EConfig:             e2eConfig,
+			ClusterctlConfigPath:  clusterctlConfigPath,
+			BootstrapClusterProxy: bootstrapClusterProxy,
+			ArtifactFolder:        artifactFolder,
+			SkipCleanup:           skipCleanup,
+			Flavor:                pointer.String("topology-dualstack-ipv4-primary"),
+			PostMachinesProvisioned: func(proxy framework.ClusterProxy, namespace, clusterName string) {
+				By("Running kubetest dualstack tests")
+				// Start running the dualstack test suite from kubetest.
+				Expect(kubetest.Run(
+					ctx,
+					kubetest.RunInput{
+						ClusterProxy:       proxy.GetWorkloadCluster(ctx, namespace, clusterName),
+						ArtifactsDirectory: artifactFolder,
+						ConfigFilePath:     "./data/kubetest/dualstack.yaml",
+					},
+				)).To(Succeed())
+			},
+		}
+	})
+})
+
+var _ = Describe("When following the Cluster API quick-start with dualstack and ipv6 primary [IPv6]", func() {
+	QuickStartSpec(ctx, func() QuickStartSpecInput {
+		return QuickStartSpecInput{
+			E2EConfig:             e2eConfig,
+			ClusterctlConfigPath:  clusterctlConfigPath,
+			BootstrapClusterProxy: bootstrapClusterProxy,
+			ArtifactFolder:        artifactFolder,
+			SkipCleanup:           skipCleanup,
+			Flavor:                pointer.String("topology-dualstack-ipv6-primary"),
+			PostMachinesProvisioned: func(proxy framework.ClusterProxy, namespace, clusterName string) {
+				By("Running kubetest dualstack tests")
+				// Start running the dualstack test suite from kubetest.
+				Expect(kubetest.Run(
+					ctx,
+					kubetest.RunInput{
+						ClusterProxy:       proxy.GetWorkloadCluster(ctx, namespace, clusterName),
+						ArtifactsDirectory: artifactFolder,
+						ConfigFilePath:     "./data/kubetest/dualstack.yaml",
+					},
+				)).To(Succeed())
+			},
+		}
+	})
+})
diff --git a/test/framework/bootstrap/kind_provider.go b/test/framework/bootstrap/kind_provider.go
@@ -74,6 +74,14 @@ func WithIPv6Family() KindClusterOption {
 	})
 }
 
+// WithDualStackFamily implements a New Option that instruct the kindClusterProvider to set the IPFamily to dual in
+// the new kind cluster.
+func WithDualStackFamily() KindClusterOption {
+	return kindClusterOptionAdapter(func(k *KindClusterProvider) {
+		k.ipFamily = clusterv1.DualStackIPFamily
+	})
+}
+
 // LogFolder implements a New Option that instruct the kindClusterProvider to dump bootstrap logs in a folder in case of errors.
 func LogFolder(path string) KindClusterOption {
 	return kindClusterOptionAdapter(func(k *KindClusterProvider) {
@@ -136,6 +144,9 @@ func (k *KindClusterProvider) createKindCluster() {
 	if k.ipFamily == clusterv1.IPv6IPFamily {
 		cfg.Networking.IPFamily = kindv1.IPv6Family
 	}
+	if k.ipFamily == clusterv1.DualStackIPFamily {
+		cfg.Networking.IPFamily = kindv1.DualStackFamily
+	}
 	kindv1.SetDefaultsCluster(cfg)
 
 	if k.withDockerSock {
diff --git a/test/framework/bootstrap/kind_util.go b/test/framework/bootstrap/kind_util.go
@@ -71,6 +71,9 @@ func CreateKindBootstrapClusterAndLoadImages(ctx context.Context, input CreateKi
 	if input.IPFamily == "IPv6" {
 		options = append(options, WithIPv6Family())
 	}
+	if input.IPFamily == "dual" {
+		options = append(options, WithDualStackFamily())
+	}
 	if input.LogFolder != "" {
 		options = append(options, LogFolder(input.LogFolder))
 	}
diff --git a/test/infrastructure/container/docker.go b/test/infrastructure/container/docker.go
@@ -400,7 +400,7 @@ func (d *dockerRuntime) RunContainer(ctx context.Context, runConfig *RunContaine
 	}
 	networkConfig := network.NetworkingConfig{}
 
-	if runConfig.IPFamily == clusterv1.IPv6IPFamily {
+	if runConfig.IPFamily == clusterv1.IPv6IPFamily || runConfig.IPFamily == clusterv1.DualStackIPFamily {
 		hostConfig.Sysctls = map[string]string{
 			"net.ipv6.conf.all.disable_ipv6": "0",
 			"net.ipv6.conf.all.forwarding":   "1",
diff --git a/test/infrastructure/docker/exp/internal/docker/nodepool.go b/test/infrastructure/docker/exp/internal/docker/nodepool.go
@@ -314,7 +314,7 @@ func (np *NodePool) reconcileMachine(ctx context.Context, machine *docker.Machin
 	if machineStatus.Addresses == nil {
 		log.Info("Fetching instance addresses", "instance", machine.Name())
 		// set address in machine status
-		machineAddress, err := externalMachine.Address(ctx)
+		machineAddresses, err := externalMachine.Address(ctx)
 		if err != nil {
 			// Requeue if there is an error, as this is likely momentary load balancer
 			// state changes during control plane provisioning.
@@ -326,14 +326,17 @@ func (np *NodePool) reconcileMachine(ctx context.Context, machine *docker.Machin
 				Type:    clusterv1.MachineHostName,
 				Address: externalMachine.ContainerName(),
 			},
-			{
-				Type:    clusterv1.MachineInternalIP,
-				Address: machineAddress,
-			},
-			{
-				Type:    clusterv1.MachineExternalIP,
-				Address: machineAddress,
-			},
+		}
+		for _, addr := range machineAddresses {
+			machineStatus.Addresses = append(machineStatus.Addresses,
+				clusterv1.MachineAddress{
+					Type:    clusterv1.MachineInternalIP,
+					Address: addr,
+				},
+				clusterv1.MachineAddress{
+					Type:    clusterv1.MachineExternalIP,
+					Address: addr,
+				})
 		}
 	}
 
diff --git a/test/infrastructure/docker/internal/controllers/dockermachine_controller.go b/test/infrastructure/docker/internal/controllers/dockermachine_controller.go
diff --git a/test/infrastructure/docker/internal/docker/machine.go b/test/infrastructure/docker/internal/docker/machine.go
