Added e2e test for multiple network support

test/e2e/common.go

@@ -21,6 +21,7 @@ import (
 	"encoding/json"
 	"errors"
 	"fmt"
+	"maps"
 	"path/filepath"
 	"strconv"
 	"strings"
@@ -575,6 +576,95 @@ func IsClusterReady(ctx context.Context, mgmtClient client.Client, cluster *clus
 	return c.Status.ControlPlaneReady && c.Status.InfrastructureReady
 }
 
+func EnsureSecondaryNetworkExists(client *cloudstack.CloudStackClient, input CommonSpecInput) (*cloudstack.Network, error) {
+	secondaryNetName := input.E2EConfig.GetVariable("CLOUDSTACK_NEW_NETWORK_NAME")
+
+	By("Fetching secondary network details")
+	// Try fetching secondary network
+	secondaryNet, _, err := client.Network.GetNetworkByName(secondaryNetName)
+	if err == nil && secondaryNet != nil {
+		By(fmt.Sprintf("Network %q already exists", secondaryNetName))
+		return secondaryNet, nil
+	}
+
+	By("Listing Zone")
+	zoneName := input.E2EConfig.GetVariable("CLOUDSTACK_ZONE_NAME")
+	pz := client.Zone.NewListZonesParams()
+	pz.SetName(zoneName)
+	listZonesResponse, err := client.Zone.ListZones(pz)
+	Expect(err).To(BeNil(), "error listing zones")
+	Expect(listZonesResponse.Count).To(Equal(1), "no zones, or more than one zone resolve to zone name %s", zoneName)
+	zoneId := listZonesResponse.Zones[0].Id
+
+	By("Listing network offerings")
+	networkOffering, _, err := client.NetworkOffering.GetNetworkOfferingByName(DefaultNetworkOffering)
+	Expect(err).To(BeNil(), "error fetching network offering %q", DefaultNetworkOffering)
+	Expect(networkOffering).ToNot(BeNil(), "network offering %q not found", DefaultNetworkOffering)
+
+	// Create new network using zone and offering from primary
+	By("Create secondary network")
+	createParams := client.Network.NewCreateNetworkParams(
+		secondaryNetName,
+		networkOffering.Id,
+		zoneId,
+	)
+
+	newNetResp, err := client.Network.CreateNetwork(createParams)
+	if err != nil {
+		return nil, fmt.Errorf("failed to create network %q: %w", secondaryNetName, err)
+	}
+
+	newNet, _, err := client.Network.GetNetworkByID(newNetResp.Id)
+	if err != nil {
+		return nil, fmt.Errorf("failed to fetch created network %q by ID: %w", newNetResp.Id, err)
+	}
+
+	By("Created secondary network")
+	By(fmt.Sprintf("Created secondary network %q", secondaryNetName))
+	return newNet, nil
+}
+
+func CheckIfNodesHaveTwoNICs(client *cloudstack.CloudStackClient, clusterName string, input CommonSpecInput) {
+	requiredNetworks := map[string]bool{
+		input.E2EConfig.GetVariable("CLOUDSTACK_NETWORK_NAME"):     false,
+		input.E2EConfig.GetVariable("CLOUDSTACK_NEW_NETWORK_NAME"): false,
+	}
+
+	Byf("Listing machines with name containing %q", clusterName)
+	listResp, err := client.VirtualMachine.ListVirtualMachines(client.VirtualMachine.NewListVirtualMachinesParams())
+	Expect(err).NotTo(HaveOccurred(), "Failed to list virtual machines from CloudStack")
+	for _, vm := range listResp.VirtualMachines {
+		if !strings.Contains(vm.Name, clusterName) {
+			continue
+		}
+
+		if len(vm.Nic) < 2 {
+			Fail(fmt.Sprintf("VM %q has fewer than 2 NICs. Found: %d", vm.Name, len(vm.Nic)))
+		}
+
+		foundNetworks := make(map[string]bool)
+		for _, nic := range vm.Nic {
+			foundNetworks[nic.Networkname] = true
+		}
+
+		for required := range requiredNetworks {
+			if !foundNetworks[required] {
+				Fail(fmt.Sprintf("VM %q is missing required network %q", vm.Name, required))
+			}
+		}
+
+		By(fmt.Sprintf("VM %q has required NICs: %v", vm.Name, maps.Keys(foundNetworks)))
+	}
+}
+
+func keys(m map[string]bool) []string {
+	var list []string
+	for k := range m {
+		list = append(list, k)
+	}
+	return list
+}
+
 func CheckDiskOfferingOfVmInstances(client *cloudstack.CloudStackClient, clusterName string, diskOfferingName string) {
 	Byf("Listing machines with %q", clusterName)
 	listResp, err := client.VirtualMachine.ListVirtualMachines(client.VirtualMachine.NewListVirtualMachinesParams())

test/e2e/config/cloudstack.yaml

@@ -102,6 +102,7 @@ providers:
       - sourcePath: "../data/infrastructure-cloudstack/v1beta3/cluster-template-kubernetes-version-upgrade-before.yaml"
       - sourcePath: "../data/infrastructure-cloudstack/v1beta3/cluster-template-kubernetes-version-upgrade-after.yaml"
       - sourcePath: "../data/infrastructure-cloudstack/v1beta3/cluster-template-k8s-cks.yaml"
+      - sourcePath: "../data/infrastructure-cloudstack/v1beta3/cluster-template-multiple-networks.yaml"
       - sourcePath: "../data/shared/v1beta1_provider/metadata.yaml"
     versions:
       - name: v1.0.0

test/e2e/data/infrastructure-cloudstack/v1beta3/cluster-template-multiple-networks/kustomization.yaml

@@ -0,0 +1,6 @@
+bases:
+  - ../bases/cluster-with-kcp.yaml
+  - ../bases/md.yaml
+
+patchesStrategicMerge:
+- ./multiple-networks.yaml

test/e2e/data/infrastructure-cloudstack/v1beta3/cluster-template-multiple-networks/multiple-networks.yaml

@@ -0,0 +1,33 @@
+---
+apiVersion: infrastructure.cluster.x-k8s.io/v1beta3
+kind: CloudStackMachineTemplate
+metadata:
+  name: ${CLUSTER_NAME}-control-plane
+spec:
+  template:
+    spec:
+      offering:
+        name: ${CLOUDSTACK_CONTROL_PLANE_MACHINE_OFFERING}
+      template:
+        name: ${CLOUDSTACK_TEMPLATE_NAME}
+      sshKey: ${CLOUDSTACK_SSH_KEY_NAME}
+      affinity: pro
+      networks:
+        - name: ${CLOUDSTACK_NETWORK_NAME}
+        - name: ${CLOUDSTACK_NEW_NETWORK_NAME}
+---
+apiVersion: infrastructure.cluster.x-k8s.io/v1beta3
+kind: CloudStackMachineTemplate
+metadata:
+  name: ${CLUSTER_NAME}-md-0
+spec:
+  template:
+    spec:
+      offering:
+        name: ${CLOUDSTACK_WORKER_MACHINE_OFFERING}
+      template:
+        name: ${CLOUDSTACK_TEMPLATE_NAME}
+      sshKey: ${CLOUDSTACK_SSH_KEY_NAME}
+      networks:
+        - name: ${CLOUDSTACK_NETWORK_NAME}
+        - name: ${CLOUDSTACK_NEW_NETWORK_NAME}

test/e2e/multiple_networks.go

@@ -0,0 +1,100 @@
+/*
+Copyright 2020 The Kubernetes Authors.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package e2e
+
+import (
+	"context"
+	"fmt"
+	"os"
+	"path/filepath"
+
+	. "github.com/onsi/ginkgo/v2"
+	. "github.com/onsi/gomega"
+	corev1 "k8s.io/api/core/v1"
+	"k8s.io/utils/pointer"
+
+	"sigs.k8s.io/cluster-api/test/framework/clusterctl"
+	"sigs.k8s.io/cluster-api/util"
+)
+
+// MultipleNetworksSpec implements a spec that creates a cluster with nodes having multiple NICs.
+func MultipleNetworksSpec(ctx context.Context, inputGetter func() CommonSpecInput) {
+	var (
+		specName         = "multiple-networks"
+		input            CommonSpecInput
+		namespace        *corev1.Namespace
+		cancelWatches    context.CancelFunc
+		clusterResources *clusterctl.ApplyClusterTemplateAndWaitResult
+	)
+
+	BeforeEach(func() {
+		Expect(ctx).NotTo(BeNil(), "ctx is required for %s spec", specName)
+		input = inputGetter()
+
+		Expect(input.E2EConfig).ToNot(BeNil(), "Invalid argument. input.E2EConfig can't be nil when calling %s spec", specName)
+		Expect(input.ClusterctlConfigPath).To(BeAnExistingFile(), "Invalid argument. input.ClusterctlConfigPath must be an existing file when calling %s spec", specName)
+		Expect(input.BootstrapClusterProxy).ToNot(BeNil(), "Invalid argument. input.BootstrapClusterProxy can't be nil when calling %s spec", specName)
+		Expect(os.MkdirAll(input.ArtifactFolder, 0750)).To(Succeed(), "Invalid argument. input.ArtifactFolder can't be created for %s spec", specName)
+
+		Expect(input.E2EConfig.Variables).To(HaveKey(KubernetesVersion))
+
+		// Setup a Namespace where to host objects for this spec and create a watcher for the namespace events.
+		namespace, cancelWatches = setupSpecNamespace(ctx, specName, input.BootstrapClusterProxy, input.ArtifactFolder)
+		clusterResources = new(clusterctl.ApplyClusterTemplateAndWaitResult)
+	})
+
+	It("Should create a workload cluster", func() {
+		By("Creating a workload cluster")
+
+		clusterName := fmt.Sprintf("%s-%s", specName, util.RandomString(6))
+
+		// Get details from ACS and ensure secondary network exists
+		csClient := CreateCloudStackClient(ctx, input.BootstrapClusterProxy.GetKubeconfigPath())
+		_, err := EnsureSecondaryNetworkExists(csClient, input)
+		Expect(err).ToNot(HaveOccurred(), "Failed to ensure secondary network exists")
+
+		clusterctl.ApplyClusterTemplateAndWait(ctx, clusterctl.ApplyClusterTemplateAndWaitInput{
+			ClusterProxy:    input.BootstrapClusterProxy,
+			CNIManifestPath: input.E2EConfig.GetVariable(CNIPath),
+			ConfigCluster: clusterctl.ConfigClusterInput{
+				LogFolder:                filepath.Join(input.ArtifactFolder, "clusters", input.BootstrapClusterProxy.GetName()),
+				ClusterctlConfigPath:     input.ClusterctlConfigPath,
+				KubeconfigPath:           input.BootstrapClusterProxy.GetKubeconfigPath(),
+				InfrastructureProvider:   clusterctl.DefaultInfrastructureProvider,
+				Flavor:                   specName,
+				Namespace:                namespace.Name,
+				ClusterName:              clusterName,
+				KubernetesVersion:        input.E2EConfig.GetVariable(KubernetesVersion),
+				ControlPlaneMachineCount: pointer.Int64Ptr(1),
+				WorkerMachineCount:       pointer.Int64Ptr(1),
+			},
+			WaitForClusterIntervals:      input.E2EConfig.GetIntervals(specName, "wait-cluster"),
+			WaitForControlPlaneIntervals: input.E2EConfig.GetIntervals(specName, "wait-control-plane"),
+			WaitForMachineDeployments:    input.E2EConfig.GetIntervals(specName, "wait-worker-nodes"),
+		}, clusterResources)
+
+		By("Verifying that each VM has two NICs")
+		CheckIfNodesHaveTwoNICs(csClient, clusterName, input)
+
+		By("PASSED!")
+	})
+
+	AfterEach(func() {
+		// Dumps all the resources in the spec namespace, then cleanups the cluster object and the spec namespace itself.
+		dumpSpecResourcesAndCleanup(ctx, specName, input.BootstrapClusterProxy, input.ArtifactFolder, namespace, cancelWatches, clusterResources.Cluster, input.E2EConfig.GetIntervals, input.SkipCleanup)
+	})
+}

test/e2e/multiple_networks_test.go

@@ -0,0 +1,40 @@
+//go:build e2e
+// +build e2e
+
+/*
+Copyright 2021 The Kubernetes Authors.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package e2e
+
+import (
+	"context"
+
+	. "github.com/onsi/ginkgo/v2"
+)
+
+var _ = Describe("Test with multiple networks for nodes", func() {
+
+	MultipleNetworksSpec(context.TODO(), func() CommonSpecInput {
+		return CommonSpecInput{
+			E2EConfig:             e2eConfig,
+			ClusterctlConfigPath:  clusterctlConfigPath,
+			BootstrapClusterProxy: bootstrapClusterProxy,
+			ArtifactFolder:        artifactFolder,
+			SkipCleanup:           skipCleanup,
+		}
+	})
+
+})