Merge pull request #1365 from vr4manta/OCPBUGS-55461

OCPBUGS-55461: Changed machineset tests to be serial

Author: openshift-merge-bot[bot]

test/e2e/util.go

@@ -10,6 +10,7 @@ import (
 	"github.com/openshift/api/machine/v1beta1"
 	configclient "github.com/openshift/client-go/config/clientset/versioned"
 	machinesetclient "github.com/openshift/client-go/machine/clientset/versioned/typed/machine/v1beta1"
+	v1 "k8s.io/api/core/v1"
 	"k8s.io/apimachinery/pkg/api/errors"
 	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
 	"k8s.io/apimachinery/pkg/runtime"
@@ -176,6 +177,13 @@ func CreateMachine(ctx context.Context, cfg *rest.Config, mc *machinesetclient.M
 			ProviderSpec: v1beta1.ProviderSpec{
 				Value: provider,
 			},
+			Taints: []v1.Taint{
+				{
+					Effect: v1.TaintEffectNoSchedule,
+					Key:    "mapi-e2e",
+					Value:  "yes",
+				},
+			},
 		},
 	}
 
@@ -223,6 +231,13 @@ func CreateMachineSet(ctx context.Context, cfg *rest.Config, mc *machinesetclien
 					ProviderSpec: v1beta1.ProviderSpec{
 						Value: provider,
 					},
+					Taints: []v1.Taint{
+						{
+							Effect: v1.TaintEffectNoSchedule,
+							Key:    "mapi-e2e",
+							Value:  "yes",
+						},
+					},
 				},
 			},
 		},

test/e2e/vsphere/machines.go

@@ -19,11 +19,11 @@ import (
 )
 
 const (
-	machineRole         = "e2e-test"
+	machineRole         = "feature-gate-test"
 	machineReadyTimeout = time.Minute * 6
 )
 
-var _ = Describe("[sig-cluster-lifecycle][OCPFeatureGate:VSphereMultiDisk][platform:vsphere] Managed cluster should", func() {
+var _ = Describe("[sig-cluster-lifecycle][OCPFeatureGate:VSphereMultiDisk][platform:vsphere][Disruptive] Managed cluster should", Label("Conformance"), Label("Serial"), func() {
 	defer GinkgoRecover()
 	ctx := context.Background()
 
@@ -46,7 +46,7 @@ var _ = Describe("[sig-cluster-lifecycle][OCPFeatureGate:VSphereMultiDisk][platf
 		Expect(err).NotTo(HaveOccurred())
 	})
 
-	It("create machines with data disks [apigroup:machine.openshift.io]", func() {
+	It("create machines with data disks [apigroup:machine.openshift.io][Suite:openshift/conformance/serial]", func() {
 		machineName := "machine-multi-test"
 		dataDisks := []v1beta1.VSphereDisk{
 			{
@@ -151,33 +151,43 @@ var _ = Describe("[sig-cluster-lifecycle][OCPFeatureGate:VSphereMultiDisk][platf
 		err = util.ScaleMachineSet(cfg, ddMachineSet.Name, 0)
 		Expect(err).NotTo(HaveOccurred())
 
+		// Verify / wait for machine is removed
+		By("verifying machine is destroyed")
+		Eventually(func() (int32, error) {
+			ms, err := mc.MachineSets(util.MachineAPINamespace).Get(ctx, ddMachineSet.Name, metav1.GetOptions{})
+			if err != nil {
+				return -1, err
+			}
+			return ms.Status.ReadyReplicas, nil
+		}, machineReadyTimeout).Should(BeEquivalentTo(0))
+
 		// Delete machineset
 		By("deleting the machineset")
 		err = mc.MachineSets(util.MachineAPINamespace).Delete(ctx, ddMachineSet.Name, metav1.DeleteOptions{})
 		Expect(err).NotTo(HaveOccurred())
 	},
-		Entry("with thin data disk [apigroup:machine.openshift.io]", "ms-thin-test", []v1beta1.VSphereDisk{
+		Entry("with thin data disk [apigroup:machine.openshift.io][Suite:openshift/conformance/serial]", "ms-thin-test", []v1beta1.VSphereDisk{
 			{
 				Name:             "thickDataDisk",
 				SizeGiB:          1,
 				ProvisioningMode: v1beta1.ProvisioningModeThick,
 			},
 		}),
-		Entry("with thick data disk [apigroup:machine.openshift.io]", "ms-thick-test", []v1beta1.VSphereDisk{
+		Entry("with thick data disk [apigroup:machine.openshift.io][Suite:openshift/conformance/serial]", "ms-thick-test", []v1beta1.VSphereDisk{
 			{
 				Name:             "thickDataDisk",
 				SizeGiB:          1,
 				ProvisioningMode: v1beta1.ProvisioningModeThick,
 			},
 		}),
-		Entry("with eagerly zeroed data disk [apigroup:machine.openshift.io]", "ms-zeroed-test", []v1beta1.VSphereDisk{
+		Entry("with eagerly zeroed data disk [apigroup:machine.openshift.io][Suite:openshift/conformance/serial]", "ms-zeroed-test", []v1beta1.VSphereDisk{
 			{
 				Name:             "zeroedDataDisk",
 				SizeGiB:          1,
 				ProvisioningMode: v1beta1.ProvisioningModeEagerlyZeroed,
 			},
 		}),
-		Entry("with a data disk using each provisioning mode [apigroup:machine.openshift.io]", "ms-multi-test", []v1beta1.VSphereDisk{
+		Entry("with a data disk using each provisioning mode [apigroup:machine.openshift.io][Suite:openshift/conformance/serial]", "ms-multi-test", []v1beta1.VSphereDisk{
 			{
 				Name:             "thinDataDisk",
 				SizeGiB:          1,

test/e2e/vsphere/multi-nic.go

@@ -41,18 +41,19 @@ func failIfNodeNotInMachineNetwork(nodes corev1.NodeList, machineNetworks []stri
 
 func failIfIncorrectPortgroupsAttachedToVMs(
 	ctx context.Context,
-	infra configv1.PlatformSpec,
+	infra *configv1.Infrastructure,
 	nodeList *corev1.NodeList,
 	vsphereCreds *corev1.Secret) {
 
 	By("checking if VMs have the correct portgroups attached")
 
-	for _, failureDomain := range infra.VSphere.FailureDomains {
-		nodes, err := getNodesInFailureDomain(infra.VSphere, failureDomain, nodeList)
+	providerSpec := infra.Spec.PlatformSpec
+	for _, failureDomain := range providerSpec.VSphere.FailureDomains {
+		nodes, err := getNodesInFailureDomain(providerSpec.VSphere, failureDomain, nodeList)
 		fmt.Printf("nodes: %d", len(nodes))
 		Expect(err).NotTo(HaveOccurred())
 
-		vmPortgroupMap, err := GetPortGroupsAttachedToVMsInFailureDomain(ctx, failureDomain, vsphereCreds, infra.VSphere.VCenters)
+		vmPortgroupMap, err := GetPortGroupsAttachedToVMsInFailureDomain(ctx, infra.Status.InfrastructureName, failureDomain, vsphereCreds, providerSpec.VSphere.VCenters)
 		if err != nil {
 			Expect(err).NotTo(HaveOccurred())
 		}
@@ -82,13 +83,21 @@ func failIfIncorrectPortgroupsAttachedToVMs(
 }
 
 func failIfNodeNetworkingInconsistentWithMachineNetwork(infra configv1.PlatformSpec, machineNetworks []string) {
+	// This can happen in scenarios where multinetwork is not enabled.
+	if len(infra.VSphere.NodeNetworking.External.NetworkSubnetCIDR) == 0 ||
+		len(infra.VSphere.NodeNetworking.Internal.NetworkSubnetCIDR) == 0 {
+		Skip("skipping test due to incomplete config")
+	}
+
 	internalNodeNetworking := infra.VSphere.NodeNetworking.Internal
 	externalNodeNetworking := infra.VSphere.NodeNetworking.External
 
+	// machineNetworks contain the VIPs now so we'll need to check each network to see if we find one that matches internal/external.
 	By("comparing nodeNetworking slices to the machine network")
 	for _, nodeNetworkingSpec := range []configv1.VSpherePlatformNodeNetworkingSpec{internalNodeNetworking, externalNodeNetworking} {
-		slices.Sort(nodeNetworkingSpec.NetworkSubnetCIDR)
-		Expect(slices.Equal(nodeNetworkingSpec.NetworkSubnetCIDR, machineNetworks)).To(BeTrue())
+		for _, network := range nodeNetworkingSpec.NetworkSubnetCIDR {
+			Expect(slices.Contains(machineNetworks, network)).To(BeTrue())
+		}
 	}
 }
 
@@ -126,7 +135,7 @@ func failIfMachineDoesNotHaveAllPortgroups(machine machinev1beta1.Machine, failu
 	Expect(slices.Equal(expectedPortgroups, portgroups)).To(BeTrue())
 }
 
-var _ = Describe("[sig-cluster-lifecycle][OCPFeatureGate:VSphereMultiNetworks][platform:vsphere] Managed cluster should", func() {
+var _ = Describe("[sig-cluster-lifecycle][OCPFeatureGate:VSphereMultiNetworks][platform:vsphere] Managed cluster should", Label("Conformance"), func() {
 	defer GinkgoRecover()
 	ctx := context.Background()
 
@@ -190,24 +199,24 @@ var _ = Describe("[sig-cluster-lifecycle][OCPFeatureGate:VSphereMultiNetworks][p
 		}
 	})
 
-	It("node addresses should be correlated with the machine network", func() {
+	It("node addresses should be correlated with the machine network [apigroup:machine.openshift.io][Suite:openshift/conformance/parallel]", func() {
 		By("checking for correlation between node internal/external IPs and the machine network")
 		failIfNodeNotInMachineNetwork(*nodes, machineNetworks)
 	})
 
-	It("machine network should be correlated with node networking", func() {
+	It("machine network should be correlated with node networking [apigroup:machine.openshift.io][Suite:openshift/conformance/parallel]", func() {
 		failIfNodeNetworkingInconsistentWithMachineNetwork(infra.Spec.PlatformSpec, machineNetworks)
 	})
 
-	It("machines should have all specified portgroup associated with their failure domain", func() {
+	It("machines should have all specified portgroup associated with their failure domain [apigroup:machine.openshift.io][Suite:openshift/conformance/parallel]", func() {
 		failIfMachinesDoNotHaveAllPortgroups(infra.Spec.PlatformSpec, machines)
 	})
 
-	It("node VMs should have all specified portgroups attached which are associated with their failure domain", func() {
-		failIfIncorrectPortgroupsAttachedToVMs(ctx, infra.Spec.PlatformSpec, nodes, vsphereCreds)
+	It("node VMs should have all specified portgroups attached which are associated with their failure domain [apigroup:machine.openshift.io][Suite:openshift/conformance/parallel]", func() {
+		failIfIncorrectPortgroupsAttachedToVMs(ctx, infra, nodes, vsphereCreds)
 	})
 
-	It("new machines should pass multi network tests", func() {
+	It("new machines should pass multi network tests [apigroup:machine.openshift.io][Suite:openshift/conformance/serial]", Label("Serial"), func() {
 		machineSets, err := e2eutil.GetMachineSets(cfg)
 		Expect(err).NotTo(HaveOccurred())
 
@@ -244,6 +253,6 @@ var _ = Describe("[sig-cluster-lifecycle][OCPFeatureGate:VSphereMultiNetworks][p
 		failIfNodeNotInMachineNetwork(*nodes, machineNetworks)
 		failIfNodeNetworkingInconsistentWithMachineNetwork(infra.Spec.PlatformSpec, machineNetworks)
 		failIfMachinesDoNotHaveAllPortgroups(infra.Spec.PlatformSpec, machines)
-		failIfIncorrectPortgroupsAttachedToVMs(ctx, infra.Spec.PlatformSpec, nodes, vsphereCreds)
+		failIfIncorrectPortgroupsAttachedToVMs(ctx, infra, nodes, vsphereCreds)
 	})
 })

test/e2e/vsphere/util.go

@@ -6,21 +6,21 @@ import (
 	"net"
 	"net/url"
 	"path"
+	"slices"
 	"strconv"
 	"time"
 
 	. "github.com/onsi/ginkgo/v2"
 	. "github.com/onsi/gomega"
-
 	configv1 "github.com/openshift/api/config/v1"
 	"github.com/openshift/api/machine/v1beta1"
 	machinev1beta1 "github.com/openshift/api/machine/v1beta1"
-	util "github.com/openshift/machine-api-operator/test/e2e"
 	"github.com/pkg/errors"
 	"github.com/vmware/govmomi"
 	"github.com/vmware/govmomi/find"
 	"github.com/vmware/govmomi/object"
 	vapirest "github.com/vmware/govmomi/vapi/rest"
+	"github.com/vmware/govmomi/vapi/tags"
 	"github.com/vmware/govmomi/vim25"
 	"github.com/vmware/govmomi/vim25/mo"
 	"github.com/vmware/govmomi/vim25/soap"
@@ -29,6 +29,7 @@ import (
 	"k8s.io/client-go/rest"
 
 	"github.com/openshift/machine-api-operator/pkg/controller/vsphere"
+	util "github.com/openshift/machine-api-operator/test/e2e"
 )
 
 func isIpInCidrRange(ip string, cidr string) (bool, error) {
@@ -255,6 +256,7 @@ func getCredentialsForVCenter(ctx context.Context, vsphereCreds *corev1.Secret,
 
 func GetPortGroupsAttachedToVMsInFailureDomain(
 	ctx context.Context,
+	clusterID string,
 	failureDomain configv1.VSpherePlatformFailureDomainSpec,
 	vsphereCreds *corev1.Secret,
 	vCenters []configv1.VSpherePlatformVCenterSpec) (map[string][]string, error) {
@@ -281,7 +283,7 @@ func GetPortGroupsAttachedToVMsInFailureDomain(
 
 	portGroupMap := make(map[string][]string)
 
-	client, _, logout, err := CreateVSphereClients(ctx, vcenterUrl, user, pass)
+	client, rClient, logout, err := CreateVSphereClients(ctx, vcenterUrl, user, pass)
 	defer logout()
 
 	if err != nil {
@@ -290,17 +292,31 @@ func GetPortGroupsAttachedToVMsInFailureDomain(
 
 	finder := NewFinder(client)
 
+	// Get tagged objects and then grab VMs
+	tm := tags.NewManager(rClient)
+	refs, err := tm.ListAttachedObjects(ctx, clusterID)
+	var vmList []types.ManagedObjectReference
+	for _, ref := range refs {
+		if ref.Reference().Type == "VirtualMachine" {
+			vmList = append(vmList, ref.Reference())
+		}
+	}
+
 	vms, err := finder.VirtualMachineList(ctx, fmt.Sprintf("/%s/vm/...", failureDomain.Topology.Datacenter))
 	if err != nil {
 		return nil, fmt.Errorf("unable to get VMs in %s. %v", failureDomain.Topology.ComputeCluster, err)
 	}
 
 	for _, vm := range vms {
+		// Make sure vm has the cluster id tag.  Note: template will be included.
+		if !slices.Contains(vmList, vm.Reference()) {
+			continue
+		}
+
 		vm, err := finder.VirtualMachine(ctx, vm.InventoryPath)
 		if err != nil {
 			return nil, fmt.Errorf("unable to get VMs in %s. %v", failureDomain.Topology.ComputeCluster, err)
 		}
-		fmt.Printf("UUID: %s\n", vm.UUID(ctx))
 		virtualDevices, err := vm.Device(ctx)
 
 		var portGroups []string