self hosted tests should check for rollouts

Author: Yuvaraj Kakaraparthi

test/e2e/clusterctl_upgrade.go

@@ -31,6 +31,9 @@ import (
 	corev1 "k8s.io/api/core/v1"
 	apierrors "k8s.io/apimachinery/pkg/api/errors"
 	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
+	"k8s.io/apimachinery/pkg/apis/meta/v1/unstructured"
+	"k8s.io/apimachinery/pkg/types"
+	"k8s.io/apimachinery/pkg/util/sets"
 	"k8s.io/client-go/discovery"
 	"k8s.io/klog/v2"
 	"k8s.io/utils/pointer"
@@ -297,7 +300,8 @@ func ClusterctlUpgradeSpec(ctx context.Context, inputGetter func() ClusterctlUpg
 
 		// Get the workloadCluster before the management cluster is upgraded to make sure that the upgrade did not trigger
 		// any unexpected rollouts.
-		preUpgradeMachineList := &clusterv1alpha3.MachineList{}
+		preUpgradeMachineList := &unstructured.UnstructuredList{}
+		preUpgradeMachineList.SetGroupVersionKind(clusterv1alpha3.GroupVersion.WithKind("MachineList"))
 		err = managementClusterProxy.GetClient().List(
 			ctx,
 			preUpgradeMachineList,
@@ -323,15 +327,16 @@ func ClusterctlUpgradeSpec(ctx context.Context, inputGetter func() ClusterctlUpg
 
 		// After the upgrade check that there were no unexpected rollouts.
 		Consistently(func() bool {
-			postUpgradeMachineList := &clusterv1.MachineList{}
+			postUpgradeMachineList := &unstructured.UnstructuredList{}
+			postUpgradeMachineList.SetGroupVersionKind(clusterv1.GroupVersion.WithKind("MachineList"))
 			err = managementClusterProxy.GetClient().List(
 				ctx,
 				postUpgradeMachineList,
 				client.InNamespace(testNamespace.Name),
 				client.MatchingLabels{clusterv1.ClusterLabelName: workLoadClusterName},
 			)
 			Expect(err).NotTo(HaveOccurred())
-			return machinesMatch(preUpgradeMachineList, postUpgradeMachineList)
+			return matchUnstructuredLists(preUpgradeMachineList, postUpgradeMachineList)
 		}, "3m", "30s").Should(BeTrue(), "Machines should remain the same after the upgrade")
 
 		// After upgrading we are sure the version is the latest version of the API,
@@ -578,23 +583,23 @@ func waitForClusterDeletedV1alpha4(ctx context.Context, input waitForClusterDele
 	}, intervals...).Should(BeTrue())
 }
 
-func machinesMatch(oldMachineList *clusterv1alpha3.MachineList, newMachineList *clusterv1.MachineList) bool {
-	if len(oldMachineList.Items) != len(newMachineList.Items) {
+func matchUnstructuredLists(l1 *unstructured.UnstructuredList, l2 *unstructured.UnstructuredList) bool {
+	if l1 == nil && l2 == nil {
+		return true
+	}
+	if l1 == nil || l2 == nil {
 		return false
 	}
-
-	// Every machine from the old list should be present in the new list
-	for _, oldMachine := range oldMachineList.Items {
-		found := false
-		for _, newMachine := range newMachineList.Items {
-			if oldMachine.Name == newMachine.Name && oldMachine.Namespace == newMachine.Namespace {
-				found = true
-				break
-			}
-		}
-		if !found {
-			return false
-		}
+	if len(l1.Items) != len(l2.Items) {
+		return false
+	}
+	s1 := sets.NewString()
+	for _, i := range l1.Items {
+		s1.Insert(types.NamespacedName{Namespace: i.GetNamespace(), Name: i.GetName()}.String())
+	}
+	s2 := sets.NewString()
+	for _, i := range l2.Items {
+		s2.Insert(types.NamespacedName{Namespace: i.GetNamespace(), Name: i.GetName()}.String())
 	}
-	return true
+	return s1.Equal(s2)
 }

test/e2e/self_hosted.go

@@ -26,6 +26,7 @@ import (
 	. "github.com/onsi/ginkgo"
 	. "github.com/onsi/gomega"
 	corev1 "k8s.io/api/core/v1"
+	"k8s.io/apimachinery/pkg/apis/meta/v1/unstructured"
 	"k8s.io/klog/v2"
 	"k8s.io/utils/pointer"
 	"sigs.k8s.io/controller-runtime/pkg/client"
@@ -82,6 +83,7 @@ func SelfHostedSpec(ctx context.Context, inputGetter func() SelfHostedSpecInput)
 	It("Should pivot the bootstrap cluster to a self-hosted cluster", func() {
 		By("Creating a workload cluster")
 
+		workloadClusterName := fmt.Sprintf("%s-%s", specName, util.RandomString(6))
 		clusterctl.ApplyClusterTemplateAndWait(ctx, clusterctl.ApplyClusterTemplateAndWaitInput{
 			ClusterProxy: input.BootstrapClusterProxy,
 			ConfigCluster: clusterctl.ConfigClusterInput{
@@ -91,7 +93,7 @@ func SelfHostedSpec(ctx context.Context, inputGetter func() SelfHostedSpecInput)
 				InfrastructureProvider:   clusterctl.DefaultInfrastructureProvider,
 				Flavor:                   input.Flavor,
 				Namespace:                namespace.Name,
-				ClusterName:              fmt.Sprintf("%s-%s", specName, util.RandomString(6)),
+				ClusterName:              workloadClusterName,
 				KubernetesVersion:        input.E2EConfig.GetVariable(KubernetesVersion),
 				ControlPlaneMachineCount: pointer.Int64Ptr(1),
 				WorkerMachineCount:       pointer.Int64Ptr(1),
@@ -164,6 +166,18 @@ func SelfHostedSpec(ctx context.Context, inputGetter func() SelfHostedSpecInput)
 			return selfHostedClusterProxy.GetClient().Get(ctx, client.ObjectKey{Name: "kube-system"}, kubeSystem)
 		}, "5s", "100ms").Should(BeNil(), "Failed to assert self-hosted API server stability")
 
+		// Get the machines of the workloadCluster before it is moved to become self-hosted to make sure that the move did not trigger
+		// any unexpected rollouts.
+		preMoveMachineList := &unstructured.UnstructuredList{}
+		preMoveMachineList.SetGroupVersionKind(clusterv1.GroupVersion.WithKind("MachineList"))
+		err := input.BootstrapClusterProxy.GetClient().List(
+			ctx,
+			preMoveMachineList,
+			client.InNamespace(namespace.Name),
+			client.MatchingLabels{clusterv1.ClusterLabelName: workloadClusterName},
+		)
+		Expect(err).NotTo(HaveOccurred(), "Failed to list machines before move")
+
 		By("Moving the cluster to self hosted")
 		clusterctl.Move(ctx, clusterctl.MoveInput{
 			LogFolder:            filepath.Join(input.ArtifactFolder, "clusters", "bootstrap"),
@@ -187,6 +201,20 @@ func SelfHostedSpec(ctx context.Context, inputGetter func() SelfHostedSpecInput)
 		})
 		Expect(controlPlane).ToNot(BeNil())
 
+		// After the move check that there were no unexpected rollouts.
+		Consistently(func() bool {
+			postMoveMachineList := &unstructured.UnstructuredList{}
+			postMoveMachineList.SetGroupVersionKind(clusterv1.GroupVersion.WithKind("MachineList"))
+			err = selfHostedClusterProxy.GetClient().List(
+				ctx,
+				postMoveMachineList,
+				client.InNamespace(namespace.Name),
+				client.MatchingLabels{clusterv1.ClusterLabelName: workloadClusterName},
+			)
+			Expect(err).NotTo(HaveOccurred(), "Failed to list machines after move")
+			return matchUnstructuredLists(preMoveMachineList, postMoveMachineList)
+		}, "3m", "30s").Should(BeTrue(), "Machines should not roll out after move to self-hosted cluster")
+
 		By("PASSED!")
 	})