Integration Tests: clusterctl move (#102)

Co-authored-by: Valentin Knabel <[email protected]>

Author: simcod

test/e2e/frmwrk/bootstrap_test.go

@@ -11,52 +11,124 @@ import (
 	. "github.com/onsi/ginkgo/v2"
 	. "github.com/onsi/gomega"
 
+	apierrors "k8s.io/apimachinery/pkg/api/errors"
+	"sigs.k8s.io/controller-runtime/pkg/client"
+
+	clusterv1 "sigs.k8s.io/cluster-api/api/v1beta1"
 	"sigs.k8s.io/cluster-api/test/framework"
+	"sigs.k8s.io/cluster-api/test/framework/clusterctl"
 )
 
-var _ = Describe("Basic Cluster Creation", Ordered, func() {
-	var (
-		ec *E2ECluster
-	)
+var _ = Describe("Basic Cluster", Ordered, func() {
 
 	BeforeAll(func() {
 		e2eCtx = NewE2EContext()
 		e2eCtx.ProvideBootstrapCluster()
 		e2eCtx.CreateClusterctlConfig(context.TODO())
 		e2eCtx.InitManagementCluster(context.TODO())
-
-		DeferCleanup(e2eCtx.Teardown, context.TODO())
-	})
-
-	BeforeEach(func() {
-		ec = nil
 	})
 
 	kubernetesVersions := strings.Split(os.Getenv("E2E_KUBERNETES_VERSIONS"), ",")
 	Expect(kubernetesVersions).ToNot(BeEmpty(), "E2E_KUBERNETES_VERSIONS must be set")
 
 	for i, v := range kubernetesVersions {
-		It(fmt.Sprintf("create new cluster with kubernetes %s", v), func() {
-			ctx := context.Background()
-
-			ec = createE2ECluster(ctx, e2eCtx, ClusterConfig{
-				SpecName:                 "basic-cluster-creation-" + v,
-				NamespaceName:            fmt.Sprintf("e2e-basic-cluster-creation-%d", i),
-				ClusterName:              fmt.Sprintf("simple-%d", i),
-				KubernetesVersion:        v,
-				ControlPlaneMachineImage: os.Getenv("E2E_CONTROL_PLANE_MACHINE_IMAGE_PREFIX") + strings.TrimPrefix(v, "v"),
-				ControlPlaneMachineCount: 1,
-				WorkerMachineImage:       os.Getenv("E2E_WORKER_MACHINE_IMAGE_PREFIX") + strings.TrimPrefix(v, "v"),
-				WorkerMachineCount:       1,
+		Context(fmt.Sprintf("with kubernetes %s", v), Ordered, func() {
+			var (
+				ec  *E2ECluster
+				ctx context.Context
+			)
+
+			BeforeEach(func() {
+				ctx = context.Background()
+			})
+
+			It("create new cluster", func() {
+				ec = createE2ECluster(ctx, e2eCtx, ClusterConfig{
+					SpecName:                 "basic-cluster-creation-" + v,
+					NamespaceName:            fmt.Sprintf("e2e-basic-cluster-creation-%d", i),
+					ClusterName:              fmt.Sprintf("simple-%d", i),
+					KubernetesVersion:        v,
+					ControlPlaneMachineImage: os.Getenv("E2E_CONTROL_PLANE_MACHINE_IMAGE_PREFIX") + strings.TrimPrefix(v, "v"),
+					ControlPlaneMachineCount: 1,
+					WorkerMachineImage:       os.Getenv("E2E_WORKER_MACHINE_IMAGE_PREFIX") + strings.TrimPrefix(v, "v"),
+					WorkerMachineCount:       1,
+				})
+				Expect(ec).ToNot(BeNil())
+			})
+
+			It("move from bootstrap to workload cluster", func() {
+				Expect(ec).NotTo(BeNil(), "e2e cluster required")
+
+				clusterctl.InitManagementClusterAndWatchControllerLogs(ctx, clusterctl.InitManagementClusterAndWatchControllerLogsInput{
+					ClusterProxy:            ec.Refs.Workload,
+					ClusterctlConfigPath:    e2eCtx.Environment.ClusterctlConfigPath,
+					InfrastructureProviders: e2eCtx.E2EConfig.InfrastructureProviders(),
+					LogFolder:               path.Join(e2eCtx.Environment.artifactsPath, "clusters", ec.ClusterName, "init"),
+				})
+
+				clusterctl.Move(ctx, clusterctl.MoveInput{
+					LogFolder:            path.Join(ec.E2EContext.Environment.artifactsPath, "clusters", ec.ClusterName, "move-to"),
+					ClusterctlConfigPath: ec.E2EContext.Environment.ClusterctlConfigPath,
+					FromKubeconfigPath:   ec.E2EContext.Environment.Bootstrap.GetKubeconfigPath(),
+					ToKubeconfigPath:     ec.Refs.Workload.GetKubeconfigPath(),
+					Namespace:            ec.NamespaceName,
+				})
+
+				cluster := &clusterv1.Cluster{}
+				err := e2eCtx.Environment.Bootstrap.GetClient().Get(ctx, client.ObjectKey{
+					Namespace: ec.NamespaceName,
+					Name:      ec.ClusterName,
+				}, cluster)
+				Expect(err).To(Satisfy(apierrors.IsNotFound), "cluster should have been moved")
+
+				cluster = &clusterv1.Cluster{}
+				err = ec.Refs.Workload.GetClient().Get(ctx, client.ObjectKey{
+					Namespace: ec.NamespaceName,
+					Name:      ec.ClusterName,
+				}, cluster)
+				Expect(err).ToNot(HaveOccurred(), "cluster should be present")
+			})
+
+			It("move from workload to bootstrap cluster", func() {
+				Expect(ec).NotTo(BeNil(), "e2e cluster required")
+
+				clusterctl.Move(ctx, clusterctl.MoveInput{
+					LogFolder:            path.Join(ec.E2EContext.Environment.artifactsPath, "clusters", ec.ClusterName, "move-back"),
+					ClusterctlConfigPath: ec.E2EContext.Environment.ClusterctlConfigPath,
+					FromKubeconfigPath:   ec.Refs.Workload.GetKubeconfigPath(),
+					ToKubeconfigPath:     ec.E2EContext.Environment.Bootstrap.GetKubeconfigPath(),
+					Namespace:            ec.NamespaceName,
+				})
+
+				cluster := &clusterv1.Cluster{}
+				err := ec.Refs.Workload.GetClient().Get(ctx, client.ObjectKey{
+					Namespace: ec.NamespaceName,
+					Name:      ec.ClusterName,
+				}, cluster)
+				Expect(err).To(Satisfy(apierrors.IsNotFound), "cluster should have been moved")
+
+				cluster = &clusterv1.Cluster{}
+				err = e2eCtx.Environment.Bootstrap.GetClient().Get(ctx, client.ObjectKey{
+					Namespace: ec.NamespaceName,
+					Name:      ec.ClusterName,
+				}, cluster)
+				Expect(err).ToNot(HaveOccurred(), "cluster should be present")
+			})
+
+			It("delete cluster", func() {
+				ec.Teardown(ctx)
+				// TODO: expect resources to be gone
 			})
-			Expect(ec).ToNot(BeNil())
 		})
 	}
+
+	It("teardown management cluster", func() {
+		e2eCtx.Teardown(context.Background())
+	})
 })
 
 func createE2ECluster(ctx context.Context, e2eCtx *E2EContext, cfg ClusterConfig) *E2ECluster {
 	ec := e2eCtx.NewE2ECluster(cfg)
-	DeferCleanup(ec.Teardown, ctx)
 
 	ec.SetupMetalStackPreconditions(ctx)
 	ec.SetupNamespace(ctx)

test/e2e/frmwrk/shared.go

@@ -527,8 +527,7 @@ func (e2e *E2ECluster) GenerateAndApplyClusterTemplate(ctx context.Context) {
 		// TODO: why does this not work with clusterctl.DefaultInfrastructureProvider?
 		InfrastructureProvider: "capms:v0.6.2",
 		LogFolder:              path.Join(e2e.E2EContext.Environment.artifactsPath, "clusters", e2e.ClusterName),
-		// KubeconfigPath:         "",
-		ClusterctlVariables: e2e.Variables(),
+		ClusterctlVariables:    e2e.Variables(),
 	})
 
 	By("Apply cluster template")
@@ -537,6 +536,12 @@ func (e2e *E2ECluster) GenerateAndApplyClusterTemplate(ctx context.Context) {
 
 	e2e.Refs.Workload = e2e.E2EContext.Environment.Bootstrap.GetWorkloadCluster(ctx, e2e.NamespaceName, e2e.ClusterName)
 
+	err = copyFile(
+		e2e.Refs.Workload.GetKubeconfigPath(),
+		path.Join(e2e.E2EContext.Environment.artifactsPath, "clusters", e2e.ClusterName, "kubeconfig"),
+	)
+	Expect(err).NotTo(HaveOccurred(), "cannot copy workload kubeconfig file")
+
 	e2e.Refs.Cluster = framework.DiscoveryAndWaitForCluster(ctx, framework.DiscoveryAndWaitForClusterInput{
 		Namespace: e2e.NamespaceName,
 		Name:      e2e.ClusterName,
@@ -594,3 +599,28 @@ func deleteClusterAndWait(ctx context.Context, input framework.DeleteClusterAndW
 		})
 	}, retryableOperationTimeout, retryableOperationInterval).Should(BeEmpty(), "There are still Cluster API resources in the %q namespace", input.Cluster.Namespace)
 }
+
+func copyFile(src, dst string) error {
+	sourceFile, err := os.Open(src)
+	if err != nil {
+		return err
+	}
+	defer func() {
+		Expect(sourceFile.Close()).ToNot(HaveOccurred(), "cannot close source file")
+	}()
+
+	destinationFile, err := os.Create(dst)
+	if err != nil {
+		return err
+	}
+	defer func() {
+		Expect(destinationFile.Close()).ToNot(HaveOccurred(), "cannot close destination file")
+	}()
+
+	_, err = io.Copy(destinationFile, sourceFile)
+	if err != nil {
+		return err
+	}
+
+	return nil
+}