test: successful :)

Author: vknabel

Makefile

@@ -113,7 +113,7 @@ E2E_METAL_API_HMAC_AUTH_TYPE ?= "$(or $(METALCTL_HMAC_AUTH_TYPE),Metal-Admin)"
 E2E_METAL_PROJECT_ID ?= "00000000-0000-0000-0000-000000000001"
 E2E_METAL_PARTITION ?= "mini-lab"
 E2E_METAL_PUBLIC_NETWORK ?= "internet-mini-lab"
-E2E_KUBERNETES_VERSIONS ?= "v1.32.9,v1.33.5,v1.34.1"
+E2E_KUBERNETES_VERSIONS ?= "v1.32.9"
 E2E_CONTROL_PLANE_MACHINE_IMAGE_PREFIX ?= "capms-ubuntu-"
 E2E_CONTROL_PLANE_MACHINE_SIZE ?= "v1-small-x86"
 E2E_WORKER_MACHINE_IMAGE_PREFIX ?= "capms-ubuntu-"
@@ -143,7 +143,7 @@ test-e2e: manifests generate fmt vet
 	FIREWALL_IMAGE=$(E2E_FIREWALL_IMAGE) \
 	FIREWALL_SIZE=$(E2E_FIREWALL_SIZE) \
 	FIREWALL_NETWORKS=$(E2E_FIREWALL_NETWORKS) \
-	go test ./test/e2e/frmwrk -v ginkgo -vv 
+	go test -v ./test/e2e/frmwrk -timeout 60m -v ginkgo -vv
 
 .PHONY: lint
 lint: golangci-lint ## Run golangci-lint linter

config/manager/kustomization.yaml

@@ -4,5 +4,5 @@ apiVersion: kustomize.config.k8s.io/v1beta1
 kind: Kustomization
 images:
 - name: controller
-  newName: ghcr.io/metal-stack/capms-controller
+  newName: ghcr.io/metal-stack/cluster-api-metal-stack-controller
   newTag: latest

test/e2e/frmwrk/bootstrap_test.go

@@ -7,11 +7,10 @@ import (
 	"path"
 	"strings"
 
+	"github.com/drone/envsubst/v2"
 	. "github.com/onsi/ginkgo/v2"
 	. "github.com/onsi/gomega"
 
-	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
-	controlplanev1 "sigs.k8s.io/cluster-api/controlplane/kubeadm/api/v1beta1"
 	"sigs.k8s.io/cluster-api/test/framework"
 )
 
@@ -36,13 +35,13 @@ var _ = Describe("Basic Cluster Creation", Ordered, func() {
 	kubernetesVersions := strings.Split(os.Getenv("E2E_KUBERNETES_VERSIONS"), ",")
 	Expect(kubernetesVersions).ToNot(BeEmpty(), "E2E_KUBERNETES_VERSIONS must be set")
 
-	for _, v := range kubernetesVersions {
+	for i, v := range kubernetesVersions {
 		It(fmt.Sprintf("create new cluster with kubernetes %s", v), func() {
 			ctx := context.Background()
 
 			ec = createE2ECluster(ctx, e2eCtx, ClusterConfig{
 				SpecName:                 "basic-cluster-creation-" + v,
-				NamespaceName:            "simple-" + v,
+				NamespaceName:            fmt.Sprintf("e2e-basic-cluster-creation-%d", i),
 				ClusterName:              "simple",
 				KubernetesVersion:        v,
 				ControlPlaneMachineImage: os.Getenv("E2E_CONTROL_PLANE_MACHINE_IMAGE_PREFIX") + strings.TrimPrefix(v, "v"),
@@ -78,38 +77,29 @@ func createE2ECluster(ctx context.Context, e2eCtx *E2EContext, cfg ClusterConfig
 
 	Expect(controlPlane).To(Not(BeNil()))
 
-	targetResources, err := os.ReadFile(path.Join(e2eCtx.Environment.artifactsPath, "config", "target", "base.yaml"))
+	By("Wait for CNI and CCM")
+	targetTemplate, err := os.ReadFile(path.Join(e2eCtx.Environment.artifactsPath, "config", "target", "base.yaml"))
 	Expect(err).ToNot(HaveOccurred())
 
-	err = ec.Refs.Workload.CreateOrUpdate(ctx, targetResources)
+	vars := ec.Variables()
+	targetResources, err := envsubst.Eval(string(targetTemplate), func(varName string) string {
+		return vars[varName]
+	})
 	Expect(err).ToNot(HaveOccurred())
 
-	framework.WaitForKubeadmControlPlaneMachinesToExist(ctx, framework.WaitForKubeadmControlPlaneMachinesToExistInput{
-		Lister:       e2eCtx.Environment.Bootstrap.GetClient(),
-		Cluster:      cluster,
-		ControlPlane: controlPlane,
-	}, e2eCtx.E2EConfig.GetIntervals("default", "wait-control-plane")...)
+	Eventually(func() error {
+		return ec.Refs.Workload.CreateOrUpdate(ctx, []byte(targetResources))
+	}, "5m", "15s").Should(Succeed())
+
+	By("Wait for kubeadm control plane")
 	framework.DiscoveryAndWaitForControlPlaneInitialized(ctx, framework.DiscoveryAndWaitForControlPlaneInitializedInput{
 		Lister:  e2eCtx.Environment.Bootstrap.GetClient(),
 		Cluster: cluster,
 	}, e2eCtx.E2EConfig.GetIntervals("default", "wait-control-plane")...)
 
-	By("Wait for control plane")
-	framework.WaitForControlPlaneToBeReady(ctx, framework.WaitForControlPlaneToBeReadyInput{
-		Getter: e2eCtx.Environment.Bootstrap.GetClient(),
-		ControlPlane: &controlplanev1.KubeadmControlPlane{
-			ObjectMeta: metav1.ObjectMeta{
-				Name:      ec.ClusterName,
-				Namespace: ec.NamespaceName,
-			},
-		},
-	}, e2eCtx.E2EConfig.GetIntervals("default", "wait-control-plane")...)
-
-	framework.WaitForClusterMachinesReady(ctx, framework.WaitForClusterMachinesReadyInput{
-		Cluster:    cluster,
-		GetLister:  e2eCtx.Environment.Bootstrap.GetClient(),
-		NodeGetter: e2eCtx.Environment.Bootstrap.GetClient(),
-	}, e2eCtx.E2EConfig.GetIntervals("default", "wait-workers")...)
-
+	framework.WaitForClusterToProvision(ctx, framework.WaitForClusterToProvisionInput{
+		Cluster: cluster,
+		Getter:  e2eCtx.Environment.Bootstrap.GetClient(),
+	}, e2eCtx.E2EConfig.GetIntervals("default", "wait-cluster-provisioned")...)
 	return ec
 }

test/e2e/frmwrk/config/capi-e2e-config.yaml

@@ -109,9 +109,6 @@ providers:
     files:
       - sourcePath: "../../../../metadata.yaml"
       - sourcePath: "../../../../config/clusterctl-templates/cluster-template.yaml"
-      - sourcePath: "../../../../config/cluster-resource-sets/calico-resource-set.yaml"
-      - sourcePath: "../../../../config/cluster-resource-sets/metal-ccm-resource-set.yaml"
-      - sourcePath: "../../../../config/cluster-resource-sets/metallb-resource-set.yaml"
     replacements:
       - old: --metrics-addr=127.0.0.1:8080
         new: --metrics-addr=:8080
@@ -137,6 +134,7 @@ variables:
 intervals:
   default/wait-controllers: ["4m", "10s"]
   default/wait-cluster: ["5m", "10s"]
+  metal-stack/wait-firewall-allocate: ["5m", "10s"]
   default/wait-control-plane: ["10m", "10s"]
   default/wait-worker-nodes: ["5m", "10s"]
   default/wait-machine-pool-nodes: ["5m", "10s"]

test/e2e/frmwrk/shared.go

@@ -20,7 +20,6 @@ import (
 	metalmodels "github.com/metal-stack/metal-go/api/models"
 
 	corev1 "k8s.io/api/core/v1"
-	apierrors "k8s.io/apimachinery/pkg/api/errors"
 	"k8s.io/apimachinery/pkg/apis/meta/v1/unstructured"
 	"k8s.io/apimachinery/pkg/runtime"
 	"k8s.io/utils/ptr"
@@ -285,6 +284,33 @@ func (e2e *E2EContext) NewE2ECluster(cfg ClusterConfig) *E2ECluster {
 	}
 }
 
+func (e2e *E2ECluster) Variables() map[string]string {
+	vars := make(map[string]string)
+
+	for k := range e2e.E2EContext.E2EConfig.Variables {
+		vars[k] = e2e.E2EContext.envOrVar(k)
+	}
+
+	vars["METAL_API_URL"] = e2e.E2EContext.envOrVar("METAL_API_URL")
+	vars["METAL_API_HMAC"] = e2e.E2EContext.envOrVar("METAL_API_HMAC")
+	vars["METAL_API_HMAC_AUTH_TYPE"] = e2e.E2EContext.envOrVar("METAL_API_HMAC_AUTH_TYPE")
+
+	vars["NAMESPACE"] = e2e.NamespaceName
+	vars["METAL_PROJECT_ID"] = e2e.E2EContext.Environment.project
+	vars["METAL_PARTITION"] = e2e.E2EContext.Environment.partition
+	vars["METAL_NODE_NETWORK_ID"] = *e2e.Refs.NodeNetwork.ID
+	vars["FIREWALL_MACHINE_SIZE"] = e2e.FirewallSize
+	vars["FIREWALL_MACHINE_IMAGE"] = e2e.FirewallImage
+	vars["FIREWALL_MACHINE_NETWORKS"] = "[" + strings.Join(e2e.FirewallNetworks, ",") + "]"
+	vars["CONTROL_PLANE_IP"] = e2e.ControlPlaneIP
+	vars["CONTROL_PLANE_MACHINE_SIZE"] = e2e.ControlPlaneMachineSize
+	vars["CONTROL_PLANE_MACHINE_IMAGE"] = e2e.ControlPlaneMachineImage
+	vars["WORKER_MACHINE_SIZE"] = e2e.WorkerMachineSize
+	vars["WORKER_MACHINE_IMAGE"] = e2e.WorkerMachineImage
+
+	return vars
+}
+
 // common
 
 func (e2e *E2ECluster) SetupNamespace(ctx context.Context) *corev1.Namespace {
@@ -322,7 +348,6 @@ func (e2e *E2ECluster) SetupMetalStackPreconditions(ctx context.Context) {
 
 func (e2e *E2ECluster) Teardown(ctx context.Context) {
 	e2e.teardownCluster(ctx)
-	e2e.teardownClusterResourceSets(ctx)
 	e2e.teardownControlPlaneIP(ctx)
 	e2e.teardownFirewall(ctx)
 	e2e.teardownNodeNetwork(ctx)
@@ -422,10 +447,15 @@ func (e2e *E2ECluster) setupFirewall(ctx context.Context) {
 		},
 	}
 
-	fw, err := e2e.E2EContext.Environment.Metal.Firewall().AllocateFirewall(metalfw.NewAllocateFirewallParamsWithContext(ctx).WithBody(fcr), nil)
-	Expect(err).ToNot(HaveOccurred(), "failed to allocate firewall")
+	Eventually(func() error {
+		fw, err := e2e.E2EContext.Environment.Metal.Firewall().AllocateFirewall(metalfw.NewAllocateFirewallParamsWithContext(ctx).WithBody(fcr), nil)
+		if err != nil {
+			return err
+		}
 
-	e2e.Refs.Firewall = fw.Payload
+		e2e.Refs.Firewall = fw.Payload
+		return nil
+	}, e2e.E2EContext.E2EConfig.GetIntervals("metal-stack", "wait-firewall-allocate")...).ShouldNot(HaveOccurred(), "firewall not available")
 }
 
 func (e2e *E2ECluster) teardownFirewall(ctx context.Context) {
@@ -440,6 +470,10 @@ func (e2e *E2ECluster) teardownFirewall(ctx context.Context) {
 }
 
 func (e2e *E2ECluster) setupControlPlaneIP(ctx context.Context) {
+	if e2e.ControlPlaneIP != "" {
+		return
+	}
+
 	By("Setup Control Plane IP")
 
 	ipr := &metalmodels.V1IPAllocateRequest{
@@ -451,12 +485,15 @@ func (e2e *E2ECluster) setupControlPlaneIP(ctx context.Context) {
 			fmt.Sprintf("%s=%s", "e2e-test", e2e.SpecName),
 		},
 		Networkid: ptr.To(e2e.E2EContext.Environment.publicNetwork),
+		Type:      ptr.To(metalmodels.V1IPAllocateRequestTypeStatic),
 	}
 
 	ip, err := e2e.E2EContext.Environment.Metal.IP().AllocateIP(metalip.NewAllocateIPParamsWithContext(ctx).WithBody(ipr), nil)
 	Expect(err).ToNot(HaveOccurred(), "failed to allocate control plane IP")
+	Expect(ip.Payload.Ipaddress).NotTo(BeNil(), "allocated control plane IP has no IP address")
 
 	e2e.Refs.ControlPlaneIP = ip.Payload
+	e2e.ControlPlaneIP = *e2e.Refs.ControlPlaneIP.Ipaddress
 }
 
 func (e2e *E2ECluster) teardownControlPlaneIP(ctx context.Context) {
@@ -488,22 +525,7 @@ func (e2e *E2ECluster) GenerateAndApplyClusterTemplate(ctx context.Context) {
 		InfrastructureProvider: "capms:v0.6.1",
 		LogFolder:              path.Join(e2e.E2EContext.Environment.artifactsPath, "clusters", e2e.ClusterName),
 		// KubeconfigPath:         "",
-		ClusterctlVariables: map[string]string{
-			// "METAL_API_URL":               "",
-			// "METAL_API_HMAC":              "",
-			"METAL_PROJECT_ID": e2e.E2EContext.Environment.project,
-			// "POD_CIDR":                    "",
-			"METAL_PARTITION":             e2e.E2EContext.Environment.partition,
-			"METAL_NODE_NETWORK_ID":       *e2e.Refs.NodeNetwork.ID,
-			"FIREWALL_MACHINE_SIZE":       e2e.FirewallSize,
-			"FIREWALL_MACHINE_IMAGE":      e2e.FirewallImage,
-			"FIREWALL_MACHINE_NETWORKS":   "[" + strings.Join(e2e.FirewallNetworks, ",") + "]",
-			"CONTROL_PLANE_IP":            e2e.ControlPlaneIP,
-			"CONTROL_PLANE_MACHINE_SIZE":  e2e.ControlPlaneMachineSize,
-			"CONTROL_PLANE_MACHINE_IMAGE": e2e.ControlPlaneMachineImage,
-			"WORKER_MACHINE_SIZE":         e2e.WorkerMachineSize,
-			"WORKER_MACHINE_IMAGE":        e2e.WorkerMachineImage,
-		},
+		ClusterctlVariables: e2e.Variables(),
 	})
 
 	By("Apply cluster template")
@@ -535,20 +557,6 @@ func (e2e *E2ECluster) teardownCluster(ctx context.Context) {
 	}, e2e.E2EContext.E2EConfig.GetIntervals("default", "wait-delete-cluster")...)
 }
 
-func (e2e *E2ECluster) teardownClusterResourceSets(ctx context.Context) {
-	sets := framework.GetClusterResourceSets(ctx, framework.GetClusterResourceSetsInput{
-		Lister:    e2e.E2EContext.Environment.Bootstrap.GetClient(),
-		Namespace: e2e.NamespaceName,
-	})
-
-	for _, s := range sets {
-		err := e2e.E2EContext.Environment.Bootstrap.GetClient().Delete(ctx, s)
-		if err != nil && !apierrors.IsNotFound(err) {
-			Expect(err).NotTo(HaveOccurred(), "failed to delete cluster resourceset")
-		}
-	}
-}
-
 // deleteClusterAndWait deletes a cluster object and waits for it to be gone.
 // TODO: remove once cluster expectation has been fixed in framework
 func deleteClusterAndWait(ctx context.Context, input framework.DeleteClusterAndWaitInput, intervals ...any) {