Merge remote-tracking branch 'origin/main' into capi-v1.11.2

# Conflicts:
#	go.mod
#	go.sum

Author: simcod

.github/workflows/test-e2e.yml

@@ -10,7 +10,7 @@ on:
       - main
 
 jobs:
-  test-e2e:
+  test-e2e-controller:
     name: E2E Tests
     runs-on: ubuntu-latest
     steps:
@@ -34,7 +34,7 @@ jobs:
       - name: Create kind cluster
         run: kind create cluster
 
-      - name: Running Test e2e
+      - name: Running Test e2e-controller
         run: |
           go mod tidy
-          make test-e2e
+          make test-e2e-controller

.gitignore

@@ -6,6 +6,8 @@
 *.dylib
 bin/*
 test/external-crds
+test/e2e/frmwrk/artifacts
+_artifacts
 Dockerfile.cross
 .release
 
@@ -17,6 +19,7 @@ infrastructure-components.yaml
 
 # Output of the go coverage tool, specifically when used with LiteIDE
 *.out
+junit.e2e_suite.xml
 
 # Go workspace file
 go.work

DEVELOPMENT.md

@@ -204,7 +204,7 @@ export WORKER_MACHINE_SIZE=
 
 export CLUSTER_NAME=
 export NAMESPACE=default
-export KUBERNETES_VERSION=v1.31.6
+export KUBERNETES_VERSION=v1.32.9
 
 export CONTROL_PLANE_MACHINE_COUNT=1
 export WORKER_MACHINE_COUNT=1

Makefile

@@ -77,7 +77,7 @@ update-test-crds:
 	go mod tidy
 	rm -rf test/external-crds
 	mkdir -p test/external-crds/cluster-api
-	cp -f $(shell go list -mod=mod -m -f '{{.Dir}}' all | grep sigs.k8s.io/cluster-api)/config/crd/bases/* test/external-crds/cluster-api
+	cp -f $(shell go list -mod=mod -m -f '{{.Dir}}' all | grep sigs.k8s.io/cluster-api@)/config/crd/bases/* test/external-crds/cluster-api
 
 .PHONY: fmt
 fmt: ## Run go fmt against code.
@@ -88,15 +88,15 @@ vet: ## Run go vet against code.
 	go vet ./...
 
 .PHONY: test
-test: manifests generate update-test-crds fmt vet envtest ## Run tests.
-	KUBEBUILDER_ASSETS="$(shell $(ENVTEST) use $(ENVTEST_K8S_VERSION) --bin-dir $(LOCALBIN) -p path)" go test $$(go list ./... | grep -v /e2e) -coverprofile cover.out
+test: manifests generate update-test-crds fmt vet envtest ginkgo ## Run tests.
+	KUBEBUILDER_ASSETS="$(shell $(ENVTEST) use $(ENVTEST_K8S_VERSION) --bin-dir $(LOCALBIN) -p path)" $(GINKGO) -r --junit-report="junit.e2e_suite.xml" --cover -coverprofile cover.out $$(go list ./... | grep -v /e2e | sed "s|github.com/metal-stack/cluster-api-provider-metal-stack|.|")
 
 # The default setup assumes Kind is pre-installed and builds/loads the Manager Docker image locally.
 # Prometheus and CertManager are installed by default; skip with:
 # - PROMETHEUS_INSTALL_SKIP=true
 # - CERT_MANAGER_INSTALL_SKIP=true
-.PHONY: test-e2e
-test-e2e: manifests generate update-test-crds fmt vet ## Run the e2e tests. Expected an isolated environment using Kind.
+.PHONY: test-e2e-controller
+test-e2e-controller: manifests generate update-test-crds fmt vet ## Run the e2e tests. Expected an isolated environment using Kind.
 	@command -v kind >/dev/null 2>&1 || { \
 		echo "Kind is not installed. Please install Kind manually."; \
 		exit 1; \
@@ -107,6 +107,46 @@ test-e2e: manifests generate update-test-crds fmt vet ## Run the e2e tests. Expe
 	}
 	go test ./test/e2e/ -v -ginkgo.v
 
+E2E_METAL_API_URL ?= "$(METALCTL_API_URL)"
+E2E_METAL_API_HMAC ?= "$(METALCTL_HMAC)"
+E2E_METAL_API_HMAC_AUTH_TYPE ?= "$(or $(METALCTL_HMAC_AUTH_TYPE),Metal-Admin)"
+E2E_METAL_PROJECT_ID ?= "00000000-0000-0000-0000-000000000001"
+E2E_METAL_PARTITION ?= "mini-lab"
+E2E_METAL_PUBLIC_NETWORK ?= "internet-mini-lab"
+E2E_KUBERNETES_VERSIONS ?= "v1.32.9"
+E2E_CONTROL_PLANE_MACHINE_IMAGE_PREFIX ?= "capms-ubuntu-"
+E2E_CONTROL_PLANE_MACHINE_SIZE ?= "v1-small-x86"
+E2E_WORKER_MACHINE_IMAGE_PREFIX ?= "capms-ubuntu-"
+E2E_WORKER_MACHINE_SIZE ?= "v1-small-x86"
+E2E_FIREWALL_IMAGE ?= "firewall-ubuntu-3.0"
+E2E_FIREWALL_SIZE ?= "v1-small-x86"
+E2E_FIREWALL_NETWORKS ?= "internet-mini-lab"
+ARTIFACTS ?= "$(PWD)/_artifacts"
+
+.PHONY: test-e2e
+test-e2e: manifests generate fmt vet ginkgo
+	rm -rf test/e2e/frmwrk/artifacts
+
+	mkdir -p $(ARTIFACTS)/config/target
+	kubectl kustomize test/e2e/frmwrk/config/target/base -o $(ARTIFACTS)/config/target/base.yaml
+
+	@METAL_API_URL=$(E2E_METAL_API_URL) \
+	METAL_API_HMAC=$(E2E_METAL_API_HMAC) \
+	METAL_API_HMAC_AUTH_TYPE=$(E2E_METAL_API_HMAC_AUTH_TYPE) \
+	METAL_PROJECT_ID=$(E2E_METAL_PROJECT_ID) \
+	METAL_PARTITION=$(E2E_METAL_PARTITION) \
+	METAL_PUBLIC_NETWORK=$(E2E_METAL_PUBLIC_NETWORK) \
+	E2E_KUBERNETES_VERSIONS=$(E2E_KUBERNETES_VERSIONS) \
+	E2E_CONTROL_PLANE_MACHINE_IMAGE_PREFIX=$(E2E_CONTROL_PLANE_MACHINE_IMAGE_PREFIX) \
+	CONTROL_PLANE_MACHINE_SIZE=$(E2E_CONTROL_PLANE_MACHINE_SIZE) \
+	E2E_WORKER_MACHINE_IMAGE_PREFIX=$(E2E_WORKER_MACHINE_IMAGE_PREFIX) \
+	WORKER_MACHINE_SIZE=$(E2E_WORKER_MACHINE_SIZE) \
+	FIREWALL_IMAGE=$(E2E_FIREWALL_IMAGE) \
+	FIREWALL_SIZE=$(E2E_FIREWALL_SIZE) \
+	FIREWALL_NETWORKS=$(E2E_FIREWALL_NETWORKS) \
+	ARTIFACTS=$(ARTIFACTS) \
+	$(GINKGO) -vv -r --junit-report="junit.e2e_suite.xml" --output-dir="$(ARTIFACTS)" -timeout 60m ./test/e2e/frmwrk 
+
 .PHONY: lint
 lint: golangci-lint ## Run golangci-lint linter
 	$(GOLANGCI_LINT) run
@@ -199,13 +239,15 @@ KUBECTL ?= kubectl
 KUSTOMIZE ?= $(LOCALBIN)/kustomize
 CONTROLLER_GEN ?= $(LOCALBIN)/controller-gen
 ENVTEST ?= $(LOCALBIN)/setup-envtest
+GINKGO ?= $(LOCALBIN)/ginkgo
 GOLANGCI_LINT = $(LOCALBIN)/golangci-lint
 
 ## Tool Versions
 KUSTOMIZE_VERSION ?= v5.4.3
 CONTROLLER_TOOLS_VERSION ?= v0.16.4
 ENVTEST_VERSION ?= release-0.19
 GOLANGCI_LINT_VERSION ?= v1.61.0
+GINKGO_VERSION ?= v2.23.3
 
 .PHONY: kustomize
 kustomize: $(KUSTOMIZE) ## Download kustomize locally if necessary.
@@ -227,6 +269,11 @@ golangci-lint: $(GOLANGCI_LINT) ## Download golangci-lint locally if necessary.
 $(GOLANGCI_LINT): $(LOCALBIN)
 	$(call go-install-tool,$(GOLANGCI_LINT),github.com/golangci/golangci-lint/cmd/golangci-lint,$(GOLANGCI_LINT_VERSION))
 
+.PHONY: ginkgo
+ginkgo: $(GINKGO) ## Download setup-envtest locally if necessary.
+$(GINKGO): $(LOCALBIN)
+	$(call go-install-tool,$(GINKGO),github.com/onsi/ginkgo/v2/ginkgo,$(GINKGO_VERSION))
+
 # go-install-tool will 'go install' any package with custom target and name of binary, if it doesn't exist
 # $1 - target path with name of binary
 # $2 - package url which can be installed

capi-lab/Makefile

@@ -7,7 +7,7 @@ KUBECONFIG := $(shell pwd)/mini-lab/.kubeconfig
 MINI_LAB_FLAVOR=capms
 
 CLUSTER_NAME ?= metal-test
-KUBERNETES_VERSION ?= 1.31.6
+KUBERNETES_VERSION ?= 1.32.9
 
 METAL_API_URL=http://metal.203.0.113.1.nip.io:8080
 METAL_API_HMAC=metal-edit
@@ -19,9 +19,9 @@ METALCTL_HMAC_AUTH_TYPE=Metal-Edit
 METAL_PARTITION ?= mini-lab
 METAL_PROJECT_ID ?= 00000000-0000-0000-0000-000000000001
 
-CONTROL_PLANE_MACHINE_IMAGE ?= ubuntu-24.0-k8s-$(KUBERNETES_VERSION)
+CONTROL_PLANE_MACHINE_IMAGE ?= capms-ubuntu-$(KUBERNETES_VERSION)
 CONTROL_PLANE_MACHINE_SIZE ?= v1-small-x86
-WORKER_MACHINE_IMAGE ?= ubuntu-24.0-k8s-$(KUBERNETES_VERSION)
+WORKER_MACHINE_IMAGE ?= capms-ubuntu-$(KUBERNETES_VERSION)
 WORKER_MACHINE_SIZE ?= v1-small-x86
 
 IMG ?= ghcr.io/metal-stack/cluster-api-metal-stack-controller:latest
@@ -46,6 +46,7 @@ cleanup:
 dev-env:
 	@echo "export METALCTL_API_URL=${METALCTL_API_URL}"
 	@echo "export METALCTL_HMAC=${METALCTL_HMAC}"
+	@echo "export METALCTL_HMAC_AUTH_TYPE=${METALCTL_HMAC_AUTH_TYPE}"
 	@echo "export KUBECONFIG=${KUBECONFIG}"
 
 .PHONY: controller

capi-lab/mini-lab

@@ -17,10 +17,22 @@ metal_api_images:
     url: https://images.metal-stack.io/metal-os/stable/firewall/3.0-ubuntu/img.tar.lz4
     features:
       - firewall
-  - id: ubuntu-24.4
-    name: Ubuntu 24.04 with Kubernetes 1.30.6
-    description: Ubuntu 24.04 with Kubernetes 1.30.6
-    url: https://images.metal-stack.io/metal-os/pull_requests/273-cluster-api-k8s-v1.30.6/cluster-api/cluster-api-k8s-1.30.6/img.tar.lz4
+  - id: capms-ubuntu-1.34.1
+    name: Ubuntu 24.04 with Kubernetes 1.34.1
+    description: Ubuntu 24.04 with Kubernetes 1.34.1
+    url: https://images.metal-stack.io/metal-os/pull_requests/352-capi-images/capms-ubuntu/1.34.1/img.tar.lz4
+    features:
+      - machine
+  - id: capms-ubuntu-1.33.5
+    name: Ubuntu 24.04 with Kubernetes 1.33.5
+    description: Ubuntu 24.04 with Kubernetes 1.33.5
+    url: https://images.metal-stack.io/metal-os/pull_requests/352-capi-images/capms-ubuntu/1.33.5/img.tar.lz4
+    features:
+      - machine
+  - id: capms-ubuntu-1.32.9
+    name: Ubuntu 24.04 with Kubernetes 1.32.9
+    description: Ubuntu 24.04 with Kubernetes 1.32.9
+    url: https://images.metal-stack.io/metal-os/pull_requests/352-capi-images/capms-ubuntu/1.32.9/img.tar.lz4
     features:
       - machine
 

capi-lab/mini-lab-overrides/extra-vars.yaml

@@ -0,0 +1,2 @@
+resources:
+  - ./metal-ccm.yaml

config/target-cluster/kustomization.yaml

@@ -0,0 +1,105 @@
+package frmwrk
+
+import (
+	"context"
+	"fmt"
+	"os"
+	"path"
+	"strings"
+
+	"github.com/drone/envsubst/v2"
+	. "github.com/onsi/ginkgo/v2"
+	. "github.com/onsi/gomega"
+
+	"sigs.k8s.io/cluster-api/test/framework"
+)
+
+var _ = Describe("Basic Cluster Creation", Ordered, func() {
+	var (
+		ec *E2ECluster
+	)
+
+	BeforeAll(func() {
+		e2eCtx = NewE2EContext()
+		e2eCtx.ProvideBootstrapCluster()
+		e2eCtx.CreateClusterctlConfig(context.TODO())
+		e2eCtx.InitManagementCluster(context.TODO())
+
+		DeferCleanup(e2eCtx.Teardown, context.TODO())
+	})
+
+	BeforeEach(func() {
+		ec = nil
+	})
+
+	kubernetesVersions := strings.Split(os.Getenv("E2E_KUBERNETES_VERSIONS"), ",")
+	Expect(kubernetesVersions).ToNot(BeEmpty(), "E2E_KUBERNETES_VERSIONS must be set")
+
+	for i, v := range kubernetesVersions {
+		It(fmt.Sprintf("create new cluster with kubernetes %s", v), func() {
+			ctx := context.Background()
+
+			ec = createE2ECluster(ctx, e2eCtx, ClusterConfig{
+				SpecName:                 "basic-cluster-creation-" + v,
+				NamespaceName:            fmt.Sprintf("e2e-basic-cluster-creation-%d", i),
+				ClusterName:              fmt.Sprintf("simple-%d", i),
+				KubernetesVersion:        v,
+				ControlPlaneMachineImage: os.Getenv("E2E_CONTROL_PLANE_MACHINE_IMAGE_PREFIX") + strings.TrimPrefix(v, "v"),
+				ControlPlaneMachineCount: 1,
+				WorkerMachineImage:       os.Getenv("E2E_WORKER_MACHINE_IMAGE_PREFIX") + strings.TrimPrefix(v, "v"),
+				WorkerMachineCount:       1,
+			})
+			Expect(ec).ToNot(BeNil())
+		})
+	}
+})
+
+func createE2ECluster(ctx context.Context, e2eCtx *E2EContext, cfg ClusterConfig) *E2ECluster {
+	ec := e2eCtx.NewE2ECluster(cfg)
+	DeferCleanup(ec.Teardown, ctx)
+
+	ec.SetupMetalStackPreconditions(ctx)
+	ec.SetupNamespace(ctx)
+	ec.GenerateAndApplyClusterTemplate(ctx)
+
+	By("Wait for cluster")
+	controlPlane := framework.GetKubeadmControlPlaneByCluster(ctx, framework.GetKubeadmControlPlaneByClusterInput{
+		Lister:      e2eCtx.Environment.Bootstrap.GetClient(),
+		ClusterName: ec.Refs.Cluster.Name,
+		Namespace:   ec.Refs.Cluster.Namespace,
+	})
+
+	framework.DiscoveryAndWaitForCluster(ctx, framework.DiscoveryAndWaitForClusterInput{
+		Getter:    e2eCtx.Environment.Bootstrap.GetClient(),
+		Namespace: ec.Refs.Cluster.Namespace,
+		Name:      ec.Refs.Cluster.Name,
+	}, e2eCtx.E2EConfig.GetIntervals("default", "wait-cluster")...)
+
+	Expect(controlPlane).To(Not(BeNil()))
+
+	By("Wait for CNI and CCM")
+	targetTemplate, err := os.ReadFile(path.Join(e2eCtx.Environment.artifactsPath, "config", "target", "base.yaml"))
+	Expect(err).ToNot(HaveOccurred())
+
+	vars := ec.Variables()
+	targetResources, err := envsubst.Eval(string(targetTemplate), func(varName string) string {
+		return vars[varName]
+	})
+	Expect(err).ToNot(HaveOccurred())
+
+	Eventually(func() error {
+		return ec.Refs.Workload.CreateOrUpdate(ctx, []byte(targetResources))
+	}, "5m", "15s").Should(Succeed())
+
+	By("Wait for kubeadm control plane")
+	framework.DiscoveryAndWaitForControlPlaneInitialized(ctx, framework.DiscoveryAndWaitForControlPlaneInitializedInput{
+		Lister:  e2eCtx.Environment.Bootstrap.GetClient(),
+		Cluster: ec.Refs.Cluster,
+	}, e2eCtx.E2EConfig.GetIntervals("default", "wait-control-plane")...)
+
+	framework.WaitForClusterToProvision(ctx, framework.WaitForClusterToProvisionInput{
+		Cluster: ec.Refs.Cluster,
+		Getter:  e2eCtx.Environment.Bootstrap.GetClient(),
+	}, e2eCtx.E2EConfig.GetIntervals("default", "wait-cluster-provisioned")...)
+	return ec
+}