Merge remote-tracking branch 'origin/main' into test-k8s-upgrade

# Conflicts:
#	test/e2e/frmwrk/shared_cases.go

Author: simcod

.gitignore

@@ -11,6 +11,7 @@ _artifacts
 Dockerfile.cross
 .release
 
+infrastructure-metal-stack
 infrastructure-components.yaml
 .capms-cluster-kubeconfig.yaml
 

DEVELOPMENT.md

@@ -44,40 +44,8 @@ kubectl get secret metal-test-kubeconfig -o jsonpath='{.data.value}' | base64 -d
 clusterctl get kubeconfig metal-test > capi-lab/.capms-cluster-kubeconfig.yaml
 ```
 
-It is now expected to deploy a CNI to the cluster:
-
-```bash
-kubectl --kubeconfig=capi-lab/.capms-cluster-kubeconfig.yaml create -f https://raw.githubusercontent.com/projectcalico/calico/v3.28.2/manifests/tigera-operator.yaml
-cat <<EOF | kubectl --kubeconfig=capi-lab/.capms-cluster-kubeconfig.yaml create -f -
-apiVersion: operator.tigera.io/v1
-kind: Installation
-metadata:
-  name: default
-spec:
-  # Configures Calico networking.
-  calicoNetwork:
-    bgp: Disabled
-    ipPools:
-    - name: default-ipv4-ippool
-      blockSize: 26
-      cidr: 10.240.0.0/12
-      encapsulation: None
-    mtu: 1440
-  cni:
-    ipam:
-      type: HostLocal
-    type: Calico
-EOF
-```
-
-> [!note]
-> Actually, Calico should be configured using BGP (no overlay), eBPF and DSR. An example will be proposed in this repository at a later point in time.
-
 The node's provider ID is provided by the [metal-ccm](https://github.com/metal-stack/metal-ccm), which needs to be deployed into the cluster:
 
-```bash
-make -C capi-lab deploy-metal-ccm
-```
 
 If you want to provide service's of type load balancer through MetalLB by the metal-ccm, you need to deploy MetalLB:
 
@@ -244,7 +212,7 @@ export control_plane_machine_id=
 metalctl machine console --ipmi $control_plane_machine_id
 # ip r
 # sudo systemctl restart kubeadm
-# crictl ps 
+# crictl ps
 # ~.
 
 clusterctl get kubeconfig > capms-cluster.kubeconfig
@@ -279,12 +247,15 @@ watch kubectl -n $NAMESPACE --kubeconfig kind-bootstrap.kubeconfig get cluster,m
 # until everything is ready
 ```
 
+> [!note]
+> Actually, Calico should be configured using BGP (no overlay), eBPF and DSR. An example will be proposed in this repository at a later point in time.
+
 Now you are able to move the cluster resources as you wish:
 
 ```bash
 clusterctl init --infrastructure metal-stack --kubeconfig capms-cluster.kubeconfig
 
-clusterctl move -n $NAMESPACE --kubeconfig kind-bootstrap.kubeconfig --to-kubeconfig capms-cluster.kubeconfig 
+clusterctl move -n $NAMESPACE --kubeconfig kind-bootstrap.kubeconfig --to-kubeconfig capms-cluster.kubeconfig
 # everything as expected
 kubectl --kubeconfig -n $NAMESPACE kind-bootstrap.kubeconfig get cluster,metalstackcluster,machine,metalstackmachine,kubeadmcontrolplanes,kubeadmconfigs
 kubectl --kubeconfig -n $NAMESPACE capms-cluster.kubeconfig get cluster,metalstackcluster,machine,metalstackmachine,kubeadmcontrolplanes,kubeadmconfigs

Makefile

@@ -46,15 +46,29 @@ help: ## Display this help.
 
 ##@ Releases
 
+LATEST_RELEASE_TAG := $(shell git describe --tags `git rev-list --tags --max-count=1`)
+
 .PHONY: release-manifests
 release-manifests: $(KUSTOMIZE) build-installer ## Builds the manifests to publish with a release
 	mkdir -p $(RELEASE_DIR)
 	$(KUSTOMIZE) build config/default > $(RELEASE_DIR)/infrastructure-components.yaml
 	sed -i 's!image: $(IMG_NAME):latest!image: $(IMG_NAME):$(IMG_TAG)!' $(RELEASE_DIR)/infrastructure-components.yaml
 	cp metadata.yaml $(RELEASE_DIR)/metadata.yaml
-	cp config/clusterctl-templates/cluster-template.yaml $(RELEASE_DIR)/cluster-template.yaml
+	cp config/clusterctl-templates/cluster-template*.yaml $(RELEASE_DIR)/
 	cp config/clusterctl-templates/example_variables.rc $(RELEASE_DIR)/example_variables.rc
 
+ifneq ($(CI),true)
+	# for devel purposes with local overwrite in clusterctl.yaml
+	# $ cat ~/.config/cluster-api/clusterctl.yaml                                                                                                                                                                                                           1.25.3 13:33:07
+	# providers:
+	#   - name: "metal-stack"
+	#     # url: "https://github.com/metal-stack/cluster-api-provider-metal-stack/releases/latest/download/infrastructure-components.yaml"
+	#     url: <your-repo-path>/infrastructure-metal-stack/$(LATEST_RELEASE_TAG)/infrastructure-components.yaml
+	#     type: InfrastructureProvider
+	rm -rf infrastructure-metal-stack
+	mkdir -p infrastructure-metal-stack && cd infrastructure-metal-stack && ln -s ../.release $(LATEST_RELEASE_TAG)
+endif
+
 ##@ Development
 
 .PHONY: push-to-capi-lab
@@ -111,6 +125,7 @@ E2E_METAL_API_URL ?= "$(METALCTL_API_URL)"
 E2E_METAL_API_HMAC ?= "$(METALCTL_HMAC)"
 E2E_METAL_API_HMAC_AUTH_TYPE ?= "$(or $(METALCTL_HMAC_AUTH_TYPE),Metal-Admin)"
 E2E_METAL_PROJECT_ID ?= "00000000-0000-0000-0000-000000000001"
+E2E_METAL_PROJECT_NAME ?= "test"
 E2E_METAL_PARTITION ?= "mini-lab"
 E2E_METAL_PUBLIC_NETWORK ?= "internet-mini-lab"
 E2E_KUBERNETES_VERSIONS ?= "v1.32.9"
@@ -122,20 +137,19 @@ E2E_FIREWALL_IMAGE ?= "firewall-ubuntu-3.0"
 E2E_FIREWALL_SIZE ?= "v1-small-x86"
 E2E_FIREWALL_NETWORKS ?= "internet-mini-lab"
 ARTIFACTS ?= "$(PWD)/_artifacts"
+E2E_DEFAULT_FLAVOR ?= "calico"
 # Can be something like: basic && !move
 E2E_LABEL_FILTER ?= ""
 
 .PHONY: test-e2e
 test-e2e: manifests generate fmt vet ginkgo
-	rm -rf $(ARTIFACTS)/config/target
-
-	mkdir -p $(ARTIFACTS)/config/target
-	kubectl kustomize test/e2e/frmwrk/config/target/base -o $(ARTIFACTS)/config/target/base.yaml
+	rm -rf $(ARTIFACTS)
 
 	@METAL_API_URL=$(E2E_METAL_API_URL) \
 	METAL_API_HMAC=$(E2E_METAL_API_HMAC) \
 	METAL_API_HMAC_AUTH_TYPE=$(E2E_METAL_API_HMAC_AUTH_TYPE) \
 	METAL_PROJECT_ID=$(E2E_METAL_PROJECT_ID) \
+	E2E_METAL_PROJECT_NAME=$(E2E_METAL_PROJECT_NAME) \
 	METAL_PARTITION=$(E2E_METAL_PARTITION) \
 	METAL_PUBLIC_NETWORK=$(E2E_METAL_PUBLIC_NETWORK) \
 	E2E_KUBERNETES_VERSIONS=$(E2E_KUBERNETES_VERSIONS) \
@@ -147,7 +161,8 @@ test-e2e: manifests generate fmt vet ginkgo
 	FIREWALL_SIZE=$(E2E_FIREWALL_SIZE) \
 	FIREWALL_NETWORKS=$(E2E_FIREWALL_NETWORKS) \
 	ARTIFACTS=$(ARTIFACTS) \
-	$(GINKGO) -vv -r --junit-report="junit.e2e_suite.xml" --output-dir="$(ARTIFACTS)" --label-filter="$(E2E_LABEL_FILTER)" -timeout 60m ./test/e2e/frmwrk 
+	E2E_DEFAULT_FLAVOR=$(E2E_DEFAULT_FLAVOR) \
+	$(GINKGO) -vv -r --junit-report="junit.e2e_suite.xml" --output-dir="$(ARTIFACTS)" --label-filter="$(E2E_LABEL_FILTER)" -timeout 60m ./test/e2e/frmwrk
 
 .PHONY: lint
 lint: golangci-lint ## Run golangci-lint linter

README.md

@@ -1,17 +1,23 @@
 # cluster-api-provider-metal-stack
 
-The Cluster API provider for metal-stack (CAPMS) implements the declarative management of Kubernetes cluster infrastructure.
+The Cluster API provider for metal-stack (CAPMS) implements the declarative management of Kubernetes cluster infrastructure on top of [metal-stack](https://metal-stack.io/) using [Cluster API (CAPI)](https://cluster-api.sigs.k8s.io/).
 
-> [!CAUTION]
-> This project is currently under heavy development and is not advised to be used in production any time soon.
-> Please use our stack on top of [Gardener](https://docs.metal-stack.io/stable/installation/deployment/#Gardener-with-metal-stack) instead.
+> [!WARNING]
+> As of now the CAPMS is not yet feature complete and there might be breaking changes in future releases.
+> In case you search for a feature stable alternative consider [Gardener on metal-stack](https://docs.metal-stack.io/stable/installation/deployment/#Gardener-with-metal-stack) instead.
 > For developing this project head to our [DEVELOPMENT.md](/DEVELOPMENT.md).
 
 Currently, we provide the following custom resources:
 
 - [`MetalStackCluster`](./api/v1alpha1/metalstackcluster_types.go) can be used as [infrastructure cluster](https://cluster-api.sigs.k8s.io/developer/providers/contracts/infra-cluster) and ensures that there is a control plane IP for the cluster.
 - [`MetalStackMachine`](./api/v1alpha1/metalstackmachine_types.go) bridges between [infrastructure machines](https://cluster-api.sigs.k8s.io/developer/providers/contracts/infra-machine) and metal-stack machines.
 
+We plan to cover more resources in the future:
+
+- Node Networks
+- Firewall Deployments
+- Improved configuration suggestion of CNIs
+
 > [!note]
 > Currently our infrastructure provider is only tested against the [Cluster API bootstrap provider Kubeadm (CABPK)](https://cluster-api.sigs.k8s.io/tasks/bootstrap/kubeadm-bootstrap/index.html?highlight=kubeadm#cluster-api-bootstrap-provider-kubeadm).
 > While other providers might work, there is no guarantee nor the goal to reach compatibility.
@@ -21,6 +27,7 @@ Currently, we provide the following custom resources:
 **Prerequisites:**
 
 - Running metal-stack installation. See our [installation](https://docs.metal-stack.io/stable/installation/deployment/) section on how to get started with metal-stack.
+- Operating system images available to metal-stack. See [metal-stack/metal-images](https://github.com/metal-stack/metal-images) for pre-built ones.
 - Management cluster (with network access to the metal-stack infrastructure).
 - CLI metalctl installed for communicating with the metal-api. Installation instructions can be found in the corresponding [repository](https://github.com/metal-stack/metalctl).
 - CLI clusterctl
@@ -91,7 +98,7 @@ export WORKER_MACHINE_IMAGE=<machine-image>
 export WORKER_MACHINE_SIZE=<machine-size>
 
 # generate manifest
-clusterctl generate cluster $CLUSTER_NAME --kubernetes-version v1.30.6 --infrastructure metal-stack
+clusterctl generate cluster $CLUSTER_NAME --kubernetes-version v1.32.9 --infrastructure metal-stack
 ```
 
 Apply the generated manifest from the `clusterctl` output.
@@ -163,3 +170,13 @@ When generating your cluster, set `POD_CIDR` to your desired value.
 ```bash
 export POD_CIDR=["10.240.0.0/12"]
 ```
+
+## Flavors
+
+You might choose from different cluster template [flavors](https://cluster-api.sigs.k8s.io/clusterctl/commands/generate-cluster.html?highlight=flavor#flavors) to generate manifests with clusterctl. Here is a table describing the available flavors:
+
+| Name      | Description                                                                                                                                                                                                                                                                            | K8s Compatibility |
+| --------- | -------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | ----------------- |
+|           | This is the default flavor without providing the `--flavor` flag. This expects the user to deploy a CNI and a CCM.                                                                                                                                                                     | >= v1.33          |
+| calico    | Installs [calico](https://docs.tigera.io/calico/latest/about/) CNI along with [metal-ccm](https://github.com/metal-stack/metal-ccm). Depends on `ClusterResourceSet` and the [Add-on Provider for Helm](https://github.com/kubernetes-sigs/cluster-api-addon-provider-helm/tree/main). | >= v1.33          |
+| pre-v1.33 | The same as the default flavor but working for K8s versions < v1.33.                                                                                                                                                                                                                   | < v1.33           |

capi-lab/Makefile

@@ -7,7 +7,7 @@ KUBECONFIG := $(shell pwd)/mini-lab/.kubeconfig
 MINI_LAB_FLAVOR=capms
 
 CLUSTER_NAME ?= metal-test
-KUBERNETES_VERSION ?= 1.32.9
+KUBERNETES_VERSION ?= 1.33.5
 
 METAL_API_URL=http://metal.203.0.113.1.nip.io:8080
 METAL_API_HMAC=metal-edit
@@ -35,7 +35,7 @@ bake:
 
 .PHONY: deploy-capi
 deploy-capi:
-	docker compose run --rm clusterctl init
+	docker compose run --rm clusterctl init --addon helm
 
 .PHONY: cleanup
 cleanup:
@@ -78,7 +78,7 @@ apply-sample-cluster:
 		--worker-machine-count 1 \
 		--control-plane-machine-count 1 \
 		--kubernetes-version $(KUBERNETES_VERSION) \
-		--from ../config/clusterctl-templates/cluster-template.yaml \
+		--from ../config/clusterctl-templates/cluster-template-calico.yaml \
 		| kubectl --kubeconfig=$(KUBECONFIG) apply -f -
 
 .PHONY: delete-sample-cluster
@@ -90,17 +90,10 @@ delete-sample-cluster:
 		--worker-machine-count 1 \
 		--control-plane-machine-count 1 \
 		--kubernetes-version $(KUBERNETES_VERSION) \
-		--from ../config/clusterctl-templates/cluster-template.yaml \
+		--from ../config/clusterctl-templates/cluster-template-calico.yaml \
 		| kubectl --kubeconfig=$(KUBECONFIG) delete -f -
 
 .PHONY: mtu-fix
 mtu-fix:
 	cd mini-lab && ssh -F files/ssh/config leaf01 'ip link set dev vtep-1001 mtu 9100 && echo done'
 	cd mini-lab && ssh -F files/ssh/config leaf02 'ip link set dev vtep-1001 mtu 9100 && echo done'
-
-.PHONY: deploy-metal-ccm
-deploy-metal-ccm:
-	$(eval NAMESPACE = $(shell kubectl get metalstackclusters.infrastructure.cluster.x-k8s.io $(CLUSTER_NAME) -ojsonpath='{.metadata.namespace}'))
-	$(eval METAL_NODE_NETWORK_ID = $(shell metalctl network list --name $(CLUSTER_NAME) -o template --template '{{ .id }}'))
-	$(eval CONTROL_PLANE_IP = $(shell metalctl network ip list --name "$(CLUSTER_NAME)-vip" -o template --template '{{ .id }}'))
-	cat metal-ccm.yaml | envsubst | kubectl --kubeconfig=.capms-cluster-kubeconfig.yaml apply -f -