Use KubeVIP Load Balancing for Control Planes. (#87)

Co-authored-by: Robert Volkmann <[email protected]>
Co-authored-by: Gerrit Schwerthelm <[email protected]>

Author: 3 people

DEVELOPMENT.md

@@ -21,7 +21,7 @@ make push-to-capi-lab
 Before creating a cluster some manual steps are required beforehand: you need to allocate a node network and a firewall.
 
 ```bash
-make -C capi-lab node-network firewall
+make -C capi-lab node-network firewall control-plane-ip
 ```
 
 A basic cluster configuration that relies on `config/clusterctl-templates/cluster-template.yaml` and uses the aforementioned node network can be generated and applied to the management cluster using a make target.
@@ -218,6 +218,7 @@ export METAL_API_URL=
 export METAL_PARTITION=
 export METAL_PROJECT_ID=
 export METAL_NODE_NETWORK_ID=
+export CONTROL_PLANE_IP=
 
 export FIREWALL_MACHINE_IMAGE=
 export FIREWALL_MACHINE_SIZE=
@@ -246,6 +247,7 @@ Create firewall if needed:
 ```bash
 metalctl project create --name $project_name --tenant $tenant_name --description "Cluster API test project"
 metalctl network allocate --description "Node network for $CLUSTER_NAME" --name $CLUSTER_NAME --project $METAL_PROJECT_ID --partition $METAL_PARTITION
+metalctl network ip create --network internet --project $METAL_PROJECT_ID --name "$CLUSTER_NAME-vip" --type static -o template --template "{{ .ipaddress }}"
 metalctl firewall create --description "Firewall for $CLUSTER_NAME cluster" --name firewall-$CLUSTER_NAME --hostname firewall-$CLUSTER_NAME --project $METAL_PROJECT_ID --partition $METAL_PARTITION --image $FIREWALL_MACHINE_IMAGE  --size $FIREWALL_MACHINE_SIZE --firewall-rules-file $repo_path/config/target-cluster/firewall-rules.yaml --networks internet,$METAL_NODE_NETWORK_ID
 ```
 

README.md

@@ -54,12 +54,19 @@ clusterctl init --infrastructure metal-stack
 
 A node network needs to be created.
 ```bash
+export CLUSTER_NAME=<cluster-name>
 export METAL_PARTITION=<partition>
 export METAL_PROJECT_ID=<project-id>
-metalctl network allocate --description "<description>" --name <name> --project $METAL_PROJECT_ID --partition $METAL_PARTITION
+metalctl network allocate --description "Node network for $CLUSTER_NAME" --name $CLUSTER_NAME --project $METAL_PROJECT_ID --partition $METAL_PARTITION
 
 # export environment variable for use in the next steps
-export METAL_NODE_NETWORK_ID=$(metalctl network list --name <name> -o template --template '{{ .id }}')
+export METAL_NODE_NETWORK_ID=$(metalctl network list --name $CLUSTER_NAME -o template --template '{{ .id }}')
+```
+
+Allocate a VIP for the control plane.
+
+```bash
+export CONTROL_PLANE_IP=$(metalctl network ip create --network internet --project $METAL_PROJECT_ID --name "$CLUSTER_NAME-vip" --type static -o template --template "{{ .ipaddress }}")
 ```
 
 A firewall needs to be created with appropriate firewall rules. An example can be found at [firewall-rules.yaml](config/target-cluster/firewall-rules.yaml).
@@ -68,14 +75,14 @@ A firewall needs to be created with appropriate firewall rules. An example can b
 export FIREWALL_MACHINE_IMAGE=<firewall-image>
 export FIREWALL_MACHINE_SIZE=<machine-size>
 
-metalctl firewall create --description <description> --name <name> --hostname <hostname> --project $METAL_PROJECT_ID --partition $METAL_PARTITION --image $FIREWALL_MACHINE_IMAGE  --size $FIREWALL_MACHINE_SIZE --firewall-rules-file=<rules.yaml> --networks internet,$METAL_NODE_NETWORK_ID
+metalctl firewall create --description "Firewall for $CLUSTER_NAME" --name "$CLUSTER_NAME-fw" --hostname "$CLUSTER_NAME-fw" --project $METAL_PROJECT_ID --partition $METAL_PARTITION --image $FIREWALL_MACHINE_IMAGE  --size $FIREWALL_MACHINE_SIZE --firewall-rules-file=<rules.yaml> --networks internet,$METAL_NODE_NETWORK_ID
 ```
 
 For your first cluster, it is advised to start with our generated template. Ensure that the namespaced cluster name is unique within the metal stack project.
 
 ```bash
 # display required environment variables
-clusterctl generate cluster <cluster-name> --infrastructure metal-stack --list-variables
+clusterctl generate cluster $CLUSTER_NAME --infrastructure metal-stack --list-variables
 
 # set additional environment variables
 export CONTROL_PLANE_MACHINE_IMAGE=<machine-image>
@@ -84,7 +91,7 @@ export WORKER_MACHINE_IMAGE=<machine-image>
 export WORKER_MACHINE_SIZE=<machine-size>
 
 # generate manifest
-clusterctl generate cluster <cluster-name> --kubernetes-version v1.30.6 --infrastructure metal-stack
+clusterctl generate cluster $CLUSTER_NAME --kubernetes-version v1.30.6 --infrastructure metal-stack
 ```
 
 Apply the generated manifest from the `clusterctl` output.
@@ -179,10 +186,10 @@ EOF
 
 ### I need to know the Control Plane IP address in advance. Can I provide a static IP address in advance?
 
-Yes, simply create a static IP address and set it to `metalstackcluster/<name>.spec.controlPlaneIP`.
+Yes, simply create a static IP address and set it to `metalstackcluster/$CLUSTER_NAME.spec.controlPlaneIP`.
 
 ```bash
-metalctl network ip create --name <name> --project $METAL_PROJECT_ID --type static
+metalctl network ip create --name $CLUSTER_NAME-vip --project $METAL_PROJECT_ID --type static
 ```
 
 ### I'd like to have a specific Pod CIDR. How can I achieve this?

capi-lab/Makefile

@@ -16,9 +16,9 @@ METALCTL_HMAC_AUTH_TYPE=Metal-Edit
 METAL_PARTITION ?= mini-lab
 METAL_PROJECT_ID ?= 00000000-0000-0000-0000-000000000001
 
-CONTROL_PLANE_MACHINE_IMAGE ?= ubuntu-24.4
+CONTROL_PLANE_MACHINE_IMAGE ?= ubuntu-24.0-k8s-1.31.6
 CONTROL_PLANE_MACHINE_SIZE ?= v1-small-x86
-WORKER_MACHINE_IMAGE ?= ubuntu-24.4
+WORKER_MACHINE_IMAGE ?= ubuntu-24.0-k8s-1.31.6
 WORKER_MACHINE_SIZE ?= v1-small-x86
 
 IMG ?= ghcr.io/metal-stack/cluster-api-metal-stack-controller:latest
@@ -60,9 +60,14 @@ firewall:
 node-network:
 	metalctl network allocate --description "node network for metal-test cluster" --name metal-test --project 00000000-0000-0000-0000-000000000001 --partition mini-lab
 
+.PHONY: control-plane-ip
+control-plane-ip:
+	metalctl network ip create --network internet-mini-lab --project $(METAL_PROJECT_ID) --name "$(CLUSTER_NAME)-vip" --type static -o template --template "{{ .ipaddress }}"
+
 .PHONY: apply-sample-cluster
 apply-sample-cluster:
 	$(eval METAL_NODE_NETWORK_ID = $(shell metalctl network list --name metal-test -o template --template '{{ .id }}'))
+	$(eval CONTROL_PLANE_IP = $(shell metalctl network ip list --name "$(CLUSTER_NAME)-vip" -o template --template '{{ .id }}'))
 	clusterctl generate cluster metal-test \
 		--kubeconfig=$(KUBECONFIG) \
 		--worker-machine-count 1 \
@@ -74,6 +79,7 @@ apply-sample-cluster:
 .PHONY: delete-sample-cluster
 delete-sample-cluster:
 	$(eval METAL_NODE_NETWORK_ID = $(shell metalctl network list --name metal-test -o template --template '{{ .id }}'))
+	$(eval CONTROL_PLANE_IP = $(shell metalctl network ip list --name "$(CLUSTER_NAME)-vip" -o template --template '{{ .id }}'))
 	clusterctl generate cluster metal-test \
 		--kubeconfig=$(KUBECONFIG) \
 		--worker-machine-count 1 \
@@ -91,4 +97,5 @@ mtu-fix:
 deploy-metal-ccm:
 	$(eval METAL_CLUSTER_ID = $(shell kubectl get metalstackclusters.infrastructure.cluster.x-k8s.io metal-test -ojsonpath='{.metadata.uid}'))
 	$(eval METAL_NODE_NETWORK_ID = $(shell metalctl network list --name metal-test -o template --template '{{ .id }}'))
+	$(eval CONTROL_PLANE_IP = $(shell metalctl network ip list --name "$(CLUSTER_NAME)-vip" -o template --template '{{ .id }}'))
 	cat metal-ccm.yaml | envsubst | kubectl --kubeconfig=.capms-cluster-kubeconfig.yaml apply -f -

config/clusterctl-templates/cluster-template.yaml

@@ -28,6 +28,7 @@ spec:
   projectID: ${METAL_PROJECT_ID}
   partition: ${METAL_PARTITION}
   nodeNetworkID: ${METAL_NODE_NETWORK_ID}
+  controlPlaneIP: ${CONTROL_PLANE_IP}
 ---
 apiVersion: infrastructure.cluster.x-k8s.io/v1alpha1
 kind: MetalStackMachineTemplate
@@ -80,10 +81,75 @@ spec:
         kubeletExtraArgs:
           cloud-provider: external
     joinConfiguration:
-      controlPlane: {}
+      controlPlane:
+        localAPIEndpoint:
+          advertiseAddress: 127.0.0.1
+          bindPort: 443
       nodeRegistration:
         kubeletExtraArgs:
           cloud-provider: external
+    files:
+      - path: /etc/kubernetes/manifests/kubevip.yaml
+        owner: root:root
+        permissions: "0644"
+        content: |
+          apiVersion: v1
+          kind: Pod
+          metadata:
+            name: kube-vip
+            namespace: kube-system
+          spec:
+            containers:
+            - args:
+              - manager
+              env:
+              - name: vip_arp
+                value: "false"
+              - name: port
+                value: "443"
+              - name: vip_interface
+                value: lo
+              - name: cp_enable
+                value: "true"
+              - name: cp_namespace
+                value: kube-system
+              - name: bgp_enable
+                value: "true"
+              - name: bgp_routerid
+                value: 127.0.0.1
+              - name: bgp_as
+                value: "METAL_MACHINE_ASN"
+              - name: bgp_peeraddress
+                value: 127.0.0.1
+              - name: bgp_peerpass
+              - name: bgp_peeras
+                value: "METAL_MACHINE_ASN"
+              - name: address
+                value: ${CONTROL_PLANE_IP}
+              image: ghcr.io/kube-vip/kube-vip:v0.8.10
+              imagePullPolicy: IfNotPresent
+              name: kube-vip
+              resources: {}
+              securityContext:
+                capabilities:
+                  add:
+                  - NET_ADMIN
+                  - NET_RAW
+                  drop:
+                  - ALL
+              volumeMounts:
+              - mountPath: /etc/kubernetes/admin.conf
+                name: kubeconfig
+            hostAliases:
+            - hostnames:
+              - kubernetes
+              ip: 127.0.0.1
+            hostNetwork: true
+            volumes:
+            - hostPath:
+                path: /etc/kubernetes/admin.conf
+              name: kubeconfig
+
 ---
 apiVersion: cluster.x-k8s.io/v1beta1
 kind: MachineDeployment

config/clusterctl-templates/example_variables.rc

@@ -10,6 +10,7 @@ export FIREWALL_MACHINE_SIZE=v1-small-x86
 export FIREWALL_MACHINE_IMAGE=
 export FIREWALL_NETWORKS=[internet]
 
+export CONTROL_PLANE_IP=203.0.113.130
 export CONTROL_PLANE_MACHINE_SIZE=v1-small-x86
 export CONTROL_PLANE_MACHINE_IMAGE=ubuntu-24.4
 

internal/controller/metalstackmachine_controller.go

@@ -46,7 +46,6 @@ import (
 	"github.com/go-logr/logr"
 	"github.com/metal-stack/cluster-api-provider-metal-stack/api/v1alpha1"
 	metalgo "github.com/metal-stack/metal-go"
-	ipmodels "github.com/metal-stack/metal-go/api/client/ip"
 	metalmachine "github.com/metal-stack/metal-go/api/client/machine"
 	"github.com/metal-stack/metal-go/api/models"
 	"github.com/metal-stack/metal-lib/pkg/pointer"
@@ -79,15 +78,8 @@ type machineReconciler struct {
 // +kubebuilder:rbac:groups=infrastructure.cluster.x-k8s.io,resources=metalstackmachines/finalizers,verbs=update
 // +kubebuilder:rbac:groups="",resources=secrets,verbs=get;list;watch
 
-// Reconcile is part of the main kubernetes reconciliation loop which aims to
-// move the current state of the cluster closer to the desired state.
-// TODO(user): Modify the Reconcile function to compare the state specified by
-// the MetalStackMachine object against the actual cluster state, and then
-// perform operations to make the cluster state reflect the state specified by
-// the user.
-//
-// For more details, check Reconcile and its Result here:
-// - https://pkg.go.dev/sigs.k8s.io/[email protected]/pkg/reconcile
+// Reconile reconciles a MetalStackMachine object.
+// Creates, updates and deletes the actual metalstack infra machine entities.
 func (r *MetalStackMachineReconciler) Reconcile(ctx context.Context, req ctrl.Request) (ctrl.Result, error) {
 	var (
 		log          = ctrllog.FromContext(ctx)
@@ -440,20 +432,6 @@ func (r *machineReconciler) create() (*models.V1MachineResponse, error) {
 		}
 	)
 
-	if util.IsControlPlaneMachine(r.clusterMachine) {
-		ips = append(ips, r.infraCluster.Spec.ControlPlaneEndpoint.Host)
-
-		resp, err := r.metalClient.IP().FindIP(ipmodels.NewFindIPParams().WithID(r.infraCluster.Spec.ControlPlaneEndpoint.Host).WithContext(r.ctx), nil)
-		if err != nil {
-			return nil, fmt.Errorf("unable to lookup control plane ip: %w", err)
-		}
-
-		nws = append(nws, &models.V1MachineAllocationNetwork{
-			Autoacquire: ptr.To(false),
-			Networkid:   resp.Payload.Networkid,
-		})
-	}
-
 	resp, err := r.metalClient.Machine().AllocateMachine(metalmachine.NewAllocateMachineParamsWithContext(r.ctx).WithBody(&models.V1MachineAllocateRequest{
 		Partitionid:   &r.infraCluster.Spec.Partition,
 		Projectid:     &r.infraCluster.Spec.ProjectID,