Bootstrap first worker node. (#15)

Author: Gerrit91

.gitignore

@@ -9,6 +9,7 @@ test/external-crds
 Dockerfile.cross
 
 infrastructure-components.yaml
+.capms-cluster-kubeconfig.yaml
 
 # Test binary, built with `go test -c`
 *.test

CONTRIBUTING.md

@@ -28,6 +28,62 @@ A basic cluster configuration resides in `config/samples`.
 kubectl apply -k config/samples
 ```
 
+For now it is required to manually create the firewall. This might be changed in the future, but for now run:
+
+```bash
+make -C capi-lab firewall
+# once the firewall is up run
+make -C capi-lab mtu-fix
+```
+
+When the control plane node was provisioned, you can obtain the kubeconfig like:
+
+```bash
+kubectl get secret metal-test-kubeconfig -o jsonpath='{.data.value}' | base64 -d > .capms-cluster-kubeconfig.yaml
+```
+
+For now, the provider ID has to be manually added to the node object because we did not integrate the [metal-ccm](https://github.com/metal-stack/metal-ccm) yet:
+
+```bash
+kubectl --kubeconfig=.capms-cluster-kubeconfig.yaml patch node <control-plane-node-name> --patch='{"spec":{"providerID": "metal://<machine-id>"}}'
+```
+
+It is now expected to deploy a CNI to the cluster:
+
+```bash
+kubectl --kubeconfig=.capms-cluster-kubeconfig.yaml create -f https://raw.githubusercontent.com/projectcalico/calico/v3.28.2/manifests/tigera-operator.yaml
+cat <<EOF | kubectl --kubeconfig=.capms-cluster-kubeconfig.yaml create -f -
+apiVersion: operator.tigera.io/v1
+kind: Installation
+metadata:
+  name: default
+spec:
+  # Configures Calico networking.
+  calicoNetwork:
+    bgp: Disabled
+    ipPools:
+    - name: default-ipv4-ippool
+      blockSize: 26
+      cidr: 192.168.0.0/16
+      encapsulation: None
+    mtu: 1440
+  cni:
+    ipam:
+      type: HostLocal
+    type: Calico
+EOF
+```
+
+> [!note]
+> Actually, Calico should be configured using BGP (no overlay), eBPF and DSR. An example will be proposed in this repository at a later point in time.
+
+As soon as the worker node was provisioned, the same provider ID patch as above is required:
+
+```bash
+kubectl --kubeconfig=.capms-cluster-kubeconfig.yaml patch node <worker-node-name> --patch='{"spec":{"providerID": "metal://<machine-id>"}}'
+```
+
+That's it!
 
 ### To Deploy on the cluster
 **Build and push your image to the location specified by `IMG`:**

capi-lab/Makefile

@@ -36,4 +36,9 @@ controller:
 
 .PHONY: firewall
 firewall:
-	metalctl firewall create --description fw --name fw --hostname fw --project 00000000-0000-0000-0000-000000000001 --partition mini-lab --image firewall-ubuntu-3.0 --size v1-small-x86 --firewall-rules-file=firewall-rules.yaml --networks internet-mini-lab,$(shell docker compose run $(DOCKER_COMPOSE_RUN_ARG) metalctl network list --name metal-test -o template --template '{{ .id }}')
+	metalctl firewall create --description fw --name fw --hostname fw --project 00000000-0000-0000-0000-000000000001 --partition mini-lab --image firewall-ubuntu-3.0 --size v1-small-x86 --firewall-rules-file=firewall-rules.yaml --networks internet-mini-lab,$(shell metalctl network list --name metal-test -o template --template '{{ .id }}')
+
+.PHONY: mtu-fix
+mtu-fix:
+	cd mini-lab && ssh -F files/ssh/config leaf01 'ip link set dev vtep-1001 mtu 9100 && echo done'
+	cd mini-lab && ssh -F files/ssh/config leaf02 'ip link set dev vtep-1001 mtu 9100 && echo done'

capi-lab/mini-lab

@@ -39,6 +39,16 @@ spec:
       image: ubuntu-24.04
       size: v1-small-x86
 ---
+apiVersion: infrastructure.cluster.x-k8s.io/v1alpha1
+kind: MetalStackMachineTemplate
+metadata:
+  name: metal-test-worker
+spec:
+  template:
+    spec:
+      image: ubuntu-24.04
+      size: v1-small-x86
+---
 kind: KubeadmControlPlane
 apiVersion: controlplane.cluster.x-k8s.io/v1beta1
 metadata:
@@ -119,3 +129,104 @@ spec:
         permissions: "0644"
         content: |
           disabled_plugins = []
+---
+apiVersion: cluster.x-k8s.io/v1beta1
+kind: MachineDeployment
+metadata:
+  name: metal-test-md-0
+  labels:
+    cluster.x-k8s.io/cluster-name: metal-test
+    nodepool: nodepool-0
+spec:
+  clusterName: metal-test
+  replicas: 1
+  selector:
+    matchLabels:
+      cluster.x-k8s.io/cluster-name: metal-test
+      nodepool: nodepool-0
+  template:
+    metadata:
+      labels:
+        cluster.x-k8s.io/cluster-name: metal-test
+        nodepool: nodepool-0
+    spec:
+      nodeDrainTimeout: 120s
+      clusterName: metal-test
+      version: "v1.30.6"
+      bootstrap:
+        configRef:
+          name: metal-test-md-0
+          apiVersion: bootstrap.cluster.x-k8s.io/v1beta1
+          kind: KubeadmConfigTemplate
+      infrastructureRef:
+        name: metal-test-worker
+        apiVersion: infrastructure.cluster.x-k8s.io/v1beta1
+        kind: MetalStackMachineTemplate
+---
+apiVersion: bootstrap.cluster.x-k8s.io/v1beta1
+kind: KubeadmConfigTemplate
+metadata:
+  name: metal-test-md-0
+spec:
+  template:
+    spec:
+      format: ignition
+      clusterConfiguration:
+        controlPlaneEndpoint: 203.0.113.129:443
+      joinConfiguration:
+        nodeRegistration: {}
+      ignition:
+        containerLinuxConfig:
+          additionalConfig: |
+            systemd:
+              units:
+              - name: cluster-api-init.service
+                enable: true
+                contents: |-
+                  [Unit]
+                  Description=Prepares the node for bootstrapping with cluster-api kubeadm
+                  Before=kubeadm.service
+                  After=network-online.target
+                  Wants=network-online.target
+                  [Service]
+                  Type=oneshot
+                  Restart=on-failure
+                  RestartSec=5
+                  StartLimitBurst=0
+                  EnvironmentFile=/etc/environment
+                  ExecStart=/var/lib/cluster-api-init/bootstrap.sh
+                  [Install]
+                  WantedBy=multi-user.target
+      files:
+        - path: /var/lib/cluster-api-init/bootstrap.sh
+          owner: "root:root"
+          permissions: "0744"
+          content: |
+            #!/usr/bin/env bash
+            set -eo pipefail
+            set +x
+
+            apt update
+            apt install conntrack
+
+            CNI_PLUGINS_VERSION="v1.3.0"
+            DEST="/opt/cni/bin"
+            mkdir -p "$DEST"
+            curl -L "https://github.com/containernetworking/plugins/releases/download/${CNI_PLUGINS_VERSION}/cni-plugins-linux-amd64-${CNI_PLUGINS_VERSION}.tgz" | tar -C "$DEST" -xz
+
+            RELEASE="v1.30.6"
+            cd /usr/local/bin
+            curl -L --remote-name-all https://dl.k8s.io/release/${RELEASE}/bin/linux/amd64/{kubeadm,kubelet,kubectl}
+            chmod +x {kubeadm,kubelet,kubectl}
+
+            RELEASE_VERSION="v0.16.2"
+            curl -sSL "https://raw.githubusercontent.com/kubernetes/release/${RELEASE_VERSION}/cmd/krel/templates/latest/kubelet/kubelet.service" | sed "s:/usr/bin:/usr/local/bin:g" | tee /usr/lib/systemd/system/kubelet.service
+            mkdir -p /usr/lib/systemd/system/kubelet.service.d
+            curl -sSL "https://raw.githubusercontent.com/kubernetes/release/${RELEASE_VERSION}/cmd/krel/templates/latest/kubeadm/10-kubeadm.conf" | sed "s:/usr/bin:/usr/local/bin:g" | tee /usr/lib/systemd/system/kubelet.service.d/10-kubeadm.conf
+
+            systemctl enable kubelet.service
+        - path: /etc/containerd/config.toml
+          owner: "root:root"
+          permissions: "0644"
+          content: |
+            disabled_plugins = []

config/samples/example-kubeadm.yaml

@@ -20,6 +20,7 @@ import (
 	"context"
 	"errors"
 	"fmt"
+	"strings"
 
 	"golang.org/x/sync/errgroup"
 	corev1 "k8s.io/api/core/v1"
@@ -230,7 +231,7 @@ func (r *machineReconciler) reconcile() error {
 		return err
 	}
 
-	r.infraMachine.Spec.ProviderID = *m.ID
+	r.infraMachine.Spec.ProviderID = "metal://" + *m.ID
 
 	err = helper.Patch(r.ctx, r.infraMachine) // TODO:check whether patch is not executed when no changes occur
 	if err != nil {
@@ -369,7 +370,7 @@ func (r *machineReconciler) status() error {
 				conditions.MarkFalse(r.infraMachine, v1alpha1.ProviderMachineHealthy, "NotHealthy", clusterv1.ConditionSeverityWarning, "machine not created")
 				conditions.MarkFalse(r.infraMachine, v1alpha1.ProviderMachineReady, "NotReady", clusterv1.ConditionSeverityWarning, "machine not created")
 			default:
-				if r.infraMachine.Spec.ProviderID == *m.ID {
+				if r.infraMachine.Spec.ProviderID == "metal://"+*m.ID {
 					conditions.MarkTrue(r.infraMachine, v1alpha1.ProviderMachineCreated)
 				} else {
 					conditions.MarkFalse(r.infraMachine, v1alpha1.ProviderMachineCreated, "NotSet", clusterv1.ConditionSeverityWarning, "provider id was not yet patched into the machine's spec")
@@ -455,7 +456,7 @@ func (r *machineReconciler) status() error {
 
 func (r *machineReconciler) findProviderMachine() (*models.V1MachineResponse, error) {
 	mfr := &models.V1MachineFindRequest{
-		ID:                r.infraMachine.Spec.ProviderID,
+		ID:                strings.TrimPrefix(r.infraMachine.Spec.ProviderID, "metal://"),
 		AllocationProject: r.infraCluster.Spec.ProjectID,
 		Tags:              r.machineTags(),
 	}