diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index 3870d65..941c975 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -31,7 +31,7 @@ make -C capi-lab node-network firewall A basic cluster configuration that relies on `config/clusterctl-templates/cluster-template.yaml` and uses the aforementioned node network can be generated and applied to the management cluster using a make target. ```bash -make apply-sample-cluster +make -C capi-lab apply-sample-cluster ``` Once the control plane node has phoned home, run: @@ -43,20 +43,14 @@ make -C capi-lab mtu-fix When the control plane node was provisioned, you can obtain the kubeconfig like: ```bash -kubectl get secret metal-test-kubeconfig -o jsonpath='{.data.value}' | base64 -d > .capms-cluster-kubeconfig.yaml -``` - -For now, the provider ID has to be manually added to the node object because we did not integrate the [metal-ccm](https://github.com/metal-stack/metal-ccm) yet: - -```bash -kubectl --kubeconfig=.capms-cluster-kubeconfig.yaml patch node --patch='{"spec":{"providerID": "metal://"}}' +kubectl get secret metal-test-kubeconfig -o jsonpath='{.data.value}' | base64 -d > capi-lab/.capms-cluster-kubeconfig.yaml ``` It is now expected to deploy a CNI to the cluster: ```bash -kubectl --kubeconfig=.capms-cluster-kubeconfig.yaml create -f https://raw.githubusercontent.com/projectcalico/calico/v3.28.2/manifests/tigera-operator.yaml -cat < [!note] > Actually, Calico should be configured using BGP (no overlay), eBPF and DSR. An example will be proposed in this repository at a later point in time. -As soon as the worker node was provisioned, the same provider ID patch as above is required: +The node's provider ID is provided by the [metal-ccm](https://github.com/metal-stack/metal-ccm), which needs to be deployed into the cluster: + +```bash +make -C capi-lab deploy-metal-ccm +``` + +If you want to provide service's of type load balancer through MetalLB by the metal-ccm, you need to deploy MetalLB: ```bash -kubectl --kubeconfig=.capms-cluster-kubeconfig.yaml patch node --patch='{"spec":{"providerID": "metal://"}}' +kubectl --kubeconfig capi-lab/.capms-cluster-kubeconfig.yaml apply --kustomize capi-lab/metallb +``` + +For each node in your Kubernetes cluster, you need to create a BGP peer configuration. Replace the placeholders ({{ +NODE_ASN }}, {{ NODE_HOSTNAME }}, and {{ NODE_ROUTER_ID }}) with the appropriate values for each node. + +```bash +cat </cluster-api-provider-metal-stack:tag ``` -> **NOTE**: If you encounter RBAC errors, you may need to grant yourself cluster-admin -privileges or be logged in as admin. +> **NOTE**: If you encounter RBAC errors, you may need to grant yourself cluster-admin privileges or be logged in as admin. **Create instances of your solution** You can apply the sample cluster configuration: ```sh -make apply-sample-cluster +make -C capi-lab apply-sample-cluster ``` ### To Uninstall **Delete the instances (CRs) from the cluster:** ```sh -make delete-sample-cluster +make -C capi-lab delete-sample-cluster ``` **Delete the APIs(CRDs) from the cluster:** diff --git a/Makefile b/Makefile index 1d7a75d..04835cd 100644 --- a/Makefile +++ b/Makefile @@ -241,36 +241,3 @@ mv $(1) $(1)-$(3) ;\ } ;\ ln -sf $(1)-$(3) $(1) endef - -# mini-lab developer environment - -export METAL_PARTITION ?= mini-lab -export METAL_PROJECT_ID ?= 00000000-0000-0000-0000-000000000001 -export METAL_NODE_NETWORK_ID ?= $(shell metalctl network list --name metal-test -o template --template '{{ .id }}') -export CONTROL_PLANE_MACHINE_IMAGE ?= ubuntu-24.04 -export CONTROL_PLANE_MACHINE_SIZE ?= v1-small-x86 -export WORKER_MACHINE_IMAGE ?= ubuntu-24.04 -export WORKER_MACHINE_SIZE ?= v1-small-x86 - -.PHONY: up -up: bake deploy-cloud-stack - -.PHONY: apply-sample-cluster -apply-sample-cluster: generate manifests - clusterctl generate cluster metal-test \ - --kubeconfig=$(KUBECONFIG) \ - --worker-machine-count 1 \ - --control-plane-machine-count 1 \ - --kubernetes-version 1.30.6 \ - --from config/clusterctl-templates/cluster-template.yaml \ - | kubectl --kubeconfig=$(KUBECONFIG) apply -f - - -.PHONY: delete-sample-cluster -delete-sample-cluster: generate manifests - clusterctl generate cluster metal-test \ - --kubeconfig=$(KUBECONFIG) \ - --worker-machine-count 1 \ - --control-plane-machine-count 1 \ - --kubernetes-version 1.30.6 \ - --from config/clusterctl-templates/cluster-template.yaml \ - | kubectl --kubeconfig=$(KUBECONFIG) delete -f - diff --git a/README.md b/README.md index eac3926..30d607d 100644 --- a/README.md +++ b/README.md @@ -57,9 +57,5 @@ clusterctl generate cluster example --kubernetes-version v1.30.6 --infrastructur > Due to the early development stage the following manual actions are needed for the cluster to operate. 1. The firewall needs to be created manually. -2. You need to install your CNI of choice. This is required due to CAPI. -3. Control plane and worker nodes need to be patched. - - ```bash - kubectl patch node --patch='{"spec":{"providerID": "metal://"}}' - ``` +2. The metal-ccm has to be deployed +3. You need to install your CNI of choice. This is required due to CAPI. diff --git a/capi-lab/Makefile b/capi-lab/Makefile index a06c67a..d1c42cf 100644 --- a/capi-lab/Makefile +++ b/capi-lab/Makefile @@ -5,9 +5,20 @@ ANSIBLE_EXTRA_VARS_FILE=$(shell pwd)/mini-lab-overrides/extra-vars.yaml KIND_EXPERIMENTAL_DOCKER_NETWORK=mini_lab_ext KUBECONFIG := $(shell pwd)/mini-lab/.kubeconfig MINI_LAB_FLAVOR=capms + +METAL_API_URL=http://metal.203.0.113.1.nip.io:8080 +METAL_API_HMAC=metal-admin METALCTL_API_URL=http://metal.203.0.113.1.nip.io:8080 METALCTL_HMAC=metal-admin +METAL_PARTITION ?= mini-lab +METAL_PROJECT_ID ?= 00000000-0000-0000-0000-000000000001 + +CONTROL_PLANE_MACHINE_IMAGE ?= ubuntu-24.04 +CONTROL_PLANE_MACHINE_SIZE ?= v1-small-x86 +WORKER_MACHINE_IMAGE ?= ubuntu-24.04 +WORKER_MACHINE_SIZE ?= v1-small-x86 + IMG ?= ghcr.io/metal-stack/cluster-api-metal-stack-controller:latest .PHONY: up @@ -47,7 +58,35 @@ firewall: node-network: metalctl network allocate --description "node network for metal-test cluster" --name metal-test --project 00000000-0000-0000-0000-000000000001 --partition mini-lab +.PHONY: apply-sample-cluster +apply-sample-cluster: + $(eval METAL_NODE_NETWORK_ID = $(shell metalctl network list --name metal-test -o template --template '{{ .id }}')) + clusterctl generate cluster metal-test \ + --kubeconfig=$(KUBECONFIG) \ + --worker-machine-count 1 \ + --control-plane-machine-count 1 \ + --kubernetes-version 1.30.6 \ + --from ../config/clusterctl-templates/cluster-template.yaml \ + | kubectl --kubeconfig=$(KUBECONFIG) apply -f - + +.PHONY: delete-sample-cluster +delete-sample-cluster: + $(eval METAL_NODE_NETWORK_ID = $(shell metalctl network list --name metal-test -o template --template '{{ .id }}')) + clusterctl generate cluster metal-test \ + --kubeconfig=$(KUBECONFIG) \ + --worker-machine-count 1 \ + --control-plane-machine-count 1 \ + --kubernetes-version 1.30.6 \ + --from ../config/clusterctl-templates/cluster-template.yaml \ + | kubectl --kubeconfig=$(KUBECONFIG) delete -f - + .PHONY: mtu-fix mtu-fix: cd mini-lab && ssh -F files/ssh/config leaf01 'ip link set dev vtep-1001 mtu 9100 && echo done' cd mini-lab && ssh -F files/ssh/config leaf02 'ip link set dev vtep-1001 mtu 9100 && echo done' + +.PHONY: deploy-metal-ccm +deploy-metal-ccm: + $(eval METAL_CLUSTER_ID = $(shell kubectl get metalstackclusters.infrastructure.cluster.x-k8s.io metal-test -ojsonpath='{.metadata.uid}')) + $(eval METAL_NODE_NETWORK_ID = $(shell metalctl network list --name metal-test -o template --template '{{ .id }}')) + cat metal-ccm.yaml | envsubst | kubectl --kubeconfig=.capms-cluster-kubeconfig.yaml apply -f - diff --git a/capi-lab/firewall-rules.yaml b/capi-lab/firewall-rules.yaml index 62e27a9..e070994 100644 --- a/capi-lab/firewall-rules.yaml +++ b/capi-lab/firewall-rules.yaml @@ -13,6 +13,12 @@ egress: protocol: TCP to: - 0.0.0.0/0 +- comment: allow outgoing traffic to control plane for ccm + ports: + - 8080 + protocol: TCP + to: + - 203.0.113.0/24 - comment: allow outgoing DNS and NTP traffic via UDP ports: - 53 diff --git a/capi-lab/metal-ccm.yaml b/capi-lab/metal-ccm.yaml new file mode 100644 index 0000000..16f9813 --- /dev/null +++ b/capi-lab/metal-ccm.yaml @@ -0,0 +1,224 @@ +--- +apiVersion: v1 +kind: Secret +metadata: + name: cloud-controller-manager + namespace: kube-system +stringData: + api-url: ${METAL_API_URL} + api-hmac: ${METAL_API_HMAC} +--- +apiVersion: v1 +kind: ServiceAccount +metadata: + name: cloud-controller-manager + namespace: kube-system +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRole +metadata: + name: cloud-controller-manager +rules: +- apiGroups: + - "" + resources: + - events + verbs: + - create + - patch + - update +- apiGroups: + - "" + resources: + - nodes + verbs: + - "*" +- apiGroups: + - "" + resources: + - nodes/status + verbs: + - patch +- apiGroups: + - "" + resources: + - services + - services/status + - endpoints + verbs: + - get + - list + - patch + - update + - watch +- apiGroups: + - "" + resources: + - serviceaccounts + - serviceaccounts/token + verbs: + - create + - get + - list + - watch +- apiGroups: + - "" + resources: + - secrets + - configmaps + verbs: + - get + - list + - watch + - update + - create + - patch + - delete +- apiGroups: + - coordination.k8s.io + resources: + - leases + verbs: + - get + - create + - update +- apiGroups: + - metallb.io + resources: + - bgppeers + verbs: + - create + - delete + - get + - list + - patch + - update + - watch +- apiGroups: + - metallb.io + resources: + - ipaddresspools + verbs: + - create + - delete + - get + - list + - patch + - update + - watch +- apiGroups: + - metallb.io + resources: + - bgpadvertisements + verbs: + - create + - delete + - get + - list + - patch + - update + - watch +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRoleBinding +metadata: + name: cloud-controller-manager +roleRef: + apiGroup: rbac.authorization.k8s.io + kind: ClusterRole + name: cloud-controller-manager +subjects: +- kind: ServiceAccount + name: cloud-controller-manager + namespace: kube-system +--- +apiVersion: apps/v1 +kind: Deployment +metadata: + labels: + app: cloud-controller-manager + name: cloud-controller-manager + namespace: kube-system +spec: + replicas: 1 + selector: + matchLabels: + app: cloud-controller-manager + strategy: + type: RollingUpdate + template: + metadata: + labels: + app: cloud-controller-manager + spec: + containers: + - command: + - ./metal-cloud-controller-manager + - --cluster-cidr=10.240.0.0/12 + - --cluster-name= + - --concurrent-service-syncs=10 + - --leader-elect=true + - --secure-port=10258 + - --use-service-account-credentials + - --v=2 + env: + - name: METAL_API_URL + valueFrom: + secretKeyRef: + key: api-url + name: cloud-controller-manager + - name: METAL_AUTH_HMAC + valueFrom: + secretKeyRef: + key: api-hmac + name: cloud-controller-manager + - name: METAL_PROJECT_ID + value: 00000000-0000-0000-0000-000000000001 + - name: METAL_PARTITION_ID + value: mini-lab + # associates service type load balancer ips with this cluster: + - name: METAL_CLUSTER_ID + value: ${METAL_CLUSTER_ID} + - name: METAL_DEFAULT_EXTERNAL_NETWORK_ID + value: internet-mini-lab + - name: METAL_ADDITIONAL_NETWORKS + value: internet-mini-lab,${METAL_NODE_NETWORK_ID} + - name: METAL_SSH_PUBLICKEY + value: "" + image: ghcr.io/metal-stack/metal-ccm:v0.9.3 + imagePullPolicy: IfNotPresent + livenessProbe: + failureThreshold: 2 + httpGet: + path: /healthz + port: 10258 + scheme: HTTPS + initialDelaySeconds: 15 + periodSeconds: 10 + successThreshold: 1 + timeoutSeconds: 15 + name: cloud-controller-manager + resources: + limits: + cpu: 250m + memory: 256Mi + requests: + cpu: 100m + memory: 64Mi + nodeSelector: + node-role.kubernetes.io/control-plane: "" + hostNetwork: true + serviceAccountName: cloud-controller-manager + tolerations: + - effect: NoSchedule + operator: Exists + key: node-role.kubernetes.io/control-plane + - effect: NoSchedule + key: node.cloudprovider.kubernetes.io/uninitialized + value: "true" + restartPolicy: Always + volumes: + - name: cloud-controller-manager + secret: + defaultMode: 420 + secretName: cloud-controller-manager diff --git a/capi-lab/metallb/kustomization.yaml b/capi-lab/metallb/kustomization.yaml new file mode 100644 index 0000000..e139915 --- /dev/null +++ b/capi-lab/metallb/kustomization.yaml @@ -0,0 +1,8 @@ +apiVersion: kustomize.config.k8s.io/v1beta1 +kind: Kustomization + +resources: + - github.com/metallb/metallb/config/native?ref=v0.14.9 + +patches: + - path: speaker.yaml diff --git a/capi-lab/metallb/speaker.yaml b/capi-lab/metallb/speaker.yaml new file mode 100644 index 0000000..edd4a03 --- /dev/null +++ b/capi-lab/metallb/speaker.yaml @@ -0,0 +1,21 @@ +--- +apiVersion: apps/v1 +kind: DaemonSet +metadata: + name: speaker + namespace: metallb-system +spec: + template: + spec: + containers: + - name: speaker + env: + - name: METALLB_NODE_NAME + valueFrom: + fieldRef: + fieldPath: spec.nodeName + - name: METALLB_HOST + valueFrom: + fieldRef: + fieldPath: status.podIP + hostNetwork: false diff --git a/capi-lab/mini-lab b/capi-lab/mini-lab index 72acbfb..4b32d5e 160000 --- a/capi-lab/mini-lab +++ b/capi-lab/mini-lab @@ -1 +1 @@ -Subproject commit 72acbfbffce1866ea6ac8352f38506c5e2f95291 +Subproject commit 4b32d5e16de1b5205ffc1deaa1e1731a9c7c913b diff --git a/config/clusterctl-templates/cluster-template.yaml b/config/clusterctl-templates/cluster-template.yaml index 65796a3..065ce82 100644 --- a/config/clusterctl-templates/cluster-template.yaml +++ b/config/clusterctl-templates/cluster-template.yaml @@ -65,15 +65,25 @@ spec: name: ${CLUSTER_NAME}-controlplane kubeadmConfigSpec: format: ignition - clusterConfiguration: {} + clusterConfiguration: + apiServer: + extraArgs: + cloud-provider: external + controllerManager: + extraArgs: + cloud-provider: external initConfiguration: localAPIEndpoint: advertiseAddress: 127.0.0.1 bindPort: 443 - nodeRegistration: {} + nodeRegistration: + kubeletExtraArgs: + cloud-provider: external joinConfiguration: controlPlane: {} - nodeRegistration: {} + nodeRegistration: + kubeletExtraArgs: + cloud-provider: external --- apiVersion: cluster.x-k8s.io/v1beta1 kind: MachineDeployment @@ -116,6 +126,14 @@ spec: template: spec: format: ignition - clusterConfiguration: {} + clusterConfiguration: + apiServer: + extraArgs: + cloud-provider: external + controllerManager: + extraArgs: + cloud-provider: external joinConfiguration: - nodeRegistration: {} + nodeRegistration: + kubeletExtraArgs: + cloud-provider: external diff --git a/internal/controller/metalstackmachine_controller.go b/internal/controller/metalstackmachine_controller.go index bd25ce4..e38dfac 100644 --- a/internal/controller/metalstackmachine_controller.go +++ b/internal/controller/metalstackmachine_controller.go @@ -42,6 +42,7 @@ import ( ipmodels "github.com/metal-stack/metal-go/api/client/ip" metalmachine "github.com/metal-stack/metal-go/api/client/machine" "github.com/metal-stack/metal-go/api/models" + "github.com/metal-stack/metal-lib/pkg/pointer" "github.com/metal-stack/metal-lib/pkg/tag" ) @@ -200,7 +201,7 @@ func (r *machineReconciler) reconcile() (ctrl.Result, error) { if m.ID == nil { return ctrl.Result{}, errors.New("machine allocated but got no provider ID") } - r.infraMachine.Spec.ProviderID = "metal://" + *m.ID + r.infraMachine.Spec.ProviderID = encodeProviderID(m) result := ctrl.Result{} @@ -370,7 +371,7 @@ func (r *machineReconciler) getMachineAddresses(m *models.V1MachineResponse) clu func (r *machineReconciler) findProviderMachine() (*models.V1MachineResponse, error) { mfr := &models.V1MachineFindRequest{ - ID: strings.TrimPrefix(r.infraMachine.Spec.ProviderID, "metal://"), + ID: decodeProviderID(r.infraMachine.Spec.ProviderID), AllocationProject: r.infraCluster.Spec.ProjectID, Tags: r.machineTags(), } @@ -403,3 +404,13 @@ func (r *machineReconciler) machineTags() []string { return tags } + +func encodeProviderID(m *models.V1MachineResponse) string { + return fmt.Sprintf("metal://%s/%s", pointer.SafeDeref(pointer.SafeDeref(m.Partition).ID), pointer.SafeDeref(m.ID)) +} + +func decodeProviderID(id string) string { + withPartition := strings.TrimPrefix(id, "metal://") + _, res, _ := strings.Cut(withPartition, "/") + return res +}