Skip to content

Commit e5f422c

Browse files
vknabelvknabel
committed
feat: cluster-template #29 
1 parent d31fff2 commit e5f422c

File tree

6 files changed

+323
-9
lines changed

6 files changed

+323
-9
lines changed

‎Makefile‎

Lines changed: 6 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -1,7 +1,7 @@
11
# Image URL to use all building/pushing image targets
2-
IMG ?= ghcr.io/metal-stack/capms-controller:latest
32
IMG_NAME ?= ghcr.io/metal-stack/capms-controller
43
IMG_TAG ?= latest
4+
IMG ?= ${IMG_NAME}:${IMG_TAG}
55
# ENVTEST_K8S_VERSION refers to the version of kubebuilder assets to be downloaded by envtest binary.
66
ENVTEST_K8S_VERSION = 1.31.0
77
RELEASE_DIR ?= .release
@@ -47,7 +47,8 @@ help: ## Display this help.
4747
##@ Releases
4848

4949
.PHONY: release-manifests
50-
release-manifests: $(KUSTOMIZE) ## Builds the manifests to publish with a release
50+
release-manifests: $(KUSTOMIZE) build-installer ## Builds the manifests to publish with a release
51+
mkdir -p $(RELEASE_DIR)
5152
$(KUSTOMIZE) build config/default > $(RELEASE_DIR)/infrastructure-components.yaml
5253
sed -i 's!image: $(IMG_NAME):latest!image: $(IMG_NAME):$(IMG_TAG)!' $(RELEASE_DIR)/infrastructure-components.yaml
5354
cp metadata.yaml $(RELEASE_DIR)/metadata.yaml
@@ -156,9 +157,9 @@ docker-buildx: ## Build and push docker image for the manager for cross-platform
156157

157158
.PHONY: build-installer
158159
build-installer: manifests generate kustomize ## Generate a consolidated YAML with CRDs and deployment.
159-
mkdir -p dist
160-
cd config/manager && $(KUSTOMIZE) edit set image controller=${IMG}
161-
$(KUSTOMIZE) build config/default > dist/install.yaml
160+
mkdir -p $(RELEASE_DIR)
161+
cd config/manager && $(KUSTOMIZE) edit set image controller=${IMG_NAME}:${IMG_TAG}
162+
$(KUSTOMIZE) build config/default > $(RELEASE_DIR)/install.yaml
162163

163164
##@ Deployment
164165

‎README.md‎

Lines changed: 56 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -4,7 +4,7 @@ The Cluster API provider for metal-stack (CAPMS) implements the declarative mana
44

55
> [!CAUTION]
66
> This project is currently under heavy development and is not advised to be used in production any time soon.
7-
> Please use or stack on top of [Gardener](https://docs.metal-stack.io/stable/installation/deployment/#Gardener-with-metal-stack) instead.
7+
> Please use our stack on top of [Gardener](https://docs.metal-stack.io/stable/installation/deployment/#Gardener-with-metal-stack) instead.
88
> User documentation will follow as soon. Until then head to our [CONTRIBUTING.md](/CONTRIBUTING.md)
99
1010
Currently we provide the following custom resources:
@@ -15,3 +15,58 @@ Currently we provide the following custom resources:
1515
> [!note]
1616
> Currently our infrastructure provider is only tested against the [Cluster API bootstrap provider Kubeadm (CABPK)](https://cluster-api.sigs.k8s.io/tasks/bootstrap/kubeadm-bootstrap/index.html?highlight=kubeadm#cluster-api-bootstrap-provider-kubeadm).
1717
> While other providers might work, there is no guarantee nor the goal to reach compatibility.
18+
19+
## Getting started
20+
21+
**Prerequisites:**
22+
23+
- a running metal-stack installation
24+
- CRDs for Prometheus
25+
- CRDs for the Firewall Controller Manager
26+
27+
First add the metal-stack infrastructure provider to your `clusterctl.yaml`:
28+
29+
```yaml
30+
# ~/.config/cluster-api/clusterctl.yaml
31+
providers:
32+
- name: "metal-stack"
33+
url: "https://github.com/metal-stack/cluster-api-provider-metal-stack/releases/latest/infrastructure-components.yaml"
34+
type: InfrastructureProvider
35+
```
36+
37+
Now you are able to install the CAPMS into your cluster:
38+
39+
```bash
40+
export METALCTL_API_URL=http://metal.172.17.0.1.nip.io:8080
41+
export METALCTL_API_HMAC=metal-admin
42+
export EXP_KUBEADM_BOOTSTRAP_FORMAT_IGNITION=true
43+
44+
clusterctl init --infrastructure metal-stack
45+
```
46+
47+
Now you should be able to create Clusters on top of metal-stack.
48+
For your first cluster it is advised to start with our generated template.
49+
50+
```bash
51+
# to display all env variables that need to be set
52+
clusterctl generate cluster example --kubernetes-version v1.30.6 --infrastructure metal-stack --list-variables
53+
```
54+
55+
> [!ATTENTION]
56+
> **Manual steps needed:**
57+
> Due to the early development stage manual actions are needed for the cluster to operate.
58+
59+
1. The firewall needs to be created manually.
60+
2. You need to install your CNI of choice.
61+
3. Control plane and worker nodes need to be patched.
62+
63+
```bash
64+
kubectl patch node <worker-node-name> --patch='{"spec":{"providerID": "metal://<machine-id>"}}'
65+
```
66+
67+
4. Remove the `NoSchedule` taint from all worker nodes.
68+
69+
```bash
70+
kubectl taint node.cluster.x-k8s.io/uninitialized:NoSchedule-
71+
```
72+

‎config/clusterctl-templates/cluster-template.yaml‎

Lines changed: 213 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,213 @@
1+
---
2+
apiVersion: cluster.x-k8s.io/v1beta1
3+
kind: Cluster
4+
metadata:
5+
name: ${CLUSTER_NAME}
6+
namespace: ${NAMESPACE}
7+
spec:
8+
clusterNetwork:
9+
pods:
10+
cidrBlocks: ${POD_CIDR:=["192.168.0.0/16"]}
11+
controlPlaneRef:
12+
apiVersion: controlplane.cluster.x-k8s.io/v1beta1
13+
kind: KubeadmControlPlane
14+
name: ${CLUSTER_NAME}-controlplane
15+
namespace: ${NAMESPACE}
16+
infrastructureRef:
17+
apiVersion: infrastructure.cluster.x-k8s.io/v1alpha1
18+
kind: MetalStackCluster
19+
name: ${CLUSTER_NAME}
20+
namespace: ${NAMESPACE}
21+
---
22+
apiVersion: infrastructure.cluster.x-k8s.io/v1alpha1
23+
kind: MetalStackCluster
24+
metadata:
25+
name: ${CLUSTER_NAME}
26+
namespace: ${NAMESPACE}
27+
spec:
28+
projectID: ${METAL_PROJECT_ID}
29+
partition: ${METAL_PARTITION}
30+
firewall:
31+
size: ${FIREWALL_MACHINE_SIZE}
32+
image: ${FIREWALL_MACHINE_IMAGE}
33+
networks: ${FIREWALL_NETWORKS:=[internet]}
34+
---
35+
apiVersion: infrastructure.cluster.x-k8s.io/v1alpha1
36+
kind: MetalStackMachineTemplate
37+
metadata:
38+
name: ${CLUSTER_NAME}-controlplane
39+
namespace: ${NAMESPACE}
40+
spec:
41+
template:
42+
spec:
43+
size: ${CONTROL_PLANE_MACHINE_SIZE}
44+
image: ${CONTROL_PLANE_MACHINE_IMAGE}
45+
---
46+
apiVersion: infrastructure.cluster.x-k8s.io/v1alpha1
47+
kind: MetalStackMachineTemplate
48+
metadata:
49+
name: ${CLUSTER_NAME}-worker
50+
spec:
51+
template:
52+
spec:
53+
size: ${WORKER_MACHINE_SIZE}
54+
image: ${WORKER_MACHINE_IMAGE}
55+
---
56+
kind: KubeadmControlPlane
57+
apiVersion: controlplane.cluster.x-k8s.io/v1beta1
58+
metadata:
59+
name: ${CLUSTER_NAME}-controlplane
60+
spec:
61+
replicas: ${CONTROL_PLANE_MACHINE_COUNT}
62+
version: ${KUBERNETES_VERSION}
63+
machineTemplate:
64+
nodeDrainTimeout: 10m
65+
infrastructureRef:
66+
kind: MetalStackMachineTemplate
67+
apiVersion: infrastructure.cluster.x-k8s.io/v1alpha1
68+
name: ${CLUSTER_NAME}-controlplane
69+
kubeadmConfigSpec:
70+
format: ignition
71+
clusterConfiguration:
72+
controlPlaneEndpoint: ${CONTROL_PLANE_ENDPOINT}
73+
initConfiguration:
74+
localAPIEndpoint:
75+
bindPort: ${CONTROL_PLANE_PORT:=443}
76+
nodeRegistration: {}
77+
joinConfiguration:
78+
controlPlane: {}
79+
nodeRegistration: {}
80+
ignition:
81+
containerLinuxConfig:
82+
additionalConfig: |
83+
systemd:
84+
units:
85+
- name: cluster-api-init.service
86+
enable: true
87+
contents: |-
88+
[Unit]
89+
Description=Prepares the node for bootstrapping with cluster-api kubeadm
90+
Before=kubeadm.service
91+
After=network-online.target
92+
Wants=network-online.target
93+
[Service]
94+
Type=oneshot
95+
Restart=on-failure
96+
RestartSec=5
97+
StartLimitBurst=0
98+
EnvironmentFile=/etc/environment
99+
ExecStart=/var/lib/cluster-api-init/bootstrap.sh
100+
[Install]
101+
WantedBy=multi-user.target
102+
files:
103+
- path: /var/lib/cluster-api-init/bootstrap.sh
104+
owner: "root:root"
105+
permissions: "0744"
106+
content: ${BOOTSTRAP_SCRIPT:="
107+
#!/usr/bin/env bash
108+
set -eo pipefail
109+
set +x
110+
111+
apt update
112+
apt install conntrack
113+
114+
CNI_PLUGINS_VERSION="v1.3.0"
115+
DEST="/opt/cni/bin"
116+
mkdir -p "$DEST"
117+
curl -L "https://github.com/containernetworking/plugins/releases/download/$CNI_PLUGINS_VERSION/cni-plugins-linux-amd64-$CNI_PLUGINS_VERSION.tgz" | tar -C "$DEST" -xz
118+
119+
RELEASE="${KUBERNETES_VERSION}"
120+
cd /usr/local/bin
121+
curl -L --remote-name-all https://dl.k8s.io/release/${RELEASE}/bin/linux/amd64/{kubeadm,kubelet,kubectl}
122+
chmod +x {kubeadm,kubelet,kubectl}
123+
124+
RELEASE_VERSION="v0.16.2"
125+
curl -sSL "https://raw.githubusercontent.com/kubernetes/release/$RELEASE_VERSION/cmd/krel/templates/latest/kubelet/kubelet.service" | sed "s:/usr/bin:/usr/local/bin:g" | tee /usr/lib/systemd/system/kubelet.service
126+
mkdir -p /usr/lib/systemd/system/kubelet.service.d
127+
curl -sSL "https://raw.githubusercontent.com/kubernetes/release/$RELEASE_VERSION/cmd/krel/templates/latest/kubeadm/10-kubeadm.conf" | sed "s:/usr/bin:/usr/local/bin:g" | tee /usr/lib/systemd/system/kubelet.service.d/10-kubeadm.conf
128+
129+
systemctl enable kubelet.service
130+
"}
131+
- path: /etc/containerd/config.toml
132+
owner: "root:root"
133+
permissions: "0644"
134+
content: |
135+
disabled_plugins = []
136+
---
137+
apiVersion: cluster.x-k8s.io/v1beta1
138+
kind: MachineDeployment
139+
metadata:
140+
name: ${CLUSTER_NAME}-md-0
141+
labels:
142+
cluster.x-k8s.io/cluster-name: ${CLUSTER_NAME}
143+
nodepool: nodepool-0
144+
spec:
145+
clusterName: ${CLUSTER_NAME}
146+
replicas: ${WORKER_MACHINE_COUNT}
147+
selector:
148+
matchLabels:
149+
cluster.x-k8s.io/cluster-name: ${CLUSTER_NAME}
150+
nodepool: nodepool-0
151+
template:
152+
metadata:
153+
labels:
154+
cluster.x-k8s.io/cluster-name: ${CLUSTER_NAME}
155+
nodepool: nodepool-0
156+
spec:
157+
nodeDrainTimeout: 120s
158+
clusterName: ${CLUSTER_NAME}
159+
version: "${KUBERNETES_VERSION}"
160+
bootstrap:
161+
configRef:
162+
name: ${CLUSTER_NAME}-md-0
163+
apiVersion: bootstrap.cluster.x-k8s.io/v1beta1
164+
kind: KubeadmConfigTemplate
165+
infrastructureRef:
166+
name: ${CLUSTER_NAME}-worker
167+
apiVersion: infrastructure.cluster.x-k8s.io/v1beta1
168+
kind: MetalStackMachineTemplate
169+
---
170+
apiVersion: bootstrap.cluster.x-k8s.io/v1beta1
171+
kind: KubeadmConfigTemplate
172+
metadata:
173+
name: ${CLUSTER_NAME}-md-0
174+
spec:
175+
template:
176+
spec:
177+
format: ignition
178+
clusterConfiguration:
179+
controlPlaneEndpoint: ${CONTROL_PLANE_ENDPOINT}
180+
joinConfiguration:
181+
nodeRegistration: {}
182+
ignition:
183+
containerLinuxConfig:
184+
additionalConfig: |
185+
systemd:
186+
units:
187+
- name: cluster-api-init.service
188+
enable: true
189+
contents: |-
190+
[Unit]
191+
Description=Prepares the node for bootstrapping with cluster-api kubeadm
192+
Before=kubeadm.service
193+
After=network-online.target
194+
Wants=network-online.target
195+
[Service]
196+
Type=oneshot
197+
Restart=on-failure
198+
RestartSec=5
199+
StartLimitBurst=0
200+
EnvironmentFile=/etc/environment
201+
ExecStart=/var/lib/cluster-api-init/bootstrap.sh
202+
[Install]
203+
WantedBy=multi-user.target
204+
files:
205+
- path: /var/lib/cluster-api-init/bootstrap.sh
206+
owner: "root:root"
207+
permissions: "0744"
208+
content: ${BOOTSTRAP_SCRIPT}
209+
- path: /etc/containerd/config.toml
210+
owner: "root:root"
211+
permissions: "0644"
212+
content: |
213+
disabled_plugins = []

‎config/clusterctl-templates/example_variables.rc‎

Lines changed: 45 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,45 @@
1+
export METAL_API_URL=http://metal.172.17.0.1.nip.io:8080
2+
export METAL_API_HMAC=metal-admin
3+
4+
export METAL_PROJECT_ID=00000000-0000-0000-0000-000000000001
5+
export POD_CIDR=192.168.0.0/16
6+
export METAL_PARTITION=mini-lab
7+
8+
export FIREWALL_MACHINE_SIZE=v1-small-x86
9+
export FIREWALL_MACHINE_IMAGE=
10+
export FIREWALL_NETWORKS=[internet]
11+
12+
export NODE_NETWORK_ID=00000000-0000-0000-0000-000000000002
13+
export CONTROL_PLANE_ENDPOINT=203.0.113.129:443
14+
export CONTROL_PLANE_PORT=443
15+
export CONTROL_PLANE_MACHINE_SIZE=v1-small-x86
16+
export CONTROL_PLANE_MACHINE_IMAGE=ubuntu-24.04
17+
18+
export WORKER_MACHINE_IMAGE=ubuntu-24.04
19+
export WORKER_MACHINE_SIZE=v1-small-x86
20+
21+
export BOOTSTRAP_SCRIPT="
22+
#!/usr/bin/env bash
23+
set -eo pipefail
24+
set +x
25+
26+
apt update
27+
apt install conntrack
28+
29+
CNI_PLUGINS_VERSION="v1.3.0"
30+
DEST="/opt/cni/bin"
31+
mkdir -p "$DEST"
32+
curl -L "https://github.com/containernetworking/plugins/releases/download/${CNI_PLUGINS_VERSION}/cni-plugins-linux-amd64-${CNI_PLUGINS_VERSION}.tgz" | tar -C "$DEST" -xz
33+
34+
RELEASE="${KUBERNETES_VERSION}"
35+
cd /usr/local/bin
36+
curl -L --remote-name-all https://dl.k8s.io/release/${RELEASE}/bin/linux/amd64/{kubeadm,kubelet,kubectl}
37+
chmod +x {kubeadm,kubelet,kubectl}
38+
39+
RELEASE_VERSION="v0.16.2"
40+
curl -sSL "https://raw.githubusercontent.com/kubernetes/release/${RELEASE_VERSION}/cmd/krel/templates/latest/kubelet/kubelet.service" | sed "s:/usr/bin:/usr/local/bin:g" | tee /usr/lib/systemd/system/kubelet.service
41+
mkdir -p /usr/lib/systemd/system/kubelet.service.d
42+
curl -sSL "https://raw.githubusercontent.com/kubernetes/release/${RELEASE_VERSION}/cmd/krel/templates/latest/kubeadm/10-kubeadm.conf" | sed "s:/usr/bin:/usr/local/bin:g" | tee /usr/lib/systemd/system/kubelet.service.d/10-kubeadm.conf
43+
44+
systemctl enable kubelet.service
45+
"

‎config/manager/kustomization.yaml‎

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -4,5 +4,5 @@ apiVersion: kustomize.config.k8s.io/v1beta1
44
kind: Kustomization
55
images:
66
- name: controller
7-
newName: capms-controller
8-
newTag: latest
7+
newName: ghcr.io/metal-stack/capms-controller
8+
newTag: xxxxxxx

‎config/manager/manager.yaml‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -49,7 +49,7 @@ spec:
4949
args:
5050
- --leader-elect
5151
- --health-probe-bind-address=:8081
52-
image: ghcr.io/metal-stack/capms-controller:latest
52+
image: controller:latest
5353
imagePullPolicy: IfNotPresent
5454
name: manager
5555
securityContext:

0 commit comments

Comments
 (0)