Skip to content

Commit 233a807

Browse files
tamcoretamcore
committed
samples: add proxmox
Signed-off-by: Philipp Born <[email protected]>
1 parent 2231531 commit 233a807

File tree

2 files changed

+372
-0
lines changed

2 files changed

+372
-0
lines changed

‎samples/proxmox/cluster-template-k3s.yaml‎

Lines changed: 289 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,289 @@
1+
---
2+
apiVersion: cluster.x-k8s.io/v1beta1
3+
kind: Cluster
4+
metadata:
5+
name: "${CLUSTER_NAME}"
6+
spec:
7+
clusterNetwork:
8+
pods:
9+
cidrBlocks:
10+
- 10.42.0.0/16
11+
services:
12+
cidrBlocks:
13+
- 10.43.0.0/16
14+
serviceDomain: cluster.local
15+
infrastructureRef:
16+
apiVersion: infrastructure.cluster.x-k8s.io/v1alpha1
17+
kind: ProxmoxCluster
18+
name: "${CLUSTER_NAME}"
19+
controlPlaneRef:
20+
apiVersion: controlplane.cluster.x-k8s.io/v1beta2
21+
kind: KThreesControlPlane
22+
name: "${CLUSTER_NAME}-control-plane"
23+
---
24+
apiVersion: infrastructure.cluster.x-k8s.io/v1alpha1
25+
kind: ProxmoxCluster
26+
metadata:
27+
name: "${CLUSTER_NAME}"
28+
spec:
29+
controlPlaneEndpoint:
30+
host: ${CONTROL_PLANE_ENDPOINT_IP}
31+
port: 6443
32+
ipv4Config:
33+
addresses: ${NODE_IP_RANGES}
34+
prefix: ${IP_PREFIX}
35+
gateway: ${GATEWAY}
36+
dnsServers: ${DNS_SERVERS}
37+
allowedNodes: ${ALLOWED_NODES:=[]}
38+
---
39+
kind: ProxmoxMachineTemplate
40+
apiVersion: infrastructure.cluster.x-k8s.io/v1alpha1
41+
metadata:
42+
name: "${CLUSTER_NAME}-control-plane"
43+
spec:
44+
template:
45+
spec:
46+
sourceNode: "${PROXMOX_SOURCENODE}"
47+
templateID: ${PROXMOX_TEMPLATE_VMID}
48+
format: "qcow2"
49+
full: true
50+
numSockets: ${NUM_SOCKETS:=1}
51+
numCores: ${NUM_CORES:=2}
52+
memoryMiB: ${MEMORY_MIB:=2048}
53+
disks:
54+
bootVolume:
55+
disk: ${BOOT_VOLUME_DEVICE:=scsi0}
56+
sizeGb: ${BOOT_VOLUME_SIZE:=32}
57+
network:
58+
default:
59+
bridge: ${BRIDGE}
60+
model: virtio
61+
vmIDRange:
62+
start: 2000
63+
end: 3000
64+
---
65+
apiVersion: infrastructure.cluster.x-k8s.io/v1alpha1
66+
kind: ProxmoxMachineTemplate
67+
metadata:
68+
name: "${CLUSTER_NAME}-worker"
69+
spec:
70+
template:
71+
spec:
72+
sourceNode: "${PROXMOX_SOURCENODE}"
73+
templateID: ${PROXMOX_TEMPLATE_VMID}
74+
format: "qcow2"
75+
full: true
76+
numSockets: ${NUM_SOCKETS:=1}
77+
numCores: ${NUM_CORES:=1}
78+
memoryMiB: ${MEMORY_MIB:=2048}
79+
disks:
80+
bootVolume:
81+
disk: ${BOOT_VOLUME_DEVICE:=scsi0}
82+
sizeGb: ${BOOT_VOLUME_SIZE:=32}
83+
network:
84+
default:
85+
bridge: ${BRIDGE}
86+
model: virtio
87+
vmIDRange:
88+
start: 2000
89+
end: 3000
90+
---
91+
apiVersion: controlplane.cluster.x-k8s.io/v1beta2
92+
kind: KThreesControlPlane
93+
metadata:
94+
name: "${CLUSTER_NAME}-control-plane"
95+
spec:
96+
machineTemplate:
97+
infrastructureRef:
98+
apiVersion: infrastructure.cluster.x-k8s.io/v1alpha1
99+
kind: ProxmoxMachineTemplate
100+
name: "${CLUSTER_NAME}-control-plane"
101+
kthreesConfigSpec:
102+
serverConfig:
103+
# cloudProviderName: "external"
104+
disableCloudController: false
105+
disableComponents: ${K3S_DISABLE_COMPONENTS:=[]}
106+
agentConfig:
107+
nodeName: "{{ ds.meta_data.local_hostname }}"
108+
kubeletArgs:
109+
- "provider-id=proxmox://{{ ds.meta_data.instance_id }}"
110+
files:
111+
- path: /var/lib/rancher/k3s/server/manifests/kube-vip.yaml
112+
owner: root:root
113+
content: |
114+
---
115+
apiVersion: v1
116+
kind: ServiceAccount
117+
metadata:
118+
name: kube-vip
119+
namespace: kube-system
120+
---
121+
apiVersion: rbac.authorization.k8s.io/v1
122+
kind: ClusterRole
123+
metadata:
124+
annotations:
125+
rbac.authorization.kubernetes.io/autoupdate: "true"
126+
name: system:kube-vip-role
127+
rules:
128+
- apiGroups: [""]
129+
resources: ["services/status"]
130+
verbs: ["update"]
131+
- apiGroups: [""]
132+
resources: ["services", "endpoints"]
133+
verbs: ["list","get","watch", "update"]
134+
- apiGroups: [""]
135+
resources: ["nodes"]
136+
verbs: ["list","get","watch", "update", "patch"]
137+
- apiGroups: ["coordination.k8s.io"]
138+
resources: ["leases"]
139+
verbs: ["list", "get", "watch", "update", "create"]
140+
- apiGroups: ["discovery.k8s.io"]
141+
resources: ["endpointslices"]
142+
verbs: ["list","get","watch", "update"]
143+
---
144+
kind: ClusterRoleBinding
145+
apiVersion: rbac.authorization.k8s.io/v1
146+
metadata:
147+
name: system:kube-vip-binding
148+
roleRef:
149+
apiGroup: rbac.authorization.k8s.io
150+
kind: ClusterRole
151+
name: system:kube-vip-role
152+
subjects:
153+
- kind: ServiceAccount
154+
name: kube-vip
155+
namespace: kube-system
156+
---
157+
apiVersion: apps/v1
158+
kind: DaemonSet
159+
metadata:
160+
creationTimestamp: null
161+
labels:
162+
app.kubernetes.io/name: kube-vip-ds
163+
app.kubernetes.io/version: v0.8.7
164+
name: kube-vip-ds
165+
namespace: kube-system
166+
spec:
167+
selector:
168+
matchLabels:
169+
app.kubernetes.io/name: kube-vip-ds
170+
template:
171+
metadata:
172+
creationTimestamp: null
173+
labels:
174+
app.kubernetes.io/name: kube-vip-ds
175+
app.kubernetes.io/version: v0.8.7
176+
spec:
177+
affinity:
178+
nodeAffinity:
179+
requiredDuringSchedulingIgnoredDuringExecution:
180+
nodeSelectorTerms:
181+
- matchExpressions:
182+
- key: node-role.kubernetes.io/master
183+
operator: Exists
184+
- matchExpressions:
185+
- key: node-role.kubernetes.io/control-plane
186+
operator: Exists
187+
containers:
188+
- args:
189+
- manager
190+
env:
191+
- name: vip_arp
192+
value: "true"
193+
- name: vip_nodename
194+
valueFrom:
195+
fieldRef:
196+
fieldPath: spec.nodeName
197+
- name: vip_interface
198+
value: ""
199+
- name: address
200+
value: ${CONTROL_PLANE_ENDPOINT_IP}
201+
- name: port
202+
value: ${CONTROL_PLANE_ENDPOINT_PORT="6443"}
203+
- name: dns_mode
204+
value: first
205+
- name: cp_enable
206+
value: "true"
207+
- name: cp_namespace
208+
value: kube-system
209+
- name: svc_enable
210+
value: "true"
211+
- name: svc_leasename
212+
value: plndr-svcs-lock
213+
- name: vip_leaderelection
214+
value: "true"
215+
- name: vip_leasename
216+
value: plndr-cp-lock
217+
- name: vip_leaseduration
218+
value: "5"
219+
- name: vip_renewdeadline
220+
value: "3"
221+
- name: vip_retryperiod
222+
value: "1"
223+
- name: prometheus_server
224+
value: :2112
225+
- name: enableUPNP
226+
value: "false"
227+
image: ghcr.io/kube-vip/kube-vip:v0.8.7
228+
imagePullPolicy: IfNotPresent
229+
name: kube-vip
230+
resources: {}
231+
securityContext:
232+
capabilities:
233+
add:
234+
- NET_ADMIN
235+
- NET_RAW
236+
hostNetwork: true
237+
serviceAccountName: kube-vip
238+
tolerations:
239+
- effect: NoSchedule
240+
operator: Exists
241+
- effect: NoExecute
242+
operator: Exists
243+
updateStrategy: {}
244+
preK3sCommands:
245+
- mkdir -p /root/.ssh
246+
- chmod 700 /root/.ssh
247+
- echo "${VM_SSH_KEYS}" > /root/.ssh/authorized_keys
248+
- chmod 600 /root/.ssh/authorized_keys
249+
replicas: ${CONTROL_PLANE_MACHINE_COUNT=1}
250+
version: "${KUBERNETES_VERSION}"
251+
---
252+
apiVersion: bootstrap.cluster.x-k8s.io/v1beta2
253+
kind: KThreesConfigTemplate
254+
metadata:
255+
name: "${CLUSTER_NAME}-worker"
256+
spec:
257+
template:
258+
spec:
259+
preK3sCommands:
260+
- mkdir -p /root/.ssh
261+
- chmod 700 /root/.ssh
262+
- echo "${VM_SSH_KEYS}" > /root/.ssh/authorized_keys
263+
- chmod 600 /root/.ssh/authorized_keys
264+
---
265+
apiVersion: cluster.x-k8s.io/v1beta1
266+
kind: MachineDeployment
267+
metadata:
268+
name: "${CLUSTER_NAME}-worker"
269+
spec:
270+
clusterName: "${CLUSTER_NAME}"
271+
replicas: ${WORKER_MACHINE_COUNT=1}
272+
selector:
273+
matchLabels: {}
274+
template:
275+
metadata:
276+
labels:
277+
node-role.kubernetes.io/node: ""
278+
spec:
279+
clusterName: "${CLUSTER_NAME}"
280+
version: "${KUBERNETES_VERSION}"
281+
bootstrap:
282+
configRef:
283+
apiVersion: bootstrap.cluster.x-k8s.io/v1beta2
284+
kind: KThreesConfigTemplate
285+
name: "${CLUSTER_NAME}-worker"
286+
infrastructureRef:
287+
apiVersion: infrastructure.cluster.x-k8s.io/v1alpha1
288+
kind: ProxmoxMachineTemplate
289+
name: "${CLUSTER_NAME}-worker"

‎samples/proxmox/setup.sh‎

Lines changed: 83 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,83 @@
1+
## Configure your Proxmox parameters
2+
3+
if [ -z "${CLUSTER_NAME}" ]; then
4+
echo "Please set CLUSTER_NAME"
5+
exit 0
6+
fi
7+
8+
if [ -z "${KUBERNETES_VERSION}" ]; then
9+
echo "Please set KUBERNETES_VERSION. For ex. v1.31.2+k3s1"
10+
exit 0
11+
fi
12+
13+
if [ -z "${CONTROL_PLANE_ENDPOINT_IP}" ]; then
14+
echo "Please set CONTROL_PLANE_ENDPOINT_IP. For ex. '10.10.10.4'"
15+
exit 0
16+
fi
17+
18+
if [ -z "${NODE_IP_RANGES}" ] || [ -z "${GATEWAY}" ] || [ -z "${IP_PREFIX}" ] || [ -z "${DNS_SERVERS}" ] || [ -z "${BRIDGE}" ]; then
19+
echo "Please set NODE_IP_RANGES. For ex. '[10.10.10.5-10.10.10.50]'"
20+
echo "Please set GATEWAY. For ex. '10.10.10.1'"
21+
echo "Please set IP_PREFIX. For ex. '24'"
22+
echo "Please set DNS_SERVERS. For ex. '[8.8.8.8,8.8.4.4]'"
23+
echo "Please set BRIDGE. For ex. 'vmbr0'"
24+
exit 0
25+
fi
26+
27+
if [ -z "${PROXMOX_URL}" ] || [ -z "${PROXMOX_TOKEN}" ] || [ -z "${PROXMOX_SECRET}" ] || [ -z "${PROXMOX_SOURCENODE}" ] || [ -z "${PROXMOX_TEMPLATE_VMID}" ]; then
28+
echo "Please set PROXMOX_URL, PROXMOX_TOKEN, PROXMOX_SECRET, PROXMOX_SOURCENODE, PROXMOX_TEMPLATE_VMID"
29+
echo "- See https://github.com/ionos-cloud/cluster-api-provider-proxmox/blob/main/docs/Usage.md"
30+
exit 0
31+
fi
32+
33+
# The device used for the boot disk.
34+
export BOOT_VOLUME_DEVICE="scsi0"
35+
# The size of the boot disk in GB.
36+
export BOOT_VOLUME_SIZE="32"
37+
# The number of sockets for the VMs.
38+
export NUM_SOCKETS="1"
39+
# The number of cores for the VMs.
40+
export NUM_CORES="1"
41+
# The memory size for the VMs.
42+
export MEMORY_MIB="4069"
43+
44+
# K3s components to disable
45+
# For example because you plan to use MetalLB over ServiceLB, or Longhorn over local-storage, or...
46+
# export K3S_DISABLE_COMPONENTS="[servicelb,local-storage,traefik,metrics-server,helm-controller]"
47+
48+
## Install your cluser-api-k3s provider correctly
49+
mkdir -p ~/.cluster-api
50+
cat samples/clusterctl.yaml | envsubst > ~/.cluster-api/clusterctl.yaml
51+
52+
cat >> ~/.cluster-api/clusterctl.yaml <<EOC
53+
- name: "in-cluster"
54+
url: https://github.com/kubernetes-sigs/cluster-api-ipam-provider-in-cluster/releases/latest/ipam-components.yaml
55+
type: "IPAMProvider"
56+
EOC
57+
58+
clusterctl init \
59+
--infrastructure proxmox \
60+
--bootstrap k3s \
61+
--control-plane k3s \
62+
--ipam in-cluster
63+
64+
kubectl wait --for=condition=Available --timeout=5m \
65+
-n capi-system deployment/capi-controller-manager
66+
kubectl wait --for=condition=Available --timeout=5m \
67+
-n capi-k3s-control-plane-system deployment/capi-k3s-control-plane-controller-manager
68+
kubectl wait --for=condition=Available --timeout=5m \
69+
-n capi-k3s-bootstrap-system deployment/capi-k3s-bootstrap-controller-manager
70+
kubectl wait --for=condition=Available --timeout=5m \
71+
-n capmox-system deployment/capmox-controller-manager
72+
73+
clusterctl generate cluster \
74+
"${CLUSTER_NAME}" \
75+
--from samples/proxmox/cluster-template-k3s.yaml \
76+
| kubectl apply -f -
77+
78+
echo "Once the cluster is up, run 'clusterctl get kubeconfig $CLUSTER_NAME > k3s.yaml' to retrieve your kubeconfig"
79+
echo "- Run 'kubectl scale kthreescontrolplane $CLUSTER_NAME-control-plane --replicas 3' to enable HA for your control-planes"
80+
echo "- or run 'kubectl scale machinedeployment $CLUSTER_NAME-worker --replicas 3' to deploy worker nodes"
81+
echo "- or to just use the single node cluster, you might need to also run the following commands:"
82+
echo " kubectl taint nodes --all node-role.kubernetes.io/control-plane-"
83+
echo " kubectl taint nodes --all node-role.kubernetes.io/master-"

0 commit comments

Comments
 (0)