Skip to content

Commit acddc38

Browse files
mcbenjemaatuunitFelix Wischke (65278)
authored
Support Flatcar and provide ignition bootstrap data (#308)
* add metadata handling for proxmox node name * First draft: flatcar and ignition support * Revert "add metadata handling for proxmox node name" This reverts commit 3dd5aae. * Make The Ignition ISO able to be unmounted * Fixes lints * Fixes Tests * remove unsed config and Fixes Test * Set ipv6 * add tests * add tests for injector * more test coverage * Fix lint * Provide a correct cluster template * Provide docs on how to provision cluster with flatcar * Address reviews * Update envtest * Address reviwes and try e2e test with skip label * Implement networkd config units with VRFs * Tidy * E2e test for flatcar * Use correct template for flatcar * Add more tests * Simpler units name * add comment for netdev index * Address reviews * remove uncessary variable * Add note about EXP_KUBEADM_BOOTSTRAP_FORMAT_IGNITION * Address reviews * Change ignition docs * Remove quotes * pkg/ignition/network.go: fix VRF routing rules policy table, add tests --------- Co-authored-by: tuunit <[email protected]> Co-authored-by: Felix Wischke (65278) <[email protected]>
1 parent 5f3ba2f commit acddc38

30 files changed

+2567
-42
lines changed

‎.gitignore‎

Lines changed: 1 addition & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -46,5 +46,4 @@ _artifacts
4646
**e2e.test
4747
test/e2e/config/*-envsubst.yaml
4848

49-
# asdf package manager
50-
.tool-versions
49+
.tool-versions

‎Makefile‎

Lines changed: 3 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -181,6 +181,7 @@ ENVTEST ?= $(LOCALBIN)/setup-envtest
181181
## Tool Versions
182182
KUSTOMIZE_VERSION ?= v5.0.0
183183
CONTROLLER_TOOLS_VERSION ?= v0.15.0
184+
ENVTEST_VERSION ?= latest
184185

185186
KUSTOMIZE_INSTALL_SCRIPT ?= "https://raw.githubusercontent.com/kubernetes-sigs/kustomize/master/hack/install_kustomize.sh"
186187
.PHONY: kustomize
@@ -201,7 +202,7 @@ $(CONTROLLER_GEN): $(LOCALBIN)
201202
.PHONY: envtest
202203
envtest: $(ENVTEST) ## Download envtest-setup locally if necessary.
203204
$(ENVTEST): $(LOCALBIN)
204-
test -s $(LOCALBIN)/setup-envtest || GOBIN=$(LOCALBIN) go install sigs.k8s.io/controller-runtime/tools/setup-envtest@v0.0.0-20240208111015-5923139bc5bd
205+
test -s $(LOCALBIN)/setup-envtest || GOBIN=$(LOCALBIN) go install sigs.k8s.io/controller-runtime/tools/setup-envtest@$(ENVTEST_VERSION)
205206

206207
##@ Test
207208

@@ -280,7 +281,7 @@ KUBETEST_CONF_PATH ?= $(abspath $(E2E_DATA_DIR)/kubetest/conformance.yaml)
280281

281282
# Allow overriding the e2e configurations
282283
GINKGO_FOCUS ?= Workload cluster creation
283-
GINKGO_SKIP ?= API Version Upgrade
284+
GINKGO_SKIP ?= Generic|Flatcar
284285
GINKGO_NODES ?= 1
285286
GINKGO_NOCOLOR ?= false
286287
GINKGO_ARGS ?=

‎api/v1alpha1/proxmoxmachine_types.go‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -297,6 +297,7 @@ type RoutingPolicySpec struct {
297297
From string `json:"from,omitempty"`
298298

299299
// Table is the routing table ID.
300+
// when used in the networks, the value should be the VRF Table.
300301
// +optional
301302
Table *uint32 `json:"table,omitempty"`
302303

‎config/crd/bases/infrastructure.cluster.x-k8s.io_proxmoxclusters.yaml‎

Lines changed: 6 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -328,8 +328,9 @@ spec:
328328
rule: (self > 0 && self < 32765) || (self
329329
> 32766)
330330
table:
331-
description: Table is the routing table
332-
ID.
331+
description: |-
332+
Table is the routing table ID.
333+
when used in the networks, the value should be the VRF Table.
333334
format: int32
334335
type: integer
335336
to:
@@ -461,8 +462,9 @@ spec:
461462
rule: (self > 0 && self < 32765) || (self
462463
> 32766)
463464
table:
464-
description: Table is the routing table
465-
ID.
465+
description: |-
466+
Table is the routing table ID.
467+
when used in the networks, the value should be the VRF Table.
466468
format: int32
467469
type: integer
468470
to:

‎config/crd/bases/infrastructure.cluster.x-k8s.io_proxmoxclustertemplates.yaml‎

Lines changed: 6 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -356,8 +356,9 @@ spec:
356356
rule: (self > 0 && self < 32765)
357357
|| (self > 32766)
358358
table:
359-
description: Table is the routing
360-
table ID.
359+
description: |-
360+
Table is the routing table ID.
361+
when used in the networks, the value should be the VRF Table.
361362
format: int32
362363
type: integer
363364
to:
@@ -498,8 +499,9 @@ spec:
498499
rule: (self > 0 && self < 32765)
499500
|| (self > 32766)
500501
table:
501-
description: Table is the routing
502-
table ID.
502+
description: |-
503+
Table is the routing table ID.
504+
when used in the networks, the value should be the VRF Table.
503505
format: int32
504506
type: integer
505507
to:

‎config/crd/bases/infrastructure.cluster.x-k8s.io_proxmoxmachines.yaml‎

Lines changed: 6 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -301,7 +301,9 @@ spec:
301301
kernel rules
302302
rule: (self > 0 && self < 32765) || (self > 32766)
303303
table:
304-
description: Table is the routing table ID.
304+
description: |-
305+
Table is the routing table ID.
306+
when used in the networks, the value should be the VRF Table.
305307
format: int32
306308
type: integer
307309
to:
@@ -427,7 +429,9 @@ spec:
427429
kernel rules
428430
rule: (self > 0 && self < 32765) || (self > 32766)
429431
table:
430-
description: Table is the routing table ID.
432+
description: |-
433+
Table is the routing table ID.
434+
when used in the networks, the value should be the VRF Table.
431435
format: int32
432436
type: integer
433437
to:

‎config/crd/bases/infrastructure.cluster.x-k8s.io_proxmoxmachinetemplates.yaml‎

Lines changed: 6 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -328,7 +328,9 @@ spec:
328328
rule: (self > 0 && self < 32765) || (self
329329
> 32766)
330330
table:
331-
description: Table is the routing table ID.
331+
description: |-
332+
Table is the routing table ID.
333+
when used in the networks, the value should be the VRF Table.
332334
format: int32
333335
type: integer
334336
to:
@@ -460,7 +462,9 @@ spec:
460462
rule: (self > 0 && self < 32765) || (self
461463
> 32766)
462464
table:
463-
description: Table is the routing table ID.
465+
description: |-
466+
Table is the routing table ID.
467+
when used in the networks, the value should be the VRF Table.
464468
format: int32
465469
type: integer
466470
to:

‎docs/Usage.md‎

Lines changed: 37 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -170,6 +170,7 @@ We provide the following templates:
170170
| default | templates/cluster-template.yaml | - |
171171
| cilium loadbalancer | templates/cluster-template-cilium-load-balancer.yaml | templates/crs/cni/cilium.yaml, templates/crs/metallb.yaml |
172172
| external-creds | templates/cluster-template-external-creds.yaml | |
173+
| flatcar | templates/cluster-template-flatcar.yaml | |
173174

174175
For more information about advanced clusters please check our [advanced setups docs](advanced-setups.md).
175176

@@ -272,6 +273,42 @@ kubectl apply -f cluster-crs.yaml
272273
kubectl delete cluster proxmox-quickstart
273274
```
274275
276+
277+
#### Provision a cluster with Flatcar Container Linux Images
278+
279+
To provision a cluster with Flatcar, you need to build a suitable image with [image-builder](https://github.com/kubernetes-sigs/image-builder)
280+
Once this PR [#1589](https://github.com/kubernetes-sigs/image-builder/pull/1589) is merged, you can follow the official docs on how to build a Flatcar image for Proxmox.
281+
After you create a VM template using Flatcar, you can provision a cluster with the following options:
282+
283+
First, make sure to enable the
284+
[experimental-feature-ignition-bootstrap-config](https://cluster-api.sigs.k8s.io/tasks/experimental-features/ignition#experimental-feature-ignition-bootstrap-config-alpha)
285+
286+
you can do so, by [Enabling Experimental Features on Existing Management Clusters](https://cluster-api.sigs.k8s.io/tasks/experimental-features/experimental-features#enabling-experimental-features-on-existing-management-clusters)
287+
288+
Then, you can generate a cluster manifest with the following command:
289+
```shell
290+
$ clusterctl generate cluster flatcar-quickstart \
291+
--infrastructure proxmox \
292+
--kubernetes-version v1.30.5 \
293+
--control-plane-machine-count 3 \
294+
--worker-machine-count 3 \
295+
--flavor flatcar > cluster.yaml
296+
297+
$ kubectl apply -f cluster.yaml
298+
```
299+
300+
**Notes**:
301+
- Make sure to define at least one ssh key in the `VM_SSH_KEYS` environment variable, or the cluster will fail to provision.
302+
- If you want more customization, you can extend the template to add multiple interfaces or dual-stack.
303+
- Make sure that the ProxmoxMachines always ignore the cloud-init status by defining `spec.checks.skipCloudInitStatus: true` in the ProxmoxMachine CR.
304+
305+
```yaml
306+
spec:
307+
checks:
308+
skipQemuGuestAgent: false
309+
skipCloudInitStatus: true
310+
```
311+
275312
### Custom cluster templates
276313
277314
If you need anything specific that requires a more complex setup, we recommend to use custom templates:

‎go.mod‎

Lines changed: 6 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -7,6 +7,7 @@ replace sigs.k8s.io/cluster-api => sigs.k8s.io/cluster-api v1.8.5
77
replace github.com/google/cel-go => github.com/google/cel-go v0.17.8
88

99
require (
10+
github.com/flatcar/ignition v0.36.2
1011
github.com/go-logr/logr v1.4.2
1112
github.com/google/uuid v1.6.0
1213
github.com/jarcoal/httpmock v1.3.1
@@ -38,6 +39,7 @@ require (
3839
github.com/Microsoft/go-winio v0.5.0 // indirect
3940
github.com/ProtonMail/go-crypto v0.0.0-20230217124315-7d5c6f04bbb8 // indirect
4041
github.com/adrg/xdg v0.5.0 // indirect
42+
github.com/ajeddeloh/go-json v0.0.0-20200220154158-5ae607161559 // indirect
4143
github.com/alessio/shellescape v1.4.2 // indirect
4244
github.com/antlr/antlr4/runtime/Go/antlr/v4 v4.0.0-20230305170008-8188dc5388df // indirect
4345
github.com/asaskevich/govalidator v0.0.0-20230301143203-a9d515a09cc2 // indirect
@@ -47,6 +49,8 @@ require (
4749
github.com/cenkalti/backoff/v4 v4.2.1 // indirect
4850
github.com/cespare/xxhash/v2 v2.2.0 // indirect
4951
github.com/cloudflare/circl v1.3.7 // indirect
52+
github.com/coreos/go-semver v0.3.1 // indirect
53+
github.com/coreos/go-systemd v0.0.0-20191104093116-d3cd4ed1dbcf // indirect
5054
github.com/davecgh/go-spew v1.1.2-0.20180830191138-d8f796af33cc // indirect
5155
github.com/diskfs/go-diskfs v1.2.0 // indirect
5256
github.com/distribution/reference v0.6.0 // indirect
@@ -119,6 +123,7 @@ require (
119123
github.com/stretchr/objx v0.5.2 // indirect
120124
github.com/subosito/gotenv v1.6.0 // indirect
121125
github.com/valyala/fastjson v1.6.4 // indirect
126+
github.com/vincent-petithory/dataurl v1.0.0 // indirect
122127
go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp v0.49.0 // indirect
123128
go.opentelemetry.io/otel v1.24.0 // indirect
124129
go.opentelemetry.io/otel/exporters/otlp/otlptrace v1.22.0 // indirect
@@ -129,6 +134,7 @@ require (
129134
go.opentelemetry.io/proto/otlp v1.0.0 // indirect
130135
go.uber.org/multierr v1.11.0 // indirect
131136
go.uber.org/zap v1.27.0 // indirect
137+
go4.org v0.0.0-20201209231011-d4a079459e60 // indirect
132138
golang.org/x/crypto v0.29.0 // indirect
133139
golang.org/x/exp v0.0.0-20240719175910-8a7402abbf56 // indirect
134140
golang.org/x/mod v0.22.0 // indirect

0 commit comments

Comments
 (0)