feat: Allow configuration of kube-proxy mode on cluster creation (#1163)

jimmidyson · web-flow · commit 255a03e816cd · 2025-06-25T19:24:33.000Z
This commit allows users to configure kube-proxy mode when creating clusters. This is enforced via CEL to prevent users from moving between kube-proxy and non-kube-proxy deployments. Supports `iptables` (default), and `nftables` modes. Skipping `kube-proxy` deployment and management by CAPI is enabled via setting the upstream `controlplane.cluster.x-k8s.io/skip-kube-proxy` annotation on the `Cluster` `spec.topology.controlPlane.metadata.annotation`. This will configure `kubeadm` to skip the `addon/kube-proxy` phase. Follow up work will allow Cilium configuration to enable their kube-proxy replacement (already possible via custom Helm values) and migration from kube-proxy to kube-proxy replacement for existing clusters. Requires #1162.
diff --git a/api/v1alpha1/clusterconfig_types.go b/api/v1alpha1/clusterconfig_types.go
@@ -212,6 +212,10 @@ type GenericClusterConfigSpec struct {
 
 	// +kubebuilder:validation:Optional
 	DNS *DNS `json:"dns,omitempty"`
+
+	// KubeProxy defines the configuration for kube-proxy.
+	// +kubebuilder:validation:Optional
+	KubeProxy *KubeProxy `json:"kubeProxy,omitempty"`
 }
 
 type Image struct {
@@ -328,6 +332,27 @@ type CoreDNS struct {
 	Image *Image `json:"image,omitempty"`
 }
 
+type KubeProxyMode string
+
+const (
+	// KubeProxyModeIPTables indicates that kube-proxy should be installed in iptables
+	// mode.
+	KubeProxyModeIPTables KubeProxyMode = "iptables"
+	// KubeProxyModeNFTables indicates that kube-proxy should be installed in nftables
+	// mode.
+	KubeProxyModeNFTables KubeProxyMode = "nftables"
+)
+
+type KubeProxy struct {
+	// Mode specifies the mode for kube-proxy:
+	// - iptables means that kube-proxy is installed in iptables mode.
+	// - nftables means that kube-proxy is installed in nftables mode.
+	// +kubebuilder:validation:Optional
+	// +kubebuilder:validation:Enum=iptables;nftables
+	// +kubebuilder:validation:XValidation:rule="self == oldSelf",message="Value cannot be changed after cluster creation"
+	Mode KubeProxyMode `json:"mode,omitempty"`
+}
+
 //nolint:gochecknoinits // Idiomatic to use init functions to register APIs with scheme.
 func init() {
 	objectTypes = append(objectTypes,
diff --git a/api/v1alpha1/crds/caren.nutanix.com_awsclusterconfigs.yaml b/api/v1alpha1/crds/caren.nutanix.com_awsclusterconfigs.yaml
@@ -560,6 +560,22 @@ spec:
                       - url
                     type: object
                   type: array
+                kubeProxy:
+                  description: KubeProxy defines the configuration for kube-proxy.
+                  properties:
+                    mode:
+                      description: |-
+                        Mode specifies the mode for kube-proxy:
+                        - iptables means that kube-proxy is installed in iptables mode.
+                        - nftables means that kube-proxy is installed in nftables mode.
+                      enum:
+                        - iptables
+                        - nftables
+                      type: string
+                      x-kubernetes-validations:
+                        - message: Value cannot be changed after cluster creation
+                          rule: self == oldSelf
+                  type: object
                 kubernetesImageRepository:
                   description: Sets the Kubernetes image repository used for the KubeadmControlPlane.
                   pattern: ^((?:[a-z0-9]([-a-z0-9]*[a-z0-9])?(\.[a-z0-9]([-a-z0-9]*[a-z0-9])?)*|\[(?:[a-fA-F0-9:]+)\])(:[0-9]+)?/)?[a-z0-9]+((?:[._]|__|[-]+)[a-z0-9]+)*(/[a-z0-9]+((?:[._]|__|[-]+)[a-z0-9]+)*)*$
diff --git a/api/v1alpha1/crds/caren.nutanix.com_dockerclusterconfigs.yaml b/api/v1alpha1/crds/caren.nutanix.com_dockerclusterconfigs.yaml
@@ -497,6 +497,22 @@ spec:
                       - url
                     type: object
                   type: array
+                kubeProxy:
+                  description: KubeProxy defines the configuration for kube-proxy.
+                  properties:
+                    mode:
+                      description: |-
+                        Mode specifies the mode for kube-proxy:
+                        - iptables means that kube-proxy is installed in iptables mode.
+                        - nftables means that kube-proxy is installed in nftables mode.
+                      enum:
+                        - iptables
+                        - nftables
+                      type: string
+                      x-kubernetes-validations:
+                        - message: Value cannot be changed after cluster creation
+                          rule: self == oldSelf
+                  type: object
                 kubernetesImageRepository:
                   description: Sets the Kubernetes image repository used for the KubeadmControlPlane.
                   pattern: ^((?:[a-z0-9]([-a-z0-9]*[a-z0-9])?(\.[a-z0-9]([-a-z0-9]*[a-z0-9])?)*|\[(?:[a-fA-F0-9:]+)\])(:[0-9]+)?/)?[a-z0-9]+((?:[._]|__|[-]+)[a-z0-9]+)*(/[a-z0-9]+((?:[._]|__|[-]+)[a-z0-9]+)*)*$
diff --git a/api/v1alpha1/crds/caren.nutanix.com_genericclusterconfigs.yaml b/api/v1alpha1/crds/caren.nutanix.com_genericclusterconfigs.yaml
@@ -175,6 +175,22 @@ spec:
                   - url
                   type: object
                 type: array
+              kubeProxy:
+                description: KubeProxy defines the configuration for kube-proxy.
+                properties:
+                  mode:
+                    description: |-
+                      Mode specifies the mode for kube-proxy:
+                      - iptables means that kube-proxy is installed in iptables mode.
+                      - nftables means that kube-proxy is installed in nftables mode.
+                    enum:
+                    - iptables
+                    - nftables
+                    type: string
+                    x-kubernetes-validations:
+                    - message: Value cannot be changed after cluster creation
+                      rule: self == oldSelf
+                type: object
               kubernetesImageRepository:
                 description: Sets the Kubernetes image repository used for the KubeadmControlPlane.
                 pattern: ^((?:[a-z0-9]([-a-z0-9]*[a-z0-9])?(\.[a-z0-9]([-a-z0-9]*[a-z0-9])?)*|\[(?:[a-fA-F0-9:]+)\])(:[0-9]+)?/)?[a-z0-9]+((?:[._]|__|[-]+)[a-z0-9]+)*(/[a-z0-9]+((?:[._]|__|[-]+)[a-z0-9]+)*)*$
diff --git a/api/v1alpha1/crds/caren.nutanix.com_nutanixclusterconfigs.yaml b/api/v1alpha1/crds/caren.nutanix.com_nutanixclusterconfigs.yaml
@@ -676,6 +676,22 @@ spec:
                       - url
                     type: object
                   type: array
+                kubeProxy:
+                  description: KubeProxy defines the configuration for kube-proxy.
+                  properties:
+                    mode:
+                      description: |-
+                        Mode specifies the mode for kube-proxy:
+                        - iptables means that kube-proxy is installed in iptables mode.
+                        - nftables means that kube-proxy is installed in nftables mode.
+                      enum:
+                        - iptables
+                        - nftables
+                      type: string
+                      x-kubernetes-validations:
+                        - message: Value cannot be changed after cluster creation
+                          rule: self == oldSelf
+                  type: object
                 kubernetesImageRepository:
                   description: Sets the Kubernetes image repository used for the KubeadmControlPlane.
                   pattern: ^((?:[a-z0-9]([-a-z0-9]*[a-z0-9])?(\.[a-z0-9]([-a-z0-9]*[a-z0-9])?)*|\[(?:[a-fA-F0-9:]+)\])(:[0-9]+)?/)?[a-z0-9]+((?:[._]|__|[-]+)[a-z0-9]+)*(/[a-z0-9]+((?:[._]|__|[-]+)[a-z0-9]+)*)*$
diff --git a/api/v1alpha1/zz_generated.deepcopy.go b/api/v1alpha1/zz_generated.deepcopy.go
diff --git a/docs/content/customization/generic/kube-proxy-mode.md b/docs/content/customization/generic/kube-proxy-mode.md
@@ -0,0 +1,81 @@
++++
+title = "kube-proxy mode"
++++
+
+This customization allows configuration of the `kube-proxy` proxy mode. Currently, only `iptables` and `nftables`
+modes are supported. By default, `kube-proxy` is enabled in `iptables` mode by `kubeadm`.
+
+## Examples
+
+### Enabling nftables kube-proxy mode
+
+Enabling `nftables` is done via the following configuration:
+
+```yaml
+apiVersion: cluster.x-k8s.io/v1beta1
+kind: Cluster
+metadata:
+  name: <NAME>
+spec:
+  topology:
+    variables:
+      - name: clusterConfig
+        value:
+          kubeProxy:
+            mode: nftables
+```
+
+Applying this configuration will result in the following configuration being applied to create a
+`KubeProxyConfiguration` and append it to the kubeadm configuration that is used when `kubeadm init`
+is executed:
+
+- `KubeadmControlPlaneTemplate`:
+
+  - ```yaml
+    spec:
+      template:
+        spec:
+          kubeadmConfigSpec:
+            files:
+              - path: "/etc/kubernetes/kubeproxy-config.yaml"
+                owner: "root:root"
+                permissions: "0644"
+                content: |-
+                  ---
+                  apiVersion: kubeproxy.config.k8s.io/v1alpha1
+                  kind: KubeProxyConfiguration
+                  mode: nftables
+          preKubeadmCommands:
+            - /bin/sh -ec 'cat /etc/kubernetes/kubeproxy-config.yaml >> /run/kubeadm/kubeadm.yaml'
+    ```
+
+### Skipping kube-proxy installation
+
+To disable the deployment and upgrade of `kube-proxy`, specify the following configuration:
+
+```yaml
+apiVersion: cluster.x-k8s.io/v1beta1
+kind: Cluster
+metadata:
+  name: <NAME>
+spec:
+  topology:
+    controlPlane:
+      metadata:
+        annotations:
+          controlplane.cluster.x-k8s.io/skip-kube-proxy: ""
+```
+
+Applying this configuration will result in the following configuration being applied:
+
+- `KubeadmControlPlaneTemplate`:
+
+  - ```yaml
+    spec:
+      template:
+        spec:
+          kubeadmConfigSpec:
+            initConfiguration:
+              skipPhases:
+                - addon/kube-proxy
+    ```
diff --git a/pkg/handlers/generic/mutation/extraapiservercertsans/inject_test.go b/pkg/handlers/generic/mutation/extraapiservercertsans/inject_test.go
@@ -175,12 +175,10 @@ var _ = Describe("Generate Extra API server certificate patches", func() {
 			utilruntime.Must(clientgoscheme.AddToScheme(clientScheme))
 			utilruntime.Must(clusterv1.AddToScheme(clientScheme))
 			cl, err := helpers.TestEnv.GetK8sClientWithScheme(clientScheme)
-			gomega.Expect(err).To(gomega.BeNil())
-			err = cl.Create(context.Background(), &tt.cluster)
-			gomega.Expect(err).To(gomega.BeNil())
+			gomega.Expect(err).ToNot(gomega.HaveOccurred())
+			gomega.Expect(cl.Create(context.Background(), &tt.cluster)).To(gomega.Succeed())
+			DeferCleanup(cl.Delete, context.Background(), &tt.cluster)
 			capitest.AssertGeneratePatches(GinkgoT(), patchGenerator, &tt.patchTest)
-			err = cl.Delete(context.Background(), &tt.cluster)
-			gomega.Expect(err).To(gomega.BeNil())
 		})
 	}
 })
diff --git a/pkg/handlers/generic/mutation/handlers.go b/pkg/handlers/generic/mutation/handlers.go
@@ -20,6 +20,7 @@ import (
 	"github.com/nutanix-cloud-native/cluster-api-runtime-extensions-nutanix/pkg/handlers/generic/mutation/extraapiservercertsans"
 	"github.com/nutanix-cloud-native/cluster-api-runtime-extensions-nutanix/pkg/handlers/generic/mutation/httpproxy"
 	"github.com/nutanix-cloud-native/cluster-api-runtime-extensions-nutanix/pkg/handlers/generic/mutation/imageregistries/credentials"
+	"github.com/nutanix-cloud-native/cluster-api-runtime-extensions-nutanix/pkg/handlers/generic/mutation/kubeproxymode"
 	"github.com/nutanix-cloud-native/cluster-api-runtime-extensions-nutanix/pkg/handlers/generic/mutation/kubernetesimagerepository"
 	"github.com/nutanix-cloud-native/cluster-api-runtime-extensions-nutanix/pkg/handlers/generic/mutation/mirrors"
 	"github.com/nutanix-cloud-native/cluster-api-runtime-extensions-nutanix/pkg/handlers/generic/mutation/noderegistration"
@@ -44,6 +45,7 @@ func MetaMutators(mgr manager.Manager) []mutation.MetaMutator {
 		containerdunprivilegedports.NewPatch(),
 		encryptionatrest.NewPatch(mgr.GetClient(), encryptionatrest.RandomTokenGenerator),
 		autorenewcerts.NewPatch(),
+		kubeproxymode.NewPatch(),
 
 		// Some patches may have changed containerd configuration.
 		// We write the configuration changes to disk, and must run a command
diff --git a/pkg/handlers/generic/mutation/kubeproxymode/embedded/kubeproxyconfig.yaml b/pkg/handlers/generic/mutation/kubeproxymode/embedded/kubeproxyconfig.yaml
@@ -0,0 +1,4 @@
+---
+apiVersion: kubeproxy.config.k8s.io/v1alpha1
+kind: KubeProxyConfiguration
+mode: {{ .Mode }}
diff --git a/pkg/handlers/generic/mutation/kubeproxymode/inject.go b/pkg/handlers/generic/mutation/kubeproxymode/inject.go
diff --git a/pkg/handlers/generic/mutation/kubeproxymode/inject_test.go b/pkg/handlers/generic/mutation/kubeproxymode/inject_test.go
diff --git a/pkg/handlers/generic/mutation/kubeproxymode/variables_test.go b/pkg/handlers/generic/mutation/kubeproxymode/variables_test.go
