feat: Allow configuration of kube-proxy mode on cluster creation

This commit allows users to configure kube-proxy mode when
creating clusters. This is enforced via CEL to prevent users
from moving between kube-proxy and non-kube-proxy deployments.

Currently only `Disabled` and `iptables` modes are supported due to
limitations in CAPI kube-proxy configuration.

Follow up work will allow Cilium configuration to enable their
kube-proxy replacement (already possible via custom Helm values)
and migration from kube-proxy to kube-proxy replacement for
existing clusters.

Author: jimmidyson

api/v1alpha1/clusterconfig_types.go

@@ -212,6 +212,10 @@ type GenericClusterConfigSpec struct {
 
 	// +kubebuilder:validation:Optional
 	DNS *DNS `json:"dns,omitempty"`
+
+	// KubeProxy defines the configuration for kube-proxy.
+	// +kubebuilder:validation:Optional
+	KubeProxy *KubeProxy `json:"kubeProxy,omitempty"`
 }
 
 type Image struct {
@@ -328,6 +332,27 @@ type CoreDNS struct {
 	Image *Image `json:"image,omitempty"`
 }
 
+type KubeProxyMode string
+
+const (
+	// KubeProxyModeDisabled indicates that kube-proxy should not be installed.
+	KubeProxyModeDisabled KubeProxyMode = "Disabled"
+	// KubeProxyModeIPTables indicates that kube-proxy should be installed in iptables
+	// mode.
+	KubeProxyModeIPTables KubeProxyMode = "iptables"
+)
+
+type KubeProxy struct {
+	// Mode specifies the mode for kube-proxy.
+	// Disabled means that kube-proxy is not installed.
+	// iptables means that kube-proxy is installed in iptables mode.
+	// +kubebuilder:validation:Optional
+	// +kubebuilder:validation:Enum=Disabled;iptables
+	// +kubebuilder:default=iptables
+	// +kubebuilder:validation:XValidation:rule="self == oldSelf",message="Value cannot be changed after cluster creation"
+	Mode KubeProxyMode `json:"mode,omitempty"`
+}
+
 //nolint:gochecknoinits // Idiomatic to use init functions to register APIs with scheme.
 func init() {
 	objectTypes = append(objectTypes,

api/v1alpha1/crds/caren.nutanix.com_awsclusterconfigs.yaml

@@ -560,6 +560,23 @@ spec:
                       - url
                     type: object
                   type: array
+                kubeProxy:
+                  description: KubeProxy defines the configuration for kube-proxy.
+                  properties:
+                    mode:
+                      default: iptables
+                      description: |-
+                        Mode specifies the mode for kube-proxy.
+                        Disabled means that kube-proxy is not installed.
+                        iptables means that kube-proxy is installed in iptables mode.
+                      enum:
+                        - Disabled
+                        - iptables
+                      type: string
+                      x-kubernetes-validations:
+                        - message: Value cannot be changed after cluster creation
+                          rule: self == oldSelf
+                  type: object
                 kubernetesImageRepository:
                   description: Sets the Kubernetes image repository used for the KubeadmControlPlane.
                   pattern: ^((?:[a-z0-9]([-a-z0-9]*[a-z0-9])?(\.[a-z0-9]([-a-z0-9]*[a-z0-9])?)*|\[(?:[a-fA-F0-9:]+)\])(:[0-9]+)?/)?[a-z0-9]+((?:[._]|__|[-]+)[a-z0-9]+)*(/[a-z0-9]+((?:[._]|__|[-]+)[a-z0-9]+)*)*$

api/v1alpha1/crds/caren.nutanix.com_dockerclusterconfigs.yaml

@@ -497,6 +497,23 @@ spec:
                       - url
                     type: object
                   type: array
+                kubeProxy:
+                  description: KubeProxy defines the configuration for kube-proxy.
+                  properties:
+                    mode:
+                      default: iptables
+                      description: |-
+                        Mode specifies the mode for kube-proxy.
+                        Disabled means that kube-proxy is not installed.
+                        iptables means that kube-proxy is installed in iptables mode.
+                      enum:
+                        - Disabled
+                        - iptables
+                      type: string
+                      x-kubernetes-validations:
+                        - message: Value cannot be changed after cluster creation
+                          rule: self == oldSelf
+                  type: object
                 kubernetesImageRepository:
                   description: Sets the Kubernetes image repository used for the KubeadmControlPlane.
                   pattern: ^((?:[a-z0-9]([-a-z0-9]*[a-z0-9])?(\.[a-z0-9]([-a-z0-9]*[a-z0-9])?)*|\[(?:[a-fA-F0-9:]+)\])(:[0-9]+)?/)?[a-z0-9]+((?:[._]|__|[-]+)[a-z0-9]+)*(/[a-z0-9]+((?:[._]|__|[-]+)[a-z0-9]+)*)*$

api/v1alpha1/crds/caren.nutanix.com_genericclusterconfigs.yaml

@@ -175,6 +175,23 @@ spec:
                   - url
                   type: object
                 type: array
+              kubeProxy:
+                description: KubeProxy defines the configuration for kube-proxy.
+                properties:
+                  mode:
+                    default: iptables
+                    description: |-
+                      Mode specifies the mode for kube-proxy.
+                      Disabled means that kube-proxy is not installed.
+                      iptables means that kube-proxy is installed in iptables mode.
+                    enum:
+                    - Disabled
+                    - iptables
+                    type: string
+                    x-kubernetes-validations:
+                    - message: Value cannot be changed after cluster creation
+                      rule: self == oldSelf
+                type: object
               kubernetesImageRepository:
                 description: Sets the Kubernetes image repository used for the KubeadmControlPlane.
                 pattern: ^((?:[a-z0-9]([-a-z0-9]*[a-z0-9])?(\.[a-z0-9]([-a-z0-9]*[a-z0-9])?)*|\[(?:[a-fA-F0-9:]+)\])(:[0-9]+)?/)?[a-z0-9]+((?:[._]|__|[-]+)[a-z0-9]+)*(/[a-z0-9]+((?:[._]|__|[-]+)[a-z0-9]+)*)*$

api/v1alpha1/crds/caren.nutanix.com_nutanixclusterconfigs.yaml

@@ -676,6 +676,23 @@ spec:
                       - url
                     type: object
                   type: array
+                kubeProxy:
+                  description: KubeProxy defines the configuration for kube-proxy.
+                  properties:
+                    mode:
+                      default: iptables
+                      description: |-
+                        Mode specifies the mode for kube-proxy.
+                        Disabled means that kube-proxy is not installed.
+                        iptables means that kube-proxy is installed in iptables mode.
+                      enum:
+                        - Disabled
+                        - iptables
+                      type: string
+                      x-kubernetes-validations:
+                        - message: Value cannot be changed after cluster creation
+                          rule: self == oldSelf
+                  type: object
                 kubernetesImageRepository:
                   description: Sets the Kubernetes image repository used for the KubeadmControlPlane.
                   pattern: ^((?:[a-z0-9]([-a-z0-9]*[a-z0-9])?(\.[a-z0-9]([-a-z0-9]*[a-z0-9])?)*|\[(?:[a-fA-F0-9:]+)\])(:[0-9]+)?/)?[a-z0-9]+((?:[._]|__|[-]+)[a-z0-9]+)*(/[a-z0-9]+((?:[._]|__|[-]+)[a-z0-9]+)*)*$

api/v1alpha1/zz_generated.deepcopy.go

@@ -0,0 +1,39 @@
++++
+title = "kube-proxy mode"
++++
+
+This customization allows configuration of the `kube-proxy` proxy mode. Currently, only `iptables` or `Disabled` modes
+are supported. `Disabled` is useful when deploying a CNI implementation that can replace `kube-proxy` to avoid
+potential conflicts. By default, `kube-proxy` is enabled in `iptables` mode.
+
+## Example
+
+To disable the deployment of `kube-proxy`, specify the following configuration:
+
+```yaml
+apiVersion: cluster.x-k8s.io/v1beta1
+kind: Cluster
+metadata:
+  name: <NAME>
+spec:
+  topology:
+    variables:
+      - name: clusterConfig
+        value:
+          kubeProxy:
+            mode: Disabled
+```
+
+Applying this configuration will result in the following configuration being applied:
+
+- `KubeadmControlPlaneTemplate`:
+
+  - ```yaml
+    spec:
+      template:
+        spec:
+          kubeadmConfigSpec:
+            initConfiguration:
+              skipPhases:
+                - addon/kube-proxy
+    ```

docs/content/customization/generic/kube-proxy-mode.md

@@ -20,6 +20,7 @@ import (
 	"github.com/nutanix-cloud-native/cluster-api-runtime-extensions-nutanix/pkg/handlers/generic/mutation/extraapiservercertsans"
 	"github.com/nutanix-cloud-native/cluster-api-runtime-extensions-nutanix/pkg/handlers/generic/mutation/httpproxy"
 	"github.com/nutanix-cloud-native/cluster-api-runtime-extensions-nutanix/pkg/handlers/generic/mutation/imageregistries/credentials"
+	"github.com/nutanix-cloud-native/cluster-api-runtime-extensions-nutanix/pkg/handlers/generic/mutation/kubeproxymode"
 	"github.com/nutanix-cloud-native/cluster-api-runtime-extensions-nutanix/pkg/handlers/generic/mutation/kubernetesimagerepository"
 	"github.com/nutanix-cloud-native/cluster-api-runtime-extensions-nutanix/pkg/handlers/generic/mutation/mirrors"
 	"github.com/nutanix-cloud-native/cluster-api-runtime-extensions-nutanix/pkg/handlers/generic/mutation/noderegistration"
@@ -44,6 +45,7 @@ func MetaMutators(mgr manager.Manager) []mutation.MetaMutator {
 		containerdunprivilegedports.NewPatch(),
 		encryptionatrest.NewPatch(mgr.GetClient(), encryptionatrest.RandomTokenGenerator),
 		autorenewcerts.NewPatch(),
+		kubeproxymode.NewPatch(),
 
 		// Some patches may have changed containerd configuration.
 		// We write the configuration changes to disk, and must run a command

pkg/handlers/generic/mutation/handlers.go

@@ -0,0 +1,126 @@
+// Copyright 2025 Nutanix. All rights reserved.
+// SPDX-License-Identifier: Apache-2.0
+
+package kubeproxymode
+
+import (
+	"context"
+	"fmt"
+
+	apiextensionsv1 "k8s.io/apiextensions-apiserver/pkg/apis/apiextensions/v1"
+	"k8s.io/apimachinery/pkg/apis/meta/v1/unstructured"
+	bootstrapv1 "sigs.k8s.io/cluster-api/bootstrap/kubeadm/api/v1beta1"
+	controlplanev1 "sigs.k8s.io/cluster-api/controlplane/kubeadm/api/v1beta1"
+	runtimehooksv1 "sigs.k8s.io/cluster-api/exp/runtime/hooks/api/v1alpha1"
+	ctrl "sigs.k8s.io/controller-runtime"
+	"sigs.k8s.io/controller-runtime/pkg/client"
+
+	"github.com/nutanix-cloud-native/cluster-api-runtime-extensions-nutanix/api/v1alpha1"
+	"github.com/nutanix-cloud-native/cluster-api-runtime-extensions-nutanix/common/pkg/capi/clustertopology/handlers/mutation"
+	"github.com/nutanix-cloud-native/cluster-api-runtime-extensions-nutanix/common/pkg/capi/clustertopology/patches"
+	"github.com/nutanix-cloud-native/cluster-api-runtime-extensions-nutanix/common/pkg/capi/clustertopology/patches/selectors"
+	"github.com/nutanix-cloud-native/cluster-api-runtime-extensions-nutanix/common/pkg/capi/clustertopology/variables"
+)
+
+const (
+	// VariableName is the external patch variable name.
+	VariableName = "kubeProxy"
+)
+
+type kubeProxyMode struct {
+	variableName      string
+	variableFieldPath []string
+}
+
+func NewPatch() *kubeProxyMode {
+	return newKubeProxyModePatch(
+		v1alpha1.ClusterConfigVariableName,
+		VariableName,
+		"mode",
+	)
+}
+
+func newKubeProxyModePatch(
+	variableName string,
+	variableFieldPath ...string,
+) *kubeProxyMode {
+	return &kubeProxyMode{
+		variableName:      variableName,
+		variableFieldPath: variableFieldPath,
+	}
+}
+
+func (h *kubeProxyMode) Mutate(
+	ctx context.Context,
+	obj *unstructured.Unstructured,
+	vars map[string]apiextensionsv1.JSON,
+	holderRef runtimehooksv1.HolderReference,
+	_ client.ObjectKey,
+	clusterGetter mutation.ClusterGetter,
+) error {
+	log := ctrl.LoggerFrom(ctx).WithValues(
+		"holderRef", holderRef,
+	)
+
+	kubeProxyMode, err := variables.Get[v1alpha1.KubeProxyMode](
+		vars,
+		h.variableName,
+		h.variableFieldPath...,
+	)
+	if err != nil {
+		if variables.IsNotFoundError(err) {
+			log.V(5).Info("kubeProxy mode variable not defined")
+			return nil
+		}
+
+		return err
+	}
+
+	log = log.WithValues(
+		"variableName",
+		h.variableName,
+		"variableFieldPath",
+		h.variableFieldPath,
+		"variableValue",
+		kubeProxyMode,
+	)
+
+	if kubeProxyMode == "" {
+		log.V(5).Info("kube proxy mode is not set, skipping mutation")
+		return nil
+	}
+
+	return patches.MutateIfApplicable(
+		obj,
+		vars,
+		&holderRef,
+		selectors.ControlPlane(),
+		log,
+		func(obj *controlplanev1.KubeadmControlPlaneTemplate) error {
+			log.WithValues(
+				"patchedObjectKind", obj.GetObjectKind().GroupVersionKind().String(),
+				"patchedObjectName", client.ObjectKeyFromObject(obj),
+			).Info("adding kube proxy mode to control plane kubeadm config spec")
+			if obj.Spec.Template.Spec.KubeadmConfigSpec.InitConfiguration == nil {
+				obj.Spec.Template.Spec.KubeadmConfigSpec.InitConfiguration = &bootstrapv1.InitConfiguration{}
+			}
+
+			switch kubeProxyMode {
+			case v1alpha1.KubeProxyModeDisabled:
+				log.Info("kube proxy mode is set to disabled, skipping kube-proxy addon")
+				obj.Spec.Template.Spec.KubeadmConfigSpec.InitConfiguration.SkipPhases = append(
+					obj.Spec.Template.Spec.KubeadmConfigSpec.InitConfiguration.SkipPhases,
+					"addon/kube-proxy",
+				)
+			case v1alpha1.KubeProxyModeIPTables:
+				log.Info(
+					"kube proxy mode is set to iptables, no patches required as this is the default mode configured by kubeadm",
+				)
+			default:
+				return fmt.Errorf("unknown kube proxy mode %q", kubeProxyMode)
+			}
+
+			return nil
+		},
+	)
+}