Skip to content

Commit 597bea7

Browse files
jimmidysonjimmidyson
committed
fixup! refactor: Rely on skip-kube-proxy annotation
1 parent c46468e commit 597bea7

File tree

9 files changed

+98
-115
lines changed

9 files changed

+98
-115
lines changed

‎api/v1alpha1/clusterconfig_types.go‎

Lines changed: 1 addition & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -335,8 +335,6 @@ type CoreDNS struct {
335335
type KubeProxyMode string
336336

337337
const (
338-
// KubeProxyModeDisabled indicates that kube-proxy should not be installed.
339-
KubeProxyModeDisabled KubeProxyMode = "Disabled"
340338
// KubeProxyModeIPTables indicates that kube-proxy should be installed in iptables
341339
// mode.
342340
KubeProxyModeIPTables KubeProxyMode = "iptables"
@@ -348,12 +346,10 @@ const (
348346
type KubeProxy struct {
349347
// Mode specifies the mode for kube-proxy:
350348
//
351-
// - Disabled means that kube-proxy is not installed.
352349
// - iptables means that kube-proxy is installed in iptables mode.
353350
// - nftables means that kube-proxy is installed in nftables mode.
354351
// +kubebuilder:validation:Optional
355-
// +kubebuilder:validation:Enum=Disabled;iptables;nftables
356-
// +kubebuilder:default=iptables
352+
// +kubebuilder:validation:Enum=iptables;nftables
357353
// +kubebuilder:validation:XValidation:rule="self == oldSelf",message="Value cannot be changed after cluster creation"
358354
Mode KubeProxyMode `json:"mode,omitempty"`
359355
}

‎api/v1alpha1/crds/caren.nutanix.com_awsclusterconfigs.yaml‎

Lines changed: 0 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -564,15 +564,12 @@ spec:
564564
description: KubeProxy defines the configuration for kube-proxy.
565565
properties:
566566
mode:
567-
default: iptables
568567
description: |-
569568
Mode specifies the mode for kube-proxy:
570569
571-
- Disabled means that kube-proxy is not installed.
572570
- iptables means that kube-proxy is installed in iptables mode.
573571
- nftables means that kube-proxy is installed in nftables mode.
574572
enum:
575-
- Disabled
576573
- iptables
577574
- nftables
578575
type: string

‎api/v1alpha1/crds/caren.nutanix.com_dockerclusterconfigs.yaml‎

Lines changed: 0 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -501,15 +501,12 @@ spec:
501501
description: KubeProxy defines the configuration for kube-proxy.
502502
properties:
503503
mode:
504-
default: iptables
505504
description: |-
506505
Mode specifies the mode for kube-proxy:
507506
508-
- Disabled means that kube-proxy is not installed.
509507
- iptables means that kube-proxy is installed in iptables mode.
510508
- nftables means that kube-proxy is installed in nftables mode.
511509
enum:
512-
- Disabled
513510
- iptables
514511
- nftables
515512
type: string

‎api/v1alpha1/crds/caren.nutanix.com_genericclusterconfigs.yaml‎

Lines changed: 0 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -179,15 +179,12 @@ spec:
179179
description: KubeProxy defines the configuration for kube-proxy.
180180
properties:
181181
mode:
182-
default: iptables
183182
description: |-
184183
Mode specifies the mode for kube-proxy:
185184
186-
- Disabled means that kube-proxy is not installed.
187185
- iptables means that kube-proxy is installed in iptables mode.
188186
- nftables means that kube-proxy is installed in nftables mode.
189187
enum:
190-
- Disabled
191188
- iptables
192189
- nftables
193190
type: string

‎api/v1alpha1/crds/caren.nutanix.com_nutanixclusterconfigs.yaml‎

Lines changed: 0 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -680,15 +680,12 @@ spec:
680680
description: KubeProxy defines the configuration for kube-proxy.
681681
properties:
682682
mode:
683-
default: iptables
684683
description: |-
685684
Mode specifies the mode for kube-proxy:
686685
687-
- Disabled means that kube-proxy is not installed.
688686
- iptables means that kube-proxy is installed in iptables mode.
689687
- nftables means that kube-proxy is installed in nftables mode.
690688
enum:
691-
- Disabled
692689
- iptables
693690
- nftables
694691
type: string

‎docs/content/customization/generic/kube-proxy-mode.md‎

Lines changed: 7 additions & 23 deletions
Original file line numberDiff line numberDiff line change
@@ -2,9 +2,8 @@
22
title = "kube-proxy mode"
33
+++
44

5-
This customization allows configuration of the `kube-proxy` proxy mode. Currently, only `iptables`, `nftables` or
6-
`Disabled` modes are supported. `Disabled` is useful when deploying a CNI implementation that can replace `kube-proxy`
7-
to avoid potential conflicts. By default, `kube-proxy` is enabled in `iptables` mode.
5+
This customization allows configuration of the `kube-proxy` proxy mode. Currently, only `iptables` and `nftables`
6+
modes are supported. By default, `kube-proxy` is enabled in `iptables` mode by `kubeadm`.
87

98
## Examples
109

@@ -52,7 +51,7 @@ is executed:
5251

5352
### Skipping kube-proxy installation
5453

55-
To disable the deployment of `kube-proxy`, specify the following configuration:
54+
To disable the deployment and upgrade of `kube-proxy`, specify the following configuration:
5655

5756
```yaml
5857
apiVersion: cluster.x-k8s.io/v1beta1
@@ -61,11 +60,10 @@ metadata:
6160
name: <NAME>
6261
spec:
6362
topology:
64-
variables:
65-
- name: clusterConfig
66-
value:
67-
kubeProxy:
68-
mode: Disabled
63+
controlPlane:
64+
metadata:
65+
annotations:
66+
controlplane.cluster.x-k8s.io/skip-kube-proxy: ""
6967
```
7068

7169
Applying this configuration will result in the following configuration being applied:
@@ -81,17 +79,3 @@ Applying this configuration will result in the following configuration being app
8179
skipPhases:
8280
- addon/kube-proxy
8381
```
84-
85-
**IMPORTANT**: If you are disabling kube-proxy in this way to manage kube-proxy yourself, then you will also need
86-
to add the following control plane annotation to your `Cluster` definition:
87-
88-
```yaml
89-
spec:
90-
topology:
91-
controlPlane:
92-
metadata:
93-
annotations:
94-
controlplane.cluster.x-k8s.io/skip-kube-proxy: ""
95-
```
96-
97-
Without this, CAPI will attempt to upgrade kube-proxy when the cluster is upgraded.

‎pkg/handlers/generic/mutation/kubeproxymode/inject.go‎

Lines changed: 29 additions & 27 deletions
Original file line numberDiff line numberDiff line change
@@ -9,6 +9,7 @@ import (
99

1010
apiextensionsv1 "k8s.io/apiextensions-apiserver/pkg/apis/apiextensions/v1"
1111
"k8s.io/apimachinery/pkg/apis/meta/v1/unstructured"
12+
clusterv1 "sigs.k8s.io/cluster-api/api/v1beta1"
1213
bootstrapv1 "sigs.k8s.io/cluster-api/bootstrap/kubeadm/api/v1beta1"
1314
controlplanev1 "sigs.k8s.io/cluster-api/controlplane/kubeadm/api/v1beta1"
1415
runtimehooksv1 "sigs.k8s.io/cluster-api/exp/runtime/hooks/api/v1alpha1"
@@ -76,17 +77,23 @@ func (h *kubeProxyMode) Mutate(
7677
"holderRef", holderRef,
7778
)
7879

80+
cluster, err := clusterGetter(ctx)
81+
if err != nil {
82+
log.Error(err, "failed to get cluster for kube proxy mode mutation")
83+
return fmt.Errorf("failed to get cluster for kube proxy mode mutation: %w", err)
84+
}
85+
86+
isSkipProxy := false
87+
if cluster.Spec.Topology != nil {
88+
_, isSkipProxy = cluster.Spec.Topology.ControlPlane.Metadata.Annotations[controlplanev1.SkipKubeProxyAnnotation]
89+
}
90+
7991
kubeProxyMode, err := variables.Get[v1alpha1.KubeProxyMode](
8092
vars,
8193
h.variableName,
8294
h.variableFieldPath...,
8395
)
84-
if err != nil {
85-
if variables.IsNotFoundError(err) {
86-
log.V(5).Info("kubeProxy mode variable not defined")
87-
return nil
88-
}
89-
96+
if err != nil && !variables.IsNotFoundError(err) {
9097
return err
9198
}
9299

@@ -99,8 +106,8 @@ func (h *kubeProxyMode) Mutate(
99106
kubeProxyMode,
100107
)
101108

102-
if kubeProxyMode == "" {
103-
log.V(5).Info("kube proxy mode is not set, skipping mutation")
109+
if kubeProxyMode == "" && !isSkipProxy {
110+
log.V(5).Info("kube proxy mode is not set or skipped, skipping mutation")
104111
return nil
105112
}
106113

@@ -116,9 +123,11 @@ func (h *kubeProxyMode) Mutate(
116123
"patchedObjectName", client.ObjectKeyFromObject(obj),
117124
).Info("adding kube proxy mode to control plane kubeadm config spec")
118125

119-
switch kubeProxyMode {
120-
case v1alpha1.KubeProxyModeDisabled:
121-
log.Info("kube proxy mode is set to disabled, skipping kube-proxy addon")
126+
if isSkipProxy {
127+
log.Info(
128+
"cluster controlplane contains controlplane.cluster.x-k8s.io/skip-kube-proxy annotation, " +
129+
"skipping kube-proxy addon",
130+
)
122131
if obj.Spec.Template.Spec.KubeadmConfigSpec.InitConfiguration == nil {
123132
obj.Spec.Template.Spec.KubeadmConfigSpec.InitConfiguration = &bootstrapv1.InitConfiguration{}
124133
}
@@ -127,15 +136,13 @@ func (h *kubeProxyMode) Mutate(
127136
initConfiguration.SkipPhases,
128137
"addon/kube-proxy",
129138
)
139+
140+
return nil
141+
}
142+
143+
switch kubeProxyMode {
130144
case v1alpha1.KubeProxyModeIPTables, v1alpha1.KubeProxyModeNFTables:
131-
kubeProxyConfigProviderTemplate, err := templateForClusterProvider(ctx, clusterGetter)
132-
if err != nil {
133-
log.Error(
134-
err,
135-
"failed to get kube proxy config template for cluster provider",
136-
)
137-
return fmt.Errorf("failed to get cluster for kube proxy mode mutation: %w", err)
138-
}
145+
kubeProxyConfigProviderTemplate := templateForClusterProvider(cluster)
139146

140147
kubeProxyConfig := bootstrapv1.File{
141148
Path: "/etc/kubernetes/kubeproxy-config.yaml",
@@ -162,16 +169,11 @@ func (h *kubeProxyMode) Mutate(
162169
}
163170

164171
// templateForClusterProvider returns the kube-proxy config template based on the cluster provider.
165-
func templateForClusterProvider(ctx context.Context, clusterGetter mutation.ClusterGetter) (string, error) {
166-
cluster, err := clusterGetter(ctx)
167-
if err != nil {
168-
return "", err
169-
}
170-
172+
func templateForClusterProvider(cluster *clusterv1.Cluster) string {
171173
switch utils.GetProvider(cluster) {
172174
case "docker":
173-
return kubeProxyConfigYAMLTemplateForDockerProvider, nil
175+
return kubeProxyConfigYAMLTemplateForDockerProvider
174176
default:
175-
return kubeProxyConfigYAMLTemplate, nil
177+
return kubeProxyConfigYAMLTemplate
176178
}
177179
}

‎pkg/handlers/generic/mutation/kubeproxymode/inject_test.go‎

Lines changed: 61 additions & 21 deletions
Original file line numberDiff line numberDiff line change
@@ -14,6 +14,7 @@ import (
1414
utilruntime "k8s.io/apimachinery/pkg/util/runtime"
1515
clientgoscheme "k8s.io/client-go/kubernetes/scheme"
1616
clusterv1 "sigs.k8s.io/cluster-api/api/v1beta1"
17+
controlplanev1 "sigs.k8s.io/cluster-api/controlplane/kubeadm/api/v1beta1"
1718
runtimehooksv1 "sigs.k8s.io/cluster-api/exp/runtime/hooks/api/v1alpha1"
1819

1920
"github.com/nutanix-cloud-native/cluster-api-runtime-extensions-nutanix/api/v1alpha1"
@@ -49,13 +50,7 @@ var _ = Describe("Generate kube proxy mode patches", func() {
4950
Vars: []runtimehooksv1.Variable{
5051
capitest.VariableWithValue(
5152
v1alpha1.ClusterConfigVariableName,
52-
v1alpha1.AWSClusterConfigSpec{
53-
GenericClusterConfigSpec: v1alpha1.GenericClusterConfigSpec{
54-
KubeProxy: &v1alpha1.KubeProxy{
55-
Mode: v1alpha1.KubeProxyModeDisabled,
56-
},
57-
},
58-
},
53+
v1alpha1.AWSClusterConfigSpec{},
5954
),
6055
},
6156
RequestItem: request.NewKubeadmControlPlaneTemplateRequestItem(""),
@@ -65,19 +60,35 @@ var _ = Describe("Generate kube proxy mode patches", func() {
6560
ValueMatcher: gomega.ConsistOf("addon/kube-proxy"),
6661
}},
6762
},
63+
cluster: &clusterv1.Cluster{
64+
ObjectMeta: metav1.ObjectMeta{
65+
Name: "test-cluster",
66+
Namespace: request.Namespace,
67+
Labels: map[string]string{
68+
clusterv1.ProviderNameLabel: "aws",
69+
},
70+
},
71+
Spec: clusterv1.ClusterSpec{
72+
Topology: &clusterv1.Topology{
73+
Version: "dummy-version",
74+
Class: "dummy-class",
75+
ControlPlane: clusterv1.ControlPlaneTopology{
76+
Metadata: clusterv1.ObjectMeta{
77+
Annotations: map[string]string{
78+
controlplanev1.SkipKubeProxyAnnotation: "",
79+
},
80+
},
81+
},
82+
},
83+
},
84+
},
6885
}, {
6986
patchTest: capitest.PatchTestDef{
7087
Name: "disable kube proxy with Docker",
7188
Vars: []runtimehooksv1.Variable{
7289
capitest.VariableWithValue(
7390
v1alpha1.ClusterConfigVariableName,
74-
v1alpha1.DockerClusterConfigSpec{
75-
GenericClusterConfigSpec: v1alpha1.GenericClusterConfigSpec{
76-
KubeProxy: &v1alpha1.KubeProxy{
77-
Mode: v1alpha1.KubeProxyModeDisabled,
78-
},
79-
},
80-
},
91+
v1alpha1.DockerClusterConfigSpec{},
8192
),
8293
},
8394
RequestItem: request.NewKubeadmControlPlaneTemplateRequestItem(""),
@@ -87,19 +98,35 @@ var _ = Describe("Generate kube proxy mode patches", func() {
8798
ValueMatcher: gomega.ConsistOf("addon/kube-proxy"),
8899
}},
89100
},
101+
cluster: &clusterv1.Cluster{
102+
ObjectMeta: metav1.ObjectMeta{
103+
Name: "test-cluster",
104+
Namespace: request.Namespace,
105+
Labels: map[string]string{
106+
clusterv1.ProviderNameLabel: "docker",
107+
},
108+
},
109+
Spec: clusterv1.ClusterSpec{
110+
Topology: &clusterv1.Topology{
111+
Version: "dummy-version",
112+
Class: "dummy-class",
113+
ControlPlane: clusterv1.ControlPlaneTopology{
114+
Metadata: clusterv1.ObjectMeta{
115+
Annotations: map[string]string{
116+
controlplanev1.SkipKubeProxyAnnotation: "",
117+
},
118+
},
119+
},
120+
},
121+
},
122+
},
90123
}, {
91124
patchTest: capitest.PatchTestDef{
92125
Name: "disable kube proxy with Nutanix",
93126
Vars: []runtimehooksv1.Variable{
94127
capitest.VariableWithValue(
95128
v1alpha1.ClusterConfigVariableName,
96-
v1alpha1.NutanixClusterConfigSpec{
97-
GenericClusterConfigSpec: v1alpha1.GenericClusterConfigSpec{
98-
KubeProxy: &v1alpha1.KubeProxy{
99-
Mode: v1alpha1.KubeProxyModeDisabled,
100-
},
101-
},
102-
},
129+
v1alpha1.NutanixClusterConfigSpec{},
103130
),
104131
},
105132
RequestItem: request.NewKubeadmControlPlaneTemplateRequestItem(""),
@@ -117,6 +144,19 @@ var _ = Describe("Generate kube proxy mode patches", func() {
117144
clusterv1.ProviderNameLabel: "nutanix",
118145
},
119146
},
147+
Spec: clusterv1.ClusterSpec{
148+
Topology: &clusterv1.Topology{
149+
Version: "dummy-version",
150+
Class: "dummy-class",
151+
ControlPlane: clusterv1.ControlPlaneTopology{
152+
Metadata: clusterv1.ObjectMeta{
153+
Annotations: map[string]string{
154+
controlplanev1.SkipKubeProxyAnnotation: "",
155+
},
156+
},
157+
},
158+
},
159+
},
120160
},
121161
}, {
122162
patchTest: capitest.PatchTestDef{

0 commit comments

Comments
 (0)