Skip to content

Commit 56df70b

Browse files
aojeaaojea
committed
kube-proxy detect IP family based on nodeIP
We were detecting the IP family that kube-proxy should use based on the bind address, however, this is not valid when using an unspecified address, because on those cases kube-proxy adopts the IP family of the address reported in the Node API object. The IP family will be determined by the nodeIP used by the proxier The order of precedence is: 1. config.bindAddress if bindAddress is not 0.0.0.0 or :: 2. the primary IP from the Node object, if set 3. if no IP is found it defaults to 127.0.0.1 and IPv4 Signed-off-by: Antonio Ojea <[email protected]>
1 parent bb11561 commit 56df70b

File tree

4 files changed

+183
-48
lines changed

4 files changed

+183
-48
lines changed

‎cmd/kube-proxy/app/BUILD‎

Lines changed: 11 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -234,7 +234,6 @@ go_test(
234234
"//pkg/proxy/apis/config:go_default_library",
235235
"//staging/src/k8s.io/apimachinery/pkg/apis/meta/v1:go_default_library",
236236
"//staging/src/k8s.io/component-base/config:go_default_library",
237-
"//staging/src/k8s.io/component-base/configz:go_default_library",
238237
"//vendor/github.com/google/go-cmp/cmp:go_default_library",
239238
"//vendor/github.com/stretchr/testify/assert:go_default_library",
240239
"//vendor/k8s.io/utils/pointer:go_default_library",
@@ -245,76 +244,87 @@ go_test(
245244
"//pkg/util/iptables:go_default_library",
246245
"//pkg/util/iptables/testing:go_default_library",
247246
"//staging/src/k8s.io/api/core/v1:go_default_library",
247+
"//staging/src/k8s.io/client-go/kubernetes/fake:go_default_library",
248248
],
249249
"@io_bazel_rules_go//go/platform:darwin": [
250250
"//pkg/proxy/ipvs:go_default_library",
251251
"//pkg/proxy/util/iptables:go_default_library",
252252
"//pkg/util/iptables:go_default_library",
253253
"//pkg/util/iptables/testing:go_default_library",
254254
"//staging/src/k8s.io/api/core/v1:go_default_library",
255+
"//staging/src/k8s.io/client-go/kubernetes/fake:go_default_library",
255256
],
256257
"@io_bazel_rules_go//go/platform:dragonfly": [
257258
"//pkg/proxy/ipvs:go_default_library",
258259
"//pkg/proxy/util/iptables:go_default_library",
259260
"//pkg/util/iptables:go_default_library",
260261
"//pkg/util/iptables/testing:go_default_library",
261262
"//staging/src/k8s.io/api/core/v1:go_default_library",
263+
"//staging/src/k8s.io/client-go/kubernetes/fake:go_default_library",
262264
],
263265
"@io_bazel_rules_go//go/platform:freebsd": [
264266
"//pkg/proxy/ipvs:go_default_library",
265267
"//pkg/proxy/util/iptables:go_default_library",
266268
"//pkg/util/iptables:go_default_library",
267269
"//pkg/util/iptables/testing:go_default_library",
268270
"//staging/src/k8s.io/api/core/v1:go_default_library",
271+
"//staging/src/k8s.io/client-go/kubernetes/fake:go_default_library",
269272
],
270273
"@io_bazel_rules_go//go/platform:ios": [
271274
"//pkg/proxy/ipvs:go_default_library",
272275
"//pkg/proxy/util/iptables:go_default_library",
273276
"//pkg/util/iptables:go_default_library",
274277
"//pkg/util/iptables/testing:go_default_library",
275278
"//staging/src/k8s.io/api/core/v1:go_default_library",
279+
"//staging/src/k8s.io/client-go/kubernetes/fake:go_default_library",
276280
],
277281
"@io_bazel_rules_go//go/platform:linux": [
278282
"//pkg/proxy/ipvs:go_default_library",
279283
"//pkg/proxy/util/iptables:go_default_library",
280284
"//pkg/util/iptables:go_default_library",
281285
"//pkg/util/iptables/testing:go_default_library",
282286
"//staging/src/k8s.io/api/core/v1:go_default_library",
287+
"//staging/src/k8s.io/client-go/kubernetes/fake:go_default_library",
283288
],
284289
"@io_bazel_rules_go//go/platform:nacl": [
285290
"//pkg/proxy/ipvs:go_default_library",
286291
"//pkg/proxy/util/iptables:go_default_library",
287292
"//pkg/util/iptables:go_default_library",
288293
"//pkg/util/iptables/testing:go_default_library",
289294
"//staging/src/k8s.io/api/core/v1:go_default_library",
295+
"//staging/src/k8s.io/client-go/kubernetes/fake:go_default_library",
290296
],
291297
"@io_bazel_rules_go//go/platform:netbsd": [
292298
"//pkg/proxy/ipvs:go_default_library",
293299
"//pkg/proxy/util/iptables:go_default_library",
294300
"//pkg/util/iptables:go_default_library",
295301
"//pkg/util/iptables/testing:go_default_library",
296302
"//staging/src/k8s.io/api/core/v1:go_default_library",
303+
"//staging/src/k8s.io/client-go/kubernetes/fake:go_default_library",
297304
],
298305
"@io_bazel_rules_go//go/platform:openbsd": [
299306
"//pkg/proxy/ipvs:go_default_library",
300307
"//pkg/proxy/util/iptables:go_default_library",
301308
"//pkg/util/iptables:go_default_library",
302309
"//pkg/util/iptables/testing:go_default_library",
303310
"//staging/src/k8s.io/api/core/v1:go_default_library",
311+
"//staging/src/k8s.io/client-go/kubernetes/fake:go_default_library",
304312
],
305313
"@io_bazel_rules_go//go/platform:plan9": [
306314
"//pkg/proxy/ipvs:go_default_library",
307315
"//pkg/proxy/util/iptables:go_default_library",
308316
"//pkg/util/iptables:go_default_library",
309317
"//pkg/util/iptables/testing:go_default_library",
310318
"//staging/src/k8s.io/api/core/v1:go_default_library",
319+
"//staging/src/k8s.io/client-go/kubernetes/fake:go_default_library",
311320
],
312321
"@io_bazel_rules_go//go/platform:solaris": [
313322
"//pkg/proxy/ipvs:go_default_library",
314323
"//pkg/proxy/util/iptables:go_default_library",
315324
"//pkg/util/iptables:go_default_library",
316325
"//pkg/util/iptables/testing:go_default_library",
317326
"//staging/src/k8s.io/api/core/v1:go_default_library",
327+
"//staging/src/k8s.io/client-go/kubernetes/fake:go_default_library",
318328
],
319329
"//conditions:default": [],
320330
}),

‎cmd/kube-proxy/app/server_others.go‎

Lines changed: 33 additions & 20 deletions
Original file line numberDiff line numberDiff line change
@@ -91,10 +91,24 @@ func newProxyServer(
9191
return nil, fmt.Errorf("unable to register configz: %s", err)
9292
}
9393

94+
hostname, err := utilnode.GetHostname(config.HostnameOverride)
95+
if err != nil {
96+
return nil, err
97+
}
98+
99+
client, eventClient, err := createClients(config.ClientConnection, master)
100+
if err != nil {
101+
return nil, err
102+
}
103+
104+
nodeIP := detectNodeIP(client, hostname, config.BindAddress)
105+
94106
protocol := utiliptables.ProtocolIPv4
95-
if net.ParseIP(config.BindAddress).To4() == nil {
96-
klog.V(0).Infof("IPv6 bind address (%s), assume IPv6 operation", config.BindAddress)
107+
if utilsnet.IsIPv6(nodeIP) {
108+
klog.V(0).Infof("kube-proxy node IP is an IPv6 address (%s), assume IPv6 operation", nodeIP.String())
97109
protocol = utiliptables.ProtocolIPv6
110+
} else {
111+
klog.V(0).Infof("kube-proxy node IP is an IPv4 address (%s), assume IPv4 operation", nodeIP.String())
98112
}
99113

100114
var iptInterface utiliptables.Interface
@@ -131,16 +145,7 @@ func newProxyServer(
131145
metrics.SetShowHidden()
132146
}
133147

134-
client, eventClient, err := createClients(config.ClientConnection, master)
135-
if err != nil {
136-
return nil, err
137-
}
138-
139148
// Create event recorder
140-
hostname, err := utilnode.GetHostname(config.HostnameOverride)
141-
if err != nil {
142-
return nil, err
143-
}
144149
eventBroadcaster := record.NewBroadcaster()
145150
recorder := eventBroadcaster.NewRecorder(proxyconfigscheme.Scheme, v1.EventSource{Component: "kube-proxy", Host: hostname})
146151

@@ -175,15 +180,6 @@ func newProxyServer(
175180
klog.Infof("NodeInfo PodCIDR: %v, PodCIDRs: %v", nodeInfo.Spec.PodCIDR, nodeInfo.Spec.PodCIDRs)
176181
}
177182

178-
nodeIP := net.ParseIP(config.BindAddress)
179-
if nodeIP.IsUnspecified() {
180-
nodeIP = utilnode.GetNodeIP(client, hostname)
181-
if nodeIP == nil {
182-
klog.V(0).Infof("can't determine this node's IP, assuming 127.0.0.1; if this is incorrect, please set the --bind-address flag")
183-
nodeIP = net.ParseIP("127.0.0.1")
184-
}
185-
}
186-
187183
klog.V(2).Info("DetectLocalMode: '", string(detectLocalMode), "'")
188184

189185
if proxyMode == proxyModeIPTables {
@@ -422,6 +418,23 @@ func waitForPodCIDR(client clientset.Interface, nodeName string) (*v1.Node, erro
422418
return nil, fmt.Errorf("event object not of type node")
423419
}
424420

421+
// detectNodeIP returns the nodeIP used by the proxier
422+
// The order of precedence is:
423+
// 1. config.bindAddress if bindAddress is not 0.0.0.0 or ::
424+
// 2. the primary IP from the Node object, if set
425+
// 3. if no IP is found it defaults to 127.0.0.1 and IPv4
426+
func detectNodeIP(client clientset.Interface, hostname, bindAddress string) net.IP {
427+
nodeIP := net.ParseIP(bindAddress)
428+
if nodeIP.IsUnspecified() {
429+
nodeIP = utilnode.GetNodeIP(client, hostname)
430+
}
431+
if nodeIP == nil {
432+
klog.V(0).Infof("can't determine this node's IP, assuming 127.0.0.1; if this is incorrect, please set the --bind-address flag")
433+
nodeIP = net.ParseIP("127.0.0.1")
434+
}
435+
return nodeIP
436+
}
437+
425438
func getDetectLocalMode(config *proxyconfigapi.KubeProxyConfiguration) (proxyconfigapi.LocalMode, error) {
426439
mode := config.DetectLocalMode
427440
switch mode {

‎cmd/kube-proxy/app/server_others_test.go‎

Lines changed: 139 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -20,10 +20,15 @@ package app
2020

2121
import (
2222
"fmt"
23+
"net"
2324
"reflect"
2425
"testing"
2526

2627
v1 "k8s.io/api/core/v1"
28+
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
29+
30+
clientsetfake "k8s.io/client-go/kubernetes/fake"
31+
2732
proxyconfigapi "k8s.io/kubernetes/pkg/proxy/apis/config"
2833
"k8s.io/kubernetes/pkg/proxy/ipvs"
2934
proxyutiliptables "k8s.io/kubernetes/pkg/proxy/util/iptables"
@@ -194,6 +199,111 @@ func Test_getDetectLocalMode(t *testing.T) {
194199
}
195200
}
196201

202+
func Test_detectNodeIP(t *testing.T) {
203+
cases := []struct {
204+
name string
205+
nodeInfo *v1.Node
206+
hostname string
207+
bindAddress string
208+
expectedIP net.IP
209+
}{
210+
{
211+
name: "Bind address IPv4 unicast address and no Node object",
212+
nodeInfo: makeNodeWithAddresses("", "", ""),
213+
hostname: "fakeHost",
214+
bindAddress: "10.0.0.1",
215+
expectedIP: net.ParseIP("10.0.0.1"),
216+
},
217+
{
218+
name: "Bind address IPv6 unicast address and no Node object",
219+
nodeInfo: makeNodeWithAddresses("", "", ""),
220+
hostname: "fakeHost",
221+
bindAddress: "fd00:4321::2",
222+
expectedIP: net.ParseIP("fd00:4321::2"),
223+
},
224+
{
225+
name: "No Valid IP found",
226+
nodeInfo: makeNodeWithAddresses("", "", ""),
227+
hostname: "fakeHost",
228+
bindAddress: "",
229+
expectedIP: net.ParseIP("127.0.0.1"),
230+
},
231+
// Disabled because the GetNodeIP method has a backoff retry mechanism
232+
// and the test takes more than 30 seconds
233+
// ok k8s.io/kubernetes/cmd/kube-proxy/app 34.136s
234+
// {
235+
// name: "No Valid IP found and unspecified bind address",
236+
// nodeInfo: makeNodeWithAddresses("", "", ""),
237+
// hostname: "fakeHost",
238+
// bindAddress: "0.0.0.0",
239+
// expectedIP: net.ParseIP("127.0.0.1"),
240+
// },
241+
{
242+
name: "Bind address 0.0.0.0 and node with IPv4 InternalIP set",
243+
nodeInfo: makeNodeWithAddresses("fakeHost", "192.168.1.1", "90.90.90.90"),
244+
hostname: "fakeHost",
245+
bindAddress: "0.0.0.0",
246+
expectedIP: net.ParseIP("192.168.1.1"),
247+
},
248+
{
249+
name: "Bind address :: and node with IPv4 InternalIP set",
250+
nodeInfo: makeNodeWithAddresses("fakeHost", "192.168.1.1", "90.90.90.90"),
251+
hostname: "fakeHost",
252+
bindAddress: "::",
253+
expectedIP: net.ParseIP("192.168.1.1"),
254+
},
255+
{
256+
name: "Bind address 0.0.0.0 and node with IPv6 InternalIP set",
257+
nodeInfo: makeNodeWithAddresses("fakeHost", "fd00:1234::1", "2001:db8::2"),
258+
hostname: "fakeHost",
259+
bindAddress: "0.0.0.0",
260+
expectedIP: net.ParseIP("fd00:1234::1"),
261+
},
262+
{
263+
name: "Bind address :: and node with IPv6 InternalIP set",
264+
nodeInfo: makeNodeWithAddresses("fakeHost", "fd00:1234::1", "2001:db8::2"),
265+
hostname: "fakeHost",
266+
bindAddress: "::",
267+
expectedIP: net.ParseIP("fd00:1234::1"),
268+
},
269+
{
270+
name: "Bind address 0.0.0.0 and node with only IPv4 ExternalIP set",
271+
nodeInfo: makeNodeWithAddresses("fakeHost", "", "90.90.90.90"),
272+
hostname: "fakeHost",
273+
bindAddress: "0.0.0.0",
274+
expectedIP: net.ParseIP("90.90.90.90"),
275+
},
276+
{
277+
name: "Bind address :: and node with only IPv4 ExternalIP set",
278+
nodeInfo: makeNodeWithAddresses("fakeHost", "", "90.90.90.90"),
279+
hostname: "fakeHost",
280+
bindAddress: "::",
281+
expectedIP: net.ParseIP("90.90.90.90"),
282+
},
283+
{
284+
name: "Bind address 0.0.0.0 and node with only IPv6 ExternalIP set",
285+
nodeInfo: makeNodeWithAddresses("fakeHost", "", "2001:db8::2"),
286+
hostname: "fakeHost",
287+
bindAddress: "0.0.0.0",
288+
expectedIP: net.ParseIP("2001:db8::2"),
289+
},
290+
{
291+
name: "Bind address :: and node with only IPv6 ExternalIP set",
292+
nodeInfo: makeNodeWithAddresses("fakeHost", "", "2001:db8::2"),
293+
hostname: "fakeHost",
294+
bindAddress: "::",
295+
expectedIP: net.ParseIP("2001:db8::2"),
296+
},
297+
}
298+
for _, c := range cases {
299+
client := clientsetfake.NewSimpleClientset(c.nodeInfo)
300+
ip := detectNodeIP(client, c.hostname, c.bindAddress)
301+
if !ip.Equal(c.expectedIP) {
302+
t.Errorf("Case[%s] Expected IP %q got %q", c.name, c.expectedIP, ip)
303+
}
304+
}
305+
}
306+
197307
func Test_getLocalDetector(t *testing.T) {
198308
cases := []struct {
199309
mode proxyconfigapi.LocalMode
@@ -474,6 +584,35 @@ func Test_getDualStackLocalDetectorTuple(t *testing.T) {
474584
}
475585
}
476586

587+
func makeNodeWithAddresses(name, internal, external string) *v1.Node {
588+
if name == "" {
589+
return &v1.Node{}
590+
}
591+
592+
node := &v1.Node{
593+
ObjectMeta: metav1.ObjectMeta{
594+
Name: name,
595+
},
596+
Status: v1.NodeStatus{
597+
Addresses: []v1.NodeAddress{},
598+
},
599+
}
600+
601+
if internal != "" {
602+
node.Status.Addresses = append(node.Status.Addresses,
603+
v1.NodeAddress{Type: v1.NodeInternalIP, Address: internal},
604+
)
605+
}
606+
607+
if external != "" {
608+
node.Status.Addresses = append(node.Status.Addresses,
609+
v1.NodeAddress{Type: v1.NodeExternalIP, Address: external},
610+
)
611+
}
612+
613+
return node
614+
}
615+
477616
func makeNodeWithPodCIDRs(cidrs ...string) *v1.Node {
478617
if len(cidrs) == 0 {
479618
return &v1.Node{}

‎cmd/kube-proxy/app/server_test.go‎

Lines changed: 0 additions & 27 deletions
Original file line numberDiff line numberDiff line change
@@ -36,36 +36,9 @@ import (
3636

3737
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
3838
componentbaseconfig "k8s.io/component-base/config"
39-
"k8s.io/component-base/configz"
4039
kubeproxyconfig "k8s.io/kubernetes/pkg/proxy/apis/config"
4140
)
4241

43-
// This test verifies that NewProxyServer does not crash when CleanupAndExit is true.
44-
func TestProxyServerWithCleanupAndExit(t *testing.T) {
45-
// Each bind address below is a separate test case
46-
bindAddresses := []string{
47-
"0.0.0.0",
48-
"::",
49-
}
50-
for _, addr := range bindAddresses {
51-
options := NewOptions()
52-
53-
options.config = &kubeproxyconfig.KubeProxyConfiguration{
54-
BindAddress: addr,
55-
}
56-
options.CleanupAndExit = true
57-
58-
proxyserver, err := NewProxyServer(options)
59-
60-
assert.Nil(t, err, "unexpected error in NewProxyServer, addr: %s", addr)
61-
assert.NotNil(t, proxyserver, "nil proxy server obj, addr: %s", addr)
62-
assert.NotNil(t, proxyserver.IptInterface, "nil iptables intf, addr: %s", addr)
63-
64-
// Clean up config for next test case
65-
configz.Delete(kubeproxyconfig.GroupName)
66-
}
67-
}
68-
6942
func TestGetConntrackMax(t *testing.T) {
7043
ncores := runtime.NumCPU()
7144
testCases := []struct {

0 commit comments

Comments
 (0)