Skip to content

Commit 48f36ac

Browse files
k8s-ci-robotk8s-ci-robot
authored
Merge pull request kubernetes#125337 from aojea/document_node_addresses
kubelet --node-ip flag using unspecified IPs and external cloud provider node addresses behavior
2 parents bd00406 + 7917e8b commit 48f36ac

File tree

5 files changed

+309
-12
lines changed

5 files changed

+309
-12
lines changed

‎cmd/kubelet/app/options/options.go‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -292,7 +292,7 @@ func (f *KubeletFlags) AddFlags(mainfs *pflag.FlagSet) {
292292

293293
fs.StringVar(&f.HostnameOverride, "hostname-override", f.HostnameOverride, "If non-empty, will use this string as identification instead of the actual hostname. If --cloud-provider is set, the cloud provider determines the name of the node (consult cloud provider documentation to determine if and how the hostname is used).")
294294

295-
fs.StringVar(&f.NodeIP, "node-ip", f.NodeIP, "IP address (or comma-separated dual-stack IP addresses) of the node. If unset, kubelet will use the node's default IPv4 address, if any, or its default IPv6 address if it has no IPv4 addresses. You can pass '::' to make it prefer the default IPv6 address rather than the default IPv4 address.")
295+
fs.StringVar(&f.NodeIP, "node-ip", f.NodeIP, "IP address (or comma-separated dual-stack IP addresses) of the node. If unset, kubelet will use the node's default IPv4 address, if any, or its default IPv6 address if it has no IPv4 addresses. You can pass '::' to make it prefer the default IPv6 address rather than the default IPv4 address. If cloud-provider is set to external, this flag will help to bootstrap the node with the corresponding IP.")
296296

297297
fs.StringVar(&f.CertDirectory, "cert-dir", f.CertDirectory, "The directory where the TLS certs are located. "+
298298
"If --tls-cert-file and --tls-private-key-file are provided, this flag will be ignored.")

‎pkg/kubelet/kubelet_node_status.go‎

Lines changed: 2 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -31,6 +31,7 @@ import (
3131
"k8s.io/apimachinery/pkg/api/resource"
3232
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
3333
"k8s.io/apimachinery/pkg/types"
34+
utilnet "k8s.io/apimachinery/pkg/util/net"
3435
"k8s.io/apimachinery/pkg/util/sets"
3536
utilfeature "k8s.io/apiserver/pkg/util/feature"
3637
cloudprovider "k8s.io/cloud-provider"
@@ -741,7 +742,7 @@ func (kl *Kubelet) defaultNodeStatusFuncs() []func(context.Context, *v1.Node) er
741742
}
742743
var setters []func(ctx context.Context, n *v1.Node) error
743744
setters = append(setters,
744-
nodestatus.NodeAddress(kl.nodeIPs, kl.nodeIPValidator, kl.hostname, kl.hostnameOverridden, kl.externalCloudProvider, kl.cloud, nodeAddressesFunc),
745+
nodestatus.NodeAddress(kl.nodeIPs, kl.nodeIPValidator, kl.hostname, kl.hostnameOverridden, kl.externalCloudProvider, kl.cloud, nodeAddressesFunc, utilnet.ResolveBindAddress),
745746
nodestatus.MachineInfo(string(kl.nodeName), kl.maxPods, kl.podsPerCore, kl.GetCachedMachineInfo, kl.containerManager.GetCapacity,
746747
kl.containerManager.GetDevicePluginResourceCapacity, kl.containerManager.GetNodeAllocatableReservation, kl.recordEvent, kl.supportLocalStorageCapacityIsolation()),
747748
nodestatus.VersionInfo(kl.cadvisor.VersionInfo, kl.containerRuntime.Type, kl.containerRuntime.Version),

‎pkg/kubelet/nodestatus/setters.go‎

Lines changed: 7 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -31,7 +31,6 @@ import (
3131
"k8s.io/apimachinery/pkg/api/resource"
3232
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
3333
"k8s.io/apimachinery/pkg/util/errors"
34-
utilnet "k8s.io/apimachinery/pkg/util/net"
3534
utilfeature "k8s.io/apiserver/pkg/util/feature"
3635
cloudprovider "k8s.io/cloud-provider"
3736
cloudproviderapi "k8s.io/cloud-provider/api"
@@ -66,6 +65,7 @@ func NodeAddress(nodeIPs []net.IP, // typically Kubelet.nodeIPs
6665
externalCloudProvider bool, // typically Kubelet.externalCloudProvider
6766
cloud cloudprovider.Interface, // typically Kubelet.cloud
6867
nodeAddressesFunc func() ([]v1.NodeAddress, error), // typically Kubelet.cloudResourceSyncManager.NodeAddresses
68+
resolveAddressFunc func(net.IP) (net.IP, error), // typically k8s.io/apimachinery/pkg/util/net.ResolveBindAddress
6969
) Setter {
7070
var nodeIP, secondaryNodeIP net.IP
7171
if len(nodeIPs) > 0 {
@@ -129,12 +129,15 @@ func NodeAddress(nodeIPs []net.IP, // typically Kubelet.nodeIPs
129129
if len(node.Status.Addresses) > 0 {
130130
return nil
131131
}
132-
// If nodeIPs are not specified wait for the external cloud-provider to set the node addresses.
132+
// If nodeIPs are not set wait for the external cloud-provider to set the node addresses.
133+
// If the nodeIP is the unspecified address 0.0.0.0 or ::, then use the IP of the default gateway of
134+
// the corresponding IP family to bootstrap the node until the out-of-tree provider overrides it later.
135+
// xref: https://github.com/kubernetes/kubernetes/issues/125348
133136
// Otherwise uses them on the assumption that the installer/administrator has the previous knowledge
134137
// required to ensure the external cloud provider will use the same addresses to avoid the issues explained
135138
// in https://github.com/kubernetes/kubernetes/issues/120720.
136139
// We are already hinting the external cloud provider via the annotation AnnotationAlphaProvidedIPAddr.
137-
if !nodeIPSpecified {
140+
if nodeIP == nil {
138141
node.Status.Addresses = []v1.NodeAddress{
139142
{Type: v1.NodeHostName, Address: hostname},
140143
}
@@ -222,7 +225,7 @@ func NodeAddress(nodeIPs []net.IP, // typically Kubelet.nodeIPs
222225
}
223226

224227
if ipAddr == nil {
225-
ipAddr, err = utilnet.ResolveBindAddress(nodeIP)
228+
ipAddr, err = resolveAddressFunc(nodeIP)
226229
}
227230
}
228231

‎pkg/kubelet/nodestatus/setters_test.go‎

Lines changed: 129 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -63,12 +63,14 @@ func TestNodeAddress(t *testing.T) {
6363
cloudProviderExternal
6464
cloudProviderNone
6565
)
66+
6667
existingNodeAddress := v1.NodeAddress{Address: "10.1.1.2"}
6768
cases := []struct {
6869
name string
6970
hostnameOverride bool
7071
nodeIP net.IP
7172
secondaryNodeIP net.IP
73+
resolvedIP net.IP
7274
cloudProviderType cloudProviderType
7375
nodeAddresses []v1.NodeAddress
7476
expectedAddresses []v1.NodeAddress
@@ -234,17 +236,120 @@ func TestNodeAddress(t *testing.T) {
234236
shouldError: false,
235237
},
236238
{
237-
name: "cloud provider is external and nodeIP unspecified",
239+
name: "no cloud provider and nodeIP IPv4 unspecified",
240+
nodeIP: netutils.ParseIPSloppy("0.0.0.0"),
241+
resolvedIP: netutils.ParseIPSloppy("10.0.0.2"),
242+
nodeAddresses: []v1.NodeAddress{},
243+
cloudProviderType: cloudProviderNone,
244+
expectedAddresses: []v1.NodeAddress{
245+
{Type: v1.NodeInternalIP, Address: "10.0.0.2"},
246+
{Type: v1.NodeHostName, Address: testKubeletHostname},
247+
},
248+
shouldError: false,
249+
},
250+
{
251+
name: "no cloud provider and nodeIP IPv6 unspecified",
252+
nodeIP: netutils.ParseIPSloppy("::"),
253+
resolvedIP: netutils.ParseIPSloppy("2001:db2::2"),
254+
nodeAddresses: []v1.NodeAddress{},
255+
cloudProviderType: cloudProviderNone,
256+
expectedAddresses: []v1.NodeAddress{
257+
{Type: v1.NodeInternalIP, Address: "2001:db2::2"},
258+
{Type: v1.NodeHostName, Address: testKubeletHostname},
259+
},
260+
shouldError: false,
261+
},
262+
{
263+
name: "legacy cloud provider and nodeIP IPv4 unspecified",
264+
nodeIP: netutils.ParseIPSloppy("0.0.0.0"),
265+
resolvedIP: netutils.ParseIPSloppy("10.0.0.2"),
266+
nodeAddresses: []v1.NodeAddress{},
267+
cloudProviderType: cloudProviderLegacy,
268+
expectedAddresses: []v1.NodeAddress{
269+
{Type: v1.NodeHostName, Address: testKubeletHostname},
270+
},
271+
shouldError: true,
272+
},
273+
{
274+
name: "legacy cloud provider and nodeIP IPv6 unspecified",
275+
nodeIP: netutils.ParseIPSloppy("::"),
276+
resolvedIP: netutils.ParseIPSloppy("2001:db2::2"),
277+
nodeAddresses: []v1.NodeAddress{},
278+
cloudProviderType: cloudProviderLegacy,
279+
expectedAddresses: []v1.NodeAddress{
280+
{Type: v1.NodeHostName, Address: testKubeletHostname},
281+
},
282+
shouldError: true,
283+
},
284+
{
285+
name: "cloud provider is external and nodeIP IPv4 unspecified",
286+
nodeIP: netutils.ParseIPSloppy("0.0.0.0"),
287+
resolvedIP: netutils.ParseIPSloppy("10.0.0.2"),
288+
nodeAddresses: []v1.NodeAddress{},
289+
cloudProviderType: cloudProviderExternal,
290+
expectedAddresses: []v1.NodeAddress{
291+
{Type: v1.NodeInternalIP, Address: "10.0.0.2"},
292+
{Type: v1.NodeHostName, Address: testKubeletHostname},
293+
},
294+
shouldError: false,
295+
},
296+
{
297+
name: "cloud provider is external and nodeIP IPv6 unspecified",
238298
nodeIP: netutils.ParseIPSloppy("::"),
299+
resolvedIP: netutils.ParseIPSloppy("2001:db2::2"),
239300
nodeAddresses: []v1.NodeAddress{},
240301
cloudProviderType: cloudProviderExternal,
241302
expectedAddresses: []v1.NodeAddress{
303+
{Type: v1.NodeInternalIP, Address: "2001:db2::2"},
242304
{Type: v1.NodeHostName, Address: testKubeletHostname},
243305
},
244306
shouldError: false,
245307
},
246308
{
247-
name: "cloud provider is external and no nodeIP",
309+
name: "no cloud provider and no nodeIP resolve IPv4",
310+
resolvedIP: netutils.ParseIPSloppy("10.0.0.2"),
311+
nodeAddresses: []v1.NodeAddress{},
312+
cloudProviderType: cloudProviderNone,
313+
expectedAddresses: []v1.NodeAddress{
314+
{Type: v1.NodeInternalIP, Address: "10.0.0.2"},
315+
{Type: v1.NodeHostName, Address: testKubeletHostname},
316+
},
317+
shouldError: false,
318+
},
319+
{
320+
name: "no cloud provider and no nodeIP resolve IPv6",
321+
resolvedIP: netutils.ParseIPSloppy("2001:db2::2"),
322+
nodeAddresses: []v1.NodeAddress{},
323+
cloudProviderType: cloudProviderNone,
324+
expectedAddresses: []v1.NodeAddress{
325+
{Type: v1.NodeInternalIP, Address: "2001:db2::2"},
326+
{Type: v1.NodeHostName, Address: testKubeletHostname},
327+
},
328+
shouldError: false,
329+
},
330+
{
331+
name: "legacy cloud provider and no nodeIP",
332+
resolvedIP: netutils.ParseIPSloppy("10.0.0.2"),
333+
nodeAddresses: []v1.NodeAddress{},
334+
cloudProviderType: cloudProviderLegacy,
335+
expectedAddresses: []v1.NodeAddress{
336+
{Type: v1.NodeHostName, Address: testKubeletHostname},
337+
},
338+
shouldError: true,
339+
},
340+
{
341+
name: "cloud provider is external and no nodeIP resolve IPv4",
342+
resolvedIP: netutils.ParseIPSloppy("10.0.0.2"),
343+
nodeAddresses: []v1.NodeAddress{},
344+
cloudProviderType: cloudProviderExternal,
345+
expectedAddresses: []v1.NodeAddress{
346+
{Type: v1.NodeHostName, Address: testKubeletHostname},
347+
},
348+
shouldError: false,
349+
},
350+
{
351+
name: "cloud provider is external and no nodeIP resolve IPv6",
352+
resolvedIP: netutils.ParseIPSloppy("2001:db2::2"),
248353
nodeAddresses: []v1.NodeAddress{},
249354
cloudProviderType: cloudProviderExternal,
250355
expectedAddresses: []v1.NodeAddress{
@@ -638,6 +743,20 @@ func TestNodeAddress(t *testing.T) {
638743
return testCase.nodeAddresses, nil
639744
}
640745

746+
net.DefaultResolver = &net.Resolver{
747+
PreferGo: true,
748+
Dial: func(ctx context.Context, network string, address string) (net.Conn, error) {
749+
return nil, fmt.Errorf("error")
750+
},
751+
}
752+
defer func() {
753+
net.DefaultResolver = &net.Resolver{}
754+
}()
755+
756+
resolveAddressFunc := func(net.IP) (net.IP, error) {
757+
return testCase.resolvedIP, nil
758+
}
759+
641760
// cloud provider is expected to be nil if external provider is set or there is no cloud provider
642761
var cloud cloudprovider.Interface
643762
if testCase.cloudProviderType == cloudProviderLegacy {
@@ -659,7 +778,9 @@ func TestNodeAddress(t *testing.T) {
659778
testCase.hostnameOverride,
660779
testCase.cloudProviderType == cloudProviderExternal,
661780
cloud,
662-
nodeAddressesFunc)
781+
nodeAddressesFunc,
782+
resolveAddressFunc,
783+
)
663784

664785
// call setter on existing node
665786
err := setter(ctx, existingNode)
@@ -739,6 +860,9 @@ func TestNodeAddress_NoCloudProvider(t *testing.T) {
739860
nodeAddressesFunc := func() ([]v1.NodeAddress, error) {
740861
return nil, fmt.Errorf("not reached")
741862
}
863+
resolvedAddressesFunc := func(net.IP) (net.IP, error) {
864+
return nil, fmt.Errorf("not reached")
865+
}
742866

743867
// construct setter
744868
setter := NodeAddress(testCase.nodeIPs,
@@ -747,7 +871,8 @@ func TestNodeAddress_NoCloudProvider(t *testing.T) {
747871
false, // hostnameOverridden
748872
false, // externalCloudProvider
749873
nil, // cloud
750-
nodeAddressesFunc)
874+
nodeAddressesFunc,
875+
resolvedAddressesFunc)
751876

752877
// call setter on existing node
753878
err := setter(ctx, existingNode)

0 commit comments

Comments
 (0)