Skip to content

Commit 900143c

Browse files
k8s-ci-robotk8s-ci-robot
authored
Merge pull request kubernetes#88935 from satyasm/kep-local-cidr
Add NodeCIDR for detect-local-mode
2 parents 0bb125e + e053fdd commit 900143c

File tree

4 files changed

+281
-12
lines changed

4 files changed

+281
-12
lines changed

‎cmd/kube-proxy/app/BUILD‎

Lines changed: 44 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -83,6 +83,9 @@ go_library(
8383
"//pkg/util/node:go_default_library",
8484
"//staging/src/k8s.io/apimachinery/pkg/types:go_default_library",
8585
"//staging/src/k8s.io/apimachinery/pkg/util/net:go_default_library",
86+
"//staging/src/k8s.io/apimachinery/pkg/watch:go_default_library",
87+
"//staging/src/k8s.io/client-go/tools/cache:go_default_library",
88+
"//staging/src/k8s.io/client-go/tools/watch:go_default_library",
8689
"//staging/src/k8s.io/component-base/metrics:go_default_library",
8790
"//vendor/k8s.io/utils/net:go_default_library",
8891
],
@@ -92,6 +95,9 @@ go_library(
9295
"//pkg/util/node:go_default_library",
9396
"//staging/src/k8s.io/apimachinery/pkg/types:go_default_library",
9497
"//staging/src/k8s.io/apimachinery/pkg/util/net:go_default_library",
98+
"//staging/src/k8s.io/apimachinery/pkg/watch:go_default_library",
99+
"//staging/src/k8s.io/client-go/tools/cache:go_default_library",
100+
"//staging/src/k8s.io/client-go/tools/watch:go_default_library",
95101
"//staging/src/k8s.io/component-base/metrics:go_default_library",
96102
"//vendor/k8s.io/utils/net:go_default_library",
97103
],
@@ -101,6 +107,9 @@ go_library(
101107
"//pkg/util/node:go_default_library",
102108
"//staging/src/k8s.io/apimachinery/pkg/types:go_default_library",
103109
"//staging/src/k8s.io/apimachinery/pkg/util/net:go_default_library",
110+
"//staging/src/k8s.io/apimachinery/pkg/watch:go_default_library",
111+
"//staging/src/k8s.io/client-go/tools/cache:go_default_library",
112+
"//staging/src/k8s.io/client-go/tools/watch:go_default_library",
104113
"//staging/src/k8s.io/component-base/metrics:go_default_library",
105114
"//vendor/k8s.io/utils/net:go_default_library",
106115
],
@@ -110,6 +119,9 @@ go_library(
110119
"//pkg/util/node:go_default_library",
111120
"//staging/src/k8s.io/apimachinery/pkg/types:go_default_library",
112121
"//staging/src/k8s.io/apimachinery/pkg/util/net:go_default_library",
122+
"//staging/src/k8s.io/apimachinery/pkg/watch:go_default_library",
123+
"//staging/src/k8s.io/client-go/tools/cache:go_default_library",
124+
"//staging/src/k8s.io/client-go/tools/watch:go_default_library",
113125
"//staging/src/k8s.io/component-base/metrics:go_default_library",
114126
"//vendor/k8s.io/utils/net:go_default_library",
115127
],
@@ -119,6 +131,9 @@ go_library(
119131
"//pkg/util/node:go_default_library",
120132
"//staging/src/k8s.io/apimachinery/pkg/types:go_default_library",
121133
"//staging/src/k8s.io/apimachinery/pkg/util/net:go_default_library",
134+
"//staging/src/k8s.io/apimachinery/pkg/watch:go_default_library",
135+
"//staging/src/k8s.io/client-go/tools/cache:go_default_library",
136+
"//staging/src/k8s.io/client-go/tools/watch:go_default_library",
122137
"//staging/src/k8s.io/component-base/metrics:go_default_library",
123138
"//vendor/k8s.io/utils/net:go_default_library",
124139
],
@@ -128,6 +143,9 @@ go_library(
128143
"//pkg/util/node:go_default_library",
129144
"//staging/src/k8s.io/apimachinery/pkg/types:go_default_library",
130145
"//staging/src/k8s.io/apimachinery/pkg/util/net:go_default_library",
146+
"//staging/src/k8s.io/apimachinery/pkg/watch:go_default_library",
147+
"//staging/src/k8s.io/client-go/tools/cache:go_default_library",
148+
"//staging/src/k8s.io/client-go/tools/watch:go_default_library",
131149
"//staging/src/k8s.io/component-base/metrics:go_default_library",
132150
"//vendor/k8s.io/utils/net:go_default_library",
133151
],
@@ -137,6 +155,9 @@ go_library(
137155
"//pkg/util/node:go_default_library",
138156
"//staging/src/k8s.io/apimachinery/pkg/types:go_default_library",
139157
"//staging/src/k8s.io/apimachinery/pkg/util/net:go_default_library",
158+
"//staging/src/k8s.io/apimachinery/pkg/watch:go_default_library",
159+
"//staging/src/k8s.io/client-go/tools/cache:go_default_library",
160+
"//staging/src/k8s.io/client-go/tools/watch:go_default_library",
140161
"//staging/src/k8s.io/component-base/metrics:go_default_library",
141162
"//vendor/k8s.io/utils/net:go_default_library",
142163
],
@@ -146,6 +167,9 @@ go_library(
146167
"//pkg/util/node:go_default_library",
147168
"//staging/src/k8s.io/apimachinery/pkg/types:go_default_library",
148169
"//staging/src/k8s.io/apimachinery/pkg/util/net:go_default_library",
170+
"//staging/src/k8s.io/apimachinery/pkg/watch:go_default_library",
171+
"//staging/src/k8s.io/client-go/tools/cache:go_default_library",
172+
"//staging/src/k8s.io/client-go/tools/watch:go_default_library",
149173
"//staging/src/k8s.io/component-base/metrics:go_default_library",
150174
"//vendor/k8s.io/utils/net:go_default_library",
151175
],
@@ -155,6 +179,9 @@ go_library(
155179
"//pkg/util/node:go_default_library",
156180
"//staging/src/k8s.io/apimachinery/pkg/types:go_default_library",
157181
"//staging/src/k8s.io/apimachinery/pkg/util/net:go_default_library",
182+
"//staging/src/k8s.io/apimachinery/pkg/watch:go_default_library",
183+
"//staging/src/k8s.io/client-go/tools/cache:go_default_library",
184+
"//staging/src/k8s.io/client-go/tools/watch:go_default_library",
158185
"//staging/src/k8s.io/component-base/metrics:go_default_library",
159186
"//vendor/k8s.io/utils/net:go_default_library",
160187
],
@@ -164,6 +191,9 @@ go_library(
164191
"//pkg/util/node:go_default_library",
165192
"//staging/src/k8s.io/apimachinery/pkg/types:go_default_library",
166193
"//staging/src/k8s.io/apimachinery/pkg/util/net:go_default_library",
194+
"//staging/src/k8s.io/apimachinery/pkg/watch:go_default_library",
195+
"//staging/src/k8s.io/client-go/tools/cache:go_default_library",
196+
"//staging/src/k8s.io/client-go/tools/watch:go_default_library",
167197
"//staging/src/k8s.io/component-base/metrics:go_default_library",
168198
"//vendor/k8s.io/utils/net:go_default_library",
169199
],
@@ -173,6 +203,9 @@ go_library(
173203
"//pkg/util/node:go_default_library",
174204
"//staging/src/k8s.io/apimachinery/pkg/types:go_default_library",
175205
"//staging/src/k8s.io/apimachinery/pkg/util/net:go_default_library",
206+
"//staging/src/k8s.io/apimachinery/pkg/watch:go_default_library",
207+
"//staging/src/k8s.io/client-go/tools/cache:go_default_library",
208+
"//staging/src/k8s.io/client-go/tools/watch:go_default_library",
176209
"//staging/src/k8s.io/component-base/metrics:go_default_library",
177210
"//vendor/k8s.io/utils/net:go_default_library",
178211
],
@@ -211,66 +244,77 @@ go_test(
211244
"//pkg/proxy/util/iptables:go_default_library",
212245
"//pkg/util/iptables:go_default_library",
213246
"//pkg/util/iptables/testing:go_default_library",
247+
"//staging/src/k8s.io/api/core/v1:go_default_library",
214248
],
215249
"@io_bazel_rules_go//go/platform:darwin": [
216250
"//pkg/proxy/ipvs:go_default_library",
217251
"//pkg/proxy/util/iptables:go_default_library",
218252
"//pkg/util/iptables:go_default_library",
219253
"//pkg/util/iptables/testing:go_default_library",
254+
"//staging/src/k8s.io/api/core/v1:go_default_library",
220255
],
221256
"@io_bazel_rules_go//go/platform:dragonfly": [
222257
"//pkg/proxy/ipvs:go_default_library",
223258
"//pkg/proxy/util/iptables:go_default_library",
224259
"//pkg/util/iptables:go_default_library",
225260
"//pkg/util/iptables/testing:go_default_library",
261+
"//staging/src/k8s.io/api/core/v1:go_default_library",
226262
],
227263
"@io_bazel_rules_go//go/platform:freebsd": [
228264
"//pkg/proxy/ipvs:go_default_library",
229265
"//pkg/proxy/util/iptables:go_default_library",
230266
"//pkg/util/iptables:go_default_library",
231267
"//pkg/util/iptables/testing:go_default_library",
268+
"//staging/src/k8s.io/api/core/v1:go_default_library",
232269
],
233270
"@io_bazel_rules_go//go/platform:ios": [
234271
"//pkg/proxy/ipvs:go_default_library",
235272
"//pkg/proxy/util/iptables:go_default_library",
236273
"//pkg/util/iptables:go_default_library",
237274
"//pkg/util/iptables/testing:go_default_library",
275+
"//staging/src/k8s.io/api/core/v1:go_default_library",
238276
],
239277
"@io_bazel_rules_go//go/platform:linux": [
240278
"//pkg/proxy/ipvs:go_default_library",
241279
"//pkg/proxy/util/iptables:go_default_library",
242280
"//pkg/util/iptables:go_default_library",
243281
"//pkg/util/iptables/testing:go_default_library",
282+
"//staging/src/k8s.io/api/core/v1:go_default_library",
244283
],
245284
"@io_bazel_rules_go//go/platform:nacl": [
246285
"//pkg/proxy/ipvs:go_default_library",
247286
"//pkg/proxy/util/iptables:go_default_library",
248287
"//pkg/util/iptables:go_default_library",
249288
"//pkg/util/iptables/testing:go_default_library",
289+
"//staging/src/k8s.io/api/core/v1:go_default_library",
250290
],
251291
"@io_bazel_rules_go//go/platform:netbsd": [
252292
"//pkg/proxy/ipvs:go_default_library",
253293
"//pkg/proxy/util/iptables:go_default_library",
254294
"//pkg/util/iptables:go_default_library",
255295
"//pkg/util/iptables/testing:go_default_library",
296+
"//staging/src/k8s.io/api/core/v1:go_default_library",
256297
],
257298
"@io_bazel_rules_go//go/platform:openbsd": [
258299
"//pkg/proxy/ipvs:go_default_library",
259300
"//pkg/proxy/util/iptables:go_default_library",
260301
"//pkg/util/iptables:go_default_library",
261302
"//pkg/util/iptables/testing:go_default_library",
303+
"//staging/src/k8s.io/api/core/v1:go_default_library",
262304
],
263305
"@io_bazel_rules_go//go/platform:plan9": [
264306
"//pkg/proxy/ipvs:go_default_library",
265307
"//pkg/proxy/util/iptables:go_default_library",
266308
"//pkg/util/iptables:go_default_library",
267309
"//pkg/util/iptables/testing:go_default_library",
310+
"//staging/src/k8s.io/api/core/v1:go_default_library",
268311
],
269312
"@io_bazel_rules_go//go/platform:solaris": [
270313
"//pkg/proxy/ipvs:go_default_library",
271314
"//pkg/proxy/util/iptables:go_default_library",
272315
"//pkg/util/iptables:go_default_library",
273316
"//pkg/util/iptables/testing:go_default_library",
317+
"//staging/src/k8s.io/api/core/v1:go_default_library",
274318
],
275319
"//conditions:default": [],
276320
}),

‎cmd/kube-proxy/app/server_others.go‎

Lines changed: 98 additions & 7 deletions
Original file line numberDiff line numberDiff line change
@@ -21,17 +21,29 @@ limitations under the License.
2121
package app
2222

2323
import (
24+
"context"
2425
"errors"
2526
"fmt"
2627
"net"
2728
"strings"
29+
"time"
30+
31+
"k8s.io/apimachinery/pkg/watch"
32+
33+
"k8s.io/apimachinery/pkg/runtime"
34+
"k8s.io/client-go/tools/cache"
35+
36+
"k8s.io/apimachinery/pkg/fields"
2837

2938
v1 "k8s.io/api/core/v1"
39+
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
3040
"k8s.io/apimachinery/pkg/types"
3141
utilnet "k8s.io/apimachinery/pkg/util/net"
3242
utilruntime "k8s.io/apimachinery/pkg/util/runtime"
3343
utilfeature "k8s.io/apiserver/pkg/util/feature"
44+
clientset "k8s.io/client-go/kubernetes"
3445
"k8s.io/client-go/tools/record"
46+
toolswatch "k8s.io/client-go/tools/watch"
3547
"k8s.io/component-base/metrics"
3648
"k8s.io/kubernetes/pkg/features"
3749
"k8s.io/kubernetes/pkg/proxy"
@@ -55,6 +67,10 @@ import (
5567
"k8s.io/klog"
5668
)
5769

70+
// timeoutForNodePodCIDR is the time to wait for allocators to assign a PodCIDR to the
71+
// node after it is registered.
72+
var timeoutForNodePodCIDR = 5 * time.Minute
73+
5874
// NewProxyServer returns a new ProxyServer.
5975
func NewProxyServer(o *Options) (*ProxyServer, error) {
6076
return newProxyServer(o.config, o.CleanupAndExit, o.master)
@@ -145,6 +161,16 @@ func newProxyServer(
145161
return nil, fmt.Errorf("cannot determine detect-local-mode: %v", err)
146162
}
147163

164+
var nodeInfo *v1.Node
165+
if detectLocalMode == proxyconfigapi.LocalModeNodeCIDR {
166+
klog.Infof("Watching for node %s, awaiting podCIDR allocation", hostname)
167+
nodeInfo, err = waitForPodCIDR(client, hostname)
168+
if err != nil {
169+
return nil, err
170+
}
171+
klog.Infof("NodeInfo PodCIDR: %v, PodCIDRs: %v", nodeInfo.Spec.PodCIDR, nodeInfo.Spec.PodCIDRs)
172+
}
173+
148174
nodeIP := net.ParseIP(config.BindAddress)
149175
if nodeIP.IsUnspecified() {
150176
nodeIP = utilnode.GetNodeIP(client, hostname)
@@ -179,7 +205,7 @@ func newProxyServer(
179205

180206
// Always ordered to match []ipt
181207
var localDetectors [2]proxyutiliptables.LocalTrafficDetector
182-
localDetectors, err = getDualStackLocalDetectorTuple(detectLocalMode, config, ipt)
208+
localDetectors, err = getDualStackLocalDetectorTuple(detectLocalMode, config, ipt, nodeInfo)
183209
if err != nil {
184210
return nil, fmt.Errorf("unable to create proxier: %v", err)
185211
}
@@ -202,7 +228,7 @@ func newProxyServer(
202228
)
203229
} else { // Create a single-stack proxier.
204230
var localDetector proxyutiliptables.LocalTrafficDetector
205-
localDetector, err = getLocalDetector(detectLocalMode, config, iptInterface)
231+
localDetector, err = getLocalDetector(detectLocalMode, config, iptInterface, nodeInfo)
206232
if err != nil {
207233
return nil, fmt.Errorf("unable to create proxier: %v", err)
208234
}
@@ -249,7 +275,7 @@ func newProxyServer(
249275

250276
// Always ordered to match []ipt
251277
var localDetectors [2]proxyutiliptables.LocalTrafficDetector
252-
localDetectors, err = getDualStackLocalDetectorTuple(detectLocalMode, config, ipt)
278+
localDetectors, err = getDualStackLocalDetectorTuple(detectLocalMode, config, ipt, nodeInfo)
253279
if err != nil {
254280
return nil, fmt.Errorf("unable to create proxier: %v", err)
255281
}
@@ -279,7 +305,7 @@ func newProxyServer(
279305
)
280306
} else {
281307
var localDetector proxyutiliptables.LocalTrafficDetector
282-
localDetector, err = getLocalDetector(detectLocalMode, config, iptInterface)
308+
localDetector, err = getLocalDetector(detectLocalMode, config, iptInterface, nodeInfo)
283309
if err != nil {
284310
return nil, fmt.Errorf("unable to create proxier: %v", err)
285311
}
@@ -355,10 +381,44 @@ func newProxyServer(
355381
}, nil
356382
}
357383

384+
func waitForPodCIDR(client clientset.Interface, nodeName string) (*v1.Node, error) {
385+
// since allocators can assign the podCIDR after the node registers, we do a watch here to wait
386+
// for podCIDR to be assigned, instead of assuming that the Get() on startup will have it.
387+
ctx, cancelFunc := context.WithTimeout(context.TODO(), timeoutForNodePodCIDR)
388+
defer cancelFunc()
389+
390+
fieldSelector := fields.OneTermEqualSelector("metadata.name", nodeName).String()
391+
lw := &cache.ListWatch{
392+
ListFunc: func(options metav1.ListOptions) (object runtime.Object, e error) {
393+
options.FieldSelector = fieldSelector
394+
return client.CoreV1().Nodes().List(ctx, options)
395+
},
396+
WatchFunc: func(options metav1.ListOptions) (i watch.Interface, e error) {
397+
options.FieldSelector = fieldSelector
398+
return client.CoreV1().Nodes().Watch(ctx, options)
399+
},
400+
}
401+
condition := func(event watch.Event) (bool, error) {
402+
if n, ok := event.Object.(*v1.Node); ok {
403+
return n.Spec.PodCIDR != "" && len(n.Spec.PodCIDRs) > 0, nil
404+
}
405+
return false, fmt.Errorf("event object not of type Node")
406+
}
407+
408+
evt, err := toolswatch.UntilWithSync(ctx, lw, &v1.Node{}, nil, condition)
409+
if err != nil {
410+
return nil, fmt.Errorf("timeout waiting for PodCIDR allocation to configure detect-local-mode %v: %v", proxyconfigapi.LocalModeNodeCIDR, err)
411+
}
412+
if n, ok := evt.Object.(*v1.Node); ok {
413+
return n, nil
414+
}
415+
return nil, fmt.Errorf("event object not of type node")
416+
}
417+
358418
func getDetectLocalMode(config *proxyconfigapi.KubeProxyConfiguration) (proxyconfigapi.LocalMode, error) {
359419
mode := config.DetectLocalMode
360420
switch mode {
361-
case proxyconfigapi.LocalModeClusterCIDR:
421+
case proxyconfigapi.LocalModeClusterCIDR, proxyconfigapi.LocalModeNodeCIDR:
362422
return mode, nil
363423
default:
364424
if strings.TrimSpace(mode.String()) != "" {
@@ -369,20 +429,26 @@ func getDetectLocalMode(config *proxyconfigapi.KubeProxyConfiguration) (proxycon
369429
}
370430
}
371431

372-
func getLocalDetector(mode proxyconfigapi.LocalMode, config *proxyconfigapi.KubeProxyConfiguration, ipt utiliptables.Interface) (proxyutiliptables.LocalTrafficDetector, error) {
432+
func getLocalDetector(mode proxyconfigapi.LocalMode, config *proxyconfigapi.KubeProxyConfiguration, ipt utiliptables.Interface, nodeInfo *v1.Node) (proxyutiliptables.LocalTrafficDetector, error) {
373433
switch mode {
374434
case proxyconfigapi.LocalModeClusterCIDR:
375435
if len(strings.TrimSpace(config.ClusterCIDR)) == 0 {
376436
klog.Warning("detect-local-mode set to ClusterCIDR, but no cluster CIDR defined")
377437
break
378438
}
379439
return proxyutiliptables.NewDetectLocalByCIDR(config.ClusterCIDR, ipt)
440+
case proxyconfigapi.LocalModeNodeCIDR:
441+
if len(strings.TrimSpace(nodeInfo.Spec.PodCIDR)) == 0 {
442+
klog.Warning("detect-local-mode set to NodeCIDR, but no PodCIDR defined at node")
443+
break
444+
}
445+
return proxyutiliptables.NewDetectLocalByCIDR(nodeInfo.Spec.PodCIDR, ipt)
380446
}
381447
klog.V(0).Info("detect-local-mode: ", string(mode), " , defaulting to no-op detect-local")
382448
return proxyutiliptables.NewNoOpLocalDetector(), nil
383449
}
384450

385-
func getDualStackLocalDetectorTuple(mode proxyconfigapi.LocalMode, config *proxyconfigapi.KubeProxyConfiguration, ipt [2]utiliptables.Interface) ([2]proxyutiliptables.LocalTrafficDetector, error) {
451+
func getDualStackLocalDetectorTuple(mode proxyconfigapi.LocalMode, config *proxyconfigapi.KubeProxyConfiguration, ipt [2]utiliptables.Interface, nodeInfo *v1.Node) ([2]proxyutiliptables.LocalTrafficDetector, error) {
386452
var err error
387453
localDetectors := [2]proxyutiliptables.LocalTrafficDetector{proxyutiliptables.NewNoOpLocalDetector(), proxyutiliptables.NewNoOpLocalDetector()}
388454
switch mode {
@@ -409,6 +475,31 @@ func getDualStackLocalDetectorTuple(mode proxyconfigapi.LocalMode, config *proxy
409475
localDetectors[1], err = proxyutiliptables.NewDetectLocalByCIDR(clusterCIDRs[1], ipt[1])
410476
}
411477
return localDetectors, err
478+
case proxyconfigapi.LocalModeNodeCIDR:
479+
if nodeInfo == nil || len(strings.TrimSpace(nodeInfo.Spec.PodCIDR)) == 0 {
480+
klog.Warning("No node info available to configure detect-local-mode NodeCIDR")
481+
break
482+
}
483+
// localDetectors, like ipt, need to be of the order [IPv4, IPv6], but PodCIDRs is setup so that PodCIDRs[0] == PodCIDR.
484+
// so have to handle the case where PodCIDR can be IPv6 and set that to localDetectors[1]
485+
if utilsnet.IsIPv6CIDRString(nodeInfo.Spec.PodCIDR) {
486+
localDetectors[1], err = proxyutiliptables.NewDetectLocalByCIDR(nodeInfo.Spec.PodCIDR, ipt[1])
487+
if err != nil {
488+
return localDetectors, err
489+
}
490+
if len(nodeInfo.Spec.PodCIDRs) > 1 {
491+
localDetectors[0], err = proxyutiliptables.NewDetectLocalByCIDR(nodeInfo.Spec.PodCIDRs[1], ipt[0])
492+
}
493+
} else {
494+
localDetectors[0], err = proxyutiliptables.NewDetectLocalByCIDR(nodeInfo.Spec.PodCIDR, ipt[0])
495+
if err != nil {
496+
return localDetectors, err
497+
}
498+
if len(nodeInfo.Spec.PodCIDRs) > 1 {
499+
localDetectors[1], err = proxyutiliptables.NewDetectLocalByCIDR(nodeInfo.Spec.PodCIDRs[1], ipt[1])
500+
}
501+
}
502+
return localDetectors, err
412503
default:
413504
klog.Warningf("unknown detect-local-mode: %v", mode)
414505
}

0 commit comments

Comments
 (0)