Skip to content

Commit ded6ee9

Browse files
k8s-ci-robotk8s-ci-robot
authored
Merge pull request kubernetes#79993 from aramase/controller-manager-multiple-cidr
Allow multiple node cidr masks in kube-controller-manager
2 parents 4e45328 + 796faba commit ded6ee9

File tree

17 files changed

+343
-198
lines changed

17 files changed

+343
-198
lines changed

‎api/api-rules/violation_exceptions.list‎

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -662,6 +662,8 @@ API rule violation: names_match,k8s.io/kube-controller-manager/config/v1alpha1,K
662662
API rule violation: names_match,k8s.io/kube-controller-manager/config/v1alpha1,NamespaceControllerConfiguration,ConcurrentNamespaceSyncs
663663
API rule violation: names_match,k8s.io/kube-controller-manager/config/v1alpha1,NamespaceControllerConfiguration,NamespaceSyncPeriod
664664
API rule violation: names_match,k8s.io/kube-controller-manager/config/v1alpha1,NodeIPAMControllerConfiguration,NodeCIDRMaskSize
665+
API rule violation: names_match,k8s.io/kube-controller-manager/config/v1alpha1,NodeIPAMControllerConfiguration,NodeCIDRMaskSizeIPv4
666+
API rule violation: names_match,k8s.io/kube-controller-manager/config/v1alpha1,NodeIPAMControllerConfiguration,NodeCIDRMaskSizeIPv6
665667
API rule violation: names_match,k8s.io/kube-controller-manager/config/v1alpha1,NodeIPAMControllerConfiguration,SecondaryServiceCIDR
666668
API rule violation: names_match,k8s.io/kube-controller-manager/config/v1alpha1,NodeIPAMControllerConfiguration,ServiceCIDR
667669
API rule violation: names_match,k8s.io/kube-controller-manager/config/v1alpha1,NodeLifecycleControllerConfiguration,EnableTaintManager

‎cmd/kube-controller-manager/app/BUILD‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -60,6 +60,7 @@ go_library(
6060
"//pkg/controller/job:go_default_library",
6161
"//pkg/controller/namespace:go_default_library",
6262
"//pkg/controller/nodeipam:go_default_library",
63+
"//pkg/controller/nodeipam/config:go_default_library",
6364
"//pkg/controller/nodeipam/ipam:go_default_library",
6465
"//pkg/controller/nodelifecycle:go_default_library",
6566
"//pkg/controller/podautoscaler:go_default_library",

‎cmd/kube-controller-manager/app/core.go‎

Lines changed: 73 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -21,6 +21,7 @@ limitations under the License.
2121
package app
2222

2323
import (
24+
"errors"
2425
"fmt"
2526
"net"
2627
"net/http"
@@ -29,7 +30,7 @@ import (
2930

3031
"k8s.io/klog"
3132

32-
v1 "k8s.io/api/core/v1"
33+
"k8s.io/api/core/v1"
3334
"k8s.io/apimachinery/pkg/runtime/schema"
3435
utilfeature "k8s.io/apiserver/pkg/util/feature"
3536
cacheddiscovery "k8s.io/client-go/discovery/cached/memory"
@@ -46,6 +47,7 @@ import (
4647
"k8s.io/kubernetes/pkg/controller/garbagecollector"
4748
namespacecontroller "k8s.io/kubernetes/pkg/controller/namespace"
4849
nodeipamcontroller "k8s.io/kubernetes/pkg/controller/nodeipam"
50+
nodeipamconfig "k8s.io/kubernetes/pkg/controller/nodeipam/config"
4951
"k8s.io/kubernetes/pkg/controller/nodeipam/ipam"
5052
lifecyclecontroller "k8s.io/kubernetes/pkg/controller/nodelifecycle"
5153
"k8s.io/kubernetes/pkg/controller/podgc"
@@ -69,6 +71,13 @@ import (
6971
netutils "k8s.io/utils/net"
7072
)
7173

74+
const (
75+
// defaultNodeMaskCIDRIPv4 is default mask size for IPv4 node cidr
76+
defaultNodeMaskCIDRIPv4 = 24
77+
// defaultNodeMaskCIDRIPv6 is default mask size for IPv6 node cidr
78+
defaultNodeMaskCIDRIPv6 = 64
79+
)
80+
7281
func startServiceController(ctx ControllerContext) (http.Handler, bool, error) {
7382
serviceController, err := servicecontroller.New(
7483
ctx.Cloud,
@@ -85,6 +94,7 @@ func startServiceController(ctx ControllerContext) (http.Handler, bool, error) {
8594
go serviceController.Run(ctx.Stop, int(ctx.ComponentConfig.ServiceController.ConcurrentServiceSyncs))
8695
return nil, true, nil
8796
}
97+
8898
func startNodeIpamController(ctx ControllerContext) (http.Handler, bool, error) {
8999
var serviceCIDR *net.IPNet
90100
var secondaryServiceCIDR *net.IPNet
@@ -147,14 +157,28 @@ func startNodeIpamController(ctx ControllerContext) (http.Handler, bool, error)
147157
}
148158
}
149159

160+
var nodeCIDRMaskSizeIPv4, nodeCIDRMaskSizeIPv6 int
161+
if utilfeature.DefaultFeatureGate.Enabled(kubefeatures.IPv6DualStack) {
162+
nodeCIDRMaskSizeIPv4, nodeCIDRMaskSizeIPv6, err = setNodeCIDRMaskSizesDualStack(ctx.ComponentConfig.NodeIPAMController)
163+
} else {
164+
nodeCIDRMaskSizeIPv4, nodeCIDRMaskSizeIPv6, err = setNodeCIDRMaskSizes(ctx.ComponentConfig.NodeIPAMController)
165+
}
166+
167+
if err != nil {
168+
return nil, false, err
169+
}
170+
171+
// get list of node cidr mask sizes
172+
nodeCIDRMaskSizes := getNodeCIDRMaskSizes(clusterCIDRs, nodeCIDRMaskSizeIPv4, nodeCIDRMaskSizeIPv6)
173+
150174
nodeIpamController, err := nodeipamcontroller.NewNodeIpamController(
151175
ctx.InformerFactory.Core().V1().Nodes(),
152176
ctx.Cloud,
153177
ctx.ClientBuilder.ClientOrDie("node-controller"),
154178
clusterCIDRs,
155179
serviceCIDR,
156180
secondaryServiceCIDR,
157-
int(ctx.ComponentConfig.NodeIPAMController.NodeCIDRMaskSize),
181+
nodeCIDRMaskSizes,
158182
ipam.CIDRAllocatorType(ctx.ComponentConfig.KubeCloudShared.CIDRAllocatorType),
159183
)
160184
if err != nil {
@@ -562,3 +586,50 @@ func processCIDRs(cidrsList string) ([]*net.IPNet, bool, error) {
562586

563587
return cidrs, dualstack, nil
564588
}
589+
590+
// setNodeCIDRMaskSizes returns the IPv4 and IPv6 node cidr mask sizes.
591+
// If --node-cidr-mask-size not set, then it will return default IPv4 and IPv6 cidr mask sizes.
592+
func setNodeCIDRMaskSizes(cfg nodeipamconfig.NodeIPAMControllerConfiguration) (int, int, error) {
593+
ipv4Mask, ipv6Mask := defaultNodeMaskCIDRIPv4, defaultNodeMaskCIDRIPv6
594+
// NodeCIDRMaskSizeIPv4 and NodeCIDRMaskSizeIPv6 can be used only for dual-stack clusters
595+
if cfg.NodeCIDRMaskSizeIPv4 != 0 || cfg.NodeCIDRMaskSizeIPv6 != 0 {
596+
return ipv4Mask, ipv6Mask, errors.New("usage of --node-cidr-mask-size-ipv4 and --node-cidr-mask-size-ipv6 are not allowed with non dual-stack clusters")
597+
}
598+
if cfg.NodeCIDRMaskSize != 0 {
599+
ipv4Mask = int(cfg.NodeCIDRMaskSize)
600+
ipv6Mask = int(cfg.NodeCIDRMaskSize)
601+
}
602+
return ipv4Mask, ipv6Mask, nil
603+
}
604+
605+
// setNodeCIDRMaskSizesDualStack returns the IPv4 and IPv6 node cidr mask sizes to the value provided
606+
// for --node-cidr-mask-size-ipv4 and --node-cidr-mask-size-ipv6 respectively. If value not provided,
607+
// then it will return default IPv4 and IPv6 cidr mask sizes.
608+
func setNodeCIDRMaskSizesDualStack(cfg nodeipamconfig.NodeIPAMControllerConfiguration) (int, int, error) {
609+
ipv4Mask, ipv6Mask := defaultNodeMaskCIDRIPv4, defaultNodeMaskCIDRIPv6
610+
// NodeCIDRMaskSize can be used only for single stack clusters
611+
if cfg.NodeCIDRMaskSize != 0 {
612+
return ipv4Mask, ipv6Mask, errors.New("usage of --node-cidr-mask-size is not allowed with dual-stack clusters")
613+
}
614+
if cfg.NodeCIDRMaskSizeIPv4 != 0 {
615+
ipv4Mask = int(cfg.NodeCIDRMaskSizeIPv4)
616+
}
617+
if cfg.NodeCIDRMaskSizeIPv6 != 0 {
618+
ipv6Mask = int(cfg.NodeCIDRMaskSizeIPv6)
619+
}
620+
return ipv4Mask, ipv6Mask, nil
621+
}
622+
623+
// getNodeCIDRMaskSizes is a helper function that helps the generate the node cidr mask
624+
// sizes slice based on the cluster cidr slice
625+
func getNodeCIDRMaskSizes(clusterCIDRs []*net.IPNet, maskSizeIPv4, maskSizeIPv6 int) []int {
626+
nodeMaskCIDRs := []int{}
627+
for _, clusterCIDR := range clusterCIDRs {
628+
if netutils.IsIPv6CIDR(clusterCIDR) {
629+
nodeMaskCIDRs = append(nodeMaskCIDRs, maskSizeIPv6)
630+
} else {
631+
nodeMaskCIDRs = append(nodeMaskCIDRs, maskSizeIPv4)
632+
}
633+
}
634+
return nodeMaskCIDRs
635+
}

‎cmd/kube-controller-manager/app/options/nodeipamcontroller.go‎

Lines changed: 3 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -36,7 +36,9 @@ func (o *NodeIPAMControllerOptions) AddFlags(fs *pflag.FlagSet) {
3636
return
3737
}
3838
fs.StringVar(&o.ServiceCIDR, "service-cluster-ip-range", o.ServiceCIDR, "CIDR Range for Services in cluster. Requires --allocate-node-cidrs to be true")
39-
fs.Int32Var(&o.NodeCIDRMaskSize, "node-cidr-mask-size", o.NodeCIDRMaskSize, "Mask size for node cidr in cluster.")
39+
fs.Int32Var(&o.NodeCIDRMaskSize, "node-cidr-mask-size", o.NodeCIDRMaskSize, "Mask size for node cidr in cluster. Default is 24 for IPv4 and 64 for IPv6.")
40+
fs.Int32Var(&o.NodeCIDRMaskSizeIPv4, "node-cidr-mask-size-ipv4", o.NodeCIDRMaskSizeIPv4, "Mask size for IPv4 node cidr in dual-stack cluster. Default is 24.")
41+
fs.Int32Var(&o.NodeCIDRMaskSizeIPv6, "node-cidr-mask-size-ipv6", o.NodeCIDRMaskSizeIPv6, "Mask size for IPv6 node cidr in dual-stack cluster. Default is 64.")
4042
}
4143

4244
// ApplyTo fills up NodeIpamController config with options.

‎cmd/kube-controller-manager/app/options/options_test.go‎

Lines changed: 5 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -117,6 +117,8 @@ func TestAddFlags(t *testing.T) {
117117
"--min-resync-period=8h",
118118
"--namespace-sync-period=10m",
119119
"--node-cidr-mask-size=48",
120+
"--node-cidr-mask-size-ipv4=48",
121+
"--node-cidr-mask-size-ipv6=108",
120122
"--node-eviction-rate=0.2",
121123
"--node-monitor-grace-period=30s",
122124
"--node-monitor-period=10s",
@@ -279,7 +281,9 @@ func TestAddFlags(t *testing.T) {
279281
},
280282
NodeIPAMController: &NodeIPAMControllerOptions{
281283
&nodeipamconfig.NodeIPAMControllerConfiguration{
282-
NodeCIDRMaskSize: 48,
284+
NodeCIDRMaskSize: 48,
285+
NodeCIDRMaskSizeIPv4: 48,
286+
NodeCIDRMaskSizeIPv6: 108,
283287
},
284288
},
285289
NodeLifecycleController: &NodeLifecycleControllerOptions{

‎pkg/controller/nodeipam/config/types.go‎

Lines changed: 5 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -22,6 +22,10 @@ type NodeIPAMControllerConfiguration struct {
2222
ServiceCIDR string
2323
// secondaryServiceCIDR is CIDR Range for Services in cluster. This is used in dual stack clusters. SecondaryServiceCIDR must be of different IP family than ServiceCIDR
2424
SecondaryServiceCIDR string
25-
// NodeCIDRMaskSize is the mask size for node cidr in cluster.
25+
// NodeCIDRMaskSize is the mask size for node cidr in single-stack cluster.
2626
NodeCIDRMaskSize int32
27+
// NodeCIDRMaskSizeIPv4 is the mask size for node cidr in dual-stack cluster.
28+
NodeCIDRMaskSizeIPv4 int32
29+
// NodeCIDRMaskSizeIPv6 is the mask size for node cidr in dual-stack cluster.
30+
NodeCIDRMaskSizeIPv6 int32
2731
}

‎pkg/controller/nodeipam/config/v1alpha1/defaults.go‎

Lines changed: 2 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -30,7 +30,6 @@ import (
3030
// be no easy way to opt-out. Instead, if you want to use this defaulting method
3131
// run it in your wrapper struct of this type in its `SetDefaults_` method.
3232
func RecommendedDefaultNodeIPAMControllerConfiguration(obj *kubectrlmgrconfigv1alpha1.NodeIPAMControllerConfiguration) {
33-
if obj.NodeCIDRMaskSize == 0 {
34-
obj.NodeCIDRMaskSize = 24
35-
}
33+
// The default mask size is not set here because we need to determine the cluster cidr family before setting the
34+
// appropriate mask size.
3635
}

‎pkg/controller/nodeipam/config/v1alpha1/zz_generated.conversion.go‎

Lines changed: 4 additions & 0 deletions
Some generated files are not rendered by default. Learn more about customizing how changed files appear on GitHub.

‎pkg/controller/nodeipam/ipam/cidr_allocator.go‎

Lines changed: 15 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -88,16 +88,29 @@ type CIDRAllocator interface {
8888
Run(stopCh <-chan struct{})
8989
}
9090

91+
// CIDRAllocatorParams is parameters that's required for creating new
92+
// cidr range allocator.
93+
type CIDRAllocatorParams struct {
94+
// ClusterCIDRs is list of cluster cidrs
95+
ClusterCIDRs []*net.IPNet
96+
// ServiceCIDR is primary service cidr for cluster
97+
ServiceCIDR *net.IPNet
98+
// SecondaryServiceCIDR is secondary service cidr for cluster
99+
SecondaryServiceCIDR *net.IPNet
100+
// NodeCIDRMaskSizes is list of node cidr mask sizes
101+
NodeCIDRMaskSizes []int
102+
}
103+
91104
// New creates a new CIDR range allocator.
92-
func New(kubeClient clientset.Interface, cloud cloudprovider.Interface, nodeInformer informers.NodeInformer, allocatorType CIDRAllocatorType, clusterCIDRs []*net.IPNet, serviceCIDR *net.IPNet, secondaryServiceCIDR *net.IPNet, nodeCIDRMaskSize int) (CIDRAllocator, error) {
105+
func New(kubeClient clientset.Interface, cloud cloudprovider.Interface, nodeInformer informers.NodeInformer, allocatorType CIDRAllocatorType, allocatorParams CIDRAllocatorParams) (CIDRAllocator, error) {
93106
nodeList, err := listNodes(kubeClient)
94107
if err != nil {
95108
return nil, err
96109
}
97110

98111
switch allocatorType {
99112
case RangeAllocatorType:
100-
return NewCIDRRangeAllocator(kubeClient, nodeInformer, clusterCIDRs, serviceCIDR, secondaryServiceCIDR, nodeCIDRMaskSize, nodeList)
113+
return NewCIDRRangeAllocator(kubeClient, nodeInformer, allocatorParams, nodeList)
101114
case CloudAllocatorType:
102115
return NewCloudCIDRAllocator(kubeClient, cloud, nodeInformer)
103116
default:

‎pkg/controller/nodeipam/ipam/cidrset/cidr_set.go‎

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -43,6 +43,8 @@ const (
4343
// The subnet mask size cannot be greater than 16 more than the cluster mask size
4444
// TODO: https://github.com/kubernetes/kubernetes/issues/44918
4545
// clusterSubnetMaxDiff limited to 16 due to the uncompressed bitmap
46+
// Due to this limitation the subnet mask for IPv6 cluster cidr needs to be >= 48
47+
// as default mask size for IPv6 is 64.
4648
clusterSubnetMaxDiff = 16
4749
// halfIPv6Len is the half of the IPv6 length
4850
halfIPv6Len = net.IPv6len / 2

0 commit comments

Comments
 (0)