Skip to content

Commit 550fb1b

Browse files
authored
Merge pull request kubernetes#79386 from khenidak/phase2-dualstack
Phase 2 dualstack
2 parents ca5babc + c27e0b0 commit 550fb1b

Some content is hidden

Large Commits have some content hidden by default. Use the searchbox below for content that may be hidden.

61 files changed

+3223
-1150
lines changed

api/api-rules/violation_exceptions.list

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -636,6 +636,7 @@ API rule violation: names_match,k8s.io/kube-controller-manager/config/v1alpha1,K
636636
API rule violation: names_match,k8s.io/kube-controller-manager/config/v1alpha1,NamespaceControllerConfiguration,ConcurrentNamespaceSyncs
637637
API rule violation: names_match,k8s.io/kube-controller-manager/config/v1alpha1,NamespaceControllerConfiguration,NamespaceSyncPeriod
638638
API rule violation: names_match,k8s.io/kube-controller-manager/config/v1alpha1,NodeIPAMControllerConfiguration,NodeCIDRMaskSize
639+
API rule violation: names_match,k8s.io/kube-controller-manager/config/v1alpha1,NodeIPAMControllerConfiguration,SecondaryServiceCIDR
639640
API rule violation: names_match,k8s.io/kube-controller-manager/config/v1alpha1,NodeIPAMControllerConfiguration,ServiceCIDR
640641
API rule violation: names_match,k8s.io/kube-controller-manager/config/v1alpha1,NodeLifecycleControllerConfiguration,EnableTaintManager
641642
API rule violation: names_match,k8s.io/kube-controller-manager/config/v1alpha1,NodeLifecycleControllerConfiguration,LargeClusterSizeThreshold

api/openapi-spec/swagger.json

Lines changed: 4 additions & 0 deletions
Some generated files are not rendered by default. Learn more about customizing how changed files appear on GitHub.

cmd/kube-apiserver/app/options/BUILD

Lines changed: 5 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -35,6 +35,7 @@ go_library(
3535
"//staging/src/k8s.io/component-base/cli/globalflag:go_default_library",
3636
"//staging/src/k8s.io/kube-aggregator/pkg/apiserver/scheme:go_default_library",
3737
"//vendor/github.com/spf13/pflag:go_default_library",
38+
"//vendor/k8s.io/utils/net:go_default_library",
3839
],
3940
)
4041

@@ -43,22 +44,26 @@ go_test(
4344
srcs = [
4445
"globalflags_test.go",
4546
"options_test.go",
47+
"validation_test.go",
4648
],
4749
embed = [":go_default_library"],
4850
deps = [
4951
"//pkg/apis/core:go_default_library",
52+
"//pkg/features:go_default_library",
5053
"//pkg/kubeapiserver/options:go_default_library",
5154
"//pkg/kubelet/client:go_default_library",
5255
"//pkg/master/reconcilers:go_default_library",
5356
"//staging/src/k8s.io/apimachinery/pkg/util/diff:go_default_library",
5457
"//staging/src/k8s.io/apiserver/pkg/server/options:go_default_library",
5558
"//staging/src/k8s.io/apiserver/pkg/storage/storagebackend:go_default_library",
59+
"//staging/src/k8s.io/apiserver/pkg/util/feature:go_default_library",
5660
"//staging/src/k8s.io/apiserver/plugin/pkg/audit/buffered:go_default_library",
5761
"//staging/src/k8s.io/apiserver/plugin/pkg/audit/dynamic:go_default_library",
5862
"//staging/src/k8s.io/apiserver/plugin/pkg/audit/truncate:go_default_library",
5963
"//staging/src/k8s.io/client-go/rest:go_default_library",
6064
"//staging/src/k8s.io/component-base/cli/flag:go_default_library",
6165
"//staging/src/k8s.io/component-base/cli/globalflag:go_default_library",
66+
"//staging/src/k8s.io/component-base/featuregate/testing:go_default_library",
6267
"//vendor/github.com/spf13/pflag:go_default_library",
6368
],
6469
)

cmd/kube-apiserver/app/options/options.go

Lines changed: 13 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -56,10 +56,16 @@ type ServerRunOptions struct {
5656
KubeletConfig kubeletclient.KubeletClientConfig
5757
KubernetesServiceNodePort int
5858
MaxConnectionBytesPerSec int64
59-
ServiceClusterIPRange net.IPNet // TODO: make this a list
60-
ServiceNodePortRange utilnet.PortRange
61-
SSHKeyfile string
62-
SSHUser string
59+
// ServiceClusterIPRange is mapped to input provided by user
60+
ServiceClusterIPRanges string
61+
//PrimaryServiceClusterIPRange and SecondaryServiceClusterIPRange are the results
62+
// of parsing ServiceClusterIPRange into actual values
63+
PrimaryServiceClusterIPRange net.IPNet
64+
SecondaryServiceClusterIPRange net.IPNet
65+
66+
ServiceNodePortRange utilnet.PortRange
67+
SSHKeyfile string
68+
SSHUser string
6369

6470
ProxyClientCertFile string
6571
ProxyClientKeyFile string
@@ -114,7 +120,7 @@ func NewServerRunOptions() *ServerRunOptions {
114120
},
115121
ServiceNodePortRange: kubeoptions.DefaultServiceNodePortRange,
116122
}
117-
s.ServiceClusterIPRange = kubeoptions.DefaultServiceIPCIDR
123+
s.ServiceClusterIPRanges = kubeoptions.DefaultServiceIPCIDR.String()
118124

119125
// Overwrite the default for storage data format.
120126
s.Etcd.DefaultStorageMediaType = "application/vnd.kubernetes.protobuf"
@@ -179,7 +185,8 @@ func (s *ServerRunOptions) Flags() (fss cliflag.NamedFlagSets) {
179185
"of type NodePort, using this as the value of the port. If zero, the Kubernetes master "+
180186
"service will be of type ClusterIP.")
181187

182-
fs.IPNetVar(&s.ServiceClusterIPRange, "service-cluster-ip-range", s.ServiceClusterIPRange, ""+
188+
// TODO (khenidak) change documentation as we move IPv6DualStack feature from ALPHA to BETA
189+
fs.StringVar(&s.ServiceClusterIPRanges, "service-cluster-ip-range", s.ServiceClusterIPRanges, ""+
183190
"A CIDR notation IP range from which to assign service cluster IPs. This must not "+
184191
"overlap with any IP ranges assigned to nodes for pods.")
185192

cmd/kube-apiserver/app/options/options_test.go

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -118,7 +118,7 @@ func TestAddFlags(t *testing.T) {
118118
// This is a snapshot of expected options parsed by args.
119119
expected := &ServerRunOptions{
120120
ServiceNodePortRange: kubeoptions.DefaultServiceNodePortRange,
121-
ServiceClusterIPRange: kubeoptions.DefaultServiceIPCIDR,
121+
ServiceClusterIPRanges: kubeoptions.DefaultServiceIPCIDR.String(),
122122
MasterCount: 5,
123123
EndpointReconcilerType: string(reconcilers.LeaseEndpointReconcilerType),
124124
AllowPrivileged: false,

cmd/kube-apiserver/app/options/validation.go

Lines changed: 46 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -19,26 +19,69 @@ package options
1919
import (
2020
"errors"
2121
"fmt"
22+
"net"
23+
"strings"
2224

2325
apiextensionsapiserver "k8s.io/apiextensions-apiserver/pkg/apiserver"
2426
utilfeature "k8s.io/apiserver/pkg/util/feature"
2527
aggregatorscheme "k8s.io/kube-aggregator/pkg/apiserver/scheme"
2628
"k8s.io/kubernetes/pkg/api/legacyscheme"
2729
"k8s.io/kubernetes/pkg/features"
30+
netutils "k8s.io/utils/net"
2831
)
2932

3033
// TODO: Longer term we should read this from some config store, rather than a flag.
34+
// validateClusterIPFlags is expected to be called after Complete()
3135
func validateClusterIPFlags(options *ServerRunOptions) []error {
3236
var errs []error
3337

34-
if options.ServiceClusterIPRange.IP == nil {
35-
errs = append(errs, errors.New("no --service-cluster-ip-range specified"))
38+
// validate that primary has been processed by user provided values or it has been defaulted
39+
if options.PrimaryServiceClusterIPRange.IP == nil {
40+
errs = append(errs, errors.New("--service-cluster-ip-range must contain at least one valid cidr"))
3641
}
37-
var ones, bits = options.ServiceClusterIPRange.Mask.Size()
42+
43+
serviceClusterIPRangeList := strings.Split(options.ServiceClusterIPRanges, ",")
44+
if len(serviceClusterIPRangeList) > 2 {
45+
errs = append(errs, errors.New("--service-cluster-ip-range must not contain more than two entries"))
46+
}
47+
48+
// Complete() expected to have set Primary* and Secondary*
49+
// primary CIDR validation
50+
var ones, bits = options.PrimaryServiceClusterIPRange.Mask.Size()
3851
if bits-ones > 20 {
3952
errs = append(errs, errors.New("specified --service-cluster-ip-range is too large"))
4053
}
4154

55+
// Secondary IP validation
56+
secondaryServiceClusterIPRangeUsed := (options.SecondaryServiceClusterIPRange.IP != nil)
57+
if secondaryServiceClusterIPRangeUsed && !utilfeature.DefaultFeatureGate.Enabled(features.IPv6DualStack) {
58+
errs = append(errs, fmt.Errorf("--secondary-service-cluster-ip-range can only be used if %v feature is enabled", string(features.IPv6DualStack)))
59+
}
60+
61+
// note: While the cluster might be dualstack (i.e. pods with multiple IPs), the user may choose
62+
// to only ingress traffic within and into the cluster on one IP family only. this family is decided
63+
// by the range set on --service-cluster-ip-range. If/when the user decides to use dual stack services
64+
// the Secondary* must be of different IPFamily than --service-cluster-ip-range
65+
if secondaryServiceClusterIPRangeUsed {
66+
// Should be dualstack IPFamily(PrimaryServiceClusterIPRange) != IPFamily(SecondaryServiceClusterIPRange)
67+
dualstack, err := netutils.IsDualStackCIDRs([]*net.IPNet{&options.PrimaryServiceClusterIPRange, &options.SecondaryServiceClusterIPRange})
68+
if err != nil {
69+
errs = append(errs, errors.New("error attempting to validate dualstack for --service-cluster-ip-range and --secondary-service-cluster-ip-range"))
70+
}
71+
72+
if !dualstack {
73+
errs = append(errs, errors.New("--service-cluster-ip-range and --secondary-service-cluster-ip-range must be of different IP family"))
74+
}
75+
76+
// should be smallish sized cidr, this thing is kept in etcd
77+
// bigger cidr (specially those offered by IPv6) will add no value
78+
// significantly increase snapshotting time.
79+
var ones, bits = options.SecondaryServiceClusterIPRange.Mask.Size()
80+
if bits-ones > 20 {
81+
errs = append(errs, errors.New("specified --secondary-service-cluster-ip-range is too large"))
82+
}
83+
}
84+
4285
return errs
4386
}
4487

Lines changed: 132 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,132 @@
1+
/*
2+
Copyright 2019 The Kubernetes Authors.
3+
4+
Licensed under the Apache License, Version 2.0 (the "License");
5+
you may not use this file except in compliance with the License.
6+
You may obtain a copy of the License at
7+
8+
http://www.apache.org/licenses/LICENSE-2.0
9+
10+
Unless required by applicable law or agreed to in writing, software
11+
distributed under the License is distributed on an "AS IS" BASIS,
12+
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
13+
See the License for the specific language governing permissions and
14+
limitations under the License.
15+
*/
16+
17+
package options
18+
19+
import (
20+
"net"
21+
"testing"
22+
23+
utilfeature "k8s.io/apiserver/pkg/util/feature"
24+
featuregatetesting "k8s.io/component-base/featuregate/testing"
25+
"k8s.io/kubernetes/pkg/features"
26+
)
27+
28+
func makeOptionsWithCIDRs(serviceCIDR string, secondaryServiceCIDR string) *ServerRunOptions {
29+
value := serviceCIDR
30+
if len(secondaryServiceCIDR) > 0 {
31+
value = value + "," + secondaryServiceCIDR
32+
}
33+
34+
var primaryCIDR, secondaryCIDR net.IPNet
35+
if len(serviceCIDR) > 0 {
36+
_, cidr, _ := net.ParseCIDR(serviceCIDR)
37+
if cidr != nil {
38+
primaryCIDR = *(cidr)
39+
}
40+
}
41+
42+
if len(secondaryServiceCIDR) > 0 {
43+
_, cidr, _ := net.ParseCIDR(secondaryServiceCIDR)
44+
if cidr != nil {
45+
secondaryCIDR = *(cidr)
46+
}
47+
}
48+
return &ServerRunOptions{
49+
ServiceClusterIPRanges: value,
50+
PrimaryServiceClusterIPRange: primaryCIDR,
51+
SecondaryServiceClusterIPRange: secondaryCIDR,
52+
}
53+
}
54+
55+
func TestClusterSerivceIPRange(t *testing.T) {
56+
testCases := []struct {
57+
name string
58+
options *ServerRunOptions
59+
enableDualStack bool
60+
expectErrors bool
61+
}{
62+
{
63+
name: "no service cidr",
64+
expectErrors: true,
65+
options: makeOptionsWithCIDRs("", ""),
66+
enableDualStack: false,
67+
},
68+
{
69+
name: "only secondary service cidr, dual stack gate on",
70+
expectErrors: true,
71+
options: makeOptionsWithCIDRs("", "10.0.0.0/16"),
72+
enableDualStack: true,
73+
},
74+
{
75+
name: "only secondary service cidr, dual stack gate off",
76+
expectErrors: true,
77+
options: makeOptionsWithCIDRs("", "10.0.0.0/16"),
78+
enableDualStack: false,
79+
},
80+
{
81+
name: "primary and secondary are provided but not dual stack v4-v4",
82+
expectErrors: true,
83+
options: makeOptionsWithCIDRs("10.0.0.0/16", "11.0.0.0/16"),
84+
enableDualStack: true,
85+
},
86+
{
87+
name: "primary and secondary are provided but not dual stack v6-v6",
88+
expectErrors: true,
89+
options: makeOptionsWithCIDRs("2000::/108", "3000::/108"),
90+
enableDualStack: true,
91+
},
92+
{
93+
name: "valid dual stack with gate disabled",
94+
expectErrors: true,
95+
options: makeOptionsWithCIDRs("10.0.0.0/16", "3000::/108"),
96+
enableDualStack: false,
97+
},
98+
/* success cases */
99+
{
100+
name: "valid primary",
101+
expectErrors: false,
102+
options: makeOptionsWithCIDRs("10.0.0.0/16", ""),
103+
enableDualStack: false,
104+
},
105+
{
106+
name: "valid v4-v6 dual stack + gate on",
107+
expectErrors: false,
108+
options: makeOptionsWithCIDRs("10.0.0.0/16", "3000::/108"),
109+
enableDualStack: true,
110+
},
111+
{
112+
name: "valid v6-v4 dual stack + gate on",
113+
expectErrors: false,
114+
options: makeOptionsWithCIDRs("3000::/108", "10.0.0.0/16"),
115+
enableDualStack: true,
116+
},
117+
}
118+
119+
for _, tc := range testCases {
120+
t.Run(tc.name, func(t *testing.T) {
121+
defer featuregatetesting.SetFeatureGateDuringTest(t, utilfeature.DefaultFeatureGate, features.IPv6DualStack, tc.enableDualStack)()
122+
errs := validateClusterIPFlags(tc.options)
123+
if len(errs) > 0 && !tc.expectErrors {
124+
t.Errorf("expected no errors, errors found %+v", errs)
125+
}
126+
127+
if len(errs) == 0 && tc.expectErrors {
128+
t.Errorf("expected errors, no errors found")
129+
}
130+
})
131+
}
132+
}

cmd/kube-apiserver/app/server.go

Lines changed: 57 additions & 7 deletions
Original file line numberDiff line numberDiff line change
@@ -306,11 +306,21 @@ func CreateKubeAPIServerConfig(
306306
PerConnectionBandwidthLimitBytesPerSec: s.MaxConnectionBytesPerSec,
307307
})
308308

309-
serviceIPRange, apiServerServiceIP, lastErr := master.DefaultServiceIPRange(s.ServiceClusterIPRange)
309+
serviceIPRange, apiServerServiceIP, lastErr := master.DefaultServiceIPRange(s.PrimaryServiceClusterIPRange)
310310
if lastErr != nil {
311311
return
312312
}
313313

314+
// defaults to empty range and ip
315+
var secondaryServiceIPRange net.IPNet
316+
// process secondary range only if provided by user
317+
if s.SecondaryServiceClusterIPRange.IP != nil {
318+
secondaryServiceIPRange, _, lastErr = master.DefaultServiceIPRange(s.SecondaryServiceClusterIPRange)
319+
if lastErr != nil {
320+
return
321+
}
322+
}
323+
314324
clientCA, lastErr := readCAorNil(s.Authentication.ClientCert.ClientCA)
315325
if lastErr != nil {
316326
return
@@ -341,8 +351,10 @@ func CreateKubeAPIServerConfig(
341351

342352
Tunneler: nodeTunneler,
343353

344-
ServiceIPRange: serviceIPRange,
345-
APIServerServiceIP: apiServerServiceIP,
354+
ServiceIPRange: serviceIPRange,
355+
APIServerServiceIP: apiServerServiceIP,
356+
SecondaryServiceIPRange: secondaryServiceIPRange,
357+
346358
APIServerServicePort: 443,
347359

348360
ServiceNodePortRange: s.ServiceNodePortRange,
@@ -548,11 +560,49 @@ func Complete(s *options.ServerRunOptions) (completedServerRunOptions, error) {
548560
if err := kubeoptions.DefaultAdvertiseAddress(s.GenericServerRunOptions, s.InsecureServing.DeprecatedInsecureServingOptions); err != nil {
549561
return options, err
550562
}
551-
serviceIPRange, apiServerServiceIP, err := master.DefaultServiceIPRange(s.ServiceClusterIPRange)
552-
if err != nil {
553-
return options, fmt.Errorf("error determining service IP ranges: %v", err)
563+
564+
// process s.ServiceClusterIPRange from list to Primary and Secondary
565+
// we process secondary only if provided by user
566+
567+
serviceClusterIPRangeList := strings.Split(s.ServiceClusterIPRanges, ",")
568+
569+
var apiServerServiceIP net.IP
570+
var serviceIPRange net.IPNet
571+
var err error
572+
// nothing provided by user, use default range (only applies to the Primary)
573+
if len(serviceClusterIPRangeList) == 0 {
574+
var primaryServiceClusterCIDR net.IPNet
575+
serviceIPRange, apiServerServiceIP, err = master.DefaultServiceIPRange(primaryServiceClusterCIDR)
576+
if err != nil {
577+
return options, fmt.Errorf("error determining service IP ranges: %v", err)
578+
}
579+
s.PrimaryServiceClusterIPRange = serviceIPRange
580+
}
581+
582+
if len(serviceClusterIPRangeList) > 0 {
583+
_, primaryServiceClusterCIDR, err := net.ParseCIDR(serviceClusterIPRangeList[0])
584+
if err != nil {
585+
return options, fmt.Errorf("service-cluster-ip-range[0] is not a valid cidr")
586+
}
587+
588+
serviceIPRange, apiServerServiceIP, err = master.DefaultServiceIPRange(*(primaryServiceClusterCIDR))
589+
if err != nil {
590+
return options, fmt.Errorf("error determining service IP ranges for primary service cidr: %v", err)
591+
}
592+
s.PrimaryServiceClusterIPRange = serviceIPRange
554593
}
555-
s.ServiceClusterIPRange = serviceIPRange
594+
595+
// user provided at least two entries
596+
if len(serviceClusterIPRangeList) > 1 {
597+
_, secondaryServiceClusterCIDR, err := net.ParseCIDR(serviceClusterIPRangeList[1])
598+
if err != nil {
599+
return options, fmt.Errorf("service-cluster-ip-range[1] is not an ip net")
600+
}
601+
602+
s.SecondaryServiceClusterIPRange = *(secondaryServiceClusterCIDR)
603+
}
604+
//note: validation asserts that the list is max of two dual stack entries
605+
556606
if err := s.SecureServing.MaybeDefaultWithSelfSignedCerts(s.GenericServerRunOptions.AdvertiseAddress.String(), []string{"kubernetes.default.svc", "kubernetes.default", "kubernetes"}, []net.IP{apiServerServiceIP}); err != nil {
557607
return options, fmt.Errorf("error creating self-signed certificates: %v", err)
558608
}

0 commit comments

Comments
 (0)