@@ -27,23 +27,29 @@ import (
2727 "github.com/google/go-cmp/cmp"
2828 "github.com/spf13/pflag"
2929
30- eventv1 "k8s.io/api/events/v1"
3130 metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
3231 utilerrors "k8s.io/apimachinery/pkg/util/errors"
32+ utilruntime "k8s.io/apimachinery/pkg/util/runtime"
33+ "k8s.io/apimachinery/pkg/util/version"
34+
3335 "k8s.io/apiserver/pkg/apis/apiserver"
3436 apiserveroptions "k8s.io/apiserver/pkg/server/options"
35- cpconfig "k8s.io/cloud-provider/config"
36- serviceconfig "k8s.io/cloud-provider/controllers/service/config"
37- cpoptions "k8s.io/cloud-provider/options"
37+ utilversion "k8s.io/apiserver/pkg/util/version"
38+
3839 componentbaseconfig "k8s.io/component-base/config"
40+ "k8s.io/component-base/featuregate"
3941 "k8s.io/component-base/logs"
4042 "k8s.io/component-base/metrics"
43+
44+ cpconfig "k8s.io/cloud-provider/config"
45+ serviceconfig "k8s.io/cloud-provider/controllers/service/config"
46+ cpoptions "k8s.io/cloud-provider/options"
47+
48+ eventv1 "k8s.io/api/events/v1"
49+ clientgofeaturegate "k8s.io/client-go/features"
4150 cmconfig "k8s.io/controller-manager/config"
4251 cmoptions "k8s.io/controller-manager/options"
4352 migration "k8s.io/controller-manager/pkg/leadermigration/options"
44- netutils "k8s.io/utils/net"
45-
46- clientgofeaturegate "k8s.io/client-go/features"
4753 kubecontrollerconfig "k8s.io/kubernetes/cmd/kube-controller-manager/app/config"
4854 kubectrlmgrconfig "k8s.io/kubernetes/pkg/controller/apis/config"
4955 csrsigningconfig "k8s.io/kubernetes/pkg/controller/certificates/signer/config"
@@ -70,6 +76,7 @@ import (
7076 attachdetachconfig "k8s.io/kubernetes/pkg/controller/volume/attachdetach/config"
7177 ephemeralvolumeconfig "k8s.io/kubernetes/pkg/controller/volume/ephemeral/config"
7278 persistentvolumeconfig "k8s.io/kubernetes/pkg/controller/volume/persistentvolume/config"
79+ netutils "k8s.io/utils/net"
7380)
7481
7582var args = []string {
@@ -165,11 +172,7 @@ var args = []string{
165172}
166173
167174func TestAddFlags (t * testing.T ) {
168- fs := pflag .NewFlagSet ("addflagstest" , pflag .ContinueOnError )
169- s , _ := NewKubeControllerManagerOptions ()
170- for _ , f := range s .Flags ([]string {"" }, []string {"" }, nil ).FlagSets {
171- fs .AddFlagSet (f )
172- }
175+ fs , s := setupControllerManagerFlagSet (t )
173176
174177 fs .Parse (args )
175178 // Sort GCIgnoredResources because it's built from a map, which means the
@@ -442,9 +445,10 @@ func TestAddFlags(t *testing.T) {
442445 AlwaysAllowPaths : []string {"/healthz" , "/readyz" , "/livez" }, // note: this does not match /healthz/ or /healthz/*
443446 AlwaysAllowGroups : []string {"system:masters" },
444447 },
445- Master : "192.168.4.20" ,
446- Metrics : & metrics.Options {},
447- Logs : logs .NewOptions (),
448+ Master : "192.168.4.20" ,
449+ Metrics : & metrics.Options {},
450+ Logs : logs .NewOptions (),
451+ ComponentGlobalsRegistry : utilversion .DefaultComponentGlobalsRegistry ,
448452 }
449453
450454 // Sort GCIgnoredResources because it's built from a map, which means the
@@ -457,12 +461,7 @@ func TestAddFlags(t *testing.T) {
457461}
458462
459463func TestApplyTo (t * testing.T ) {
460- fs := pflag .NewFlagSet ("addflagstest" , pflag .ContinueOnError )
461- s , _ := NewKubeControllerManagerOptions ()
462- // flag set to parse the args that are required to start the kube controller manager
463- for _ , f := range s .Flags ([]string {"" }, []string {"" }, nil ).FlagSets {
464- fs .AddFlagSet (f )
465- }
464+ fs , s := setupControllerManagerFlagSet (t )
466465
467466 fs .Parse (args )
468467 // Sort GCIgnoredResources because it's built from a map, which means the
@@ -657,6 +656,96 @@ func TestApplyTo(t *testing.T) {
657656 }
658657}
659658
659+ func TestEmulatedVersion (t * testing.T ) {
660+ var cleanupAndSetupFunc = func () featuregate.FeatureGate {
661+ componentGlobalsRegistry := utilversion .DefaultComponentGlobalsRegistry
662+ componentGlobalsRegistry .Reset () // make sure this test have a clean state
663+ t .Cleanup (func () {
664+ componentGlobalsRegistry .Reset () // make sure this test doesn't leak a dirty state
665+ })
666+
667+ verKube := utilversion .NewEffectiveVersion ("1.32" )
668+ fg := featuregate .NewVersionedFeatureGate (version .MustParse ("1.32" ))
669+ utilruntime .Must (fg .AddVersioned (map [featuregate.Feature ]featuregate.VersionedSpecs {
670+ "kubeA" : {
671+ {Version : version .MustParse ("1.32" ), Default : true , LockToDefault : true , PreRelease : featuregate .GA },
672+ {Version : version .MustParse ("1.30" ), Default : false , PreRelease : featuregate .Beta },
673+ },
674+ "kubeB" : {
675+ {Version : version .MustParse ("1.31" ), Default : false , PreRelease : featuregate .Alpha },
676+ },
677+ }))
678+ utilruntime .Must (componentGlobalsRegistry .Register (utilversion .DefaultKubeComponent , verKube , fg ))
679+ return fg
680+ }
681+
682+ testcases := []struct {
683+ name string
684+ flags []string // not a good place to test flagParse error
685+ wantErr bool // this won't apply to flagParse, it only apply to KubeControllerManagerOptions.Validate
686+ errorSubString string
687+ wantFeaturesGates map [string ]bool
688+ }{
689+ {
690+ name : "default feature gates at binary version" ,
691+ flags : []string {},
692+ wantErr : false ,
693+ wantFeaturesGates : map [string ]bool {"kubeA" : true , "kubeB" : false },
694+ },
695+ {
696+ name : "emulating version out of range" ,
697+ flags : []string {
698+ "--emulated-version=1.28" ,
699+ },
700+ wantErr : true ,
701+ errorSubString : "emulation version 1.28 is not between" ,
702+ wantFeaturesGates : nil ,
703+ },
704+ {
705+ name : "default feature gates at emulated version" ,
706+ flags : []string {
707+ "--emulated-version=1.31" ,
708+ },
709+ wantFeaturesGates : map [string ]bool {"kubeA" : false , "kubeB" : false },
710+ },
711+ {
712+ name : "set feature gates at emulated version" ,
713+ flags : []string {
714+ "--emulated-version=1.31" ,
715+ "--feature-gates=kubeA=false,kubeB=true" ,
716+ },
717+ wantFeaturesGates : map [string ]bool {"kubeA" : false , "kubeB" : true },
718+ },
719+ {
720+ name : "cannot set locked feature gate" ,
721+ flags : []string {
722+ "--emulated-version=1.32" ,
723+ "--feature-gates=kubeA=false,kubeB=true" ,
724+ },
725+ errorSubString : "cannot set feature gate kubeA to false, feature is locked to true" ,
726+ wantErr : true ,
727+ },
728+ }
729+
730+ for _ , tc := range testcases {
731+ t .Run (tc .name , func (t * testing.T ) {
732+ fg := cleanupAndSetupFunc ()
733+
734+ fs , s := setupControllerManagerFlagSet (t )
735+ err := fs .Parse (tc .flags )
736+ checkTestError (t , err , false , "" )
737+ err = s .Validate ([]string {"" }, []string {"" }, nil )
738+ checkTestError (t , err , tc .wantErr , tc .errorSubString )
739+
740+ for feature , expected := range tc .wantFeaturesGates {
741+ if fg .Enabled (featuregate .Feature (feature )) != expected {
742+ t .Errorf ("expected %s to be %v" , feature , expected )
743+ }
744+ }
745+ })
746+ }
747+ }
748+
660749func TestValidateControllersOptions (t * testing.T ) {
661750 testCases := []struct {
662751 name string
@@ -1334,7 +1423,11 @@ func TestWatchListClientFlagUsage(t *testing.T) {
13341423
13351424func TestWatchListClientFlagChange (t * testing.T ) {
13361425 fs := pflag .NewFlagSet ("addflagstest" , pflag .ContinueOnError )
1337- s , _ := NewKubeControllerManagerOptions ()
1426+ s , err := NewKubeControllerManagerOptions ()
1427+ if err != nil {
1428+ t .Fatal (fmt .Errorf ("NewKubeControllerManagerOptions failed with %w" , err ))
1429+ }
1430+
13381431 for _ , f := range s .Flags ([]string {"" }, []string {"" }, nil ).FlagSets {
13391432 fs .AddFlagSet (f )
13401433 }
@@ -1344,7 +1437,13 @@ func TestWatchListClientFlagChange(t *testing.T) {
13441437
13451438 args := []string {fmt .Sprintf ("--feature-gates=%v=true" , clientgofeaturegate .WatchListClient )}
13461439 if err := fs .Parse (args ); err != nil {
1347- t .Fatal (err )
1440+ t .Fatal (fmt .Errorf ("FlatSet.Parse failed with %w" , err ))
1441+ }
1442+
1443+ // this is needed to Apply parsed flags to GlobalRegistry, so the DefaultFeatureGate values can be set from the flag
1444+ err = s .ComponentGlobalsRegistry .Set ()
1445+ if err != nil {
1446+ t .Fatal (fmt .Errorf ("ComponentGlobalsRegistry.Set failed with %w" , err ))
13481447 }
13491448
13501449 watchListClientValue := clientgofeaturegate .FeatureGates ().Enabled (clientgofeaturegate .WatchListClient )
@@ -1373,6 +1472,39 @@ func assertWatchListCommandLineDefaultValue(t *testing.T, fs *pflag.FlagSet) {
13731472 }
13741473}
13751474
1475+ func setupControllerManagerFlagSet (t * testing.T ) (* pflag.FlagSet , * KubeControllerManagerOptions ) {
1476+ fs := pflag .NewFlagSet ("addflagstest" , pflag .ContinueOnError )
1477+ s , err := NewKubeControllerManagerOptions ()
1478+ if err != nil {
1479+ t .Fatal (fmt .Errorf ("NewKubeControllerManagerOptions failed with %w" , err ))
1480+ }
1481+
1482+ for _ , f := range s .Flags ([]string {"" }, []string {"" }, nil ).FlagSets {
1483+ fs .AddFlagSet (f )
1484+ }
1485+ return fs , s
1486+ }
1487+
1488+ // caution: checkTestError use t.Fatal, to simplify caller handling.
1489+ // it also means it may break test code execution flow.
1490+ func checkTestError (t * testing.T , err error , expectingErr bool , expectedErrorSubString string ) {
1491+ if ! expectingErr {
1492+ if err != nil { // not expecting, but got error
1493+ t .Fatal (fmt .Errorf ("expected no error, got %w" , err ))
1494+ }
1495+ return // not expecting, and no error
1496+ }
1497+
1498+ // from this point we do expecting error
1499+ if err == nil {
1500+ t .Fatal ("expected error, got nil" )
1501+ }
1502+
1503+ if expectedErrorSubString != "" && ! strings .Contains (err .Error (), expectedErrorSubString ) {
1504+ t .Fatalf ("expected error to contain %q, but got %q" , expectedErrorSubString , err .Error ())
1505+ }
1506+ }
1507+
13761508type sortedGCIgnoredResources []garbagecollectorconfig.GroupResource
13771509
13781510func (r sortedGCIgnoredResources ) Len () int {
0 commit comments