add DefaultComponentGlobalsRegistry flags in ServerRunOptions

Signed-off-by: Siyuan Zhang <[email protected]>

Author: siyuanfoundation

cmd/kube-apiserver/app/options/options.go

@@ -26,8 +26,6 @@ import (
 	utilnet "k8s.io/apimachinery/pkg/util/net"
 	cliflag "k8s.io/component-base/cli/flag"
 
-	utilversion "k8s.io/apiserver/pkg/util/version"
-	"k8s.io/component-base/featuregate"
 	api "k8s.io/kubernetes/pkg/apis/core"
 	"k8s.io/kubernetes/pkg/cluster/ports"
 	controlplaneapiserver "k8s.io/kubernetes/pkg/controlplane/apiserver/options"
@@ -66,9 +64,9 @@ type Extra struct {
 }
 
 // NewServerRunOptions creates and returns ServerRunOptions according to the given featureGate and effectiveVersion of the server binary to run.
-func NewServerRunOptions(featureGate featuregate.FeatureGate, effectiveVersion utilversion.EffectiveVersion) *ServerRunOptions {
+func NewServerRunOptions() *ServerRunOptions {
 	s := ServerRunOptions{
-		Options:       controlplaneapiserver.NewOptions(featureGate, effectiveVersion),
+		Options:       controlplaneapiserver.NewOptions(),
 		CloudProvider: kubeoptions.NewCloudProviderOptions(),
 
 		Extra: Extra{

cmd/kube-apiserver/app/options/options_test.go

@@ -48,17 +48,17 @@ import (
 )
 
 func TestAddFlags(t *testing.T) {
+	componentGlobalsRegistry := utilversion.DefaultComponentGlobalsRegistry
+	t.Cleanup(func() {
+		componentGlobalsRegistry.Reset()
+	})
 	fs := pflag.NewFlagSet("addflagstest", pflag.PanicOnError)
 
-	featureGate := featuregate.NewFeatureGate()
-	componentRegistry := utilversion.NewComponentGlobalsRegistry()
-	effectiveVersion := utilversion.NewEffectiveVersion("1.32")
-	utilruntime.Must(componentRegistry.Register("test", effectiveVersion, featureGate))
-	s := NewServerRunOptions(featureGate, effectiveVersion)
+	utilruntime.Must(componentGlobalsRegistry.Register("test", utilversion.NewEffectiveVersion("1.32"), featuregate.NewFeatureGate()))
+	s := NewServerRunOptions()
 	for _, f := range s.Flags().FlagSets {
 		fs.AddFlagSet(f)
 	}
-	componentRegistry.AddFlags(fs)
 
 	args := []string{
 		"--enable-admission-plugins=AlwaysDeny",
@@ -133,7 +133,7 @@ func TestAddFlags(t *testing.T) {
 		"--emulated-version=test=1.31",
 	}
 	fs.Parse(args)
-	utilruntime.Must(componentRegistry.Set())
+	utilruntime.Must(componentGlobalsRegistry.Set())
 
 	// This is a snapshot of expected options parsed by args.
 	expected := &ServerRunOptions{
@@ -147,8 +147,8 @@ func TestAddFlags(t *testing.T) {
 				MinRequestTimeout:           1800,
 				JSONPatchMaxCopyBytes:       int64(3 * 1024 * 1024),
 				MaxRequestBodyBytes:         int64(3 * 1024 * 1024),
-				FeatureGate:                 featureGate,
-				EffectiveVersion:            effectiveVersion,
+				ComponentGlobalsRegistry:    componentGlobalsRegistry,
+				ComponentName:               utilversion.DefaultKubeComponent,
 			},
 			Admission: &kubeoptions.AdmissionOptions{
 				GenericAdmission: &apiserveroptions.AdmissionOptions{
@@ -350,8 +350,8 @@ func TestAddFlags(t *testing.T) {
 	if !reflect.DeepEqual(expected, s) {
 		t.Errorf("Got different run options than expected.\nDifference detected on:\n%s", cmp.Diff(expected, s, cmpopts.IgnoreUnexported(admission.Plugins{}, kubeoptions.OIDCAuthenticationOptions{})))
 	}
-
-	if s.GenericServerRunOptions.EffectiveVersion.EmulationVersion().String() != "1.31" {
-		t.Errorf("Got emulation version %s, wanted %s", s.GenericServerRunOptions.EffectiveVersion.EmulationVersion().String(), "1.31")
+	testEffectiveVersion := s.GenericServerRunOptions.ComponentGlobalsRegistry.EffectiveVersionFor("test")
+	if testEffectiveVersion.EmulationVersion().String() != "1.31" {
+		t.Errorf("Got emulation version %s, wanted %s", testEffectiveVersion.EmulationVersion().String(), "1.31")
 	}
 }

cmd/kube-apiserver/app/options/validation.go

@@ -26,6 +26,7 @@ import (
 	utilfeature "k8s.io/apiserver/pkg/util/feature"
 	netutils "k8s.io/utils/net"
 
+	"k8s.io/apimachinery/pkg/util/version"
 	controlplaneapiserver "k8s.io/kubernetes/pkg/controlplane/apiserver/options"
 	"k8s.io/kubernetes/pkg/controlplane/reconcilers"
 	"k8s.io/kubernetes/pkg/features"
@@ -139,5 +140,14 @@ func (s CompletedOptions) Validate() []error {
 		errs = append(errs, fmt.Errorf("--apiserver-count should be a positive number, but value '%d' provided", s.MasterCount))
 	}
 
+	// TODO: remove in 1.32
+	// emulationVersion is introduced in 1.31, so it is only allowed to be equal to the binary version at 1.31.
+	effectiveVersion := s.GenericServerRunOptions.ComponentGlobalsRegistry.EffectiveVersionFor(s.GenericServerRunOptions.ComponentName)
+	binaryVersion := version.MajorMinor(effectiveVersion.BinaryVersion().Major(), effectiveVersion.BinaryVersion().Minor())
+	if binaryVersion.EqualTo(version.MajorMinor(1, 31)) && !effectiveVersion.EmulationVersion().EqualTo(binaryVersion) {
+		errs = append(errs, fmt.Errorf("emulation version needs to be equal to binary version(%s) in compatibility-version alpha, got %s",
+			binaryVersion.String(), effectiveVersion.EmulationVersion().String()))
+	}
+
 	return errs
 }

cmd/kube-apiserver/app/server.go

@@ -64,9 +64,9 @@ func init() {
 
 // NewAPIServerCommand creates a *cobra.Command object with default parameters
 func NewAPIServerCommand() *cobra.Command {
-	effectiveVersion, featureGate := utilversion.DefaultComponentGlobalsRegistry.ComponentGlobalsOrRegister(
+	_, featureGate := utilversion.DefaultComponentGlobalsRegistry.ComponentGlobalsOrRegister(
 		utilversion.DefaultKubeComponent, utilversion.DefaultBuildEffectiveVersion(), utilfeature.DefaultMutableFeatureGate)
-	s := options.NewServerRunOptions(featureGate, effectiveVersion)
+	s := options.NewServerRunOptions()
 
 	cmd := &cobra.Command{
 		Use: "kube-apiserver",
@@ -124,8 +124,6 @@ cluster's shared state through which all other components interact.`,
 	fs := cmd.Flags()
 	namedFlagSets := s.Flags()
 	verflag.AddFlags(namedFlagSets.FlagSet("global"))
-	utilversion.DefaultComponentGlobalsRegistry.AddFlags(namedFlagSets.FlagSet("global"))
-
 	globalflag.AddGlobalFlags(namedFlagSets.FlagSet("global"), cmd.Name(), logs.SkipLoggingConfigurationFlags())
 	options.AddCustomGlobalFlags(namedFlagSets.FlagSet("generic"))
 	for _, f := range namedFlagSets.FlagSets {

cmd/kube-apiserver/app/testing/testserver.go

@@ -55,6 +55,7 @@ import (
 	clientgotransport "k8s.io/client-go/transport"
 	"k8s.io/client-go/util/cert"
 	"k8s.io/client-go/util/keyutil"
+	featuregatetesting "k8s.io/component-base/featuregate/testing"
 	logsapi "k8s.io/component-base/logs/api/v1"
 	"k8s.io/klog/v2"
 	"k8s.io/kube-aggregator/pkg/apiserver"
@@ -187,15 +188,16 @@ func StartTestServer(t ktesting.TB, instanceOptions *TestServerInstanceOptions,
 	if instanceOptions.BinaryVersion != "" {
 		effectiveVersion = utilversion.NewEffectiveVersion(instanceOptions.BinaryVersion)
 	}
+	// need to call SetFeatureGateEmulationVersionDuringTest to reset the feature gate emulation version at the end of the test.
+	featuregatetesting.SetFeatureGateEmulationVersionDuringTest(t, featureGate, effectiveVersion.EmulationVersion())
 	utilversion.DefaultComponentGlobalsRegistry.Reset()
 	utilruntime.Must(utilversion.DefaultComponentGlobalsRegistry.Register(utilversion.DefaultKubeComponent, effectiveVersion, featureGate))
 
-	s := options.NewServerRunOptions(featureGate, effectiveVersion)
+	s := options.NewServerRunOptions()
 
 	for _, f := range s.Flags().FlagSets {
 		fs.AddFlagSet(f)
 	}
-	utilversion.DefaultComponentGlobalsRegistry.AddFlags(fs)
 
 	s.SecureServing.Listener, s.SecureServing.BindPort, err = createLocalhostListenerOnFreePort()
 	if err != nil {

pkg/controlplane/apiserver/config_test.go

@@ -24,8 +24,6 @@ import (
 	"k8s.io/apimachinery/pkg/runtime"
 	"k8s.io/apimachinery/pkg/runtime/schema"
 	apiserveroptions "k8s.io/apiserver/pkg/server/options"
-	utilversion "k8s.io/apiserver/pkg/util/version"
-	"k8s.io/component-base/featuregate"
 	aggregatorscheme "k8s.io/kube-aggregator/pkg/apiserver/scheme"
 	"k8s.io/kubernetes/pkg/api/legacyscheme"
 	"k8s.io/kubernetes/pkg/controlplane/apiserver/options"
@@ -34,9 +32,7 @@ import (
 )
 
 func TestBuildGenericConfig(t *testing.T) {
-	featureGate := featuregate.NewFeatureGate()
-	effectiveVersion := utilversion.DefaultKubeEffectiveVersion()
-	opts := options.NewOptions(featureGate, effectiveVersion)
+	opts := options.NewOptions()
 	s := (&apiserveroptions.SecureServingOptions{
 		BindAddress: netutils.ParseIPSloppy("127.0.0.1"),
 	}).WithLoopback()

pkg/controlplane/apiserver/options/options.go

@@ -28,10 +28,8 @@ import (
 	peerreconcilers "k8s.io/apiserver/pkg/reconcilers"
 	genericoptions "k8s.io/apiserver/pkg/server/options"
 	"k8s.io/apiserver/pkg/storage/storagebackend"
-	utilversion "k8s.io/apiserver/pkg/util/version"
 	"k8s.io/client-go/util/keyutil"
 	cliflag "k8s.io/component-base/cli/flag"
-	"k8s.io/component-base/featuregate"
 	"k8s.io/component-base/logs"
 	logsapi "k8s.io/component-base/logs/api/v1"
 	"k8s.io/component-base/metrics"
@@ -100,9 +98,9 @@ type CompletedOptions struct {
 }
 
 // NewOptions creates a new ServerRunOptions object with default parameters
-func NewOptions(featureGate featuregate.FeatureGate, effectiveVersion utilversion.EffectiveVersion) *Options {
+func NewOptions() *Options {
 	s := Options{
-		GenericServerRunOptions: genericoptions.NewServerRunOptions(featureGate, effectiveVersion),
+		GenericServerRunOptions: genericoptions.NewServerRunOptions(),
 		Etcd:                    genericoptions.NewEtcdOptions(storagebackend.NewDefaultConfig(kubeoptions.DefaultEtcdPathPrefix, nil)),
 		SecureServing:           kubeoptions.NewSecureServingOptions(),
 		Audit:                   genericoptions.NewAuditOptions(),

pkg/controlplane/apiserver/options/options_test.go

@@ -44,18 +44,18 @@ import (
 )
 
 func TestAddFlags(t *testing.T) {
+	componentGlobalsRegistry := utilversion.DefaultComponentGlobalsRegistry
+	t.Cleanup(func() {
+		componentGlobalsRegistry.Reset()
+	})
 	fs := pflag.NewFlagSet("addflagstest", pflag.PanicOnError)
-	featureGate := featuregate.NewFeatureGate()
-	effectiveVersion := utilversion.NewEffectiveVersion("1.32")
-	componentRegistry := utilversion.NewComponentGlobalsRegistry()
-	utilruntime.Must(componentRegistry.Register("test", effectiveVersion, featureGate))
-	s := NewOptions(featureGate, effectiveVersion)
+	utilruntime.Must(componentGlobalsRegistry.Register("test", utilversion.NewEffectiveVersion("1.32"), featuregate.NewFeatureGate()))
+	s := NewOptions()
 	var fss cliflag.NamedFlagSets
 	s.AddFlags(&fss)
 	for _, f := range fss.FlagSets {
 		fs.AddFlagSet(f)
 	}
-	componentRegistry.AddFlags(fs)
 
 	args := []string{
 		"--enable-admission-plugins=AlwaysDeny",
@@ -119,7 +119,7 @@ func TestAddFlags(t *testing.T) {
 		"--emulated-version=test=1.31",
 	}
 	fs.Parse(args)
-	utilruntime.Must(componentRegistry.Set())
+	utilruntime.Must(componentGlobalsRegistry.Set())
 
 	// This is a snapshot of expected options parsed by args.
 	expected := &Options{
@@ -132,8 +132,8 @@ func TestAddFlags(t *testing.T) {
 			MinRequestTimeout:           1800,
 			JSONPatchMaxCopyBytes:       int64(3 * 1024 * 1024),
 			MaxRequestBodyBytes:         int64(3 * 1024 * 1024),
-			FeatureGate:                 featureGate,
-			EffectiveVersion:            effectiveVersion,
+			ComponentGlobalsRegistry:    componentGlobalsRegistry,
+			ComponentName:               utilversion.DefaultKubeComponent,
 		},
 		Admission: &kubeoptions.AdmissionOptions{
 			GenericAdmission: &apiserveroptions.AdmissionOptions{
@@ -305,7 +305,8 @@ func TestAddFlags(t *testing.T) {
 		t.Errorf("Got different run options than expected.\nDifference detected on:\n%s", cmp.Diff(expected, s, cmpopts.IgnoreUnexported(admission.Plugins{}, kubeoptions.OIDCAuthenticationOptions{})))
 	}
 
-	if s.GenericServerRunOptions.EffectiveVersion.EmulationVersion().String() != "1.31" {
-		t.Errorf("Got emulation version %s, wanted %s", s.GenericServerRunOptions.EffectiveVersion.EmulationVersion().String(), "1.31")
+	testEffectiveVersion := s.GenericServerRunOptions.ComponentGlobalsRegistry.EffectiveVersionFor("test")
+	if testEffectiveVersion.EmulationVersion().String() != "1.31" {
+		t.Errorf("Got emulation version %s, wanted %s", testEffectiveVersion.EmulationVersion().String(), "1.31")
 	}
 }

pkg/controlplane/apiserver/options/validation_test.go

@@ -201,7 +201,7 @@ func TestValidateOptions(t *testing.T) {
 			name:         "validate master count equal 0",
 			expectErrors: true,
 			options: &Options{
-				GenericServerRunOptions: &genericoptions.ServerRunOptions{FeatureGate: utilfeature.DefaultFeatureGate.DeepCopy(), EffectiveVersion: utilversion.NewEffectiveVersion("1.32")},
+				GenericServerRunOptions: &genericoptions.ServerRunOptions{ComponentGlobalsRegistry: utilversion.DefaultComponentGlobalsRegistry},
 				Etcd:                    &genericoptions.EtcdOptions{},
 				SecureServing:           &genericoptions.SecureServingOptionsWithLoopback{},
 				Audit:                   &genericoptions.AuditOptions{},
@@ -228,7 +228,7 @@ func TestValidateOptions(t *testing.T) {
 			name:         "validate token request enable not attempted",
 			expectErrors: true,
 			options: &Options{
-				GenericServerRunOptions: &genericoptions.ServerRunOptions{FeatureGate: utilfeature.DefaultFeatureGate.DeepCopy(), EffectiveVersion: utilversion.NewEffectiveVersion("1.32")},
+				GenericServerRunOptions: &genericoptions.ServerRunOptions{ComponentGlobalsRegistry: utilversion.DefaultComponentGlobalsRegistry},
 				Etcd:                    &genericoptions.EtcdOptions{},
 				SecureServing:           &genericoptions.SecureServingOptionsWithLoopback{},
 				Audit:                   &genericoptions.AuditOptions{},

staging/src/k8s.io/apiextensions-apiserver/pkg/cmd/server/options/options.go

@@ -39,11 +39,9 @@ import (
 	flowcontrolrequest "k8s.io/apiserver/pkg/util/flowcontrol/request"
 	"k8s.io/apiserver/pkg/util/openapi"
 	"k8s.io/apiserver/pkg/util/proxy"
-	utilversion "k8s.io/apiserver/pkg/util/version"
 	"k8s.io/apiserver/pkg/util/webhook"
 	scheme "k8s.io/client-go/kubernetes/scheme"
 	corev1 "k8s.io/client-go/listers/core/v1"
-	"k8s.io/component-base/featuregate"
 	netutils "k8s.io/utils/net"
 )
 
@@ -60,9 +58,9 @@ type CustomResourceDefinitionsServerOptions struct {
 }
 
 // NewCustomResourceDefinitionsServerOptions creates default options of an apiextensions-apiserver.
-func NewCustomResourceDefinitionsServerOptions(out, errOut io.Writer, featureGate featuregate.FeatureGate, effectiveVersion utilversion.EffectiveVersion) *CustomResourceDefinitionsServerOptions {
+func NewCustomResourceDefinitionsServerOptions(out, errOut io.Writer) *CustomResourceDefinitionsServerOptions {
 	o := &CustomResourceDefinitionsServerOptions{
-		ServerRunOptions: genericoptions.NewServerRunOptions(featureGate, effectiveVersion),
+		ServerRunOptions: genericoptions.NewServerRunOptions(),
 		RecommendedOptions: genericoptions.NewRecommendedOptions(
 			defaultEtcdPathPrefix,
 			apiserver.Codecs.LegacyCodec(v1beta1.SchemeGroupVersion, v1.SchemeGroupVersion),