apiserver: Add API emulation versioning.

Co-authored-by: Siyuan Zhang <[email protected]>
Co-authored-by: Joe Betz <[email protected]>
Co-authored-by: Alex Zielenski <[email protected]>

Signed-off-by: Siyuan Zhang <[email protected]>

Author: siyuanfoundation

cmd/kube-apiserver/app/options/options.go

@@ -26,6 +26,8 @@ import (
 	utilnet "k8s.io/apimachinery/pkg/util/net"
 	cliflag "k8s.io/component-base/cli/flag"
 
+	utilversion "k8s.io/apiserver/pkg/util/version"
+	"k8s.io/component-base/featuregate"
 	api "k8s.io/kubernetes/pkg/apis/core"
 	"k8s.io/kubernetes/pkg/cluster/ports"
 	controlplaneapiserver "k8s.io/kubernetes/pkg/controlplane/apiserver/options"
@@ -63,10 +65,10 @@ type Extra struct {
 	MasterCount int
 }
 
-// NewServerRunOptions creates a new ServerRunOptions object with default parameters
-func NewServerRunOptions() *ServerRunOptions {
+// NewServerRunOptions creates and returns ServerRunOptions according to the given featureGate and effectiveVersion of the server binary to run.
+func NewServerRunOptions(featureGate featuregate.FeatureGate, effectiveVersion utilversion.EffectiveVersion) *ServerRunOptions {
 	s := ServerRunOptions{
-		Options:       controlplaneapiserver.NewOptions(),
+		Options:       controlplaneapiserver.NewOptions(featureGate, effectiveVersion),
 		CloudProvider: kubeoptions.NewCloudProviderOptions(),
 
 		Extra: Extra{

cmd/kube-apiserver/app/options/options_test.go

@@ -31,9 +31,11 @@ import (
 	apiserveroptions "k8s.io/apiserver/pkg/server/options"
 	"k8s.io/apiserver/pkg/storage/etcd3"
 	"k8s.io/apiserver/pkg/storage/storagebackend"
+	utilversion "k8s.io/apiserver/pkg/util/version"
 	auditbuffered "k8s.io/apiserver/plugin/pkg/audit/buffered"
 	audittruncate "k8s.io/apiserver/plugin/pkg/audit/truncate"
 	cliflag "k8s.io/component-base/cli/flag"
+	"k8s.io/component-base/featuregate"
 	"k8s.io/component-base/logs"
 	"k8s.io/component-base/metrics"
 	kapi "k8s.io/kubernetes/pkg/apis/core"
@@ -46,10 +48,15 @@ import (
 
 func TestAddFlags(t *testing.T) {
 	fs := pflag.NewFlagSet("addflagstest", pflag.PanicOnError)
-	s := NewServerRunOptions()
+
+	featureGate := featuregate.NewFeatureGate()
+	effectiveVersion := utilversion.NewEffectiveVersion("1.32")
+	s := NewServerRunOptions(featureGate, effectiveVersion)
 	for _, f := range s.Flags().FlagSets {
 		fs.AddFlagSet(f)
 	}
+	featureGate.AddFlag(fs, "")
+	effectiveVersion.AddFlags(fs, "")
 
 	args := []string{
 		"--enable-admission-plugins=AlwaysDeny",
@@ -121,6 +128,7 @@ func TestAddFlags(t *testing.T) {
 		"--storage-backend=etcd3",
 		"--service-cluster-ip-range=192.168.128.0/17",
 		"--lease-reuse-duration-seconds=100",
+		"--emulated-version=1.31",
 	}
 	fs.Parse(args)
 
@@ -136,6 +144,8 @@ func TestAddFlags(t *testing.T) {
 				MinRequestTimeout:           1800,
 				JSONPatchMaxCopyBytes:       int64(3 * 1024 * 1024),
 				MaxRequestBodyBytes:         int64(3 * 1024 * 1024),
+				FeatureGate:                 featureGate,
+				EffectiveVersion:            effectiveVersion,
 			},
 			Admission: &kubeoptions.AdmissionOptions{
 				GenericAdmission: &apiserveroptions.AdmissionOptions{
@@ -337,4 +347,8 @@ func TestAddFlags(t *testing.T) {
 	if !reflect.DeepEqual(expected, s) {
 		t.Errorf("Got different run options than expected.\nDifference detected on:\n%s", cmp.Diff(expected, s, cmpopts.IgnoreUnexported(admission.Plugins{}, kubeoptions.OIDCAuthenticationOptions{})))
 	}
+
+	if s.GenericServerRunOptions.EffectiveVersion.EmulationVersion().String() != "1.31" {
+		t.Errorf("Got emulation version %s, wanted %s", s.GenericServerRunOptions.EffectiveVersion.EmulationVersion().String(), "1.31")
+	}
 }

cmd/kube-apiserver/app/server.go

@@ -36,6 +36,7 @@ import (
 	serverstorage "k8s.io/apiserver/pkg/server/storage"
 	utilfeature "k8s.io/apiserver/pkg/util/feature"
 	"k8s.io/apiserver/pkg/util/notfoundhandler"
+	utilversion "k8s.io/apiserver/pkg/util/version"
 	"k8s.io/apiserver/pkg/util/webhook"
 	clientgoinformers "k8s.io/client-go/informers"
 	"k8s.io/client-go/rest"
@@ -63,7 +64,10 @@ func init() {
 
 // NewAPIServerCommand creates a *cobra.Command object with default parameters
 func NewAPIServerCommand() *cobra.Command {
-	s := options.NewServerRunOptions()
+	effectiveVersion, featureGate := utilversion.DefaultComponentGlobalsRegistry.ComponentGlobalsOrRegister(
+		utilversion.ComponentGenericAPIServer, utilversion.DefaultBuildEffectiveVersion(), utilfeature.DefaultMutableFeatureGate)
+	s := options.NewServerRunOptions(featureGate, effectiveVersion)
+
 	cmd := &cobra.Command{
 		Use: "kube-apiserver",
 		Long: `The Kubernetes API server validates and configures data
@@ -83,9 +87,13 @@ cluster's shared state through which all other components interact.`,
 			verflag.PrintAndExitIfRequested()
 			fs := cmd.Flags()
 
+			if err := utilversion.DefaultComponentGlobalsRegistry.SetAllComponents(); err != nil {
+				return err
+			}
+
 			// Activate logging as soon as possible, after that
 			// show flags with the final logging configuration.
-			if err := logsapi.ValidateAndApply(s.Logs, utilfeature.DefaultFeatureGate); err != nil {
+			if err := logsapi.ValidateAndApply(s.Logs, featureGate); err != nil {
 				return err
 			}
 			cliflag.PrintFlags(fs)
@@ -101,7 +109,7 @@ cluster's shared state through which all other components interact.`,
 				return utilerrors.NewAggregate(errs)
 			}
 			// add feature enablement metrics
-			utilfeature.DefaultMutableFeatureGate.AddMetrics()
+			featureGate.AddMetrics()
 			return Run(cmd.Context(), completedOptions)
 		},
 		Args: func(cmd *cobra.Command, args []string) error {
@@ -118,6 +126,9 @@ cluster's shared state through which all other components interact.`,
 	fs := cmd.Flags()
 	namedFlagSets := s.Flags()
 	verflag.AddFlags(namedFlagSets.FlagSet("global"))
+	featureGate.AddFlag(namedFlagSets.FlagSet("global"), "")
+	effectiveVersion.AddFlags(namedFlagSets.FlagSet("global"), "")
+
 	globalflag.AddGlobalFlags(namedFlagSets.FlagSet("global"), cmd.Name(), logs.SkipLoggingConfigurationFlags())
 	options.AddCustomGlobalFlags(namedFlagSets.FlagSet("generic"))
 	for _, f := range namedFlagSets.FlagSets {

cmd/kube-apiserver/app/testing/testserver.go

@@ -48,6 +48,7 @@ import (
 	"k8s.io/apiserver/pkg/storage/storagebackend"
 	"k8s.io/apiserver/pkg/storageversion"
 	utilfeature "k8s.io/apiserver/pkg/util/feature"
+	utilversion "k8s.io/apiserver/pkg/util/version"
 	"k8s.io/client-go/kubernetes"
 	restclient "k8s.io/client-go/rest"
 	clientgotransport "k8s.io/client-go/transport"
@@ -98,6 +99,9 @@ type TestServerInstanceOptions struct {
 	// We specify this as on option to pass a common proxyCA to multiple apiservers to simulate
 	// an apiserver version skew scenario where all apiservers use the same proxyCA to verify client connections.
 	ProxyCA *ProxyCA
+	// Set the BinaryVersion of server effective version.
+	// Default to 1.31
+	BinaryVersion string
 }
 
 // TestServer return values supplied by kube-test-ApiServer
@@ -177,10 +181,21 @@ func StartTestServer(t ktesting.TB, instanceOptions *TestServerInstanceOptions,
 
 	fs := pflag.NewFlagSet("test", pflag.PanicOnError)
 
-	s := options.NewServerRunOptions()
+	featureGate := utilfeature.DefaultMutableFeatureGate
+	binaryVersion := utilversion.DefaultKubeEffectiveVersion().BinaryVersion().String()
+	if instanceOptions.BinaryVersion != "" {
+		binaryVersion = instanceOptions.BinaryVersion
+	}
+	effectiveVersion := utilversion.NewEffectiveVersion(binaryVersion)
+	_ = utilversion.DefaultComponentGlobalsRegistry.Register(utilversion.ComponentGenericAPIServer, effectiveVersion, featureGate, true)
+
+	s := options.NewServerRunOptions(featureGate, effectiveVersion)
+
 	for _, f := range s.Flags().FlagSets {
 		fs.AddFlagSet(f)
 	}
+	featureGate.AddFlag(fs, "")
+	effectiveVersion.AddFlags(fs, "")
 
 	s.SecureServing.Listener, s.SecureServing.BindPort, err = createLocalhostListenerOnFreePort()
 	if err != nil {
@@ -321,6 +336,10 @@ func StartTestServer(t ktesting.TB, instanceOptions *TestServerInstanceOptions,
 		return result, err
 	}
 
+	if err := utilversion.DefaultComponentGlobalsRegistry.SetAllComponents(); err != nil {
+		return result, err
+	}
+
 	saSigningKeyFile, err := os.CreateTemp("/tmp", "insecure_test_key")
 	if err != nil {
 		t.Fatalf("create temp file failed: %v", err)

cmd/kube-controller-manager/app/options/options.go

@@ -273,7 +273,7 @@ func (s *KubeControllerManagerOptions) Flags(allControllers []string, disabledBy
 	fs := fss.FlagSet("misc")
 	fs.StringVar(&s.Master, "master", s.Master, "The address of the Kubernetes API server (overrides any value in kubeconfig).")
 	fs.StringVar(&s.Generic.ClientConnection.Kubeconfig, "kubeconfig", s.Generic.ClientConnection.Kubeconfig, "Path to kubeconfig file with authorization and master location information (the master location can be overridden by the master flag).")
-	utilfeature.DefaultMutableFeatureGate.AddFlag(fss.FlagSet("generic"))
+	utilfeature.DefaultMutableFeatureGate.AddFlag(fss.FlagSet("generic"), "")
 
 	return fss
 }

cmd/kube-scheduler/app/options/options.go

@@ -189,7 +189,7 @@ func (o *Options) initFlags() {
 	o.Authorization.AddFlags(nfs.FlagSet("authorization"))
 	o.Deprecated.AddFlags(nfs.FlagSet("deprecated"))
 	options.BindLeaderElectionFlags(o.LeaderElection, nfs.FlagSet("leader election"))
-	utilfeature.DefaultMutableFeatureGate.AddFlag(nfs.FlagSet("feature gate"))
+	utilfeature.DefaultMutableFeatureGate.AddFlag(nfs.FlagSet("feature gate"), "")
 	o.Metrics.AddFlags(nfs.FlagSet("metrics"))
 	logsapi.AddFlags(o.Logs, nfs.FlagSet("logs"))
 

pkg/controlplane/apiserver/apiextensions.go

@@ -17,7 +17,6 @@ limitations under the License.
 package apiserver
 
 import (
-	"k8s.io/apiextensions-apiserver/pkg/apis/apiextensions/v1"
 	"k8s.io/apiextensions-apiserver/pkg/apis/apiextensions/v1beta1"
 	apiextensionsapiserver "k8s.io/apiextensions-apiserver/pkg/apiserver"
 	apiextensionsoptions "k8s.io/apiextensions-apiserver/pkg/cmd/server/options"
@@ -27,6 +26,7 @@ import (
 	"k8s.io/apiserver/pkg/server"
 	"k8s.io/apiserver/pkg/util/webhook"
 	"k8s.io/client-go/informers"
+	v1 "k8s.io/kube-aggregator/pkg/apis/apiregistration/v1"
 
 	"k8s.io/kubernetes/pkg/controlplane/apiserver/options"
 )

pkg/controlplane/apiserver/apis.go

@@ -88,7 +88,7 @@ func (s *Server) InstallAPIs(restStorageProviders ...RESTStorageProvider) error
 	nonLegacy := []*genericapiserver.APIGroupInfo{}
 
 	// used later in the loop to filter the served resource by those that have expired.
-	resourceExpirationEvaluator, err := genericapiserver.NewResourceExpirationEvaluator(*s.GenericAPIServer.Version)
+	resourceExpirationEvaluator, err := genericapiserver.NewResourceExpirationEvaluator(s.GenericAPIServer.EffectiveVersion.EmulationVersion())
 	if err != nil {
 		return err
 	}

pkg/controlplane/apiserver/config.go

@@ -185,7 +185,9 @@ func BuildGenericConfig(
 	}
 
 	storageFactoryConfig := kubeapiserver.NewStorageFactoryConfig()
+	storageFactoryConfig.CurrentVersion = genericConfig.EffectiveVersion
 	storageFactoryConfig.APIResourceConfig = genericConfig.MergedResourceConfig
+	storageFactoryConfig.DefaultResourceEncoding.SetEffectiveVersion(genericConfig.EffectiveVersion)
 	storageFactory, lastErr = storageFactoryConfig.Complete(s.Etcd).New()
 	if lastErr != nil {
 		return

pkg/controlplane/apiserver/config_test.go

@@ -24,6 +24,8 @@ import (
 	"k8s.io/apimachinery/pkg/runtime"
 	"k8s.io/apimachinery/pkg/runtime/schema"
 	apiserveroptions "k8s.io/apiserver/pkg/server/options"
+	utilversion "k8s.io/apiserver/pkg/util/version"
+	"k8s.io/component-base/featuregate"
 	aggregatorscheme "k8s.io/kube-aggregator/pkg/apiserver/scheme"
 	"k8s.io/kubernetes/pkg/api/legacyscheme"
 	"k8s.io/kubernetes/pkg/controlplane/apiserver/options"
@@ -32,7 +34,9 @@ import (
 )
 
 func TestBuildGenericConfig(t *testing.T) {
-	opts := options.NewOptions()
+	featureGate := featuregate.NewFeatureGate()
+	effectiveVersion := utilversion.DefaultKubeEffectiveVersion()
+	opts := options.NewOptions(featureGate, effectiveVersion)
 	s := (&apiserveroptions.SecureServingOptions{
 		BindAddress: netutils.ParseIPSloppy("127.0.0.1"),
 	}).WithLoopback()
@@ -66,7 +70,7 @@ func TestBuildGenericConfig(t *testing.T) {
 		t.Errorf("There are different StorageObjectCountTracker in genericConfig and storageFactory")
 	}
 
-	restOptions, err := genericConfig.RESTOptionsGetter.GetRESTOptions(schema.GroupResource{Group: "", Resource: ""})
+	restOptions, err := genericConfig.RESTOptionsGetter.GetRESTOptions(schema.GroupResource{Group: "", Resource: ""}, nil)
 	if err != nil {
 		t.Fatal(err)
 	}