Add option to explicitly enable future gv or gvr in runtime-config.

Signed-off-by: Siyuan Zhang <[email protected]>

Author: siyuanfoundation

cmd/kube-apiserver/app/options/options_test.go

@@ -129,6 +129,7 @@ func TestAddFlags(t *testing.T) {
 		"--lease-reuse-duration-seconds=100",
 		"--emulated-version=test=1.31",
 		"--emulation-forward-compatible=true",
+		"--runtime-config-emulation-forward-compatible=true",
 	}
 	fs.Parse(args)
 	utilruntime.Must(componentGlobalsRegistry.Set())
@@ -137,18 +138,19 @@ func TestAddFlags(t *testing.T) {
 	expected := &ServerRunOptions{
 		Options: &controlplaneapiserver.Options{
 			GenericServerRunOptions: &apiserveroptions.ServerRunOptions{
-				AdvertiseAddress:             netutils.ParseIPSloppy("192.168.10.10"),
-				CorsAllowedOriginList:        []string{"10.10.10.100", "10.10.10.200"},
-				MaxRequestsInFlight:          400,
-				MaxMutatingRequestsInFlight:  200,
-				RequestTimeout:               time.Duration(2) * time.Minute,
-				MinRequestTimeout:            1800,
-				StorageInitializationTimeout: time.Minute,
-				JSONPatchMaxCopyBytes:        int64(3 * 1024 * 1024),
-				MaxRequestBodyBytes:          int64(3 * 1024 * 1024),
-				ComponentGlobalsRegistry:     componentGlobalsRegistry,
-				ComponentName:                basecompatibility.DefaultKubeComponent,
-				EmulationForwardCompatible:   true,
+				AdvertiseAddress:                        netutils.ParseIPSloppy("192.168.10.10"),
+				CorsAllowedOriginList:                   []string{"10.10.10.100", "10.10.10.200"},
+				MaxRequestsInFlight:                     400,
+				MaxMutatingRequestsInFlight:             200,
+				RequestTimeout:                          time.Duration(2) * time.Minute,
+				MinRequestTimeout:                       1800,
+				StorageInitializationTimeout:            time.Minute,
+				JSONPatchMaxCopyBytes:                   int64(3 * 1024 * 1024),
+				MaxRequestBodyBytes:                     int64(3 * 1024 * 1024),
+				ComponentGlobalsRegistry:                componentGlobalsRegistry,
+				ComponentName:                           basecompatibility.DefaultKubeComponent,
+				EmulationForwardCompatible:              true,
+				RuntimeConfigEmulationForwardCompatible: true,
 			},
 			Admission: &kubeoptions.AdmissionOptions{
 				GenericAdmission: &apiserveroptions.AdmissionOptions{

pkg/controlplane/apiserver/apis.go

@@ -89,7 +89,12 @@ func (s *Server) InstallAPIs(restStorageProviders ...RESTStorageProvider) error
 	nonLegacy := []*genericapiserver.APIGroupInfo{}
 
 	// used later in the loop to filter the served resource by those that have expired.
-	resourceExpirationEvaluator, err := genericapiserver.NewResourceExpirationEvaluator(s.GenericAPIServer.EffectiveVersion.EmulationVersion(), s.GenericAPIServer.EmulationForwardCompatible)
+	resourceExpirationEvaluatorOpts := genericapiserver.ResourceExpirationEvaluatorOptions{
+		CurrentVersion:                          s.GenericAPIServer.EffectiveVersion.EmulationVersion(),
+		EmulationForwardCompatible:              s.GenericAPIServer.EmulationForwardCompatible,
+		RuntimeConfigEmulationForwardCompatible: s.GenericAPIServer.RuntimeConfigEmulationForwardCompatible,
+	}
+	resourceExpirationEvaluator, err := genericapiserver.NewResourceExpirationEvaluatorFromOptions(resourceExpirationEvaluatorOpts)
 	if err != nil {
 		return err
 	}
@@ -107,11 +112,13 @@ func (s *Server) InstallAPIs(restStorageProviders ...RESTStorageProvider) error
 			continue
 		}
 
-		// Remove resources that serving kinds that are removed.
+		// Remove resources that serving kinds that are removed or not introduced yet at the current version.
 		// We do this here so that we don't accidentally serve versions without resources or openapi information that for kinds we don't serve.
 		// This is a spot above the construction of individual storage handlers so that no sig accidentally forgets to check.
-		resourceExpirationEvaluator.RemoveDeletedKinds(groupName, apiGroupInfo.Scheme, apiGroupInfo.VersionedResourcesStorageMap)
-		resourceExpirationEvaluator.RemoveUnIntroducedKinds(groupName, apiGroupInfo.Scheme, apiGroupInfo.VersionedResourcesStorageMap)
+		err = resourceExpirationEvaluator.RemoveUnavailableKinds(groupName, apiGroupInfo.Scheme, apiGroupInfo.VersionedResourcesStorageMap, s.APIResourceConfigSource)
+		if err != nil {
+			return err
+		}
 		if len(apiGroupInfo.VersionedResourcesStorageMap) == 0 {
 			klog.V(1).Infof("Removing API group %v because it is time to stop serving it because it has no versions per APILifecycle.", groupName)
 			continue

staging/src/k8s.io/apiserver/pkg/server/config.go

@@ -154,10 +154,15 @@ type Config struct {
 	// EffectiveVersion determines which apis and features are available
 	// based on when the api/feature lifecyle.
 	EffectiveVersion basecompatibility.EffectiveVersion
-	// EmulationForwardCompatible indicates APIs introduced after the emulation version are installed.
-	// If true, APIs that have higher priority than the APIs of the same group resource enabled at the emulation version will be installed.
-	// This is useful if a controller has switched to use newer APIs in the binary version, and we want it still functional in an older emulation version.
+	// EmulationForwardCompatible is an option to implicitly enable all APIs which are introduced after the emulation version and
+	// have higher priority than APIs of the same group resource enabled at the emulation version.
+	// If true, all APIs that have higher priority than the APIs of the same group resource enabled at the emulation version will be installed.
+	// This is needed when a controller implementation migrates to newer API versions, for the binary version, and also uses the newer API versions even when emulation version is set.
 	EmulationForwardCompatible bool
+	// RuntimeConfigEmulationForwardCompatible is an option to explicitly enable specific APIs introduced after the emulation version through the runtime-config.
+	// If true, APIs identified by group/version that are enabled in the --runtime-config flag will be installed even if it is introduced after the emulation version. --runtime-config flag values that identify multiple APIs, such as api/all,api/ga,api/beta, are not influenced by this flag and will only enable APIs available at the current emulation version.
+	// If false, error would be thrown if any GroupVersion or GroupVersionResource explicitly enabled in the --runtime-config flag is introduced after the emulation version.
+	RuntimeConfigEmulationForwardCompatible bool
 	// FeatureGate is a way to plumb feature gate through if you have them.
 	FeatureGate featuregate.FeatureGate
 	// AuditBackend is where audit events are sent to.
@@ -843,9 +848,10 @@ func (c completedConfig) New(name string, delegationTarget DelegationTarget) (*G
 		StorageReadinessHook:  NewStorageReadinessHook(c.StorageInitializationTimeout),
 		StorageVersionManager: c.StorageVersionManager,
 
-		EffectiveVersion:           c.EffectiveVersion,
-		EmulationForwardCompatible: c.EmulationForwardCompatible,
-		FeatureGate:                c.FeatureGate,
+		EffectiveVersion:                        c.EffectiveVersion,
+		EmulationForwardCompatible:              c.EmulationForwardCompatible,
+		RuntimeConfigEmulationForwardCompatible: c.RuntimeConfigEmulationForwardCompatible,
+		FeatureGate:                             c.FeatureGate,
 
 		muxAndDiscoveryCompleteSignals: map[string]<-chan struct{}{},
 	}