Add emulation forward compatibility into api enablement and RemoveDeletedKinds.

Signed-off-by: Siyuan Zhang <[email protected]>

Author: siyuanfoundation

cmd/kube-apiserver/app/options/options_test.go

@@ -128,6 +128,7 @@ func TestAddFlags(t *testing.T) {
 		"--service-cluster-ip-range=192.168.128.0/17",
 		"--lease-reuse-duration-seconds=100",
 		"--emulated-version=test=1.31",
+		"--emulation-forward-compatible=true",
 	}
 	fs.Parse(args)
 	utilruntime.Must(componentGlobalsRegistry.Set())
@@ -147,6 +148,7 @@ func TestAddFlags(t *testing.T) {
 				MaxRequestBodyBytes:          int64(3 * 1024 * 1024),
 				ComponentGlobalsRegistry:     componentGlobalsRegistry,
 				ComponentName:                basecompatibility.DefaultKubeComponent,
+				EmulationForwardCompatible:   true,
 			},
 			Admission: &kubeoptions.AdmissionOptions{
 				GenericAdmission: &apiserveroptions.AdmissionOptions{

pkg/controlplane/apiserver/apis.go

@@ -89,7 +89,7 @@ func (s *Server) InstallAPIs(restStorageProviders ...RESTStorageProvider) error
 	nonLegacy := []*genericapiserver.APIGroupInfo{}
 
 	// used later in the loop to filter the served resource by those that have expired.
-	resourceExpirationEvaluator, err := genericapiserver.NewResourceExpirationEvaluator(s.GenericAPIServer.EffectiveVersion.EmulationVersion())
+	resourceExpirationEvaluator, err := genericapiserver.NewResourceExpirationEvaluator(s.GenericAPIServer.EffectiveVersion.EmulationVersion(), s.GenericAPIServer.EmulationForwardCompatible)
 	if err != nil {
 		return err
 	}
@@ -111,6 +111,7 @@ func (s *Server) InstallAPIs(restStorageProviders ...RESTStorageProvider) error
 		// We do this here so that we don't accidentally serve versions without resources or openapi information that for kinds we don't serve.
 		// This is a spot above the construction of individual storage handlers so that no sig accidentally forgets to check.
 		resourceExpirationEvaluator.RemoveDeletedKinds(groupName, apiGroupInfo.Scheme, apiGroupInfo.VersionedResourcesStorageMap)
+		resourceExpirationEvaluator.RemoveUnIntroducedKinds(groupName, apiGroupInfo.Scheme, apiGroupInfo.VersionedResourcesStorageMap)
 		if len(apiGroupInfo.VersionedResourcesStorageMap) == 0 {
 			klog.V(1).Infof("Removing API group %v because it is time to stop serving it because it has no versions per APILifecycle.", groupName)
 			continue

staging/src/k8s.io/apimachinery/pkg/runtime/interfaces.go

@@ -259,6 +259,7 @@ type ObjectDefaulter interface {
 
 type ObjectVersioner interface {
 	ConvertToVersion(in Object, gv GroupVersioner) (out Object, err error)
+	PrioritizedVersionsForGroup(group string) []schema.GroupVersion
 }
 
 // ObjectConvertor converts an object to a different version.

staging/src/k8s.io/apiserver/pkg/server/config.go

@@ -154,6 +154,10 @@ type Config struct {
 	// EffectiveVersion determines which apis and features are available
 	// based on when the api/feature lifecyle.
 	EffectiveVersion basecompatibility.EffectiveVersion
+	// EmulationForwardCompatible indicates APIs introduced after the emulation version are installed.
+	// If true, APIs that have higher priority than the APIs of the same group resource enabled at the emulation version will be installed.
+	// This is useful if a controller has switched to use newer APIs in the binary version, and we want it still functional in an older emulation version.
+	EmulationForwardCompatible bool
 	// FeatureGate is a way to plumb feature gate through if you have them.
 	FeatureGate featuregate.FeatureGate
 	// AuditBackend is where audit events are sent to.
@@ -839,8 +843,9 @@ func (c completedConfig) New(name string, delegationTarget DelegationTarget) (*G
 		StorageReadinessHook:  NewStorageReadinessHook(c.StorageInitializationTimeout),
 		StorageVersionManager: c.StorageVersionManager,
 
-		EffectiveVersion: c.EffectiveVersion,
-		FeatureGate:      c.FeatureGate,
+		EffectiveVersion:           c.EffectiveVersion,
+		EmulationForwardCompatible: c.EmulationForwardCompatible,
+		FeatureGate:                c.FeatureGate,
 
 		muxAndDiscoveryCompleteSignals: map[string]<-chan struct{}{},
 	}

staging/src/k8s.io/apiserver/pkg/server/deleted_kinds.go

@@ -19,6 +19,7 @@ package server
 import (
 	"fmt"
 	"os"
+	"regexp"
 	"strconv"
 	"strings"
 
@@ -31,10 +32,13 @@ import (
 	"k8s.io/klog/v2"
 )
 
+var alphaPattern = regexp.MustCompile(`^v\d+alpha\d+$`)
+
 // resourceExpirationEvaluator holds info for deciding if a particular rest.Storage needs to excluded from the API
 type resourceExpirationEvaluator struct {
-	currentVersion *apimachineryversion.Version
-	isAlpha        bool
+	currentVersion             *apimachineryversion.Version
+	emulationForwardCompatible bool
+	isAlpha                    bool
 	// Special flag checking for the existence of alpha.0
 	// alpha.0 is a special case where everything merged to master is auto propagated to the release-1.n branch
 	isAlphaZero bool
@@ -53,17 +57,21 @@ type ResourceExpirationEvaluator interface {
 	// RemoveDeletedKinds inspects the storage map and modifies it in place by removing storage for kinds that have been deleted.
 	// versionedResourcesStorageMap mirrors the field on APIGroupInfo, it's a map from version to resource to the storage.
 	RemoveDeletedKinds(groupName string, versioner runtime.ObjectVersioner, versionedResourcesStorageMap map[string]map[string]rest.Storage)
+	// RemoveUnIntroducedKinds inspects the storage map and modifies it in place by removing storage for kinds that are introduced after the current version.
+	// versionedResourcesStorageMap mirrors the field on APIGroupInfo, it's a map from version to resource to the storage.
+	RemoveUnIntroducedKinds(groupName string, versioner runtime.ObjectVersioner, versionedResourcesStorageMap map[string]map[string]rest.Storage)
 	// ShouldServeForVersion returns true if a particular version cut off is after the current version
 	ShouldServeForVersion(majorRemoved, minorRemoved int) bool
 }
 
-func NewResourceExpirationEvaluator(currentVersion *apimachineryversion.Version) (ResourceExpirationEvaluator, error) {
+func NewResourceExpirationEvaluator(currentVersion *apimachineryversion.Version, emulationForwardCompatible bool) (ResourceExpirationEvaluator, error) {
 	if currentVersion == nil {
 		return nil, fmt.Errorf("empty NewResourceExpirationEvaluator currentVersion")
 	}
 	klog.V(1).Infof("NewResourceExpirationEvaluator with currentVersion: %s.", currentVersion)
 	ret := &resourceExpirationEvaluator{
 		strictRemovedHandlingInAlpha: false,
+		emulationForwardCompatible:   emulationForwardCompatible,
 	}
 	// Only keeps the major and minor versions from input version.
 	ret.currentVersion = apimachineryversion.MajorMinor(currentVersion.Major(), currentVersion.Minor())
@@ -89,7 +97,7 @@ func NewResourceExpirationEvaluator(currentVersion *apimachineryversion.Version)
 	return ret, nil
 }
 
-func (e *resourceExpirationEvaluator) shouldServe(gv schema.GroupVersion, versioner runtime.ObjectVersioner, resourceServingInfo rest.Storage) bool {
+func (e *resourceExpirationEvaluator) isNotRemoved(gv schema.GroupVersion, versioner runtime.ObjectVersioner, resourceServingInfo rest.Storage) bool {
 	internalPtr := resourceServingInfo.New()
 
 	target := gv
@@ -104,15 +112,6 @@ func (e *resourceExpirationEvaluator) shouldServe(gv schema.GroupVersion, versio
 		return false
 	}
 
-	introduced, ok := versionedPtr.(introducedInterface)
-	if ok {
-		majorIntroduced, minorIntroduced := introduced.APILifecycleIntroduced()
-		verIntroduced := apimachineryversion.MajorMinor(uint(majorIntroduced), uint(minorIntroduced))
-		if e.currentVersion.LessThan(verIntroduced) {
-			return false
-		}
-	}
-
 	removed, ok := versionedPtr.(removedInterface)
 	if !ok {
 		return true
@@ -153,15 +152,15 @@ type introducedInterface interface {
 	APILifecycleIntroduced() (major, minor int)
 }
 
-// removeDeletedKinds inspects the storage map and modifies it in place by removing storage for kinds that have been deleted.
+// RemoveDeletedKinds inspects the storage map and modifies it in place by removing storage for kinds that have been deleted.
 // versionedResourcesStorageMap mirrors the field on APIGroupInfo, it's a map from version to resource to the storage.
 func (e *resourceExpirationEvaluator) RemoveDeletedKinds(groupName string, versioner runtime.ObjectVersioner, versionedResourcesStorageMap map[string]map[string]rest.Storage) {
 	versionsToRemove := sets.NewString()
 	for apiVersion := range sets.StringKeySet(versionedResourcesStorageMap) {
 		versionToResource := versionedResourcesStorageMap[apiVersion]
 		resourcesToRemove := sets.NewString()
 		for resourceName, resourceServingInfo := range versionToResource {
-			if !e.shouldServe(schema.GroupVersion{Group: groupName, Version: apiVersion}, versioner, resourceServingInfo) {
+			if !e.isNotRemoved(schema.GroupVersion{Group: groupName, Version: apiVersion}, versioner, resourceServingInfo) {
 				resourcesToRemove.Insert(resourceName)
 			}
 		}
@@ -189,6 +188,82 @@ func (e *resourceExpirationEvaluator) RemoveDeletedKinds(groupName string, versi
 	}
 }
 
+// RemoveUnIntroducedKinds inspects the storage map and modifies it in place by removing storage for kinds that are introduced after the current version.
+// versionedResourcesStorageMap mirrors the field on APIGroupInfo, it's a map from version to resource to the storage.
+func (e *resourceExpirationEvaluator) RemoveUnIntroducedKinds(groupName string, versioner runtime.ObjectVersioner, versionedResourcesStorageMap map[string]map[string]rest.Storage) {
+	versionsToRemove := sets.NewString()
+	prioritizedVersions := versioner.PrioritizedVersionsForGroup(groupName)
+	enabledResources := sets.NewString()
+
+	// iterate from the end to the front, so that we remove the older versions first.
+	for i := len(prioritizedVersions) - 1; i >= 0; i-- {
+		apiVersion := prioritizedVersions[i].Version
+		versionToResource := versionedResourcesStorageMap[apiVersion]
+		resourcesToRemove := sets.NewString()
+		for resourceName, resourceServingInfo := range versionToResource {
+			// if an earlier version of the resource has been enabled, the same resource with higher priority
+			// should also be enabled if emulationForwardCompatible.
+			if e.emulationForwardCompatible && enabledResources.Has(resourceName) {
+				continue
+			}
+			verIntroduced := versionIntroduced(schema.GroupVersion{Group: groupName, Version: apiVersion}, versioner, resourceServingInfo)
+			if e.currentVersion.LessThan(verIntroduced) {
+				resourcesToRemove.Insert(resourceName)
+			} else {
+				// emulation forward compatibility is not applicable to alpha apis.
+				if !alphaPattern.MatchString(apiVersion) {
+					enabledResources.Insert(resourceName)
+				}
+			}
+		}
+
+		for resourceName := range versionedResourcesStorageMap[apiVersion] {
+			if !shouldRemoveResourceAndSubresources(resourcesToRemove, resourceName) {
+				continue
+			}
+
+			klog.V(1).Infof("Removing resource %v.%v.%v because it is introduced after the current version %s per APILifecycle.", resourceName, apiVersion, groupName, e.currentVersion.String())
+			storage := versionToResource[resourceName]
+			storage.Destroy()
+			delete(versionToResource, resourceName)
+		}
+		versionedResourcesStorageMap[apiVersion] = versionToResource
+
+		if len(versionedResourcesStorageMap[apiVersion]) == 0 {
+			versionsToRemove.Insert(apiVersion)
+		}
+	}
+
+	for _, apiVersion := range versionsToRemove.List() {
+		klog.V(1).Infof("Removing version %v.%v because it is introduced after the current version %s and because it has no resources per APILifecycle.", apiVersion, groupName, e.currentVersion.String())
+		delete(versionedResourcesStorageMap, apiVersion)
+	}
+}
+
+func versionIntroduced(gv schema.GroupVersion, versioner runtime.ObjectVersioner, resourceServingInfo rest.Storage) *apimachineryversion.Version {
+	defaultVer := apimachineryversion.MajorMinor(0, 0)
+	internalPtr := resourceServingInfo.New()
+
+	target := gv
+	// honor storage that overrides group version (used for things like scale subresources)
+	if versionProvider, ok := resourceServingInfo.(rest.GroupVersionKindProvider); ok {
+		target = versionProvider.GroupVersionKind(target).GroupVersion()
+	}
+
+	versionedPtr, err := versioner.ConvertToVersion(internalPtr, target)
+	if err != nil {
+		utilruntime.HandleError(err)
+		return defaultVer
+	}
+
+	introduced, ok := versionedPtr.(introducedInterface)
+	if ok {
+		majorIntroduced, minorIntroduced := introduced.APILifecycleIntroduced()
+		return apimachineryversion.MajorMinor(uint(majorIntroduced), uint(minorIntroduced))
+	}
+	return defaultVer
+}
+
 func shouldRemoveResourceAndSubresources(resourcesToRemove sets.String, resourceName string) bool {
 	for _, resourceToRemove := range resourcesToRemove.List() {
 		if resourceName == resourceToRemove {