Let kube-apiserver host the storage version API

Co-authored-by: Haowei Cai <[email protected]>

Author: Chao Xu

cmd/kube-apiserver/app/aggregator.go

@@ -275,6 +275,7 @@ var apiVersionPriorities = map[schema.GroupVersion]priority{
 	{Group: "discovery.k8s.io", Version: "v1beta1"}:              {group: 16200, version: 12},
 	{Group: "discovery.k8s.io", Version: "v1alpha1"}:             {group: 16200, version: 9},
 	{Group: "flowcontrol.apiserver.k8s.io", Version: "v1alpha1"}: {group: 16100, version: 9},
+	{Group: "internal.apiserver.k8s.io", Version: "v1alpha1"}:    {group: 16000, version: 9},
 	// Append a new group to the end of the list if unsure.
 	// You can use min(existing group)-100 as the initial value for a group.
 	// Version can be set to 9 (to have space around) for a new group.

pkg/apis/apiserverinternal/install/install.go

@@ -21,10 +21,15 @@ package install
 import (
 	"k8s.io/apimachinery/pkg/runtime"
 	utilruntime "k8s.io/apimachinery/pkg/util/runtime"
+	"k8s.io/kubernetes/pkg/api/legacyscheme"
 	"k8s.io/kubernetes/pkg/apis/apiserverinternal"
 	"k8s.io/kubernetes/pkg/apis/apiserverinternal/v1alpha1"
 )
 
+func init() {
+	Install(legacyscheme.Scheme)
+}
+
 // Install registers the API group and adds types to a scheme
 func Install(scheme *runtime.Scheme) {
 	utilruntime.Must(apiserverinternal.AddToScheme(scheme))

pkg/apis/apiserverinternal/validation/BUILD

@@ -7,6 +7,7 @@ go_library(
     visibility = ["//visibility:public"],
     deps = [
         "//pkg/apis/apiserverinternal:go_default_library",
+        "//pkg/apis/core/validation:go_default_library",
         "//staging/src/k8s.io/apimachinery/pkg/api/validation:go_default_library",
         "//staging/src/k8s.io/apimachinery/pkg/util/validation:go_default_library",
         "//staging/src/k8s.io/apimachinery/pkg/util/validation/field:go_default_library",

pkg/apis/apiserverinternal/validation/validation.go

@@ -25,10 +25,42 @@ import (
 	utilvalidation "k8s.io/apimachinery/pkg/util/validation"
 	"k8s.io/apimachinery/pkg/util/validation/field"
 	"k8s.io/kubernetes/pkg/apis/apiserverinternal"
+	apivalidation "k8s.io/kubernetes/pkg/apis/core/validation"
 )
 
 // ValidateStorageVersion validate the storage version object.
 func ValidateStorageVersion(sv *apiserverinternal.StorageVersion) field.ErrorList {
+	var allErrs field.ErrorList
+	allErrs = append(allErrs, apivalidation.ValidateObjectMeta(&sv.ObjectMeta, false, ValidateStorageVersionName, field.NewPath("metadata"))...)
+	allErrs = append(allErrs, validateStorageVersionStatus(sv.Status, field.NewPath("status"))...)
+	return allErrs
+}
+
+// ValidateStorageVersionName is a ValidateNameFunc for storage version names
+func ValidateStorageVersionName(name string, prefix bool) []string {
+	var allErrs []string
+	idx := strings.LastIndex(name, ".")
+	if idx < 0 {
+		allErrs = append(allErrs, "name must be in the form of <group>.<resource>")
+	} else {
+		for _, msg := range utilvalidation.IsDNS1123Subdomain(name[:idx]) {
+			allErrs = append(allErrs, "the group segment "+msg)
+		}
+		for _, msg := range utilvalidation.IsDNS1035Label(name[idx+1:]) {
+			allErrs = append(allErrs, "the resource segment "+msg)
+		}
+	}
+	return allErrs
+}
+
+// ValidateStorageVersionUpdate tests if an update to a StorageVersion is valid.
+func ValidateStorageVersionUpdate(sv, oldSV *apiserverinternal.StorageVersion) field.ErrorList {
+	// no error since StorageVersionSpec is an empty spec
+	return field.ErrorList{}
+}
+
+// ValidateStorageVersionStatusUpdate tests if an update to a StorageVersionStatus is valid.
+func ValidateStorageVersionStatusUpdate(sv, oldSV *apiserverinternal.StorageVersion) field.ErrorList {
 	var allErrs field.ErrorList
 	allErrs = append(allErrs, validateStorageVersionStatus(sv.Status, field.NewPath("status"))...)
 	return allErrs

pkg/apis/apiserverinternal/validation/validation_test.go

@@ -284,3 +284,66 @@ func TestValidateStorageVersionCondition(t *testing.T) {
 		}
 	}
 }
+
+func TestValidateStorageVersionName(t *testing.T) {
+	cases := []struct {
+		name        string
+		expectedErr string
+	}{
+		{
+			name:        "",
+			expectedErr: `name must be in the form of <group>.<resource>`,
+		},
+		{
+			name:        "pods",
+			expectedErr: `name must be in the form of <group>.<resource>`,
+		},
+		{
+			name:        "core.pods",
+			expectedErr: "",
+		},
+		{
+			name:        "authentication.k8s.io.tokenreviews",
+			expectedErr: "",
+		},
+		{
+			name:        strings.Repeat("x", 253) + ".tokenreviews",
+			expectedErr: "",
+		},
+		{
+			name:        strings.Repeat("x", 254) + ".tokenreviews",
+			expectedErr: `the group segment must be no more than 253 characters`,
+		},
+		{
+			name:        "authentication.k8s.io." + strings.Repeat("x", 63),
+			expectedErr: "",
+		},
+		{
+			name:        "authentication.k8s.io." + strings.Repeat("x", 64),
+			expectedErr: `the resource segment must be no more than 63 characters`,
+		},
+	}
+	for _, tc := range cases {
+		errs := ValidateStorageVersionName(tc.name, false)
+		if errs == nil && len(tc.expectedErr) == 0 {
+			continue
+		}
+		if errs != nil && len(tc.expectedErr) == 0 {
+			t.Errorf("unexpected error %v", errs)
+			continue
+		}
+		if errs == nil && len(tc.expectedErr) != 0 {
+			t.Errorf("unexpected empty error")
+			continue
+		}
+		found := false
+		for _, msg := range errs {
+			if msg == tc.expectedErr {
+				found = true
+			}
+		}
+		if !found {
+			t.Errorf("expected error to contain %s, got %v", tc.expectedErr, errs)
+		}
+	}
+}

pkg/controlplane/master.go

@@ -27,6 +27,7 @@ import (
 
 	admissionregistrationv1 "k8s.io/api/admissionregistration/v1"
 	admissionregistrationv1beta1 "k8s.io/api/admissionregistration/v1beta1"
+	apiserverinternalv1alpha1 "k8s.io/api/apiserverinternal/v1alpha1"
 	appsv1 "k8s.io/api/apps/v1"
 	authenticationv1 "k8s.io/api/authentication/v1"
 	authenticationv1beta1 "k8s.io/api/authentication/v1beta1"
@@ -93,6 +94,7 @@ import (
 
 	// RESTStorage installers
 	admissionregistrationrest "k8s.io/kubernetes/pkg/registry/admissionregistration/rest"
+	apiserverinternalrest "k8s.io/kubernetes/pkg/registry/apiserverinternal/rest"
 	appsrest "k8s.io/kubernetes/pkg/registry/apps/rest"
 	authenticationrest "k8s.io/kubernetes/pkg/registry/authentication/rest"
 	authorizationrest "k8s.io/kubernetes/pkg/registry/authorization/rest"
@@ -415,6 +417,7 @@ func (c completedConfig) New(delegationTarget genericapiserver.DelegationTarget)
 	// TODO: describe the priority all the way down in the RESTStorageProviders and plumb it back through the various discovery
 	// handlers that we have.
 	restStorageProviders := []RESTStorageProvider{
+		apiserverinternalrest.StorageProvider{},
 		authenticationrest.RESTStorageProvider{Authenticator: c.GenericConfig.Authentication.Authenticator, APIAudiences: c.GenericConfig.Authentication.APIAudiences},
 		authorizationrest.RESTStorageProvider{Authorizer: c.GenericConfig.Authorization.Authorizer, RuleResolver: c.GenericConfig.RuleResolver},
 		autoscalingrest.RESTStorageProvider{},
@@ -633,6 +636,7 @@ func DefaultAPIResourceConfigSource() *serverstorage.ResourceConfig {
 	)
 	// disable alpha versions explicitly so we have a full list of what's possible to serve
 	ret.DisableVersions(
+		apiserverinternalv1alpha1.SchemeGroupVersion,
 		batchapiv2alpha1.SchemeGroupVersion,
 		nodev1alpha1.SchemeGroupVersion,
 		rbacv1alpha1.SchemeGroupVersion,

pkg/printers/internalversion/printers.go

@@ -25,6 +25,7 @@ import (
 	"strings"
 	"time"
 
+	apiserverinternalv1alpha1 "k8s.io/api/apiserverinternal/v1alpha1"
 	appsv1beta1 "k8s.io/api/apps/v1beta1"
 	autoscalingv2beta1 "k8s.io/api/autoscaling/v2beta1"
 	batchv1 "k8s.io/api/batch/v1"
@@ -48,6 +49,7 @@ import (
 	"k8s.io/apimachinery/pkg/util/sets"
 	utilfeature "k8s.io/apiserver/pkg/util/feature"
 	"k8s.io/kubernetes/pkg/apis/admissionregistration"
+	"k8s.io/kubernetes/pkg/apis/apiserverinternal"
 	"k8s.io/kubernetes/pkg/apis/apps"
 	"k8s.io/kubernetes/pkg/apis/autoscaling"
 	"k8s.io/kubernetes/pkg/apis/batch"
@@ -571,6 +573,15 @@ func AddHandlers(h printers.PrintHandler) {
 	}
 	h.TableHandler(priorityLevelColumnDefinitions, printPriorityLevelConfiguration)
 	h.TableHandler(priorityLevelColumnDefinitions, printPriorityLevelConfigurationList)
+
+	storageVersionColumnDefinitions := []metav1.TableColumnDefinition{
+		{Name: "Name", Type: "string", Format: "name", Description: metav1.ObjectMeta{}.SwaggerDoc()["name"]},
+		{Name: "CommonEncodingVersion", Type: "string", Description: apiserverinternalv1alpha1.StorageVersionStatus{}.SwaggerDoc()["commonEncodingVersion"]},
+		{Name: "StorageVersions", Type: "string", Description: apiserverinternalv1alpha1.StorageVersionStatus{}.SwaggerDoc()["storageVersions"]},
+		{Name: "Age", Type: "string", Description: metav1.ObjectMeta{}.SwaggerDoc()["creationTimestamp"]},
+	}
+	h.TableHandler(storageVersionColumnDefinitions, printStorageVersion)
+	h.TableHandler(storageVersionColumnDefinitions, printStorageVersionList)
 }
 
 // Pass ports=nil for all ports.
@@ -2477,6 +2488,46 @@ func printFlowSchemaList(list *flowcontrol.FlowSchemaList, options printers.Gene
 	return rows, nil
 }
 
+func printStorageVersion(obj *apiserverinternal.StorageVersion, options printers.GenerateOptions) ([]metav1.TableRow, error) {
+	row := metav1.TableRow{
+		Object: runtime.RawExtension{Object: obj},
+	}
+	commonEncodingVersion := "<unset>"
+	if obj.Status.CommonEncodingVersion != nil {
+		commonEncodingVersion = *obj.Status.CommonEncodingVersion
+	}
+	row.Cells = append(row.Cells, obj.Name, commonEncodingVersion, formatStorageVersions(obj.Status.StorageVersions), translateTimestampSince(obj.CreationTimestamp))
+	return []metav1.TableRow{row}, nil
+}
+
+func formatStorageVersions(storageVersions []apiserverinternal.ServerStorageVersion) string {
+	list := []string{}
+	max := 3
+	more := false
+	count := 0
+	for _, sv := range storageVersions {
+		if len(list) < max {
+			list = append(list, fmt.Sprintf("%s=%s", sv.APIServerID, sv.EncodingVersion))
+		} else if len(list) == max {
+			more = true
+		}
+		count++
+	}
+	return listWithMoreString(list, more, count, max)
+}
+
+func printStorageVersionList(list *apiserverinternal.StorageVersionList, options printers.GenerateOptions) ([]metav1.TableRow, error) {
+	rows := make([]metav1.TableRow, 0, len(list.Items))
+	for i := range list.Items {
+		r, err := printStorageVersion(&list.Items[i], options)
+		if err != nil {
+			return nil, err
+		}
+		rows = append(rows, r...)
+	}
+	return rows, nil
+}
+
 func printPriorityLevelConfiguration(obj *flowcontrol.PriorityLevelConfiguration, options printers.GenerateOptions) ([]metav1.TableRow, error) {
 	row := metav1.TableRow{
 		Object: runtime.RawExtension{Object: obj},

pkg/printers/internalversion/printers_test.go

@@ -27,6 +27,7 @@ import (
 	"k8s.io/apimachinery/pkg/runtime/schema"
 	"k8s.io/apimachinery/pkg/util/diff"
 	"k8s.io/apimachinery/pkg/util/intstr"
+	"k8s.io/kubernetes/pkg/apis/apiserverinternal"
 	"k8s.io/kubernetes/pkg/apis/apps"
 	"k8s.io/kubernetes/pkg/apis/autoscaling"
 	"k8s.io/kubernetes/pkg/apis/batch"
@@ -5311,3 +5312,125 @@ func TestPrintPriorityLevelConfiguration(t *testing.T) {
 		}
 	}
 }
+
+func TestPrintStorageVersion(t *testing.T) {
+	commonEncodingVersion := "v1"
+	tests := []struct {
+		sv       apiserverinternal.StorageVersion
+		expected []metav1.TableRow
+	}{
+		{
+			sv: apiserverinternal.StorageVersion{
+				ObjectMeta: metav1.ObjectMeta{
+					Name:              "empty",
+					CreationTimestamp: metav1.Time{Time: time.Now().Add(1.9e9)},
+				},
+				Status: apiserverinternal.StorageVersionStatus{},
+			},
+			// Columns: Name, CommonEncodingVersion, StorageVersions, Age
+			expected: []metav1.TableRow{{Cells: []interface{}{"empty", "<unset>", "<unset>", "0s"}}},
+		},
+		{
+			sv: apiserverinternal.StorageVersion{
+				ObjectMeta: metav1.ObjectMeta{
+					Name:              "valid",
+					CreationTimestamp: metav1.Time{Time: time.Now().Add(1.9e9)},
+				},
+				Status: apiserverinternal.StorageVersionStatus{
+					StorageVersions: []apiserverinternal.ServerStorageVersion{
+						{
+							APIServerID:       "1",
+							EncodingVersion:   "v1",
+							DecodableVersions: []string{"v1"},
+						},
+						{
+							APIServerID:       "2",
+							EncodingVersion:   "v1",
+							DecodableVersions: []string{"v1", "v2"},
+						},
+					},
+					CommonEncodingVersion: &commonEncodingVersion,
+				},
+			},
+			// Columns: Name, CommonEncodingVersion, StorageVersions, Age
+			expected: []metav1.TableRow{{Cells: []interface{}{"valid", "v1", "1=v1,2=v1", "0s"}}},
+		},
+		{
+			sv: apiserverinternal.StorageVersion{
+				ObjectMeta: metav1.ObjectMeta{
+					Name:              "disagree",
+					CreationTimestamp: metav1.Time{Time: time.Now().Add(1.9e9)},
+				},
+				Status: apiserverinternal.StorageVersionStatus{
+					StorageVersions: []apiserverinternal.ServerStorageVersion{
+						{
+							APIServerID:       "1",
+							EncodingVersion:   "v1",
+							DecodableVersions: []string{"v1"},
+						},
+						{
+							APIServerID:       "2",
+							EncodingVersion:   "v1",
+							DecodableVersions: []string{"v1", "v2"},
+						},
+						{
+							APIServerID:       "3",
+							EncodingVersion:   "v2",
+							DecodableVersions: []string{"v2"},
+						},
+					},
+				},
+			},
+			// Columns: Name, CommonEncodingVersion, StorageVersions, Age
+			expected: []metav1.TableRow{{Cells: []interface{}{"disagree", "<unset>", "1=v1,2=v1,3=v2", "0s"}}},
+		},
+		{
+			sv: apiserverinternal.StorageVersion{
+				ObjectMeta: metav1.ObjectMeta{
+					Name:              "agreeWithMore",
+					CreationTimestamp: metav1.Time{Time: time.Now().Add(1.9e9)},
+				},
+				Status: apiserverinternal.StorageVersionStatus{
+					StorageVersions: []apiserverinternal.ServerStorageVersion{
+						{
+							APIServerID:       "1",
+							EncodingVersion:   "v1",
+							DecodableVersions: []string{"v1"},
+						},
+						{
+							APIServerID:       "2",
+							EncodingVersion:   "v1",
+							DecodableVersions: []string{"v1", "v2"},
+						},
+						{
+							APIServerID:       "3",
+							EncodingVersion:   "v1",
+							DecodableVersions: []string{"v1", "v2"},
+						},
+						{
+							APIServerID:       "4",
+							EncodingVersion:   "v1",
+							DecodableVersions: []string{"v1", "v2", "v3alpha1"},
+						},
+					},
+					CommonEncodingVersion: &commonEncodingVersion,
+				},
+			},
+			// Columns: Name, CommonEncodingVersion, StorageVersions, Age
+			expected: []metav1.TableRow{{Cells: []interface{}{"agreeWithMore", "v1", "1=v1,2=v1,3=v1 + 1 more...", "0s"}}},
+		},
+	}
+
+	for i, test := range tests {
+		rows, err := printStorageVersion(&test.sv, printers.GenerateOptions{})
+		if err != nil {
+			t.Fatal(err)
+		}
+		for i := range rows {
+			rows[i].Object.Object = nil
+		}
+		if !reflect.DeepEqual(test.expected, rows) {
+			t.Errorf("%d mismatch: %s", i, diff.ObjectReflectDiff(test.expected, rows))
+		}
+	}
+}