Merge pull request kubernetes#72694 from liggitt/integration-api-versions

Use apps/v1 APIs in integration, explicitly enable all resources for print/etcd/scale tests

Author: k8s-ci-robot

hack/update-openapi-spec.sh

@@ -66,7 +66,7 @@ kube::log::status "Starting kube-apiserver"
   --etcd-servers="http://${ETCD_HOST}:${ETCD_PORT}" \
   --advertise-address="10.10.10.10" \
   --cert-dir="${TMP_DIR}/certs" \
-  --runtime-config="api/all=true" \
+  --runtime-config="api/all=true,extensions/v1beta1/daemonsets=true,extensions/v1beta1/deployments=true,extensions/v1beta1/replicasets=true,extensions/v1beta1/networkpolicies=true,extensions/v1beta1/podsecuritypolicies=true,extensions/v1beta1/replicationcontrollers=true" \
   --token-auth-file="${TMP_DIR}/tokenauth.csv" \
   --logtostderr \
   --v=2 \

test/integration/apiserver/BUILD

@@ -20,16 +20,17 @@ go_test(
     ],
     deps = [
         "//pkg/api/legacyscheme:go_default_library",
-        "//pkg/api/testapi:go_default_library",
-        "//pkg/apis/core:go_default_library",
         "//pkg/kubectl/cmd/util:go_default_library",
         "//pkg/master:go_default_library",
         "//pkg/printers:go_default_library",
         "//pkg/printers/internalversion:go_default_library",
         "//staging/src/k8s.io/api/apps/v1:go_default_library",
+        "//staging/src/k8s.io/api/apps/v1beta1:go_default_library",
+        "//staging/src/k8s.io/api/apps/v1beta2:go_default_library",
         "//staging/src/k8s.io/api/auditregistration/v1alpha1:go_default_library",
         "//staging/src/k8s.io/api/batch/v2alpha1:go_default_library",
         "//staging/src/k8s.io/api/core/v1:go_default_library",
+        "//staging/src/k8s.io/api/extensions/v1beta1:go_default_library",
         "//staging/src/k8s.io/api/rbac/v1alpha1:go_default_library",
         "//staging/src/k8s.io/api/scheduling/v1beta1:go_default_library",
         "//staging/src/k8s.io/api/settings/v1alpha1:go_default_library",

test/integration/apiserver/apiserver_test.go

@@ -23,6 +23,7 @@ import (
 	"io/ioutil"
 	"net/http"
 	"net/http/httptest"
+	"path"
 	"reflect"
 	"testing"
 
@@ -39,17 +40,20 @@ import (
 	restclient "k8s.io/client-go/rest"
 	"k8s.io/client-go/tools/pager"
 	"k8s.io/klog"
-	"k8s.io/kubernetes/pkg/api/testapi"
-	api "k8s.io/kubernetes/pkg/apis/core"
 	"k8s.io/kubernetes/pkg/master"
 	"k8s.io/kubernetes/test/integration/framework"
 )
 
 func setup(t *testing.T, groupVersions ...schema.GroupVersion) (*httptest.Server, clientset.Interface, framework.CloseFunc) {
+	return setupWithResources(t, groupVersions, nil)
+}
+
+func setupWithResources(t *testing.T, groupVersions []schema.GroupVersion, resources []schema.GroupVersionResource) (*httptest.Server, clientset.Interface, framework.CloseFunc) {
 	masterConfig := framework.NewIntegrationTestMasterConfig()
-	if len(groupVersions) > 0 {
+	if len(groupVersions) > 0 || len(resources) > 0 {
 		resourceConfig := master.DefaultAPIResourceConfigSource()
 		resourceConfig.EnableVersions(groupVersions...)
+		resourceConfig.EnableResources(resources...)
 		masterConfig.ExtraConfig.APIResourceConfigSource = resourceConfig
 	}
 	_, s, closeFn := framework.RunAMaster(masterConfig)
@@ -82,10 +86,6 @@ func verifyStatusCode(t *testing.T, verb, URL, body string, expectedStatusCode i
 	}
 }
 
-func path(resource, namespace, name string) string {
-	return testapi.Extensions.ResourcePath(resource, namespace, name)
-}
-
 func newRS(namespace string) *apps.ReplicaSet {
 	return &apps.ReplicaSet{
 		TypeMeta: metav1.TypeMeta{
@@ -118,7 +118,7 @@ func newRS(namespace string) *apps.ReplicaSet {
 var cascDel = `
 {
   "kind": "DeleteOptions",
-  "apiVersion": "` + testapi.Groups[api.GroupName].GroupVersion().String() + `",
+  "apiVersion": "v1",
   "orphanDependents": false
 }
 `
@@ -139,7 +139,7 @@ func Test202StatusCode(t *testing.T) {
 	if err != nil {
 		t.Fatalf("Failed to create rs: %v", err)
 	}
-	verifyStatusCode(t, "DELETE", s.URL+path("replicasets", ns.Name, rs.Name), "", 200)
+	verifyStatusCode(t, "DELETE", s.URL+path.Join("/apis/apps/v1/namespaces", ns.Name, "replicasets", rs.Name), "", 200)
 
 	// 2. Create the resource with a finalizer so that the resource is not immediately deleted and then delete it without setting DeleteOptions.
 	// Verify that the apiserver still returns 200 since DeleteOptions.OrphanDependents is not set.
@@ -149,7 +149,7 @@ func Test202StatusCode(t *testing.T) {
 	if err != nil {
 		t.Fatalf("Failed to create rs: %v", err)
 	}
-	verifyStatusCode(t, "DELETE", s.URL+path("replicasets", ns.Name, rs.Name), "", 200)
+	verifyStatusCode(t, "DELETE", s.URL+path.Join("/apis/apps/v1/namespaces", ns.Name, "replicasets", rs.Name), "", 200)
 
 	// 3. Create the resource and then delete it with DeleteOptions.OrphanDependents=false.
 	// Verify that the server still returns 200 since the resource is immediately deleted.
@@ -158,7 +158,7 @@ func Test202StatusCode(t *testing.T) {
 	if err != nil {
 		t.Fatalf("Failed to create rs: %v", err)
 	}
-	verifyStatusCode(t, "DELETE", s.URL+path("replicasets", ns.Name, rs.Name), cascDel, 200)
+	verifyStatusCode(t, "DELETE", s.URL+path.Join("/apis/apps/v1/namespaces", ns.Name, "replicasets", rs.Name), cascDel, 200)
 
 	// 4. Create the resource with a finalizer so that the resource is not immediately deleted and then delete it with DeleteOptions.OrphanDependents=false.
 	// Verify that the server returns 202 in this case.
@@ -168,7 +168,7 @@ func Test202StatusCode(t *testing.T) {
 	if err != nil {
 		t.Fatalf("Failed to create rs: %v", err)
 	}
-	verifyStatusCode(t, "DELETE", s.URL+path("replicasets", ns.Name, rs.Name), cascDel, 202)
+	verifyStatusCode(t, "DELETE", s.URL+path.Join("/apis/apps/v1/namespaces", ns.Name, "replicasets", rs.Name), cascDel, 202)
 }
 
 func TestAPIListChunking(t *testing.T) {

test/integration/apiserver/print_test.go

@@ -26,15 +26,19 @@ import (
 	"testing"
 	"time"
 
+	appsv1beta1 "k8s.io/api/apps/v1beta1"
+	appsv1beta2 "k8s.io/api/apps/v1beta2"
 	auditregv1alpha1 "k8s.io/api/auditregistration/v1alpha1"
 	batchv2alpha1 "k8s.io/api/batch/v2alpha1"
+	extensionsv1beta1 "k8s.io/api/extensions/v1beta1"
 	rbacv1alpha1 "k8s.io/api/rbac/v1alpha1"
 	schedulerapi "k8s.io/api/scheduling/v1beta1"
 	settingsv1alpha1 "k8s.io/api/settings/v1alpha1"
 	storagev1alpha1 "k8s.io/api/storage/v1alpha1"
 	"k8s.io/apimachinery/pkg/api/meta"
 	metav1beta1 "k8s.io/apimachinery/pkg/apis/meta/v1beta1"
 	"k8s.io/apimachinery/pkg/runtime"
+	"k8s.io/apimachinery/pkg/runtime/schema"
 	"k8s.io/cli-runtime/pkg/genericclioptions"
 	"k8s.io/client-go/discovery"
 	"k8s.io/client-go/tools/clientcmd"
@@ -131,14 +135,27 @@ var missingHanlders = sets.NewString(
 )
 
 func TestServerSidePrint(t *testing.T) {
-	s, _, closeFn := setup(t,
+	s, _, closeFn := setupWithResources(t,
 		// additional groupversions needed for the test to run
-		auditregv1alpha1.SchemeGroupVersion,
-		batchv2alpha1.SchemeGroupVersion,
-		rbacv1alpha1.SchemeGroupVersion,
-		settingsv1alpha1.SchemeGroupVersion,
-		schedulerapi.SchemeGroupVersion,
-		storagev1alpha1.SchemeGroupVersion)
+		[]schema.GroupVersion{
+			auditregv1alpha1.SchemeGroupVersion,
+			batchv2alpha1.SchemeGroupVersion,
+			rbacv1alpha1.SchemeGroupVersion,
+			settingsv1alpha1.SchemeGroupVersion,
+			schedulerapi.SchemeGroupVersion,
+			storagev1alpha1.SchemeGroupVersion,
+			appsv1beta1.SchemeGroupVersion,
+			appsv1beta2.SchemeGroupVersion,
+			extensionsv1beta1.SchemeGroupVersion,
+		},
+		[]schema.GroupVersionResource{
+			extensionsv1beta1.SchemeGroupVersion.WithResource("daemonsets"),
+			extensionsv1beta1.SchemeGroupVersion.WithResource("deployments"),
+			extensionsv1beta1.SchemeGroupVersion.WithResource("networkpolicies"),
+			extensionsv1beta1.SchemeGroupVersion.WithResource("podsecuritypolicies"),
+			extensionsv1beta1.SchemeGroupVersion.WithResource("replicasets"),
+		},
+	)
 	defer closeFn()
 
 	ns := framework.CreateTestingNamespace("server-print", s, t)

test/integration/etcd/BUILD

@@ -15,6 +15,7 @@ go_test(
         "integration",
     ],
     deps = [
+        "//cmd/kube-apiserver/app/options:go_default_library",
         "//staging/src/k8s.io/api/core/v1:go_default_library",
         "//staging/src/k8s.io/apimachinery/pkg/api/equality:go_default_library",
         "//staging/src/k8s.io/apimachinery/pkg/api/meta:go_default_library",

test/integration/etcd/etcd_storage_path_test.go

@@ -35,6 +35,7 @@ import (
 	"k8s.io/apimachinery/pkg/util/diff"
 	"k8s.io/apimachinery/pkg/util/sets"
 	"k8s.io/client-go/dynamic"
+	"k8s.io/kubernetes/cmd/kube-apiserver/app/options"
 )
 
 // Only add kinds to this list when this a virtual resource with get and create verbs that doesn't actually
@@ -49,7 +50,15 @@ const testNamespace = "etcdstoragepathtestnamespace"
 // It will also fail when a type gets moved to a different location. Be very careful in this situation because
 // it essentially means that you will be break old clusters unless you create some migration path for the old data.
 func TestEtcdStoragePath(t *testing.T) {
-	master := StartRealMasterOrDie(t)
+	master := StartRealMasterOrDie(t, func(opts *options.ServerRunOptions) {
+		// force enable all resources so we can check storage.
+		// TODO: drop these once we stop allowing them to be served.
+		opts.APIEnablement.RuntimeConfig["extensions/v1beta1/deployments"] = "true"
+		opts.APIEnablement.RuntimeConfig["extensions/v1beta1/daemonsets"] = "true"
+		opts.APIEnablement.RuntimeConfig["extensions/v1beta1/replicasets"] = "true"
+		opts.APIEnablement.RuntimeConfig["extensions/v1beta1/podsecuritypolicies"] = "true"
+		opts.APIEnablement.RuntimeConfig["extensions/v1beta1/networkpolicies"] = "true"
+	})
 	defer master.Cleanup()
 	defer dumpEtcdKVOnFailure(t, master.KV)
 

test/integration/etcd/server.go

@@ -54,7 +54,7 @@ import (
 )
 
 // StartRealMasterOrDie starts an API master that is appropriate for use in tests that require one of every resource
-func StartRealMasterOrDie(t *testing.T) *Master {
+func StartRealMasterOrDie(t *testing.T, configFuncs ...func(*options.ServerRunOptions)) *Master {
 	certDir, err := ioutil.TempDir("", t.Name())
 	if err != nil {
 		t.Fatal(err)
@@ -79,13 +79,14 @@ func StartRealMasterOrDie(t *testing.T) *Master {
 	kubeAPIServerOptions.ServiceClusterIPRange = *defaultServiceClusterIPRange
 	kubeAPIServerOptions.Authorization.Modes = []string{"RBAC"}
 	kubeAPIServerOptions.Admission.GenericAdmission.DisablePlugins = []string{"ServiceAccount"}
+	kubeAPIServerOptions.APIEnablement.RuntimeConfig["api/all"] = "true"
+	for _, f := range configFuncs {
+		f(kubeAPIServerOptions)
+	}
 	completedOptions, err := app.Complete(kubeAPIServerOptions)
 	if err != nil {
 		t.Fatal(err)
 	}
-	if err := completedOptions.APIEnablement.RuntimeConfig.Set("api/all=true"); err != nil {
-		t.Fatal(err)
-	}
 
 	// get etcd client before starting API server
 	rawClient, kvClient, err := integration.GetEtcdClients(completedOptions.Etcd.StorageConfig.Transport)

test/integration/scale/scale_test.go

@@ -52,7 +52,11 @@ func TestMain(m *testing.M) {
 }
 
 func TestScaleSubresources(t *testing.T) {
-	clientSet, tearDown := setup(t)
+	clientSet, tearDown := setupWithOptions(t, nil, []string{
+		"--runtime-config",
+		// TODO(liggitt): remove these once apps/v1beta1, apps/v1beta2, and extensions/v1beta1 can no longer be served
+		"api/all=true,extensions/v1beta1/deployments=true,extensions/v1beta1/replicationcontrollers=true,extensions/v1beta1/replicasets=true",
+	})
 	defer tearDown()
 
 	resourceLists, err := clientSet.Discovery().ServerResources()
@@ -63,6 +67,7 @@ func TestScaleSubresources(t *testing.T) {
 	expectedScaleSubresources := map[schema.GroupVersionResource]schema.GroupVersionKind{
 		makeGVR("", "v1", "replicationcontrollers/scale"): makeGVK("autoscaling", "v1", "Scale"),
 
+		// TODO(liggitt): remove these once apps/v1beta1, apps/v1beta2, and extensions/v1beta1 can no longer be served
 		makeGVR("extensions", "v1beta1", "deployments/scale"):            makeGVK("extensions", "v1beta1", "Scale"),
 		makeGVR("extensions", "v1beta1", "replicationcontrollers/scale"): makeGVK("extensions", "v1beta1", "Scale"),
 		makeGVR("extensions", "v1beta1", "replicasets/scale"):            makeGVK("extensions", "v1beta1", "Scale"),
@@ -215,7 +220,11 @@ var (
 )
 
 func setup(t *testing.T) (client kubernetes.Interface, tearDown func()) {
-	result := apitesting.StartTestServerOrDie(t, nil, nil, framework.SharedEtcd())
+	return setupWithOptions(t, nil, nil)
+}
+
+func setupWithOptions(t *testing.T, instanceOptions *apitesting.TestServerInstanceOptions, flags []string) (client kubernetes.Interface, tearDown func()) {
+	result := apitesting.StartTestServerOrDie(t, instanceOptions, flags, framework.SharedEtcd())
 
 	// TODO: Disable logging here until we resolve teardown issues which result in
 	// massive log spam. Another path forward would be to refactor

test/integration/statefulset/BUILD

@@ -8,15 +8,15 @@ go_library(
     deps = [
         #"//pkg/api:go_default_library",
         "//pkg/controller/statefulset:go_default_library",
-        "//staging/src/k8s.io/api/apps/v1beta1:go_default_library",
+        "//staging/src/k8s.io/api/apps/v1:go_default_library",
         "//staging/src/k8s.io/api/core/v1:go_default_library",
         "//staging/src/k8s.io/apimachinery/pkg/api/resource:go_default_library",
         "//staging/src/k8s.io/apimachinery/pkg/apis/meta/v1:go_default_library",
         "//staging/src/k8s.io/apimachinery/pkg/labels:go_default_library",
         "//staging/src/k8s.io/apimachinery/pkg/util/wait:go_default_library",
         "//staging/src/k8s.io/client-go/informers:go_default_library",
         "//staging/src/k8s.io/client-go/kubernetes:go_default_library",
-        "//staging/src/k8s.io/client-go/kubernetes/typed/apps/v1beta1:go_default_library",
+        "//staging/src/k8s.io/client-go/kubernetes/typed/apps/v1:go_default_library",
         "//staging/src/k8s.io/client-go/kubernetes/typed/core/v1:go_default_library",
         "//staging/src/k8s.io/client-go/rest:go_default_library",
         "//staging/src/k8s.io/client-go/util/retry:go_default_library",
@@ -34,7 +34,7 @@ go_test(
     embed = [":go_default_library"],
     tags = ["integration"],
     deps = [
-        "//staging/src/k8s.io/api/apps/v1beta1:go_default_library",
+        "//staging/src/k8s.io/api/apps/v1:go_default_library",
         "//staging/src/k8s.io/api/core/v1:go_default_library",
         "//staging/src/k8s.io/apimachinery/pkg/apis/meta/v1:go_default_library",
         "//staging/src/k8s.io/apimachinery/pkg/util/wait:go_default_library",

test/integration/statefulset/statefulset_test.go

@@ -20,7 +20,7 @@ import (
 	"fmt"
 	"testing"
 
-	"k8s.io/api/apps/v1beta1"
+	appsv1 "k8s.io/api/apps/v1"
 	"k8s.io/api/core/v1"
 	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
 	"k8s.io/apimachinery/pkg/util/wait"
@@ -37,7 +37,7 @@ func TestSpecReplicasChange(t *testing.T) {
 
 	createHeadlessService(t, c, newHeadlessService(ns.Name))
 	sts := newSTS("sts", ns.Name, 2)
-	stss, _ := createSTSsPods(t, c, []*v1beta1.StatefulSet{sts}, []*v1.Pod{})
+	stss, _ := createSTSsPods(t, c, []*appsv1.StatefulSet{sts}, []*v1.Pod{})
 	sts = stss[0]
 	waitSTSStable(t, c, sts)
 
@@ -48,9 +48,9 @@ func TestSpecReplicasChange(t *testing.T) {
 
 	// Add a template annotation change to test STS's status does update
 	// without .Spec.Replicas change
-	stsClient := c.AppsV1beta1().StatefulSets(ns.Name)
+	stsClient := c.AppsV1().StatefulSets(ns.Name)
 	var oldGeneration int64
-	newSTS := updateSTS(t, stsClient, sts.Name, func(sts *v1beta1.StatefulSet) {
+	newSTS := updateSTS(t, stsClient, sts.Name, func(sts *appsv1.StatefulSet) {
 		oldGeneration = sts.Generation
 		sts.Spec.Template.Annotations = map[string]string{"test": "annotation"}
 	})
@@ -64,7 +64,7 @@ func TestSpecReplicasChange(t *testing.T) {
 		if err != nil {
 			return false, err
 		}
-		return *newSTS.Status.ObservedGeneration >= savedGeneration, nil
+		return newSTS.Status.ObservedGeneration >= savedGeneration, nil
 	}); err != nil {
 		t.Fatalf("failed to verify .Status.ObservedGeneration has incremented for sts %s: %v", sts.Name, err)
 	}
@@ -80,7 +80,7 @@ func TestDeletingAndFailedPods(t *testing.T) {
 
 	labelMap := labelMap()
 	sts := newSTS("sts", ns.Name, 2)
-	stss, _ := createSTSsPods(t, c, []*v1beta1.StatefulSet{sts}, []*v1.Pod{})
+	stss, _ := createSTSsPods(t, c, []*appsv1.StatefulSet{sts}, []*v1.Pod{})
 	sts = stss[0]
 	waitSTSStable(t, c, sts)