test: remove k8s.io/apiextensions-apiserver from framework

There are two reason why this is useful:

1. less code to vendor into external users of the framework

The following dependencies become obsolete due to this change (from `dep`):

(8/23) Removed unused project github.com/grpc-ecosystem/go-grpc-prometheus
(9/23) Removed unused project github.com/coreos/etcd
(10/23) Removed unused project github.com/globalsign/mgo
(11/23) Removed unused project github.com/go-openapi/strfmt
(12/23) Removed unused project github.com/asaskevich/govalidator
(13/23) Removed unused project github.com/mitchellh/mapstructure
(14/23) Removed unused project github.com/NYTimes/gziphandler
(15/23) Removed unused project gopkg.in/natefinch/lumberjack.v2
(16/23) Removed unused project github.com/go-openapi/errors
(17/23) Removed unused project github.com/go-openapi/analysis
(18/23) Removed unused project github.com/go-openapi/runtime
(19/23) Removed unused project sigs.k8s.io/structured-merge-diff
(20/23) Removed unused project github.com/go-openapi/validate
(21/23) Removed unused project github.com/coreos/go-systemd
(22/23) Removed unused project github.com/go-openapi/loads
(23/23) Removed unused project github.com/munnerz/goautoneg

2. works around kubernetes#75338
   which currently breaks vendoring

Some recent changes to crd_util.go must now be pulling in the broken
k8s.io/apiextensions-apiserver packages, because it was still working
in revision 2e90d92 (as demonstrated by
https://github.com/intel/pmem-CSI/tree/586ae281ac2810cb4da6f1e160cf165c7daf0d80).

Author: pohly

test/e2e/apimachinery/crd_conversion_webhook.go

@@ -32,6 +32,7 @@ import (
 	"k8s.io/client-go/dynamic"
 	clientset "k8s.io/client-go/kubernetes"
 	"k8s.io/kubernetes/test/e2e/framework"
+	"k8s.io/kubernetes/test/utils/crd"
 	imageutils "k8s.io/kubernetes/test/utils/image"
 
 	. "github.com/onsi/ginkgo"
@@ -100,7 +101,7 @@ var _ = SIGDescribe("CustomResourceConversionWebhook [Feature:CustomResourceWebh
 	})
 
 	It("Should be able to convert from CR v1 to CR v2", func() {
-		testcrd, err := framework.CreateMultiVersionTestCRD(f, "stable.example.com", apiVersions,
+		testcrd, err := crd.CreateMultiVersionTestCRD(f, "stable.example.com", apiVersions,
 			&v1beta1.WebhookClientConfig{
 				CABundle: context.signingCert,
 				Service: &v1beta1.ServiceReference{
@@ -116,7 +117,7 @@ var _ = SIGDescribe("CustomResourceConversionWebhook [Feature:CustomResourceWebh
 	})
 
 	It("Should be able to convert a non homogeneous list of CRs", func() {
-		testcrd, err := framework.CreateMultiVersionTestCRD(f, "stable.example.com", apiVersions,
+		testcrd, err := crd.CreateMultiVersionTestCRD(f, "stable.example.com", apiVersions,
 			&v1beta1.WebhookClientConfig{
 				CABundle: context.signingCert,
 				Service: &v1beta1.ServiceReference{
@@ -325,7 +326,7 @@ func testCustomResourceConversionWebhook(f *framework.Framework, crd *v1beta1.Cu
 	verifyV2Object(f, crd, v2crd)
 }
 
-func testCRListConversion(f *framework.Framework, testCrd *framework.TestCrd) {
+func testCRListConversion(f *framework.Framework, testCrd *crd.TestCrd) {
 	crd := testCrd.Crd
 	customResourceClients := testCrd.DynamicClients
 	name1 := "cr-instance-1"

test/e2e/apimachinery/crd_publish_openapi.go

@@ -37,6 +37,7 @@ import (
 	k8sclientset "k8s.io/client-go/kubernetes"
 	openapiutil "k8s.io/kube-openapi/pkg/util"
 	"k8s.io/kubernetes/test/e2e/framework"
+	"k8s.io/kubernetes/test/utils/crd"
 	"sigs.k8s.io/yaml"
 )
 
@@ -327,7 +328,7 @@ var _ = SIGDescribe("CustomResourcePublishOpenAPI [Feature:CustomResourcePublish
 	})
 })
 
-func setupCRD(f *framework.Framework, schema []byte, groupSuffix string, versions ...string) (*framework.TestCrd, error) {
+func setupCRD(f *framework.Framework, schema []byte, groupSuffix string, versions ...string) (*crd.TestCrd, error) {
 	group := fmt.Sprintf("%s-test-%s.k8s.io", f.BaseName, groupSuffix)
 	if len(versions) == 0 {
 		return nil, fmt.Errorf("require at least one version for CRD")
@@ -343,7 +344,7 @@ func setupCRD(f *framework.Framework, schema []byte, groupSuffix string, version
 	}
 	apiVersions[0].Storage = true
 
-	crd, err := framework.CreateMultiVersionTestCRD(f, group, apiVersions, nil)
+	crd, err := crd.CreateMultiVersionTestCRD(f, group, apiVersions, nil)
 	if err != nil {
 		return nil, fmt.Errorf("failed to create CRD: %v", err)
 	}
@@ -366,7 +367,7 @@ func setupCRD(f *framework.Framework, schema []byte, groupSuffix string, version
 	return crd, nil
 }
 
-func cleanupCRD(f *framework.Framework, crd *framework.TestCrd) error {
+func cleanupCRD(f *framework.Framework, crd *crd.TestCrd) error {
 	crd.CleanUp()
 	for _, v := range crd.Versions {
 		name := definitionName(crd, v.Name)
@@ -378,7 +379,7 @@ func cleanupCRD(f *framework.Framework, crd *framework.TestCrd) error {
 }
 
 // patchSchema takes schema in YAML and patches it to given CRD in given version
-func patchSchema(schema []byte, crd *framework.TestCrd) error {
+func patchSchema(schema []byte, crd *crd.TestCrd) error {
 	s, err := utilyaml.ToJSON(schema)
 	if err != nil {
 		return fmt.Errorf("failed to create json patch: %v", err)
@@ -484,7 +485,7 @@ func verifyKubectlExplain(name, pattern string) error {
 }
 
 // definitionName returns the openapi definition name for given CRD in given version
-func definitionName(crd *framework.TestCrd, version string) string {
+func definitionName(crd *crd.TestCrd, version string) string {
 	return openapiutil.ToRESTFriendlyName(fmt.Sprintf("%s/%s/%s", crd.ApiGroup, version, crd.Kind))
 }
 

test/e2e/apimachinery/discovery.go

@@ -20,6 +20,7 @@ import (
 	utilversion "k8s.io/apimachinery/pkg/util/version"
 	"k8s.io/apiserver/pkg/endpoints/discovery"
 	"k8s.io/kubernetes/test/e2e/framework"
+	"k8s.io/kubernetes/test/utils/crd"
 
 	. "github.com/onsi/ginkgo"
 )
@@ -40,7 +41,7 @@ var _ = SIGDescribe("Discovery", func() {
 	})
 
 	It("[Feature:StorageVersionHash] Custom resource should have storage version hash", func() {
-		testcrd, err := framework.CreateTestCRD(f)
+		testcrd, err := crd.CreateTestCRD(f)
 		if err != nil {
 			return
 		}

test/e2e/apimachinery/webhook.go

@@ -38,6 +38,7 @@ import (
 	"k8s.io/client-go/dynamic"
 	clientset "k8s.io/client-go/kubernetes"
 	"k8s.io/kubernetes/test/e2e/framework"
+	"k8s.io/kubernetes/test/utils/crd"
 	imageutils "k8s.io/kubernetes/test/utils/image"
 
 	. "github.com/onsi/ginkgo"
@@ -130,7 +131,7 @@ var _ = SIGDescribe("AdmissionWebhook", func() {
 	})
 
 	It("Should be able to deny custom resource creation", func() {
-		testcrd, err := framework.CreateTestCRD(f)
+		testcrd, err := crd.CreateTestCRD(f)
 		if err != nil {
 			return
 		}
@@ -167,7 +168,7 @@ var _ = SIGDescribe("AdmissionWebhook", func() {
 	})
 
 	It("Should mutate custom resource", func() {
-		testcrd, err := framework.CreateTestCRD(f)
+		testcrd, err := crd.CreateTestCRD(f)
 		if err != nil {
 			return
 		}
@@ -185,7 +186,7 @@ var _ = SIGDescribe("AdmissionWebhook", func() {
 	})
 
 	It("Should mutate custom resource with different stored version", func() {
-		testcrd, err := framework.CreateMultiVersionTestCRDWithV1Storage(f)
+		testcrd, err := crd.CreateMultiVersionTestCRDWithV1Storage(f)
 		if err != nil {
 			return
 		}
@@ -1186,7 +1187,7 @@ func cleanWebhookTest(client clientset.Interface, namespaceName string) {
 	_ = client.RbacV1beta1().RoleBindings("kube-system").Delete(roleBindingName, nil)
 }
 
-func registerWebhookForCustomResource(f *framework.Framework, context *certContext, testcrd *framework.TestCrd) func() {
+func registerWebhookForCustomResource(f *framework.Framework, context *certContext, testcrd *crd.TestCrd) func() {
 	client := f.ClientSet
 	By("Registering the custom resource webhook via the AdmissionRegistration API")
 
@@ -1227,7 +1228,7 @@ func registerWebhookForCustomResource(f *framework.Framework, context *certConte
 	}
 }
 
-func registerMutatingWebhookForCustomResource(f *framework.Framework, context *certContext, testcrd *framework.TestCrd) func() {
+func registerMutatingWebhookForCustomResource(f *framework.Framework, context *certContext, testcrd *crd.TestCrd) func() {
 	client := f.ClientSet
 	By("Registering the mutating webhook for a custom resource via the AdmissionRegistration API")
 
@@ -1338,7 +1339,7 @@ func testMutatingCustomResourceWebhook(f *framework.Framework, crd *apiextension
 	}
 }
 
-func testMultiVersionCustomResourceWebhook(f *framework.Framework, testcrd *framework.TestCrd) {
+func testMultiVersionCustomResourceWebhook(f *framework.Framework, testcrd *crd.TestCrd) {
 	customResourceClient := testcrd.GetV1DynamicClient()
 	By("Creating a custom resource while v1 is storage version")
 	crName := "cr-instance-1"
@@ -1427,7 +1428,7 @@ func testCRDDenyWebhook(f *framework.Framework) {
 			Storage: true,
 		},
 	}
-	testcrd := &framework.TestCrd{
+	testcrd := &crd.TestCrd{
 		Name:     name,
 		Kind:     kind,
 		ApiGroup: group,

test/e2e/framework/framework.go

@@ -32,7 +32,6 @@ import (
 	"time"
 
 	"k8s.io/api/core/v1"
-	apiextensionsclient "k8s.io/apiextensions-apiserver/pkg/client/clientset/clientset"
 	apierrors "k8s.io/apimachinery/pkg/api/errors"
 	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
 	"k8s.io/apimachinery/pkg/labels"
@@ -75,7 +74,6 @@ type Framework struct {
 
 	ClientSet                        clientset.Interface
 	KubemarkExternalClusterClientSet clientset.Interface
-	APIExtensionsClientSet           apiextensionsclient.Interface
 
 	InternalClientset *internalclientset.Clientset
 	AggregatorClient  *aggregatorclient.Clientset
@@ -182,8 +180,6 @@ func (f *Framework) BeforeEach() {
 		}
 		f.ClientSet, err = clientset.NewForConfig(config)
 		ExpectNoError(err)
-		f.APIExtensionsClientSet, err = apiextensionsclient.NewForConfig(config)
-		ExpectNoError(err)
 		f.InternalClientset, err = internalclientset.NewForConfig(config)
 		ExpectNoError(err)
 		f.AggregatorClient, err = aggregatorclient.NewForConfig(config)

test/e2e/framework/util.go

@@ -1060,7 +1060,7 @@ func WaitForPersistentVolumeClaimsPhase(phase v1.PersistentVolumeClaimPhase, c c
 func findAvailableNamespaceName(baseName string, c clientset.Interface) (string, error) {
 	var name string
 	err := wait.PollImmediate(Poll, 30*time.Second, func() (bool, error) {
-		name = fmt.Sprintf("%v-%v", baseName, randomSuffix())
+		name = fmt.Sprintf("%v-%v", baseName, RandomSuffix())
 		_, err := c.CoreV1().Namespaces().Get(name, metav1.GetOptions{})
 		if err == nil {
 			// Already taken
@@ -2154,7 +2154,7 @@ func LoadClientset() (*clientset.Clientset, error) {
 //       for pods and replication controllers so we don't
 //       need to use such a function and can instead
 //       use the UUID utility function.
-func randomSuffix() string {
+func RandomSuffix() string {
 	r := rand.New(rand.NewSource(time.Now().UnixNano()))
 	return strconv.Itoa(r.Int() % 10000)
 }

test/e2e/kubectl/kubectl.go

@@ -61,6 +61,7 @@ import (
 	"k8s.io/kubernetes/test/e2e/framework/testfiles"
 	"k8s.io/kubernetes/test/e2e/scheduling"
 	testutils "k8s.io/kubernetes/test/utils"
+	"k8s.io/kubernetes/test/utils/crd"
 	uexec "k8s.io/utils/exec"
 
 	"github.com/onsi/ginkgo"
@@ -823,7 +824,7 @@ metadata:
 	framework.KubeDescribe("Kubectl client-side validation", func() {
 		ginkgo.It("should create/apply a CR with unknown fields for CRD with no validation schema", func() {
 			ginkgo.By("create CRD with no validation schema")
-			crd, err := framework.CreateTestCRD(f)
+			crd, err := crd.CreateTestCRD(f)
 			if err != nil {
 				framework.Failf("failed to create test CRD: %v", err)
 			}
@@ -841,7 +842,7 @@ metadata:
 
 		ginkgo.It("should create/apply a valid CR for CRD with validation schema", func() {
 			ginkgo.By("prepare CRD with validation schema")
-			crd, err := framework.CreateTestCRD(f)
+			crd, err := crd.CreateTestCRD(f)
 			if err != nil {
 				framework.Failf("failed to create test CRD: %v", err)
 			}
@@ -862,7 +863,7 @@ metadata:
 
 		ginkgo.It("should create/apply a valid CR with arbitrary-extra properties for CRD with partially-specified validation schema", func() {
 			ginkgo.By("prepare CRD with partially-specified validation schema")
-			crd, err := framework.CreateTestCRD(f)
+			crd, err := crd.CreateTestCRD(f)
 			if err != nil {
 				framework.Failf("failed to create test CRD: %v", err)
 			}
@@ -2226,7 +2227,7 @@ func startLocalProxy() (srv *httptest.Server, logs *bytes.Buffer) {
 
 // createApplyCustomResource asserts that given CustomResource be created and applied
 // without being rejected by client-side validation
-func createApplyCustomResource(resource, namespace, name string, crd *framework.TestCrd) error {
+func createApplyCustomResource(resource, namespace, name string, crd *crd.TestCrd) error {
 	ns := fmt.Sprintf("--namespace=%v", namespace)
 	ginkgo.By("successfully create CR")
 	if _, err := framework.RunKubectlInput(resource, ns, "create", "-f", "-"); err != nil {

test/e2e/framework/crd_util.go renamed to test/utils/crd/crd_util.go

@@ -14,7 +14,7 @@ See the License for the specific language governing permissions and
 limitations under the License.
 */
 
-package framework
+package crd
 
 import (
 	"fmt"
@@ -27,6 +27,7 @@ import (
 	"k8s.io/apimachinery/pkg/types"
 	utilyaml "k8s.io/apimachinery/pkg/util/yaml"
 	"k8s.io/client-go/dynamic"
+	"k8s.io/kubernetes/test/e2e/framework"
 )
 
 // CleanCrdFn declares the clean up function needed to remove the CRD
@@ -45,8 +46,8 @@ type TestCrd struct {
 }
 
 // CreateTestCRD creates a new CRD specifically for the calling test.
-func CreateMultiVersionTestCRD(f *Framework, group string, apiVersions []apiextensionsv1beta1.CustomResourceDefinitionVersion, conversionWebhook *apiextensionsv1beta1.WebhookClientConfig) (*TestCrd, error) {
-	suffix := randomSuffix()
+func CreateMultiVersionTestCRD(f *framework.Framework, group string, apiVersions []apiextensionsv1beta1.CustomResourceDefinitionVersion, conversionWebhook *apiextensionsv1beta1.WebhookClientConfig) (*TestCrd, error) {
+	suffix := framework.RandomSuffix()
 	name := fmt.Sprintf("e2e-test-%s-%s-crd", f.BaseName, suffix)
 	kind := fmt.Sprintf("E2e-test-%s-%s-crd", f.BaseName, suffix)
 	testcrd := &TestCrd{
@@ -57,19 +58,19 @@ func CreateMultiVersionTestCRD(f *Framework, group string, apiVersions []apiexte
 	}
 
 	// Creating a custom resource definition for use by assorted tests.
-	config, err := LoadConfig()
+	config, err := framework.LoadConfig()
 	if err != nil {
-		Failf("failed to load config: %v", err)
+		framework.Failf("failed to load config: %v", err)
 		return nil, err
 	}
 	apiExtensionClient, err := crdclientset.NewForConfig(config)
 	if err != nil {
-		Failf("failed to initialize apiExtensionClient: %v", err)
+		framework.Failf("failed to initialize apiExtensionClient: %v", err)
 		return nil, err
 	}
 	dynamicClient, err := dynamic.NewForConfig(config)
 	if err != nil {
-		Failf("failed to initialize dynamic client: %v", err)
+		framework.Failf("failed to initialize dynamic client: %v", err)
 		return nil, err
 	}
 
@@ -85,7 +86,7 @@ func CreateMultiVersionTestCRD(f *Framework, group string, apiVersions []apiexte
 	//create CRD and waits for the resource to be recognized and available.
 	crd, err = fixtures.CreateNewCustomResourceDefinitionWatchUnsafe(crd, apiExtensionClient)
 	if err != nil {
-		Failf("failed to create CustomResourceDefinition: %v", err)
+		framework.Failf("failed to create CustomResourceDefinition: %v", err)
 		return nil, err
 	}
 
@@ -103,15 +104,15 @@ func CreateMultiVersionTestCRD(f *Framework, group string, apiVersions []apiexte
 	testcrd.CleanUp = func() error {
 		err := fixtures.DeleteCustomResourceDefinition(crd, apiExtensionClient)
 		if err != nil {
-			Failf("failed to delete CustomResourceDefinition(%s): %v", name, err)
+			framework.Failf("failed to delete CustomResourceDefinition(%s): %v", name, err)
 		}
 		return err
 	}
 	return testcrd, nil
 }
 
 // CreateTestCRD creates a new CRD specifically for the calling test.
-func CreateTestCRD(f *Framework) (*TestCrd, error) {
+func CreateTestCRD(f *framework.Framework) (*TestCrd, error) {
 	group := fmt.Sprintf("%s-crd-test.k8s.io", f.BaseName)
 	apiVersions := []apiextensionsv1beta1.CustomResourceDefinitionVersion{
 		{
@@ -124,7 +125,7 @@ func CreateTestCRD(f *Framework) (*TestCrd, error) {
 }
 
 // CreateTestCRD creates a new CRD specifically for the calling test.
-func CreateMultiVersionTestCRDWithV1Storage(f *Framework) (*TestCrd, error) {
+func CreateMultiVersionTestCRDWithV1Storage(f *framework.Framework) (*TestCrd, error) {
 	group := fmt.Sprintf("%s-multiversion-crd-test.k8s.io", f.BaseName)
 	apiVersions := []apiextensionsv1beta1.CustomResourceDefinitionVersion{
 		{