test/e2e: refactor sdk tests with improvements from pkg/test (#441)

* test/e2e: refactor sdk tests with improvements from pkg/test

The framework in pkg/test was originally based off of the sdk's
internal e2e tests. They have since been drastically improved and
many changes are applicable to the old sdk tests. This commit
refactors the sdk's internal tests with these improvements
(specifically, using the dynamic client and the apimachinery wait
util).

* test/e2e/framework: add comment about createCRFromYAML use

Author: AlexNPavel

test/e2e/e2eutil/check_util.go

@@ -20,13 +20,14 @@ import (
 
 	apierrors "k8s.io/apimachinery/pkg/api/errors"
 	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
+	"k8s.io/apimachinery/pkg/util/wait"
 	"k8s.io/client-go/kubernetes"
 )
 
 var retryInterval = time.Second * 5
 
 func DeploymentReplicaCheck(t *testing.T, kubeclient kubernetes.Interface, namespace, name string, replicas, retries int) error {
-	err := Retry(retryInterval, retries, func() (done bool, err error) {
+	err := wait.Poll(retryInterval, time.Duration(retries)*retryInterval, func() (done bool, err error) {
 		deployment, err := kubeclient.AppsV1().Deployments(namespace).Get(name, metav1.GetOptions{IncludeUninitialized: true})
 		if err != nil {
 			if apierrors.IsNotFound(err) {

test/e2e/e2eutil/retry_util.go

@@ -15,23 +15,42 @@
 package framework
 
 import (
+	goctx "context"
 	"flag"
+	"fmt"
 	"log"
 	"os"
+	"sync"
+	"time"
 
-	extensions "k8s.io/apiextensions-apiserver/pkg/client/clientset/clientset"
+	extscheme "k8s.io/apiextensions-apiserver/pkg/client/clientset/clientset/scheme"
+	"k8s.io/apimachinery/pkg/api/meta"
+	"k8s.io/apimachinery/pkg/runtime"
+	"k8s.io/apimachinery/pkg/runtime/serializer"
+	"k8s.io/apimachinery/pkg/util/wait"
+	"k8s.io/client-go/discovery"
+	"k8s.io/client-go/discovery/cached"
 	"k8s.io/client-go/kubernetes"
+	cgoscheme "k8s.io/client-go/kubernetes/scheme"
 	"k8s.io/client-go/rest"
 	"k8s.io/client-go/tools/clientcmd"
+	dynclient "sigs.k8s.io/controller-runtime/pkg/client"
 )
 
-var Global *Framework
+var (
+	// mutex for AddToFrameworkScheme
+	mutex = sync.Mutex{}
+	// Global framework struct
+	Global *Framework
+)
 
 type Framework struct {
-	KubeConfig       *rest.Config
-	KubeClient       kubernetes.Interface
-	ExtensionsClient *extensions.Clientset
-	ImageName        *string
+	KubeConfig     *rest.Config
+	KubeClient     kubernetes.Interface
+	Scheme         *runtime.Scheme
+	DynamicClient  dynclient.Client
+	DynamicDecoder runtime.Decoder
+	ImageName      *string
 }
 
 func setup() error {
@@ -54,15 +73,49 @@ func setup() error {
 	if err != nil {
 		return err
 	}
-	extensionsClient, err := extensions.NewForConfig(kubeconfig)
+	scheme := runtime.NewScheme()
+	cgoscheme.AddToScheme(scheme)
+	extscheme.AddToScheme(scheme)
+	dynClient, err := dynclient.New(kubeconfig, dynclient.Options{Scheme: scheme})
 	if err != nil {
-		return err
+		return fmt.Errorf("failed to build the dynamic client: %v", err)
 	}
+	dynDec := serializer.NewCodecFactory(scheme).UniversalDeserializer()
 	Global = &Framework{
-		KubeConfig:       kubeconfig,
-		KubeClient:       kubeclient,
-		ExtensionsClient: extensionsClient,
-		ImageName:        imageName,
+		KubeConfig:     kubeconfig,
+		KubeClient:     kubeclient,
+		Scheme:         scheme,
+		DynamicClient:  dynClient,
+		DynamicDecoder: dynDec,
+		ImageName:      imageName,
+	}
+	return nil
+}
+
+type addToSchemeFunc func(*runtime.Scheme) error
+
+func AddToFrameworkScheme(addToScheme addToSchemeFunc, obj runtime.Object) error {
+	mutex.Lock()
+	defer mutex.Unlock()
+	err := addToScheme(Global.Scheme)
+	if err != nil {
+		return err
+	}
+	cachedDiscoveryClient := cached.NewMemCacheClient(Global.KubeClient.Discovery())
+	restMapper := discovery.NewDeferredDiscoveryRESTMapper(cachedDiscoveryClient, meta.InterfacesForUnstructured)
+	restMapper.Reset()
+	Global.DynamicClient, err = dynclient.New(Global.KubeConfig, dynclient.Options{Scheme: Global.Scheme, Mapper: restMapper})
+	err = wait.PollImmediate(time.Second, time.Second*10, func() (done bool, err error) {
+		err = Global.DynamicClient.List(goctx.TODO(), &dynclient.ListOptions{Namespace: "default"}, obj)
+		if err != nil {
+			restMapper.Reset()
+			return false, nil
+		}
+		return true, nil
+	})
+	if err != nil {
+		return fmt.Errorf("failed to build the dynamic client: %v", err)
 	}
+	Global.DynamicDecoder = serializer.NewCodecFactory(Global.Scheme).UniversalDeserializer()
 	return nil
 }

test/e2e/framework/framework.go

@@ -15,17 +15,13 @@
 package framework
 
 import (
-	"errors"
+	"bytes"
+	goctx "context"
 	"strings"
 
 	y2j "github.com/ghodss/yaml"
 	yaml "gopkg.in/yaml.v2"
-	apps "k8s.io/api/apps/v1"
 	core "k8s.io/api/core/v1"
-	"k8s.io/api/rbac/v1beta1"
-	crd "k8s.io/apiextensions-apiserver/pkg/apis/apiextensions/v1beta1"
-	extensions_scheme "k8s.io/apiextensions-apiserver/pkg/client/clientset/clientset/scheme"
-	apierrors "k8s.io/apimachinery/pkg/api/errors"
 	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
 	"k8s.io/apimachinery/pkg/runtime/schema"
 	"k8s.io/apimachinery/pkg/runtime/serializer"
@@ -77,7 +73,7 @@ func (ctx *TestCtx) GetCRClient(yamlCR []byte) (*rest.RESTClient, error) {
 	return ctx.CRClient, nil
 }
 
-func (ctx *TestCtx) createCRFromYAML(yamlFile []byte, resourceName string) error {
+func (ctx *TestCtx) createCRFromYAML(yamlFile []byte) error {
 	client, err := ctx.GetCRClient(yamlFile)
 	if err != nil {
 		return err
@@ -86,6 +82,13 @@ func (ctx *TestCtx) createCRFromYAML(yamlFile []byte, resourceName string) error
 	if err != nil {
 		return err
 	}
+	yamlMap := make(map[interface{}]interface{})
+	err = yaml.Unmarshal(yamlFile, &yamlMap)
+	if err != nil {
+		return err
+	}
+	kind := yamlMap["kind"].(string)
+	resourceName := kind + "s"
 	jsonDat, err := y2j.YAMLToJSON(yamlFile)
 	err = client.Post().
 		Namespace(namespace).
@@ -104,80 +107,41 @@ func (ctx *TestCtx) createCRFromYAML(yamlFile []byte, resourceName string) error
 	return err
 }
 
-func (ctx *TestCtx) createCRDFromYAML(yamlFile []byte) error {
-	decode := extensions_scheme.Codecs.UniversalDeserializer().Decode
-	obj, _, err := decode(yamlFile, nil, nil)
+func setNamespaceYAML(yamlFile []byte, namespace string) ([]byte, error) {
+	yamlMap := make(map[interface{}]interface{})
+	err := yaml.Unmarshal(yamlFile, &yamlMap)
 	if err != nil {
-		return err
-	}
-	switch o := obj.(type) {
-	case *crd.CustomResourceDefinition:
-		_, err = Global.ExtensionsClient.ApiextensionsV1beta1().CustomResourceDefinitions().Create(o)
-		ctx.AddFinalizerFn(func() error {
-			err = Global.ExtensionsClient.ApiextensionsV1beta1().CustomResourceDefinitions().Delete(o.Name, metav1.NewDeleteOptions(0))
-			if err != nil && !apierrors.IsNotFound(err) {
-				return err
-			}
-			return nil
-		})
-		if apierrors.IsAlreadyExists(err) {
-			return nil
-		}
-		return err
-	default:
-		return errors.New("non-CRD resource in createCRDFromYAML function")
+		return nil, err
 	}
+	yamlMap["metadata"].(map[interface{}]interface{})["namespace"] = namespace
+	return yaml.Marshal(yamlMap)
 }
 
 func (ctx *TestCtx) CreateFromYAML(yamlFile []byte) error {
-	yamlMap := make(map[interface{}]interface{})
-	err := yaml.Unmarshal(yamlFile, &yamlMap)
+	namespace, err := ctx.GetNamespace()
 	if err != nil {
 		return err
 	}
-	kind := yamlMap["kind"].(string)
-	switch kind {
-	case "Role":
-	case "RoleBinding":
-	case "Deployment":
-	case "CustomResourceDefinition":
-		return ctx.createCRDFromYAML(yamlFile)
-	// we assume that all custom resources are from operator-sdk and thus follow
-	// a common naming convention
-	default:
-		return ctx.createCRFromYAML(yamlFile, strings.ToLower(kind)+"s")
-	}
+	yamlSplit := bytes.Split(yamlFile, []byte("\n---\n"))
+	for _, yamlSpec := range yamlSplit {
+		yamlSpec, err = setNamespaceYAML(yamlSpec, namespace)
+		if err != nil {
+			return err
+		}
 
-	decode := scheme.Codecs.UniversalDeserializer().Decode
-	obj, _, err := decode(yamlFile, nil, nil)
-	if err != nil {
-		return err
-	}
+		obj, _, err := Global.DynamicDecoder.Decode(yamlSpec, nil, nil)
+		if err != nil {
+			// DynamicClient/DynamicDecoder can only handle standard and extensions kubernetes resources.
+			// If a resource is not recognized by the decoder, assume it's a custom resource and fall back
+			// to createCRFromYAML.
+			return ctx.createCRFromYAML(yamlFile)
+		}
 
-	namespace, err := ctx.GetNamespace()
-	if err != nil {
-		return err
-	}
-	switch o := obj.(type) {
-	case *v1beta1.Role:
-		_, err = Global.KubeClient.RbacV1beta1().Roles(namespace).Create(o)
-		ctx.AddFinalizerFn(func() error {
-			return Global.KubeClient.RbacV1beta1().Roles(namespace).Delete(o.Name, metav1.NewDeleteOptions(0))
-		})
-		return err
-	case *v1beta1.RoleBinding:
-		_, err = Global.KubeClient.RbacV1beta1().RoleBindings(namespace).Create(o)
-		ctx.AddFinalizerFn(func() error {
-			return Global.KubeClient.RbacV1beta1().RoleBindings(namespace).Delete(o.Name, metav1.NewDeleteOptions(0))
-		})
-		return err
-	case *apps.Deployment:
-		_, err = Global.KubeClient.AppsV1().Deployments(namespace).Create(o)
-		ctx.AddFinalizerFn(func() error {
-			return Global.KubeClient.AppsV1().Deployments(namespace).Delete(o.Name, metav1.NewDeleteOptions(0))
-		})
-		return err
-	default:
-		return errors.New("unhandled resource type")
+		err = Global.DynamicClient.Create(goctx.TODO(), obj)
+		if err != nil {
+			return err
+		}
+		ctx.AddFinalizerFn(func() error { return Global.DynamicClient.Delete(goctx.TODO(), obj) })
 	}
+	return nil
 }

test/e2e/framework/resource_creator.go

@@ -29,6 +29,7 @@ import (
 	framework "github.com/operator-framework/operator-sdk/test/e2e/framework"
 
 	"k8s.io/apimachinery/pkg/types"
+	"k8s.io/apimachinery/pkg/util/wait"
 )
 
 const (
@@ -185,7 +186,7 @@ func MemcachedLocal(t *testing.T) {
 	ctx.AddFinalizerFn(func() error { return cmd.Process.Signal(os.Interrupt) })
 
 	// wait for operator to start (may take a minute to compile the command...)
-	err = e2eutil.Retry(time.Second*5, 16, func() (done bool, err error) {
+	err = wait.Poll(time.Second*5, time.Second*80, func() (done bool, err error) {
 		file, err := ioutil.ReadFile("stderr.txt")
 		if err != nil {
 			return false, err