Improves dev_cluster_deploy and add optional fake server

Signed-off-by: JoshVanL <[email protected]>

Author: JoshVanL

test/e2e/framework/helper/deploy.go

@@ -209,7 +209,7 @@ func (h *Helper) DeployIssuer(ns string) (*util.KeyBundle, *url.URL, error) {
 	return bundle, appURL, nil
 }
 
-func (h *Helper) DeployFakeAPIServer(ns string) (*util.KeyBundle, *url.URL, error) {
+func (h *Helper) DeployFakeAPIServer(ns string) ([]corev1.Volume, *url.URL, error) {
 	cnt := corev1.Container{
 		Name:            FakeAPIServerName,
 		Image:           FakeAPIServerName,
@@ -239,7 +239,31 @@ func (h *Helper) DeployFakeAPIServer(ns string) (*util.KeyBundle, *url.URL, erro
 		return nil, nil, err
 	}
 
-	return bundle, appURL, nil
+	sec, err := h.KubeClient.CoreV1().Secrets(ns).Create(&corev1.Secret{
+		ObjectMeta: metav1.ObjectMeta{
+			GenerateName: "fake-apiserver-ca-",
+			Namespace:    ns,
+		},
+		Data: map[string][]byte{
+			"ca.pem": bundle.CertBytes,
+		},
+	})
+	if err != nil {
+		return nil, nil, err
+	}
+
+	extraVolumes := []corev1.Volume{
+		{
+			Name: "fake-apiserver",
+			VolumeSource: corev1.VolumeSource{
+				Secret: &corev1.SecretVolumeSource{
+					SecretName: sec.Name,
+				},
+			},
+		},
+	}
+
+	return extraVolumes, appURL, nil
 }
 
 func (h *Helper) deployApp(ns, name string, serviceType corev1.ServiceType, container corev1.Container, volumes ...corev1.Volume) (*util.KeyBundle, *url.URL, error) {

test/e2e/suite/cases/headers/headers.go

@@ -4,16 +4,12 @@ package headers
 import (
 	"fmt"
 	"net/http"
-	"net/url"
 	"strings"
 	"time"
 
 	. "github.com/onsi/ginkgo"
 	. "github.com/onsi/gomega"
 
-	corev1 "k8s.io/api/core/v1"
-	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
-
 	"github.com/jetstack/kube-oidc-proxy/test/e2e/framework"
 	testutil "github.com/jetstack/kube-oidc-proxy/test/util"
 )
@@ -28,10 +24,11 @@ var _ = framework.CasesDescribe("Headers", func() {
 	})
 
 	It("should not respond with any extra headers if none are set on the proxy", func() {
-		fakeAPIServerURL, extraVolumes := deployFakeAPIServer(f)
+		extraOIDCVolumes, fakeAPIServerURL, err := f.Helper().DeployFakeAPIServer(f.Namespace.Name)
+		Expect(err).NotTo(HaveOccurred())
 
 		By("Redeploying proxy to send traffic to fake API server")
-		f.DeployProxyWith(extraVolumes, fmt.Sprintf("--server=%s", fakeAPIServerURL), "--certificate-authority=/fake-apiserver/ca.pem")
+		f.DeployProxyWith(extraOIDCVolumes, fmt.Sprintf("--server=%s", fakeAPIServerURL), "--certificate-authority=/fake-apiserver/ca.pem")
 
 		resp := sendRequestToProxy(f)
 
@@ -44,10 +41,12 @@ var _ = framework.CasesDescribe("Headers", func() {
 	})
 
 	It("should respond with remote address and custom extra headers when they are set", func() {
-		fakeAPIServerURL, extraVolumes := deployFakeAPIServer(f)
+		By("Deploying fake API Server")
+		extraOIDCVolumes, fakeAPIServerURL, err := f.Helper().DeployFakeAPIServer(f.Namespace.Name)
+		Expect(err).NotTo(HaveOccurred())
 
 		By("Redeploying proxy to send traffic to fake API server with extra headers set")
-		f.DeployProxyWith(extraVolumes, fmt.Sprintf("--server=%s", fakeAPIServerURL), "--certificate-authority=/fake-apiserver/ca.pem",
+		f.DeployProxyWith(extraOIDCVolumes, fmt.Sprintf("--server=%s", fakeAPIServerURL), "--certificate-authority=/fake-apiserver/ca.pem",
 			"--extra-user-header-client-ip", "--extra-user-headers=key1=foo,key2=foo,key1=bar")
 
 		resp := sendRequestToProxy(f)
@@ -86,36 +85,6 @@ var _ = framework.CasesDescribe("Headers", func() {
 	})
 })
 
-func deployFakeAPIServer(f *framework.Framework) (*url.URL, []corev1.Volume) {
-	By("Deploying fake API Server")
-	fAPIServerBundle, fakeAPIServerURL, err := f.Helper().DeployFakeAPIServer(f.Namespace.Name)
-	Expect(err).NotTo(HaveOccurred())
-
-	sec, err := f.KubeClientSet.CoreV1().Secrets(f.Namespace.Name).Create(&corev1.Secret{
-		ObjectMeta: metav1.ObjectMeta{
-			GenerateName: "fake-apiserver-ca-",
-			Namespace:    f.Namespace.Name,
-		},
-		Data: map[string][]byte{
-			"ca.pem": fAPIServerBundle.CertBytes,
-		},
-	})
-	Expect(err).NotTo(HaveOccurred())
-
-	extraVolumes := []corev1.Volume{
-		{
-			Name: "fake-apiserver",
-			VolumeSource: corev1.VolumeSource{
-				Secret: &corev1.SecretVolumeSource{
-					SecretName: sec.Name,
-				},
-			},
-		},
-	}
-
-	return fakeAPIServerURL, extraVolumes
-}
-
 func sendRequestToProxy(f *framework.Framework) *http.Response {
 	By("Building request to proxy")
 	tokenPayload := f.Helper().NewTokenPayload(

test/environment/dev/dev.go

@@ -3,8 +3,10 @@ package main
 
 import (
 	"fmt"
+	"net/url"
 	"os"
 	"path/filepath"
+	"strings"
 	"time"
 
 	corev1 "k8s.io/api/core/v1"
@@ -20,6 +22,8 @@ import (
 
 const (
 	clientID = "kube-oidc-proxy-e2e-client-id"
+
+	EnvFakeAPIServer = "KUBE_OIDC_PROXY_FAKE_APISERVER"
 )
 
 func main() {
@@ -44,23 +48,28 @@ func main() {
 }
 
 func create() {
-	env, err := environment.Create(1, 1)
+	env, err := environment.New(1, 1)
 	errExit(err)
 
+	errExit(env.Create())
+
 	fmt.Printf("> dev environment created.\n")
 	fmt.Printf("export KUBECONFIG=%s\n", env.KubeConfigPath())
 }
 
 func deploy() {
-	k := new(kind.Kind)
-	kubeconfig := k.KubeConfigPath()
-	rootPath, err := environment.RootPath()
+	env, err := environment.New(1, 1)
 	errExit(err)
 
+	fmt.Printf("> reloading all images\n")
+	errExit(env.Kind().LoadAllImages())
+
+	kubeconfig := env.KubeConfigPath()
+
 	cfg := &config.Config{
 		KubeConfigPath: kubeconfig,
-		RepoRoot:       rootPath,
-		Kubectl:        filepath.Join(rootPath, "bin", "kubectl"),
+		RepoRoot:       env.RootPath(),
+		Kubectl:        filepath.Join(env.RootPath(), "bin", "kubectl"),
 	}
 
 	err = cfg.Validate()
@@ -93,9 +102,24 @@ func deploy() {
 
 	fmt.Printf("> deployed issuer at url %s\n", issuerURL)
 
-	_, proxyURL, err := helper.DeployProxy(ns, issuerURL,
-		"kube-oidc-proxy-e2e-client-id", issuerKeyBundle, nil)
-	errExit(err)
+	var proxyURL *url.URL
+	if e := os.Getenv(EnvFakeAPIServer); strings.ToLower(e) == "true" {
+		extraOIDCVolume, fURL, err := helper.DeployFakeAPIServer(ns.Name)
+		errExit(err)
+
+		fmt.Printf("> deployed fake API server at url %s\n", fURL)
+
+		_, proxyURL, err = helper.DeployProxy(ns, issuerURL,
+			"kube-oidc-proxy-e2e-client-id", issuerKeyBundle, extraOIDCVolume,
+			fmt.Sprintf("--server=%s", fURL), "--certificate-authority=/fake-apiserver/ca.pem")
+		errExit(err)
+
+	} else {
+		_, proxyURL, err = helper.DeployProxy(ns, issuerURL,
+			"kube-oidc-proxy-e2e-client-id", issuerKeyBundle, nil)
+		errExit(err)
+	}
+
 	fmt.Printf("> deployed proxy at url %s\n", proxyURL)
 
 	tokenPayload := helper.NewTokenPayload(issuerURL, clientID, time.Now().Add(time.Hour*48))

test/environment/environment.go

@@ -20,10 +20,11 @@ const (
 type Environment struct {
 	kind *kind.Kind
 
-	rootPath string
+	rootPath  string
+	nodeImage string
 }
 
-func Create(masterNodes, workerNodes int) (*Environment, error) {
+func New(masterNodesCount, workerNodesCount int) (*Environment, error) {
 	nodeImage := os.Getenv("KUBE_OIDC_PROXY_K8S_VERSION")
 	if nodeImage == "" {
 		nodeImage = defaultNodeImage
@@ -35,27 +36,25 @@ func Create(masterNodes, workerNodes int) (*Environment, error) {
 		return nil, err
 	}
 
-	k, err := kind.New(rootPath, nodeImage, masterNodes, workerNodes)
-	if err != nil {
-		return nil, fmt.Errorf("failed to create kind cluster: %s", err)
-	}
+	kind := kind.New(rootPath, nodeImage, masterNodesCount, workerNodesCount)
 
-	if err := k.LoadKubeOIDCProxy(); err != nil {
-		return nil, err
-	}
+	return &Environment{
+		rootPath:  rootPath,
+		nodeImage: nodeImage,
+		kind:      kind,
+	}, nil
+}
 
-	if err := k.LoadIssuer(); err != nil {
-		return nil, err
+func (e *Environment) Create() error {
+	if err := e.kind.Create(); err != nil {
+		return fmt.Errorf("failed to create kind cluster: %s", err)
 	}
 
-	if err := k.LoadFakeAPIServer(); err != nil {
-		return nil, err
+	if err := e.kind.LoadAllImages(); err != nil {
+		return err
 	}
 
-	return &Environment{
-		kind:     k,
-		rootPath: rootPath,
-	}, nil
+	return nil
 }
 
 func (e *Environment) Destory() error {
@@ -101,6 +100,10 @@ func (e *Environment) Node(name string) (*nodes.Node, error) {
 	return node, nil
 }
 
+func (e *Environment) Kind() *kind.Kind {
+	return e.kind
+}
+
 func RootPath() (string, error) {
 	rootPath := os.Getenv("KUBE_OIDC_PROXY_ROOT_PATH")
 	if rootPath == "" {

test/kind/image.go

@@ -15,6 +15,22 @@ import (
 	"sigs.k8s.io/kind/pkg/cluster/nodeutils"
 )
 
+func (k *Kind) LoadAllImages() error {
+	if err := k.LoadKubeOIDCProxy(); err != nil {
+		return err
+	}
+
+	if err := k.LoadIssuer(); err != nil {
+		return err
+	}
+
+	if err := k.LoadFakeAPIServer(); err != nil {
+		return err
+	}
+
+	return nil
+}
+
 func (k *Kind) LoadKubeOIDCProxy() error {
 	binPath := filepath.Join(k.rootPath, "./bin/kube-oidc-proxy")
 	mainPath := filepath.Join(k.rootPath, "./cmd/.")
@@ -24,17 +40,17 @@ func (k *Kind) LoadKubeOIDCProxy() error {
 }
 
 func (k *Kind) LoadIssuer() error {
-	binPath := filepath.Join(k.rootPath, "./test/e2e/framework/issuer/bin/oidc-issuer")
-	dockerfilePath := filepath.Join(k.rootPath, "./test/e2e/framework/issuer")
+	binPath := filepath.Join(k.rootPath, "./test/tools/issuer/bin/oidc-issuer")
+	dockerfilePath := filepath.Join(k.rootPath, "./test/tools/issuer")
 	mainPath := filepath.Join(dockerfilePath, "cmd")
 	image := "oidc-issuer-e2e"
 
 	return k.loadImage(binPath, mainPath, image, dockerfilePath)
 }
 
 func (k *Kind) LoadFakeAPIServer() error {
-	binPath := filepath.Join(k.rootPath, "./test/e2e/framework/fake-apiserver/bin/fake-apiserver")
-	dockerfilePath := filepath.Join(k.rootPath, "./test/e2e/framework/fake-apiserver")
+	binPath := filepath.Join(k.rootPath, "./test/tools/fake-apiserver/bin/fake-apiserver")
+	dockerfilePath := filepath.Join(k.rootPath, "./test/tools/fake-apiserver")
 	mainPath := filepath.Join(dockerfilePath, "cmd")
 	image := "fake-apiserver-e2e"