Adds e2e tests for auditing (both file and webhook)

Signed-off-by: JoshVanL <[email protected]>

Author: JoshVanL

test/e2e/framework/helper/deploy.go

@@ -22,6 +22,7 @@ const (
 	ProxyName         = "kube-oidc-proxy-e2e"
 	IssuerName        = "oidc-issuer-e2e"
 	FakeAPIServerName = "fake-apiserver-e2e"
+	AuditWebhookName  = "audit-webhook-e2e"
 )
 
 func (h *Helper) DeployProxy(ns *corev1.Namespace, issuerURL *url.URL, clientID string,
@@ -266,6 +267,62 @@ func (h *Helper) DeployFakeAPIServer(ns string) ([]corev1.Volume, *url.URL, erro
 	return extraVolumes, appURL, nil
 }
 
+func (h *Helper) DeployAuditWebhook(ns, logPath string) (corev1.Volume, *url.URL, error) {
+	cnt := corev1.Container{
+		Name:            AuditWebhookName,
+		Image:           AuditWebhookName,
+		ImagePullPolicy: corev1.PullNever,
+		Args: []string{
+			"audit-webhook",
+			"--secure-port=6443",
+			"--tls-cert-file=/tls/cert.pem",
+			"--tls-private-key-file=/tls/key.pem",
+			"--audit-file-path=" + logPath,
+		},
+		VolumeMounts: []corev1.VolumeMount{
+			corev1.VolumeMount{
+				MountPath: "/tls",
+				Name:      "tls",
+				ReadOnly:  true,
+			},
+		},
+		Ports: []corev1.ContainerPort{
+			corev1.ContainerPort{
+				ContainerPort: 6443,
+			},
+		},
+	}
+
+	bundle, appURL, err := h.deployApp(ns, AuditWebhookName, corev1.ServiceTypeClusterIP, cnt)
+	if err != nil {
+		return corev1.Volume{}, nil, err
+	}
+
+	sec, err := h.KubeClient.CoreV1().Secrets(ns).Create(&corev1.Secret{
+		ObjectMeta: metav1.ObjectMeta{
+			GenerateName: "audit-webhook-ca-",
+			Namespace:    ns,
+		},
+		Data: map[string][]byte{
+			"ca.pem": bundle.CertBytes,
+		},
+	})
+	if err != nil {
+		return corev1.Volume{}, nil, err
+	}
+
+	auditWebhookCAVol := corev1.Volume{
+		Name: "audit-webhook-ca",
+		VolumeSource: corev1.VolumeSource{
+			Secret: &corev1.SecretVolumeSource{
+				SecretName: sec.Name,
+			},
+		},
+	}
+
+	return auditWebhookCAVol, appURL, nil
+}
+
 func (h *Helper) deployApp(ns, name string, serviceType corev1.ServiceType, container corev1.Container, volumes ...corev1.Volume) (*util.KeyBundle, *url.URL, error) {
 	host, appURL := h.appURL(ns, name, "6443")
 

test/e2e/suite/cases/audit/audit.go

@@ -0,0 +1,269 @@
+// Copyright Jetstack Ltd. See LICENSE for details.
+package passthrough
+
+import (
+	"bufio"
+	"bytes"
+	"encoding/json"
+	"fmt"
+	"io/ioutil"
+	"net/http"
+	"os"
+	"path/filepath"
+	"reflect"
+	"time"
+
+	. "github.com/onsi/ginkgo"
+	. "github.com/onsi/gomega"
+
+	authnv1 "k8s.io/api/authentication/v1"
+	corev1 "k8s.io/api/core/v1"
+	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
+	auditv1 "k8s.io/apiserver/pkg/apis/audit/v1"
+
+	"github.com/jetstack/kube-oidc-proxy/test/e2e/framework"
+)
+
+var _ = framework.CasesDescribe("Audit", func() {
+	f := framework.NewDefaultFramework("audit")
+
+	It("should be able to write audit logs to file", func() {
+		By("Creating policy file ConfigMap")
+		cm, err := f.Helper().KubeClient.CoreV1().ConfigMaps(f.Namespace.Name).Create(&corev1.ConfigMap{
+			ObjectMeta: metav1.ObjectMeta{
+				GenerateName: "kube-oidc-proxy-policy-",
+			},
+			Data: map[string]string{
+				"audit.yaml": `apiVersion: audit.k8s.io/v1
+kind: Policy
+rules:
+- level: RequestResponse`,
+			},
+		})
+		Expect(err).NotTo(HaveOccurred())
+
+		vols := []corev1.Volume{
+			corev1.Volume{
+				Name: "audit",
+				VolumeSource: corev1.VolumeSource{
+					ConfigMap: &corev1.ConfigMapVolumeSource{
+						LocalObjectReference: corev1.LocalObjectReference{
+							Name: cm.Name,
+						},
+					},
+				},
+			},
+		}
+
+		By("Deploying proxy with audit policy enabled")
+		f.DeployProxyWith(vols, "--audit-log-path=/audit-log", "--audit-policy-file=/audit/audit.yaml")
+
+		testAuditLogs(f, "app=kube-oidc-proxy-e2e")
+	})
+
+	It("should be able to write audit logs to webhook", func() {
+		By("Creating policy file ConfigMap")
+		cmPolicy, err := f.Helper().KubeClient.CoreV1().ConfigMaps(f.Namespace.Name).Create(&corev1.ConfigMap{
+			ObjectMeta: metav1.ObjectMeta{
+				GenerateName: "kube-oidc-proxy-policy-",
+			},
+			Data: map[string]string{
+				"audit.yaml": `apiVersion: audit.k8s.io/v1
+kind: Policy
+rules:
+- level: RequestResponse`,
+			},
+		})
+		Expect(err).NotTo(HaveOccurred())
+
+		extraWebhookVol, webhookURL, err := f.Helper().DeployAuditWebhook(f.Namespace.Name, "/audit-log")
+		Expect(err).NotTo(HaveOccurred())
+
+		cmWebhook, err := f.Helper().KubeClient.CoreV1().ConfigMaps(f.Namespace.Name).Create(&corev1.ConfigMap{
+			ObjectMeta: metav1.ObjectMeta{
+				GenerateName: "kube-oidc-proxy-webhook-config-",
+			},
+			Data: map[string]string{
+				"kubeconfig.yaml": `apiVersion: v1
+kind: Config
+clusters:
+- cluster:
+    server: ` + webhookURL.String() + `
+    certificate-authority: /audit-webhook-ca/ca.pem
+  name: logstash
+contexts:
+- context:
+    cluster: logstash
+    user: ""
+  name: default-context
+current-context: default-context
+preferences: {}
+users: []`,
+			},
+		})
+		Expect(err).NotTo(HaveOccurred())
+
+		vols := []corev1.Volume{
+			corev1.Volume{
+				Name: "audit",
+				VolumeSource: corev1.VolumeSource{
+					ConfigMap: &corev1.ConfigMapVolumeSource{
+						LocalObjectReference: corev1.LocalObjectReference{
+							Name: cmPolicy.Name,
+						},
+					},
+				},
+			},
+			corev1.Volume{
+				Name: "audit-webhook",
+				VolumeSource: corev1.VolumeSource{
+					ConfigMap: &corev1.ConfigMapVolumeSource{
+						LocalObjectReference: corev1.LocalObjectReference{
+							Name: cmWebhook.Name,
+						},
+					},
+				},
+			},
+			extraWebhookVol,
+		}
+
+		By("Deploying proxy with audit policy enabled")
+		f.DeployProxyWith(vols, "--audit-webhook-config-file=/audit-webhook/kubeconfig.yaml",
+			"--audit-policy-file=/audit/audit.yaml", "--audit-webhook-initial-backoff=1s", "--audit-webhook-batch-max-wait=1s")
+
+		testAuditLogs(f, "app=audit-webhook-e2e")
+	})
+})
+
+func testAuditLogs(f *framework.Framework, podLabelSelector string) {
+	By("Making calls to proxy to ensure audit get created")
+	token := f.Helper().NewTokenPayload(f.IssuerURL(), f.ClientID(), time.Now().Add(time.Second*5))
+	signedToken, err := f.Helper().SignToken(f.IssuerKeyBundle(), token)
+	Expect(err).NotTo(HaveOccurred())
+
+	proxyConfig := f.NewProxyRestConfig()
+	requester := f.Helper().NewRequester(proxyConfig.Transport, signedToken)
+
+	target := fmt.Sprintf("%s/api/v1/namespaces/kube-system/pods", proxyConfig.Host)
+
+	// Make request that should succeed
+	_, _, err = requester.Get(target)
+	Expect(err).NotTo(HaveOccurred())
+
+	// Make request that should be unauthenticated
+	requester = f.Helper().NewRequester(proxyConfig.Transport, "foo")
+	_, resp, err := requester.Get(target)
+	Expect(err).NotTo(HaveOccurred())
+
+	if resp.StatusCode != http.StatusUnauthorized {
+		Expect(fmt.Errorf("expected to get unauthorized, got=%d", resp.StatusCode)).NotTo(HaveOccurred())
+	}
+
+	By("Waiting for audit logs to be written")
+	time.Sleep(time.Second * 5)
+
+	By("Copying audit log from proxy locally")
+	// Get pod
+	pods, err := f.Helper().KubeClient.CoreV1().Pods(f.Namespace.Name).List(metav1.ListOptions{
+		LabelSelector: podLabelSelector,
+	})
+	Expect(err).NotTo(HaveOccurred())
+	if len(pods.Items) != 1 {
+		Expect(fmt.Errorf("expected single kube-oidc-proxy pod running, got=%d", len(pods.Items))).NotTo(HaveOccurred())
+	}
+
+	tmpDir, err := ioutil.TempDir(os.TempDir(), "kube-oidc-proxy-e2e")
+	Expect(err).NotTo(HaveOccurred())
+
+	defer func() {
+		err := os.RemoveAll(tmpDir)
+		Expect(err).NotTo(HaveOccurred())
+	}()
+
+	logFile := filepath.Join(tmpDir, "log.txt")
+
+	err = f.Helper().Kubectl(f.Namespace.Name).Run(
+		"cp", fmt.Sprintf("%s:audit-log", pods.Items[0].Name), logFile)
+	Expect(err).NotTo(HaveOccurred())
+
+	logs, err := ioutil.ReadFile(logFile)
+	Expect(err).NotTo(HaveOccurred())
+
+	scanner := bufio.NewScanner(bytes.NewReader(logs))
+
+	expAuditEvents := []auditv1.Event{
+		auditv1.Event{
+			Level:      auditv1.LevelRequestResponse,
+			Stage:      auditv1.StageRequestReceived,
+			RequestURI: "/api/v1/namespaces/kube-system/pods",
+			Verb:       "get",
+			User: authnv1.UserInfo{
+				Username: "[email protected]",
+				Groups:   []string{"group-1", "group-2"},
+			},
+		},
+		auditv1.Event{
+			Level:      auditv1.LevelRequestResponse,
+			Stage:      auditv1.StageResponseComplete,
+			RequestURI: "/api/v1/namespaces/kube-system/pods",
+			Verb:       "get",
+			User: authnv1.UserInfo{
+				Username: "[email protected]",
+				Groups:   []string{"group-1", "group-2"},
+			},
+			ResponseStatus: &metav1.Status{
+				Code: 403,
+			},
+		},
+		auditv1.Event{
+			Level:      auditv1.LevelRequestResponse,
+			Stage:      auditv1.StageResponseStarted,
+			RequestURI: "/api/v1/namespaces/kube-system/pods",
+			Verb:       "get",
+			ResponseStatus: &metav1.Status{
+				Code:    401,
+				Message: "Authentication failed, attempted: bearer",
+			},
+		},
+	}
+
+	By("Testing for expected audit logs")
+	var i int
+	for scanner.Scan() {
+		if i > len(expAuditEvents) {
+			Expect(fmt.Errorf("more proxy audit logs than expected, exp=%d got=%s", len(expAuditEvents), logs)).NotTo(HaveOccurred())
+		}
+
+		var auditEvent auditv1.Event
+		err = json.Unmarshal(scanner.Bytes(), &auditEvent)
+		Expect(err).NotTo(HaveOccurred())
+
+		gotAuditEvent := auditv1.Event{
+			Level:      auditEvent.Level,
+			Stage:      auditEvent.Stage,
+			RequestURI: auditEvent.RequestURI,
+			Verb:       auditEvent.Verb,
+			User: authnv1.UserInfo{
+				Username: auditEvent.User.Username,
+				Groups:   auditEvent.User.Groups,
+			},
+		}
+
+		if auditEvent.ResponseStatus != nil {
+			gotAuditEvent.ResponseStatus = &metav1.Status{
+				Code:    auditEvent.ResponseStatus.Code,
+				Message: auditEvent.ResponseStatus.Message,
+			}
+		}
+
+		if !reflect.DeepEqual(expAuditEvents[i], gotAuditEvent) {
+			Expect(fmt.Errorf("unexpected audit event\nexp=%v\ngot=%v", expAuditEvents[i], gotAuditEvent)).NotTo(HaveOccurred())
+		}
+
+		i++
+	}
+
+	if i != len(expAuditEvents) {
+		Expect(fmt.Errorf("less proxy audit logs then expected, exp=%d, got=%s", len(expAuditEvents), logs)).NotTo(HaveOccurred())
+	}
+}

test/e2e/suite/cases/doc.go

@@ -2,6 +2,7 @@
 package cases
 
 import (
+	_ "github.com/jetstack/kube-oidc-proxy/test/e2e/suite/cases/audit"
 	_ "github.com/jetstack/kube-oidc-proxy/test/e2e/suite/cases/headers"
 	_ "github.com/jetstack/kube-oidc-proxy/test/e2e/suite/cases/impersonation"
 	_ "github.com/jetstack/kube-oidc-proxy/test/e2e/suite/cases/passthrough"

test/e2e/suite/cases/upgrade/upgrade.go

@@ -140,7 +140,7 @@ var _ = framework.CasesDescribe("Upgrade", func() {
 			Expect(err).NotTo(HaveOccurred())
 		}
 
-		By(fmt.Sprintf("exec outpu t%s/%s: %s", pod.Namespace, pod.Name, execOut.String()))
+		By(fmt.Sprintf("exec output %s/%s: %s", pod.Namespace, pod.Name, execOut.String()))
 
 		// should have correct stdout output from echo server
 		if !strings.HasSuffix(execOut.String(), "BODY:\nhello world") {

test/kind/image.go

@@ -28,6 +28,10 @@ func (k *Kind) LoadAllImages() error {
 		return err
 	}
 
+	if err := k.LoadAuditWebhook(); err != nil {
+		return err
+	}
+
 	return nil
 }
 
@@ -57,6 +61,15 @@ func (k *Kind) LoadFakeAPIServer() error {
 	return k.loadImage(binPath, mainPath, image, dockerfilePath)
 }
 
+func (k *Kind) LoadAuditWebhook() error {
+	binPath := filepath.Join(k.rootPath, "./test/tools/audit-webhook/bin/audit-webhook")
+	dockerfilePath := filepath.Join(k.rootPath, "./test/tools/audit-webhook")
+	mainPath := filepath.Join(dockerfilePath, "cmd")
+	image := "audit-webhook-e2e"
+
+	return k.loadImage(binPath, mainPath, image, dockerfilePath)
+}
+
 func (k *Kind) loadImage(binPath, mainPath, image, dockerfilePath string) error {
 	log.Infof("kind: building %q", mainPath)
 

test/tools/audit-webhook/.gitignore

@@ -0,0 +1 @@
+/bin

test/tools/audit-webhook/Dockerfile

@@ -0,0 +1,10 @@
+# Copyright Jetstack Ltd. See LICENSE for details.
+FROM alpine:3.10
+
+LABEL description="A audit webhook sink to read audit events and write to file."
+
+RUN apk --no-cache --update add ca-certificates
+
+COPY ./bin/audit-webhook /usr/bin/audit-webhook
+
+CMD ["/usr/bin/audit-webhook"]