Adds proxy to main proxy and it's handlers

Signed-off-by: JoshVanL <[email protected]>

Author: JoshVanL

pkg/proxy/context/context.go

@@ -3,6 +3,7 @@ package context
 
 import (
 	"net/http"
+	"time"
 
 	"github.com/sebest/xff"
 	"k8s.io/apiserver/pkg/endpoints/request"
@@ -21,8 +22,11 @@ const (
 	// bearerTokenKey is the context key for the bearer token.
 	bearerTokenKey
 
-	// bearerTokenKey is the context key for the client address.
+	// clientAddressKey is the context key for the client address.
 	clientAddressKey
+
+	// clientRequestTimestampKey is the context key for the timestamp of a client request.
+	clientRequestTimestampKey
 )
 
 // WithNoImpersonation returns a copy of the request in which the noImpersonation context value is set.
@@ -58,6 +62,17 @@ func BearerToken(req *http.Request) string {
 	return token
 }
 
+// WithClientRequestTimestamp will add the current timestamp to the request context.
+func WithClientRequestTimestamp(req *http.Request) *http.Request {
+	return req.WithContext(request.WithValue(req.Context(), clientRequestTimestampKey, time.Now()))
+}
+
+// ClientRequestTimestampKey will return thetimestamp that the client request was received.
+func ClientRequestTimestamp(req *http.Request) time.Time {
+	stamp, _ := req.Context().Value(clientRequestTimestampKey).(time.Time)
+	return stamp
+}
+
 // RemoteAddress will attempt to return the source client address if available
 // in the request context. If it is not, it will be gathered from the request
 // and entered into the context.

pkg/proxy/handlers.go

@@ -2,8 +2,10 @@
 package proxy
 
 import (
+	"errors"
 	"net/http"
 	"strings"
+	"time"
 
 	authuser "k8s.io/apiserver/pkg/authentication/user"
 	genericapirequest "k8s.io/apiserver/pkg/endpoints/request"
@@ -14,14 +16,33 @@ import (
 	"github.com/jetstack/kube-oidc-proxy/pkg/proxy/context"
 )
 
+const (
+	UserHeaderClientIPKey = "Remote-Client-IP"
+)
+
+var (
+	errUnauthorized          = errors.New("Unauthorized")
+	errImpersonateHeader     = errors.New("Impersonate-User in header")
+	errNoName                = errors.New("No name in OIDC info")
+	errNoImpersonationConfig = errors.New("No impersonation configuration in context")
+
+	// http headers are case-insensitive
+	impersonateUserHeader  = strings.ToLower(transport.ImpersonateUserHeader)
+	impersonateGroupHeader = strings.ToLower(transport.ImpersonateGroupHeader)
+	impersonateExtraHeader = strings.ToLower(transport.ImpersonateUserExtraHeaderPrefix)
+)
+
 func (p *Proxy) withHandlers(handler http.Handler) http.Handler {
 	// Set up proxy handlers
+	handler = p.withClientTimestamp(handler)
 	handler = p.auditor.WithRequest(handler)
 	handler = p.withImpersonateRequest(handler)
 	handler = p.withAuthenticateRequest(handler)
 
 	// Add the auditor backend as a shutdown hook
 	p.hooks.AddPreShutdownHook("AuditBackend", p.auditor.Shutdown)
+	// Add the metrics server as a shutdown hook
+	p.hooks.AddPreShutdownHook("Metrics", p.metrics.Shutdown)
 
 	return handler
 }
@@ -31,27 +52,29 @@ func (p *Proxy) withAuthenticateRequest(handler http.Handler) http.Handler {
 	tokenReviewHandler := p.withTokenReview(handler)
 
 	return http.HandlerFunc(func(rw http.ResponseWriter, req *http.Request) {
+		req, remoteAddr := context.RemoteAddr(req)
+
 		// Auth request and handle unauthed
 		info, ok, err := p.oidcRequestAuther.AuthenticateRequest(req)
 		if err != nil {
 			// Since we have failed OIDC auth, we will try a token review, if enabled.
+			p.metrics.IncrementOIDCAuthCount(false, remoteAddr, "")
 			tokenReviewHandler.ServeHTTP(rw, req)
 			return
 		}
 
 		// Failed authorization
 		if !ok {
+			p.metrics.IncrementOIDCAuthCount(false, remoteAddr, "")
 			p.handleError(rw, req, errUnauthorized)
 			return
 		}
 
-		var remoteAddr string
-		req, remoteAddr = context.RemoteAddr(req)
-
 		klog.V(4).Infof("authenticated request: %s", remoteAddr)
 
 		// Add the user info to the request context
 		req = req.WithContext(genericapirequest.WithUser(req.Context(), info.User))
+		p.metrics.IncrementOIDCAuthCount(true, remoteAddr, info.User.GetName())
 		handler.ServeHTTP(rw, req)
 	})
 }
@@ -89,8 +112,7 @@ func (p *Proxy) withImpersonateRequest(handler http.Handler) http.Handler {
 			return
 		}
 
-		var remoteAddr string
-		req, remoteAddr = context.RemoteAddr(req)
+		req, remoteAddr := context.RemoteAddr(req)
 
 		// If we have disabled impersonation we can forward the request right away
 		if p.config.DisableImpersonation {
@@ -163,14 +185,35 @@ func (p *Proxy) withImpersonateRequest(handler http.Handler) http.Handler {
 	})
 }
 
+// withClientTimestamp adds the current timestamp for the client request to the
+// request contect.
+func (p *Proxy) withClientTimestamp(handler http.Handler) http.Handler {
+	return http.HandlerFunc(func(rw http.ResponseWriter, req *http.Request) {
+		req = context.WithClientRequestTimestamp(req)
+		handler.ServeHTTP(rw, req)
+	})
+}
+
 // newErrorHandler returns a handler failed requests.
-func (p *Proxy) newErrorHandler() func(rw http.ResponseWriter, r *http.Request, err error) {
-	unauthedHandler := audit.NewUnauthenticatedHandler(p.auditor, func(rw http.ResponseWriter, r *http.Request) {
-		klog.V(2).Infof("unauthenticated user request %s", r.RemoteAddr)
+func (p *Proxy) newErrorHandler() func(rw http.ResponseWriter, req *http.Request, err error) {
+
+	// Setup unauthed handler so that it is passed through the audit
+	unauthedHandler := audit.NewUnauthenticatedHandler(p.auditor, func(rw http.ResponseWriter, req *http.Request) {
+		_, remoteAddr := context.RemoteAddr(req)
+		klog.V(2).Infof("unauthenticated user request %s", remoteAddr)
 		http.Error(rw, "Unauthorized", http.StatusUnauthorized)
 	})
 
-	return func(rw http.ResponseWriter, r *http.Request, err error) {
+	return func(rw http.ResponseWriter, req *http.Request, err error) {
+		var statusCode int
+		req, remoteAddr := context.RemoteAddr(req)
+
+		// Update client duration metrics from error
+		defer func() {
+			clientDuration := context.ClientRequestTimestamp(req)
+			p.metrics.ObserveClient(statusCode, req.URL.Path, remoteAddr, time.Since(clientDuration))
+		}()
+
 		if err == nil {
 			klog.Error("error was called with no error")
 			http.Error(rw, "", http.StatusInternalServerError)
@@ -182,30 +225,35 @@ func (p *Proxy) newErrorHandler() func(rw http.ResponseWriter, r *http.Request,
 		// Failed auth
 		case errUnauthorized:
 			// If Unauthorized then error and report to audit
-			unauthedHandler.ServeHTTP(rw, r)
+			statusCode = http.StatusUnauthorized
+			unauthedHandler.ServeHTTP(rw, req)
 			return
 
 			// User request with impersonation
 		case errImpersonateHeader:
-			klog.V(2).Infof("impersonation user request %s", r.RemoteAddr)
+			statusCode = http.StatusForbidden
+			klog.V(2).Infof("impersonation user request %s", remoteAddr)
 			http.Error(rw, "Impersonation requests are disabled when using kube-oidc-proxy", http.StatusForbidden)
 			return
 
 			// No name given or available in oidc request
 		case errNoName:
-			klog.V(2).Infof("no name available in oidc info %s", r.RemoteAddr)
+			statusCode = http.StatusForbidden
+			klog.V(2).Infof("no name available in oidc info %s", remoteAddr)
 			http.Error(rw, "Username claim not available in OIDC Issuer response", http.StatusForbidden)
 			return
 
 			// No impersonation configuration found in context
 		case errNoImpersonationConfig:
-			klog.Errorf("if you are seeing this, there is likely a bug in the proxy (%s): %s", r.RemoteAddr, err)
+			statusCode = http.StatusInternalServerError
+			klog.Errorf("if you are seeing this, there is likely a bug in the proxy (%s): %s", remoteAddr, err)
 			http.Error(rw, "", http.StatusInternalServerError)
 			return
 
 			// Server or unknown error
 		default:
-			klog.Errorf("unknown error (%s): %s", r.RemoteAddr, err)
+			statusCode = http.StatusInternalServerError
+			klog.Errorf("unknown error (%s): %s", remoteAddr, err)
 			http.Error(rw, "", http.StatusInternalServerError)
 		}
 	}

pkg/proxy/proxy.go

@@ -2,12 +2,10 @@
 package proxy
 
 import (
-	"errors"
 	"fmt"
 	"net/http"
 	"net/http/httputil"
 	"net/url"
-	"strings"
 	"time"
 
 	"k8s.io/apiserver/pkg/authentication/authenticator"
@@ -19,28 +17,13 @@ import (
 	"k8s.io/klog"
 
 	"github.com/jetstack/kube-oidc-proxy/cmd/app/options"
+	"github.com/jetstack/kube-oidc-proxy/pkg/metrics"
 	"github.com/jetstack/kube-oidc-proxy/pkg/proxy/audit"
 	"github.com/jetstack/kube-oidc-proxy/pkg/proxy/context"
 	"github.com/jetstack/kube-oidc-proxy/pkg/proxy/hooks"
 	"github.com/jetstack/kube-oidc-proxy/pkg/proxy/tokenreview"
 )
 
-const (
-	UserHeaderClientIPKey = "Remote-Client-IP"
-)
-
-var (
-	errUnauthorized          = errors.New("Unauthorized")
-	errImpersonateHeader     = errors.New("Impersonate-User in header")
-	errNoName                = errors.New("No name in OIDC info")
-	errNoImpersonationConfig = errors.New("No impersonation configuration in context")
-
-	// http headers are case-insensitive
-	impersonateUserHeader  = strings.ToLower(transport.ImpersonateUserHeader)
-	impersonateGroupHeader = strings.ToLower(transport.ImpersonateGroupHeader)
-	impersonateExtraHeader = strings.ToLower(transport.ImpersonateUserExtraHeaderPrefix)
-)
-
 type Config struct {
 	DisableImpersonation bool
 	TokenReview          bool
@@ -68,6 +51,7 @@ type Proxy struct {
 	config *Config
 
 	hooks       *hooks.Hooks
+	metrics     *metrics.Metrics
 	handleError errorHandlerFn
 }
 
@@ -76,6 +60,8 @@ func New(restConfig *rest.Config,
 	auditOptions *options.AuditOptions,
 	tokenReviewer *tokenreview.TokenReview,
 	ssinfo *server.SecureServingInfo,
+	hooks *hooks.Hooks,
+	metrics *metrics.Metrics,
 	config *Config) (*Proxy, error) {
 
 	// generate tokenAuther from oidc config
@@ -101,7 +87,8 @@ func New(restConfig *rest.Config,
 
 	return &Proxy{
 		restConfig:        restConfig,
-		hooks:             hooks.New(),
+		hooks:             hooks,
+		metrics:           metrics,
 		tokenReviewer:     tokenReviewer,
 		secureServingInfo: ssinfo,
 		config:            config,
@@ -199,8 +186,22 @@ func (p *Proxy) RoundTrip(req *http.Request) (*http.Response, error) {
 	// Set up impersonation request.
 	rt := transport.NewImpersonatingRoundTripper(*conf, p.clientTransport)
 
+	req, remoteAddr := context.RemoteAddr(req)
+	serverDuration := time.Now()
+	clientDuration := context.ClientRequestTimestamp(req)
+
 	// Push request through round trippers to the API server.
-	return rt.RoundTrip(req)
+	resp, err := rt.RoundTrip(req)
+
+	var statusCode int
+	if resp != nil {
+		statusCode = resp.StatusCode
+	}
+
+	p.metrics.ObserveClient(statusCode, req.URL.Path, remoteAddr, time.Since(clientDuration))
+	p.metrics.ObserveServer(statusCode, req.URL.Path, remoteAddr, time.Since(serverDuration))
+
+	return resp, err
 }
 
 func (p *Proxy) reviewToken(rw http.ResponseWriter, req *http.Request) bool {
@@ -259,7 +260,3 @@ func (p *Proxy) roundTripperForRestConfig(config *rest.Config) (http.RoundTrippe
 func (p *Proxy) OIDCTokenAuthenticator() authenticator.Token {
 	return p.tokenAuther
 }
-
-func (p *Proxy) RunPreShutdownHooks() error {
-	return p.hooks.RunPreShutdownHooks()
-}

pkg/proxy/proxy_test.go

@@ -21,6 +21,7 @@ import (
 	"k8s.io/apiserver/pkg/server"
 
 	"github.com/jetstack/kube-oidc-proxy/cmd/app/options"
+	"github.com/jetstack/kube-oidc-proxy/pkg/metrics"
 	"github.com/jetstack/kube-oidc-proxy/pkg/mocks"
 	"github.com/jetstack/kube-oidc-proxy/pkg/proxy/audit"
 	"github.com/jetstack/kube-oidc-proxy/pkg/proxy/hooks"
@@ -63,6 +64,7 @@ func (f *fakeRW) Header() http.Header {
 func newFakeR() *http.Request {
 	return &http.Request{
 		RemoteAddr: "fakeAddr",
+		URL:        new(url.URL),
 	}
 }
 
@@ -117,8 +119,7 @@ func (f *fakeRT) RoundTrip(h *http.Request) (*http.Response, error) {
 }
 
 func tryError(t *testing.T, expCode int, err error) *fakeRW {
-	p := new(Proxy)
-	p.handleError = p.newErrorHandler()
+	p := newTestProxy(t)
 
 	frw := newFakeRW()
 	fr := newFakeR()
@@ -169,7 +170,7 @@ func TestError(t *testing.T) {
 }
 
 func TestHasImpersonation(t *testing.T) {
-	p := new(Proxy)
+	p := newTestProxy(t)
 
 	// no impersonation headers
 	noImpersonation := []http.Header{
@@ -269,6 +270,7 @@ func newTestProxy(t *testing.T) *fakeProxy {
 			noAuthClientTransport: fakeRT,
 			config:                new(Config),
 			hooks:                 hooks.New(),
+			metrics:               metrics.New(),
 		},
 	}
 
@@ -425,7 +427,7 @@ func TestHandlers(t *testing.T) {
 			expAuthToken: "fake-token",
 			authResponse: &authResponse{
 				resp: &authenticator.Response{
-					User: nil,
+					User: &user.DefaultInfo{},
 				},
 				pass: true,
 				err:  nil,