Moves all probe logic into ./pkg/probe

Signed-off-by: JoshVanL <[email protected]>

Author: JoshVanL

cmd/app/run.go

@@ -11,8 +11,6 @@ import (
 	"k8s.io/apiserver/pkg/server"
 	apiserveroptions "k8s.io/apiserver/pkg/server/options"
 	"k8s.io/apiserver/pkg/util/term"
-	"k8s.io/apiserver/plugin/pkg/authenticator/token/oidc"
-	"k8s.io/cli-runtime/pkg/genericclioptions"
 	"k8s.io/client-go/rest"
 	cliflag "k8s.io/component-base/cli/flag"
 	"k8s.io/component-base/cli/globalflag"
@@ -21,6 +19,7 @@ import (
 	"github.com/jetstack/kube-oidc-proxy/pkg/probe"
 	"github.com/jetstack/kube-oidc-proxy/pkg/proxy"
 	"github.com/jetstack/kube-oidc-proxy/pkg/proxy/tokenreview"
+	"github.com/jetstack/kube-oidc-proxy/pkg/util"
 	"github.com/jetstack/kube-oidc-proxy/pkg/version"
 )
 
@@ -47,8 +46,6 @@ func NewRunCommand(stopCh <-chan struct{}) *cobra.Command {
 
 	clientConfigOptions := options.NewClientFlags()
 
-	healthCheck := probe.New(strconv.Itoa(readinessProbePort))
-
 	// proxy command
 	cmd := &cobra.Command{
 		Use:  "kube-oidc-proxy",
@@ -69,14 +66,14 @@ func NewRunCommand(stopCh <-chan struct{}) *cobra.Command {
 			}
 
 			var restConfig *rest.Config
-
 			if clientConfigOptions.ClientFlagsChanged(cmd) {
 				// one or more client flags have been set to use client flag built
 				// config
 				restConfig, err = clientConfigOptions.ToRESTConfig()
 				if err != nil {
 					return err
 				}
+
 			} else {
 				// no client flags have been set so default to in-cluster config
 				restConfig, err = rest.InClusterConfig()
@@ -85,7 +82,7 @@ func NewRunCommand(stopCh <-chan struct{}) *cobra.Command {
 				}
 			}
 
-			// Init token reviewer if enabled
+			// Initialise token reviewer if enabled
 			var tokenReviewer *tokenreview.TokenReview
 			if kopOptions.TokenPassthrough.Enabled {
 				tokenReviewer, err = tokenreview.New(restConfig, kopOptions.TokenPassthrough.Audiences)
@@ -94,7 +91,7 @@ func NewRunCommand(stopCh <-chan struct{}) *cobra.Command {
 				}
 			}
 
-			// oidc auther from config
+			// Initialise Secure Serving Config
 			secureServingInfo := new(server.SecureServingInfo)
 			if err := ssoptionsWithLB.ApplyTo(&secureServingInfo, nil); err != nil {
 				return err
@@ -105,10 +102,26 @@ func NewRunCommand(stopCh <-chan struct{}) *cobra.Command {
 				DisableImpersonation: kopOptions.DisableImpersonation,
 			}
 
-			p := proxy.New(restConfig, oidcOptions,
-				tokenReviewer, secureServingInfo, healthCheck, proxyOptions)
+			// Initialise proxy with OIDC token authenticator
+			p, err := proxy.New(restConfig, oidcOptions,
+				tokenReviewer, secureServingInfo, proxyOptions)
+			if err != nil {
+				return err
+			}
+
+			// Create a fake JWT to set up readiness probe
+			fakeJWT, err := util.FakeJWT(oidcOptions.IssuerURL, oidcOptions.APIAudiences)
+			if err != nil {
+				return err
+			}
+
+			// Start readiness probe
+			if err := probe.Run(strconv.Itoa(readinessProbePort),
+				fakeJWT, p.OIDCAuthenticator()); err != nil {
+				return err
+			}
 
-			// run proxy
+			// Run proxy
 			waitCh, err := p.Run(stopCh)
 			if err != nil {
 				return err

pkg/probe/probe.go

@@ -2,26 +2,37 @@
 package probe
 
 import (
-	"errors"
+	"fmt"
 	"net"
 	"net/http"
-	"sync"
+	"strings"
 	"time"
 
-	"k8s.io/klog"
-
 	"github.com/heptiolabs/healthcheck"
+	"k8s.io/apiserver/pkg/authentication/request/bearertoken"
+	"k8s.io/klog"
 )
 
 type HealthCheck struct {
 	handler healthcheck.Handler
-	mu      sync.Mutex
-	ready   bool
+
+	oidcAuther *bearertoken.Authenticator
+	req        *http.Request
+
+	ready bool
 }
 
-func New(port string) *HealthCheck {
+func Run(port, fakeJWT string, oidcAuther *bearertoken.Authenticator) error {
+	req, err := http.NewRequest("GET", "http://fake", nil)
+	if err != nil {
+		return fmt.Errorf("failed to build fake probe request: %s", err)
+	}
+	req.Header.Set("Authorization", fmt.Sprintf("Bearer %s", fakeJWT))
+
 	h := &HealthCheck{
-		handler: healthcheck.NewHandler(),
+		handler:    healthcheck.NewHandler(),
+		oidcAuther: oidcAuther,
+		req:        req,
 	}
 
 	h.handler.AddReadinessCheck("secure serving", h.Check)
@@ -36,30 +47,25 @@ func New(port string) *HealthCheck {
 		}
 	}()
 
-	return h
+	return nil
 }
 
 func (h *HealthCheck) Check() error {
-	h.mu.Lock()
-	defer h.mu.Unlock()
-
-	if !h.ready {
-		return errors.New("not ready")
+	if h.ready {
+		return nil
 	}
 
-	return nil
-}
-
-func (h *HealthCheck) SetReady() {
-	h.mu.Lock()
-	defer h.mu.Unlock()
+	_, _, err := h.oidcAuther.AuthenticateRequest(h.req)
+	if err != nil && strings.HasSuffix(err.Error(), "authenticator not initialized") {
+		err = fmt.Errorf("OIDC provider not yet initialized: %s", err)
+		klog.V(4).Infof(err.Error())
+		return err
+	}
 
 	h.ready = true
-}
 
-func (h *HealthCheck) SetNotReady() {
-	h.mu.Lock()
-	defer h.mu.Unlock()
+	klog.Info("OIDC provider initialized, proxy ready")
+	klog.V(4).Infof("OIDC provider initialized, readiness check returned error: %s", err)
 
-	h.ready = false
+	return nil
 }

pkg/proxy/proxy.go

@@ -10,8 +10,6 @@ import (
 	"strings"
 	"time"
 
-	"gopkg.in/square/go-jose.v2"
-	"gopkg.in/square/go-jose.v2/jwt"
 	utilnet "k8s.io/apimachinery/pkg/util/net"
 	"k8s.io/apiserver/pkg/authentication/request/bearertoken"
 	authuser "k8s.io/apiserver/pkg/authentication/user"
@@ -22,7 +20,6 @@ import (
 	"k8s.io/klog"
 
 	"github.com/jetstack/kube-oidc-proxy/cmd/app/options"
-	"github.com/jetstack/kube-oidc-proxy/pkg/probe"
 	"github.com/jetstack/kube-oidc-proxy/pkg/proxy/tokenreview"
 )
 
@@ -44,49 +41,49 @@ type Options struct {
 
 type Proxy struct {
 	oidcAuther        *bearertoken.Authenticator
-	oidcOptions       *options.OIDCAuthenticationOptions
 	tokenReviewer     *tokenreview.TokenReview
 	secureServingInfo *server.SecureServingInfo
 
 	restConfig            *rest.Config
 	clientTransport       http.RoundTripper
 	noAuthClientTransport http.RoundTripper
-	healthCheck           *probe.HealthCheck
 
 	options *Options
 }
 
 func New(restConfig *rest.Config, oidcOptions *options.OIDCAuthenticationOptions,
-	tokenReviewer *tokenreview.TokenReview, ssinfo *server.SecureServingInfo, healthCheck *probe.HealthCheck, options *Options) *Proxy {
+	tokenReviewer *tokenreview.TokenReview, ssinfo *server.SecureServingInfo,
+	options *Options) (*Proxy, error) {
+
+	// generate tokenAuther from oidc config
+	tokenAuther, err := oidc.New(oidc.Options{
+		APIAudiences:         oidcOptions.APIAudiences,
+		CAFile:               oidcOptions.CAFile,
+		ClientID:             oidcOptions.ClientID,
+		GroupsClaim:          oidcOptions.GroupsClaim,
+		GroupsPrefix:         oidcOptions.GroupsPrefix,
+		IssuerURL:            oidcOptions.IssuerURL,
+		RequiredClaims:       oidcOptions.RequiredClaims,
+		SupportedSigningAlgs: oidcOptions.SigningAlgs,
+		UsernameClaim:        oidcOptions.UsernameClaim,
+		UsernamePrefix:       oidcOptions.UsernamePrefix,
+	})
+	if err != nil {
+		return nil, err
+	}
+
+	oidcAuther := bearertoken.New(tokenAuther)
+
 	return &Proxy{
 		restConfig:        restConfig,
-		oidcOptions:       oidcOptions,
 		tokenReviewer:     tokenReviewer,
 		secureServingInfo: ssinfo,
-		healthCheck:       healthCheck,
 		options:           options,
-	}
+		oidcAuther:        oidcAuther,
+	}, nil
 }
 
 func (p *Proxy) Run(stopCh <-chan struct{}) (<-chan struct{}, error) {
-	// generate oidcAuther from oidc config
-	oidcAuther, err := oidc.New(oidc.Options{
-		APIAudiences:         p.oidcOptions.APIAudiences,
-		CAFile:               p.oidcOptions.CAFile,
-		ClientID:             p.oidcOptions.ClientID,
-		GroupsClaim:          p.oidcOptions.GroupsClaim,
-		GroupsPrefix:         p.oidcOptions.GroupsPrefix,
-		IssuerURL:            p.oidcOptions.IssuerURL,
-		RequiredClaims:       p.oidcOptions.RequiredClaims,
-		SupportedSigningAlgs: p.oidcOptions.SigningAlgs,
-		UsernameClaim:        p.oidcOptions.UsernameClaim,
-		UsernamePrefix:       p.oidcOptions.UsernamePrefix,
-	})
-	if err != nil {
-		return nil, err
-	}
-	p.oidcAuther = bearertoken.New(oidcAuther)
-
 	// standard round tripper for proxy to API Server
 	clientRT, err := p.roundTripperForRestConfig(p.restConfig)
 	if err != nil {
@@ -123,37 +120,6 @@ func (p *Proxy) Run(stopCh <-chan struct{}) (<-chan struct{}, error) {
 	proxyHandler.Transport = p
 	proxyHandler.ErrorHandler = p.Error
 
-	// probe for readiness
-	go func() {
-		ticker := time.NewTicker(500 * time.Millisecond)
-		for {
-			<-ticker.C
-			fr, err := http.NewRequest("GET", "http://fake", nil)
-			if err != nil {
-				klog.Infof("error during readiness check: %v", err)
-				continue
-			}
-			jwt, err := p.fakeJWT()
-			if err != nil {
-				klog.Infof("error during readiness check: %v", err)
-				continue
-			}
-			fr.Header.Set("Authorization", fmt.Sprintf("Bearer %s", jwt))
-
-			_, _, err = p.oidcAuther.AuthenticateRequest(fr)
-			if strings.HasSuffix(err.Error(), "authenticator not initialized") {
-				klog.V(4).Infof("OIDC provider not yet initialized")
-				continue
-			}
-
-			p.healthCheck.SetReady()
-			klog.Info("OIDC provider initialized, proxy ready")
-			klog.V(4).Infof("OIDC provider initialized, readiness check returned error: %+v", err)
-			return
-		}
-
-	}()
-
 	waitCh, err := p.serve(proxyHandler, stopCh)
 	if err != nil {
 		return nil, err
@@ -242,24 +208,6 @@ func (p *Proxy) RoundTrip(req *http.Request) (*http.Response, error) {
 	return rt.RoundTrip(reqCpy)
 }
 
-// fakeJWT generates a JWT that passes the first offline validity checks. It is
-// used to test if the OIDC provider is initialised
-func (p *Proxy) fakeJWT() (string, error) {
-	key := []byte("secret")
-	sig, err := jose.NewSigner(jose.SigningKey{Algorithm: jose.HS256, Key: key}, (&jose.SignerOptions{}).WithType("JWT"))
-	if err != nil {
-		return "", err
-	}
-
-	cl := jwt.Claims{
-		Subject:   "readiness",
-		Issuer:    p.oidcOptions.IssuerURL,
-		NotBefore: jwt.NewNumericDate(time.Date(2016, 1, 1, 0, 0, 0, 0, time.UTC)),
-		Audience:  jwt.Audience(p.oidcOptions.APIAudiences),
-	}
-	return jwt.Signed(sig).Claims(cl).CompactSerialize()
-}
-
 func (p *Proxy) tokenReview(req *http.Request) (*http.Response, error) {
 	klog.V(4).Infof("attempting to validate a token in request using TokenReview endpoint(%s)",
 		req.RemoteAddr)
@@ -356,3 +304,8 @@ func (p *Proxy) roundTripperForRestConfig(config *rest.Config) (http.RoundTrippe
 
 	return clientRT, nil
 }
+
+// Return the proxy OIDC token authenticator
+func (p *Proxy) OIDCAuthenticator() *bearertoken.Authenticator {
+	return p.oidcAuther
+}

pkg/util/token.go

@@ -4,6 +4,10 @@ package util
 import (
 	"net/http"
 	"strings"
+	"time"
+
+	"gopkg.in/square/go-jose.v2"
+	"gopkg.in/square/go-jose.v2/jwt"
 )
 
 // Return just the token from the header of the request, without 'bearer'.
@@ -31,3 +35,26 @@ func ParseTokenFromRequest(req *http.Request) (string, bool) {
 
 	return token, true
 }
+
+// fakeJWT generates a valid JWT using the passed input parameters which is
+// signed by a generated key. This is useful for checking the status of a
+// signer.
+func FakeJWT(issuerURL string, apiAudiences []string) (string, error) {
+	key := []byte("secret")
+
+	sig, err := jose.NewSigner(
+		jose.SigningKey{Algorithm: jose.HS256, Key: key},
+		(&jose.SignerOptions{}).WithType("JWT"))
+	if err != nil {
+		return "", err
+	}
+
+	cl := jwt.Claims{
+		Subject:   "fake",
+		Issuer:    issuerURL,
+		NotBefore: jwt.NewNumericDate(time.Date(2016, 1, 1, 0, 0, 0, 0, time.UTC)),
+		Audience:  jwt.Audience(apiAudiences),
+	}
+
+	return jwt.Signed(sig).Claims(cl).CompactSerialize()
+}