Enriched Span Attributes on upstream auth (#56)

* Update authenticator.go

* create a new reason for when upstream auth header is empty

* Added Http Headers to Span and enriched spans for upstream auth

* added "upstream_auth_empty_tokens_total" metric

Author: SamMHD

pkg/auth/authenticator.go

@@ -2,6 +2,7 @@ package auth
 
 import (
 	"context"
+	"errors"
 	"net/http"
 	"net/url"
 	"os"
@@ -15,6 +16,7 @@ import (
 	"github.com/snapp-incubator/Cerberus/internal/tracing"
 	"go.opentelemetry.io/otel/attribute"
 	otelcodes "go.opentelemetry.io/otel/codes"
+	"go.opentelemetry.io/otel/trace"
 	"google.golang.org/grpc/codes"
 	"google.golang.org/grpc/status"
 )
@@ -159,27 +161,8 @@ func (a *Authenticator) Check(ctx context.Context, request *Request) (finalRespo
 	wsvc, ns, reason := readRequestContext(request)
 
 	// generate opentelemetry span with given parameters
-	parentCtx := tracing.ReadParentSpanFromRequest(ctx, request.Request)
-	ctx, span := tracing.StartSpan(parentCtx, "CheckFunction",
-		attribute.String("webservice", wsvc),
-		attribute.String("namespace", ns),
-	)
-	defer func() {
-		extraAttrs := []attribute.KeyValue{
-			attribute.String("cerberus-reason", string(reason)),
-		}
-		if finalResponse != nil {
-			extraAttrs = append(extraAttrs,
-				attribute.Bool("final-response-ok", finalResponse.Allow),
-			)
-			for k, v := range finalResponse.Response.Header {
-				extraAttrs = append(extraAttrs,
-					attribute.String("final-extra-headers-"+k, strings.Join(v, ",")),
-				)
-			}
-		}
-		tracing.EndSpan(span, start_time, extraAttrs...)
-	}()
+	ctx, span := startSpan(ctx, request.Request, wsvc, ns)
+	defer endSpan(span, start_time, finalResponse, reason)
 
 	if reason != "" {
 		return generateResponse(reason, nil), nil
@@ -193,6 +176,7 @@ func (a *Authenticator) Check(ctx context.Context, request *Request) (finalRespo
 	var extraHeaders ExtraHeaders
 
 	reason, wsvcCacheEntry := a.readService(wsvc)
+	addHeadersToSpan(request.Request, wsvcCacheEntry, span)
 	if reason == "" {
 		var cerberusExtraHeaders CerberusExtraHeaders
 
@@ -214,6 +198,53 @@ func (a *Authenticator) Check(ctx context.Context, request *Request) (finalRespo
 	return
 }
 
+// startSpan starts span for Check Function
+func startSpan(ctx context.Context, request http.Request, wsvc string, ns string) (context.Context, trace.Span) {
+	parentCtx := tracing.ReadParentSpanFromRequest(ctx, request)
+	return tracing.StartSpan(parentCtx, "CheckFunction",
+		attribute.String("webservice", wsvc),
+		attribute.String("namespace", ns),
+	)
+}
+
+// endSpan ends Check Function span and adds attributes.
+func endSpan(span trace.Span, start_time time.Time, finalResponse *Response, reason CerberusReason) {
+	extraAttrs := []attribute.KeyValue{
+		attribute.String("cerberus-reason", string(reason)),
+	}
+	if finalResponse != nil {
+		extraAttrs = append(extraAttrs,
+			attribute.Bool("final-response-ok", finalResponse.Allow),
+		)
+		for k, v := range finalResponse.Response.Header {
+			extraAttrs = append(extraAttrs,
+				attribute.String("final-extra-headers-"+k, strings.Join(v, ",")),
+			)
+		}
+	}
+	tracing.EndSpan(span, start_time, extraAttrs...)
+}
+
+// addHeadersToSpan adds request headers to Span
+func addHeadersToSpan(request http.Request, service WebservicesCacheEntry, span trace.Span) {
+	// Add request headers to span
+	for headerName, headerValues := range request.Header {
+		headerValue := strings.Join(headerValues, ",")
+
+		// For Authorization header, only include first 20 chars
+		if headerName == service.Spec.LookupHeader && len(headerValue) > 20 {
+			headerValue = headerValue[:20]
+		}
+		if hasUpstreamAuth(service) && headerName == service.Spec.UpstreamHttpAuth.ReadTokenFrom && len(headerValue) > 20 {
+			headerValue = headerValue[:20]
+		}
+
+		span.SetAttributes(
+			attribute.String("http.header."+headerName, headerValue),
+		)
+	}
+}
+
 func readRequestContext(request *Request) (wsvc string, ns string, reason CerberusReason) {
 	wsvc = request.Context["webservice"]
 	if wsvc == "" {
@@ -253,14 +284,23 @@ func NewAuthenticator(logger logr.Logger) *Authenticator {
 // validateUpstreamAuthRequest validates the service before calling the upstream.
 // when calling the upstream authentication, one of read or write tokens must be
 // empty and the upstream address must be a valid url.
-func validateUpstreamAuthRequest(service WebservicesCacheEntry) CerberusReason {
+func validateUpstreamAuthRequest(service WebservicesCacheEntry, request *Request) CerberusReason {
 	if service.Spec.UpstreamHttpAuth.ReadTokenFrom == "" ||
 		service.Spec.UpstreamHttpAuth.WriteTokenTo == "" {
 		return CerberusReasonTargetAuthTokenEmpty
 	}
 	if !govalidator.IsRequestURL(service.Spec.UpstreamHttpAuth.Address) {
 		return CerberusReasonInvalidUpstreamAddress
 	}
+	if request != nil {
+		token := request.Request.Header.Get(service.Spec.UpstreamHttpAuth.ReadTokenFrom)
+		if token == "" {
+			upstreamAuthEmptyTokens.Inc()
+
+			// uncomment if you want to stop upstream auth call when token is empty
+			// return CerberusReasonUpstreamAuthHeaderEmpty
+		}
+	}
 	return ""
 }
 
@@ -332,12 +372,16 @@ func (a *Authenticator) checkServiceUpstreamAuth(service WebservicesCacheEntry,
 		)
 	}()
 
-	if reason := validateUpstreamAuthRequest(service); reason != "" {
+	if reason := validateUpstreamAuthRequest(service, request); reason != "" {
+		span.RecordError(errors.New("upstream auth request validation:" + string(reason)))
+		span.SetStatus(otelcodes.Error, "upstream auth http request faild")
 		return reason
 	}
 	upstreamAuth := service.Spec.UpstreamHttpAuth
 	req, err := setupUpstreamAuthRequest(&upstreamAuth, request)
 	if err != nil {
+		span.RecordError(err)
+		span.SetStatus(otelcodes.Error, "failed to create upstream auth request")
 		return CerberusReasonUpstreamAuthNoReq
 	}
 	a.adjustTimeout(upstreamAuth.Timeout, downstreamDeadline, hasDownstreamDeadline)
@@ -368,6 +412,8 @@ func (a *Authenticator) checkServiceUpstreamAuth(service WebservicesCacheEntry,
 	}
 
 	if resp.StatusCode != http.StatusOK {
+		span.RecordError(err)
+		span.SetStatus(otelcodes.Error, "upstream auth non 200 status code")
 		return CerberusReasonUnauthorized
 	}
 	// add requested careHeaders to extraHeaders for response

pkg/auth/authenticator_test.go

@@ -768,37 +768,37 @@ func TestValidateUpstreamAuthRequest(t *testing.T) {
 	service := WebservicesCacheEntry{}
 	service.Spec.UpstreamHttpAuth.ReadTokenFrom = ""
 	service.Spec.UpstreamHttpAuth.WriteTokenTo = ""
-	reason := validateUpstreamAuthRequest(service)
+	reason := validateUpstreamAuthRequest(service, nil)
 	assert.Equal(t, CerberusReasonTargetAuthTokenEmpty, reason, "Expected target auth token empty")
 
 	// Test case 2: WriteTokenTo is empty
 	service = WebservicesCacheEntry{}
 	service.Spec.UpstreamHttpAuth.ReadTokenFrom = "token"
 	service.Spec.UpstreamHttpAuth.WriteTokenTo = ""
-	reason = validateUpstreamAuthRequest(service)
+	reason = validateUpstreamAuthRequest(service, nil)
 	assert.Equal(t, CerberusReasonTargetAuthTokenEmpty, reason, "Expected target auth token empty")
 
 	// Test case 3: ReadTokenFrom is empty
 	service = WebservicesCacheEntry{}
 	service.Spec.UpstreamHttpAuth.ReadTokenFrom = ""
 	service.Spec.UpstreamHttpAuth.WriteTokenTo = "token"
-	reason = validateUpstreamAuthRequest(service)
+	reason = validateUpstreamAuthRequest(service, nil)
 	assert.Equal(t, CerberusReasonTargetAuthTokenEmpty, reason, "Expected target auth token empty")
 
 	// Test case 4: Address is invalid
 	service = WebservicesCacheEntry{}
 	service.Spec.UpstreamHttpAuth.ReadTokenFrom = "token"
 	service.Spec.UpstreamHttpAuth.WriteTokenTo = "token"
 	service.Spec.UpstreamHttpAuth.Address = "not a valid URL"
-	reason = validateUpstreamAuthRequest(service)
+	reason = validateUpstreamAuthRequest(service, nil)
 	assert.Equal(t, CerberusReasonInvalidUpstreamAddress, reason, "Expected invalid upstream address")
 
 	// Test case 5: Everything is valid
 	service = WebservicesCacheEntry{}
 	service.Spec.UpstreamHttpAuth.ReadTokenFrom = "token"
 	service.Spec.UpstreamHttpAuth.WriteTokenTo = "token"
 	service.Spec.UpstreamHttpAuth.Address = "http://example.com"
-	reason = validateUpstreamAuthRequest(service)
+	reason = validateUpstreamAuthRequest(service, nil)
 	assert.Empty(t, reason, "Expected no reason")
 }
 

pkg/auth/cerberus_reasons.go

@@ -77,6 +77,10 @@ const (
 	// has an invalid upstream address in it's manifest
 	CerberusReasonInvalidUpstreamAddress CerberusReason = "invalid-auth-upstream"
 
+	// CerberusReasonUpstreamAuthHeaderEmpty means that request header that
+	// should be forwarded to upstream auth service is empty
+	CerberusReasonUpstreamAuthHeaderEmpty CerberusReason = "upstream-auth-header-empty"
+
 	// CerberusReasonSourceAuthTokenEmpty means that requested webservice
 	// does not contain source upstream auth lookup header in it's manifest
 	CerberusReasonSourceAuthTokenEmpty CerberusReason = "upstream-source-identifier-empty"

pkg/auth/metrics.go

@@ -13,7 +13,7 @@ const (
 	HasUpstreamAuth             = "upstream_auth_enabled"
 	ObjectKindLabel             = "kind"
 	WithDownstreamDeadlineLabel = "with_downstream_deadline"
-	WebserviceLabel 			= "webservice" 
+	WebserviceLabel             = "webservice"
 
 	MetricsKindSecret                  = "secret"
 	MetricsKindWebservice              = "webservice"
@@ -131,6 +131,13 @@ var (
 		},
 		[]string{WithDownstreamDeadlineLabel},
 	)
+
+	upstreamAuthEmptyTokens = prometheus.NewCounter(
+		prometheus.CounterOpts{
+			Name: "upstream_auth_empty_tokens_total",
+			Help: "Total number of UpstreamAuth requests that token were empty",
+		},
+	)
 )
 
 func init() {
@@ -196,9 +203,9 @@ func AddWithDownstreamDeadlineLabel(labels prometheus.Labels, hasDeadline bool)
 }
 
 func AddWebserviceLabel(labels prometheus.Labels, wsvc string) prometheus.Labels {
-    if labels == nil {
-        labels = prometheus.Labels{}
-    }
-    labels[WebserviceLabel] = wsvc
-    return labels
-}
+	if labels == nil {
+		labels = prometheus.Labels{}
+	}
+	labels[WebserviceLabel] = wsvc
+	return labels
+}