update tracing stuff (#1472)

* jaeger deprecated config stuff
* add call_id trace tag, proper app and fn trace span attributes

turns out the tag keys don't get added to the trace span attributes, have to
annotate them differently, kinda sucks but whatever... these are useful for
debugging (particularly call id for me right now, anyway)

Author: rdallman

api/agent/agent.go

@@ -25,6 +25,7 @@ import (
 	"github.com/sirupsen/logrus"
 	"go.opencensus.io/plugin/ochttp"
 	"go.opencensus.io/stats"
+	"go.opencensus.io/tag"
 	"go.opencensus.io/trace"
 	"go.opencensus.io/trace/propagation"
 )
@@ -232,19 +233,24 @@ func (a *agent) Close() error {
 
 func (a *agent) Submit(callI Call) error {
 	call := callI.(*call)
-	ctx, span := trace.StartSpan(call.req.Context(), "agent_submit")
-	defer span.End()
 
-	statsCalls(ctx)
+	ctx := call.req.Context()
 
-	if !a.shutWg.AddSession(1) {
-		statsTooBusy(ctx)
-		return models.ErrCallTimeoutServerBusy
+	callIDKey, err := tag.NewKey("agent.call_id")
+	if err != nil {
+		return err
+	}
+	ctx, err = tag.New(ctx, tag.Insert(callIDKey, call.ID))
+	if err != nil {
+		return err
 	}
-	defer a.shutWg.DoneSession()
 
-	err := a.submit(ctx, call)
-	return err
+	ctx, span := trace.StartSpan(ctx, "agent_submit")
+	defer span.End()
+
+	span.AddAttributes(trace.StringAttribute("agent.call_id", call.ID))
+
+	return a.submit(ctx, call)
 }
 
 func (a *agent) startStateTrackers(ctx context.Context, call *call) {
@@ -256,6 +262,14 @@ func (a *agent) endStateTrackers(ctx context.Context, call *call) {
 }
 
 func (a *agent) submit(ctx context.Context, call *call) error {
+	statsCalls(ctx)
+
+	if !a.shutWg.AddSession(1) {
+		statsTooBusy(ctx)
+		return models.ErrCallTimeoutServerBusy
+	}
+	defer a.shutWg.DoneSession()
+
 	statsEnqueue(ctx)
 
 	a.startStateTrackers(ctx, call)

api/server/gin_middlewares.go

@@ -66,20 +66,15 @@ func optionalCorsWrap(r *gin.Engine) {
 
 // we should use http grr
 func traceWrap(c *gin.Context) {
-	appKey, err := tag.NewKey("fn_appname")
+	appIDKey, err := tag.NewKey("fn.app_id")
 	if err != nil {
 		logrus.Fatal(err)
 	}
-	appIDKey, err := tag.NewKey("fn_app_id")
-	if err != nil {
-		logrus.Fatal(err)
-	}
-	fnKey, err := tag.NewKey("fn_fn_id")
+	fnKey, err := tag.NewKey("fn.fn_id")
 	if err != nil {
 		logrus.Fatal(err)
 	}
 	ctx, err := tag.New(c.Request.Context(),
-		tag.Insert(appKey, c.Param(api.AppName)),
 		tag.Insert(appIDKey, c.Param(api.AppID)),
 		tag.Insert(fnKey, c.Param(api.FnID)),
 	)
@@ -90,8 +85,14 @@ func traceWrap(c *gin.Context) {
 	// TODO inspect opencensus more and see if we need to define a header ourselves
 	// to trigger per-request spans (we will want this), we can set sampler here per request.
 
-	ctx, serverSpan := trace.StartSpan(ctx, "serve_http")
-	defer serverSpan.End()
+	ctx, span := trace.StartSpan(ctx, "serve_http")
+	defer span.End()
+
+	// spans like these, not tags
+	span.AddAttributes(
+		trace.StringAttribute("fn.app_id", c.Param(api.AppID)),
+		trace.StringAttribute("fn.fn_id", c.Param(api.FnID)),
+	)
 
 	c.Request = c.Request.WithContext(ctx)
 	c.Next()

api/server/server.go

@@ -664,14 +664,15 @@ func WithoutFnInvokeEndpoints() Option {
 // WithJaeger maps EnvJaegerURL
 func WithJaeger(jaegerURL string) Option {
 	return func(ctx context.Context, s *Server) error {
-		// ex: "http://localhost:14268"
+		// ex: "http://localhost:14268/api/traces?format=jaeger.thrift"
 		if jaegerURL == "" {
 			return nil
 		}
 
 		exporter, err := jaeger.NewExporter(jaeger.Options{
-			Endpoint:    jaegerURL,
-			ServiceName: "fn",
+			CollectorEndpoint: jaegerURL,
+			Process:           jaeger.Process{ServiceName: "fnserver"},
+			OnError:           func(err error) { logrus.WithError(err).Error("Error when uploading spans to Jaeger") },
 		})
 		if err != nil {
 			return fmt.Errorf("error connecting to jaeger: %v", err)