[FSSDK-9705] feat: add OpenTelemetry tracing (#400)

* add a basic tracing pipeline

* add tracing config

* update tracing config

* use context

* update handler with middleware

* support both http & grpc protocal

* refactor config

* add unit test

* update unit test

* refactor code

* make tracing disabled by default

* add config doc

* fix typo

* collect more attributes

* add common http attributes

* collect http status code attribute

* collect sdk key attribute

* support distributed tracing

* add unit test for trace id generator

* update unit test

* fix typo

* fix typo in config file

* update config.yaml

* make trace_id configurable for distributed tracing

Author: pulak-opti

cmd/optimizely/main.go

@@ -18,6 +18,8 @@ package main
 import (
 	"bytes"
 	"context"
+	"errors"
+	"fmt"
 	"os"
 	"os/signal"
 	"runtime"
@@ -32,6 +34,7 @@ import (
 
 	"github.com/optimizely/agent/config"
 	"github.com/optimizely/agent/pkg/metrics"
+	"github.com/optimizely/agent/pkg/middleware"
 	"github.com/optimizely/agent/pkg/optimizely"
 	"github.com/optimizely/agent/pkg/routers"
 	"github.com/optimizely/agent/pkg/server"
@@ -44,6 +47,14 @@ import (
 	// Initiate the loading of the odpCache plugins
 	_ "github.com/optimizely/agent/plugins/odpcache/all"
 	"github.com/optimizely/go-sdk/pkg/logging"
+	"go.opentelemetry.io/otel"
+	"go.opentelemetry.io/otel/exporters/otlp/otlptrace"
+	"go.opentelemetry.io/otel/exporters/otlp/otlptrace/otlptracegrpc"
+	"go.opentelemetry.io/otel/exporters/otlp/otlptrace/otlptracehttp"
+	"go.opentelemetry.io/otel/exporters/stdout/stdouttrace"
+	"go.opentelemetry.io/otel/sdk/resource"
+	sdktrace "go.opentelemetry.io/otel/sdk/trace"
+	semconv "go.opentelemetry.io/otel/semconv/v1.4.0"
 )
 
 // Version holds the admin version
@@ -119,6 +130,97 @@ func initLogging(conf config.LogConfig) {
 	}
 }
 
+func getStdOutTraceProvider(conf config.OTELTracingConfig) (*sdktrace.TracerProvider, error) {
+	f, err := os.Create(conf.Services.StdOut.Filename)
+	if err != nil {
+		return nil, fmt.Errorf("failed to create the trace file, error: %s", err.Error())
+	}
+
+	exp, err := stdouttrace.New(
+		stdouttrace.WithPrettyPrint(),
+		stdouttrace.WithWriter(f),
+	)
+	if err != nil {
+		return nil, fmt.Errorf("failed to create the collector exporter, error: %s", err.Error())
+	}
+
+	res, err := resource.New(
+		context.Background(),
+		resource.WithAttributes(
+			semconv.ServiceNameKey.String(conf.ServiceName),
+			semconv.DeploymentEnvironmentKey.String(conf.Env),
+		),
+	)
+	if err != nil {
+		return nil, fmt.Errorf("failed to create the otel resource, error: %s", err.Error())
+	}
+
+	return sdktrace.NewTracerProvider(
+		sdktrace.WithBatcher(exp),
+		sdktrace.WithResource(res),
+		sdktrace.WithIDGenerator(middleware.NewTraceIDGenerator(conf.TraceIDHeaderKey)),
+	), nil
+}
+
+func getOTELTraceClient(conf config.OTELTracingConfig) (otlptrace.Client, error) {
+	switch conf.Services.Remote.Protocol {
+	case config.TracingRemoteProtocolHTTP:
+		return otlptracehttp.NewClient(
+			otlptracehttp.WithInsecure(),
+			otlptracehttp.WithEndpoint(conf.Services.Remote.Endpoint),
+		), nil
+	case config.TracingRemoteProtocolGRPC:
+		return otlptracegrpc.NewClient(
+			otlptracegrpc.WithInsecure(),
+			otlptracegrpc.WithEndpoint(conf.Services.Remote.Endpoint),
+		), nil
+	default:
+		return nil, errors.New("unknown remote tracing protocal")
+	}
+}
+
+func getRemoteTraceProvider(conf config.OTELTracingConfig) (*sdktrace.TracerProvider, error) {
+	res, err := resource.New(
+		context.Background(),
+		resource.WithAttributes(
+			semconv.ServiceNameKey.String(conf.ServiceName),
+			semconv.DeploymentEnvironmentKey.String(conf.Env),
+		),
+	)
+	if err != nil {
+		return nil, fmt.Errorf("failed to create the otel resource, error: %s", err.Error())
+	}
+
+	traceClient, err := getOTELTraceClient(conf)
+	if err != nil {
+		return nil, fmt.Errorf("failed to create the remote trace client, error: %s", err.Error())
+	}
+
+	traceExporter, err := otlptrace.New(context.Background(), traceClient)
+	if err != nil {
+		return nil, fmt.Errorf("failed to create the remote trace exporter, error: %s", err.Error())
+	}
+
+	bsp := sdktrace.NewBatchSpanProcessor(traceExporter)
+	return sdktrace.NewTracerProvider(
+		sdktrace.WithSampler(sdktrace.ParentBased(sdktrace.TraceIDRatioBased(conf.Services.Remote.SampleRate))),
+		sdktrace.WithResource(res),
+		sdktrace.WithSpanProcessor(bsp),
+		sdktrace.WithIDGenerator(middleware.NewTraceIDGenerator(conf.TraceIDHeaderKey)),
+	), nil
+}
+
+func initTracing(conf config.OTELTracingConfig) (*sdktrace.TracerProvider, error) {
+	switch conf.Default {
+	case config.TracingServiceTypeRemote:
+		return getRemoteTraceProvider(conf)
+	case config.TracingServiceTypeStdOut:
+		return getStdOutTraceProvider(conf)
+	default:
+		return nil, errors.New("unknown tracing service type")
+	}
+}
+
 func setRuntimeEnvironment(conf config.RuntimeConfig) {
 	if conf.BlockProfileRate != 0 {
 		log.Warn().Msgf("Setting non-zero blockProfileRate is NOT recommended for production")
@@ -140,6 +242,22 @@ func main() {
 	conf := loadConfig(v)
 	initLogging(conf.Log)
 
+	if conf.Tracing.Enabled {
+		tp, err := initTracing(conf.Tracing.OpenTelemetry)
+		if err != nil {
+			log.Panic().Err(err).Msg("Unable to initialize tracing")
+		}
+		defer func() {
+			if err := tp.Shutdown(context.Background()); err != nil {
+				log.Error().Err(err).Msg("Failed to shutdown tracing")
+			}
+		}()
+		otel.SetTracerProvider(tp)
+		log.Info().Msg(fmt.Sprintf("Tracing enabled with service %q", conf.Tracing.OpenTelemetry.Default))
+	} else {
+		log.Info().Msg("Tracing disabled")
+	}
+
 	conf.LogConfigWarnings()
 
 	setRuntimeEnvironment(conf.Runtime)
@@ -164,7 +282,7 @@ func main() {
 		cancel()
 	}()
 
-	apiRouter := routers.NewDefaultAPIRouter(optlyCache, conf.API, agentMetricsRegistry)
+	apiRouter := routers.NewDefaultAPIRouter(optlyCache, *conf, agentMetricsRegistry)
 	adminRouter := routers.NewAdminRouter(*conf)
 
 	log.Info().Str("version", conf.Version).Msg("Starting services.")

cmd/optimizely/main_test.go

@@ -418,3 +418,92 @@ func TestLoggingWithIncludeSdkKey(t *testing.T) {
 	})
 	assert.False(t, optimizely.ShouldIncludeSDKKey)
 }
+
+func Test_initTracing(t *testing.T) {
+	type args struct {
+		conf config.OTELTracingConfig
+	}
+	tests := []struct {
+		name    string
+		args    args
+		wantErr bool
+	}{
+		{
+			name: "should return error when exporter type is not supported",
+			args: args{
+				conf: config.OTELTracingConfig{
+					Default: "unsupported",
+				},
+			},
+			wantErr: true,
+		},
+		{
+			name: "should return no error stdout tracing exporter",
+			args: args{
+				conf: config.OTELTracingConfig{
+					Default: "stdout",
+					Services: config.TracingServiceConfig{
+						StdOut: config.TracingStdOutConfig{
+							Filename: "trace.out",
+						},
+					},
+				},
+			},
+			wantErr: false,
+		},
+		{
+			name: "should return no error for remote tracing exporter with http protocal",
+			args: args{
+				conf: config.OTELTracingConfig{
+					Default: "remote",
+					Services: config.TracingServiceConfig{
+						Remote: config.TracingRemoteConfig{
+							Endpoint: "localhost:1234",
+							Protocol: "http",
+						},
+					},
+				},
+			},
+			wantErr: false,
+		},
+		{
+			name: "should return no error for remote tracing exporter with grpc protocal",
+			args: args{
+				conf: config.OTELTracingConfig{
+					Default: "remote",
+					Services: config.TracingServiceConfig{
+						Remote: config.TracingRemoteConfig{
+							Endpoint: "localhost:1234",
+							Protocol: "grpc",
+						},
+					},
+				},
+			},
+			wantErr: false,
+		},
+		{
+			name: "should return no error for remote tracing exporter with invalid protocal",
+			args: args{
+				conf: config.OTELTracingConfig{
+					Default: "remote",
+					Services: config.TracingServiceConfig{
+						Remote: config.TracingRemoteConfig{
+							Endpoint: "localhost:1234",
+							Protocol: "udp/invalid",
+						},
+					},
+				},
+			},
+			wantErr: true,
+		},
+	}
+	for _, tt := range tests {
+		t.Run(tt.name, func(t *testing.T) {
+			_, err := initTracing(tt.args.conf)
+			if (err != nil) != tt.wantErr {
+				t.Errorf("initTracing() error = %v, wantErr %v", err, tt.wantErr)
+				return
+			}
+		})
+	}
+}

config.yaml

@@ -26,6 +26,45 @@ log:
     ## to set whether or not the SDK key is included in the logging output.
     includeSdkKey: true
 
+##
+## tracing: tracing configuration
+##
+tracing:
+    ## bydefault tracing is disabled
+    ## to enable tracing set enabled to true
+    enabled: false
+    # opentelemetry tracing configuration
+    opentelemetry:
+        ## bydefault stdout exporter is enabled
+        ## to enable remote exporter set default as "remote"
+        default: "stdout"
+        ## tracing service name
+        serviceName: "optimizely-agent"
+        ## tracing environment name
+        ## example: for production environment env can be set as "prod"
+        env: "dev"
+        ## HTTP Header Key for TraceID in Distributed Tracing
+        ## The value set in HTTP Header must be a hex compliant with the W3C trace-context specification.
+        ## See more at https://www.w3.org/TR/trace-context/#trace-id
+        traceIDHeaderKey: "X-Optimizely-Trace-ID"
+        ## tracing service configuration
+        services:
+            ## stdout exporter configuration
+            stdout:
+                ## for stdout tracing data is saved in the specified file
+                filename: "trace.out"
+            ## remote exporter configuration
+            remote:
+                ## remote collector endpoint
+                endpoint: "localhost:4317"
+                ## supported protocols are "http" and "grpc"
+                protocol: "grpc"
+                ## "sampleRate" refers to the rate at which traces are collected and recorded.
+                ## sampleRate >= 1 will always sample.
+                ## sampleRate < 0 are treated as zero i.e. never sample.
+                sampleRate: 1.0
+              
+
 ##
 ## http server configuration
 ##

config/config.go

@@ -71,6 +71,9 @@ func NewDefaultConfig() *AgentConfig {
 			IncludeSDKKey: true,
 			Level:         "info",
 		},
+		Tracing: TracingConfig{
+			Enabled: false,
+		},
 		Client: ClientConfig{
 			PollingInterval:     1 * time.Minute,
 			BatchSize:           10,
@@ -139,6 +142,7 @@ type AgentConfig struct {
 	Admin   AdminConfig   `json:"admin"`
 	API     APIConfig     `json:"api"`
 	Log     LogConfig     `json:"log"`
+	Tracing TracingConfig `json:"tracing"`
 	Client  ClientConfig  `json:"client"`
 	Runtime RuntimeConfig `json:"runtime"`
 	Server  ServerConfig  `json:"server"`
@@ -201,6 +205,48 @@ type LogConfig struct {
 	Level         string `json:"level"`
 }
 
+type TracingConfig struct {
+	Enabled       bool              `json:"enabled"`
+	OpenTelemetry OTELTracingConfig `json:"opentelemetry"`
+}
+
+type TracingServiceType string
+
+const (
+	TracingServiceTypeStdOut TracingServiceType = "stdout"
+	TracingServiceTypeRemote TracingServiceType = "remote"
+)
+
+type TracingRemoteProtocol string
+
+const (
+	TracingRemoteProtocolGRPC TracingRemoteProtocol = "grpc"
+	TracingRemoteProtocolHTTP TracingRemoteProtocol = "http"
+)
+
+type OTELTracingConfig struct {
+	Default          TracingServiceType   `json:"default"`
+	ServiceName      string               `json:"serviceName"`
+	Env              string               `json:"env"`
+	TraceIDHeaderKey string               `json:"traceIDHeaderKey"`
+	Services         TracingServiceConfig `json:"services"`
+}
+
+type TracingServiceConfig struct {
+	StdOut TracingStdOutConfig `json:"stdout"`
+	Remote TracingRemoteConfig `json:"remote"`
+}
+
+type TracingStdOutConfig struct {
+	Filename string `json:"filename"`
+}
+
+type TracingRemoteConfig struct {
+	Endpoint   string                `json:"endpoint"`
+	Protocol   TracingRemoteProtocol `json:"protocol"`
+	SampleRate float64               `json:"sampleRate"`
+}
+
 // PluginConfigs defines the generic mapping of middleware plugins
 type PluginConfigs map[string]interface{}