add sampling rate

Author: meobilivang

main.go

@@ -84,6 +84,7 @@ var (
 	leaderElectionRenewDeadline time.Duration
 	leaderElectionRetryPeriod   time.Duration
 	enableTracing               bool
+	samplingRate                float64
 )
 
 // Add RBAC for the authorized diagnostics endpoint.
@@ -165,7 +166,7 @@ func main() {
 	ctx := ctrl.SetupSignalHandler()
 
 	if enableTracing {
-		if err := ot.RegisterTracing(ctx, setupLog); err != nil {
+		if err := ot.RegisterTracing(ctx, samplingRate, setupLog); err != nil {
 			setupLog.Error(err, "unable to set up tracing")
 			os.Exit(1)
 		}
@@ -389,6 +390,12 @@ func initFlags(fs *pflag.FlagSet) {
 		"Enable collecting and sending traces to opentelemetry-collector service",
 	)
 
+	fs.Float64Var(&samplingRate,
+		"trace-sampling-rate",
+		0.6,
+		"The fraction of all the traces will be sample",
+	)
+
 	flags.AddManagerOptions(fs, &managerOptions)
 
 	feature.MutableGates.AddFlag(fs)

pkg/otel/tracing.go

@@ -20,9 +20,9 @@ import (
 	semconv "go.opentelemetry.io/otel/semconv/v1.4.0"
 )
 
-func RegisterTracing(ctx context.Context, log logr.Logger) error {
+func RegisterTracing(ctx context.Context, samplingRate float64, log logr.Logger) error {
 
-	tracerProvider, err := SetUpTracing(ctx)
+	tracerProvider, err := SetUpTracing(ctx, samplingRate)
 	if err != nil {
 		return err
 	}
@@ -64,7 +64,7 @@ func newExporter(ctx context.Context) (*otlptrace.Exporter, error) {
 	return traceExporter, nil
 }
 
-func SetUpTracing(ctx context.Context) (*trace.TracerProvider, error) {
+func SetUpTracing(ctx context.Context, samplingRate float64) (*trace.TracerProvider, error) {
 
 	traceExporter, err := newExporter(ctx)
 
@@ -89,9 +89,8 @@ func SetUpTracing(ctx context.Context) (*trace.TracerProvider, error) {
 	traceProvider := trace.NewTracerProvider(
 		trace.WithBatcher(traceExporter),
 		trace.WithResource(resource),
-		// TODO: dynamic sampling rate?
-		// sampling rate based on parent span = 60%
-		trace.WithSampler(trace.ParentBased(trace.TraceIDRatioBased(0.6))),
+		// 0 < samplingRate <= 1 (< 0 -> be treated as 0; >= 1 -> always sample)
+		trace.WithSampler(trace.ParentBased(trace.TraceIDRatioBased(samplingRate))),
 	)
 
 	otel.SetTracerProvider(traceProvider)