Use OTel library from tracing while keeping Jaeger exporter config (#11249)

* Migrate to OTel tracing

Signed-off-by: Oleg Zaytsev <[email protected]>

* Apply suggestions from code review

Co-authored-by: Taylor C <[email protected]>

* Update OTel case in pkg/util/spanlogger/spanlogger.go

Co-authored-by: Arve Knudsen <[email protected]>

* Update docs/sources/mimir/configure/configure-tracing.md

Co-authored-by: Arve Knudsen <[email protected]>

---------

Signed-off-by: Oleg Zaytsev <[email protected]>
Co-authored-by: Taylor C <[email protected]>
Co-authored-by: Arve Knudsen <[email protected]>

Author: 3 people

Makefile

@@ -456,6 +456,11 @@ lint: check-makefiles
 		"github.com/twmb/franz-go/pkg/kgo.{AllowAutoTopicCreation}" \
 		./pkg/... ./cmd/... ./tools/... ./integration/...
 
+	# We don't use opentracing anymore.
+	faillint -paths \
+		"github.com/opentracing/opentracing-go,github.com/opentracing/opentracing-go/log,github.com/uber/jaeger-client-go,github.com/opentracing-contrib/go-stdlib/nethttp" \
+		./pkg/... ./cmd/... ./tools/... ./integration/...
+
 	# Ensure lines are sorted after lint:sorted directives.
 	go run ./tools/lint-sorted/ \
 		-path ./pkg \

cmd/mimir-continuous-test/main.go

@@ -16,7 +16,6 @@ import (
 	"github.com/grafana/dskit/tracing"
 	"github.com/prometheus/client_golang/prometheus"
 	"github.com/prometheus/client_golang/prometheus/collectors"
-	jaegercfg "github.com/uber/jaeger-client-go/config"
 
 	"github.com/grafana/mimir/pkg/continuoustest"
 	"github.com/grafana/mimir/pkg/util/instrumentation"
@@ -44,7 +43,7 @@ func main() {
 	level.Warn(util_log.Logger).Log("msg", "The mimir-continuous-test binary you are using is deprecated. Please use the Mimir binary module `mimir -target=continuous-test`.")
 
 	// Setting the environment variable JAEGER_AGENT_HOST enables tracing.
-	if trace, err := tracing.NewFromEnv("mimir-continuous-test", jaegercfg.MaxTagValueLength(16e3)); err != nil {
+	if trace, err := tracing.NewOTelFromJaegerEnv("mimir-continuous-test"); err != nil {
 		level.Error(util_log.Logger).Log("msg", "Failed to setup tracing", "err", err.Error())
 	} else {
 		defer trace.Close()

cmd/mimir/main.go

@@ -22,7 +22,6 @@ import (
 	"github.com/grafana/dskit/tracing"
 	"github.com/pkg/errors"
 	"github.com/prometheus/client_golang/prometheus"
-	jaegercfg "github.com/uber/jaeger-client-go/config"
 	"gopkg.in/yaml.v3"
 
 	"github.com/grafana/mimir/pkg/mimir"
@@ -193,7 +192,7 @@ func main() {
 		}
 
 		// Setting the environment variable JAEGER_AGENT_HOST enables tracing.
-		if trace, err := tracing.NewFromEnv(name, jaegercfg.MaxTagValueLength(16e3)); err != nil {
+		if trace, err := tracing.NewOTelFromJaegerEnv(name); err != nil {
 			level.Error(util_log.Logger).Log("msg", "Failed to setup tracing", "err", err.Error())
 		} else {
 			defer trace.Close()

cmd/query-tee/main.go

@@ -19,7 +19,6 @@ import (
 	"github.com/prometheus/client_golang/prometheus"
 	"github.com/prometheus/client_golang/prometheus/collectors"
 	"github.com/prometheus/common/model"
-	jaegercfg "github.com/uber/jaeger-client-go/config"
 
 	"github.com/grafana/mimir/pkg/util/instrumentation"
 	util_log "github.com/grafana/mimir/pkg/util/log"
@@ -87,7 +86,7 @@ func initTracing() io.Closer {
 		name = "query-tee"
 	}
 
-	trace, err := tracing.NewFromEnv(name, jaegercfg.MaxTagValueLength(16e3))
+	trace, err := tracing.NewOTelFromJaegerEnv(name)
 	if err != nil {
 		level.Error(util_log.Logger).Log("msg", "Failed to setup tracing", "err", err.Error())
 		return nil

docs/sources/mimir/configure/configure-tracing.md

@@ -11,36 +11,29 @@ weight: 100
 
 # Configure Grafana Mimir tracing
 
-Grafana Mimir uses [Jaeger](https://www.jaegertracing.io/) to implement distributed
-tracing. Jaeger is a valuable tool for troubleshooting the behavior of
-Grafana Mimir in production.
+Distributed tracing is a valuable tool for troubleshooting the behavior of Grafana Mimir in production.
+
+Grafana Mimir is transitioning from [Jaeger](https://www.jaegertracing.io/) to [OpenTelemetry](https://opentelemetry.io/docs/languages/go/getting-started/) to implement distributed tracing.
+During this transition, Mimir uses OTel libraries, but performs the configuration using Jaeger environment variables.
 
 ## Dependencies
 
-Set up Jaeger deployment to collect and store traces from Grafana Mimir. A
-deployment includes either the Jaeger all-in-one binary, or a distributed
-system of agents, collectors, and queriers. If you run Grafana Mimir on Kubernetes, refer to [Jaeger
-Kubernetes](https://github.com/jaegertracing/jaeger-kubernetes).
+Set up a Jaeger deployment to collect and store traces from Grafana Mimir.
+A deployment includes either the Jaeger all-in-one binary or a distributed system of agents, collectors, and queriers.
+If you run Grafana Mimir on Kubernetes, refer to [Jaeger Kubernetes](https://github.com/jaegertracing/jaeger-kubernetes).
 
 ## Configuration
 
 To configure Grafana Mimir to send traces, perform the following steps:
 
-1. Set the `JAEGER_AGENT_HOST` environment variable in all components to point
-   to the Jaeger agent.
+1. Set the `JAEGER_AGENT_HOST` environment variable in all components to point to the Jaeger agent.
 1. Enable sampling in the appropriate components:
-   - The ingester and ruler self-initiate traces and should have sampling
-     explicitly enabled.
-   - Sampling for the distributor and query-frontend can be enabled in Grafana Mimir
-     or in an upstream service, like a proxy or gateway running in front of Grafana Mimir.
-
-To enable sampling in Grafana Mimir components you can specify either
-`JAEGER_SAMPLER_MANAGER_HOST_PORT` for remote sampling, or
-`JAEGER_SAMPLER_TYPE` and `JAEGER_SAMPLER_PARAM` to manually set sampling
-configuration. Refer to [Jaeger Client Go
-documentation](https://github.com/jaegertracing/jaeger-client-go#environment-variables)
-for the full list of environment variables you can configure.
-
-Note that you must specify one of `JAEGER_AGENT_HOST` or
-`JAEGER_SAMPLER_MANAGER_HOST_PORT` in each component for Jaeger to be enabled,
-even if you plan to use the default values.
+   - The ingester and ruler self-initiate traces. You should have sampling explicitly enabled.
+   - You can enable sampling for the distributor and query-frontend in Grafana Mimir or in an upstream service, like a proxy or gateway running in front of Grafana Mimir.
+
+To enable sampling in Grafana Mimir components, you can specify either `JAEGER_SAMPLER_MANAGER_HOST_PORT` for remote sampling, or `JAEGER_SAMPLER_TYPE` and `JAEGER_SAMPLER_PARAM` to manually set sampling configuration.
+Refer to [Jaeger Client Go documentation](https://github.com/jaegertracing/jaeger-client-go#environment-variables)for the full list of environment variables you can configure.
+
+{{< admonition type="note" >}}
+You must specify one of `JAEGER_AGENT_HOST` or `JAEGER_SAMPLER_MANAGER_HOST_PORT` in each component for Jaeger to be enabled, even if you plan to use the default values.
+{{< /admonition >}}

go.mod

@@ -26,9 +26,9 @@ require (
 	github.com/minio/minio-go/v7 v7.0.91
 	github.com/mitchellh/go-wordwrap v1.0.1
 	github.com/oklog/ulid v1.3.1 // indirect
-	github.com/opentracing-contrib/go-grpc v0.1.2
-	github.com/opentracing-contrib/go-stdlib v1.1.0
-	github.com/opentracing/opentracing-go v1.2.1-0.20220228012449-10b1cf09e00b
+	github.com/opentracing-contrib/go-grpc v0.1.2 // indirect
+	github.com/opentracing-contrib/go-stdlib v1.1.0 // indirect
+	github.com/opentracing/opentracing-go v1.2.1-0.20220228012449-10b1cf09e00b // indirect
 	github.com/pkg/errors v0.9.1
 	github.com/prometheus/alertmanager v0.28.1
 	github.com/prometheus/client_golang v1.22.0
@@ -39,7 +39,7 @@ require (
 	github.com/sirupsen/logrus v1.9.3
 	github.com/spf13/afero v1.11.0
 	github.com/stretchr/testify v1.10.0
-	github.com/uber/jaeger-client-go v2.30.0+incompatible
+	github.com/uber/jaeger-client-go v2.30.0+incompatible // indirect
 	go.uber.org/atomic v1.11.0
 	go.uber.org/goleak v1.3.0
 	golang.org/x/crypto v0.38.0
@@ -85,7 +85,10 @@ require (
 	github.com/twmb/franz-go/plugin/kprom v1.1.0
 	github.com/xlab/treeprint v1.2.0
 	go.opentelemetry.io/collector/pdata v1.31.0
+	go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc v0.60.0
+	go.opentelemetry.io/contrib/propagators/jaeger v1.35.0
 	go.opentelemetry.io/otel v1.35.0
+	go.opentelemetry.io/otel/sdk v1.35.0
 	go.opentelemetry.io/otel/trace v1.35.0
 	go.opentelemetry.io/proto/otlp v1.5.0
 	go.uber.org/multierr v1.11.0
@@ -177,13 +180,10 @@ require (
 	go.opentelemetry.io/collector/processor v1.31.0 // indirect
 	go.opentelemetry.io/contrib/bridges/otelzap v0.10.0 // indirect
 	go.opentelemetry.io/contrib/detectors/gcp v1.35.0 // indirect
-	go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc v0.60.0 // indirect
 	go.opentelemetry.io/contrib/instrumentation/net/http/httptrace/otelhttptrace v0.60.0 // indirect
-	go.opentelemetry.io/contrib/propagators/jaeger v1.35.0 // indirect
 	go.opentelemetry.io/contrib/samplers/jaegerremote v0.29.0 // indirect
 	go.opentelemetry.io/otel/exporters/jaeger v1.17.0 // indirect
 	go.opentelemetry.io/otel/log v0.11.0 // indirect
-	go.opentelemetry.io/otel/sdk v1.35.0 // indirect
 	go.opentelemetry.io/otel/sdk/metric v1.35.0 // indirect
 	gopkg.in/alexcesaro/quotedprintable.v3 v3.0.0-20150716171945-2caba252f4dc // indirect
 	gopkg.in/mail.v2 v2.3.1 // indirect
@@ -311,7 +311,7 @@ require (
 	go.etcd.io/etcd/client/v3 v3.5.4 // indirect
 	go.mongodb.org/mongo-driver v1.14.0 // indirect
 	go.opentelemetry.io/collector/semconv v0.125.0
-	go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp v0.60.0 // indirect
+	go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp v0.60.0
 	go.opentelemetry.io/otel/metric v1.35.0 // indirect
 	go.uber.org/zap v1.27.0 // indirect
 	golang.org/x/exp v0.0.0-20250106191152-7588d65b2ba8 // indirect

pkg/alertmanager/alertmanager.go

@@ -63,6 +63,7 @@ import (
 	commoncfg "github.com/prometheus/common/config"
 	"github.com/prometheus/common/model"
 	"github.com/prometheus/common/route"
+	"go.opentelemetry.io/otel"
 	"go.uber.org/atomic"
 	"golang.org/x/time/rate"
 
@@ -84,6 +85,8 @@ const (
 	silencesStateKeyPrefix = "sil:"
 )
 
+var tracer = otel.Tracer("pkg/alertmanager")
+
 // Config configures an Alertmanager.
 type Config struct {
 	UserID                            string

pkg/alertmanager/alertmanager_client.go

@@ -16,6 +16,7 @@ import (
 	"github.com/pkg/errors"
 	"github.com/prometheus/client_golang/prometheus"
 	"github.com/prometheus/client_golang/prometheus/promauto"
+	"go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc"
 	"google.golang.org/grpc"
 	"google.golang.org/grpc/health/grpc_health_v1"
 
@@ -111,6 +112,8 @@ func dialAlertmanagerClient(cfg grpcclient.Config, inst ring.InstanceDesc, reque
 	if err != nil {
 		return nil, err
 	}
+	opts = append(opts, grpc.WithStatsHandler(otelgrpc.NewClientHandler()))
+
 	// nolint:staticcheck // grpc.Dial() has been deprecated; we'll address it before upgrading to gRPC 2.
 	conn, err := grpc.Dial(inst.Addr, opts...)
 	if err != nil {

pkg/alertmanager/distributor.go

@@ -23,7 +23,6 @@ import (
 	"github.com/grafana/dskit/services"
 	"github.com/grafana/dskit/tenant"
 	"github.com/grafana/dskit/user"
-	"github.com/opentracing/opentracing-go"
 	"github.com/pkg/errors"
 	"github.com/prometheus/client_golang/prometheus"
 
@@ -198,8 +197,8 @@ func (d *Distributor) doAll(userID string, w http.ResponseWriter, r *http.Reques
 
 	results, err := replicationSet.Do(r.Context(), 0, func(ctx context.Context, instance *ring.InstanceDesc) (any, error) {
 		ctx = user.InjectOrgID(ctx, userID)
-		sp, ctx := opentracing.StartSpanFromContext(ctx, "Distributor.doAll")
-		defer sp.Finish()
+		ctx, sp := tracer.Start(ctx, "Distributor.doAll")
+		defer sp.End()
 
 		resp, err := d.doRequest(ctx, *instance, &httpgrpc.HTTPRequest{
 			Method:  r.Method,
@@ -259,8 +258,8 @@ func (d *Distributor) doQuorum(userID string, w http.ResponseWriter, r *http.Req
 	err = ring.DoBatchWithOptions(r.Context(), RingOp, d.alertmanagerRing, []uint32{shardByUser(userID)}, func(am ring.InstanceDesc, _ []int) error {
 		// Use a background context to make sure all alertmanagers get the request even if we return early.
 		localCtx := user.InjectOrgID(context.Background(), userID)
-		sp, localCtx := opentracing.StartSpanFromContext(localCtx, "Distributor.doQuorum")
-		defer sp.Finish()
+		localCtx, sp := tracer.Start(localCtx, "Distributor.doQuorum")
+		defer sp.End()
 
 		resp, err := d.doRequest(localCtx, am, &httpgrpc.HTTPRequest{
 			Method:  r.Method,
@@ -327,8 +326,8 @@ func (d *Distributor) doUnary(userID string, w http.ResponseWriter, r *http.Requ
 		Headers: httpToHttpgrpcHeaders(r.Header),
 	}
 
-	sp, ctx := opentracing.StartSpanFromContext(r.Context(), "Distributor.doUnary")
-	defer sp.Finish()
+	ctx, sp := tracer.Start(r.Context(), "Distributor.doUnary")
+	defer sp.End()
 	// Until we have a mechanism to combine the results from multiple alertmanagers,
 	// we forward the request to only one of the alertmanagers.
 	amDesc := replicationSet.Instances[rand.Intn(len(replicationSet.Instances))]

pkg/blockbuilder/blockbuilder.go

@@ -15,13 +15,13 @@ import (
 	"github.com/grafana/dskit/backoff"
 	"github.com/grafana/dskit/runutil"
 	"github.com/grafana/dskit/services"
-	otgrpc "github.com/opentracing-contrib/go-grpc"
-	"github.com/opentracing/opentracing-go"
 	"github.com/prometheus/client_golang/prometheus"
 	"github.com/prometheus/prometheus/tsdb"
 	"github.com/thanos-io/objstore"
 	"github.com/twmb/franz-go/pkg/kadm"
 	"github.com/twmb/franz-go/pkg/kgo"
+	"go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc"
+	"go.opentelemetry.io/otel"
 	"go.uber.org/atomic"
 	"google.golang.org/grpc"
 
@@ -35,6 +35,8 @@ import (
 	"github.com/grafana/mimir/pkg/util/validation"
 )
 
+var tracer = otel.Tracer("pkg/blockbuilder")
+
 type BlockBuilder struct {
 	services.Service
 
@@ -128,17 +130,18 @@ func newWithSchedulerClient(
 }
 
 func (b *BlockBuilder) makeSchedulerClient() (schedulerpb.SchedulerClient, *grpc.ClientConn, error) {
-	dialOpts, err := b.cfg.SchedulerConfig.GRPCClientConfig.DialOption(
-		[]grpc.UnaryClientInterceptor{otgrpc.OpenTracingClientInterceptor(opentracing.GlobalTracer())},
+	opts, err := b.cfg.SchedulerConfig.GRPCClientConfig.DialOption(
+		nil,
 		nil,
 		util.NewInvalidClusterValidationReporter(b.cfg.SchedulerConfig.GRPCClientConfig.ClusterValidation.Label, b.blockBuilderMetrics.invalidClusterValidation, b.logger),
 	)
 	if err != nil {
 		return nil, nil, err
 	}
+	opts = append(opts, grpc.WithStatsHandler(otelgrpc.NewClientHandler()))
 
 	// nolint:staticcheck // grpc.Dial() has been deprecated; we'll address it before upgrading to gRPC 2.
-	conn, err := grpc.Dial(b.cfg.SchedulerConfig.Address, dialOpts...)
+	conn, err := grpc.Dial(b.cfg.SchedulerConfig.Address, opts...)
 	if err != nil {
 		return nil, nil, err
 	}
@@ -242,7 +245,7 @@ func (b *BlockBuilder) consumeJob(ctx context.Context, key schedulerpb.JobKey, s
 		b.blockBuilderMetrics.consumeJobDuration.WithLabelValues(success).Observe(time.Since(start).Seconds())
 	}(time.Now())
 
-	sp, ctx := spanlogger.NewWithLogger(ctx, b.logger, "BlockBuilder.consumeJob")
+	sp, ctx := spanlogger.New(ctx, b.logger, tracer, "BlockBuilder.consumeJob")
 	defer sp.Finish()
 
 	logger := log.With(sp, "partition", spec.Partition, "job_id", key.Id, "job_epoch", key.Epoch)
@@ -445,7 +448,7 @@ func PartitionStateFromLag(logger log.Logger, lag kadm.GroupMemberLag, fallbackM
 // consumePartition consumes records from the given partition until the cycleEnd timestamp.
 // If the partition is lagging behind, it takes care of consuming it in sections.
 func (b *BlockBuilder) consumePartition(ctx context.Context, partition int32, state PartitionState, cycleEndTime time.Time, cycleEndOffset int64, logger log.Logger) (finalState PartitionState, err error) {
-	sp, ctx := spanlogger.NewWithLogger(ctx, logger, "BlockBuilder.consumePartition")
+	sp, ctx := spanlogger.New(ctx, logger, tracer, "BlockBuilder.consumePartition")
 	defer sp.Finish()
 
 	logger = log.With(sp, "partition", partition, "cycle_end", cycleEndTime, "cycle_end_offset", cycleEndOffset)