diff --git a/CHANGELOG.md b/CHANGELOG.md index bc6c571bb4a..c3c5bdd5878 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -24,6 +24,7 @@ This project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0.htm - Add experimental observability metrics for manual reader in `go.opentelemetry.io/otel/sdk/metric`. (#7524) - Add experimental observability metrics for periodic reader in `go.opentelemetry.io/otel/sdk/metric`. (#7571) - Support `OTEL_EXPORTER_OTLP_LOGS_INSECURE` and `OTEL_EXPORTER_OTLP_INSECURE` environmental variables in `go.opentelemetry.io/otel/exporters/otlp/otlplog/otlploghttp`. (#7608) +- Add new `SpanStartOption`s to control the creation of `"runtime/trace"` Tasks and Regions. (#7628) ### Fixed diff --git a/sdk/instrumentation/scope.go b/sdk/instrumentation/scope.go index 34852a47b21..097769f67c0 100644 --- a/sdk/instrumentation/scope.go +++ b/sdk/instrumentation/scope.go @@ -3,7 +3,9 @@ package instrumentation // import "go.opentelemetry.io/otel/sdk/instrumentation" -import "go.opentelemetry.io/otel/attribute" +import ( + "go.opentelemetry.io/otel/attribute" +) // Scope represents the instrumentation scope. type Scope struct { diff --git a/sdk/trace/profiling_span_test.go b/sdk/trace/profiling_span_test.go new file mode 100644 index 00000000000..42def77fd7f --- /dev/null +++ b/sdk/trace/profiling_span_test.go @@ -0,0 +1,361 @@ +// Copyright The OpenTelemetry Authors +// SPDX-License-Identifier: Apache-2.0 + +package trace + +import ( + "context" + "sync/atomic" + "testing" + + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" + + "go.opentelemetry.io/otel/trace" +) + +// mockRuntimeTracer is a simple mock implementation of runtimeTracer for testing. +type mockRuntimeTracer struct { + isEnabled bool + isEnabledCalls atomic.Uint64 + newTaskCalls atomic.Uint64 + startRegionCalls atomic.Uint64 + taskEndCalls atomic.Uint64 + regionEndCalls atomic.Uint64 +} + +func newMockRuntimeTracer(enabled bool) *mockRuntimeTracer { + return &mockRuntimeTracer{ + isEnabled: enabled, + } +} + +func (m *mockRuntimeTracer) IsEnabled() bool { + m.isEnabledCalls.Add(1) + return m.isEnabled +} + +func (m *mockRuntimeTracer) NewTask(ctx context.Context, _ string) (context.Context, runtimeTraceEndFn) { + m.newTaskCalls.Add(1) + endFunc := func() { + m.taskEndCalls.Add(1) + } + return ctx, endFunc +} + +func (m *mockRuntimeTracer) StartRegion(_ context.Context, _ string) runtimeTraceEndFn { + m.startRegionCalls.Add(1) + endFunc := func() { + m.regionEndCalls.Add(1) + } + return endFunc +} + +func assertCalls(t *testing.T, m *mockRuntimeTracer, isEnabled, newTask, startRegion, taskEnd, regionEnd int) { + assert.Equal(t, uint64(isEnabled), m.isEnabledCalls.Load()) + assert.Equal(t, uint64(newTask), m.newTaskCalls.Load()) + assert.Equal(t, uint64(startRegion), m.startRegionCalls.Load()) + assert.Equal(t, uint64(taskEnd), m.taskEndCalls.Load()) + assert.Equal(t, uint64(regionEnd), m.regionEndCalls.Load()) +} + +func TestDefaultInstrumentation(t *testing.T) { + // The cases in this test will be the default behavior for all OTel users that do not specify any runtime/trace + // instrumentation options. The default behavior is to create a task for each root span and nothing else. + + originalRuntimeTracer := globalRuntimeTracer + t.Cleanup(func() { + globalRuntimeTracer = originalRuntimeTracer + }) + + tracerProvider := NewTracerProvider(WithSampler(AlwaysSample())) + tracer := tracerProvider.Tracer("TestDefaultInstrumentation") + + mockRT := newMockRuntimeTracer(true) + globalRuntimeTracer = mockRT + + ctx := t.Context() + + t.Run("local root span creates a task", func(t *testing.T) { + _, span := tracer.Start(ctx, "root-span") + assertCalls(t, mockRT, 1, 1, 0, 0, 0) + profilingSpan, ok := span.(profilingSpan) + require.True(t, ok) + assert.True(t, profilingSpan.profilingStarted()) + span.End() + assertCalls(t, mockRT, 1, 1, 0, 1, 0) + }) + + rootCtx, _ := tracer.Start(ctx, "root-span", trace.NoProfiling()) + t.Run("disable default instrumentation individually on root span", func(t *testing.T) { + assertCalls(t, mockRT, 2, 1, 0, 1, 0) // no new task this time + }) + + t.Run("local child spans do nothing", func(t *testing.T) { + _, childSpan := tracer.Start(rootCtx, "child-span") + assertCalls(t, mockRT, 3, 1, 0, 1, 0) + profilingSpan, ok := childSpan.(profilingSpan) + require.True(t, ok) + assert.False(t, profilingSpan.profilingStarted()) + childSpan.End() + assertCalls(t, mockRT, 3, 1, 0, 1, 0) + }) + + t.Run("force manual task for child span", func(t *testing.T) { + _, childSpan := tracer.Start(rootCtx, "child-span-task", trace.ProfileTask()) + assertCalls(t, mockRT, 4, 2, 0, 1, 0) + profilingSpan, ok := childSpan.(profilingSpan) + require.True(t, ok) + assert.True(t, profilingSpan.profilingStarted()) + childSpan.End() + assertCalls(t, mockRT, 4, 2, 0, 2, 0) + }) + + t.Run("force manual region for child span", func(t *testing.T) { + _, childSpan := tracer.Start(rootCtx, "child-span-region", trace.ProfileRegion()) + assertCalls(t, mockRT, 5, 2, 1, 2, 0) + profilingSpan, ok := childSpan.(profilingSpan) + require.True(t, ok) + assert.True(t, profilingSpan.profilingStarted()) + childSpan.End() + assertCalls(t, mockRT, 5, 2, 1, 2, 1) + }) + + t.Run("force auto instrumentation for child span", func(t *testing.T) { + // Not sure how useful this is for users, but trace.ProfilingAuto > trace.ProfilingDefault, so it works. + _, childSpan := tracer.Start( + rootCtx, + "child-span-auto", + trace.WithProfileTask(trace.ProfilingAuto), + trace.WithProfileRegion(trace.ProfilingAuto), + ) + assertCalls(t, mockRT, 6, 2, 2, 2, 1) + profilingSpan, ok := childSpan.(profilingSpan) + require.True(t, ok) + assert.True(t, profilingSpan.profilingStarted()) + childSpan.End() + assertCalls(t, mockRT, 6, 2, 2, 2, 2) + }) +} + +func TestManualInstrumentation(t *testing.T) { + // Notice that the tracer is created with trace.WithProfilingMode(trace.ProfilingManual), and all spans won't create + // tasks or region unless they are explicitly tagged with trace.ProfileTask() or trace.ProfileRegion(). + + originalRuntimeTracer := globalRuntimeTracer + t.Cleanup(func() { + globalRuntimeTracer = originalRuntimeTracer + }) + + tracerProvider := NewTracerProvider(WithSampler(AlwaysSample())) + tracer := tracerProvider.Tracer("TestManualInstrumentation", trace.WithProfilingMode(trace.ProfilingManual)) + + mockRT := newMockRuntimeTracer(true) + globalRuntimeTracer = mockRT + + ctx := t.Context() + + t.Run("root span with task", func(t *testing.T) { + _, rootSpanWithTask := tracer.Start(ctx, "root-span-with-task", trace.ProfileTask()) + assertCalls(t, mockRT, 1, 1, 0, 0, 0) + profilingSpan, ok := rootSpanWithTask.(profilingSpan) + require.True(t, ok) + assert.True(t, profilingSpan.profilingStarted()) + rootSpanWithTask.End() + assertCalls(t, mockRT, 1, 1, 0, 1, 0) + }) + + t.Run("root span with region", func(t *testing.T) { + // This is a special case where the region does not have a task + // associated but is allowed by the runtime/trace API. The region will + // be associated with the background task. + _, rootSpanWithRegion := tracer.Start(ctx, "root-span-with-region", trace.ProfileRegion()) + assertCalls(t, mockRT, 2, 1, 1, 1, 0) + profilingSpan, ok := rootSpanWithRegion.(profilingSpan) + require.True(t, ok) + assert.True(t, profilingSpan.profilingStarted()) + rootSpanWithRegion.End() + assertCalls(t, mockRT, 2, 1, 1, 1, 1) + }) + + t.Run("root span without profiling", func(t *testing.T) { + _, rootSpanWithoutProfiling := tracer.Start(ctx, "root-span-without-profiling") + assertCalls(t, mockRT, 3, 1, 1, 1, 1) + profilingSpan, ok := rootSpanWithoutProfiling.(profilingSpan) + require.True(t, ok) + assert.False(t, profilingSpan.profilingStarted()) + rootSpanWithoutProfiling.End() + assertCalls(t, mockRT, 3, 1, 1, 1, 1) + }) + + t.Run("root span with both task and region", func(t *testing.T) { + // Not sure of the utility of this case, but our API is flexible/not opinionated. + _, rootSpanWithBothTaskAndRegion := tracer.Start( + ctx, + "root-span-with-both-task-and-region", + trace.WithProfileTask(trace.ProfilingManual), + trace.WithProfileRegion(trace.ProfilingManual), + ) + assertCalls(t, mockRT, 4, 2, 2, 1, 1) + profilingSpan, ok := rootSpanWithBothTaskAndRegion.(profilingSpan) + require.True(t, ok) + assert.True(t, profilingSpan.profilingStarted()) + rootSpanWithBothTaskAndRegion.End() + assertCalls(t, mockRT, 4, 2, 2, 2, 2) + }) + + rootCtx, _ := tracer.Start(ctx, "root-span") + assertCalls(t, mockRT, 5, 2, 2, 2, 2) + + t.Run("child span with task", func(t *testing.T) { + _, childSpanWithTask := tracer.Start(rootCtx, "child-span-with-task", trace.ProfileTask()) + assertCalls(t, mockRT, 6, 3, 2, 2, 2) + profilingSpan, ok := childSpanWithTask.(profilingSpan) + require.True(t, ok) + assert.True(t, profilingSpan.profilingStarted()) + childSpanWithTask.End() + assertCalls(t, mockRT, 6, 3, 2, 3, 2) + }) + + t.Run("child span with region", func(t *testing.T) { + // Notice the parent span is not attached to a task, so this region is not associated with any task. + // This is fine for runtime/trace API, the region will be associated with the background task. + _, childSpanWithRegion := tracer.Start(rootCtx, "child-span-with-region", trace.ProfileRegion()) + assertCalls(t, mockRT, 7, 3, 3, 3, 2) + profilingSpan, ok := childSpanWithRegion.(profilingSpan) + require.True(t, ok) + assert.True(t, profilingSpan.profilingStarted()) + childSpanWithRegion.End() + assertCalls(t, mockRT, 7, 3, 3, 3, 3) + }) + + t.Run("child span without profiling", func(t *testing.T) { + _, childSpanWithoutProfiling := tracer.Start(rootCtx, "child-span-without-profiling") + assertCalls(t, mockRT, 8, 3, 3, 3, 3) + profilingSpan, ok := childSpanWithoutProfiling.(profilingSpan) + require.True(t, ok) + assert.False(t, profilingSpan.profilingStarted()) + childSpanWithoutProfiling.End() + assertCalls(t, mockRT, 8, 3, 3, 3, 3) + }) +} + +func TestAutoInstrumentation(t *testing.T) { + // Notice the usage of trace.AutoProfiling(), trace.AsyncEnd() and trace.NoProfiling() in this test. + + originalRuntimeTracer := globalRuntimeTracer + t.Cleanup(func() { + globalRuntimeTracer = originalRuntimeTracer + }) + + tracerProvider := NewTracerProvider(WithSampler(AlwaysSample())) + tracer := tracerProvider.Tracer("TestAutoProfiling", trace.AutoProfiling()) + + t.Run("local root creates a task and children create regions", func(t *testing.T) { + mockRT := newMockRuntimeTracer(true) + globalRuntimeTracer = mockRT + + ctx := t.Context() + spanCtx, span := tracer.Start(ctx, "root-span") + assertCalls(t, mockRT, 1, 1, 0, 0, 0) + + childSpanCtx, childSpan := tracer.Start(spanCtx, "child-span") + assertCalls(t, mockRT, 2, 1, 1, 0, 0) + + _, grandchildSpan := tracer.Start(childSpanCtx, "grandchild-span") + assertCalls(t, mockRT, 3, 1, 2, 0, 0) + + grandchildSpan.End() + assertCalls(t, mockRT, 3, 1, 2, 0, 1) + + childSpan.End() + assertCalls(t, mockRT, 3, 1, 2, 0, 2) + + span.End() + assertCalls(t, mockRT, 3, 1, 2, 1, 2) + }) + + t.Run("async spans create tasks instead of regions", func(t *testing.T) { + mockRT := newMockRuntimeTracer(true) + globalRuntimeTracer = mockRT + + ctx := t.Context() + rootCtx, rootSpan := tracer.Start(ctx, "root-span") + assertCalls(t, mockRT, 1, 1, 0, 0, 0) + + _, childSpan := tracer.Start(rootCtx, "child-span", trace.AsyncEnd()) + assertCalls(t, mockRT, 2, 2, 0, 0, 0) // async span creates a task instead of a region + + childSpan.End() + assertCalls(t, mockRT, 2, 2, 0, 1, 0) + + rootSpan.End() + assertCalls(t, mockRT, 2, 2, 0, 2, 0) + }) + + t.Run("force manual instrumentation on individual span", func(t *testing.T) { + mockRT := newMockRuntimeTracer(true) + globalRuntimeTracer = mockRT + + ctx := t.Context() + rootCtx, rootSpan := tracer.Start(ctx, "root-span") + assertCalls(t, mockRT, 1, 1, 0, 0, 0) + + _, childSpan := tracer.Start(rootCtx, "child-span", trace.ProfileTask()) + assertCalls(t, mockRT, 2, 2, 0, 0, 0) + + childSpan.End() + assertCalls(t, mockRT, 2, 2, 0, 1, 0) + + rootSpan.End() + assertCalls(t, mockRT, 2, 2, 0, 2, 0) + }) + + t.Run("disable profiling at span level", func(t *testing.T) { + mockRT := newMockRuntimeTracer(true) + globalRuntimeTracer = mockRT + + ctx := t.Context() + _, span := tracer.Start(ctx, "root-span", trace.NoProfiling()) + assertCalls(t, mockRT, 1, 0, 0, 0, 0) + + span.End() + assertCalls(t, mockRT, 1, 0, 0, 0, 0) + }) +} + +func TestRuntimeTracerDisabled(t *testing.T) { + originalRuntimeTracer := globalRuntimeTracer + t.Cleanup(func() { + globalRuntimeTracer = originalRuntimeTracer + }) + + mockRT := newMockRuntimeTracer(false) + globalRuntimeTracer = mockRT + + tracerProvider := NewTracerProvider(WithSampler(AlwaysSample())) + tracer := tracerProvider.Tracer("TestRuntimeTracerDisabled") + // even manually tagged spans won't create tasks + _, span := tracer.Start(t.Context(), "root-span", trace.ProfileTask()) + assertCalls(t, mockRT, 1, 0, 0, 0, 0) + span.End() + assertCalls(t, mockRT, 1, 0, 0, 0, 0) +} + +func TestInstrumentationDisabled(t *testing.T) { + originalRuntimeTracer := globalRuntimeTracer + t.Cleanup(func() { + globalRuntimeTracer = originalRuntimeTracer + }) + + mockRT := newMockRuntimeTracer(true) + globalRuntimeTracer = mockRT + + tracerProvider := NewTracerProvider(WithSampler(AlwaysSample())) + tracer := tracerProvider.Tracer("TestInstrumentationDisabled", trace.WithProfilingMode(trace.ProfilingDisabled)) + // even manually tagged spans won't create tasks + _, span := tracer.Start(t.Context(), "root-span", trace.ProfileTask()) + assertCalls(t, mockRT, 1, 0, 0, 0, 0) + span.End() + assertCalls(t, mockRT, 1, 0, 0, 0, 0) +} diff --git a/sdk/trace/provider.go b/sdk/trace/provider.go index d2cf4ebd3e7..adf0e96bda0 100644 --- a/sdk/trace/provider.go +++ b/sdk/trace/provider.go @@ -157,6 +157,7 @@ func (p *TracerProvider) Tracer(name string, opts ...trace.TracerOption) trace.T t = &tracer{ provider: p, instrumentationScope: is, + profilingMode: c.ProfilingMode(), } var err error diff --git a/sdk/trace/runtime_trace.go b/sdk/trace/runtime_trace.go new file mode 100644 index 00000000000..7b6fae20a34 --- /dev/null +++ b/sdk/trace/runtime_trace.go @@ -0,0 +1,61 @@ +// Copyright The OpenTelemetry Authors +// SPDX-License-Identifier: Apache-2.0 + +package trace // import "go.opentelemetry.io/otel/sdk/trace" + +import ( + "context" + + runtimetrace "runtime/trace" + + "go.opentelemetry.io/otel/trace" +) + +type runtimeTraceEndFn func() + +// runtimeTracer abstracts the runtime/trace package so it can be mocked +// in tests. The runtime/trace API provides local, runtime-level +// instrumentation, recorded in Go execution profiles, similar to distributed +// tracing but confined to a single Go process. This interface allows +// integrating that instrumentation with distributed tracing without +// duplicating logic. +type runtimeTracer interface { + IsEnabled() bool + NewTask(ctx context.Context, name string) (context.Context, runtimeTraceEndFn) + StartRegion(ctx context.Context, name string) runtimeTraceEndFn +} + +// standardRuntimeTracer is the default implementation of runtimeTracer. +// It simply wraps the runtime/trace package. +type standardRuntimeTracer struct{} + +func (standardRuntimeTracer) IsEnabled() bool { + return runtimetrace.IsEnabled() +} + +func (standardRuntimeTracer) NewTask(ctx context.Context, name string) (context.Context, runtimeTraceEndFn) { + nctx, task := runtimetrace.NewTask(ctx, name) + return nctx, task.End +} + +func (standardRuntimeTracer) StartRegion(ctx context.Context, name string) runtimeTraceEndFn { + region := runtimetrace.StartRegion(ctx, name) + return region.End +} + +// globalRuntimeTracer is the variable that holds the global runtimeTracer +// implementation. It defaults to the real implementation but can be swapped +// for testing. +var globalRuntimeTracer runtimeTracer = standardRuntimeTracer{} + +// profilingSpan is an interface for spans that can integrate with +// runtimeTracer. +type profilingSpan interface { + // startProfiling may start a "runtime/trace" Task (returning a new + // context) or a Region (no context change). If tracing is disabled + // (globally or for the span), it does nothing. Concrete implementations + // may have their own defaults when config is not explicit. + startProfiling(ctx context.Context, config *trace.SpanConfig, tracerSetting trace.ProfilingMode) context.Context + endProfiling() + profilingStarted() bool +} diff --git a/sdk/trace/span.go b/sdk/trace/span.go index 8cfd9f62e3f..9930f07a383 100644 --- a/sdk/trace/span.go +++ b/sdk/trace/span.go @@ -8,7 +8,6 @@ import ( "fmt" "reflect" "runtime" - rt "runtime/trace" "slices" "strings" "sync" @@ -146,8 +145,8 @@ type recordingSpan struct { // links are stored in FIFO queue capped by configured limit. links evictedQueue[Link] - // executionTracerTaskEnd ends the execution tracer span. - executionTracerTaskEnd func() + // runtimeTraceEnd ends the "runtime/trace" task or region. + runtimeTraceEnd runtimeTraceEndFn // tracer is the SDK tracer that created this span. tracer *tracer @@ -161,7 +160,7 @@ type recordingSpan struct { var ( _ ReadWriteSpan = (*recordingSpan)(nil) - _ runtimeTracer = (*recordingSpan)(nil) + _ profilingSpan = (*recordingSpan)(nil) ) func (s *recordingSpan) setOrigCtx(ctx context.Context) { @@ -492,9 +491,9 @@ func (s *recordingSpan) End(options ...trace.SpanEndOption) { s.addEvent(semconv.ExceptionEventName, opts...) } - if s.executionTracerTaskEnd != nil { + if s.profilingStarted() { s.mu.Unlock() - s.executionTracerTaskEnd() + s.endProfiling() s.mu.Lock() } @@ -878,20 +877,96 @@ func (s *recordingSpan) addChild() { func (*recordingSpan) private() {} -// runtimeTrace starts a "runtime/trace".Task for the span and returns a -// context containing the task. -func (s *recordingSpan) runtimeTrace(ctx context.Context) context.Context { - if !rt.IsEnabled() { +func (s *recordingSpan) shouldCreateTask(config *trace.SpanConfig, tracerSetting trace.ProfilingMode) bool { + if config.ProfileTask() > tracerSetting { + tracerSetting = config.ProfileTask() + } + + switch tracerSetting { + case trace.ProfilingDefault: + isLocalRoot := !s.parent.IsValid() || s.parent.IsRemote() + return isLocalRoot + case trace.ProfilingAuto: + if isLocalRoot := !s.parent.IsValid() || s.parent.IsRemote(); isLocalRoot { + return true + } + return tracerSetting >= config.ProfileRegion() && config.AsyncEnd() + case trace.ProfilingManual: + return config.ProfileTask() == trace.ProfilingManual + case trace.ProfilingDisabled: + return false + default: + return false // unrecognized value + } +} + +func (s *recordingSpan) shouldCreateRegion(config *trace.SpanConfig, tracerSetting trace.ProfilingMode) bool { + if config.ProfileRegion() > tracerSetting { + tracerSetting = config.ProfileRegion() + } + + switch tracerSetting { + case trace.ProfilingDefault: + return false + case trace.ProfilingAuto: + if isLocalRoot := !s.parent.IsValid() || s.parent.IsRemote(); isLocalRoot { + return false + } + return !config.AsyncEnd() + case trace.ProfilingManual: + return config.ProfileRegion() == trace.ProfilingManual + case trace.ProfilingDisabled: + return false + default: + return false // unrecognized value + } +} + +// startProfiling implements profilingSpan. +func (s *recordingSpan) startProfiling( + ctx context.Context, + config *trace.SpanConfig, + tracerSetting trace.ProfilingMode, +) context.Context { + if !globalRuntimeTracer.IsEnabled() { // Avoid additional overhead if runtime/trace is not enabled. return ctx } - nctx, task := rt.NewTask(ctx, s.name) - s.mu.Lock() - s.executionTracerTaskEnd = task.End - s.mu.Unlock() + var endFn runtimeTraceEndFn + if s.shouldCreateTask(config, tracerSetting) { + ctx, endFn = globalRuntimeTracer.NewTask(ctx, s.name) + } + + if s.shouldCreateRegion(config, tracerSetting) { + regionEndFn := globalRuntimeTracer.StartRegion(ctx, s.name) + if endFn == nil { + endFn = regionEndFn + } else { + taskEndFn := endFn + endFn = func() { + regionEndFn() + taskEndFn() + } + } + } + + if endFn != nil { + s.mu.Lock() + s.runtimeTraceEnd = endFn + s.mu.Unlock() + } + return ctx +} + +// endProfiling implements profilingSpan. +func (s *recordingSpan) endProfiling() { + s.runtimeTraceEnd() +} - return nctx +// profilingStarted implements profilingSpan. +func (s *recordingSpan) profilingStarted() bool { + return s.runtimeTraceEnd != nil } // nonRecordingSpan is a minimal implementation of the OpenTelemetry Span API diff --git a/sdk/trace/trace_test.go b/sdk/trace/trace_test.go index a5c6f5e2229..3776e04d6a4 100644 --- a/sdk/trace/trace_test.go +++ b/sdk/trace/trace_test.go @@ -11,7 +11,6 @@ import ( "strconv" "strings" "sync" - "sync/atomic" "testing" "time" @@ -1171,33 +1170,11 @@ func TestNonRecordingSpanDoesNotTrackRuntimeTracerTask(t *testing.T) { tr := tp.Tracer("TestNonRecordingSpanDoesNotTrackRuntimeTracerTask") _, apiSpan := tr.Start(t.Context(), "foo") - if _, ok := apiSpan.(runtimeTracer); ok { + if _, ok := apiSpan.(profilingSpan); ok { t.Fatalf("non recording span implements runtime trace task tracking") } } -func TestRecordingSpanRuntimeTracerTaskEnd(t *testing.T) { - tp := NewTracerProvider(WithSampler(AlwaysSample())) - tr := tp.Tracer("TestRecordingSpanRuntimeTracerTaskEnd") - - var n uint64 - executionTracerTaskEnd := func() { - atomic.AddUint64(&n, 1) - } - _, apiSpan := tr.Start(t.Context(), "foo") - s, ok := apiSpan.(*recordingSpan) - if !ok { - t.Fatal("recording span not returned from always sampled Tracer") - } - - s.executionTracerTaskEnd = executionTracerTaskEnd - s.End() - - if n != 1 { - t.Error("recording span did not end runtime trace task") - } -} - func TestCustomStartEndTime(t *testing.T) { te := NewTestExporter() tp := NewTracerProvider(WithSyncer(te), WithSampler(AlwaysSample())) diff --git a/sdk/trace/tracer.go b/sdk/trace/tracer.go index e1d08fd4d8d..1cfdcfe7073 100644 --- a/sdk/trace/tracer.go +++ b/sdk/trace/tracer.go @@ -20,6 +20,8 @@ type tracer struct { instrumentationScope instrumentation.Scope inst observ.Tracer + + profilingMode trace.ProfilingMode } var _ trace.Tracer = &tracer{} @@ -70,19 +72,13 @@ func (tr *tracer) Start( sp.sp.OnStart(ctx, rw) } } - if rtt, ok := s.(runtimeTracer); ok { - newCtx = rtt.runtimeTrace(newCtx) + if profilingSpan, ok := s.(profilingSpan); ok { + newCtx = profilingSpan.startProfiling(newCtx, &config, tr.profilingMode) } return newCtx, s } -type runtimeTracer interface { - // runtimeTrace starts a "runtime/trace".Task for the span and - // returns a context containing the task. - runtimeTrace(ctx context.Context) context.Context -} - // newSpan returns a new configured span. func (tr *tracer) newSpan(ctx context.Context, name string, config *trace.SpanConfig) trace.Span { // If told explicitly to make this a new root use a zero value SpanContext diff --git a/trace/config.go b/trace/config.go index d9ecef1cad2..df14a6af1af 100644 --- a/trace/config.go +++ b/trace/config.go @@ -14,8 +14,9 @@ import ( type TracerConfig struct { instrumentationVersion string // Schema URL of the telemetry emitted by the Tracer. - schemaURL string - attrs attribute.Set + schemaURL string + attrs attribute.Set + profilingMode ProfilingMode } // InstrumentationVersion returns the version of the library providing instrumentation. @@ -34,6 +35,11 @@ func (t *TracerConfig) SchemaURL() string { return t.schemaURL } +// ProfilingMode returns the profiling mode for the tracer. +func (t *TracerConfig) ProfilingMode() ProfilingMode { + return t.profilingMode +} + // NewTracerConfig applies all the options to a returned TracerConfig. func NewTracerConfig(options ...TracerOption) TracerConfig { var config TracerConfig @@ -56,12 +62,15 @@ func (fn tracerOptionFunc) apply(cfg TracerConfig) TracerConfig { // SpanConfig is a group of options for a Span. type SpanConfig struct { - attributes []attribute.KeyValue - timestamp time.Time - links []Link - newRoot bool - spanKind SpanKind - stackTrace bool + attributes []attribute.KeyValue + timestamp time.Time + links []Link + newRoot bool + spanKind SpanKind + stackTrace bool + profileRegion ProfilingMode + profileTask ProfilingMode + asyncEnd bool } // Attributes describe the associated qualities of a Span. @@ -96,6 +105,24 @@ func (cfg *SpanConfig) SpanKind() SpanKind { return cfg.spanKind } +// ProfileRegion reports whether the span should create a runtime/trace.Region. +// The returned mode may be overridden by tracer-level profiling settings. +func (cfg *SpanConfig) ProfileRegion() ProfilingMode { + return cfg.profileRegion +} + +// ProfileTask reports whether the span should create a runtime/trace.Task. +// The returned mode may be overridden by tracer-level profiling settings. +func (cfg *SpanConfig) ProfileTask() ProfilingMode { + return cfg.profileTask +} + +// AsyncEnd reports whether the span will be ended on a different goroutine +// than the one it was started on. +func (cfg *SpanConfig) AsyncEnd() bool { + return cfg.asyncEnd +} + // NewSpanStartConfig applies all the options to a returned SpanConfig. // No validation is performed on the returned SpanConfig (e.g. no uniqueness // checking or bounding of data), it is left to the SDK to perform this @@ -132,6 +159,17 @@ func (fn spanOptionFunc) applySpanStart(cfg SpanConfig) SpanConfig { return fn(cfg) } +// ComposeSpanStartOptions combines the given options into one, applying them +// sequentially with later options taking precedence. +func ComposeSpanStartOptions(options ...SpanStartOption) SpanStartOption { + return spanOptionFunc(func(cfg SpanConfig) SpanConfig { + for _, option := range options { + cfg = option.applySpanStart(cfg) + } + return cfg + }) +} + // SpanEndOption applies an option to a SpanConfig. These options are // applicable only when the span is ended. type SpanEndOption interface { @@ -270,6 +308,70 @@ func WithStackTrace(b bool) SpanEndEventOption { return stackTraceOption(b) } +// WithProfileRegion controls whether the span should create a +// runtime/trace.Region. +// - ProfilingManual: the span always creates a Region. +// - ProfilingDisabled: the span does not create a Region. +// - ProfilingDefault: equivalent to ProfilingDisabled. +// - ProfilingAuto: the tracer decides whether to create a Region. This is +// typically configured at the tracer level via WithAutoProfiling, but it +// may also be set per span. +// +// Note: when profiling is set to ProfilingAuto, spans that end on a different +// goroutine than they started must be annotated with AsyncEnd(). +func WithProfileRegion(profileRegion ProfilingMode) SpanStartOption { + return spanOptionFunc(func(cfg SpanConfig) SpanConfig { + cfg.profileRegion = profileRegion + return cfg + }) +} + +// ProfileRegion is equivalent to applying both +// WithProfileRegion(ProfilingManual) and WithProfileTask(ProfilingDisabled). +func ProfileRegion() SpanStartOption { + return ComposeSpanStartOptions(WithProfileTask(ProfilingDisabled), WithProfileRegion(ProfilingManual)) +} + +// WithProfileTask controls whether the span should create a runtime/trace.Task. +// - ProfilingManual: the span always creates a Task. +// - ProfilingDisabled: the span does not create a Task. +// - ProfilingDefault: only root local spans create a Task. +// - ProfilingAuto: the tracer decides whether to create a Task. This is +// typically configured at the tracer level via WithAutoProfiling, but it +// may also be set per span. +func WithProfileTask(profileTask ProfilingMode) SpanStartOption { + return spanOptionFunc(func(cfg SpanConfig) SpanConfig { + cfg.profileTask = profileTask + return cfg + }) +} + +// ProfileTask is equivalent to applying both +// WithProfileTask(ProfilingManual) and WithProfileRegion(ProfilingDisabled). +func ProfileTask() SpanStartOption { + return ComposeSpanStartOptions(WithProfileRegion(ProfilingDisabled), WithProfileTask(ProfilingManual)) +} + +// NoProfiling is equivalent to applying both +// WithProfileRegion(ProfilingDisabled) and WithProfileTask(ProfilingDisabled). +func NoProfiling() SpanStartOption { + return ComposeSpanStartOptions(WithProfileRegion(ProfilingDisabled), WithProfileTask(ProfilingDisabled)) +} + +// WithAsyncEnd hints the tracer that the span will be ended on a different +// goroutine than the one it was started on. +func WithAsyncEnd(asyncEnd bool) SpanStartOption { + return spanOptionFunc(func(cfg SpanConfig) SpanConfig { + cfg.asyncEnd = asyncEnd + return cfg + }) +} + +// AsyncEnd is equivalent to WithAsyncEnd(true). +func AsyncEnd() SpanStartOption { + return WithAsyncEnd(true) +} + // WithLinks adds links to a Span. The links are added to the existing Span // links, i.e. this does not overwrite. Links with invalid span context are ignored. func WithLinks(links ...Link) SpanStartOption { @@ -360,3 +462,57 @@ func WithSchemaURL(schemaURL string) TracerOption { return cfg }) } + +// WithProfilingMode controls the profiling behavior of the tracer. +// +// Tracer-level profiling settings can be overridden by span-level settings, +// following the hierarchy: +// +// ProfilingDefault < ProfilingAuto < ProfilingManual < ProfilingDisabled +// +// A span may override the tracer’s mode only by selecting a *less strict* +// setting. For example, if the tracer is configured with ProfilingAuto, +// a span configured with ProfilingManual can take over and perform its own +// instrumentation. However, if the tracer is configured with ProfilingDisabled, +// no span-level option can re-enable profiling. +// +// When profiling is set to ProfilingAuto, spans that end on a different +// goroutine than they started must be annotated with AsyncEnd(). +func WithProfilingMode(mode ProfilingMode) TracerOption { + return tracerOptionFunc(func(cfg TracerConfig) TracerConfig { + cfg.profilingMode = mode + return cfg + }) +} + +// AutoProfiling is equivalent to WithProfilingMode(ProfilingAuto). +func AutoProfiling() TracerOption { + return WithProfilingMode(ProfilingAuto) +} + +// ProfilingMode defines the profiling behavior that can be applied at the +// tracer or span level. +type ProfilingMode int + +const ( + // ProfilingDefault is the default profiling mode. Root local spans create + // a runtime/trace.Task, while child spans do not create a Task or Region + // unless overridden at the span level. + ProfilingDefault ProfilingMode = iota + + // ProfilingAuto delegates instrumentation decisions to the tracer. Because + // tasks must begin and end on the same goroutine, spans that end on a + // different goroutine than they started must be annotated with AsyncEnd() + // when using this mode. + ProfilingAuto + + // ProfilingManual disables default and automatic profiling. The user is + // responsible for explicitly creating runtime/trace.Task and + // runtime/trace.Region instrumentation at the span level. + ProfilingManual + + // ProfilingDisabled disables profiling entirely. All profiler-related settings + // at the span level are ignored, and no runtime/trace instrumentation is + // produced. + ProfilingDisabled +) diff --git a/trace/config_test.go b/trace/config_test.go index 40667fa01b3..256a1d86001 100644 --- a/trace/config_test.go +++ b/trace/config_test.go @@ -30,24 +30,31 @@ func TestNewSpanConfig(t *testing.T) { } tests := []struct { - options []SpanStartOption - expected SpanConfig + name string + options []SpanStartOption + expected SpanConfig + customAssertFunction func(t *testing.T, cfg SpanConfig) }{ { // No non-zero-values should be set. + "Zero value", []SpanStartOption{}, SpanConfig{}, + nil, }, { + "WithAttributes", []SpanStartOption{ WithAttributes(k1v1), }, SpanConfig{ attributes: []attribute.KeyValue{k1v1}, }, + nil, }, { // Multiple calls should append not overwrite. + "WithAttributes multiple calls", []SpanStartOption{ WithAttributes(k1v1), WithAttributes(k1v2), @@ -57,8 +64,10 @@ func TestNewSpanConfig(t *testing.T) { // No uniqueness is guaranteed by the API. attributes: []attribute.KeyValue{k1v1, k1v2, k2v2}, }, + nil, }, { + "WithAttributes multiple values", []SpanStartOption{ WithAttributes(k1v1, k1v2, k2v2), }, @@ -66,16 +75,20 @@ func TestNewSpanConfig(t *testing.T) { // No uniqueness is guaranteed by the API. attributes: []attribute.KeyValue{k1v1, k1v2, k2v2}, }, + nil, }, { + "WithTimestamp", []SpanStartOption{ WithTimestamp(timestamp0), }, SpanConfig{ timestamp: timestamp0, }, + nil, }, { + "WithTimestamp multiple calls", []SpanStartOption{ // Multiple calls overwrites with last-one-wins. WithTimestamp(timestamp0), @@ -84,16 +97,20 @@ func TestNewSpanConfig(t *testing.T) { SpanConfig{ timestamp: timestamp1, }, + nil, }, { + "WithLinks", []SpanStartOption{ WithLinks(link1), }, SpanConfig{ links: []Link{link1}, }, + nil, }, { + "WithLinks multiple calls", []SpanStartOption{ // Multiple calls should append not overwrite. WithLinks(link1), @@ -103,16 +120,20 @@ func TestNewSpanConfig(t *testing.T) { // No uniqueness is guaranteed by the API. links: []Link{link1, link1, link2}, }, + nil, }, { + "WithNewRoot", []SpanStartOption{ WithNewRoot(), }, SpanConfig{ newRoot: true, }, + nil, }, { + "WithNewRoot multiple calls", []SpanStartOption{ // Multiple calls should not change NewRoot state. WithNewRoot(), @@ -121,16 +142,20 @@ func TestNewSpanConfig(t *testing.T) { SpanConfig{ newRoot: true, }, + nil, }, { + "WithSpanKind", []SpanStartOption{ WithSpanKind(SpanKindConsumer), }, SpanConfig{ spanKind: SpanKindConsumer, }, + nil, }, { + "WithSpanKind multiple calls", []SpanStartOption{ // Multiple calls overwrites with last-one-wins. WithSpanKind(SpanKindClient), @@ -139,27 +164,237 @@ func TestNewSpanConfig(t *testing.T) { SpanConfig{ spanKind: SpanKindConsumer, }, + nil, + }, + { + "WithProfileTask: ProfilingDefault", + []SpanStartOption{ + WithProfileTask(ProfilingDefault), + }, + SpanConfig{ + profileTask: ProfilingDefault, + }, + func(t *testing.T, cfg SpanConfig) { + assert.Equal(t, ProfilingDefault, cfg.ProfileTask()) + }, + }, + { + "WithProfileTask: ProfilingAuto", + []SpanStartOption{ + WithProfileTask(ProfilingAuto), + }, + SpanConfig{ + profileTask: ProfilingAuto, + }, + func(t *testing.T, cfg SpanConfig) { + assert.Equal(t, ProfilingAuto, cfg.ProfileTask()) + }, + }, + { + "WithProfileTask: ProfilingManual", + []SpanStartOption{ + WithProfileTask(ProfilingManual), + }, + SpanConfig{ + profileTask: ProfilingManual, + }, + func(t *testing.T, cfg SpanConfig) { + assert.Equal(t, ProfilingManual, cfg.ProfileTask()) + }, + }, + { + "WithProfileTask: ProfilingDisabled", + []SpanStartOption{ + WithProfileTask(ProfilingDisabled), + }, + SpanConfig{ + profileTask: ProfilingDisabled, + }, + func(t *testing.T, cfg SpanConfig) { + assert.Equal(t, ProfilingDisabled, cfg.ProfileTask()) + }, + }, + { + "ProfileTask", + []SpanStartOption{ + ProfileTask(), + }, + SpanConfig{ + profileTask: ProfilingManual, + profileRegion: ProfilingDisabled, + }, + func(t *testing.T, cfg SpanConfig) { + assert.Equal(t, ProfilingManual, cfg.ProfileTask()) + assert.Equal(t, ProfilingDisabled, cfg.ProfileRegion()) + }, + }, + { + "WithProfileTask multiple calls", + []SpanStartOption{ + // Multiple calls overwrites with last-one-wins. + WithProfileTask(ProfilingDisabled), + WithProfileTask(ProfilingManual), + }, + SpanConfig{ + profileTask: ProfilingManual, + }, + nil, + }, + { + "WithProfileRegion: ProfilingDefault", + []SpanStartOption{ + // Multiple calls overwrites with last-one-wins. + WithProfileTask(ProfilingDisabled), + WithProfileTask(ProfilingManual), + }, + SpanConfig{ + profileTask: ProfilingManual, + }, + nil, + }, + { + "WithProfileRegion: ProfilingAuto", + []SpanStartOption{ + WithProfileRegion(ProfilingAuto), + }, + SpanConfig{ + profileRegion: ProfilingAuto, + }, + func(t *testing.T, cfg SpanConfig) { + assert.Equal(t, ProfilingAuto, cfg.ProfileRegion()) + }, + }, + { + "WithProfileRegion: ProfilingManual", + []SpanStartOption{ + WithProfileRegion(ProfilingManual), + }, + SpanConfig{ + profileRegion: ProfilingManual, + }, + func(t *testing.T, cfg SpanConfig) { + assert.Equal(t, ProfilingManual, cfg.ProfileRegion()) + }, + }, + { + "WithProfileRegion: ProfilingDisabled", + []SpanStartOption{ + WithProfileRegion(ProfilingDisabled), + }, + SpanConfig{ + profileRegion: ProfilingDisabled, + }, + func(t *testing.T, cfg SpanConfig) { + assert.Equal(t, ProfilingDisabled, cfg.ProfileRegion()) + }, + }, + { + "ProfileRegion", + []SpanStartOption{ + ProfileRegion(), + }, + SpanConfig{ + profileRegion: ProfilingManual, + profileTask: ProfilingDisabled, + }, + func(t *testing.T, cfg SpanConfig) { + assert.Equal(t, ProfilingManual, cfg.ProfileRegion()) + assert.Equal(t, ProfilingDisabled, cfg.ProfileTask()) + }, + }, + { + "WithProfileRegion multiple calls", + []SpanStartOption{ + // Multiple calls overwrites with last-one-wins. + WithProfileRegion(ProfilingDisabled), + WithProfileRegion(ProfilingManual), + }, + SpanConfig{ + profileRegion: ProfilingManual, + }, + nil, + }, + { + "WithAsyncEnd: true", + []SpanStartOption{ + WithAsyncEnd(true), + }, + SpanConfig{ + asyncEnd: true, + }, + func(t *testing.T, cfg SpanConfig) { + assert.True(t, cfg.AsyncEnd()) + }, + }, + { + "WithAsyncEnd: false", + []SpanStartOption{ + WithAsyncEnd(false), + }, + SpanConfig{ + asyncEnd: false, + }, + func(t *testing.T, cfg SpanConfig) { + assert.False(t, cfg.AsyncEnd()) + }, + }, + { + "AsyncEnd", + []SpanStartOption{ + AsyncEnd(), + }, + SpanConfig{ + asyncEnd: true, + }, + func(t *testing.T, cfg SpanConfig) { + assert.True(t, cfg.AsyncEnd()) + }, + }, + { + "NoProfiling", + []SpanStartOption{ + NoProfiling(), + }, + SpanConfig{ + profileRegion: ProfilingDisabled, + profileTask: ProfilingDisabled, + }, + nil, }, { // Everything should work together. + "Everything together", []SpanStartOption{ WithAttributes(k1v1), WithTimestamp(timestamp0), WithLinks(link1, link2), WithNewRoot(), WithSpanKind(SpanKindConsumer), + WithProfileTask(ProfilingManual), + WithProfileRegion(ProfilingManual), + AsyncEnd(), }, SpanConfig{ - attributes: []attribute.KeyValue{k1v1}, - timestamp: timestamp0, - links: []Link{link1, link2}, - newRoot: true, - spanKind: SpanKindConsumer, + attributes: []attribute.KeyValue{k1v1}, + timestamp: timestamp0, + links: []Link{link1, link2}, + newRoot: true, + spanKind: SpanKindConsumer, + profileTask: ProfilingManual, + profileRegion: ProfilingManual, + asyncEnd: true, }, + nil, }, } for _, test := range tests { - assert.Equal(t, test.expected, NewSpanStartConfig(test.options...)) + t.Run(test.name, func(t *testing.T) { + config := NewSpanStartConfig(test.options...) + assert.Equal(t, test.expected, config) + if test.customAssertFunction != nil { + test.customAssertFunction(t, config) + } + }) } } @@ -224,11 +459,13 @@ func TestTracerConfig(t *testing.T) { WithInstrumentationVersion(v2), WithSchemaURL(schemaURL), WithInstrumentationAttributes(attrs.ToSlice()...), + WithProfilingMode(ProfilingAuto), ) assert.Equal(t, v2, c.InstrumentationVersion(), "instrumentation version") assert.Equal(t, schemaURL, c.SchemaURL(), "schema URL") assert.Equal(t, attrs, c.InstrumentationAttributes(), "instrumentation attributes") + assert.Equal(t, ProfilingAuto, c.ProfilingMode()) } func TestWithInstrumentationAttributesNotLazy(t *testing.T) { @@ -300,6 +537,30 @@ func BenchmarkNewTracerConfig(b *testing.B) { WithInstrumentationAttributeSet(attribute.NewSet(attribute.String("key", "value"))), }, }, + { + name: "with default profiling", + options: []TracerOption{ + WithProfilingMode(ProfilingDefault), + }, + }, + { + name: "with auto profiling", + options: []TracerOption{ + WithProfilingMode(ProfilingAuto), + }, + }, + { + name: "with manual profiling", + options: []TracerOption{ + WithProfilingMode(ProfilingManual), + }, + }, + { + name: "with no profiling", + options: []TracerOption{ + WithProfilingMode(ProfilingDisabled), + }, + }, } { b.Run(bb.name, func(b *testing.B) { b.ReportAllocs() @@ -364,6 +625,30 @@ func BenchmarkNewSpanStartConfig(b *testing.B) { WithSpanKind(SpanKindClient), }, }, + { + name: "profile task", + options: []SpanStartOption{ + ProfileTask(), + }, + }, + { + name: "profile region", + options: []SpanStartOption{ + ProfileRegion(), + }, + }, + { + name: "async end", + options: []SpanStartOption{ + AsyncEnd(), + }, + }, + { + name: "no profiling", + options: []SpanStartOption{ + NoProfiling(), + }, + }, } { b.Run(bb.name, func(b *testing.B) { b.ReportAllocs()