From 6aa74c984b64d936f06f715376cd0f62763ffd9d Mon Sep 17 00:00:00 2001 From: Praful Date: Thu, 7 Aug 2025 21:07:34 +0530 Subject: [PATCH 01/11] Add self-observability metrics to stdoutlog exporter Signed-off-by: Praful --- exporters/stdout/stdoutlog/config.go | 24 ++++++-- exporters/stdout/stdoutlog/doc.go | 19 +++++++ exporters/stdout/stdoutlog/exporter.go | 24 +++++++- exporters/stdout/stdoutlog/go.mod | 9 +++ .../stdout/stdoutlog/selfobservability.go | 55 +++++++++++++++++++ 5 files changed, 122 insertions(+), 9 deletions(-) create mode 100644 exporters/stdout/stdoutlog/selfobservability.go diff --git a/exporters/stdout/stdoutlog/config.go b/exporters/stdout/stdoutlog/config.go index 1b8f8bbb2ca..2c2d8834009 100644 --- a/exporters/stdout/stdoutlog/config.go +++ b/exporters/stdout/stdoutlog/config.go @@ -6,12 +6,14 @@ package stdoutlog // import "go.opentelemetry.io/otel/exporters/stdout/stdoutlog import ( "io" "os" + "strings" ) var ( - defaultWriter io.Writer = os.Stdout - defaultPrettyPrint = false - defaultTimestamps = true + defaultWriter io.Writer = os.Stdout + defaultPrettyPrint = false + defaultTimestamps = true + defaultSelfObservabilityEnabled = false ) // config contains options for the STDOUT exporter. @@ -26,18 +28,28 @@ type config struct { // Timestamps specifies if timestamps should be printed. Default is // true. Timestamps bool + + // Enable self-observability for the exporter. [Experimental] Default is + // false. + SelfObservability bool } // newConfig creates a validated Config configured with options. func newConfig(options []Option) config { cfg := config{ - Writer: defaultWriter, - PrettyPrint: defaultPrettyPrint, - Timestamps: defaultTimestamps, + Writer: defaultWriter, + PrettyPrint: defaultPrettyPrint, + Timestamps: defaultTimestamps, + SelfObservability: defaultSelfObservabilityEnabled, } for _, opt := range options { cfg = opt.apply(cfg) } + + if v := os.Getenv("OTEL_GO_X_SELF_OBSERVABILITY"); v != "" { + cfg.SelfObservability = strings.EqualFold(v, "true") + } + return cfg } diff --git a/exporters/stdout/stdoutlog/doc.go b/exporters/stdout/stdoutlog/doc.go index d400ab8c587..9707ba62417 100644 --- a/exporters/stdout/stdoutlog/doc.go +++ b/exporters/stdout/stdoutlog/doc.go @@ -9,4 +9,23 @@ // format for OpenTelemetry that is supported with any stability or // compatibility guarantees. If these are needed features, please use the OTLP // exporter instead. +// +// # Self-Observability +// +// The exporter provides a self-observability feature that allows you to monitor +// the exporter itself. To enable this feature, set the environment variable +// OTEL_GO_X_SELF_OBSERVABILITY to the case-insensitive string value of "true" +// or use the WithSelfObservability option when creating the exporter. +// +// When enabled, the exporter will create the following metrics using the global +// MeterProvider: +// +// - otel.sdk.exporter.log.inflight +// - otel.sdk.exporter.log.exported +// - otel.sdk.exporter.operation.duration +// +// Please see the [Semantic conventions for OpenTelemetry SDK metrics] documentation +// for more details on these metrics. +// +// [Semantic conventions for OpenTelemetry SDK metrics]: https://github.com/open-telemetry/semantic-conventions/blob/v1.36.0/docs/otel/sdk-metrics.md package stdoutlog // import "go.opentelemetry.io/otel/exporters/stdout/stdoutlog" diff --git a/exporters/stdout/stdoutlog/exporter.go b/exporters/stdout/stdoutlog/exporter.go index 3d48d67081e..dad6806b1f2 100644 --- a/exporters/stdout/stdoutlog/exporter.go +++ b/exporters/stdout/stdoutlog/exporter.go @@ -9,6 +9,7 @@ import ( "sync/atomic" "go.opentelemetry.io/otel/sdk/log" + "go.opentelemetry.io/otel/semconv/v1.36.0/otelconv" ) var _ log.Exporter = &Exporter{} @@ -16,21 +17,34 @@ var _ log.Exporter = &Exporter{} // Exporter writes JSON-encoded log records to an [io.Writer] ([os.Stdout] by default). // Exporter must be created with [New]. type Exporter struct { - encoder atomic.Pointer[json.Encoder] - timestamps bool + encoder atomic.Pointer[json.Encoder] + timestamps bool + selfObservability *selfObservability +} + +type selfObservability struct { + inflight otelconv.SDKExporterLogInflight + exported otelconv.SDKExporterLogExported + duration otelconv.SDKExporterOperationDuration } // New creates an [Exporter]. func New(options ...Option) (*Exporter, error) { cfg := newConfig(options) + var selfObs *selfObservability + if cfg.SelfObservability { + selfObs = newSelfObservability() + } + enc := json.NewEncoder(cfg.Writer) if cfg.PrettyPrint { enc.SetIndent("", "\t") } e := Exporter{ - timestamps: cfg.Timestamps, + timestamps: cfg.Timestamps, + selfObservability: selfObs, } e.encoder.Store(enc) @@ -39,10 +53,14 @@ func New(options ...Option) (*Exporter, error) { // Export exports log records to writer. func (e *Exporter) Export(ctx context.Context, records []log.Record) error { + if len(records) == 0 { + return nil + } enc := e.encoder.Load() if enc == nil { return nil } + e.initSelfObservability(ctx, &records) for _, record := range records { // Honor context cancellation. diff --git a/exporters/stdout/stdoutlog/go.mod b/exporters/stdout/stdoutlog/go.mod index 7b5850a7302..0b201db690e 100644 --- a/exporters/stdout/stdoutlog/go.mod +++ b/exporters/stdout/stdoutlog/go.mod @@ -6,6 +6,15 @@ go 1.24.0 retract v0.12.0 require ( + github.com/stretchr/testify v1.10.0 + go.opentelemetry.io/otel v1.37.0 + go.opentelemetry.io/otel/log v0.13.0 + go.opentelemetry.io/otel/metric v1.37.0 + go.opentelemetry.io/otel/sdk v1.37.0 + go.opentelemetry.io/otel/sdk/log v0.13.0 + go.opentelemetry.io/otel/sdk/log/logtest v0.13.0 + go.opentelemetry.io/otel/sdk/metric v1.37.0 + go.opentelemetry.io/otel/trace v1.37.0 github.com/stretchr/testify v1.11.1 go.opentelemetry.io/otel v1.38.0 go.opentelemetry.io/otel/log v0.14.0 diff --git a/exporters/stdout/stdoutlog/selfobservability.go b/exporters/stdout/stdoutlog/selfobservability.go new file mode 100644 index 00000000000..9d186a7ad0b --- /dev/null +++ b/exporters/stdout/stdoutlog/selfobservability.go @@ -0,0 +1,55 @@ +// Copyright The OpenTelemetry Authors +// SPDX-License-Identifier: Apache-2.0 + +package stdoutlog + +import ( + "context" + "time" + + "go.opentelemetry.io/otel" + "go.opentelemetry.io/otel/metric" + "go.opentelemetry.io/otel/sdk" + "go.opentelemetry.io/otel/sdk/log" + semconv "go.opentelemetry.io/otel/semconv/v1.36.0" + "go.opentelemetry.io/otel/semconv/v1.36.0/otelconv" +) + +func newSelfObservability() *selfObservability { + mp := otel.GetMeterProvider() + m := mp.Meter("go.opentelemetry.io/otel/exporters/stdout/stdoutlog", + metric.WithInstrumentationVersion(sdk.Version()), + metric.WithSchemaURL(semconv.SchemaURL)) + + so := selfObservability{} + + var err error + if so.inflight, err = otelconv.NewSDKExporterLogInflight(m); err != nil { + otel.Handle(err) + } + if so.exported, err = otelconv.NewSDKExporterLogExported(m); err != nil { + otel.Handle(err) + } + if so.duration, err = otelconv.NewSDKExporterOperationDuration(m); err != nil { + otel.Handle(err) + } + return &so +} + +func (e *Exporter) initSelfObservability(ctx context.Context, records *[]log.Record) { + if records == nil { + return + } else if e.selfObservability == nil { + return + } + + e.selfObservability.inflight.Add(ctx, int64(len(*records))) + + start := time.Now() + defer func() { + dur := float64(time.Since(start).Nanoseconds()) + e.selfObservability.duration.Record(ctx, dur) + }() + + e.selfObservability.exported.Add(ctx, int64(len(*records))) +} From 924260127e59e3af28ff92e161e4c05399635ba6 Mon Sep 17 00:00:00 2001 From: Praful Date: Thu, 7 Aug 2025 21:09:53 +0530 Subject: [PATCH 02/11] Add test case for self-observability metrics for stdoutlog exporter Signed-off-by: Praful --- .../stdoutlog/selfobservability_test.go | 209 ++++++++++++++++++ 1 file changed, 209 insertions(+) create mode 100644 exporters/stdout/stdoutlog/selfobservability_test.go diff --git a/exporters/stdout/stdoutlog/selfobservability_test.go b/exporters/stdout/stdoutlog/selfobservability_test.go new file mode 100644 index 00000000000..15056d96abd --- /dev/null +++ b/exporters/stdout/stdoutlog/selfobservability_test.go @@ -0,0 +1,209 @@ +// Copyright The OpenTelemetry Authors +// SPDX-License-Identifier: Apache-2.0 + +package stdoutlog + +import ( + "context" + "testing" + "time" + + "github.com/stretchr/testify/require" + + "go.opentelemetry.io/otel" + "go.opentelemetry.io/otel/log" + sdklog "go.opentelemetry.io/otel/sdk/log" + "go.opentelemetry.io/otel/sdk/log/logtest" + "go.opentelemetry.io/otel/sdk/metric" + "go.opentelemetry.io/otel/sdk/metric/metricdata" +) + +func TestSelfObservability(t *testing.T) { + testCases := []struct { + name string + test func(t *testing.T, scopeMetrics func() metricdata.ScopeMetrics) + }{ + { + name: "inflight", + test: func(t *testing.T, scopeMetrics func() metricdata.ScopeMetrics) { + exporter, err := New() + require.NoError(t, err) + + rf := logtest.RecordFactory{ + Timestamp: time.Now(), + Body: log.StringValue("test log 1"), + } + record1 := rf.NewRecord() + + rf.Body = log.StringValue("test log 2") + record2 := rf.NewRecord() + + records := []sdklog.Record{record1, record2} + + err = exporter.Export(context.Background(), records) + require.NoError(t, err) + + got := scopeMetrics() + require.NotEmpty(t, got.Metrics) + + var inflightMetric metricdata.Metrics + for _, m := range got.Metrics { + if m.Name == "otel.sdk.exporter.log.inflight" { + inflightMetric = m + break + } + } + require.NotEmpty(t, inflightMetric.Name, "inflight metric should be present") + + sum, ok := inflightMetric.Data.(metricdata.Sum[int64]) + require.True(t, ok, "inflight metric should be a sum") + require.Len(t, sum.DataPoints, 1, "should have one data point") + require.Equal(t, int64(2), sum.DataPoints[0].Value, "should record 2 inflight records") + }, + }, + { + name: "exported", + test: func(t *testing.T, scopeMetrics func() metricdata.ScopeMetrics) { + exporter, err := New() + require.NoError(t, err) + + rf := logtest.RecordFactory{ + Timestamp: time.Now(), + Body: log.StringValue("test log 1"), + } + record1 := rf.NewRecord() + + rf.Body = log.StringValue("test log 2") + record2 := rf.NewRecord() + + rf.Body = log.StringValue("test log 3") + record3 := rf.NewRecord() + + records := []sdklog.Record{record1, record2, record3} + + err = exporter.Export(context.Background(), records) + require.NoError(t, err) + + got := scopeMetrics() + require.NotEmpty(t, got.Metrics) + + var exportedMetric metricdata.Metrics + for _, m := range got.Metrics { + if m.Name == "otel.sdk.exporter.log.exported" { + exportedMetric = m + break + } + } + require.NotEmpty(t, exportedMetric.Name, "exported metric should be present") + + sum, ok := exportedMetric.Data.(metricdata.Sum[int64]) + require.True(t, ok, "exported metric should be a sum") + require.Len(t, sum.DataPoints, 1, "should have one data point") + require.Equal(t, int64(3), sum.DataPoints[0].Value, "should record 3 exported records") + }, + }, + { + name: "duration", + test: func(t *testing.T, scopeMetrics func() metricdata.ScopeMetrics) { + exporter, err := New() + require.NoError(t, err) + + rf := logtest.RecordFactory{ + Timestamp: time.Now(), + Body: log.StringValue("test log"), + } + record := rf.NewRecord() + records := []sdklog.Record{record} + + err = exporter.Export(context.Background(), records) + require.NoError(t, err) + + got := scopeMetrics() + require.NotEmpty(t, got.Metrics) + + var durationMetric metricdata.Metrics + for _, m := range got.Metrics { + if m.Name == "otel.sdk.exporter.operation.duration" { + durationMetric = m + break + } + } + require.NotEmpty(t, durationMetric.Name, "duration metric should be present") + + histogram, ok := durationMetric.Data.(metricdata.Histogram[float64]) + require.True(t, ok, "duration metric should be a histogram") + require.Len(t, histogram.DataPoints, 1, "should have one data point") + require.Greater(t, histogram.DataPoints[0].Sum, 0.0, "duration should be greater than 0") + }, + }, + { + name: "multiple_exports", + test: func(t *testing.T, scopeMetrics func() metricdata.ScopeMetrics) { + exporter, err := New() + require.NoError(t, err) + + for i := 0; i < 3; i++ { + rf := logtest.RecordFactory{ + Timestamp: time.Now(), + Body: log.StringValue("test log"), + } + record := rf.NewRecord() + records := []sdklog.Record{record} + err = exporter.Export(context.Background(), records) + require.NoError(t, err) + } + + got := scopeMetrics() + require.NotEmpty(t, got.Metrics) + + var exportedMetric metricdata.Metrics + for _, m := range got.Metrics { + if m.Name == "otel.sdk.exporter.log.exported" { + exportedMetric = m + break + } + } + require.NotEmpty(t, exportedMetric.Name, "exported metric should be present") + + sum, ok := exportedMetric.Data.(metricdata.Sum[int64]) + require.True(t, ok, "exported metric should be a sum") + require.Len(t, sum.DataPoints, 1, "should have one data point") + require.Equal(t, int64(3), sum.DataPoints[0].Value, "should record 3 total exported records") + }, + }, + { + name: "empty_records", + test: func(t *testing.T, scopeMetrics func() metricdata.ScopeMetrics) { + exporter, err := New() + require.NoError(t, err) + + err = exporter.Export(context.Background(), []sdklog.Record{}) + require.NoError(t, err) + + got := scopeMetrics() + require.Empty(t, got.Metrics, "no metrics should be recorded for empty records") + }, + }, + } + for _, tc := range testCases { + t.Run(tc.name, func(t *testing.T) { + t.Setenv("OTEL_GO_X_SELF_OBSERVABILITY", "true") + prev := otel.GetMeterProvider() + defer otel.SetMeterProvider(prev) + r := metric.NewManualReader() + mp := metric.NewMeterProvider(metric.WithReader(r)) + otel.SetMeterProvider(mp) + + scopeMetrics := func() metricdata.ScopeMetrics { + var got metricdata.ResourceMetrics + err := r.Collect(context.Background(), &got) + require.NoError(t, err) + if len(got.ScopeMetrics) == 0 { + return metricdata.ScopeMetrics{} + } + return got.ScopeMetrics[0] + } + tc.test(t, scopeMetrics) + }) + } +} From 36b381a7e77167fd9149e89a941284088087074b Mon Sep 17 00:00:00 2001 From: Praful Date: Thu, 7 Aug 2025 21:15:58 +0530 Subject: [PATCH 03/11] simplify nil checks Signed-off-by: Praful --- exporters/stdout/stdoutlog/selfobservability.go | 6 ++---- 1 file changed, 2 insertions(+), 4 deletions(-) diff --git a/exporters/stdout/stdoutlog/selfobservability.go b/exporters/stdout/stdoutlog/selfobservability.go index 9d186a7ad0b..b143b18a000 100644 --- a/exporters/stdout/stdoutlog/selfobservability.go +++ b/exporters/stdout/stdoutlog/selfobservability.go @@ -1,7 +1,7 @@ // Copyright The OpenTelemetry Authors // SPDX-License-Identifier: Apache-2.0 -package stdoutlog +package stdoutlog // import "go.opentelemetry.io/otel/exporters/stdout/stdoutlog" import ( "context" @@ -37,9 +37,7 @@ func newSelfObservability() *selfObservability { } func (e *Exporter) initSelfObservability(ctx context.Context, records *[]log.Record) { - if records == nil { - return - } else if e.selfObservability == nil { + if records == nil || e.selfObservability == nil { return } From f8b91e15e7f0eebd4a5d9d278fac28c365782256 Mon Sep 17 00:00:00 2001 From: Praful Date: Mon, 11 Aug 2025 18:30:45 +0530 Subject: [PATCH 04/11] follow existing convention for including experimental features in an internal/x Signed-off-by: Praful --- exporters/stdout/stdoutlog/config.go | 26 ++++--- exporters/stdout/stdoutlog/exporter.go | 30 ++++++-- .../stdout/stdoutlog/internal/x/README.md | 22 ++++++ exporters/stdout/stdoutlog/internal/x/x.go | 62 +++++++++++++++++ .../stdout/stdoutlog/internal/x/x_test.go | 68 +++++++++++++++++++ .../stdout/stdoutlog/selfobservability.go | 21 +----- 6 files changed, 191 insertions(+), 38 deletions(-) create mode 100644 exporters/stdout/stdoutlog/internal/x/README.md create mode 100644 exporters/stdout/stdoutlog/internal/x/x.go create mode 100644 exporters/stdout/stdoutlog/internal/x/x_test.go diff --git a/exporters/stdout/stdoutlog/config.go b/exporters/stdout/stdoutlog/config.go index 2c2d8834009..ddc2998f5de 100644 --- a/exporters/stdout/stdoutlog/config.go +++ b/exporters/stdout/stdoutlog/config.go @@ -6,14 +6,14 @@ package stdoutlog // import "go.opentelemetry.io/otel/exporters/stdout/stdoutlog import ( "io" "os" - "strings" + + "go.opentelemetry.io/otel/exporters/stdout/stdoutlog/internal/x" ) var ( - defaultWriter io.Writer = os.Stdout - defaultPrettyPrint = false - defaultTimestamps = true - defaultSelfObservabilityEnabled = false + defaultWriter io.Writer = os.Stdout + defaultPrettyPrint = false + defaultTimestamps = true ) // config contains options for the STDOUT exporter. @@ -29,26 +29,24 @@ type config struct { // true. Timestamps bool - // Enable self-observability for the exporter. [Experimental] Default is - // false. + // SelfObservability enables exporter self-observability metrics. + // This is an experimental feature controlled by OTEL_GO_X_SELF_OBSERVABILITY. + // Default is false. SelfObservability bool } // newConfig creates a validated Config configured with options. func newConfig(options []Option) config { cfg := config{ - Writer: defaultWriter, - PrettyPrint: defaultPrettyPrint, - Timestamps: defaultTimestamps, - SelfObservability: defaultSelfObservabilityEnabled, + Writer: defaultWriter, + PrettyPrint: defaultPrettyPrint, + Timestamps: defaultTimestamps, } for _, opt := range options { cfg = opt.apply(cfg) } - if v := os.Getenv("OTEL_GO_X_SELF_OBSERVABILITY"); v != "" { - cfg.SelfObservability = strings.EqualFold(v, "true") - } + cfg.SelfObservability = x.SelfObservability.Enabled() return cfg } diff --git a/exporters/stdout/stdoutlog/exporter.go b/exporters/stdout/stdoutlog/exporter.go index dad6806b1f2..2f4ab5f8c8f 100644 --- a/exporters/stdout/stdoutlog/exporter.go +++ b/exporters/stdout/stdoutlog/exporter.go @@ -7,6 +7,7 @@ import ( "context" "encoding/json" "sync/atomic" + "time" "go.opentelemetry.io/otel/sdk/log" "go.opentelemetry.io/otel/semconv/v1.36.0/otelconv" @@ -53,14 +54,11 @@ func New(options ...Option) (*Exporter, error) { // Export exports log records to writer. func (e *Exporter) Export(ctx context.Context, records []log.Record) error { - if len(records) == 0 { - return nil - } enc := e.encoder.Load() if enc == nil { return nil } - e.initSelfObservability(ctx, &records) + e.recordSelfObservabilityMetrics(ctx, &records) for _, record := range records { // Honor context cancellation. @@ -88,3 +86,27 @@ func (e *Exporter) Shutdown(context.Context) error { func (*Exporter) ForceFlush(context.Context) error { return nil } + +// recordSelfObservabilityMetrics records self-observability metrics if the feature is enabled. +func (e *Exporter) recordSelfObservabilityMetrics(ctx context.Context, records *[]log.Record) { + start := time.Now() + if e.selfObservability == nil { + return + } + + if len(*records) == 0 { + return + } + + componentName := e.selfObservability.inflight.AttrComponentName("stdoutlog/0") + componentType := e.selfObservability.inflight.AttrComponentType("go.opentelemetry.io/otel/exporters/stdout/stdoutlog.Exporter") + + e.selfObservability.inflight.Add(ctx, int64(len(*records)), componentName, componentType) + + defer func() { + dur := float64(time.Since(start).Seconds()) + e.selfObservability.duration.Record(ctx, dur, componentName, componentType) + }() + + e.selfObservability.exported.Add(ctx, int64(len(*records)), componentName, componentType) +} diff --git a/exporters/stdout/stdoutlog/internal/x/README.md b/exporters/stdout/stdoutlog/internal/x/README.md new file mode 100644 index 00000000000..280fe0d54d7 --- /dev/null +++ b/exporters/stdout/stdoutlog/internal/x/README.md @@ -0,0 +1,22 @@ +# Experimental Features + +This package documents the experimental features available for [go.opentelemetry.io/otel/exporters/stdout/stdoutlog]. + +## Self-Observability + +The self-observability feature allows the stdout log exporter to emit metrics. When enabled, the exporter will record metrics for: + +- Number of log records currently being processed (inflight) +- Total number of log records exported +- Duration of export operations + +To enable this feature, set the `OTEL_GO_X_SELF_OBSERVABILITY` environment variable to `true`. + +## Compatibility and Stability + +Experimental features do not fall within the scope of the OpenTelemetry Go versioning and stability [policy](../../../../../VERSIONING.md). +These features may be removed or modified in successive version releases, including patch versions. + +When an experimental feature is promoted to a stable feature, a migration path will be included in the changelog entry of the release. +There is no guarantee that any environment variable feature flags that enabled the experimental feature will be supported by the stable version. +If they are supported, they may be accompanied with a deprecation notice stating a timeline for the removal of that support. \ No newline at end of file diff --git a/exporters/stdout/stdoutlog/internal/x/x.go b/exporters/stdout/stdoutlog/internal/x/x.go new file mode 100644 index 00000000000..38c95d537ef --- /dev/null +++ b/exporters/stdout/stdoutlog/internal/x/x.go @@ -0,0 +1,62 @@ +// Copyright The OpenTelemetry Authors +// SPDX-License-Identifier: Apache-2.0 + +// Package x documents experimental features for [go.opentelemetry.io/otel/exporters/stdout/stdoutlog]. +package x // import "go.opentelemetry.io/otel/exporters/stdout/stdoutlog/internal/x" + +import ( + "os" + "strings" +) + +// SelfObservability is an experimental feature flag that determines if stdout +// log exporter self-observability metrics are enabled. +// +// To enable this feature set the OTEL_GO_X_SELF_OBSERVABILITY environment variable +// to the case-insensitive string value. +var SelfObservability = newFeature("SELF_OBSERVABILITY", func(v string) (string, bool) { + if strings.EqualFold(v, "true") { + return v, true + } + return "", false +}) + +// Feature is an experimental feature control flag. It provides a uniform way +// to interact with these feature flags and parse their values. +type Feature[T any] struct { + key string + parse func(v string) (T, bool) +} + +func newFeature[T any](suffix string, parse func(string) (T, bool)) Feature[T] { + const envKeyRoot = "OTEL_GO_X_" + return Feature[T]{ + key: envKeyRoot + suffix, + parse: parse, + } +} + +// Key returns the environment variable key that needs to be set to enable the +// feature. +func (f Feature[T]) Key() string { return f.key } + +// Lookup returns the user configured value for the feature and true if the +// user has enabled the feature. Otherwise, if the feature is not enabled, a +// zero-value and false are returned. +func (f Feature[T]) Lookup() (v T, ok bool) { + // https://github.com/open-telemetry/opentelemetry-specification/blob/62effed618589a0bec416a87e559c0a9d96289bb/specification/configuration/sdk-environment-variables.md#parsing-empty-value + // + // > The SDK MUST interpret an empty value of an environment variable the + // > same way as when the variable is unset. + vRaw := os.Getenv(f.key) + if vRaw == "" { + return v, ok + } + return f.parse(vRaw) +} + +// Enabled reports whether the feature is enabled. +func (f Feature[T]) Enabled() bool { + _, ok := f.Lookup() + return ok +} diff --git a/exporters/stdout/stdoutlog/internal/x/x_test.go b/exporters/stdout/stdoutlog/internal/x/x_test.go new file mode 100644 index 00000000000..d6dae980ff9 --- /dev/null +++ b/exporters/stdout/stdoutlog/internal/x/x_test.go @@ -0,0 +1,68 @@ +// Copyright The OpenTelemetry Authors +// SPDX-License-Identifier: Apache-2.0 + +package x + +import ( + "testing" + + "github.com/stretchr/testify/assert" +) + +func TestSelfObservabilityFeature(t *testing.T) { + testCases := []struct { + name string + envValue string + enabled bool + }{ + { + name: "enabled_lowercase", + envValue: "true", + enabled: true, + }, + { + name: "enabled_uppercase", + envValue: "TRUE", + enabled: true, + }, + { + name: "enabled_mixed_case", + envValue: "True", + enabled: true, + }, + { + name: "disabled_false", + envValue: "false", + enabled: false, + }, + { + name: "disabled_invalid", + envValue: "invalid", + enabled: false, + }, + { + name: "disabled_empty", + envValue: "", + enabled: false, + }, + } + + for _, tc := range testCases { + t.Run(tc.name, func(t *testing.T) { + if tc.envValue != "" { + t.Setenv(SelfObservability.Key(), tc.envValue) + } + + assert.Equal(t, tc.enabled, SelfObservability.Enabled()) + + value, ok := SelfObservability.Lookup() + if tc.enabled { + assert.True(t, ok) + assert.Equal(t, tc.envValue, value) + } else { + assert.False(t, ok) + assert.Empty(t, value) + } + }) + } +} diff --git a/exporters/stdout/stdoutlog/selfobservability.go b/exporters/stdout/stdoutlog/selfobservability.go index b143b18a000..26cc5f7d22b 100644 --- a/exporters/stdout/stdoutlog/selfobservability.go +++ b/exporters/stdout/stdoutlog/selfobservability.go @@ -4,17 +4,14 @@ package stdoutlog // import "go.opentelemetry.io/otel/exporters/stdout/stdoutlog" import ( - "context" - "time" - "go.opentelemetry.io/otel" "go.opentelemetry.io/otel/metric" "go.opentelemetry.io/otel/sdk" - "go.opentelemetry.io/otel/sdk/log" semconv "go.opentelemetry.io/otel/semconv/v1.36.0" "go.opentelemetry.io/otel/semconv/v1.36.0/otelconv" ) +// newSelfObservability creates a new selfObservability instance with the required metrics. func newSelfObservability() *selfObservability { mp := otel.GetMeterProvider() m := mp.Meter("go.opentelemetry.io/otel/exporters/stdout/stdoutlog", @@ -35,19 +32,3 @@ func newSelfObservability() *selfObservability { } return &so } - -func (e *Exporter) initSelfObservability(ctx context.Context, records *[]log.Record) { - if records == nil || e.selfObservability == nil { - return - } - - e.selfObservability.inflight.Add(ctx, int64(len(*records))) - - start := time.Now() - defer func() { - dur := float64(time.Since(start).Nanoseconds()) - e.selfObservability.duration.Record(ctx, dur) - }() - - e.selfObservability.exported.Add(ctx, int64(len(*records))) -} From a35fc1bd87dff75a0433535456e9d97dc839b1d6 Mon Sep 17 00:00:00 2001 From: Praful Date: Mon, 11 Aug 2025 18:31:08 +0530 Subject: [PATCH 05/11] updates testcase and using AssertEqual Signed-off-by: Praful follow existing convention for including experimental features in an internal/x Signed-off-by: Praful --- exporters/stdout/stdoutlog/exporter.go | 125 +++++-- exporters/stdout/stdoutlog/exporter_test.go | 333 ++++++++++++++++++ .../stdout/stdoutlog/selfobservability.go | 34 -- .../stdoutlog/selfobservability_test.go | 209 ----------- 4 files changed, 424 insertions(+), 277 deletions(-) delete mode 100644 exporters/stdout/stdoutlog/selfobservability.go delete mode 100644 exporters/stdout/stdoutlog/selfobservability_test.go diff --git a/exporters/stdout/stdoutlog/exporter.go b/exporters/stdout/stdoutlog/exporter.go index 2f4ab5f8c8f..4866f9b0502 100644 --- a/exporters/stdout/stdoutlog/exporter.go +++ b/exporters/stdout/stdoutlog/exporter.go @@ -6,13 +6,23 @@ package stdoutlog // import "go.opentelemetry.io/otel/exporters/stdout/stdoutlog import ( "context" "encoding/json" + "fmt" "sync/atomic" "time" + "go.opentelemetry.io/otel" + "go.opentelemetry.io/otel/attribute" + "go.opentelemetry.io/otel/exporters/stdout/stdoutlog/internal/x" + "go.opentelemetry.io/otel/metric" + "go.opentelemetry.io/otel/sdk" "go.opentelemetry.io/otel/sdk/log" + semconv "go.opentelemetry.io/otel/semconv/v1.36.0" "go.opentelemetry.io/otel/semconv/v1.36.0/otelconv" ) +// otelComponentType is a name identifying the type of the OpenTelemetry component. +const otelComponentType = "stdout_log_exporter" + var _ log.Exporter = &Exporter{} // Exporter writes JSON-encoded log records to an [io.Writer] ([os.Stdout] by default). @@ -24,51 +34,114 @@ type Exporter struct { } type selfObservability struct { - inflight otelconv.SDKExporterLogInflight - exported otelconv.SDKExporterLogExported - duration otelconv.SDKExporterOperationDuration + enabled bool + attrs []attribute.KeyValue + inflightMetric otelconv.SDKExporterLogInflight + exportedMetric otelconv.SDKExporterLogExported + operationDurationMetric otelconv.SDKExporterOperationDuration } // New creates an [Exporter]. func New(options ...Option) (*Exporter, error) { cfg := newConfig(options) - var selfObs *selfObservability - if cfg.SelfObservability { - selfObs = newSelfObservability() - } - enc := json.NewEncoder(cfg.Writer) if cfg.PrettyPrint { enc.SetIndent("", "\t") } e := Exporter{ - timestamps: cfg.Timestamps, - selfObservability: selfObs, + timestamps: cfg.Timestamps, } e.encoder.Store(enc) + e.initSelfObservability() return &e, nil } +// initSelfObservability initializes self-observability for the exporter if enabled. +func (e *Exporter) initSelfObservability() { + if !x.SelfObservability.Enabled() { + return + } + + e.selfObservability = &selfObservability{ + enabled: true, + attrs: []attribute.KeyValue{ + semconv.OTelComponentName(fmt.Sprintf("%s/%d", otelComponentType, nextExporterID())), + semconv.OTelComponentTypeKey.String(otelComponentType), + }, + } + + mp := otel.GetMeterProvider() + m := mp.Meter("go.opentelemetry.io/otel/exporters/stdout/stdoutlog", + metric.WithInstrumentationVersion(sdk.Version()), + metric.WithSchemaURL(semconv.SchemaURL), + ) + + var err error + if e.selfObservability.inflightMetric, err = otelconv.NewSDKExporterLogInflight(m); err != nil { + otel.Handle(err) + } + if e.selfObservability.exportedMetric, err = otelconv.NewSDKExporterLogExported(m); err != nil { + otel.Handle(err) + } + if e.selfObservability.operationDurationMetric, err = otelconv.NewSDKExporterOperationDuration(m); err != nil { + otel.Handle(err) + } +} + // Export exports log records to writer. func (e *Exporter) Export(ctx context.Context, records []log.Record) error { enc := e.encoder.Load() if enc == nil { return nil } - e.recordSelfObservabilityMetrics(ctx, &records) + var err error + if e.selfObservability != nil && e.selfObservability.enabled { + err = e.exportWithSelfObservability(ctx, records) + } else { + err = e.exportWithoutSelfObservability(ctx, records) + } + return err +} + +// exportWithSelfObservability exports logs with self-observability metrics. +func (e *Exporter) exportWithSelfObservability(ctx context.Context, records []log.Record) error { + if len(records) == 0 { + return nil + } + + count := int64(len(records)) + start := time.Now() + + e.selfObservability.inflightMetric.Add(context.Background(), count, e.selfObservability.attrs...) + defer func() { + addAttrs := make([]attribute.KeyValue, len(e.selfObservability.attrs), len(e.selfObservability.attrs)+1) + copy(addAttrs, e.selfObservability.attrs) + if err := ctx.Err(); err != nil { + addAttrs = append(addAttrs, semconv.ErrorType(err)) + } + + e.selfObservability.inflightMetric.Add(context.Background(), -count, e.selfObservability.attrs...) + e.selfObservability.exportedMetric.Add(context.Background(), count, addAttrs...) + e.selfObservability.operationDurationMetric.Record(context.Background(), time.Since(start).Seconds(), addAttrs...) + }() + + return e.exportWithoutSelfObservability(ctx, records) +} + +// exportWithoutSelfObservability exports logs without self-observability metrics. +func (e *Exporter) exportWithoutSelfObservability(ctx context.Context, records []log.Record) error { for _, record := range records { // Honor context cancellation. if err := ctx.Err(); err != nil { return err } - // Encode record, one by one. recordJSON := e.newRecordJSON(record) - if err := enc.Encode(recordJSON); err != nil { + if err := e.encoder.Load().Encode(recordJSON); err != nil { return err } } @@ -87,26 +160,10 @@ func (*Exporter) ForceFlush(context.Context) error { return nil } -// recordSelfObservabilityMetrics records self-observability metrics if the feature is enabled. -func (e *Exporter) recordSelfObservabilityMetrics(ctx context.Context, records *[]log.Record) { - start := time.Now() - if e.selfObservability == nil { - return - } - - if len(*records) == 0 { - return - } - - componentName := e.selfObservability.inflight.AttrComponentName("stdoutlog/0") - componentType := e.selfObservability.inflight.AttrComponentType("go.opentelemetry.io/otel/exporters/stdout/stdoutlog.Exporter") - - e.selfObservability.inflight.Add(ctx, int64(len(*records)), componentName, componentType) - - defer func() { - dur := float64(time.Since(start).Seconds()) - e.selfObservability.duration.Record(ctx, dur, componentName, componentType) - }() +var exporterIDCounter atomic.Int64 - e.selfObservability.exported.Add(ctx, int64(len(*records)), componentName, componentType) +// nextExporterID returns a new unique ID for an exporter. +// the starting value is 0, and it increments by 1 for each call. +func nextExporterID() int64 { + return exporterIDCounter.Add(1) - 1 } diff --git a/exporters/stdout/stdoutlog/exporter_test.go b/exporters/stdout/stdoutlog/exporter_test.go index 8772e9790bf..292e06da23f 100644 --- a/exporters/stdout/stdoutlog/exporter_test.go +++ b/exporters/stdout/stdoutlog/exporter_test.go @@ -14,12 +14,17 @@ import ( "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" + "go.opentelemetry.io/otel" "go.opentelemetry.io/otel/attribute" "go.opentelemetry.io/otel/log" "go.opentelemetry.io/otel/sdk/instrumentation" sdklog "go.opentelemetry.io/otel/sdk/log" "go.opentelemetry.io/otel/sdk/log/logtest" + "go.opentelemetry.io/otel/sdk/metric" + "go.opentelemetry.io/otel/sdk/metric/metricdata" + "go.opentelemetry.io/otel/sdk/metric/metricdata/metricdatatest" "go.opentelemetry.io/otel/sdk/resource" + "go.opentelemetry.io/otel/semconv/v1.36.0/otelconv" "go.opentelemetry.io/otel/trace" ) @@ -456,3 +461,331 @@ func TestValueMarshalJSON(t *testing.T) { }) } } + +func TestSelfObservability(t *testing.T) { + testCases := []struct { + name string + enable bool + test func(t *testing.T, scopeMetrics func() metricdata.ScopeMetrics) + }{ + { + name: "inflight", + enable: true, + test: func(t *testing.T, scopeMetrics func() metricdata.ScopeMetrics) { + exporter, err := New() + require.NoError(t, err) + + rf := logtest.RecordFactory{ + Timestamp: time.Now(), + Body: log.StringValue("test log 1"), + } + record1 := rf.NewRecord() + + rf.Body = log.StringValue("test log 2") + record2 := rf.NewRecord() + + records := []sdklog.Record{record1, record2} + + err = exporter.Export(context.Background(), records) + require.NoError(t, err) + + got := scopeMetrics() + assert.NotEmpty(t, got.Metrics) + + assert.Equal(t, "go.opentelemetry.io/otel/exporters/stdout/stdoutlog", got.Scope.Name) + assert.NotEmpty(t, got.Scope.Version) + + var inflightMetric metricdata.Metrics + inflightInstrument := otelconv.SDKExporterLogInflight{} + for _, m := range got.Metrics { + if m.Name == inflightInstrument.Name() { + inflightMetric = m + break + } + } + require.NotEmpty(t, inflightMetric, "inflight metric not found") + + expected := metricdata.Metrics{ + Name: inflightInstrument.Name(), + Description: inflightInstrument.Description(), + Unit: inflightInstrument.Unit(), + Data: metricdata.Sum[int64]{ + Temporality: metricdata.CumulativeTemporality, + IsMonotonic: false, + DataPoints: []metricdata.DataPoint[int64]{ + { + Value: 0, + Attributes: attribute.NewSet( + attribute.KeyValue{Key: "otel.component.name", Value: attribute.StringValue("stdout_log_exporter/0")}, + attribute.KeyValue{Key: "otel.component.type", Value: attribute.StringValue("stdout_log_exporter")}, + ), + }, + }, + }, + } + + metricdatatest.AssertEqual(t, expected, inflightMetric, metricdatatest.IgnoreTimestamp(), metricdatatest.IgnoreValue()) + }, + }, + { + name: "exported", + enable: true, + test: func(t *testing.T, scopeMetrics func() metricdata.ScopeMetrics) { + exporter, err := New() + require.NoError(t, err) + + rf := logtest.RecordFactory{ + Timestamp: time.Now(), + Body: log.StringValue("test log 1"), + } + record1 := rf.NewRecord() + + rf.Body = log.StringValue("test log 2") + record2 := rf.NewRecord() + + rf.Body = log.StringValue("test log 3") + record3 := rf.NewRecord() + + records := []sdklog.Record{record1, record2, record3} + + err = exporter.Export(context.Background(), records) + require.NoError(t, err) + + got := scopeMetrics() + assert.NotEmpty(t, got.Metrics) + + assert.Equal(t, "go.opentelemetry.io/otel/exporters/stdout/stdoutlog", got.Scope.Name) + assert.NotEmpty(t, got.Scope.Version) + + var exportedMetric metricdata.Metrics + exportedInstrument := otelconv.SDKExporterLogExported{} + for _, m := range got.Metrics { + if m.Name == exportedInstrument.Name() { + exportedMetric = m + break + } + } + require.NotEmpty(t, exportedMetric, "exported metric not found") + + expected := metricdata.Metrics{ + Name: exportedInstrument.Name(), + Description: exportedInstrument.Description(), + Unit: exportedInstrument.Unit(), + Data: metricdata.Sum[int64]{ + Temporality: metricdata.CumulativeTemporality, + IsMonotonic: true, + DataPoints: []metricdata.DataPoint[int64]{ + { + Value: 3, + Attributes: attribute.NewSet( + attribute.KeyValue{Key: "otel.component.name", Value: attribute.StringValue("stdout_log_exporter/1")}, + attribute.KeyValue{Key: "otel.component.type", Value: attribute.StringValue("stdout_log_exporter")}, + ), + }, + }, + }, + } + + metricdatatest.AssertEqual(t, expected, exportedMetric, metricdatatest.IgnoreTimestamp(), metricdatatest.IgnoreValue()) + }, + }, + { + name: "duration", + enable: true, + test: func(t *testing.T, scopeMetrics func() metricdata.ScopeMetrics) { + exporter, err := New() + require.NoError(t, err) + + rf := logtest.RecordFactory{ + Timestamp: time.Now(), + Body: log.StringValue("test log"), + } + record := rf.NewRecord() + records := []sdklog.Record{record} + + err = exporter.Export(context.Background(), records) + require.NoError(t, err) + + got := scopeMetrics() + assert.NotEmpty(t, got.Metrics) + + assert.Equal(t, "go.opentelemetry.io/otel/exporters/stdout/stdoutlog", got.Scope.Name) + assert.NotEmpty(t, got.Scope.Version) + + var durationMetric metricdata.Metrics + durationInstrument := otelconv.SDKExporterOperationDuration{} + for _, m := range got.Metrics { + if m.Name == durationInstrument.Name() { + durationMetric = m + break + } + } + require.NotEmpty(t, durationMetric, "duration metric not found") + + expected := metricdata.Metrics{ + Name: durationInstrument.Name(), + Description: durationInstrument.Description(), + Unit: durationInstrument.Unit(), + Data: metricdata.Histogram[float64]{ + Temporality: metricdata.CumulativeTemporality, + DataPoints: []metricdata.HistogramDataPoint[float64]{ + { + Count: 1, + Attributes: attribute.NewSet( + attribute.KeyValue{Key: "otel.component.name", Value: attribute.StringValue("stdout_log_exporter/2")}, + attribute.KeyValue{Key: "otel.component.type", Value: attribute.StringValue("stdout_log_exporter")}, + ), + }, + }, + }, + } + + metricdatatest.AssertEqual(t, expected, durationMetric, metricdatatest.IgnoreTimestamp(), metricdatatest.IgnoreValue()) + }, + }, + { + name: "multiple_exports", + enable: true, + test: func(t *testing.T, scopeMetrics func() metricdata.ScopeMetrics) { + exporter, err := New() + require.NoError(t, err) + + for i := 0; i < 3; i++ { + rf := logtest.RecordFactory{ + Timestamp: time.Now(), + Body: log.StringValue("test log"), + } + record := rf.NewRecord() + records := []sdklog.Record{record} + err = exporter.Export(context.Background(), records) + require.NoError(t, err) + } + + got := scopeMetrics() + assert.NotEmpty(t, got.Metrics) + + assert.Equal(t, "go.opentelemetry.io/otel/exporters/stdout/stdoutlog", got.Scope.Name) + assert.NotEmpty(t, got.Scope.Version) + + expectedAttrs := attribute.NewSet( + attribute.KeyValue{Key: "otel.component.name", Value: attribute.StringValue("stdout_log_exporter/3")}, + attribute.KeyValue{Key: "otel.component.type", Value: attribute.StringValue("stdout_log_exporter")}, + ) + + expected := metricdata.ScopeMetrics{ + Scope: got.Scope, + Metrics: []metricdata.Metrics{ + { + Name: otelconv.SDKExporterLogInflight{}.Name(), + Description: otelconv.SDKExporterLogInflight{}.Description(), + Unit: otelconv.SDKExporterLogInflight{}.Unit(), + Data: metricdata.Sum[int64]{ + Temporality: metricdata.CumulativeTemporality, + IsMonotonic: false, + DataPoints: []metricdata.DataPoint[int64]{ + { + Value: 0, + Attributes: expectedAttrs, + }, + }, + }, + }, + { + Name: otelconv.SDKExporterLogExported{}.Name(), + Description: otelconv.SDKExporterLogExported{}.Description(), + Unit: otelconv.SDKExporterLogExported{}.Unit(), + Data: metricdata.Sum[int64]{ + Temporality: metricdata.CumulativeTemporality, + IsMonotonic: true, + DataPoints: []metricdata.DataPoint[int64]{ + { + Value: 3, + Attributes: expectedAttrs, + }, + }, + }, + }, + { + Name: otelconv.SDKExporterOperationDuration{}.Name(), + Description: otelconv.SDKExporterOperationDuration{}.Description(), + Unit: otelconv.SDKExporterOperationDuration{}.Unit(), + Data: metricdata.Histogram[float64]{ + Temporality: metricdata.CumulativeTemporality, + DataPoints: []metricdata.HistogramDataPoint[float64]{ + { + Count: 3, + Attributes: expectedAttrs, + }, + }, + }, + }, + }, + } + + metricdatatest.AssertEqual(t, expected, got, metricdatatest.IgnoreTimestamp(), metricdatatest.IgnoreValue()) + }, + }, + { + name: "empty_records", + enable: true, + test: func(t *testing.T, scopeMetrics func() metricdata.ScopeMetrics) { + exporter, err := New() + require.NoError(t, err) + + err = exporter.Export(context.Background(), []sdklog.Record{}) + require.NoError(t, err) + + got := scopeMetrics() + if len(got.Metrics) > 0 { + assert.Equal(t, "go.opentelemetry.io/otel/exporters/stdout/stdoutlog", got.Scope.Name) + assert.NotEmpty(t, got.Scope.Version) + } + assert.Empty(t, got.Metrics, "no metrics should be recorded for empty records") + }, + }, + { + name: "self_observability_disabled", + enable: false, + test: func(t *testing.T, scopeMetrics func() metricdata.ScopeMetrics) { + exporter, err := New() + require.NoError(t, err) + + rf := logtest.RecordFactory{ + Timestamp: time.Now(), + Body: log.StringValue("test log"), + } + record := rf.NewRecord() + records := []sdklog.Record{record} + + err = exporter.Export(context.Background(), records) + require.NoError(t, err) + + got := scopeMetrics() + assert.Empty(t, got.Metrics, "no metrics should be recorded when self-observability is disabled") + }, + }, + } + for _, tc := range testCases { + t.Run(tc.name, func(t *testing.T) { + if tc.enable { + t.Setenv("OTEL_GO_X_SELF_OBSERVABILITY", "true") + } + prev := otel.GetMeterProvider() + otel.SetMeterProvider(prev) + r := metric.NewManualReader() + mp := metric.NewMeterProvider(metric.WithReader(r)) + otel.SetMeterProvider(mp) + + scopeMetrics := func() metricdata.ScopeMetrics { + var got metricdata.ResourceMetrics + err := r.Collect(context.Background(), &got) + require.NoError(t, err) + if len(got.ScopeMetrics) == 0 { + return metricdata.ScopeMetrics{} + } + return got.ScopeMetrics[0] + } + tc.test(t, scopeMetrics) + }) + } +} diff --git a/exporters/stdout/stdoutlog/selfobservability.go b/exporters/stdout/stdoutlog/selfobservability.go deleted file mode 100644 index 26cc5f7d22b..00000000000 --- a/exporters/stdout/stdoutlog/selfobservability.go +++ /dev/null @@ -1,34 +0,0 @@ -// Copyright The OpenTelemetry Authors -// SPDX-License-Identifier: Apache-2.0 - -package stdoutlog // import "go.opentelemetry.io/otel/exporters/stdout/stdoutlog" - -import ( - "go.opentelemetry.io/otel" - "go.opentelemetry.io/otel/metric" - "go.opentelemetry.io/otel/sdk" - semconv "go.opentelemetry.io/otel/semconv/v1.36.0" - "go.opentelemetry.io/otel/semconv/v1.36.0/otelconv" -) - -// newSelfObservability creates a new selfObservability instance with the required metrics. -func newSelfObservability() *selfObservability { - mp := otel.GetMeterProvider() - m := mp.Meter("go.opentelemetry.io/otel/exporters/stdout/stdoutlog", - metric.WithInstrumentationVersion(sdk.Version()), - metric.WithSchemaURL(semconv.SchemaURL)) - - so := selfObservability{} - - var err error - if so.inflight, err = otelconv.NewSDKExporterLogInflight(m); err != nil { - otel.Handle(err) - } - if so.exported, err = otelconv.NewSDKExporterLogExported(m); err != nil { - otel.Handle(err) - } - if so.duration, err = otelconv.NewSDKExporterOperationDuration(m); err != nil { - otel.Handle(err) - } - return &so -} diff --git a/exporters/stdout/stdoutlog/selfobservability_test.go b/exporters/stdout/stdoutlog/selfobservability_test.go deleted file mode 100644 index 15056d96abd..00000000000 --- a/exporters/stdout/stdoutlog/selfobservability_test.go +++ /dev/null @@ -1,209 +0,0 @@ -// Copyright The OpenTelemetry Authors -// SPDX-License-Identifier: Apache-2.0 - -package stdoutlog - -import ( - "context" - "testing" - "time" - - "github.com/stretchr/testify/require" - - "go.opentelemetry.io/otel" - "go.opentelemetry.io/otel/log" - sdklog "go.opentelemetry.io/otel/sdk/log" - "go.opentelemetry.io/otel/sdk/log/logtest" - "go.opentelemetry.io/otel/sdk/metric" - "go.opentelemetry.io/otel/sdk/metric/metricdata" -) - -func TestSelfObservability(t *testing.T) { - testCases := []struct { - name string - test func(t *testing.T, scopeMetrics func() metricdata.ScopeMetrics) - }{ - { - name: "inflight", - test: func(t *testing.T, scopeMetrics func() metricdata.ScopeMetrics) { - exporter, err := New() - require.NoError(t, err) - - rf := logtest.RecordFactory{ - Timestamp: time.Now(), - Body: log.StringValue("test log 1"), - } - record1 := rf.NewRecord() - - rf.Body = log.StringValue("test log 2") - record2 := rf.NewRecord() - - records := []sdklog.Record{record1, record2} - - err = exporter.Export(context.Background(), records) - require.NoError(t, err) - - got := scopeMetrics() - require.NotEmpty(t, got.Metrics) - - var inflightMetric metricdata.Metrics - for _, m := range got.Metrics { - if m.Name == "otel.sdk.exporter.log.inflight" { - inflightMetric = m - break - } - } - require.NotEmpty(t, inflightMetric.Name, "inflight metric should be present") - - sum, ok := inflightMetric.Data.(metricdata.Sum[int64]) - require.True(t, ok, "inflight metric should be a sum") - require.Len(t, sum.DataPoints, 1, "should have one data point") - require.Equal(t, int64(2), sum.DataPoints[0].Value, "should record 2 inflight records") - }, - }, - { - name: "exported", - test: func(t *testing.T, scopeMetrics func() metricdata.ScopeMetrics) { - exporter, err := New() - require.NoError(t, err) - - rf := logtest.RecordFactory{ - Timestamp: time.Now(), - Body: log.StringValue("test log 1"), - } - record1 := rf.NewRecord() - - rf.Body = log.StringValue("test log 2") - record2 := rf.NewRecord() - - rf.Body = log.StringValue("test log 3") - record3 := rf.NewRecord() - - records := []sdklog.Record{record1, record2, record3} - - err = exporter.Export(context.Background(), records) - require.NoError(t, err) - - got := scopeMetrics() - require.NotEmpty(t, got.Metrics) - - var exportedMetric metricdata.Metrics - for _, m := range got.Metrics { - if m.Name == "otel.sdk.exporter.log.exported" { - exportedMetric = m - break - } - } - require.NotEmpty(t, exportedMetric.Name, "exported metric should be present") - - sum, ok := exportedMetric.Data.(metricdata.Sum[int64]) - require.True(t, ok, "exported metric should be a sum") - require.Len(t, sum.DataPoints, 1, "should have one data point") - require.Equal(t, int64(3), sum.DataPoints[0].Value, "should record 3 exported records") - }, - }, - { - name: "duration", - test: func(t *testing.T, scopeMetrics func() metricdata.ScopeMetrics) { - exporter, err := New() - require.NoError(t, err) - - rf := logtest.RecordFactory{ - Timestamp: time.Now(), - Body: log.StringValue("test log"), - } - record := rf.NewRecord() - records := []sdklog.Record{record} - - err = exporter.Export(context.Background(), records) - require.NoError(t, err) - - got := scopeMetrics() - require.NotEmpty(t, got.Metrics) - - var durationMetric metricdata.Metrics - for _, m := range got.Metrics { - if m.Name == "otel.sdk.exporter.operation.duration" { - durationMetric = m - break - } - } - require.NotEmpty(t, durationMetric.Name, "duration metric should be present") - - histogram, ok := durationMetric.Data.(metricdata.Histogram[float64]) - require.True(t, ok, "duration metric should be a histogram") - require.Len(t, histogram.DataPoints, 1, "should have one data point") - require.Greater(t, histogram.DataPoints[0].Sum, 0.0, "duration should be greater than 0") - }, - }, - { - name: "multiple_exports", - test: func(t *testing.T, scopeMetrics func() metricdata.ScopeMetrics) { - exporter, err := New() - require.NoError(t, err) - - for i := 0; i < 3; i++ { - rf := logtest.RecordFactory{ - Timestamp: time.Now(), - Body: log.StringValue("test log"), - } - record := rf.NewRecord() - records := []sdklog.Record{record} - err = exporter.Export(context.Background(), records) - require.NoError(t, err) - } - - got := scopeMetrics() - require.NotEmpty(t, got.Metrics) - - var exportedMetric metricdata.Metrics - for _, m := range got.Metrics { - if m.Name == "otel.sdk.exporter.log.exported" { - exportedMetric = m - break - } - } - require.NotEmpty(t, exportedMetric.Name, "exported metric should be present") - - sum, ok := exportedMetric.Data.(metricdata.Sum[int64]) - require.True(t, ok, "exported metric should be a sum") - require.Len(t, sum.DataPoints, 1, "should have one data point") - require.Equal(t, int64(3), sum.DataPoints[0].Value, "should record 3 total exported records") - }, - }, - { - name: "empty_records", - test: func(t *testing.T, scopeMetrics func() metricdata.ScopeMetrics) { - exporter, err := New() - require.NoError(t, err) - - err = exporter.Export(context.Background(), []sdklog.Record{}) - require.NoError(t, err) - - got := scopeMetrics() - require.Empty(t, got.Metrics, "no metrics should be recorded for empty records") - }, - }, - } - for _, tc := range testCases { - t.Run(tc.name, func(t *testing.T) { - t.Setenv("OTEL_GO_X_SELF_OBSERVABILITY", "true") - prev := otel.GetMeterProvider() - defer otel.SetMeterProvider(prev) - r := metric.NewManualReader() - mp := metric.NewMeterProvider(metric.WithReader(r)) - otel.SetMeterProvider(mp) - - scopeMetrics := func() metricdata.ScopeMetrics { - var got metricdata.ResourceMetrics - err := r.Collect(context.Background(), &got) - require.NoError(t, err) - if len(got.ScopeMetrics) == 0 { - return metricdata.ScopeMetrics{} - } - return got.ScopeMetrics[0] - } - tc.test(t, scopeMetrics) - }) - } -} From 8ab2b0b57a0aca01cbf2c0611cb4c551fc3add7d Mon Sep 17 00:00:00 2001 From: Praful Date: Thu, 14 Aug 2025 19:13:49 +0530 Subject: [PATCH 06/11] feat(stdoutlog): refactor and added memory pooling Signed-off-by: Praful --- exporters/stdout/stdoutlog/config.go | 9 --- exporters/stdout/stdoutlog/doc.go | 20 +---- exporters/stdout/stdoutlog/exporter.go | 57 +++++++++---- exporters/stdout/stdoutlog/exporter_test.go | 79 ++++++++++++++++--- .../stdout/stdoutlog/internal/x/README.md | 20 +++-- 5 files changed, 125 insertions(+), 60 deletions(-) diff --git a/exporters/stdout/stdoutlog/config.go b/exporters/stdout/stdoutlog/config.go index ddc2998f5de..795802fd667 100644 --- a/exporters/stdout/stdoutlog/config.go +++ b/exporters/stdout/stdoutlog/config.go @@ -6,8 +6,6 @@ package stdoutlog // import "go.opentelemetry.io/otel/exporters/stdout/stdoutlog import ( "io" "os" - - "go.opentelemetry.io/otel/exporters/stdout/stdoutlog/internal/x" ) var ( @@ -28,11 +26,6 @@ type config struct { // Timestamps specifies if timestamps should be printed. Default is // true. Timestamps bool - - // SelfObservability enables exporter self-observability metrics. - // This is an experimental feature controlled by OTEL_GO_X_SELF_OBSERVABILITY. - // Default is false. - SelfObservability bool } // newConfig creates a validated Config configured with options. @@ -46,8 +39,6 @@ func newConfig(options []Option) config { cfg = opt.apply(cfg) } - cfg.SelfObservability = x.SelfObservability.Enabled() - return cfg } diff --git a/exporters/stdout/stdoutlog/doc.go b/exporters/stdout/stdoutlog/doc.go index 9707ba62417..fc721f825dd 100644 --- a/exporters/stdout/stdoutlog/doc.go +++ b/exporters/stdout/stdoutlog/doc.go @@ -10,22 +10,6 @@ // compatibility guarantees. If these are needed features, please use the OTLP // exporter instead. // -// # Self-Observability -// -// The exporter provides a self-observability feature that allows you to monitor -// the exporter itself. To enable this feature, set the environment variable -// OTEL_GO_X_SELF_OBSERVABILITY to the case-insensitive string value of "true" -// or use the WithSelfObservability option when creating the exporter. -// -// When enabled, the exporter will create the following metrics using the global -// MeterProvider: -// -// - otel.sdk.exporter.log.inflight -// - otel.sdk.exporter.log.exported -// - otel.sdk.exporter.operation.duration -// -// Please see the [Semantic conventions for OpenTelemetry SDK metrics] documentation -// for more details on these metrics. -// -// [Semantic conventions for OpenTelemetry SDK metrics]: https://github.com/open-telemetry/semantic-conventions/blob/v1.36.0/docs/otel/sdk-metrics.md +// See [go.opentelemetry.io/otel/exporters/stdout/stdoutlog/internal/x] for information about +// the experimental features. package stdoutlog // import "go.opentelemetry.io/otel/exporters/stdout/stdoutlog" diff --git a/exporters/stdout/stdoutlog/exporter.go b/exporters/stdout/stdoutlog/exporter.go index 4866f9b0502..56a059ce0a8 100644 --- a/exporters/stdout/stdoutlog/exporter.go +++ b/exporters/stdout/stdoutlog/exporter.go @@ -7,6 +7,7 @@ import ( "context" "encoding/json" "fmt" + "sync" "sync/atomic" "time" @@ -21,7 +22,7 @@ import ( ) // otelComponentType is a name identifying the type of the OpenTelemetry component. -const otelComponentType = "stdout_log_exporter" +const otelComponentType = "go.opentelemetry.io/otel/exporters/stdout/stdoutlog" var _ log.Exporter = &Exporter{} @@ -93,6 +94,10 @@ func (e *Exporter) initSelfObservability() { // Export exports log records to writer. func (e *Exporter) Export(ctx context.Context, records []log.Record) error { + if len(records) == 0 { + return nil + } + enc := e.encoder.Load() if enc == nil { return nil @@ -107,33 +112,55 @@ func (e *Exporter) Export(ctx context.Context, records []log.Record) error { return err } -// exportWithSelfObservability exports logs with self-observability metrics. -func (e *Exporter) exportWithSelfObservability(ctx context.Context, records []log.Record) error { - if len(records) == 0 { - return nil - } +const bufferSize = 1024 +var selfObservabilityBuffer = sync.Pool{ + New: func() any { + buf := make([]attribute.KeyValue, 0, bufferSize) + return &buf + }, +} + +// exportWithSelfObservability exports logs with self-observability metrics. +func (e *Exporter) exportWithSelfObservability(ctx context.Context, records []log.Record) (err error) { count := int64(len(records)) start := time.Now() - e.selfObservability.inflightMetric.Add(context.Background(), count, e.selfObservability.attrs...) + e.selfObservability.inflightMetric.Add(ctx, count, e.selfObservability.attrs...) + defer func() { - addAttrs := make([]attribute.KeyValue, len(e.selfObservability.attrs), len(e.selfObservability.attrs)+1) - copy(addAttrs, e.selfObservability.attrs) - if err := ctx.Err(); err != nil { + bufPtrAny := selfObservabilityBuffer.Get() + bufPtr, ok := bufPtrAny.(*[]attribute.KeyValue) + if !ok || bufPtr == nil { + bufPtr = &[]attribute.KeyValue{} + } + + addAttrs := (*bufPtr)[:0] + addAttrs = append(addAttrs, e.selfObservability.attrs...) + + if err != nil { addAttrs = append(addAttrs, semconv.ErrorType(err)) + } else { + e.selfObservability.exportedMetric.Add(ctx, count, addAttrs...) } - e.selfObservability.inflightMetric.Add(context.Background(), -count, e.selfObservability.attrs...) - e.selfObservability.exportedMetric.Add(context.Background(), count, addAttrs...) - e.selfObservability.operationDurationMetric.Record(context.Background(), time.Since(start).Seconds(), addAttrs...) + e.selfObservability.inflightMetric.Add(ctx, -count, e.selfObservability.attrs...) + e.selfObservability.operationDurationMetric.Record(ctx, time.Since(start).Seconds(), addAttrs...) + + *bufPtr = addAttrs[:0] + selfObservabilityBuffer.Put(bufPtr) }() - return e.exportWithoutSelfObservability(ctx, records) + err = e.exportWithoutSelfObservability(ctx, records) + return } // exportWithoutSelfObservability exports logs without self-observability metrics. func (e *Exporter) exportWithoutSelfObservability(ctx context.Context, records []log.Record) error { + enc := e.encoder.Load() + if enc == nil { + return nil + } for _, record := range records { // Honor context cancellation. if err := ctx.Err(); err != nil { @@ -141,7 +168,7 @@ func (e *Exporter) exportWithoutSelfObservability(ctx context.Context, records [ } recordJSON := e.newRecordJSON(record) - if err := e.encoder.Load().Encode(recordJSON); err != nil { + if err := enc.Encode(recordJSON); err != nil { return err } } diff --git a/exporters/stdout/stdoutlog/exporter_test.go b/exporters/stdout/stdoutlog/exporter_test.go index 292e06da23f..a8156be2dc8 100644 --- a/exporters/stdout/stdoutlog/exporter_test.go +++ b/exporters/stdout/stdoutlog/exporter_test.go @@ -516,15 +516,27 @@ func TestSelfObservability(t *testing.T) { { Value: 0, Attributes: attribute.NewSet( - attribute.KeyValue{Key: "otel.component.name", Value: attribute.StringValue("stdout_log_exporter/0")}, - attribute.KeyValue{Key: "otel.component.type", Value: attribute.StringValue("stdout_log_exporter")}, + attribute.KeyValue{ + Key: "otel.component.name", + Value: attribute.StringValue(otelComponentType + "/0"), + }, + attribute.KeyValue{ + Key: "otel.component.type", + Value: attribute.StringValue(otelComponentType), + }, ), }, }, }, } - metricdatatest.AssertEqual(t, expected, inflightMetric, metricdatatest.IgnoreTimestamp(), metricdatatest.IgnoreValue()) + metricdatatest.AssertEqual( + t, + expected, + inflightMetric, + metricdatatest.IgnoreTimestamp(), + metricdatatest.IgnoreValue(), + ) }, }, { @@ -578,15 +590,27 @@ func TestSelfObservability(t *testing.T) { { Value: 3, Attributes: attribute.NewSet( - attribute.KeyValue{Key: "otel.component.name", Value: attribute.StringValue("stdout_log_exporter/1")}, - attribute.KeyValue{Key: "otel.component.type", Value: attribute.StringValue("stdout_log_exporter")}, + attribute.KeyValue{ + Key: "otel.component.name", + Value: attribute.StringValue(otelComponentType + "/0"), + }, + attribute.KeyValue{ + Key: "otel.component.type", + Value: attribute.StringValue(otelComponentType), + }, ), }, }, }, } - metricdatatest.AssertEqual(t, expected, exportedMetric, metricdatatest.IgnoreTimestamp(), metricdatatest.IgnoreValue()) + metricdatatest.AssertEqual( + t, + expected, + exportedMetric, + metricdatatest.IgnoreTimestamp(), + metricdatatest.IgnoreValue(), + ) }, }, { @@ -632,15 +656,27 @@ func TestSelfObservability(t *testing.T) { { Count: 1, Attributes: attribute.NewSet( - attribute.KeyValue{Key: "otel.component.name", Value: attribute.StringValue("stdout_log_exporter/2")}, - attribute.KeyValue{Key: "otel.component.type", Value: attribute.StringValue("stdout_log_exporter")}, + attribute.KeyValue{ + Key: "otel.component.name", + Value: attribute.StringValue(otelComponentType + "/0"), + }, + attribute.KeyValue{ + Key: "otel.component.type", + Value: attribute.StringValue(otelComponentType), + }, ), }, }, }, } - metricdatatest.AssertEqual(t, expected, durationMetric, metricdatatest.IgnoreTimestamp(), metricdatatest.IgnoreValue()) + metricdatatest.AssertEqual( + t, + expected, + durationMetric, + metricdatatest.IgnoreTimestamp(), + metricdatatest.IgnoreValue(), + ) }, }, { @@ -668,8 +704,11 @@ func TestSelfObservability(t *testing.T) { assert.NotEmpty(t, got.Scope.Version) expectedAttrs := attribute.NewSet( - attribute.KeyValue{Key: "otel.component.name", Value: attribute.StringValue("stdout_log_exporter/3")}, - attribute.KeyValue{Key: "otel.component.type", Value: attribute.StringValue("stdout_log_exporter")}, + attribute.KeyValue{ + Key: "otel.component.name", + Value: attribute.StringValue(otelComponentType + "/0"), + }, + attribute.KeyValue{Key: "otel.component.type", Value: attribute.StringValue(otelComponentType)}, ) expected := metricdata.ScopeMetrics{ @@ -722,7 +761,13 @@ func TestSelfObservability(t *testing.T) { }, } - metricdatatest.AssertEqual(t, expected, got, metricdatatest.IgnoreTimestamp(), metricdatatest.IgnoreValue()) + metricdatatest.AssertEqual( + t, + expected, + got, + metricdatatest.IgnoreTimestamp(), + metricdatatest.IgnoreValue(), + ) }, }, { @@ -765,13 +810,20 @@ func TestSelfObservability(t *testing.T) { }, }, } + ranOnce := false for _, tc := range testCases { t.Run(tc.name, func(t *testing.T) { if tc.enable { t.Setenv("OTEL_GO_X_SELF_OBSERVABILITY", "true") } + + if ranOnce { + // Reset the global exporter ID counter for deterministic tests + exporterIDCounter.Store(0) // First call to nextExporterID() will return 0 + } + prev := otel.GetMeterProvider() - otel.SetMeterProvider(prev) + t.Cleanup(func() { otel.SetMeterProvider(prev) }) r := metric.NewManualReader() mp := metric.NewMeterProvider(metric.WithReader(r)) otel.SetMeterProvider(mp) @@ -787,5 +839,6 @@ func TestSelfObservability(t *testing.T) { } tc.test(t, scopeMetrics) }) + ranOnce = true } } diff --git a/exporters/stdout/stdoutlog/internal/x/README.md b/exporters/stdout/stdoutlog/internal/x/README.md index 280fe0d54d7..73faa10856d 100644 --- a/exporters/stdout/stdoutlog/internal/x/README.md +++ b/exporters/stdout/stdoutlog/internal/x/README.md @@ -1,17 +1,27 @@ # Experimental Features -This package documents the experimental features available for [go.opentelemetry.io/otel/exporters/stdout/stdoutlog]. +The stdout log exporter contains features that have not yet stabilized in the OpenTelemetry specification. +These features are added prior to stabilization so that users can start experimenting with them and provide feedback. -## Self-Observability +These features may change in backwards incompatible ways as feedback is applied. +See the [Compatibility and Stability](#compatibility-and-stability) section for more information. -The self-observability feature allows the stdout log exporter to emit metrics. When enabled, the exporter will record metrics for: +## Features + +- [Self-Observability](#self-observability) + +### Self-Observability + +The exporter provides a self-observability feature that allows you to monitor the exporter itself. + +To opt-in, set the environment variable `OTEL_GO_X_SELF_OBSERVABILITY` to `true`. + +When enabled, the exporter will record metrics for: - Number of log records currently being processed (inflight) - Total number of log records exported - Duration of export operations -To enable this feature, set the `OTEL_GO_X_SELF_OBSERVABILITY` environment variable to `true`. - ## Compatibility and Stability Experimental features do not fall within the scope of the OpenTelemetry Go versioning and stability [policy](../../../../../VERSIONING.md). From 55909d09183ce554791c2d42a3c0ecb3e1a14f0f Mon Sep 17 00:00:00 2001 From: Praful Date: Thu, 14 Aug 2025 19:19:45 +0530 Subject: [PATCH 07/11] fix: resolve merge conflicts Signed-off-by: Praful fix: resolve linter err Signed-off-by: Praful --- exporters/stdout/stdoutlog/internal/x/README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/exporters/stdout/stdoutlog/internal/x/README.md b/exporters/stdout/stdoutlog/internal/x/README.md index 73faa10856d..78f101a64e7 100644 --- a/exporters/stdout/stdoutlog/internal/x/README.md +++ b/exporters/stdout/stdoutlog/internal/x/README.md @@ -29,4 +29,4 @@ These features may be removed or modified in successive version releases, includ When an experimental feature is promoted to a stable feature, a migration path will be included in the changelog entry of the release. There is no guarantee that any environment variable feature flags that enabled the experimental feature will be supported by the stable version. -If they are supported, they may be accompanied with a deprecation notice stating a timeline for the removal of that support. \ No newline at end of file +If they are supported, they may be accompanied with a deprecation notice stating a timeline for the removal of that support. From 253afc2e4299a8d7113c291f1e797076bf6356cf Mon Sep 17 00:00:00 2001 From: Praful Date: Sat, 16 Aug 2025 19:14:01 +0530 Subject: [PATCH 08/11] refactor: use standalone newSelfObservability instead of e.initSelfObservability Signed-off-by: Praful refactor: test case from TestSelfObservability to TestNewSelfObservability Signed-off-by: Praful --- exporters/stdout/stdoutlog/exporter.go | 21 +++++++++------------ exporters/stdout/stdoutlog/exporter_test.go | 18 ++++++++++++------ 2 files changed, 21 insertions(+), 18 deletions(-) diff --git a/exporters/stdout/stdoutlog/exporter.go b/exporters/stdout/stdoutlog/exporter.go index 56a059ce0a8..d68a6593e00 100644 --- a/exporters/stdout/stdoutlog/exporter.go +++ b/exporters/stdout/stdoutlog/exporter.go @@ -55,18 +55,17 @@ func New(options ...Option) (*Exporter, error) { timestamps: cfg.Timestamps, } e.encoder.Store(enc) - e.initSelfObservability() + e.selfObservability = newSelfObservability() return &e, nil } -// initSelfObservability initializes self-observability for the exporter if enabled. -func (e *Exporter) initSelfObservability() { +func newSelfObservability() *selfObservability { if !x.SelfObservability.Enabled() { - return + return nil } - e.selfObservability = &selfObservability{ + selfObservability := &selfObservability{ enabled: true, attrs: []attribute.KeyValue{ semconv.OTelComponentName(fmt.Sprintf("%s/%d", otelComponentType, nextExporterID())), @@ -81,23 +80,21 @@ func (e *Exporter) initSelfObservability() { ) var err error - if e.selfObservability.inflightMetric, err = otelconv.NewSDKExporterLogInflight(m); err != nil { + if selfObservability.inflightMetric, err = otelconv.NewSDKExporterLogInflight(m); err != nil { otel.Handle(err) } - if e.selfObservability.exportedMetric, err = otelconv.NewSDKExporterLogExported(m); err != nil { + if selfObservability.exportedMetric, err = otelconv.NewSDKExporterLogExported(m); err != nil { otel.Handle(err) } - if e.selfObservability.operationDurationMetric, err = otelconv.NewSDKExporterOperationDuration(m); err != nil { + if selfObservability.operationDurationMetric, err = otelconv.NewSDKExporterOperationDuration(m); err != nil { otel.Handle(err) } + + return selfObservability } // Export exports log records to writer. func (e *Exporter) Export(ctx context.Context, records []log.Record) error { - if len(records) == 0 { - return nil - } - enc := e.encoder.Load() if enc == nil { return nil diff --git a/exporters/stdout/stdoutlog/exporter_test.go b/exporters/stdout/stdoutlog/exporter_test.go index a8156be2dc8..e4cd54e78ec 100644 --- a/exporters/stdout/stdoutlog/exporter_test.go +++ b/exporters/stdout/stdoutlog/exporter_test.go @@ -462,7 +462,7 @@ func TestValueMarshalJSON(t *testing.T) { } } -func TestSelfObservability(t *testing.T) { +func TestNewSelfObservability(t *testing.T) { testCases := []struct { name string enable bool @@ -776,16 +776,22 @@ func TestSelfObservability(t *testing.T) { test: func(t *testing.T, scopeMetrics func() metricdata.ScopeMetrics) { exporter, err := New() require.NoError(t, err) - err = exporter.Export(context.Background(), []sdklog.Record{}) require.NoError(t, err) got := scopeMetrics() - if len(got.Metrics) > 0 { - assert.Equal(t, "go.opentelemetry.io/otel/exporters/stdout/stdoutlog", got.Scope.Name) - assert.NotEmpty(t, got.Scope.Version) + assert.Equal(t, "go.opentelemetry.io/otel/exporters/stdout/stdoutlog", got.Scope.Name) + assert.NotEmpty(t, got.Scope.Version) + assert.NotEmpty(t, got.Metrics, "metrics should be recorded even for empty records") + + assert.Len(t, got.Metrics, 3, "should have 3 metrics for self-observability") + metricNames := make(map[string]bool) + for _, metric := range got.Metrics { + metricNames[metric.Name] = true } - assert.Empty(t, got.Metrics, "no metrics should be recorded for empty records") + assert.True(t, metricNames["otel.sdk.exporter.log.exported"], "exported metric should be present") + assert.True(t, metricNames["otel.sdk.exporter.operation.duration"], "duration metric should be present") + assert.True(t, metricNames["otel.sdk.exporter.log.inflight"], "inflight metric should be present") }, }, { From 26a279b1c3f94a131389d376f587e2e31c240f61 Mon Sep 17 00:00:00 2001 From: Praful Date: Sun, 17 Aug 2025 13:18:00 +0530 Subject: [PATCH 09/11] refactor: record both success and error metrics in exportWithSelfObservability Signed-off-by: Praful --- exporters/stdout/stdoutlog/exporter.go | 33 +++--- exporters/stdout/stdoutlog/exporter_test.go | 107 ++++++++++++++++++++ 2 files changed, 121 insertions(+), 19 deletions(-) diff --git a/exporters/stdout/stdoutlog/exporter.go b/exporters/stdout/stdoutlog/exporter.go index d68a6593e00..26c08884af6 100644 --- a/exporters/stdout/stdoutlog/exporter.go +++ b/exporters/stdout/stdoutlog/exporter.go @@ -35,7 +35,6 @@ type Exporter struct { } type selfObservability struct { - enabled bool attrs []attribute.KeyValue inflightMetric otelconv.SDKExporterLogInflight exportedMetric otelconv.SDKExporterLogExported @@ -55,18 +54,21 @@ func New(options ...Option) (*Exporter, error) { timestamps: cfg.Timestamps, } e.encoder.Store(enc) - e.selfObservability = newSelfObservability() + selfObs, err := newSelfObservability() + if err != nil { + return nil, err + } + e.selfObservability = selfObs return &e, nil } -func newSelfObservability() *selfObservability { +func newSelfObservability() (*selfObservability, error) { if !x.SelfObservability.Enabled() { - return nil + return nil, nil } selfObservability := &selfObservability{ - enabled: true, attrs: []attribute.KeyValue{ semconv.OTelComponentName(fmt.Sprintf("%s/%d", otelComponentType, nextExporterID())), semconv.OTelComponentTypeKey.String(otelComponentType), @@ -81,27 +83,22 @@ func newSelfObservability() *selfObservability { var err error if selfObservability.inflightMetric, err = otelconv.NewSDKExporterLogInflight(m); err != nil { - otel.Handle(err) + return nil, err } if selfObservability.exportedMetric, err = otelconv.NewSDKExporterLogExported(m); err != nil { - otel.Handle(err) + return nil, err } if selfObservability.operationDurationMetric, err = otelconv.NewSDKExporterOperationDuration(m); err != nil { - otel.Handle(err) + return nil, err } - return selfObservability + return selfObservability, nil } // Export exports log records to writer. func (e *Exporter) Export(ctx context.Context, records []log.Record) error { - enc := e.encoder.Load() - if enc == nil { - return nil - } - var err error - if e.selfObservability != nil && e.selfObservability.enabled { + if e.selfObservability != nil && x.SelfObservability.Enabled() { err = e.exportWithSelfObservability(ctx, records) } else { err = e.exportWithoutSelfObservability(ctx, records) @@ -109,7 +106,7 @@ func (e *Exporter) Export(ctx context.Context, records []log.Record) error { return err } -const bufferSize = 1024 +const bufferSize = 4 var selfObservabilityBuffer = sync.Pool{ New: func() any { @@ -137,10 +134,8 @@ func (e *Exporter) exportWithSelfObservability(ctx context.Context, records []lo if err != nil { addAttrs = append(addAttrs, semconv.ErrorType(err)) - } else { - e.selfObservability.exportedMetric.Add(ctx, count, addAttrs...) } - + e.selfObservability.exportedMetric.Add(ctx, count, addAttrs...) e.selfObservability.inflightMetric.Add(ctx, -count, e.selfObservability.attrs...) e.selfObservability.operationDurationMetric.Record(ctx, time.Since(start).Seconds(), addAttrs...) diff --git a/exporters/stdout/stdoutlog/exporter_test.go b/exporters/stdout/stdoutlog/exporter_test.go index e4cd54e78ec..811c872cb18 100644 --- a/exporters/stdout/stdoutlog/exporter_test.go +++ b/exporters/stdout/stdoutlog/exporter_test.go @@ -794,6 +794,113 @@ func TestNewSelfObservability(t *testing.T) { assert.True(t, metricNames["otel.sdk.exporter.log.inflight"], "inflight metric should be present") }, }, + { + name: "export_with_error", + enable: true, + test: func(t *testing.T, scopeMetrics func() metricdata.ScopeMetrics) { + exporter, err := New() + require.NoError(t, err) + + ctx, cancel := context.WithCancel(context.Background()) + cancel() + + rf := logtest.RecordFactory{ + Timestamp: time.Now(), + Body: log.StringValue("test log"), + } + record := rf.NewRecord() + records := []sdklog.Record{record, record} + + err = exporter.Export(ctx, records) + require.Error(t, err) + require.Equal(t, context.Canceled, err) + + got := scopeMetrics() + assert.NotEmpty(t, got.Metrics) + + var exportedMetric metricdata.Metrics + exportedInstrument := otelconv.SDKExporterLogExported{} + for _, m := range got.Metrics { + if m.Name == exportedInstrument.Name() { + exportedMetric = m + break + } + } + require.NotEmpty(t, exportedMetric, "exported metric not found") + + data := exportedMetric.Data.(metricdata.Sum[int64]) + require.Len(t, data.DataPoints, 1) + + dataPoint := data.DataPoints[0] + assert.Equal(t, int64(2), dataPoint.Value) + + attrs := dataPoint.Attributes + errorTypeAttr, found := attrs.Value(attribute.Key("error.type")) + assert.True(t, found, "error.type attribute should be present") + assert.Equal(t, "*errors.errorString", errorTypeAttr.AsString()) + }, + }, + { + name: "multiple_exports_mixed_success_failure", + enable: true, + test: func(t *testing.T, scopeMetrics func() metricdata.ScopeMetrics) { + exporter, err := New() + require.NoError(t, err) + + rf := logtest.RecordFactory{ + Timestamp: time.Now(), + Body: log.StringValue("test log"), + } + + record1 := rf.NewRecord() + record2 := rf.NewRecord() + err = exporter.Export(context.Background(), []sdklog.Record{record1, record2}) + require.NoError(t, err) + + record3 := rf.NewRecord() + err = exporter.Export(context.Background(), []sdklog.Record{record3}) + require.NoError(t, err) + + ctx, cancel := context.WithCancel(context.Background()) + cancel() + record4 := rf.NewRecord() + record5 := rf.NewRecord() + record6 := rf.NewRecord() + err = exporter.Export(ctx, []sdklog.Record{record4, record5, record6}) + require.Error(t, err) + + got := scopeMetrics() + assert.NotEmpty(t, got.Metrics) + + var exportedMetric metricdata.Metrics + exportedInstrument := otelconv.SDKExporterLogExported{} + for _, m := range got.Metrics { + if m.Name == exportedInstrument.Name() { + exportedMetric = m + break + } + } + require.NotEmpty(t, exportedMetric, "exported metric not found") + data := exportedMetric.Data.(metricdata.Sum[int64]) + require.Len(t, data.DataPoints, 2) + + successPoint := data.DataPoints[0] + errorPoint := data.DataPoints[1] + + if _, hasError := successPoint.Attributes.Value(attribute.Key("error.type")); hasError { + successPoint, errorPoint = errorPoint, successPoint + } + + assert.Equal(t, int64(3), successPoint.Value) + _, hasError := successPoint.Attributes.Value(attribute.Key("error.type")) + assert.False(t, hasError, "success data point should not have error.type attribute") + + assert.Equal(t, int64(3), errorPoint.Value) + errorTypeAttr, hasError := errorPoint.Attributes.Value(attribute.Key("error.type")) + assert.True(t, hasError, "error data point should have error.type attribute") + assert.Equal(t, "*errors.errorString", errorTypeAttr.AsString()) + }, + }, { name: "self_observability_disabled", enable: false, From 670c3ab89a9759840989ab355fe55463cf5b0264 Mon Sep 17 00:00:00 2001 From: Praful Khanduri Date: Thu, 11 Sep 2025 18:19:03 +0530 Subject: [PATCH 10/11] refactor (stdoutlog): implement self-observability metrics following contributing guidelines Signed-off-by: Praful Khanduri added chloggen Signed-off-by: Praful Khanduri fix: return err Signed-off-by: Praful Khanduri --- CHANGELOG.md | 2 + exporters/stdout/stdoutlog/config.go | 1 - exporters/stdout/stdoutlog/exporter.go | 86 +++++++++---------- exporters/stdout/stdoutlog/exporter_test.go | 5 +- exporters/stdout/stdoutlog/go.mod | 11 +-- .../stdoutlog/internal/counter/counter.go | 31 +++++++ .../internal/counter/counter_test.go | 65 ++++++++++++++ exporters/stdout/stdoutlog/internal/gen.go | 9 ++ 8 files changed, 153 insertions(+), 57 deletions(-) create mode 100644 exporters/stdout/stdoutlog/internal/counter/counter.go create mode 100644 exporters/stdout/stdoutlog/internal/counter/counter_test.go create mode 100644 exporters/stdout/stdoutlog/internal/gen.go diff --git a/CHANGELOG.md b/CHANGELOG.md index 0a17cf4dce5..5d1bec18284 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -10,6 +10,8 @@ This project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0.htm ### Added +- Add experimental self-observability log exporter metrics in `go.opentelemetry.io/otel/exporters/stdout/stdoutlog`. + Check the `go.opentelemetry.io/otel/exporters/stdout/stdoutlog/internal/x` package documentation for more information. - Add `WithInstrumentationAttributeSet` option to `go.opentelemetry.io/otel/log`, `go.opentelemetry.io/otel/metric`, and `go.opentelemetry.io/otel/trace` packages. This provides a concurrent-safe and performant alternative to `WithInstrumentationAttributes` by accepting a pre-constructed `attribute.Set`. (#7287) - Greatly reduce the cost of recording metrics in `go.opentelemetry.io/otel/sdk/metric` using hashing for map keys. (#7175) diff --git a/exporters/stdout/stdoutlog/config.go b/exporters/stdout/stdoutlog/config.go index 795802fd667..1b8f8bbb2ca 100644 --- a/exporters/stdout/stdoutlog/config.go +++ b/exporters/stdout/stdoutlog/config.go @@ -38,7 +38,6 @@ func newConfig(options []Option) config { for _, opt := range options { cfg = opt.apply(cfg) } - return cfg } diff --git a/exporters/stdout/stdoutlog/exporter.go b/exporters/stdout/stdoutlog/exporter.go index 26c08884af6..753e97da3bb 100644 --- a/exporters/stdout/stdoutlog/exporter.go +++ b/exporters/stdout/stdoutlog/exporter.go @@ -6,6 +6,7 @@ package stdoutlog // import "go.opentelemetry.io/otel/exporters/stdout/stdoutlog import ( "context" "encoding/json" + "errors" "fmt" "sync" "sync/atomic" @@ -13,12 +14,13 @@ import ( "go.opentelemetry.io/otel" "go.opentelemetry.io/otel/attribute" + "go.opentelemetry.io/otel/exporters/stdout/stdoutlog/internal/counter" "go.opentelemetry.io/otel/exporters/stdout/stdoutlog/internal/x" "go.opentelemetry.io/otel/metric" "go.opentelemetry.io/otel/sdk" "go.opentelemetry.io/otel/sdk/log" - semconv "go.opentelemetry.io/otel/semconv/v1.36.0" - "go.opentelemetry.io/otel/semconv/v1.36.0/otelconv" + semconv "go.opentelemetry.io/otel/semconv/v1.37.0" + "go.opentelemetry.io/otel/semconv/v1.37.0/otelconv" ) // otelComponentType is a name identifying the type of the OpenTelemetry component. @@ -29,12 +31,12 @@ var _ log.Exporter = &Exporter{} // Exporter writes JSON-encoded log records to an [io.Writer] ([os.Stdout] by default). // Exporter must be created with [New]. type Exporter struct { - encoder atomic.Pointer[json.Encoder] - timestamps bool - selfObservability *selfObservability + encoder atomic.Pointer[json.Encoder] + timestamps bool + inst *instrumentationImpl } -type selfObservability struct { +type instrumentationImpl struct { attrs []attribute.KeyValue inflightMetric otelconv.SDKExporterLogInflight exportedMetric otelconv.SDKExporterLogExported @@ -54,23 +56,23 @@ func New(options ...Option) (*Exporter, error) { timestamps: cfg.Timestamps, } e.encoder.Store(enc) - selfObs, err := newSelfObservability() + selfObs, err := newInstrumentation() if err != nil { return nil, err } - e.selfObservability = selfObs + e.inst = selfObs return &e, nil } -func newSelfObservability() (*selfObservability, error) { +func newInstrumentation() (*instrumentationImpl, error) { if !x.SelfObservability.Enabled() { return nil, nil } - selfObservability := &selfObservability{ + inst := &instrumentationImpl{ attrs: []attribute.KeyValue{ - semconv.OTelComponentName(fmt.Sprintf("%s/%d", otelComponentType, nextExporterID())), + semconv.OTelComponentName(fmt.Sprintf("%s/%d", otelComponentType, counter.NextExporterID())), semconv.OTelComponentTypeKey.String(otelComponentType), }, } @@ -81,24 +83,23 @@ func newSelfObservability() (*selfObservability, error) { metric.WithSchemaURL(semconv.SchemaURL), ) - var err error - if selfObservability.inflightMetric, err = otelconv.NewSDKExporterLogInflight(m); err != nil { - return nil, err - } - if selfObservability.exportedMetric, err = otelconv.NewSDKExporterLogExported(m); err != nil { - return nil, err - } - if selfObservability.operationDurationMetric, err = otelconv.NewSDKExporterOperationDuration(m); err != nil { - return nil, err - } + var err, e error + inst.inflightMetric, e = otelconv.NewSDKExporterLogInflight(m) + err = errors.Join(err, e) + + inst.exportedMetric, e = otelconv.NewSDKExporterLogExported(m) + err = errors.Join(err, e) + + inst.operationDurationMetric, e = otelconv.NewSDKExporterOperationDuration(m) + err = errors.Join(err, e) - return selfObservability, nil + return inst, err } // Export exports log records to writer. func (e *Exporter) Export(ctx context.Context, records []log.Record) error { var err error - if e.selfObservability != nil && x.SelfObservability.Enabled() { + if e.inst != nil && x.SelfObservability.Enabled() { err = e.exportWithSelfObservability(ctx, records) } else { err = e.exportWithoutSelfObservability(ctx, records) @@ -108,7 +109,7 @@ func (e *Exporter) Export(ctx context.Context, records []log.Record) error { const bufferSize = 4 -var selfObservabilityBuffer = sync.Pool{ +var attrPool = sync.Pool{ New: func() any { buf := make([]attribute.KeyValue, 0, bufferSize) return &buf @@ -120,31 +121,34 @@ func (e *Exporter) exportWithSelfObservability(ctx context.Context, records []lo count := int64(len(records)) start := time.Now() - e.selfObservability.inflightMetric.Add(ctx, count, e.selfObservability.attrs...) + e.inst.inflightMetric.Add(ctx, count, e.inst.attrs...) + bufPtrAny := attrPool.Get() + bufPtr, ok := bufPtrAny.(*[]attribute.KeyValue) + if !ok || bufPtr == nil { + bufPtr = &[]attribute.KeyValue{} + } defer func() { - bufPtrAny := selfObservabilityBuffer.Get() - bufPtr, ok := bufPtrAny.(*[]attribute.KeyValue) - if !ok || bufPtr == nil { - bufPtr = &[]attribute.KeyValue{} - } + *bufPtr = (*bufPtr)[:0] + attrPool.Put(bufPtr) + }() + defer func() { addAttrs := (*bufPtr)[:0] - addAttrs = append(addAttrs, e.selfObservability.attrs...) + addAttrs = append(addAttrs, e.inst.attrs...) if err != nil { addAttrs = append(addAttrs, semconv.ErrorType(err)) } - e.selfObservability.exportedMetric.Add(ctx, count, addAttrs...) - e.selfObservability.inflightMetric.Add(ctx, -count, e.selfObservability.attrs...) - e.selfObservability.operationDurationMetric.Record(ctx, time.Since(start).Seconds(), addAttrs...) + e.inst.exportedMetric.Add(ctx, count, addAttrs...) + e.inst.inflightMetric.Add(ctx, -count, e.inst.attrs...) + e.inst.operationDurationMetric.Record(ctx, time.Since(start).Seconds(), addAttrs...) - *bufPtr = addAttrs[:0] - selfObservabilityBuffer.Put(bufPtr) + *bufPtr = addAttrs }() err = e.exportWithoutSelfObservability(ctx, records) - return + return err } // exportWithoutSelfObservability exports logs without self-observability metrics. @@ -178,11 +182,3 @@ func (e *Exporter) Shutdown(context.Context) error { func (*Exporter) ForceFlush(context.Context) error { return nil } - -var exporterIDCounter atomic.Int64 - -// nextExporterID returns a new unique ID for an exporter. -// the starting value is 0, and it increments by 1 for each call. -func nextExporterID() int64 { - return exporterIDCounter.Add(1) - 1 -} diff --git a/exporters/stdout/stdoutlog/exporter_test.go b/exporters/stdout/stdoutlog/exporter_test.go index 811c872cb18..f79856522dd 100644 --- a/exporters/stdout/stdoutlog/exporter_test.go +++ b/exporters/stdout/stdoutlog/exporter_test.go @@ -16,6 +16,7 @@ import ( "go.opentelemetry.io/otel" "go.opentelemetry.io/otel/attribute" + "go.opentelemetry.io/otel/exporters/stdout/stdoutlog/internal/counter" "go.opentelemetry.io/otel/log" "go.opentelemetry.io/otel/sdk/instrumentation" sdklog "go.opentelemetry.io/otel/sdk/log" @@ -24,7 +25,7 @@ import ( "go.opentelemetry.io/otel/sdk/metric/metricdata" "go.opentelemetry.io/otel/sdk/metric/metricdata/metricdatatest" "go.opentelemetry.io/otel/sdk/resource" - "go.opentelemetry.io/otel/semconv/v1.36.0/otelconv" + "go.opentelemetry.io/otel/semconv/v1.37.0/otelconv" "go.opentelemetry.io/otel/trace" ) @@ -932,7 +933,7 @@ func TestNewSelfObservability(t *testing.T) { if ranOnce { // Reset the global exporter ID counter for deterministic tests - exporterIDCounter.Store(0) // First call to nextExporterID() will return 0 + counter.SetExporterID(0) // First call to NextExporterID() will return 0 } prev := otel.GetMeterProvider() diff --git a/exporters/stdout/stdoutlog/go.mod b/exporters/stdout/stdoutlog/go.mod index 0b201db690e..167df3d8065 100644 --- a/exporters/stdout/stdoutlog/go.mod +++ b/exporters/stdout/stdoutlog/go.mod @@ -6,21 +6,14 @@ go 1.24.0 retract v0.12.0 require ( - github.com/stretchr/testify v1.10.0 - go.opentelemetry.io/otel v1.37.0 - go.opentelemetry.io/otel/log v0.13.0 - go.opentelemetry.io/otel/metric v1.37.0 - go.opentelemetry.io/otel/sdk v1.37.0 - go.opentelemetry.io/otel/sdk/log v0.13.0 - go.opentelemetry.io/otel/sdk/log/logtest v0.13.0 - go.opentelemetry.io/otel/sdk/metric v1.37.0 - go.opentelemetry.io/otel/trace v1.37.0 github.com/stretchr/testify v1.11.1 go.opentelemetry.io/otel v1.38.0 go.opentelemetry.io/otel/log v0.14.0 + go.opentelemetry.io/otel/metric v1.38.0 go.opentelemetry.io/otel/sdk v1.38.0 go.opentelemetry.io/otel/sdk/log v0.14.0 go.opentelemetry.io/otel/sdk/log/logtest v0.14.0 + go.opentelemetry.io/otel/sdk/metric v1.38.0 go.opentelemetry.io/otel/trace v1.38.0 ) diff --git a/exporters/stdout/stdoutlog/internal/counter/counter.go b/exporters/stdout/stdoutlog/internal/counter/counter.go new file mode 100644 index 00000000000..bbb4cd7ddc9 --- /dev/null +++ b/exporters/stdout/stdoutlog/internal/counter/counter.go @@ -0,0 +1,31 @@ +// Code generated by gotmpl. DO NOT MODIFY. +// source: internal/shared/counter/counter.go.tmpl + +// Copyright The OpenTelemetry Authors +// SPDX-License-Identifier: Apache-2.0 + +// Package counter provides a simple counter for generating unique IDs. +// +// This package is used to generate unique IDs while allowing testing packages +// to reset the counter. +package counter // import "go.opentelemetry.io/otel/exporters/stdout/stdoutlog/internal/counter" + +import "sync/atomic" + +// exporterN is a global 0-based count of the number of exporters created. +var exporterN atomic.Int64 + +// NextExporterID returns the next unique ID for an exporter. +func NextExporterID() int64 { + const inc = 1 + return exporterN.Add(inc) - inc +} + +// SetExporterID sets the exporter ID counter to v and returns the previous +// value. +// +// This function is useful for testing purposes, allowing you to reset the +// counter. It should not be used in production code. +func SetExporterID(v int64) int64 { + return exporterN.Swap(v) +} diff --git a/exporters/stdout/stdoutlog/internal/counter/counter_test.go b/exporters/stdout/stdoutlog/internal/counter/counter_test.go new file mode 100644 index 00000000000..f3e380d3325 --- /dev/null +++ b/exporters/stdout/stdoutlog/internal/counter/counter_test.go @@ -0,0 +1,65 @@ +// Code generated by gotmpl. DO NOT MODIFY. +// source: internal/shared/counter/counter_test.go.tmpl + +// Copyright The OpenTelemetry Authors +// SPDX-License-Identifier: Apache-2.0 + +package counter + +import ( + "sync" + "testing" +) + +func TestNextExporterID(t *testing.T) { + SetExporterID(0) + + var expected int64 + for range 10 { + id := NextExporterID() + if id != expected { + t.Errorf("NextExporterID() = %d; want %d", id, expected) + } + expected++ + } +} + +func TestSetExporterID(t *testing.T) { + SetExporterID(0) + + prev := SetExporterID(42) + if prev != 0 { + t.Errorf("SetExporterID(42) returned %d; want 0", prev) + } + + id := NextExporterID() + if id != 42 { + t.Errorf("NextExporterID() = %d; want 42", id) + } +} + +func TestNextExporterIDConcurrentSafe(t *testing.T) { + SetExporterID(0) + + const goroutines = 100 + const increments = 10 + + var wg sync.WaitGroup + wg.Add(goroutines) + + for range goroutines { + go func() { + defer wg.Done() + for range increments { + NextExporterID() + } + }() + } + + wg.Wait() + + expected := int64(goroutines * increments) + if id := NextExporterID(); id != expected { + t.Errorf("NextExporterID() = %d; want %d", id, expected) + } +} \ No newline at end of file diff --git a/exporters/stdout/stdoutlog/internal/gen.go b/exporters/stdout/stdoutlog/internal/gen.go new file mode 100644 index 00000000000..3baadef2304 --- /dev/null +++ b/exporters/stdout/stdoutlog/internal/gen.go @@ -0,0 +1,9 @@ +// Copyright The OpenTelemetry Authors +// SPDX-License-Identifier: Apache-2.0 + +// Package internal provides internal functionality for the stdoutlog +// package. +package internal // import "go.opentelemetry.io/otel/exporters/stdout/stdoutlog/internal" + +//go:generate gotmpl --body=../../../../internal/shared/counter/counter.go.tmpl "--data={ \"pkg\": \"go.opentelemetry.io/otel/exporters/stdout/stdoutlog/internal/counter\" }" --out=counter/counter.go +//go:generate gotmpl --body=../../../../internal/shared/counter/counter_test.go.tmpl "--data={}" --out=counter/counter_test.go From 6eac6ed45f4f8a70503d695840b26c2c3da24621 Mon Sep 17 00:00:00 2001 From: Praful Date: Sat, 27 Sep 2025 15:16:55 +0000 Subject: [PATCH 11/11] refactor(stdoutlog): improve test cases and standardize code style Signed-off-by: Praful --- CHANGELOG.md | 4 +- exporters/stdout/stdoutlog/exporter.go | 141 ++++---------- exporters/stdout/stdoutlog/exporter_test.go | 18 +- exporters/stdout/stdoutlog/go.mod | 1 - .../internal/observ/instrumentation.go | 175 ++++++++++++++++++ .../stdout/stdoutlog/internal/version.go | 8 + .../stdout/stdoutlog/internal/x/README.md | 6 +- exporters/stdout/stdoutlog/internal/x/x.go | 7 +- .../stdout/stdoutlog/internal/x/x_test.go | 94 +++++----- exporters/stdout/stdoutlog/record.go | 4 +- 10 files changed, 276 insertions(+), 182 deletions(-) create mode 100644 exporters/stdout/stdoutlog/internal/observ/instrumentation.go create mode 100644 exporters/stdout/stdoutlog/internal/version.go diff --git a/CHANGELOG.md b/CHANGELOG.md index 5d1bec18284..ae8c8285a90 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -10,13 +10,13 @@ This project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0.htm ### Added -- Add experimental self-observability log exporter metrics in `go.opentelemetry.io/otel/exporters/stdout/stdoutlog`. - Check the `go.opentelemetry.io/otel/exporters/stdout/stdoutlog/internal/x` package documentation for more information. - Add `WithInstrumentationAttributeSet` option to `go.opentelemetry.io/otel/log`, `go.opentelemetry.io/otel/metric`, and `go.opentelemetry.io/otel/trace` packages. This provides a concurrent-safe and performant alternative to `WithInstrumentationAttributes` by accepting a pre-constructed `attribute.Set`. (#7287) - Greatly reduce the cost of recording metrics in `go.opentelemetry.io/otel/sdk/metric` using hashing for map keys. (#7175) - Add experimental observability for the prometheus exporter in `go.opentelemetry.io/otel/exporters/prometheus`. Check the `go.opentelemetry.io/otel/exporters/prometheus/internal/x` package documentation for more information. (#7345) +- Add experimental observability log exporter metrics in `go.opentelemetry.io/otel/exporters/stdout/stdoutlog`. + Check the `go.opentelemetry.io/otel/exporters/stdout/stdoutlog/internal/x` package documentation for more information. (#7351) ### Fixed diff --git a/exporters/stdout/stdoutlog/exporter.go b/exporters/stdout/stdoutlog/exporter.go index 753e97da3bb..15e97a24d7c 100644 --- a/exporters/stdout/stdoutlog/exporter.go +++ b/exporters/stdout/stdoutlog/exporter.go @@ -6,41 +6,32 @@ package stdoutlog // import "go.opentelemetry.io/otel/exporters/stdout/stdoutlog import ( "context" "encoding/json" - "errors" - "fmt" - "sync" "sync/atomic" - "time" - "go.opentelemetry.io/otel" - "go.opentelemetry.io/otel/attribute" + "go.opentelemetry.io/otel/exporters/stdout/stdoutlog/internal" "go.opentelemetry.io/otel/exporters/stdout/stdoutlog/internal/counter" - "go.opentelemetry.io/otel/exporters/stdout/stdoutlog/internal/x" - "go.opentelemetry.io/otel/metric" - "go.opentelemetry.io/otel/sdk" + "go.opentelemetry.io/otel/exporters/stdout/stdoutlog/internal/observ" "go.opentelemetry.io/otel/sdk/log" - semconv "go.opentelemetry.io/otel/semconv/v1.37.0" - "go.opentelemetry.io/otel/semconv/v1.37.0/otelconv" ) // otelComponentType is a name identifying the type of the OpenTelemetry component. -const otelComponentType = "go.opentelemetry.io/otel/exporters/stdout/stdoutlog" +const ( + otelComponentType = "go.opentelemetry.io/otel/exporters/stdout/stdoutlog" + + // Version is the current version of this instrumentation. + // + // This matches the version of the exporter. + Version = internal.Version +) var _ log.Exporter = &Exporter{} // Exporter writes JSON-encoded log records to an [io.Writer] ([os.Stdout] by default). // Exporter must be created with [New]. type Exporter struct { - encoder atomic.Pointer[json.Encoder] - timestamps bool - inst *instrumentationImpl -} - -type instrumentationImpl struct { - attrs []attribute.KeyValue - inflightMetric otelconv.SDKExporterLogInflight - exportedMetric otelconv.SDKExporterLogExported - operationDurationMetric otelconv.SDKExporterOperationDuration + encoder atomic.Pointer[json.Encoder] + timestamps bool + instrumentation *observ.Instrumentation } // New creates an [Exporter]. @@ -56,119 +47,51 @@ func New(options ...Option) (*Exporter, error) { timestamps: cfg.Timestamps, } e.encoder.Store(enc) - selfObs, err := newInstrumentation() + + exporterID := counter.NextExporterID() + inst, err := observ.NewInstrumentation(otelComponentType, exporterID) if err != nil { return nil, err } - e.inst = selfObs + e.instrumentation = inst return &e, nil } -func newInstrumentation() (*instrumentationImpl, error) { - if !x.SelfObservability.Enabled() { - return nil, nil - } - - inst := &instrumentationImpl{ - attrs: []attribute.KeyValue{ - semconv.OTelComponentName(fmt.Sprintf("%s/%d", otelComponentType, counter.NextExporterID())), - semconv.OTelComponentTypeKey.String(otelComponentType), - }, - } - - mp := otel.GetMeterProvider() - m := mp.Meter("go.opentelemetry.io/otel/exporters/stdout/stdoutlog", - metric.WithInstrumentationVersion(sdk.Version()), - metric.WithSchemaURL(semconv.SchemaURL), - ) - - var err, e error - inst.inflightMetric, e = otelconv.NewSDKExporterLogInflight(m) - err = errors.Join(err, e) - - inst.exportedMetric, e = otelconv.NewSDKExporterLogExported(m) - err = errors.Join(err, e) - - inst.operationDurationMetric, e = otelconv.NewSDKExporterOperationDuration(m) - err = errors.Join(err, e) - - return inst, err -} - // Export exports log records to writer. func (e *Exporter) Export(ctx context.Context, records []log.Record) error { - var err error - if e.inst != nil && x.SelfObservability.Enabled() { - err = e.exportWithSelfObservability(ctx, records) - } else { - err = e.exportWithoutSelfObservability(ctx, records) + if inst := e.instrumentation; inst != nil { + done := inst.ExportLogs(ctx, len(records)) + exported, err := e.exportRecords(ctx, records) + done(exported, err) + return err } - return err -} - -const bufferSize = 4 - -var attrPool = sync.Pool{ - New: func() any { - buf := make([]attribute.KeyValue, 0, bufferSize) - return &buf - }, -} - -// exportWithSelfObservability exports logs with self-observability metrics. -func (e *Exporter) exportWithSelfObservability(ctx context.Context, records []log.Record) (err error) { - count := int64(len(records)) - start := time.Now() - - e.inst.inflightMetric.Add(ctx, count, e.inst.attrs...) - - bufPtrAny := attrPool.Get() - bufPtr, ok := bufPtrAny.(*[]attribute.KeyValue) - if !ok || bufPtr == nil { - bufPtr = &[]attribute.KeyValue{} - } - defer func() { - *bufPtr = (*bufPtr)[:0] - attrPool.Put(bufPtr) - }() - defer func() { - addAttrs := (*bufPtr)[:0] - addAttrs = append(addAttrs, e.inst.attrs...) - - if err != nil { - addAttrs = append(addAttrs, semconv.ErrorType(err)) - } - e.inst.exportedMetric.Add(ctx, count, addAttrs...) - e.inst.inflightMetric.Add(ctx, -count, e.inst.attrs...) - e.inst.operationDurationMetric.Record(ctx, time.Since(start).Seconds(), addAttrs...) - - *bufPtr = addAttrs - }() - - err = e.exportWithoutSelfObservability(ctx, records) + _, err := e.exportRecords(ctx, records) return err } -// exportWithoutSelfObservability exports logs without self-observability metrics. -func (e *Exporter) exportWithoutSelfObservability(ctx context.Context, records []log.Record) error { +func (e *Exporter) exportRecords(ctx context.Context, records []log.Record) (int64, error) { enc := e.encoder.Load() if enc == nil { - return nil + return 0, nil } + + var exported int64 for _, record := range records { // Honor context cancellation. if err := ctx.Err(); err != nil { - return err + return exported, err } recordJSON := e.newRecordJSON(record) if err := enc.Encode(recordJSON); err != nil { - return err + return exported, err } + exported++ } - return nil + + return exported, nil } // Shutdown shuts down the Exporter. diff --git a/exporters/stdout/stdoutlog/exporter_test.go b/exporters/stdout/stdoutlog/exporter_test.go index f79856522dd..42cc2c0bf81 100644 --- a/exporters/stdout/stdoutlog/exporter_test.go +++ b/exporters/stdout/stdoutlog/exporter_test.go @@ -18,7 +18,7 @@ import ( "go.opentelemetry.io/otel/attribute" "go.opentelemetry.io/otel/exporters/stdout/stdoutlog/internal/counter" "go.opentelemetry.io/otel/log" - "go.opentelemetry.io/otel/sdk/instrumentation" + sdkinstrumentation "go.opentelemetry.io/otel/sdk/instrumentation" sdklog "go.opentelemetry.io/otel/sdk/log" "go.opentelemetry.io/otel/sdk/log/logtest" "go.opentelemetry.io/otel/sdk/metric" @@ -327,7 +327,7 @@ func getRecord(now time.Time) sdklog.Record { "https://example.com/custom-resource-schema", attribute.String("foo", "bar"), ), - InstrumentationScope: &instrumentation.Scope{ + InstrumentationScope: &sdkinstrumentation.Scope{ Name: "name", Version: "version", SchemaURL: "https://example.com/custom-schema", @@ -785,7 +785,7 @@ func TestNewSelfObservability(t *testing.T) { assert.NotEmpty(t, got.Scope.Version) assert.NotEmpty(t, got.Metrics, "metrics should be recorded even for empty records") - assert.Len(t, got.Metrics, 3, "should have 3 metrics for self-observability") + assert.Len(t, got.Metrics, 3, "should have 3 metrics for observability") metricNames := make(map[string]bool) for _, metric := range got.Metrics { metricNames[metric.Name] = true @@ -920,22 +920,19 @@ func TestNewSelfObservability(t *testing.T) { require.NoError(t, err) got := scopeMetrics() - assert.Empty(t, got.Metrics, "no metrics should be recorded when self-observability is disabled") + assert.Empty(t, got.Metrics, "no metrics should be recorded when observability is disabled") }, }, } - ranOnce := false for _, tc := range testCases { t.Run(tc.name, func(t *testing.T) { + // Reset the global exporter ID counter for deterministic tests + counter.SetExporterID(0) // First call to NextExporterID() will return 0 + if tc.enable { t.Setenv("OTEL_GO_X_SELF_OBSERVABILITY", "true") } - if ranOnce { - // Reset the global exporter ID counter for deterministic tests - counter.SetExporterID(0) // First call to NextExporterID() will return 0 - } - prev := otel.GetMeterProvider() t.Cleanup(func() { otel.SetMeterProvider(prev) }) r := metric.NewManualReader() @@ -953,6 +950,5 @@ func TestNewSelfObservability(t *testing.T) { } tc.test(t, scopeMetrics) }) - ranOnce = true } } diff --git a/exporters/stdout/stdoutlog/go.mod b/exporters/stdout/stdoutlog/go.mod index 167df3d8065..d708f754e59 100644 --- a/exporters/stdout/stdoutlog/go.mod +++ b/exporters/stdout/stdoutlog/go.mod @@ -24,7 +24,6 @@ require ( github.com/google/uuid v1.6.0 // indirect github.com/pmezard/go-difflib v1.0.0 // indirect go.opentelemetry.io/auto/sdk v1.2.1 // indirect - go.opentelemetry.io/otel/metric v1.38.0 // indirect golang.org/x/sys v0.36.0 // indirect gopkg.in/yaml.v3 v3.0.1 // indirect ) diff --git a/exporters/stdout/stdoutlog/internal/observ/instrumentation.go b/exporters/stdout/stdoutlog/internal/observ/instrumentation.go new file mode 100644 index 00000000000..5df39b02d47 --- /dev/null +++ b/exporters/stdout/stdoutlog/internal/observ/instrumentation.go @@ -0,0 +1,175 @@ +// Copyright The OpenTelemetry Authors +// SPDX-License-Identifier: Apache-2.0 + +// Package observ provides experimental observability instrumentation for the +// stdout log exporter. +package observ // import "go.opentelemetry.io/otel/exporters/stdout/stdoutlog/internal/observ" + +import ( + "context" + "errors" + "fmt" + "sync" + "time" + + "go.opentelemetry.io/otel" + "go.opentelemetry.io/otel/attribute" + "go.opentelemetry.io/otel/exporters/stdout/stdoutlog/internal" + "go.opentelemetry.io/otel/exporters/stdout/stdoutlog/internal/x" + "go.opentelemetry.io/otel/metric" + semconv "go.opentelemetry.io/otel/semconv/v1.37.0" + "go.opentelemetry.io/otel/semconv/v1.37.0/otelconv" +) + +// InstrumentationVersion matches the stdout log exporter version. +const InstrumentationVersion = internal.Version + +var ( + attrsPool = &sync.Pool{ + New: func() any { + // component.name + component.type + error.type + const n = 1 + 1 + 1 + s := make([]attribute.KeyValue, 0, n) + return &s + }, + } + + addOptPool = &sync.Pool{ + New: func() any { + const n = 1 // WithAttributeSet + o := make([]metric.AddOption, 0, n) + return &o + }, + } + + recordOptPool = &sync.Pool{ + New: func() any { + const n = 1 // WithAttributeSet + o := make([]metric.RecordOption, 0, n) + return &o + }, + } +) + +func get[T any](p *sync.Pool) *[]T { return p.Get().(*[]T) } + +func put[T any](p *sync.Pool, s *[]T) { + *s = (*s)[:0] + p.Put(s) +} + +// Instrumentation instruments the stdout log exporter. +type Instrumentation struct { + inflight metric.Int64UpDownCounter + exported metric.Int64Counter + duration metric.Float64Histogram + + attrs []attribute.KeyValue + setOpt metric.MeasurementOption +} + +// ExportLogsDone completes an export observation. +type ExportLogsDone func(success int64, err error) + +// NewInstrumentation returns instrumentation for the stdout log exporter with +// the provided component type and exporter identifier using the global +// MeterProvider. +// +// If the experimental observability feature is disabled, nil is returned. +func NewInstrumentation(componentType string, exporterID int64) (*Instrumentation, error) { + if !x.SelfObservability.Enabled() { + return nil, nil + } + + attrs := []attribute.KeyValue{ + semconv.OTelComponentName(fmt.Sprintf("%s/%d", componentType, exporterID)), + semconv.OTelComponentTypeKey.String(componentType), + } + + inst := &Instrumentation{ + attrs: attrs, + } + + set := attribute.NewSet(attrs...) + inst.setOpt = metric.WithAttributeSet(set) + + mp := otel.GetMeterProvider() + meter := mp.Meter( + componentType, + metric.WithInstrumentationVersion(InstrumentationVersion), + metric.WithSchemaURL(semconv.SchemaURL), + ) + + var err error + + inflight, e := otelconv.NewSDKExporterLogInflight(meter) + if e != nil { + err = errors.Join(err, fmt.Errorf("failed to create log inflight metric: %w", e)) + } + inst.inflight = inflight.Inst() + + exported, e := otelconv.NewSDKExporterLogExported(meter) + if e != nil { + err = errors.Join(err, fmt.Errorf("failed to create log exported metric: %w", e)) + } + inst.exported = exported.Inst() + + duration, e := otelconv.NewSDKExporterOperationDuration(meter) + if e != nil { + err = errors.Join(err, fmt.Errorf("failed to create export duration metric: %w", e)) + } + inst.duration = duration.Inst() + + return inst, err +} + +// ExportLogs instruments the exporter Export method. It returns a callback that +// MUST be invoked when the export completes with the number of successfully +// exported records and the resulting error. +func (i *Instrumentation) ExportLogs(ctx context.Context, total int) ExportLogsDone { + start := time.Now() + + addOpt := get[metric.AddOption](addOptPool) + *addOpt = append(*addOpt, i.setOpt) + i.inflight.Add(ctx, int64(total), *addOpt...) + put(addOptPool, addOpt) + + return func(success int64, err error) { + addOpt := get[metric.AddOption](addOptPool) + defer put(addOptPool, addOpt) + *addOpt = append(*addOpt, i.setOpt) + + n := int64(total) + i.inflight.Add(ctx, -n, *addOpt...) + if success > 0 || (n == 0 && err == nil) { + i.exported.Add(ctx, success, *addOpt...) + } + + measurementOpt := i.setOpt + + if err != nil { + attrs := get[attribute.KeyValue](attrsPool) + defer put(attrsPool, attrs) + *attrs = append(*attrs, i.attrs...) + *attrs = append(*attrs, semconv.ErrorType(err)) + + set := attribute.NewSet(*attrs...) + measurementOpt = metric.WithAttributeSet(set) + + *addOpt = append((*addOpt)[:0], measurementOpt) + failures := n - success + if failures < 0 { + failures = 0 + } + if failures > 0 || (n == 0 && err != nil) { + i.exported.Add(ctx, failures, *addOpt...) + } + } + + recordOpt := get[metric.RecordOption](recordOptPool) + defer put(recordOptPool, recordOpt) + *recordOpt = append(*recordOpt, measurementOpt) + + i.duration.Record(ctx, time.Since(start).Seconds(), *recordOpt...) + } +} diff --git a/exporters/stdout/stdoutlog/internal/version.go b/exporters/stdout/stdoutlog/internal/version.go new file mode 100644 index 00000000000..6c23639cb17 --- /dev/null +++ b/exporters/stdout/stdoutlog/internal/version.go @@ -0,0 +1,8 @@ +// Copyright The OpenTelemetry Authors +// SPDX-License-Identifier: Apache-2.0 + +package internal // import "go.opentelemetry.io/otel/exporters/stdout/stdoutlog/internal/version" + +// Version is the current release version of the OpenTelemetry stdoutlog +// exporter in use. +const Version = "v0.14.0" diff --git a/exporters/stdout/stdoutlog/internal/x/README.md b/exporters/stdout/stdoutlog/internal/x/README.md index 78f101a64e7..82844005490 100644 --- a/exporters/stdout/stdoutlog/internal/x/README.md +++ b/exporters/stdout/stdoutlog/internal/x/README.md @@ -8,11 +8,11 @@ See the [Compatibility and Stability](#compatibility-and-stability) section for ## Features -- [Self-Observability](#self-observability) +- [Observability](#observability) -### Self-Observability +### Observability -The exporter provides a self-observability feature that allows you to monitor the exporter itself. +The exporter provides observability features that allow you to monitor the exporter itself. To opt-in, set the environment variable `OTEL_GO_X_SELF_OBSERVABILITY` to `true`. diff --git a/exporters/stdout/stdoutlog/internal/x/x.go b/exporters/stdout/stdoutlog/internal/x/x.go index 38c95d537ef..218fbf68d8d 100644 --- a/exporters/stdout/stdoutlog/internal/x/x.go +++ b/exporters/stdout/stdoutlog/internal/x/x.go @@ -10,10 +10,11 @@ import ( ) // SelfObservability is an experimental feature flag that determines if stdout -// log exporter self-observability metrics are enabled. +// log exporter observability metrics are enabled. // -// To enable this feature set the OTEL_GO_X_SELF_OBSERVABILITY environment variable -// to the case-insensitive string value. +// To enable this feature set the OTEL_GO_X_SELF_OBSERVABILITY environment +// variable to the case-insensitive string value of "true" (i.e. "True" and +// "TRUE" will also enable this). var SelfObservability = newFeature("SELF_OBSERVABILITY", func(v string) (string, bool) { if strings.EqualFold(v, "true") { return v, true diff --git a/exporters/stdout/stdoutlog/internal/x/x_test.go b/exporters/stdout/stdoutlog/internal/x/x_test.go index d6dae980ff9..d1697ea3c2d 100644 --- a/exporters/stdout/stdoutlog/internal/x/x_test.go +++ b/exporters/stdout/stdoutlog/internal/x/x_test.go @@ -7,62 +7,54 @@ import ( "testing" "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" ) -func TestSelfObservabilityFeature(t *testing.T) { - testCases := []struct { - name string - envValue string - enabled bool - }{ - { - name: "enabled_lowercase", - envValue: "true", - enabled: true, - }, - { - name: "enabled_uppercase", - envValue: "TRUE", - enabled: true, - }, - { - name: "enabled_mixed_case", - envValue: "True", - enabled: true, - }, - { - name: "disabled_false", - envValue: "false", - enabled: false, - }, - { - name: "disabled_invalid", - envValue: "invalid", - enabled: false, - }, - { - name: "disabled_empty", - envValue: "", - enabled: false, - }, +func TestObservability(t *testing.T) { + const key = "OTEL_GO_X_SELF_OBSERVABILITY" + require.Equal(t, key, SelfObservability.Key()) + + t.Run("true", run(setenv(key, "true"), assertEnabled(SelfObservability, "true"))) + t.Run("True", run(setenv(key, "True"), assertEnabled(SelfObservability, "True"))) + t.Run("TRUE", run(setenv(key, "TRUE"), assertEnabled(SelfObservability, "TRUE"))) + t.Run("false", run(setenv(key, "false"), assertDisabled(SelfObservability))) + t.Run("100", run(setenv(key, "100"), assertDisabled(SelfObservability))) + t.Run("empty", run(assertDisabled(SelfObservability))) +} + +func run(steps ...func(*testing.T)) func(*testing.T) { + return func(t *testing.T) { + t.Helper() + for _, step := range steps { + step(t) + } + } +} + +func setenv(k, v string) func(t *testing.T) { //nolint:unparam + return func(t *testing.T) { t.Setenv(k, v) } +} + +func assertEnabled[T any](f Feature[T], want T) func(*testing.T) { + return func(t *testing.T) { + t.Helper() + assert.True(t, f.Enabled(), "not enabled") + + v, ok := f.Lookup() + assert.True(t, ok, "Lookup state") + assert.Equal(t, want, v, "Lookup value") } +} - for _, tc := range testCases { - t.Run(tc.name, func(t *testing.T) { - if tc.envValue != "" { - t.Setenv(SelfObservability.Key(), tc.envValue) - } +func assertDisabled[T any](f Feature[T]) func(*testing.T) { + var zero T + return func(t *testing.T) { + t.Helper() - assert.Equal(t, tc.enabled, SelfObservability.Enabled()) + assert.False(t, f.Enabled(), "enabled") - value, ok := SelfObservability.Lookup() - if tc.enabled { - assert.True(t, ok) - assert.Equal(t, tc.envValue, value) - } else { - assert.False(t, ok) - assert.Empty(t, value) - } - }) + v, ok := f.Lookup() + assert.False(t, ok, "Lookup state") + assert.Equal(t, zero, v, "Lookup value") } } diff --git a/exporters/stdout/stdoutlog/record.go b/exporters/stdout/stdoutlog/record.go index 6cb0c8c01d6..db91ec53e0c 100644 --- a/exporters/stdout/stdoutlog/record.go +++ b/exporters/stdout/stdoutlog/record.go @@ -9,7 +9,7 @@ import ( "time" "go.opentelemetry.io/otel/log" - "go.opentelemetry.io/otel/sdk/instrumentation" + sdkinstrumentation "go.opentelemetry.io/otel/sdk/instrumentation" sdklog "go.opentelemetry.io/otel/sdk/log" "go.opentelemetry.io/otel/sdk/resource" "go.opentelemetry.io/otel/trace" @@ -88,7 +88,7 @@ type recordJSON struct { SpanID trace.SpanID TraceFlags trace.TraceFlags Resource *resource.Resource - Scope instrumentation.Scope + Scope sdkinstrumentation.Scope DroppedAttributes int }