[WIP] sdk/log: self observability: batch log processor metrics

CHANGELOG.md

@@ -19,6 +19,7 @@ This project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0.htm
 - Add experimental observability metrics in `go.opentelemetry.io/otel/exporters/otlp/otlplog/otlploggrpc`. (#7353)
 - Add experimental observability metrics in `go.opentelemetry.io/otel/exporters/otlp/otlptrace/otlptracegrpc`. (#7459)
 - Add experimental observability metrics in `go.opentelemetry.io/otel/exporters/otlp/otlptrace/otlptracehttp`. (#7486)
+- Add experimental observability metrics in `go.opentelemetry.io/otel/sdk/log`. (#7124)
 
 ### Fixed
 

sdk/log/batch.go

@@ -11,7 +11,10 @@ import (
 	"sync/atomic"
 	"time"
 
+	"go.opentelemetry.io/otel"
 	"go.opentelemetry.io/otel/internal/global"
+	"go.opentelemetry.io/otel/sdk/log/internal/counter"
+	"go.opentelemetry.io/otel/sdk/log/internal/observ"
 )
 
 const (
@@ -98,6 +101,9 @@ type BatchProcessor struct {
 	// stopped holds the stopped state of the BatchProcessor.
 	stopped atomic.Bool
 
+	// inst is the instrumentation for observability (nil when disabled).
+	inst *observ.BLP
+
 	noCmp [0]func() //nolint: unused  // This is indeed used.
 }
 
@@ -111,6 +117,31 @@ func NewBatchProcessor(exporter Exporter, opts ...BatchProcessorOption) *BatchPr
 		// Do not panic on nil export.
 		exporter = defaultNoopExporter
 	}
+
+	b := &BatchProcessor{
+		q:           newQueue(cfg.maxQSize.Value),
+		batchSize:   cfg.expMaxBatchSize.Value,
+		pollTrigger: make(chan struct{}, 1),
+		pollKill:    make(chan struct{}),
+	}
+
+	var err error
+	b.inst, err = observ.NewBLP(
+		counter.NextExporterID(),
+		func() int64 { return int64(b.q.Len()) },
+		int64(cfg.maxQSize.Value),
+	)
+	if err != nil {
+		otel.Handle(err)
+	}
+
+	// Wrap exporter with metrics recording if observability is enabled.
+	// This must be the innermost wrapper (closest to user exporter) to record
+	// metrics just before calling the actual exporter.
+	if b.inst != nil {
+		exporter = newMetricsExporter(exporter, b.inst)
+	}
+
 	// Order is important here. Wrap the timeoutExporter with the chunkExporter
 	// to ensure each export completes in timeout (instead of all chunked
 	// exports).
@@ -119,15 +150,9 @@ func NewBatchProcessor(exporter Exporter, opts ...BatchProcessorOption) *BatchPr
 	// appropriately on export.
 	exporter = newChunkExporter(exporter, cfg.expMaxBatchSize.Value)
 
-	b := &BatchProcessor{
-		exporter: newBufferExporter(exporter, cfg.expBufferSize.Value),
-
-		q:           newQueue(cfg.maxQSize.Value),
-		batchSize:   cfg.expMaxBatchSize.Value,
-		pollTrigger: make(chan struct{}, 1),
-		pollKill:    make(chan struct{}),
-	}
+	b.exporter = newBufferExporter(exporter, cfg.expBufferSize.Value)
 	b.pollDone = b.poll(cfg.expInterval.Value)
+
 	return b
 }
 
@@ -143,6 +168,8 @@ func (b *BatchProcessor) poll(interval time.Duration) (done chan struct{}) {
 		defer close(done)
 		defer ticker.Stop()
 
+		ctx := context.Background()
+
 		for {
 			select {
 			case <-ticker.C:
@@ -154,6 +181,9 @@ func (b *BatchProcessor) poll(interval time.Duration) (done chan struct{}) {
 
 			if d := b.q.Dropped(); d > 0 {
 				global.Warn("dropped log records", "dropped", d)
+				if b.inst != nil {
+					b.inst.ProcessedQueueFull(ctx, int64(d))
+				}
 			}
 
 			var qLen int
@@ -220,6 +250,9 @@ func (b *BatchProcessor) Shutdown(ctx context.Context) error {
 
 	// Flush remaining queued before exporter shutdown.
 	err := b.exporter.Export(ctx, b.q.Flush())
+	if b.inst != nil {
+		err = errors.Join(err, b.inst.Shutdown())
+	}
 	return errors.Join(err, b.exporter.Shutdown(ctx))
 }
 

sdk/log/exporter.go

@@ -12,6 +12,7 @@ import (
 	"time"
 
 	"go.opentelemetry.io/otel"
+	"go.opentelemetry.io/otel/sdk/log/internal/observ"
 )
 
 // Exporter handles the delivery of log records to external receivers.
@@ -324,3 +325,26 @@ func (e *bufferExporter) Shutdown(ctx context.Context) error {
 	}
 	return e.Exporter.Shutdown(ctx)
 }
+
+// metricsExporter wraps an Exporter to record log processing metrics
+// just before calling the wrapped exporter.
+type metricsExporter struct {
+	Exporter
+	inst *observ.BLP
+}
+
+// newMetricsExporter creates a metricsExporter that wraps the given exporter.
+func newMetricsExporter(exporter Exporter, inst *observ.BLP) Exporter {
+	return &metricsExporter{
+		Exporter: exporter,
+		inst:     inst,
+	}
+}
+
+// Export records the number of log records as a metric then forwards
+// them to the wrapped Exporter. Error returned from wrapped exporter
+// is not considered as per specification (to be measured by exporter).
+func (e *metricsExporter) Export(ctx context.Context, records []Record) error {
+	e.inst.Processed(ctx, int64(len(records)))
+	return e.Exporter.Export(ctx, records)
+}

sdk/log/internal/gen.go

@@ -6,3 +6,6 @@ package internal // import "go.opentelemetry.io/otel/sdk/log/internal"
 
 //go:generate gotmpl --body=../../../internal/shared/x/x.go.tmpl "--data={ \"pkg\": \"go.opentelemetry.io/otel/sdk/log\" }" --out=x/x.go
 //go:generate gotmpl --body=../../../internal/shared/x/x_test.go.tmpl "--data={}" --out=x/x_test.go
+
+//go:generate gotmpl --body=../../../internal/shared/counter/counter.go.tmpl "--data={ \"pkg\": \"go.opentelemetry.io/otel/sdk/log\" }" --out=counter/counter.go
+//go:generate gotmpl --body=../../../internal/shared/counter/counter_test.go.tmpl "--data={}" --out=counter/counter_test.go

sdk/log/internal/observ/batch_log_processor.go

@@ -0,0 +1,124 @@
+// Copyright The OpenTelemetry Authors
+// SPDX-License-Identifier: Apache-2.0
+
+package observ // import "go.opentelemetry.io/otel/sdk/log/internal/observ"
+
+import (
+	"context"
+	"errors"
+	"fmt"
+
+	"go.opentelemetry.io/otel"
+	"go.opentelemetry.io/otel/attribute"
+	"go.opentelemetry.io/otel/metric"
+	"go.opentelemetry.io/otel/sdk"
+	"go.opentelemetry.io/otel/sdk/log/internal/x"
+	semconv "go.opentelemetry.io/otel/semconv/v1.37.0"
+	"go.opentelemetry.io/otel/semconv/v1.37.0/otelconv"
+)
+
+const (
+	// ScopeName is the name of the instrumentation scope.
+	ScopeName = "go.opentelemetry.io/otel/sdk/log"
+
+	// SchemaURL is the schema URL of the instrumentation.
+	SchemaURL = semconv.SchemaURL
+)
+
+// ErrQueueFull is the attribute value for the "queue_full" error type.
+var ErrQueueFull = otelconv.SDKProcessorLogProcessed{}.AttrErrorType("queue_full")
+
+// BLPComponentName returns the component name attribute for a
+// BatchLogProcessor with the given ID.
+func BLPComponentName(id int64) attribute.KeyValue {
+	t := otelconv.ComponentTypeBatchingLogProcessor
+	name := fmt.Sprintf("%s/%d", t, id)
+	return semconv.OTelComponentName(name)
+}
+
+// BLP is the instrumentation for an OTel SDK BatchLogProcessor.
+type BLP struct {
+	reg metric.Registration
+
+	processed              metric.Int64Counter
+	processedOpts          []metric.AddOption
+	processedQueueFullOpts []metric.AddOption
+}
+
+// NewBLP creates a new BatchLogProcessor instrumentation.
+// Returns nil if observability is not enabled.
+func NewBLP(id int64, qLen func() int64, qMax int64) (*BLP, error) {
+	if !x.Observability.Enabled() {
+		return nil, nil
+	}
+
+	meter := otel.GetMeterProvider().Meter(
+		ScopeName,
+		metric.WithInstrumentationVersion(sdk.Version()),
+		metric.WithSchemaURL(SchemaURL),
+	)
+
+	var err error
+	qCap, e := otelconv.NewSDKProcessorLogQueueCapacity(meter)
+	if e != nil {
+		e = fmt.Errorf("failed to create BLP queue capacity metric: %w", e)
+		err = errors.Join(err, e)
+	}
+	qCapInst := qCap.Inst()
+
+	qSize, e := otelconv.NewSDKProcessorLogQueueSize(meter)
+	if e != nil {
+		e = fmt.Errorf("failed to create BLP queue size metric: %w", e)
+		err = errors.Join(err, e)
+	}
+	qSizeInst := qSize.Inst()
+
+	cmpntT := semconv.OTelComponentTypeBatchingLogProcessor
+	cmpnt := BLPComponentName(id)
+	set := attribute.NewSet(cmpnt, cmpntT)
+
+	// Register callback for async metrics
+	obsOpts := []metric.ObserveOption{metric.WithAttributeSet(set)}
+	reg, e := meter.RegisterCallback(
+		func(_ context.Context, o metric.Observer) error {
+			o.ObserveInt64(qSizeInst, qLen(), obsOpts...)
+			o.ObserveInt64(qCapInst, qMax, obsOpts...)
+			return nil
+		},
+		qSizeInst,
+		qCapInst,
+	)
+	if e != nil {
+		e = fmt.Errorf("failed to register BLP queue size/capacity callback: %w", e)
+		err = errors.Join(err, e)
+	}
+
+	processed, e := otelconv.NewSDKProcessorLogProcessed(meter)
+	if e != nil {
+		e = fmt.Errorf("failed to create BLP processed logs metric: %w", e)
+		err = errors.Join(err, e)
+	}
+
+	processedOpts := []metric.AddOption{metric.WithAttributeSet(set)}
+	setWithError := attribute.NewSet(cmpnt, cmpntT, ErrQueueFull)
+	processedQueueFullOpts := []metric.AddOption{metric.WithAttributeSet(setWithError)}
+
+	return &BLP{
+		reg:                    reg,
+		processed:              processed.Inst(),
+		processedOpts:          processedOpts,
+		processedQueueFullOpts: processedQueueFullOpts,
+	}, err
+}
+
+func (b *BLP) Shutdown() error {
+	return b.reg.Unregister()
+}
+
+func (b *BLP) Processed(ctx context.Context, n int64) {
+	b.processed.Add(ctx, n, b.processedOpts...)
+}
+
+func (b *BLP) ProcessedQueueFull(ctx context.Context, n int64) {
+	b.processed.Add(ctx, n, b.processedQueueFullOpts...)
+}