feat(tracing): skip OTLP error recording for selected errors

internal/controller/ledger/controller_default.go

@@ -368,8 +368,10 @@ func (ctrl *DefaultController) importLog(ctx context.Context, store Store, log l
 
 			return nil, nil
 		},
-		trace.WithNewRoot(),
-		trace.WithLinks(trace.LinkFromContext(ctx)),
+		tracing.WithSpanStartOptions(
+			trace.WithNewRoot(),
+			trace.WithLinks(trace.LinkFromContext(ctx)),
+		),
 	)
 	return err
 }

internal/storage/ledger/logs.go

@@ -132,5 +132,6 @@ func (store *Store) ReadLogWithIdempotencyKey(ctx context.Context, key string) (
 
 			return pointer.For(ret.ToCore()), nil
 		},
+		tracing.SkipErrorRecordingIf(postgres.IsNotFoundError),
 	)
 }

internal/storage/ledger/transactions.go

@@ -137,14 +137,12 @@ func (store *Store) InsertTransaction(ctx context.Context, tx *ledger.Transactio
 				}
 			}
 
-			return tx, nil
-		},
-		func(ctx context.Context, tx *ledger.Transaction) {
 			trace.SpanFromContext(ctx).SetAttributes(
 				attribute.String("transaction.id", fmt.Sprint(tx.ID)),
 				attribute.String("transaction.timestamp", tx.Timestamp.Format(time.RFC3339Nano)),
 				attribute.String("transaction.reference", tx.Reference),
 			)
+			return tx, nil
 		},
 	))
 }

internal/tracing/tracing.go

@@ -104,7 +104,7 @@ func TraceWithMetric[RET any](
 	tracer trace.Tracer,
 	histogram metric.Int64Histogram,
 	fn func(ctx context.Context) (RET, error),
-	finalizers ...func(ctx context.Context, ret RET),
+	opts ...TraceOption,
 ) (RET, error) {
 	var zeroRet RET
 
@@ -120,11 +120,45 @@ func TraceWithMetric[RET any](
 		histogram.Record(ctx, latency.Milliseconds())
 		trace.SpanFromContext(ctx).SetAttributes(attribute.String("latency", latency.String()))
 
-		for _, finalizer := range finalizers {
-			finalizer(ctx, ret)
+		return ret, nil
+	}, opts...)
+}
+
+type traceSettings struct {
+	spanStartOptions   []trace.SpanStartOption
+	skipRecordErrorIfs []func(error) bool
+}
+
+func (s *traceSettings) shouldSkipErrorRecording(err error) bool {
+	for _, predicate := range s.skipRecordErrorIfs {
+		if predicate(err) {
+			return true
 		}
+	}
+	return false
+}
 
-		return ret, nil
+// TraceOption configures Trace and TraceWithMetric.
+type TraceOption interface {
+	applyTrace(*traceSettings)
+}
+
+type traceOptionFunc func(*traceSettings)
+
+func (f traceOptionFunc) applyTrace(s *traceSettings) { f(s) }
+
+// WithSpanStartOptions passes options to tracer.Start (for example trace.WithAttributes).
+func WithSpanStartOptions(opts ...trace.SpanStartOption) TraceOption {
+	return traceOptionFunc(func(s *traceSettings) {
+		s.spanStartOptions = append(s.spanStartOptions, opts...)
+	})
+}
+
+// SkipErrorRecordingIf skips otlp.RecordError when predicate returns true for the returned error.
+// The error is still returned to the caller unchanged. Multiple options compose with OR semantics.
+func SkipErrorRecordingIf(predicate func(error) bool) TraceOption {
+	return traceOptionFunc(func(s *traceSettings) {
+		s.skipRecordErrorIfs = append(s.skipRecordErrorIfs, predicate)
 	})
 }
 
@@ -133,14 +167,21 @@ func Trace[RET any](
 	tracer trace.Tracer,
 	name string,
 	fn func(ctx context.Context) (RET, error),
-	spanOptions ...trace.SpanStartOption,
+	opts ...TraceOption,
 ) (RET, error) {
-	ctx, span := tracer.Start(ctx, name, spanOptions...)
+	var settings traceSettings
+	for _, o := range opts {
+		o.applyTrace(&settings)
+	}
+
+	ctx, span := tracer.Start(ctx, name, settings.spanStartOptions...)
 	defer span.End()
 
 	ret, err := fn(ctx)
 	if err != nil {
-		otlp.RecordError(ctx, err)
+		if !settings.shouldSkipErrorRecording(err) {
+			otlp.RecordError(ctx, err)
+		}
 		return ret, err
 	}