diff --git a/stats/opentelemetry/client_metrics.go b/stats/opentelemetry/client_metrics.go index ee778080c251..13df118d8da9 100644 --- a/stats/opentelemetry/client_metrics.go +++ b/stats/opentelemetry/client_metrics.go @@ -76,8 +76,9 @@ func getOrCreateCallInfo(ctx context.Context, cc *grpc.ClientConn, method string logger.Info("Creating new CallInfo since its not present in context") } ci = &callInfo{ - target: cc.CanonicalTarget(), - method: determineMethod(method, opts...), + target: cc.CanonicalTarget(), + method: determineMethod(method, opts...), + previousRPCAttempts: new(atomic.Uint32), } ctx = setCallInfo(ctx, ci) } diff --git a/stats/opentelemetry/client_tracing.go b/stats/opentelemetry/client_tracing.go index 868d6a2fc9c1..6b725fb67366 100644 --- a/stats/opentelemetry/client_tracing.go +++ b/stats/opentelemetry/client_tracing.go @@ -21,6 +21,7 @@ import ( "log" "strings" + "go.opentelemetry.io/otel/attribute" otelcodes "go.opentelemetry.io/otel/codes" "go.opentelemetry.io/otel/trace" "google.golang.org/grpc" @@ -83,7 +84,10 @@ func (h *clientTracingHandler) finishTrace(err error, ts trace.Span) { // It creates a new outgoing carrier which serializes information about this // span into gRPC Metadata, if TextMapPropagator is provided in the trace // options. if TextMapPropagator is not provided, it returns the context as is. -func (h *clientTracingHandler) traceTagRPC(ctx context.Context, ai *attemptInfo, nameResolutionDelayed bool) (context.Context, *attemptInfo) { +// +// Note: The passed attemptInfo pointer (ai) is mutated in-place. Fields such as +// ai.traceSpan are updated directly. No new attemptInfo is returned. +func (h *clientTracingHandler) traceTagRPC(ctx context.Context, ai *attemptInfo, nameResolutionDelayed bool) context.Context { // Add a "Delayed name resolution complete" event to the call span // if there was name resolution delay. In case of multiple retry attempts, // ensure that event is added only once. @@ -98,7 +102,7 @@ func (h *clientTracingHandler) traceTagRPC(ctx context.Context, ai *attemptInfo, carrier := otelinternaltracing.NewOutgoingCarrier(ctx) h.options.TraceOptions.TextMapPropagator.Inject(ctx, carrier) ai.traceSpan = span - return carrier.Context(), ai + return carrier.Context() } // createCallTraceSpan creates a call span to put in the provided context using @@ -121,7 +125,12 @@ func (h *clientTracingHandler) HandleConn(context.Context, stats.ConnStats) {} // TagRPC implements per RPC attempt context management for traces. func (h *clientTracingHandler) TagRPC(ctx context.Context, info *stats.RPCTagInfo) context.Context { ctx, ai := getOrCreateRPCAttemptInfo(ctx) - ctx, ai = h.traceTagRPC(ctx, ai, info.NameResolutionDelay) + ci := getCallInfo(ctx) + if ci == nil { + logger.Error("context passed into client side stats handler (TagRPC) has no call info") + return ctx + } + ctx = h.traceTagRPC(ctx, ai, info.NameResolutionDelay) return setRPCInfo(ctx, &rpcInfo{ai: ai}) } @@ -132,5 +141,15 @@ func (h *clientTracingHandler) HandleRPC(ctx context.Context, rs stats.RPCStats) logger.Error("ctx passed into client side tracing handler trace event handling has no client attempt data present") return } + + // Client-specific Begin attributes. + if begin, ok := rs.(*stats.Begin); ok { + ci := getCallInfo(ctx) + previousRPCAttempts := ci.previousRPCAttempts.Add(1) - 1 + ri.ai.traceSpan.SetAttributes( + attribute.Int64("previous-rpc-attempts", int64(previousRPCAttempts)), + attribute.Bool("transparent-retry", begin.IsTransparentRetryAttempt), + ) + } populateSpan(rs, ri.ai) } diff --git a/stats/opentelemetry/e2e_test.go b/stats/opentelemetry/e2e_test.go index 2d575bfe06c5..9f345517e8c7 100644 --- a/stats/opentelemetry/e2e_test.go +++ b/stats/opentelemetry/e2e_test.go @@ -89,13 +89,7 @@ type traceSpanInfo struct { name string events []trace.Event attributes []attribute.KeyValue -} - -// traceSpanInfoMapKey is the key struct for constructing a map of trace spans -// retrievable by span name and span kind -type traceSpanInfoMapKey struct { - spanName string - spanKind string + status otelcodes.Code } // defaultMetricsOptions creates default metrics options @@ -276,46 +270,33 @@ func validateTraces(t *testing.T, spans tracetest.SpanStubs, wantSpanInfos []tra } } - // Constructs a map from a slice of traceSpanInfo to retrieve the - // corresponding expected span info based on span name and span kind - // for comparison. - wantSpanInfosMap := make(map[traceSpanInfoMapKey]traceSpanInfo) - for _, info := range wantSpanInfos { - key := traceSpanInfoMapKey{spanName: info.name, spanKind: info.spanKind} - wantSpanInfosMap[key] = info - } - - // Compare retrieved spans with expected spans. - for _, span := range spans { - // Check that the attempt span has the correct status. - if got, want := span.Status.Code, otelcodes.Ok; got != want { - t.Errorf("Got status code %v, want %v", got, want) - } - - // Retrieve the corresponding expected span info based on span name and - // span kind to compare. - want, ok := wantSpanInfosMap[traceSpanInfoMapKey{spanName: span.Name, spanKind: span.SpanKind.String()}] - if !ok { - t.Errorf("Unexpected span: %v", span) - continue - } - - // comparers - attributesSort := cmpopts.SortSlices(func(a, b attribute.KeyValue) bool { - return a.Key < b.Key - }) - attributesValueComparable := cmpopts.EquateComparable(attribute.KeyValue{}.Value) - eventsTimeIgnore := cmpopts.IgnoreFields(trace.Event{}, "Time") - - // attributes - if diff := cmp.Diff(want.attributes, span.Attributes, attributesSort, attributesValueComparable); diff != "" { - t.Errorf("Attributes mismatch for span %s (-want +got):\n%s", span.Name, diff) - } - // events - if diff := cmp.Diff(want.events, span.Events, attributesSort, attributesValueComparable, eventsTimeIgnore); diff != "" { - t.Errorf("Events mismatch for span %s (-want +got):\n%s", span.Name, diff) + // Convert spans to traceSpanInfo for cmp.Diff comparison. + actualSpanInfos := make([]traceSpanInfo, len(spans)) + for i, s := range spans { + actualSpanInfos[i] = traceSpanInfo{ + name: s.Name, + spanKind: s.SpanKind.String(), + attributes: s.Attributes, + events: s.Events, + status: s.Status.Code, } } + opts := []cmp.Option{ + cmpopts.SortSlices(func(a, b traceSpanInfo) bool { + if a.name == b.name { + return a.spanKind < b.spanKind + } + return a.name < b.name + }), + cmpopts.SortSlices(func(a, b trace.Event) bool { return a.Name < b.Name }), + cmpopts.IgnoreFields(trace.Event{}, "Time"), + cmpopts.EquateComparable(attribute.KeyValue{}, attribute.Value{}, attribute.Set{}), + cmpopts.IgnoreFields(tracetest.SpanStub{}, "InstrumentationScope"), + cmp.AllowUnexported(traceSpanInfo{}), + } + if diff := cmp.Diff(wantSpanInfos, actualSpanInfos, opts...); diff != "" { + t.Errorf("Spans mismatch (-want +got):\n%s", diff) + } } // TestMethodAttributeFilter tests the method attribute filter. The method @@ -857,26 +838,10 @@ func (s) TestMetricsAndTracesOptionEnabled(t *testing.T) { wantSpanInfos := []traceSpanInfo{ { - name: "Recv.grpc.testing.TestService.UnaryCall", - spanKind: oteltrace.SpanKindServer.String(), - attributes: []attribute.KeyValue{ - { - Key: "Client", - Value: attribute.BoolValue(false), - }, - { - Key: "FailFast", - Value: attribute.BoolValue(false), - }, - { - Key: "previous-rpc-attempts", - Value: attribute.IntValue(0), - }, - { - Key: "transparent-retry", - Value: attribute.BoolValue(false), - }, - }, + name: "Recv.grpc.testing.TestService.UnaryCall", + spanKind: oteltrace.SpanKindServer.String(), + status: otelcodes.Ok, + attributes: nil, events: []trace.Event{ { Name: "Inbound message", @@ -917,15 +882,8 @@ func (s) TestMetricsAndTracesOptionEnabled(t *testing.T) { { name: "Attempt.grpc.testing.TestService.UnaryCall", spanKind: oteltrace.SpanKindInternal.String(), + status: otelcodes.Ok, attributes: []attribute.KeyValue{ - { - Key: "Client", - Value: attribute.BoolValue(true), - }, - { - Key: "FailFast", - Value: attribute.BoolValue(true), - }, { Key: "previous-rpc-attempts", Value: attribute.IntValue(0), @@ -975,50 +933,29 @@ func (s) TestMetricsAndTracesOptionEnabled(t *testing.T) { { name: "Sent.grpc.testing.TestService.UnaryCall", spanKind: oteltrace.SpanKindClient.String(), + status: otelcodes.Ok, attributes: nil, events: nil, }, { - name: "Recv.grpc.testing.TestService.FullDuplexCall", - spanKind: oteltrace.SpanKindServer.String(), - attributes: []attribute.KeyValue{ - { - Key: "Client", - Value: attribute.BoolValue(false), - }, - { - Key: "FailFast", - Value: attribute.BoolValue(false), - }, - { - Key: "previous-rpc-attempts", - Value: attribute.IntValue(0), - }, - { - Key: "transparent-retry", - Value: attribute.BoolValue(false), - }, - }, - events: nil, + name: "Recv.grpc.testing.TestService.FullDuplexCall", + spanKind: oteltrace.SpanKindServer.String(), + status: otelcodes.Ok, + attributes: nil, + events: nil, }, { name: "Sent.grpc.testing.TestService.FullDuplexCall", spanKind: oteltrace.SpanKindClient.String(), + status: otelcodes.Ok, attributes: nil, events: nil, }, { name: "Attempt.grpc.testing.TestService.FullDuplexCall", spanKind: oteltrace.SpanKindInternal.String(), + status: otelcodes.Ok, attributes: []attribute.KeyValue{ - { - Key: "Client", - Value: attribute.BoolValue(true), - }, - { - Key: "FailFast", - Value: attribute.BoolValue(true), - }, { Key: "previous-rpc-attempts", Value: attribute.IntValue(0), @@ -1081,26 +1018,10 @@ func (s) TestSpan(t *testing.T) { wantSpanInfos := []traceSpanInfo{ { - name: "Recv.grpc.testing.TestService.UnaryCall", - spanKind: oteltrace.SpanKindServer.String(), - attributes: []attribute.KeyValue{ - { - Key: "Client", - Value: attribute.BoolValue(false), - }, - { - Key: "FailFast", - Value: attribute.BoolValue(false), - }, - { - Key: "previous-rpc-attempts", - Value: attribute.IntValue(0), - }, - { - Key: "transparent-retry", - Value: attribute.BoolValue(false), - }, - }, + name: "Recv.grpc.testing.TestService.UnaryCall", + spanKind: oteltrace.SpanKindServer.String(), + status: otelcodes.Ok, + attributes: nil, events: []trace.Event{ { Name: "Inbound message", @@ -1133,15 +1054,8 @@ func (s) TestSpan(t *testing.T) { { name: "Attempt.grpc.testing.TestService.UnaryCall", spanKind: oteltrace.SpanKindInternal.String(), + status: otelcodes.Ok, attributes: []attribute.KeyValue{ - { - Key: "Client", - Value: attribute.BoolValue(true), - }, - { - Key: "FailFast", - Value: attribute.BoolValue(true), - }, { Key: "previous-rpc-attempts", Value: attribute.IntValue(0), @@ -1183,50 +1097,29 @@ func (s) TestSpan(t *testing.T) { { name: "Sent.grpc.testing.TestService.UnaryCall", spanKind: oteltrace.SpanKindClient.String(), + status: otelcodes.Ok, attributes: nil, events: nil, }, { - name: "Recv.grpc.testing.TestService.FullDuplexCall", - spanKind: oteltrace.SpanKindServer.String(), - attributes: []attribute.KeyValue{ - { - Key: "Client", - Value: attribute.BoolValue(false), - }, - { - Key: "FailFast", - Value: attribute.BoolValue(false), - }, - { - Key: "previous-rpc-attempts", - Value: attribute.IntValue(0), - }, - { - Key: "transparent-retry", - Value: attribute.BoolValue(false), - }, - }, - events: nil, + name: "Recv.grpc.testing.TestService.FullDuplexCall", + spanKind: oteltrace.SpanKindServer.String(), + status: otelcodes.Ok, + attributes: nil, + events: nil, }, { name: "Sent.grpc.testing.TestService.FullDuplexCall", spanKind: oteltrace.SpanKindClient.String(), + status: otelcodes.Ok, attributes: nil, events: nil, }, { name: "Attempt.grpc.testing.TestService.FullDuplexCall", spanKind: oteltrace.SpanKindInternal.String(), + status: otelcodes.Ok, attributes: []attribute.KeyValue{ - { - Key: "Client", - Value: attribute.BoolValue(true), - }, - { - Key: "FailFast", - Value: attribute.BoolValue(true), - }, { Key: "previous-rpc-attempts", Value: attribute.IntValue(0), @@ -1291,26 +1184,10 @@ func (s) TestSpan_WithW3CContextPropagator(t *testing.T) { wantSpanInfos := []traceSpanInfo{ { - name: "Recv.grpc.testing.TestService.UnaryCall", - spanKind: oteltrace.SpanKindServer.String(), - attributes: []attribute.KeyValue{ - { - Key: "Client", - Value: attribute.BoolValue(false), - }, - { - Key: "FailFast", - Value: attribute.BoolValue(false), - }, - { - Key: "previous-rpc-attempts", - Value: attribute.IntValue(0), - }, - { - Key: "transparent-retry", - Value: attribute.BoolValue(false), - }, - }, + name: "Recv.grpc.testing.TestService.UnaryCall", + spanKind: oteltrace.SpanKindServer.String(), + status: otelcodes.Ok, + attributes: nil, events: []trace.Event{ { Name: "Inbound message", @@ -1343,15 +1220,8 @@ func (s) TestSpan_WithW3CContextPropagator(t *testing.T) { { name: "Attempt.grpc.testing.TestService.UnaryCall", spanKind: oteltrace.SpanKindInternal.String(), + status: otelcodes.Ok, attributes: []attribute.KeyValue{ - { - Key: "Client", - Value: attribute.BoolValue(true), - }, - { - Key: "FailFast", - Value: attribute.BoolValue(true), - }, { Key: "previous-rpc-attempts", Value: attribute.IntValue(0), @@ -1393,50 +1263,29 @@ func (s) TestSpan_WithW3CContextPropagator(t *testing.T) { { name: "Sent.grpc.testing.TestService.UnaryCall", spanKind: oteltrace.SpanKindClient.String(), + status: otelcodes.Ok, attributes: nil, events: nil, }, { - name: "Recv.grpc.testing.TestService.FullDuplexCall", - spanKind: oteltrace.SpanKindServer.String(), - attributes: []attribute.KeyValue{ - { - Key: "Client", - Value: attribute.BoolValue(false), - }, - { - Key: "FailFast", - Value: attribute.BoolValue(false), - }, - { - Key: "previous-rpc-attempts", - Value: attribute.IntValue(0), - }, - { - Key: "transparent-retry", - Value: attribute.BoolValue(false), - }, - }, - events: nil, + name: "Recv.grpc.testing.TestService.FullDuplexCall", + spanKind: oteltrace.SpanKindServer.String(), + status: otelcodes.Ok, + attributes: nil, + events: nil, }, { name: "Sent.grpc.testing.TestService.FullDuplexCall", spanKind: oteltrace.SpanKindClient.String(), + status: otelcodes.Ok, attributes: nil, events: nil, }, { name: "Attempt.grpc.testing.TestService.FullDuplexCall", spanKind: oteltrace.SpanKindInternal.String(), + status: otelcodes.Ok, attributes: []attribute.KeyValue{ - { - Key: "Client", - Value: attribute.BoolValue(true), - }, - { - Key: "FailFast", - Value: attribute.BoolValue(true), - }, { Key: "previous-rpc-attempts", Value: attribute.IntValue(0), @@ -1550,10 +1399,10 @@ const delayedResolutionEventName = "Delayed name resolution complete" // only once if any of the retry attempt encountered a delay in name resolution func (s) TestTraceSpan_WithRetriesAndNameResolutionDelay(t *testing.T) { tests := []struct { - name string - setupStub func() *stubserver.StubServer - doCall func(context.Context, testgrpc.TestServiceClient) error - spanName string + name string + setupStub func() *stubserver.StubServer + doCall func(context.Context, testgrpc.TestServiceClient) error + wantSpanInfos []traceSpanInfo }{ { name: "unary", @@ -1576,7 +1425,136 @@ func (s) TestTraceSpan_WithRetriesAndNameResolutionDelay(t *testing.T) { _, err := client.UnaryCall(ctx, &testpb.SimpleRequest{}) return err }, - spanName: "Sent.grpc.testing.TestService.UnaryCall", + wantSpanInfos: []traceSpanInfo{ + { + name: "Recv.grpc.testing.TestService.UnaryCall", + spanKind: oteltrace.SpanKindServer.String(), + status: otelcodes.Error, + attributes: nil, + events: []trace.Event{ + { + Name: "Inbound message", + Attributes: []attribute.KeyValue{ + attribute.Int("sequence-number", 0), + attribute.Int("message-size", 0), + }, + }, + }, + }, + // RPC attempt #1 + { + name: "Attempt.grpc.testing.TestService.UnaryCall", + spanKind: oteltrace.SpanKindInternal.String(), + status: otelcodes.Error, + attributes: []attribute.KeyValue{ + attribute.Int("previous-rpc-attempts", 0), + attribute.Bool("transparent-retry", false), + }, + events: []trace.Event{ + { + Name: "Delayed LB pick complete", + }, + { + Name: "Outbound message", + Attributes: []attribute.KeyValue{ + attribute.Int("sequence-number", 0), + attribute.Int("message-size", 0), + }, + }, + }, + }, + { + name: "Recv.grpc.testing.TestService.UnaryCall", + spanKind: oteltrace.SpanKindServer.String(), + status: otelcodes.Error, + attributes: nil, + events: []trace.Event{ + { + Name: "Inbound message", + Attributes: []attribute.KeyValue{ + attribute.Int("sequence-number", 0), + attribute.Int("message-size", 0), + }, + }, + }, + }, + // RPC attempt #2 + { + name: "Attempt.grpc.testing.TestService.UnaryCall", + spanKind: oteltrace.SpanKindInternal.String(), + status: otelcodes.Error, + attributes: []attribute.KeyValue{ + attribute.Int("previous-rpc-attempts", 1), + attribute.Bool("transparent-retry", false), + }, + events: []trace.Event{ + { + Name: "Outbound message", + Attributes: []attribute.KeyValue{ + attribute.Int("sequence-number", 0), + attribute.Int("message-size", 0), + }, + }, + }, + }, + { + name: "Recv.grpc.testing.TestService.UnaryCall", + spanKind: oteltrace.SpanKindServer.String(), + status: otelcodes.Ok, + attributes: nil, + events: []trace.Event{ + { + Name: "Inbound message", + Attributes: []attribute.KeyValue{ + attribute.Int("sequence-number", 0), + attribute.Int("message-size", 0), + }, + }, + { + Name: "Outbound message", + Attributes: []attribute.KeyValue{ + attribute.Int("sequence-number", 0), + attribute.Int("message-size", 0), + }, + }, + }, + }, + // RPC attempt #3 + { + name: "Attempt.grpc.testing.TestService.UnaryCall", + spanKind: oteltrace.SpanKindInternal.String(), + status: otelcodes.Ok, + attributes: []attribute.KeyValue{ + attribute.Int("previous-rpc-attempts", 2), + attribute.Bool("transparent-retry", false), + }, + events: []trace.Event{ + { + Name: "Outbound message", + Attributes: []attribute.KeyValue{ + attribute.Int("sequence-number", 0), + attribute.Int("message-size", 0), + }, + }, + { + Name: "Inbound message", + Attributes: []attribute.KeyValue{ + attribute.Int("sequence-number", 0), + attribute.Int("message-size", 0), + }, + }, + }, + }, + { + name: "Sent.grpc.testing.TestService.UnaryCall", + spanKind: oteltrace.SpanKindClient.String(), + status: otelcodes.Ok, + attributes: nil, + events: []trace.Event{ + {Name: delayedResolutionEventName}, + }, + }, + }, }, { name: "streaming", @@ -1620,7 +1598,106 @@ func (s) TestTraceSpan_WithRetriesAndNameResolutionDelay(t *testing.T) { } return nil }, - spanName: "Sent.grpc.testing.TestService.FullDuplexCall", + wantSpanInfos: []traceSpanInfo{ + { + name: "Recv.grpc.testing.TestService.FullDuplexCall", + spanKind: oteltrace.SpanKindServer.String(), + status: otelcodes.Error, + attributes: nil, + events: nil, + }, + // RPC attempt #1 + { + name: "Attempt.grpc.testing.TestService.FullDuplexCall", + spanKind: oteltrace.SpanKindInternal.String(), + status: otelcodes.Error, + attributes: []attribute.KeyValue{ + attribute.Int("previous-rpc-attempts", 0), + attribute.Bool("transparent-retry", false), + }, + events: []trace.Event{ + { + Name: "Delayed LB pick complete", + }, + { + Name: "Outbound message", + Attributes: []attribute.KeyValue{ + attribute.Int("sequence-number", 0), + attribute.Int("message-size", 0), + }, + }, + }, + }, + { + name: "Recv.grpc.testing.TestService.FullDuplexCall", + spanKind: oteltrace.SpanKindServer.String(), + status: otelcodes.Error, + attributes: nil, + events: nil, + }, + // RPC attempt #2 + { + name: "Attempt.grpc.testing.TestService.FullDuplexCall", + spanKind: oteltrace.SpanKindInternal.String(), + status: otelcodes.Error, + attributes: []attribute.KeyValue{ + attribute.Int("previous-rpc-attempts", 1), + attribute.Bool("transparent-retry", false), + }, + events: []trace.Event{ + { + Name: "Outbound message", + Attributes: []attribute.KeyValue{ + attribute.Int("sequence-number", 0), + attribute.Int("message-size", 0), + }, + }, + }, + }, + { + name: "Recv.grpc.testing.TestService.FullDuplexCall", + spanKind: oteltrace.SpanKindServer.String(), + status: otelcodes.Ok, + attributes: nil, + events: []trace.Event{ + { + Name: "Inbound message", + Attributes: []attribute.KeyValue{ + attribute.Int("sequence-number", 0), + attribute.Int("message-size", 0), + }, + }, + }, + }, + // RPC attempt #3 + { + name: "Attempt.grpc.testing.TestService.FullDuplexCall", + spanKind: oteltrace.SpanKindInternal.String(), + status: otelcodes.Ok, + attributes: []attribute.KeyValue{ + attribute.Int("previous-rpc-attempts", 2), + attribute.Bool("transparent-retry", false), + }, + events: []trace.Event{ + { + Name: "Outbound message", + Attributes: []attribute.KeyValue{ + attribute.Int("sequence-number", 0), + attribute.Int("message-size", 0), + }, + }, + }, + }, + { + name: "Sent.grpc.testing.TestService.FullDuplexCall", + spanKind: oteltrace.SpanKindClient.String(), + status: otelcodes.Ok, + attributes: nil, + events: []trace.Event{ + {Name: delayedResolutionEventName}, + }, + }, + }, }, } @@ -1676,38 +1753,15 @@ func (s) TestTraceSpan_WithRetriesAndNameResolutionDelay(t *testing.T) { if err := tt.doCall(ctx, client); err != nil { t.Fatalf("%s call failed: %v", tt.name, err) } - - wantSpanInfo := traceSpanInfo{ - name: tt.spanName, - spanKind: oteltrace.SpanKindClient.String(), - events: []trace.Event{{Name: delayedResolutionEventName}}, - } - spans, err := waitForTraceSpans(ctx, exporter, []traceSpanInfo{wantSpanInfo}) + spans, err := waitForTraceSpans(ctx, exporter, tt.wantSpanInfos) if err != nil { t.Fatal(err) } - verifyTrace(t, spans, wantSpanInfo) + validateTraces(t, spans, tt.wantSpanInfos) }) } } -func verifyTrace(t *testing.T, spans tracetest.SpanStubs, want traceSpanInfo) { - match := false - for _, span := range spans { - if span.Name == want.name && span.SpanKind.String() == want.spanKind { - match = true - if diff := cmp.Diff(want.events, span.Events, cmpopts.IgnoreFields(trace.Event{}, "Time")); diff != "" { - t.Errorf("Span event mismatch for %q (kind: %s) (-want +got):\n%s", - want.name, want.spanKind, diff) - } - break - } - } - if !match { - t.Errorf("Expected span not found: %q (kind: %s)", want.name, want.spanKind) - } -} - // TestStreamingRPC_TraceSequenceNumbers verifies that sequence numbers // are incremented correctly for multiple messages sent and received // during a streaming RPC. @@ -1756,27 +1810,23 @@ func (s) TestStreamingRPC_TraceSequenceNumbers(t *testing.T) { { name: "Sent.grpc.testing.TestService.FullDuplexCall", spanKind: oteltrace.SpanKindClient.String(), + status: otelcodes.Ok, events: nil, attributes: nil, }, { - name: "Recv.grpc.testing.TestService.FullDuplexCall", - spanKind: oteltrace.SpanKindServer.String(), - events: wantInboundEvents, - attributes: []attribute.KeyValue{ - attribute.Bool("Client", false), - attribute.Bool("FailFast", false), - attribute.Int("previous-rpc-attempts", 0), - attribute.Bool("transparent-retry", false), - }, + name: "Recv.grpc.testing.TestService.FullDuplexCall", + spanKind: oteltrace.SpanKindServer.String(), + status: otelcodes.Ok, + events: wantInboundEvents, + attributes: nil, }, { name: "Attempt.grpc.testing.TestService.FullDuplexCall", spanKind: oteltrace.SpanKindInternal.String(), + status: otelcodes.Ok, events: wantOutboundEvents, attributes: []attribute.KeyValue{ - attribute.Bool("Client", true), - attribute.Bool("FailFast", true), attribute.Int("previous-rpc-attempts", 0), attribute.Bool("transparent-retry", false), }, diff --git a/stats/opentelemetry/opentelemetry.go b/stats/opentelemetry/opentelemetry.go index cd01f86c4981..b97625d9ad64 100644 --- a/stats/opentelemetry/opentelemetry.go +++ b/stats/opentelemetry/opentelemetry.go @@ -179,6 +179,9 @@ type callInfo struct { // nameResolutionEventAdded is set when the resolver delay trace event // is added. Prevents duplicate events, since it is reported per-attempt. nameResolutionEventAdded atomic.Bool + // previousRPCAttempts holds the count of RPC attempts that have happened + // before current attempt. Transparent retries are excluded. + previousRPCAttempts *atomic.Uint32 } type callInfoKey struct{} @@ -239,9 +242,8 @@ type attemptInfo struct { // message counters for sent and received messages (used for // generating message IDs), and the number of previous RPC attempts for the // associated call. - countSentMsg uint32 - countRecvMsg uint32 - previousRPCAttempts uint32 + countSentMsg uint32 + countRecvMsg uint32 } type clientMetrics struct { diff --git a/stats/opentelemetry/trace.go b/stats/opentelemetry/trace.go index 40ac7a1b6ef5..3ee66d1e8cc7 100644 --- a/stats/opentelemetry/trace.go +++ b/stats/opentelemetry/trace.go @@ -17,8 +17,6 @@ package opentelemetry import ( - "sync/atomic" - "go.opentelemetry.io/otel/attribute" otelcodes "go.opentelemetry.io/otel/codes" "go.opentelemetry.io/otel/trace" @@ -40,18 +38,6 @@ func populateSpan(rs stats.RPCStats, ai *attemptInfo) { span := ai.traceSpan switch rs := rs.(type) { - case *stats.Begin: - // Note: Go always added Client and FailFast attributes even though they are not - // defined by the OpenCensus gRPC spec. Thus, they are unimportant for - // correctness. - span.SetAttributes( - attribute.Bool("Client", rs.Client), - attribute.Bool("FailFast", rs.FailFast), - attribute.Int64("previous-rpc-attempts", int64(ai.previousRPCAttempts)), - attribute.Bool("transparent-retry", rs.IsTransparentRetryAttempt), - ) - // increment previous rpc attempts applicable for next attempt - atomic.AddUint32(&ai.previousRPCAttempts, 1) case *stats.DelayedPickComplete: span.AddEvent("Delayed LB pick complete") case *stats.InPayload: