Skip to content

chore(x-goog-spanner-request-id): set span attribute x_goog_spanner_request_id #3905

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Open
wants to merge 5 commits into
base: main
Choose a base branch
from
Open

chore(x-goog-spanner-request-id): set span attribute x_goog_spanner_request_id #3905

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
5 commits
Select commit Hold shift + click to select a range
436cf2e
chore(x-goog-spanner-request-id): set span attribute x_goog_spanner_r…
odeke-em Jun 5, 2025
625c7dd
Simply matcher and usert assertTrue
odeke-em Jun 11, 2025
db91c93
Merge branch 'main' into x-goog-spanner-request-id-set-span-attribute
odeke-em Jul 16, 2025
ecf7d97
Merge branch 'main' into x-goog-spanner-request-id-set-span-attribute
odeke-em Jul 29, 2025
9ce0799
enabled tests, sort requestIds before asserting monotonicity
tharoldD Aug 1, 2025
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
4 changes: 4 additions & 0 deletions google-cloud-spanner/src/main/java/com/google/cloud/spanner/XGoogSpannerRequestId.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -198,4 +198,8 @@ XGoogSpannerRequestId withNthClientId(long replacementClientId) {
return XGoogSpannerRequestId.of(
replacementClientId, this.nthChannelId, this.nthRequest, this.attempt);
}

public static String getRequestIdFromMetadata(Metadata md) {
return md.get(REQUEST_HEADER_KEY);
}
}
5 changes: 5 additions & 0 deletions google-cloud-spanner/src/main/java/com/google/cloud/spanner/spi/v1/HeaderInterceptor.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -28,6 +28,7 @@
import com.google.cloud.spanner.CompositeTracer;
import com.google.cloud.spanner.SpannerExceptionFactory;
import com.google.cloud.spanner.SpannerRpcMetrics;
import com.google.cloud.spanner.XGoogSpannerRequestId;
import com.google.common.cache.Cache;
import com.google.common.cache.CacheBuilder;
import com.google.spanner.admin.database.v1.DatabaseName;
Expand Down Expand Up @@ -174,6 +175,10 @@ private void processHeader(
}
if (span != null) {
span.setAttribute("gfe_latency", String.valueOf(gfeLatency));
String reqId = XGoogSpannerRequestId.getRequestIdFromMetadata(metadata);
Copy link
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

@odeke-em Have you tested these changes, I dont think it will work, since request_id is a request header and not response header.

I have added the similar changes in Exemplar work #3997

Copy link
Contributor Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

We encountered an oddity that the Java Spanner client tests for OpenTelemetry only work for certain versions of Java as @olavloite mentioned in our group, so even testing anything requires y'all to help me run the bots /cc @tharoldD to help out with the updates.

if (reqId != null) {
span.setAttribute("x_goog_spanner_request_id", reqId);
}
}
} else {
measureMap.put(SPANNER_GFE_HEADER_MISSING_COUNT, 1L).record(tagContext);
Expand Down
26 changes: 7 additions & 19 deletions google-cloud-spanner/src/test/java/com/google/cloud/spanner/DatabaseClientImplTest.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -1417,8 +1417,7 @@ public void testWriteAtLeastOnceAborted() {

List<CommitRequest> commitRequests = mockSpanner.getRequestsOfType(CommitRequest.class);
assertEquals(2, commitRequests.size());
// TODO(@odeke-em): Enable in later PR.
// xGoogReqIdInterceptor.assertIntegrity();
xGoogReqIdInterceptor.assertIntegrity();
}

@Test
Expand Down Expand Up @@ -2920,9 +2919,7 @@ public void testPartitionedDmlDoesNotTimeout() {
"google.spanner.v1.Spanner/ExecuteStreamingSql",
new XGoogSpannerRequestId(NON_DETERMINISTIC, channelId, 6, 1)),
};
if (false) { // TODO(@odeke-em): enable in next PRs.
xGoogReqIdInterceptor.checkExpectedStreamingXGoogRequestIds(wantStreamingValues);
}
xGoogReqIdInterceptor.checkExpectedStreamingXGoogRequestIds(wantStreamingValues);

XGoogSpannerRequestIdTest.MethodAndRequestId[] wantUnaryValues = {
XGoogSpannerRequestIdTest.ofMethodAndRequestId(
Expand All @@ -2935,9 +2932,7 @@ public void testPartitionedDmlDoesNotTimeout() {
"google.spanner.v1.Spanner/ExecuteSql",
new XGoogSpannerRequestId(NON_DETERMINISTIC, channelId, 8, 1)),
};
if (false) { // TODO(@odeke-em): enable in next PRs.
xGoogReqIdInterceptor.checkExpectedUnaryXGoogRequestIdsAsSuffixes(wantUnaryValues);
}
xGoogReqIdInterceptor.checkExpectedUnaryXGoogRequestIdsAsSuffixes(wantUnaryValues);
}
}

Expand Down Expand Up @@ -3034,9 +3029,7 @@ public void testPartitionedDmlWithHigherTimeout() {
new XGoogSpannerRequestId(NON_DETERMINISTIC, channelId, 6, 1)),
};

if (false) { // TODO(@odeke-em): enable in next PRs.
xGoogReqIdInterceptor.checkExpectedStreamingXGoogRequestIds(wantStreamingValues);
}
xGoogReqIdInterceptor.checkExpectedStreamingXGoogRequestIds(wantStreamingValues);

XGoogSpannerRequestIdTest.MethodAndRequestId[] wantUnaryValues = {
XGoogSpannerRequestIdTest.ofMethodAndRequestId(
Expand All @@ -3049,9 +3042,7 @@ public void testPartitionedDmlWithHigherTimeout() {
"google.spanner.v1.Spanner/ExecuteSql",
new XGoogSpannerRequestId(NON_DETERMINISTIC, channelId, 8, 1)),
};
if (false) { // TODO(@odeke-em): enable in next PRs.
xGoogReqIdInterceptor.checkExpectedUnaryXGoogRequestIdsAsSuffixes(wantUnaryValues);
}
xGoogReqIdInterceptor.checkExpectedUnaryXGoogRequestIdsAsSuffixes(wantUnaryValues);
}
}

Expand Down Expand Up @@ -5304,8 +5295,7 @@ public void testSelectHasXGoogRequestIdHeader() {
assertEquals(1L, resultSet.getLong(0));
assertFalse(resultSet.next());
} finally {
// TODO(@odeke-em): Enable in later PR.
// xGoogReqIdInterceptor.assertIntegrity();
xGoogReqIdInterceptor.assertIntegrity();
}
}

Expand Down Expand Up @@ -5394,9 +5384,7 @@ public void testSessionPoolExhaustedError_containsStackTraces() {
"google.spanner.v1.Spanner/CreateSession",
new XGoogSpannerRequestId(NON_DETERMINISTIC, 0, 1, 1)),
};
if (false) { // TODO(@odeke-em): enable in next PRs.
xGoogReqIdInterceptor.checkExpectedUnaryXGoogRequestIdsAsSuffixes(wantUnaryValues);
}
xGoogReqIdInterceptor.checkExpectedUnaryXGoogRequestIdsAsSuffixes(wantUnaryValues);
}
}

Expand Down
13 changes: 12 additions & 1 deletion google-cloud-spanner/src/test/java/com/google/cloud/spanner/OpenTelemetrySpanTest.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -748,6 +748,13 @@ public void testTransactionRunnerWithRetryOnBeginTransaction() {
beginTransactionSpan.toString(),
beginTransactionSpan.getEvents().stream()
.anyMatch(event -> event.getName().equals("Starting RPC retry 1")));
verifyAtLeast1SpanHasXGoogSpannerRequestIdAttribute(finishedSpans);
}

private void verifyAtLeast1SpanHasXGoogSpannerRequestIdAttribute(List<SpanData> finishedSpans) {
AttributeKey<String> attributeKey = AttributeKey.stringKey("x_goog_spanner_request_id");
assertTrue(
finishedSpans.stream().anyMatch(span -> !span.getAttributes().get(attributeKey).isEmpty()));
}

@Test
Expand Down Expand Up @@ -798,6 +805,7 @@ public void testSingleUseRetryOnExecuteStreamingSql() {
executeStreamingQuery.toString(),
executeStreamingQuery.getEvents().stream()
.anyMatch(event -> event.getName().contains("Stream broken. Safe to retry")));
verifyAtLeast1SpanHasXGoogSpannerRequestIdAttribute(finishedSpans);
}

@Test
Expand Down Expand Up @@ -845,6 +853,7 @@ public void testRetryOnExecuteSql() {
executeSqlSpan.toString(),
executeSqlSpan.getEvents().stream()
.anyMatch(event -> event.getName().equals("Starting RPC retry 1")));
verifyAtLeast1SpanHasXGoogSpannerRequestIdAttribute(finishedSpans);
}

@Test
Expand All @@ -866,12 +875,14 @@ public void testTableAttributes() {
}
return null;
});
List<SpanData> finishedSpans = spanExporter.getFinishedSpanItems();
SpanData spanData =
spanExporter.getFinishedSpanItems().stream()
finishedSpans.stream()
.filter(x -> x.getName().equals("CloudSpannerOperation.ExecuteStreamingRead"))
.findFirst()
.get();
verifyTableAttributes(spanData);
verifyAtLeast1SpanHasXGoogSpannerRequestIdAttribute(finishedSpans);
}

private void waitForFinishedSpans(int numExpectedSpans) {
Expand Down
27 changes: 12 additions & 15 deletions google-cloud-spanner/src/test/java/com/google/cloud/spanner/XGoogSpannerRequestIdTest.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -73,11 +73,9 @@ public static class ServerHeaderEnforcer implements ServerInterceptor {
private Set<String> checkMethods;

ServerHeaderEnforcer(Set<String> checkMethods) {
this.gotValues = new CopyOnWriteArrayList<String>();
this.unaryResults =
new ConcurrentHashMap<String, CopyOnWriteArrayList<XGoogSpannerRequestId>>();
this.streamingResults =
new ConcurrentHashMap<String, CopyOnWriteArrayList<XGoogSpannerRequestId>>();
this.gotValues = new CopyOnWriteArrayList<>();
this.unaryResults = new ConcurrentHashMap<>();
this.streamingResults = new ConcurrentHashMap<>();
this.checkMethods = checkMethods;
}

Expand Down Expand Up @@ -128,17 +126,16 @@ public String[] accumulatedValues() {
}

public void assertIntegrity() {
this.unaryResults.forEach(
(String method, CopyOnWriteArrayList<XGoogSpannerRequestId> values) -> {
assertMonotonicityOfIds(method, values);
});
this.streamingResults.forEach(
(String method, CopyOnWriteArrayList<XGoogSpannerRequestId> values) -> {
assertMonotonicityOfIds(method, values);
});
this.unaryResults.forEach(this::assertMonotonicityOfIds);
this.streamingResults.forEach(this::assertMonotonicityOfIds);
}

private void assertMonotonicityOfIds(String prefix, List<XGoogSpannerRequestId> reqIds) {
reqIds.sort(
(id1, id2) -> {
if (id1.equals(id2)) return 0;
return id1.isGreaterThan(id2) ? 1 : -1;
});
int size = reqIds.size();

List<String> violations = new ArrayList<>();
Expand All @@ -161,7 +158,7 @@ private void assertMonotonicityOfIds(String prefix, List<XGoogSpannerRequestId>
}

public MethodAndRequestId[] accumulatedUnaryValues() {
List<MethodAndRequestId> accumulated = new ArrayList();
List<MethodAndRequestId> accumulated = new ArrayList<>();
this.unaryResults.forEach(
(String method, CopyOnWriteArrayList<XGoogSpannerRequestId> values) -> {
for (int i = 0; i < values.size(); i++) {
Expand All @@ -172,7 +169,7 @@ public MethodAndRequestId[] accumulatedUnaryValues() {
}

public MethodAndRequestId[] accumulatedStreamingValues() {
List<MethodAndRequestId> accumulated = new ArrayList();
List<MethodAndRequestId> accumulated = new ArrayList<>();
this.streamingResults.forEach(
(String method, CopyOnWriteArrayList<XGoogSpannerRequestId> values) -> {
for (int i = 0; i < values.size(); i++) {
Expand Down
Loading