Skip to content

Commit 085d5b8

Browse files
surbhigarg92surbhigarg92
committed
Adding gfe_latencies metric to built-in metrics
1 parent ea1ebad commit 085d5b8

File tree

8 files changed

+202
-34
lines changed

8 files changed

+202
-34
lines changed

‎google-cloud-spanner/src/main/java/com/google/cloud/spanner/BuiltInMetricsConstant.java‎

Lines changed: 17 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -37,6 +37,9 @@ public class BuiltInMetricsConstant {
3737

3838
public static final String GAX_METER_NAME = OpenTelemetryMetricsRecorder.GAX_METER_NAME;
3939

40+
static final String SPANNER_METER_NAME = "spanner-java";
41+
42+
static final String GFE_LATENCIES_NAME = "gfe_latencies";
4043
static final String OPERATION_LATENCIES_NAME = "operation_latencies";
4144
static final String ATTEMPT_LATENCIES_NAME = "attempt_latencies";
4245
static final String OPERATION_LATENCY_NAME = "operation_latency";
@@ -114,27 +117,39 @@ static Map<InstrumentSelector, View> getAllViews() {
114117
ImmutableMap.Builder<InstrumentSelector, View> views = ImmutableMap.builder();
115118
defineView(
116119
views,
120+
BuiltInMetricsConstant.GAX_METER_NAME,
117121
BuiltInMetricsConstant.OPERATION_LATENCY_NAME,
118122
BuiltInMetricsConstant.OPERATION_LATENCIES_NAME,
119123
BuiltInMetricsConstant.AGGREGATION_WITH_MILLIS_HISTOGRAM,
120124
InstrumentType.HISTOGRAM,
121125
"ms");
122126
defineView(
123127
views,
128+
BuiltInMetricsConstant.GAX_METER_NAME,
124129
BuiltInMetricsConstant.ATTEMPT_LATENCY_NAME,
125130
BuiltInMetricsConstant.ATTEMPT_LATENCIES_NAME,
126131
BuiltInMetricsConstant.AGGREGATION_WITH_MILLIS_HISTOGRAM,
127132
InstrumentType.HISTOGRAM,
128133
"ms");
129134
defineView(
130135
views,
136+
BuiltInMetricsConstant.SPANNER_METER_NAME,
137+
BuiltInMetricsConstant.GFE_LATENCIES_NAME,
138+
BuiltInMetricsConstant.GFE_LATENCIES_NAME,
139+
BuiltInMetricsConstant.AGGREGATION_WITH_MILLIS_HISTOGRAM,
140+
InstrumentType.HISTOGRAM,
141+
"ms");
142+
defineView(
143+
views,
144+
BuiltInMetricsConstant.GAX_METER_NAME,
131145
BuiltInMetricsConstant.OPERATION_COUNT_NAME,
132146
BuiltInMetricsConstant.OPERATION_COUNT_NAME,
133147
Aggregation.sum(),
134148
InstrumentType.COUNTER,
135149
"1");
136150
defineView(
137151
views,
152+
BuiltInMetricsConstant.GAX_METER_NAME,
138153
BuiltInMetricsConstant.ATTEMPT_COUNT_NAME,
139154
BuiltInMetricsConstant.ATTEMPT_COUNT_NAME,
140155
Aggregation.sum(),
@@ -145,6 +160,7 @@ static Map<InstrumentSelector, View> getAllViews() {
145160

146161
private static void defineView(
147162
ImmutableMap.Builder<InstrumentSelector, View> viewMap,
163+
String meterName,
148164
String metricName,
149165
String metricViewName,
150166
Aggregation aggregation,
@@ -153,7 +169,7 @@ private static void defineView(
153169
InstrumentSelector selector =
154170
InstrumentSelector.builder()
155171
.setName(BuiltInMetricsConstant.METER_NAME + '/' + metricName)
156-
.setMeterName(BuiltInMetricsConstant.GAX_METER_NAME)
172+
.setMeterName(meterName)
157173
.setType(type)
158174
.setUnit(unit)
159175
.build();

‎google-cloud-spanner/src/main/java/com/google/cloud/spanner/BuiltInOpenTelemetryMetricsProvider.java‎

Lines changed: 29 additions & 10 deletions
Original file line numberDiff line numberDiff line change
@@ -28,6 +28,8 @@
2828
import com.google.cloud.opentelemetry.detection.AttributeKeys;
2929
import com.google.cloud.opentelemetry.detection.DetectedPlatform;
3030
import com.google.cloud.opentelemetry.detection.GCPPlatformDetector;
31+
import com.google.common.cache.Cache;
32+
import com.google.common.cache.CacheBuilder;
3133
import com.google.common.hash.HashFunction;
3234
import com.google.common.hash.Hashing;
3335
import io.opentelemetry.api.OpenTelemetry;
@@ -42,6 +44,7 @@
4244
import java.util.HashMap;
4345
import java.util.Map;
4446
import java.util.UUID;
47+
import java.util.concurrent.ExecutionException;
4548
import java.util.logging.Level;
4649
import java.util.logging.Logger;
4750
import javax.annotation.Nullable;
@@ -57,6 +60,9 @@ final class BuiltInOpenTelemetryMetricsProvider {
5760

5861
private OpenTelemetry openTelemetry;
5962

63+
private final Cache<String, Map<String, String>> clientAttributesCache =
64+
CacheBuilder.newBuilder().maximumSize(1000).build();
65+
6066
private BuiltInOpenTelemetryMetricsProvider() {}
6167

6268
OpenTelemetry getOrCreateOpenTelemetry(String projectId, @Nullable Credentials credentials) {
@@ -78,16 +84,29 @@ OpenTelemetry getOrCreateOpenTelemetry(String projectId, @Nullable Credentials c
7884
}
7985
}
8086

81-
Map<String, String> createClientAttributes(String projectId, String client_name) {
82-
Map<String, String> clientAttributes = new HashMap<>();
83-
clientAttributes.put(LOCATION_ID_KEY.getKey(), detectClientLocation());
84-
clientAttributes.put(PROJECT_ID_KEY.getKey(), projectId);
85-
clientAttributes.put(INSTANCE_CONFIG_ID_KEY.getKey(), "unknown");
86-
clientAttributes.put(CLIENT_NAME_KEY.getKey(), client_name);
87-
String clientUid = getDefaultTaskValue();
88-
clientAttributes.put(CLIENT_UID_KEY.getKey(), clientUid);
89-
clientAttributes.put(CLIENT_HASH_KEY.getKey(), generateClientHash(clientUid));
90-
return clientAttributes;
87+
Map<String, String> createOrGetClientAttributes(String projectId, String client_name) {
88+
try {
89+
String key = projectId + client_name;
90+
return clientAttributesCache.get(
91+
key,
92+
() -> {
93+
Map<String, String> clientAttributes = new HashMap<>();
94+
clientAttributes.put(LOCATION_ID_KEY.getKey(), detectClientLocation());
95+
clientAttributes.put(PROJECT_ID_KEY.getKey(), projectId);
96+
clientAttributes.put(INSTANCE_CONFIG_ID_KEY.getKey(), "unknown");
97+
clientAttributes.put(CLIENT_NAME_KEY.getKey(), client_name);
98+
String clientUid = getDefaultTaskValue();
99+
clientAttributes.put(CLIENT_UID_KEY.getKey(), clientUid);
100+
clientAttributes.put(CLIENT_HASH_KEY.getKey(), generateClientHash(clientUid));
101+
return clientAttributes;
102+
});
103+
} catch (ExecutionException executionException) {
104+
logger.log(
105+
Level.WARNING,
106+
"Unable to get Client Attributes for client side metrics, will skip exporting client side metrics",
107+
executionException);
108+
return null;
109+
}
91110
}
92111

93112
/**

‎google-cloud-spanner/src/main/java/com/google/cloud/spanner/BuiltInOpenTelemetryMetricsRecorder.java‎

Lines changed: 82 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,82 @@
1+
/*
2+
* Copyright 2024 Google LLC
3+
*
4+
* Licensed under the Apache License, Version 2.0 (the "License");
5+
* you may not use this file except in compliance with the License.
6+
* You may obtain a copy of the License at
7+
*
8+
* http://www.apache.org/licenses/LICENSE-2.0
9+
*
10+
* Unless required by applicable law or agreed to in writing, software
11+
* distributed under the License is distributed on an "AS IS" BASIS,
12+
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
13+
* See the License for the specific language governing permissions and
14+
* limitations under the License.
15+
*/
16+
17+
package com.google.cloud.spanner;
18+
19+
import com.google.api.gax.core.GaxProperties;
20+
import com.google.common.annotations.VisibleForTesting;
21+
import com.google.common.base.Preconditions;
22+
import io.opentelemetry.api.OpenTelemetry;
23+
import io.opentelemetry.api.common.Attributes;
24+
import io.opentelemetry.api.common.AttributesBuilder;
25+
import io.opentelemetry.api.metrics.DoubleHistogram;
26+
import io.opentelemetry.api.metrics.Meter;
27+
import java.util.HashMap;
28+
import java.util.Map;
29+
30+
/** OpenTelemetry implementation of recording built in metrics. */
31+
public class BuiltInOpenTelemetryMetricsRecorder {
32+
33+
private final DoubleHistogram gfeLatencyRecorder;
34+
private final Map<String, String> attributes = new HashMap<>();
35+
36+
/**
37+
* Creates the following instruments for the following metrics:
38+
*
39+
* <ul>
40+
* <li>GFE Latency: Histogram
41+
* </ul>
42+
*
43+
* @param openTelemetry OpenTelemetry instance
44+
*/
45+
public BuiltInOpenTelemetryMetricsRecorder(
46+
OpenTelemetry openTelemetry, Map<String, String> clientAttributes) {
47+
Meter meter =
48+
openTelemetry
49+
.meterBuilder(BuiltInMetricsConstant.SPANNER_METER_NAME)
50+
.setInstrumentationVersion(GaxProperties.getLibraryVersion(getClass()))
51+
.build();
52+
this.gfeLatencyRecorder =
53+
meter
54+
.histogramBuilder(
55+
BuiltInMetricsConstant.METER_NAME + '/' + BuiltInMetricsConstant.GFE_LATENCIES_NAME)
56+
.setDescription(
57+
"Latency between Google's network receiving an RPC and reading back the first byte of the response")
58+
.setUnit("ms")
59+
.build();
60+
this.attributes.putAll(clientAttributes);
61+
}
62+
63+
/**
64+
* Record the latency between Google's network receiving an RPC and reading back the first byte of
65+
* the response. Data is stored in a Histogram.
66+
*
67+
* @param gfeLatency Attempt Latency in ms
68+
* @param attributes Map of the attributes to store
69+
*/
70+
public void recordGFELatency(double gfeLatency, Map<String, String> attributes) {
71+
this.attributes.putAll(attributes);
72+
gfeLatencyRecorder.record(gfeLatency, toOtelAttributes(this.attributes));
73+
}
74+
75+
@VisibleForTesting
76+
Attributes toOtelAttributes(Map<String, String> attributes) {
77+
Preconditions.checkNotNull(attributes, "Attributes map cannot be null");
78+
AttributesBuilder attributesBuilder = Attributes.builder();
79+
attributes.forEach(attributesBuilder::put);
80+
return attributesBuilder.build();
81+
}
82+
}

‎google-cloud-spanner/src/main/java/com/google/cloud/spanner/SpannerOptions.java‎

Lines changed: 19 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -1680,6 +1680,20 @@ public OpenTelemetry getOpenTelemetry() {
16801680
}
16811681
}
16821682

1683+
/**
1684+
* Returns an instance of OpenTelemetry. If OpenTelemetry object is not set via SpannerOptions
1685+
* then GlobalOpenTelemetry will be used as fallback.
1686+
*/
1687+
public OpenTelemetry getBuiltInMetricsOpenTelemetry() {
1688+
return this.builtInOpenTelemetryMetricsProvider.getOrCreateOpenTelemetry(
1689+
this.getProjectId(), getCredentials());
1690+
}
1691+
1692+
public Map<String, String> getBuiltInMetricsClientAttributes() {
1693+
return builtInOpenTelemetryMetricsProvider.createOrGetClientAttributes(
1694+
this.getProjectId(), "spanner-java/" + GaxProperties.getLibraryVersion(getClass()));
1695+
}
1696+
16831697
@Override
16841698
public ApiTracerFactory getApiTracerFactory() {
16851699
return createApiTracerFactory(false, false);
@@ -1729,11 +1743,13 @@ private ApiTracerFactory createMetricsApiTracerFactory() {
17291743
this.builtInOpenTelemetryMetricsProvider.getOrCreateOpenTelemetry(
17301744
this.getProjectId(), getCredentials());
17311745

1732-
return openTelemetry != null
1746+
Map<String, String> clientAttributes =
1747+
builtInOpenTelemetryMetricsProvider.createOrGetClientAttributes(
1748+
this.getProjectId(), "spanner-java/" + GaxProperties.getLibraryVersion(getClass()));
1749+
return openTelemetry != null && clientAttributes != null
17331750
? new MetricsTracerFactory(
17341751
new OpenTelemetryMetricsRecorder(openTelemetry, BuiltInMetricsConstant.METER_NAME),
1735-
builtInOpenTelemetryMetricsProvider.createClientAttributes(
1736-
this.getProjectId(), "spanner-java/" + GaxProperties.getLibraryVersion(getClass())))
1752+
clientAttributes)
17371753
: null;
17381754
}
17391755

‎google-cloud-spanner/src/main/java/com/google/cloud/spanner/spi/v1/GapicSpannerRpc.java‎

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -357,6 +357,8 @@ public GapicSpannerRpc(final SpannerOptions options) {
357357
options.getInterceptorProvider(),
358358
SpannerInterceptorProvider.createDefault(
359359
options.getOpenTelemetry(),
360+
options.getBuiltInMetricsOpenTelemetry(),
361+
options.getBuiltInMetricsClientAttributes(),
360362
(() -> directPathEnabledSupplier.get()))))
361363
// This sets the trace context headers.
362364
.withTraceContext(endToEndTracingEnabled, options.getOpenTelemetry())

‎google-cloud-spanner/src/main/java/com/google/cloud/spanner/spi/v1/HeaderInterceptor.java‎

Lines changed: 35 additions & 15 deletions
Original file line numberDiff line numberDiff line change
@@ -25,6 +25,7 @@
2525

2626
import com.google.api.gax.tracing.ApiTracer;
2727
import com.google.cloud.spanner.BuiltInMetricsConstant;
28+
import com.google.cloud.spanner.BuiltInOpenTelemetryMetricsRecorder;
2829
import com.google.cloud.spanner.CompositeTracer;
2930
import com.google.cloud.spanner.SpannerExceptionFactory;
3031
import com.google.cloud.spanner.SpannerRpcMetrics;
@@ -94,12 +95,17 @@ class HeaderInterceptor implements ClientInterceptor {
9495
private static final Level LEVEL = Level.INFO;
9596
private final SpannerRpcMetrics spannerRpcMetrics;
9697

98+
private final BuiltInOpenTelemetryMetricsRecorder builtInOpenTelemetryMetricsRecorder;
99+
97100
private final Supplier<Boolean> directPathEnabledSupplier;
98101

99102
HeaderInterceptor(
100-
SpannerRpcMetrics spannerRpcMetrics, Supplier<Boolean> directPathEnabledSupplier) {
103+
SpannerRpcMetrics spannerRpcMetrics,
104+
BuiltInOpenTelemetryMetricsRecorder builtInOpenTelemetryMetricsRecorder,
105+
Supplier<Boolean> directPathEnabledSupplier) {
101106
this.spannerRpcMetrics = spannerRpcMetrics;
102107
this.directPathEnabledSupplier = directPathEnabledSupplier;
108+
this.builtInOpenTelemetryMetricsRecorder = builtInOpenTelemetryMetricsRecorder;
103109
}
104110

105111
@Override
@@ -118,17 +124,22 @@ public void start(Listener<RespT> responseListener, Metadata headers) {
118124
TagContext tagContext = getTagContext(key, method.getFullMethodName(), databaseName);
119125
Attributes attributes =
120126
getMetricAttributes(key, method.getFullMethodName(), databaseName);
121-
Map<String, String> builtInMetricsAttributes =
122-
getBuiltInMetricAttributes(key, databaseName);
127+
Map<String, String> commonBuiltInMetricAttributes =
128+
getCommonBuiltInMetricAttributes(key, databaseName);
123129
super.start(
124130
new SimpleForwardingClientCallListener<RespT>(responseListener) {
125131
@Override
126132
public void onHeaders(Metadata metadata) {
127133
Boolean isDirectPathUsed =
128134
isDirectPathUsed(getAttributes().get(Grpc.TRANSPORT_ATTR_REMOTE_ADDR));
129135
addBuiltInMetricAttributes(
130-
compositeTracer, builtInMetricsAttributes, isDirectPathUsed);
131-
processHeader(metadata, tagContext, attributes, span);
136+
compositeTracer, commonBuiltInMetricAttributes, isDirectPathUsed);
137+
processHeader(
138+
metadata,
139+
tagContext,
140+
attributes,
141+
span,
142+
getBuiltInMetricAttributes(commonBuiltInMetricAttributes, isDirectPathUsed));
132143
super.onHeaders(metadata);
133144
}
134145
},
@@ -142,7 +153,11 @@ public void onHeaders(Metadata metadata) {
142153
}
143154

144155
private void processHeader(
145-
Metadata metadata, TagContext tagContext, Attributes attributes, Span span) {
156+
Metadata metadata,
157+
TagContext tagContext,
158+
Attributes attributes,
159+
Span span,
160+
Map<String, String> builtInMetricsAttributes) {
146161
MeasureMap measureMap = STATS_RECORDER.newMeasureMap();
147162
String serverTiming = metadata.get(SERVER_TIMING_HEADER_KEY);
148163
if (serverTiming != null && serverTiming.startsWith(SERVER_TIMING_HEADER_PREFIX)) {
@@ -154,6 +169,7 @@ private void processHeader(
154169

155170
spannerRpcMetrics.recordGfeLatency(latency, attributes);
156171
spannerRpcMetrics.recordGfeHeaderMissingCount(0L, attributes);
172+
builtInOpenTelemetryMetricsRecorder.recordGFELatency(latency, builtInMetricsAttributes);
157173

158174
if (span != null) {
159175
span.setAttribute("gfe_latency", String.valueOf(latency));
@@ -224,8 +240,8 @@ private Attributes getMetricAttributes(String key, String method, DatabaseName d
224240
});
225241
}
226242

227-
private Map<String, String> getBuiltInMetricAttributes(String key, DatabaseName databaseName)
228-
throws ExecutionException {
243+
private Map<String, String> getCommonBuiltInMetricAttributes(
244+
String key, DatabaseName databaseName) throws ExecutionException {
229245
return builtInAttributesCache.get(
230246
key,
231247
() -> {
@@ -240,17 +256,21 @@ private Map<String, String> getBuiltInMetricAttributes(String key, DatabaseName
240256
});
241257
}
242258

259+
private Map<String, String> getBuiltInMetricAttributes(
260+
Map<String, String> commonBuiltInMetricsAttributes, Boolean isDirectPathUsed) {
261+
Map<String, String> builtInMetricAttributes = new HashMap<>(commonBuiltInMetricsAttributes);
262+
builtInMetricAttributes.put(
263+
BuiltInMetricsConstant.DIRECT_PATH_USED_KEY.getKey(), Boolean.toString(isDirectPathUsed));
264+
return builtInMetricAttributes;
265+
}
266+
243267
private void addBuiltInMetricAttributes(
244268
CompositeTracer compositeTracer,
245-
Map<String, String> builtInMetricsAttributes,
269+
Map<String, String> commonBuiltInMetricsAttributes,
246270
Boolean isDirectPathUsed) {
247271
if (compositeTracer != null) {
248-
// Direct Path used attribute
249-
Map<String, String> attributes = new HashMap<>(builtInMetricsAttributes);
250-
attributes.put(
251-
BuiltInMetricsConstant.DIRECT_PATH_USED_KEY.getKey(), Boolean.toString(isDirectPathUsed));
252-
253-
compositeTracer.addAttributes(attributes);
272+
compositeTracer.addAttributes(
273+
getBuiltInMetricAttributes(commonBuiltInMetricsAttributes, isDirectPathUsed));
254274
}
255275
}
256276

0 commit comments

Comments
 (0)