Skip to content

Commit 9de6bd0

Browse files
surbhigarg92surbhigarg92
committed
Adding gfe_latencies metric to built-in metrics
1 parent ea1ebad commit 9de6bd0

File tree

6 files changed

+160
-20
lines changed

6 files changed

+160
-20
lines changed

‎google-cloud-spanner/src/main/java/com/google/cloud/spanner/BuiltInMetricsConstant.java‎

Lines changed: 18 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -37,7 +37,11 @@ public class BuiltInMetricsConstant {
3737

3838
public static final String GAX_METER_NAME = OpenTelemetryMetricsRecorder.GAX_METER_NAME;
3939

40+
public static final String SPANNER_METER_NAME = "spanner-java";
41+
4042
static final String OPERATION_LATENCIES_NAME = "operation_latencies";
43+
44+
static final String GFE_LATENCIES_NAME = "gfe_latencies";
4145
static final String ATTEMPT_LATENCIES_NAME = "attempt_latencies";
4246
static final String OPERATION_LATENCY_NAME = "operation_latency";
4347
static final String ATTEMPT_LATENCY_NAME = "attempt_latency";
@@ -114,27 +118,39 @@ static Map<InstrumentSelector, View> getAllViews() {
114118
ImmutableMap.Builder<InstrumentSelector, View> views = ImmutableMap.builder();
115119
defineView(
116120
views,
121+
BuiltInMetricsConstant.GAX_METER_NAME,
117122
BuiltInMetricsConstant.OPERATION_LATENCY_NAME,
118123
BuiltInMetricsConstant.OPERATION_LATENCIES_NAME,
119124
BuiltInMetricsConstant.AGGREGATION_WITH_MILLIS_HISTOGRAM,
120125
InstrumentType.HISTOGRAM,
121126
"ms");
122127
defineView(
123128
views,
129+
BuiltInMetricsConstant.GAX_METER_NAME,
124130
BuiltInMetricsConstant.ATTEMPT_LATENCY_NAME,
125131
BuiltInMetricsConstant.ATTEMPT_LATENCIES_NAME,
126132
BuiltInMetricsConstant.AGGREGATION_WITH_MILLIS_HISTOGRAM,
127133
InstrumentType.HISTOGRAM,
128134
"ms");
129135
defineView(
130136
views,
137+
BuiltInMetricsConstant.SPANNER_METER_NAME,
138+
BuiltInMetricsConstant.GFE_LATENCIES_NAME,
139+
BuiltInMetricsConstant.GFE_LATENCIES_NAME,
140+
BuiltInMetricsConstant.AGGREGATION_WITH_MILLIS_HISTOGRAM,
141+
InstrumentType.HISTOGRAM,
142+
"ms");
143+
defineView(
144+
views,
145+
BuiltInMetricsConstant.GAX_METER_NAME,
131146
BuiltInMetricsConstant.OPERATION_COUNT_NAME,
132147
BuiltInMetricsConstant.OPERATION_COUNT_NAME,
133148
Aggregation.sum(),
134149
InstrumentType.COUNTER,
135150
"1");
136151
defineView(
137152
views,
153+
BuiltInMetricsConstant.GAX_METER_NAME,
138154
BuiltInMetricsConstant.ATTEMPT_COUNT_NAME,
139155
BuiltInMetricsConstant.ATTEMPT_COUNT_NAME,
140156
Aggregation.sum(),
@@ -145,6 +161,7 @@ static Map<InstrumentSelector, View> getAllViews() {
145161

146162
private static void defineView(
147163
ImmutableMap.Builder<InstrumentSelector, View> viewMap,
164+
String meterName,
148165
String metricName,
149166
String metricViewName,
150167
Aggregation aggregation,
@@ -153,7 +170,7 @@ private static void defineView(
153170
InstrumentSelector selector =
154171
InstrumentSelector.builder()
155172
.setName(BuiltInMetricsConstant.METER_NAME + '/' + metricName)
156-
.setMeterName(BuiltInMetricsConstant.GAX_METER_NAME)
173+
.setMeterName(meterName)
157174
.setType(type)
158175
.setUnit(unit)
159176
.build();

‎google-cloud-spanner/src/main/java/com/google/cloud/spanner/BuiltInOpenTelemetryMetricsRecorder.java‎

Lines changed: 85 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,85 @@
1+
/*
2+
* Copyright 2024 Google LLC
3+
*
4+
* Licensed under the Apache License, Version 2.0 (the "License");
5+
* you may not use this file except in compliance with the License.
6+
* You may obtain a copy of the License at
7+
*
8+
* http://www.apache.org/licenses/LICENSE-2.0
9+
*
10+
* Unless required by applicable law or agreed to in writing, software
11+
* distributed under the License is distributed on an "AS IS" BASIS,
12+
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
13+
* See the License for the specific language governing permissions and
14+
* limitations under the License.
15+
*/
16+
17+
package com.google.cloud.spanner;
18+
19+
import com.google.api.gax.core.GaxProperties;
20+
import com.google.common.annotations.VisibleForTesting;
21+
import com.google.common.base.Preconditions;
22+
import io.opentelemetry.api.OpenTelemetry;
23+
import io.opentelemetry.api.common.Attributes;
24+
import io.opentelemetry.api.common.AttributesBuilder;
25+
import io.opentelemetry.api.metrics.DoubleHistogram;
26+
import io.opentelemetry.api.metrics.Meter;
27+
import java.util.Map;
28+
29+
/**
30+
* OpenTelemetry implementation of recording metrics. This implementation collections the
31+
* measurements related to the lifecyle of an RPC.
32+
*
33+
* <p>For the Otel implementation, an attempt is a single RPC invocation and an operation is the
34+
* collection of all the attempts made before a response is returned (either as a success or an
35+
* error). A single call (i.e. `EchoClient.echo()`) should have an operation_count of 1 and may have
36+
* an attempt_count of 1+ (depending on the retry configurations).
37+
*/
38+
public class BuiltInOpenTelemetryMetricsRecorder {
39+
40+
private final DoubleHistogram gfeLatencyRecorder;
41+
42+
/**
43+
* Creates the following instruments for the following metrics:
44+
*
45+
* <ul>
46+
* <li>GFE Latency: Histogram
47+
* </ul>
48+
*
49+
* @param openTelemetry OpenTelemetry instance
50+
*/
51+
public BuiltInOpenTelemetryMetricsRecorder(OpenTelemetry openTelemetry) {
52+
Meter meter =
53+
openTelemetry
54+
.meterBuilder(BuiltInMetricsConstant.SPANNER_METER_NAME)
55+
.setInstrumentationVersion(GaxProperties.getLibraryVersion(getClass()))
56+
.build();
57+
this.gfeLatencyRecorder =
58+
meter
59+
.histogramBuilder(
60+
BuiltInMetricsConstant.METER_NAME + '/' + BuiltInMetricsConstant.GFE_LATENCIES_NAME)
61+
.setDescription(
62+
"Latency between Google's network receiving an RPC and reading back the first byte of the response")
63+
.setUnit("ms")
64+
.build();
65+
}
66+
67+
/**
68+
* Record the latency between Google's network receiving an RPC and reading back the first byte of
69+
* the response. Data is stored in a Histogram.
70+
*
71+
* @param gfeLatency Attempt Latency in ms
72+
* @param attributes Map of the attributes to store
73+
*/
74+
public void recordGFELatency(double gfeLatency, Map<String, String> attributes) {
75+
gfeLatencyRecorder.record(gfeLatency, toOtelAttributes(attributes));
76+
}
77+
78+
@VisibleForTesting
79+
Attributes toOtelAttributes(Map<String, String> attributes) {
80+
Preconditions.checkNotNull(attributes, "Attributes map cannot be null");
81+
AttributesBuilder attributesBuilder = Attributes.builder();
82+
attributes.forEach(attributesBuilder::put);
83+
return attributesBuilder.build();
84+
}
85+
}

‎google-cloud-spanner/src/main/java/com/google/cloud/spanner/SpannerOptions.java‎

Lines changed: 9 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -1680,6 +1680,15 @@ public OpenTelemetry getOpenTelemetry() {
16801680
}
16811681
}
16821682

1683+
/**
1684+
* Returns an instance of OpenTelemetry. If OpenTelemetry object is not set via SpannerOptions
1685+
* then GlobalOpenTelemetry will be used as fallback.
1686+
*/
1687+
public OpenTelemetry getBuiltInMetricsOpenTelemetry() {
1688+
return this.builtInOpenTelemetryMetricsProvider.getOrCreateOpenTelemetry(
1689+
this.getProjectId(), getCredentials());
1690+
}
1691+
16831692
@Override
16841693
public ApiTracerFactory getApiTracerFactory() {
16851694
return createApiTracerFactory(false, false);

‎google-cloud-spanner/src/main/java/com/google/cloud/spanner/spi/v1/GapicSpannerRpc.java‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -357,6 +357,7 @@ public GapicSpannerRpc(final SpannerOptions options) {
357357
options.getInterceptorProvider(),
358358
SpannerInterceptorProvider.createDefault(
359359
options.getOpenTelemetry(),
360+
options.getBuiltInMetricsOpenTelemetry(),
360361
(() -> directPathEnabledSupplier.get()))))
361362
// This sets the trace context headers.
362363
.withTraceContext(endToEndTracingEnabled, options.getOpenTelemetry())

‎google-cloud-spanner/src/main/java/com/google/cloud/spanner/spi/v1/HeaderInterceptor.java‎

Lines changed: 35 additions & 15 deletions
Original file line numberDiff line numberDiff line change
@@ -25,6 +25,7 @@
2525

2626
import com.google.api.gax.tracing.ApiTracer;
2727
import com.google.cloud.spanner.BuiltInMetricsConstant;
28+
import com.google.cloud.spanner.BuiltInOpenTelemetryMetricsRecorder;
2829
import com.google.cloud.spanner.CompositeTracer;
2930
import com.google.cloud.spanner.SpannerExceptionFactory;
3031
import com.google.cloud.spanner.SpannerRpcMetrics;
@@ -94,12 +95,17 @@ class HeaderInterceptor implements ClientInterceptor {
9495
private static final Level LEVEL = Level.INFO;
9596
private final SpannerRpcMetrics spannerRpcMetrics;
9697

98+
private final BuiltInOpenTelemetryMetricsRecorder builtInOpenTelemetryMetricsRecorder;
99+
97100
private final Supplier<Boolean> directPathEnabledSupplier;
98101

99102
HeaderInterceptor(
100-
SpannerRpcMetrics spannerRpcMetrics, Supplier<Boolean> directPathEnabledSupplier) {
103+
SpannerRpcMetrics spannerRpcMetrics,
104+
BuiltInOpenTelemetryMetricsRecorder builtInOpenTelemetryMetricsRecorder,
105+
Supplier<Boolean> directPathEnabledSupplier) {
101106
this.spannerRpcMetrics = spannerRpcMetrics;
102107
this.directPathEnabledSupplier = directPathEnabledSupplier;
108+
this.builtInOpenTelemetryMetricsRecorder = builtInOpenTelemetryMetricsRecorder;
103109
}
104110

105111
@Override
@@ -118,17 +124,22 @@ public void start(Listener<RespT> responseListener, Metadata headers) {
118124
TagContext tagContext = getTagContext(key, method.getFullMethodName(), databaseName);
119125
Attributes attributes =
120126
getMetricAttributes(key, method.getFullMethodName(), databaseName);
121-
Map<String, String> builtInMetricsAttributes =
122-
getBuiltInMetricAttributes(key, databaseName);
127+
Map<String, String> commonBuiltInMetricAttributes =
128+
getCommonBuiltInMetricAttributes(key, databaseName);
123129
super.start(
124130
new SimpleForwardingClientCallListener<RespT>(responseListener) {
125131
@Override
126132
public void onHeaders(Metadata metadata) {
127133
Boolean isDirectPathUsed =
128134
isDirectPathUsed(getAttributes().get(Grpc.TRANSPORT_ATTR_REMOTE_ADDR));
129135
addBuiltInMetricAttributes(
130-
compositeTracer, builtInMetricsAttributes, isDirectPathUsed);
131-
processHeader(metadata, tagContext, attributes, span);
136+
compositeTracer, commonBuiltInMetricAttributes, isDirectPathUsed);
137+
processHeader(
138+
metadata,
139+
tagContext,
140+
attributes,
141+
span,
142+
getBuiltInMetricAttributes(commonBuiltInMetricAttributes, isDirectPathUsed));
132143
super.onHeaders(metadata);
133144
}
134145
},
@@ -142,7 +153,11 @@ public void onHeaders(Metadata metadata) {
142153
}
143154

144155
private void processHeader(
145-
Metadata metadata, TagContext tagContext, Attributes attributes, Span span) {
156+
Metadata metadata,
157+
TagContext tagContext,
158+
Attributes attributes,
159+
Span span,
160+
Map<String, String> builtInMetricsAttributes) {
146161
MeasureMap measureMap = STATS_RECORDER.newMeasureMap();
147162
String serverTiming = metadata.get(SERVER_TIMING_HEADER_KEY);
148163
if (serverTiming != null && serverTiming.startsWith(SERVER_TIMING_HEADER_PREFIX)) {
@@ -154,6 +169,7 @@ private void processHeader(
154169

155170
spannerRpcMetrics.recordGfeLatency(latency, attributes);
156171
spannerRpcMetrics.recordGfeHeaderMissingCount(0L, attributes);
172+
builtInOpenTelemetryMetricsRecorder.recordGFELatency(latency, builtInMetricsAttributes);
157173

158174
if (span != null) {
159175
span.setAttribute("gfe_latency", String.valueOf(latency));
@@ -224,8 +240,8 @@ private Attributes getMetricAttributes(String key, String method, DatabaseName d
224240
});
225241
}
226242

227-
private Map<String, String> getBuiltInMetricAttributes(String key, DatabaseName databaseName)
228-
throws ExecutionException {
243+
private Map<String, String> getCommonBuiltInMetricAttributes(
244+
String key, DatabaseName databaseName) throws ExecutionException {
229245
return builtInAttributesCache.get(
230246
key,
231247
() -> {
@@ -240,17 +256,21 @@ private Map<String, String> getBuiltInMetricAttributes(String key, DatabaseName
240256
});
241257
}
242258

259+
private Map<String, String> getBuiltInMetricAttributes(
260+
Map<String, String> commonBuiltInMetricsAttributes, Boolean isDirectPathUsed) {
261+
Map<String, String> builtInMetricAttributes = new HashMap<>(commonBuiltInMetricsAttributes);
262+
builtInMetricAttributes.put(
263+
BuiltInMetricsConstant.DIRECT_PATH_USED_KEY.getKey(), Boolean.toString(isDirectPathUsed));
264+
return builtInMetricAttributes;
265+
}
266+
243267
private void addBuiltInMetricAttributes(
244268
CompositeTracer compositeTracer,
245-
Map<String, String> builtInMetricsAttributes,
269+
Map<String, String> commonBuiltInMetricsAttributes,
246270
Boolean isDirectPathUsed) {
247271
if (compositeTracer != null) {
248-
// Direct Path used attribute
249-
Map<String, String> attributes = new HashMap<>(builtInMetricsAttributes);
250-
attributes.put(
251-
BuiltInMetricsConstant.DIRECT_PATH_USED_KEY.getKey(), Boolean.toString(isDirectPathUsed));
252-
253-
compositeTracer.addAttributes(attributes);
272+
compositeTracer.addAttributes(
273+
getBuiltInMetricAttributes(commonBuiltInMetricsAttributes, isDirectPathUsed));
254274
}
255275
}
256276

‎google-cloud-spanner/src/main/java/com/google/cloud/spanner/spi/v1/SpannerInterceptorProvider.java‎

Lines changed: 12 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -18,6 +18,7 @@
1818
import com.google.api.core.InternalApi;
1919
import com.google.api.core.ObsoleteApi;
2020
import com.google.api.gax.grpc.GrpcInterceptorProvider;
21+
import com.google.cloud.spanner.BuiltInOpenTelemetryMetricsRecorder;
2122
import com.google.cloud.spanner.SpannerRpcMetrics;
2223
import com.google.common.base.Supplier;
2324
import com.google.common.base.Suppliers;
@@ -44,26 +45,33 @@ private SpannerInterceptorProvider(List<ClientInterceptor> clientInterceptors) {
4445

4546
@ObsoleteApi("This method always uses Global OpenTelemetry")
4647
public static SpannerInterceptorProvider createDefault() {
47-
return createDefault(GlobalOpenTelemetry.get());
48+
return createDefault(GlobalOpenTelemetry.get(), GlobalOpenTelemetry.get());
4849
}
4950

50-
public static SpannerInterceptorProvider createDefault(OpenTelemetry openTelemetry) {
51+
public static SpannerInterceptorProvider createDefault(
52+
OpenTelemetry openTelemetry, OpenTelemetry builtInMetricsopenTelemetry) {
5153
return createDefault(
5254
openTelemetry,
55+
builtInMetricsopenTelemetry,
5356
Suppliers.memoize(
5457
() -> {
5558
return false;
5659
}));
5760
}
5861

5962
public static SpannerInterceptorProvider createDefault(
60-
OpenTelemetry openTelemetry, Supplier<Boolean> directPathEnabledSupplier) {
63+
OpenTelemetry openTelemetry,
64+
OpenTelemetry builtInMetricsopenTelemetry,
65+
Supplier<Boolean> directPathEnabledSupplier) {
6166
List<ClientInterceptor> defaultInterceptorList = new ArrayList<>();
6267
defaultInterceptorList.add(new SpannerErrorInterceptor());
6368
defaultInterceptorList.add(
6469
new LoggingInterceptor(Logger.getLogger(GapicSpannerRpc.class.getName()), Level.FINER));
6570
defaultInterceptorList.add(
66-
new HeaderInterceptor(new SpannerRpcMetrics(openTelemetry), directPathEnabledSupplier));
71+
new HeaderInterceptor(
72+
new SpannerRpcMetrics(openTelemetry),
73+
new BuiltInOpenTelemetryMetricsRecorder(builtInMetricsopenTelemetry),
74+
directPathEnabledSupplier));
6775
return new SpannerInterceptorProvider(ImmutableList.copyOf(defaultInterceptorList));
6876
}
6977

0 commit comments

Comments
 (0)