Skip to content

Commit 43b48a7

Browse files
surbhigarg92surbhigarg92
committed
afe_env and unit tests
1 parent 21dddef commit 43b48a7

File tree

9 files changed

+151
-37
lines changed

9 files changed

+151
-37
lines changed

‎google-cloud-spanner/src/main/java/com/google/cloud/spanner/BuiltInMetricsRecorder.java‎

Lines changed: 9 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -95,15 +95,20 @@ class BuiltInMetricsRecorder extends OpenTelemetryMetricsRecorder {
9595
* @param attributes Map of the attributes to store
9696
*/
9797
void recordServerTimingHeaderMetrics(
98-
double gfeLatency,
99-
double afeLatency,
98+
Long gfeLatency,
99+
Long afeLatency,
100100
Long gfeHeaderMissingCount,
101101
Long afeHeaderMissingCount,
102102
Map<String, String> attributes) {
103103
io.opentelemetry.api.common.Attributes otelAttributes = toOtelAttributes(attributes);
104-
gfeLatencyRecorder.record(gfeLatency, otelAttributes);
104+
if (gfeLatency != null) {
105+
gfeLatencyRecorder.record(gfeLatency, otelAttributes);
106+
}
105107
gfeHeaderMissingCountRecorder.add(gfeHeaderMissingCount, otelAttributes);
106-
afeLatencyRecorder.record(afeLatency, otelAttributes);
108+
109+
if (afeLatency != null) {
110+
afeLatencyRecorder.record(afeLatency, otelAttributes);
111+
}
107112
afeHeaderMissingCountRecorder.add(afeHeaderMissingCount, otelAttributes);
108113
}
109114

‎google-cloud-spanner/src/main/java/com/google/cloud/spanner/BuiltInMetricsTracer.java‎

Lines changed: 15 additions & 25 deletions
Original file line numberDiff line numberDiff line change
@@ -60,11 +60,9 @@ class BuiltInMetricsTracer extends MetricsTracer implements ApiTracer {
6060
@Override
6161
public void attemptSucceeded() {
6262
super.attemptSucceeded();
63-
if (gfeLatency != null) {
64-
attributes.put(STATUS_ATTRIBUTE, StatusCode.Code.OK.toString());
65-
builtInOpenTelemetryMetricsRecorder.recordServerTimingHeaderMetrics(
66-
gfeLatency, afeLatency, gfeHeaderMissingCount, afeHeaderMissingCount, attributes);
67-
}
63+
attributes.put(STATUS_ATTRIBUTE, StatusCode.Code.OK.toString());
64+
builtInOpenTelemetryMetricsRecorder.recordServerTimingHeaderMetrics(
65+
gfeLatency, afeLatency, gfeHeaderMissingCount, afeHeaderMissingCount, attributes);
6866
}
6967

7068
/**
@@ -74,11 +72,9 @@ public void attemptSucceeded() {
7472
@Override
7573
public void attemptCancelled() {
7674
super.attemptCancelled();
77-
if (gfeLatency != null) {
78-
attributes.put(STATUS_ATTRIBUTE, StatusCode.Code.CANCELLED.toString());
79-
builtInOpenTelemetryMetricsRecorder.recordServerTimingHeaderMetrics(
80-
gfeLatency, afeLatency, gfeHeaderMissingCount, afeHeaderMissingCount, attributes);
81-
}
75+
attributes.put(STATUS_ATTRIBUTE, StatusCode.Code.CANCELLED.toString());
76+
builtInOpenTelemetryMetricsRecorder.recordServerTimingHeaderMetrics(
77+
gfeLatency, afeLatency, gfeHeaderMissingCount, afeHeaderMissingCount, attributes);
8278
}
8379

8480
/**
@@ -92,11 +88,9 @@ public void attemptCancelled() {
9288
@Override
9389
public void attemptFailedDuration(Throwable error, java.time.Duration delay) {
9490
super.attemptFailedDuration(error, delay);
95-
if (gfeLatency != null) {
96-
attributes.put(STATUS_ATTRIBUTE, extractStatus(error));
97-
builtInOpenTelemetryMetricsRecorder.recordServerTimingHeaderMetrics(
98-
gfeLatency, afeLatency, gfeHeaderMissingCount, afeHeaderMissingCount, attributes);
99-
}
91+
attributes.put(STATUS_ATTRIBUTE, extractStatus(error));
92+
builtInOpenTelemetryMetricsRecorder.recordServerTimingHeaderMetrics(
93+
gfeLatency, afeLatency, gfeHeaderMissingCount, afeHeaderMissingCount, attributes);
10094
}
10195

10296
/**
@@ -109,11 +103,9 @@ public void attemptFailedDuration(Throwable error, java.time.Duration delay) {
109103
@Override
110104
public void attemptFailedRetriesExhausted(Throwable error) {
111105
super.attemptFailedRetriesExhausted(error);
112-
if (gfeLatency != null) {
113-
attributes.put(STATUS_ATTRIBUTE, extractStatus(error));
114-
builtInOpenTelemetryMetricsRecorder.recordServerTimingHeaderMetrics(
115-
gfeLatency, afeLatency, gfeHeaderMissingCount, afeHeaderMissingCount, attributes);
116-
}
106+
attributes.put(STATUS_ATTRIBUTE, extractStatus(error));
107+
builtInOpenTelemetryMetricsRecorder.recordServerTimingHeaderMetrics(
108+
gfeLatency, afeLatency, gfeHeaderMissingCount, afeHeaderMissingCount, attributes);
117109
}
118110

119111
/**
@@ -126,11 +118,9 @@ public void attemptFailedRetriesExhausted(Throwable error) {
126118
@Override
127119
public void attemptPermanentFailure(Throwable error) {
128120
super.attemptPermanentFailure(error);
129-
if (gfeLatency != null) {
130-
attributes.put(STATUS_ATTRIBUTE, extractStatus(error));
131-
builtInOpenTelemetryMetricsRecorder.recordServerTimingHeaderMetrics(
132-
gfeLatency, afeLatency, gfeHeaderMissingCount, afeHeaderMissingCount, attributes);
133-
}
121+
attributes.put(STATUS_ATTRIBUTE, extractStatus(error));
122+
builtInOpenTelemetryMetricsRecorder.recordServerTimingHeaderMetrics(
123+
gfeLatency, afeLatency, gfeHeaderMissingCount, afeHeaderMissingCount, attributes);
134124
}
135125

136126
void recordGFELatency(Long gfeLatency) {

‎google-cloud-spanner/src/main/java/com/google/cloud/spanner/SpannerOptions.java‎

Lines changed: 15 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -848,6 +848,10 @@ default boolean isEnableBuiltInMetrics() {
848848
return true;
849849
}
850850

851+
default boolean isEnableAFEServerTiming() {
852+
return true;
853+
}
854+
851855
default boolean isEnableEndToEndTracing() {
852856
return false;
853857
}
@@ -878,6 +882,8 @@ private static class SpannerEnvironmentImpl implements SpannerEnvironment {
878882
private static final String SPANNER_ENABLE_END_TO_END_TRACING =
879883
"SPANNER_ENABLE_END_TO_END_TRACING";
880884
private static final String SPANNER_DISABLE_BUILTIN_METRICS = "SPANNER_DISABLE_BUILTIN_METRICS";
885+
private static final String SPANNER_DISABLE_AFE_SERVER_TIMING =
886+
"SPANNER_DISABLE_AFE_SERVER_TIMING";
881887
private static final String SPANNER_MONITORING_HOST = "SPANNER_MONITORING_HOST";
882888

883889
private SpannerEnvironmentImpl() {}
@@ -910,6 +916,11 @@ public boolean isEnableBuiltInMetrics() {
910916
return !Boolean.parseBoolean(System.getenv(SPANNER_DISABLE_BUILTIN_METRICS));
911917
}
912918

919+
@Override
920+
public boolean isEnableAFEServerTiming() {
921+
return !Boolean.parseBoolean(System.getenv(SPANNER_DISABLE_AFE_SERVER_TIMING));
922+
}
923+
913924
@Override
914925
public boolean isEnableEndToEndTracing() {
915926
return Boolean.parseBoolean(System.getenv(SPANNER_ENABLE_END_TO_END_TRACING));
@@ -2075,6 +2086,10 @@ public boolean isEndToEndTracingEnabled() {
20752086
return enableEndToEndTracing;
20762087
}
20772088

2089+
public boolean isEnableAFEServerTiming() {
2090+
return SpannerOptions.environment.isEnableAFEServerTiming();
2091+
}
2092+
20782093
/** Returns the default query options to use for the specific database. */
20792094
public QueryOptions getDefaultQueryOptions(DatabaseId databaseId) {
20802095
// Use the specific query options for the database if any have been specified. These have

‎google-cloud-spanner/src/main/java/com/google/cloud/spanner/spi/v1/GapicSpannerRpc.java‎

Lines changed: 6 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -275,6 +275,8 @@ public class GapicSpannerRpc implements SpannerRpc {
275275
new ConcurrentHashMap<>();
276276
private final boolean leaderAwareRoutingEnabled;
277277
private final boolean endToEndTracingEnabled;
278+
279+
private final boolean afeServerTimingEnabled;
278280
private final int numChannels;
279281
private final boolean isGrpcGcpExtensionEnabled;
280282

@@ -331,6 +333,7 @@ public GapicSpannerRpc(final SpannerOptions options) {
331333
this.compressorName = options.getCompressorName();
332334
this.leaderAwareRoutingEnabled = options.isLeaderAwareRoutingEnabled();
333335
this.endToEndTracingEnabled = options.isEndToEndTracingEnabled();
336+
this.afeServerTimingEnabled = options.isEnableAFEServerTiming();
334337
this.numChannels = options.getNumChannels();
335338
this.isGrpcGcpExtensionEnabled = options.isGrpcGcpExtensionEnabled();
336339

@@ -2030,6 +2033,9 @@ <ReqT, RespT> GrpcCallContext newCallContext(
20302033
if (endToEndTracingEnabled) {
20312034
context = context.withExtraHeaders(metadataProvider.newEndToEndTracingHeader());
20322035
}
2036+
if (afeServerTimingEnabled) {
2037+
context = context.withExtraHeaders(metadataProvider.newAfeServerTimingHeader());
2038+
}
20332039
if (callCredentialsProvider != null) {
20342040
CallCredentials callCredentials = callCredentialsProvider.getCallCredentials();
20352041
if (callCredentials != null) {

‎google-cloud-spanner/src/main/java/com/google/cloud/spanner/spi/v1/HeaderInterceptor.java‎

Lines changed: 0 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -186,8 +186,6 @@ private void processHeader(
186186
}
187187
}
188188

189-
// Record AFE latency
190-
// TODO: Add condition to check if AFE is enabled
191189
if (compositeTracer != null) {
192190
if (serverTimingMetrics.containsKey(AFE_TIMING_HEADER)) {
193191
long afeLatency = serverTimingMetrics.get(AFE_TIMING_HEADER);

‎google-cloud-spanner/src/main/java/com/google/cloud/spanner/spi/v1/SpannerMetadataProvider.java‎

Lines changed: 8 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -38,6 +38,8 @@ class SpannerMetadataProvider {
3838
private final String resourceHeaderKey;
3939
private static final String ROUTE_TO_LEADER_HEADER_KEY = "x-goog-spanner-route-to-leader";
4040
private static final String END_TO_END_TRACING_HEADER_KEY = "x-goog-spanner-end-to-end-tracing";
41+
private static final String AFE_SERVER_TIMING_HEADER_KEY =
42+
"x-goog-spanner-enable-afe-server-timing";
4143
private static final Pattern[] RESOURCE_TOKEN_PATTERNS = {
4244
Pattern.compile("^(?<headerValue>projects/[^/]*/instances/[^/]*/databases/[^/]*)(.*)?"),
4345
Pattern.compile("^(?<headerValue>projects/[^/]*/instances/[^/]*)(.*)?")
@@ -47,6 +49,8 @@ class SpannerMetadataProvider {
4749
ImmutableMap.of(ROUTE_TO_LEADER_HEADER_KEY, Collections.singletonList("true"));
4850
private static final Map<String, List<String>> END_TO_END_TRACING_HEADER_MAP =
4951
ImmutableMap.of(END_TO_END_TRACING_HEADER_KEY, Collections.singletonList("true"));
52+
private static final Map<String, List<String>> AFE_SERVER_TIMING_HEADER_MAP =
53+
ImmutableMap.of(AFE_SERVER_TIMING_HEADER_KEY, Collections.singletonList("true"));
5054

5155
private SpannerMetadataProvider(Map<String, String> headers, String resourceHeaderKey) {
5256
this.resourceHeaderKey = resourceHeaderKey;
@@ -96,6 +100,10 @@ Map<String, List<String>> newEndToEndTracingHeader() {
96100
return END_TO_END_TRACING_HEADER_MAP;
97101
}
98102

103+
Map<String, List<String>> newAfeServerTimingHeader() {
104+
return AFE_SERVER_TIMING_HEADER_MAP;
105+
}
106+
99107
private Map<Metadata.Key<String>, String> constructHeadersAsMetadata(
100108
Map<String, String> headers) {
101109
ImmutableMap.Builder<Metadata.Key<String>, String> headersAsMetadataBuilder =

‎google-cloud-spanner/src/test/java/com/google/cloud/spanner/AbstractNettyMockServerTest.java‎

Lines changed: 6 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -47,6 +47,9 @@ abstract class AbstractNettyMockServerTest {
4747
protected static AtomicInteger fakeServerTiming =
4848
new AtomicInteger(new Random().nextInt(1000) + 1);
4949

50+
protected static AtomicInteger fakeAFEServerTiming =
51+
new AtomicInteger(new Random().nextInt(500) + 1);
52+
5053
protected Spanner spanner;
5154

5255
@BeforeClass
@@ -72,7 +75,9 @@ public <ReqT, RespT> ServerCall.Listener<ReqT> interceptCall(
7275
public void sendHeaders(Metadata headers) {
7376
headers.put(
7477
Metadata.Key.of("server-timing", Metadata.ASCII_STRING_MARSHALLER),
75-
String.format("gfet4t7; dur=%d", fakeServerTiming.get()));
78+
String.format(
79+
"afet4t7; dur=%d, gfet4t7; dur=%d",
80+
fakeAFEServerTiming.get(), fakeServerTiming.get()));
7681
super.sendHeaders(headers);
7782
}
7883
},

‎google-cloud-spanner/src/test/java/com/google/cloud/spanner/OpenTelemetryBuiltInMetricsTracerTest.java‎

Lines changed: 81 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -38,7 +38,9 @@
3838
import com.google.common.collect.ImmutableList;
3939
import com.google.common.collect.Range;
4040
import io.grpc.ManagedChannelBuilder;
41+
import io.grpc.Server;
4142
import io.grpc.Status;
43+
import io.grpc.netty.shaded.io.grpc.netty.NettyServerBuilder;
4244
import io.opentelemetry.api.OpenTelemetry;
4345
import io.opentelemetry.api.common.Attributes;
4446
import io.opentelemetry.sdk.OpenTelemetrySdk;
@@ -47,6 +49,8 @@
4749
import io.opentelemetry.sdk.metrics.data.LongPointData;
4850
import io.opentelemetry.sdk.metrics.data.MetricData;
4951
import io.opentelemetry.sdk.testing.exporter.InMemoryMetricReader;
52+
import java.io.IOException;
53+
import java.net.InetSocketAddress;
5054
import java.time.Duration;
5155
import java.util.Collection;
5256
import java.util.List;
@@ -73,6 +77,7 @@ public class OpenTelemetryBuiltInMetricsTracerTest extends AbstractNettyMockServ
7377

7478
private static Attributes expectedCommonBaseAttributes;
7579
private static Attributes expectedCommonRequestAttributes;
80+
private static ApiTracerFactory metricsTracerFactory;
7681

7782
private static final long MIN_LATENCY = 0;
7883

@@ -110,6 +115,11 @@ public static void setup() {
110115

111116
expectedCommonRequestAttributes =
112117
Attributes.builder().put(BuiltInMetricsConstant.DIRECT_PATH_USED_KEY, "false").build();
118+
119+
metricsTracerFactory =
120+
new BuiltInMetricsTracerFactory(
121+
new BuiltInMetricsRecorder(openTelemetry, BuiltInMetricsConstant.METER_NAME),
122+
attributes);
113123
}
114124

115125
@BeforeClass
@@ -122,16 +132,12 @@ public static void setupResults() {
122132
@After
123133
public void clearRequests() {
124134
mockSpanner.clearRequests();
135+
metricReader.forceFlush();
125136
}
126137

127138
@Override
128139
public void createSpannerInstance() {
129140
SpannerOptions.Builder builder = SpannerOptions.newBuilder();
130-
131-
ApiTracerFactory metricsTracerFactory =
132-
new BuiltInMetricsTracerFactory(
133-
new BuiltInMetricsRecorder(openTelemetry, BuiltInMetricsConstant.METER_NAME),
134-
attributes);
135141
// Set a quick polling algorithm to prevent this from slowing down the test unnecessarily.
136142
builder
137143
.getDatabaseAdminStubSettingsBuilder()
@@ -209,6 +215,19 @@ public void testMetricsSingleUseQuery() {
209215
getMetricData(metricReader, BuiltInMetricsConstant.GFE_LATENCIES_NAME);
210216
long gfeLatencyValue = getAggregatedValue(gfeLatencyMetricData, expectedAttributes);
211217
assertEquals(fakeServerTiming.get(), gfeLatencyValue, 0);
218+
219+
MetricData afeLatencyMetricData =
220+
getMetricData(metricReader, BuiltInMetricsConstant.AFE_LATENCIES_NAME);
221+
long afeLatencyValue = getAggregatedValue(afeLatencyMetricData, expectedAttributes);
222+
assertEquals(fakeAFEServerTiming.get(), afeLatencyValue, 0);
223+
224+
MetricData gfeConnectivityMetricData =
225+
getMetricData(metricReader, BuiltInMetricsConstant.GFE_CONNECTIVITY_ERROR_NAME);
226+
assertThat(getAggregatedValue(gfeConnectivityMetricData, expectedAttributes)).isEqualTo(0);
227+
228+
MetricData afeConnectivityMetricData =
229+
getMetricData(metricReader, BuiltInMetricsConstant.AFE_CONNECTIVITY_ERROR_NAME);
230+
assertThat(getAggregatedValue(afeConnectivityMetricData, expectedAttributes)).isEqualTo(0);
212231
}
213232

214233
@Test
@@ -345,6 +364,63 @@ public void testNoNetworkConnection() {
345364
1, getAggregatedValue(attemptCountMetricData, expectedAttributesCreateSessionFailed));
346365
}
347366

367+
// @Test
368+
public void testNoServerTimingHeader() throws IOException, InterruptedException {
369+
// Create Spanner Object without headers
370+
InetSocketAddress addressNoHeader = new InetSocketAddress("localhost", 0);
371+
Server serverNoHeader =
372+
NettyServerBuilder.forAddress(addressNoHeader).addService(mockSpanner).build().start();
373+
String endpoint = address.getHostString() + ":" + serverNoHeader.getPort();
374+
Spanner spannerNoHeader =
375+
SpannerOptions.newBuilder()
376+
.setProjectId("test-project")
377+
.setChannelConfigurator(ManagedChannelBuilder::usePlaintext)
378+
.setHost("http://" + endpoint)
379+
.setCredentials(NoCredentials.getInstance())
380+
.setSessionPoolOption(
381+
SessionPoolOptions.newBuilder()
382+
.setWaitForMinSessionsDuration(Duration.ofSeconds(5L))
383+
.setFailOnSessionLeak()
384+
.setSkipVerifyingBeginTransactionForMuxRW(true)
385+
.build())
386+
// Setting this to false so that Spanner Options does not register Metrics Tracer
387+
// factory again.
388+
.setBuiltInMetricsEnabled(false)
389+
.setApiTracerFactory(metricsTracerFactory)
390+
.build()
391+
.getService();
392+
DatabaseClient databaseClientNoHeader =
393+
spannerNoHeader.getDatabaseClient(DatabaseId.of("test-project", "i", "d"));
394+
395+
try (ResultSet resultSet = databaseClientNoHeader.singleUse().executeQuery(SELECT_RANDOM)) {
396+
assertTrue(resultSet.next());
397+
assertFalse(resultSet.next());
398+
}
399+
400+
databaseClientNoHeader
401+
.readWriteTransaction()
402+
.run(transaction -> transaction.executeUpdate(UPDATE_RANDOM));
403+
404+
Attributes expectedAttributes =
405+
expectedCommonBaseAttributes
406+
.toBuilder()
407+
.putAll(expectedCommonRequestAttributes)
408+
.put(BuiltInMetricsConstant.STATUS_KEY, "OK")
409+
.put(BuiltInMetricsConstant.METHOD_KEY, "Spanner.ExecuteSql")
410+
.build();
411+
412+
MetricData gfeConnectivityMetricData =
413+
getMetricData(metricReader, BuiltInMetricsConstant.GFE_CONNECTIVITY_ERROR_NAME);
414+
assertThat(getAggregatedValue(gfeConnectivityMetricData, expectedAttributes)).isEqualTo(1);
415+
416+
MetricData afeConnectivityMetricData =
417+
getMetricData(metricReader, BuiltInMetricsConstant.AFE_CONNECTIVITY_ERROR_NAME);
418+
assertThat(getAggregatedValue(afeConnectivityMetricData, expectedAttributes)).isEqualTo(1);
419+
spannerNoHeader.close();
420+
serverNoHeader.shutdown();
421+
serverNoHeader.awaitTermination();
422+
}
423+
348424
private MetricData getMetricData(InMemoryMetricReader reader, String metricName) {
349425
String fullMetricName = BuiltInMetricsConstant.METER_NAME + "/" + metricName;
350426
Collection<MetricData> allMetricData;

‎google-cloud-spanner/src/test/java/com/google/cloud/spanner/spi/v1/SpannerMetadataProviderTest.java‎

Lines changed: 11 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -105,6 +105,17 @@ public void testNewEndToEndTracingHeader() {
105105
assertTrue(Maps.difference(extraHeaders, expectedHeaders).areEqual());
106106
}
107107

108+
@Test
109+
public void testNewAfeServerTimingHeader() {
110+
SpannerMetadataProvider metadataProvider =
111+
SpannerMetadataProvider.create(ImmutableMap.of(), "header1");
112+
Map<String, List<String>> extraHeaders = metadataProvider.newAfeServerTimingHeader();
113+
Map<String, List<String>> expectedHeaders =
114+
ImmutableMap.<String, List<String>>of(
115+
"x-goog-spanner-enable-afe-server-timing", ImmutableList.of("true"));
116+
assertTrue(Maps.difference(extraHeaders, expectedHeaders).areEqual());
117+
}
118+
108119
private String getResourceHeaderValue(
109120
SpannerMetadataProvider headerProvider, String resourceTokenTemplate) {
110121
Metadata metadata = headerProvider.newMetadata(resourceTokenTemplate, "projects/p");

0 commit comments

Comments
 (0)