Skip to content

Commit 0f261ab

Browse files
surbhigarg92surbhigarg92
committed
poc exemplars
1 parent d79855a commit 0f261ab

File tree

12 files changed

+189
-29
lines changed

12 files changed

+189
-29
lines changed

‎google-cloud-spanner/pom.xml‎

Lines changed: 13 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -17,8 +17,8 @@
1717
<site.installationModule>google-cloud-spanner</site.installationModule>
1818
<opencensus.version>0.31.1</opencensus.version>
1919
<spanner.testenv.config.class>com.google.cloud.spanner.GceTestEnvConfig</spanner.testenv.config.class>
20-
<spanner.testenv.instance>projects/gcloud-devel/instances/spanner-testing-east1</spanner.testenv.instance>
21-
<spanner.gce.config.project_id>gcloud-devel</spanner.gce.config.project_id>
20+
<spanner.testenv.instance>projects/span-cloud-testing/instances/gargsurbhi-testing</spanner.testenv.instance>
21+
<spanner.gce.config.project_id>span-cloud-testing</spanner.gce.config.project_id>
2222
<spanner.testenv.kms_key.name>projects/gcloud-devel/locations/us-east1/keyRings/cmek-test-key-ring/cryptoKeys/cmek-test-key</spanner.testenv.kms_key.name>
2323
</properties>
2424

@@ -265,6 +265,17 @@
265265
<dependency>
266266
<groupId>com.google.cloud.opentelemetry</groupId>
267267
<artifactId>detector-resources-support</artifactId>
268+
<version>0.33.0</version>
269+
</dependency>
270+
<dependency>
271+
<groupId>com.google.cloud.opentelemetry</groupId>
272+
<artifactId>exporter-trace</artifactId>
273+
<version>0.33.0</version>
274+
</dependency>
275+
<dependency>
276+
<groupId>com.google.cloud.opentelemetry</groupId>
277+
<artifactId>exporter-metrics</artifactId>
278+
<version>0.33.0</version>
268279
</dependency>
269280
<dependency>
270281
<groupId>com.google.cloud</groupId>
@@ -445,7 +456,6 @@
445456
<dependency>
446457
<groupId>io.opentelemetry</groupId>
447458
<artifactId>opentelemetry-sdk-trace</artifactId>
448-
<scope>test</scope>
449459
</dependency>
450460
<dependency>
451461
<groupId>io.opentelemetry</groupId>

‎google-cloud-spanner/src/main/java/com/google/cloud/spanner/BuiltInMetricsProvider.java‎

Lines changed: 72 additions & 12 deletions
Original file line numberDiff line numberDiff line change
@@ -17,9 +17,8 @@
1717
package com.google.cloud.spanner;
1818

1919
import static com.google.cloud.opentelemetry.detection.GCPPlatformDetector.SupportedPlatform.GOOGLE_KUBERNETES_ENGINE;
20-
import static com.google.cloud.spanner.BuiltInMetricsConstant.CLIENT_HASH_KEY;
21-
import static com.google.cloud.spanner.BuiltInMetricsConstant.CLIENT_NAME_KEY;
2220
import static com.google.cloud.spanner.BuiltInMetricsConstant.CLIENT_UID_KEY;
21+
import static com.google.cloud.spanner.BuiltInMetricsConstant.SPANNER_RESOURCE_TYPE;
2322
import static com.google.cloud.spanner.BuiltInMetricsConstant.INSTANCE_CONFIG_ID_KEY;
2423
import static com.google.cloud.spanner.BuiltInMetricsConstant.INSTANCE_ID_KEY;
2524
import static com.google.cloud.spanner.BuiltInMetricsConstant.LOCATION_ID_KEY;
@@ -32,6 +31,12 @@
3231
import com.google.cloud.opentelemetry.detection.AttributeKeys;
3332
import com.google.cloud.opentelemetry.detection.DetectedPlatform;
3433
import com.google.cloud.opentelemetry.detection.GCPPlatformDetector;
34+
import com.google.cloud.opentelemetry.metric.GoogleCloudMetricExporter;
35+
import com.google.cloud.opentelemetry.metric.MetricConfiguration;
36+
import com.google.cloud.opentelemetry.metric.MonitoredResourceDescription;
37+
import com.google.cloud.opentelemetry.trace.TraceConfiguration;
38+
import com.google.cloud.opentelemetry.trace.TraceExporter;
39+
import com.google.common.collect.ImmutableSet;
3540
import com.google.common.hash.HashFunction;
3641
import com.google.common.hash.Hashing;
3742
import io.grpc.ManagedChannelBuilder;
@@ -43,7 +48,10 @@
4348
import io.opentelemetry.sdk.metrics.SdkMeterProvider;
4449
import io.opentelemetry.sdk.metrics.SdkMeterProviderBuilder;
4550
import io.opentelemetry.sdk.resources.Resource;
46-
import java.io.IOException;
51+
import io.opentelemetry.sdk.metrics.export.MetricExporter;
52+
import io.opentelemetry.sdk.trace.SdkTracerProvider;
53+
import io.opentelemetry.sdk.trace.export.BatchSpanProcessor;
54+
import io.opentelemetry.sdk.trace.samplers.Sampler;
4755
import java.lang.management.ManagementFactory;
4856
import java.lang.reflect.Method;
4957
import java.net.InetAddress;
@@ -72,16 +80,64 @@ OpenTelemetry getOrCreateOpenTelemetry(
7280
try {
7381
if (this.openTelemetry == null) {
7482
SdkMeterProviderBuilder sdkMeterProviderBuilder = SdkMeterProvider.builder();
75-
BuiltInMetricsView.registerBuiltinMetrics(
76-
SpannerCloudMonitoringExporter.create(projectId, credentials, monitoringHost),
77-
sdkMeterProviderBuilder);
83+
84+
// Use GoogleCloudMetricExporter with service time series as the current custom exporter
85+
// does not export exemplars
86+
// GoogleCloudMetricExporter expect to pass the resource labels at the time of creating
87+
// exporter. Hence we need to hardcode instance_id here.
88+
MonitoredResourceDescription monitoredResourceDescription =
89+
new MonitoredResourceDescription(
90+
SPANNER_RESOURCE_TYPE,
91+
ImmutableSet.of(
92+
"project_id", "instance_id", "location", "instance_config", "client_hash"));
93+
94+
MetricExporter metricExporter =
95+
GoogleCloudMetricExporter.createWithConfiguration(
96+
MetricConfiguration.builder()
97+
.setProjectId(projectId)
98+
.setMonitoredResourceDescription(monitoredResourceDescription)
99+
.setInstrumentationLibraryLabelsEnabled(false)
100+
.setMetricServiceEndpoint("monitoring.googleapis.com:443")
101+
.setPrefix("spanner.googleapis.com/internal/client")
102+
.setUseServiceTimeSeries(true)
103+
.build());
104+
AttributesBuilder attributesBuilder =
105+
Attributes.builder()
106+
.put("gcp.resource_type", SPANNER_RESOURCE_TYPE)
107+
.put("project_id", projectId)
108+
.put("instance_id", "spanner-testing-east2")
109+
.put("instance_config", "unknown")
110+
.put("location", detectClientLocation())
111+
.put("client_hash", generateClientHash(getDefaultTaskValue()));
112+
BuiltInMetricsView.registerBuiltinMetrics(metricExporter, sdkMeterProviderBuilder);
78113
sdkMeterProviderBuilder.setResource(Resource.create(createResourceAttributes(projectId)));
79-
SdkMeterProvider sdkMeterProvider = sdkMeterProviderBuilder.build();
80-
this.openTelemetry = OpenTelemetrySdk.builder().setMeterProvider(sdkMeterProvider).build();
114+
SdkMeterProvider sdkMeterProvider =
115+
sdkMeterProviderBuilder.setResource(Resource.create(attributesBuilder.build())).build();
116+
117+
// Registering the trace provider with the same OpenTelemetry object used for metrics.
118+
Resource resource =
119+
Resource.getDefault().merge(Resource.builder().put("service.name", "surbhi").build());
120+
121+
io.opentelemetry.sdk.trace.export.SpanExporter traceExporter =
122+
TraceExporter.createWithConfiguration(TraceConfiguration.builder().build());
123+
BatchSpanProcessor otlpGrpcSpanProcessor =
124+
BatchSpanProcessor.builder(traceExporter).build();
125+
SdkTracerProvider sdkTracerProvider =
126+
SdkTracerProvider.builder()
127+
.addSpanProcessor(otlpGrpcSpanProcessor)
128+
.setResource(resource)
129+
.setSampler(Sampler.traceIdRatioBased(1))
130+
.build();
131+
132+
this.openTelemetry =
133+
OpenTelemetrySdk.builder()
134+
.setMeterProvider(sdkMeterProvider)
135+
// .setTracerProvider(sdkTracerProvider)
136+
.build();
81137
Runtime.getRuntime().addShutdownHook(new Thread(sdkMeterProvider::close));
82138
}
83139
return this.openTelemetry;
84-
} catch (IOException ex) {
140+
} catch (Exception ex) {
85141
logger.log(
86142
Level.WARNING,
87143
"Unable to get OpenTelemetry object for client side metrics, will skip exporting client"
@@ -129,9 +185,13 @@ Attributes createResourceAttributes(String projectId) {
129185

130186
Map<String, String> createClientAttributes() {
131187
Map<String, String> clientAttributes = new HashMap<>();
132-
clientAttributes.put(
133-
CLIENT_NAME_KEY.getKey(), "spanner-java/" + GaxProperties.getLibraryVersion(getClass()));
134-
clientAttributes.put(CLIENT_UID_KEY.getKey(), getDefaultTaskValue());
188+
// clientAttributes.put(LOCATION_ID_KEY.getKey(), detectClientLocation());
189+
// clientAttributes.put(PROJECT_ID_KEY.getKey(), projectId);
190+
// clientAttributes.put(INSTANCE_CONFIG_ID_KEY.getKey(), "unknown");
191+
// clientAttributes.put(CLIENT_NAME_KEY.getKey(), client_name);
192+
String clientUid = getDefaultTaskValue();
193+
clientAttributes.put(CLIENT_UID_KEY.getKey(), clientUid);
194+
// clientAttributes.put(CLIENT_HASH_KEY.getKey(), generateClientHash(clientUid));
135195
return clientAttributes;
136196
}
137197

‎google-cloud-spanner/src/main/java/com/google/cloud/spanner/BuiltInMetricsRecorder.java‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -25,6 +25,7 @@
2525
import io.opentelemetry.api.metrics.DoubleHistogram;
2626
import io.opentelemetry.api.metrics.LongCounter;
2727
import io.opentelemetry.api.metrics.Meter;
28+
import io.opentelemetry.context.Context;
2829
import java.util.Map;
2930

3031
/**

‎google-cloud-spanner/src/main/java/com/google/cloud/spanner/BuiltInMetricsTracer.java‎

Lines changed: 25 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -21,6 +21,8 @@
2121
import com.google.api.gax.tracing.ApiTracer;
2222
import com.google.api.gax.tracing.MethodName;
2323
import com.google.api.gax.tracing.MetricsTracer;
24+
import io.opentelemetry.api.common.Attributes;
25+
import io.opentelemetry.api.common.AttributesBuilder;
2426
import java.util.HashMap;
2527
import java.util.Map;
2628
import java.util.concurrent.CancellationException;
@@ -39,14 +41,24 @@ class BuiltInMetricsTracer extends MetricsTracer implements ApiTracer {
3941
private final Map<String, String> attributes = new HashMap<>();
4042
private Float gfeLatency = null;
4143
private Float afeLatency = null;
44+
45+
private final TraceWrapper traceWrapper;
4246
private long gfeHeaderMissingCount = 0;
4347
private long afeHeaderMissingCount = 0;
4448

4549
BuiltInMetricsTracer(
46-
MethodName methodName, BuiltInMetricsRecorder builtInOpenTelemetryMetricsRecorder) {
50+
MethodName methodName,
51+
BuiltInMetricsRecorder builtInOpenTelemetryMetricsRecorder,
52+
TraceWrapper traceWrapper) {
4753
super(methodName, builtInOpenTelemetryMetricsRecorder);
4854
this.builtInOpenTelemetryMetricsRecorder = builtInOpenTelemetryMetricsRecorder;
4955
this.attributes.put(METHOD_ATTRIBUTE, methodName.toString());
56+
// Metrics attributes which are filtered from metrics views are sent to exemplars as
57+
// filtered_attributes.
58+
// Below testmetric attribute will be available in exemplar as we have added a attributefilter
59+
// for our metric views.
60+
this.attributes.put("testmetric", "testm");
61+
this.traceWrapper = traceWrapper;
5062
}
5163

5264
/**
@@ -55,10 +67,19 @@ class BuiltInMetricsTracer extends MetricsTracer implements ApiTracer {
5567
*/
5668
@Override
5769
public void attemptSucceeded() {
58-
super.attemptSucceeded();
59-
attributes.put(STATUS_ATTRIBUTE, StatusCode.Code.OK.toString());
60-
builtInOpenTelemetryMetricsRecorder.recordServerTimingHeaderMetrics(
70+
// For exemplars to worj metrics should be recorded with the span context.
71+
// Creating a new span to verify this.
72+
ISpan currentSpan = this.traceWrapper.getCurrentSpan();
73+
AttributesBuilder builder = Attributes.builder();
74+
builder.put("test1", "abc");
75+
ISpan span = this.traceWrapper.spanBuilder("attempt succeeded", builder.build());
76+
try (IScope s = this.traceWrapper.withSpan(span)) {
77+
super.attemptSucceeded();
78+
attributes.put(STATUS_ATTRIBUTE, StatusCode.Code.OK.toString());
79+
builtInOpenTelemetryMetricsRecorder.recordServerTimingHeaderMetrics(
6180
gfeLatency, afeLatency, gfeHeaderMissingCount, afeHeaderMissingCount, attributes);
81+
}
82+
span.end();
6283
}
6384

6485
/**

‎google-cloud-spanner/src/main/java/com/google/cloud/spanner/BuiltInMetricsTracerFactory.java‎

Lines changed: 7 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -38,24 +38,29 @@ class BuiltInMetricsTracerFactory extends MetricsTracerFactory {
3838

3939
protected BuiltInMetricsRecorder builtInMetricsRecorder;
4040
private final Map<String, String> attributes;
41+
private final TraceWrapper traceWrapper;
4142

4243
/**
4344
* Pass in a Map of client level attributes which will be added to every single MetricsTracer
4445
* created from the ApiTracerFactory.
4546
*/
4647
public BuiltInMetricsTracerFactory(
47-
BuiltInMetricsRecorder builtInMetricsRecorder, Map<String, String> attributes) {
48+
BuiltInMetricsRecorder builtInMetricsRecorder,
49+
Map<String, String> attributes,
50+
TraceWrapper traceWrapper) {
4851
super(builtInMetricsRecorder, attributes);
4952
this.builtInMetricsRecorder = builtInMetricsRecorder;
5053
this.attributes = ImmutableMap.copyOf(attributes);
54+
this.traceWrapper = traceWrapper;
5155
}
5256

5357
@Override
5458
public ApiTracer newTracer(ApiTracer parent, SpanName spanName, OperationType operationType) {
5559
BuiltInMetricsTracer metricsTracer =
5660
new BuiltInMetricsTracer(
5761
MethodName.of(spanName.getClientName(), spanName.getMethodName()),
58-
builtInMetricsRecorder);
62+
builtInMetricsRecorder,
63+
this.traceWrapper);
5964
metricsTracer.addAttributes(attributes);
6065
return metricsTracer;
6166
}

‎google-cloud-spanner/src/main/java/com/google/cloud/spanner/SpannerOptions.java‎

Lines changed: 14 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -78,6 +78,7 @@
7878
import io.grpc.netty.shaded.io.grpc.netty.GrpcSslContexts;
7979
import io.grpc.netty.shaded.io.grpc.netty.NettyChannelBuilder;
8080
import io.grpc.netty.shaded.io.netty.handler.ssl.SslContext;
81+
import io.opencensus.trace.Tracing;
8182
import io.opentelemetry.api.GlobalOpenTelemetry;
8283
import io.opentelemetry.api.OpenTelemetry;
8384
import io.opentelemetry.api.common.Attributes;
@@ -178,7 +179,7 @@ public class SpannerOptions extends ServiceOptions<Spanner, SpannerOptions> {
178179
private final boolean enableDirectAccess;
179180
private final DirectedReadOptions directedReadOptions;
180181
private final boolean useVirtualThreads;
181-
private final OpenTelemetry openTelemetry;
182+
private OpenTelemetry openTelemetry;
182183
private final boolean enableApiTracing;
183184
private final boolean enableBuiltInMetrics;
184185
private final boolean enableExtendedTracing;
@@ -2011,6 +2012,9 @@ public OpenTelemetry getOpenTelemetry() {
20112012
return this.openTelemetry;
20122013
} else {
20132014
return GlobalOpenTelemetry.get();
2015+
// this.openTelemetry = this.builtInMetricsProvider.getOrCreateOpenTelemetry(
2016+
// this.getProjectId(), getCredentials(), this.monitoringHost);
2017+
// return this.openTelemetry;
20142018
}
20152019
}
20162020

@@ -2070,10 +2074,18 @@ private ApiTracerFactory createMetricsApiTracerFactory() {
20702074
this.builtInMetricsProvider.getOrCreateOpenTelemetry(
20712075
this.getProjectId(), getCredentials(), this.monitoringHost);
20722076

2077+
// this.openTelemetry = openTelemetry;
2078+
20732079
return openTelemetry != null
20742080
? new BuiltInMetricsTracerFactory(
20752081
new BuiltInMetricsRecorder(openTelemetry, BuiltInMetricsConstant.METER_NAME),
2076-
new HashMap<>())
2082+
new HashMap<>(),
2083+
new TraceWrapper(
2084+
Tracing.getTracer(),
2085+
openTelemetry.getTracer(
2086+
MetricRegistryConstants.INSTRUMENTATION_SCOPE,
2087+
GaxProperties.getLibraryVersion(getClass())),
2088+
true))
20772089
: null;
20782090
}
20792091

‎google-cloud-spanner/src/main/java/com/google/cloud/spanner/spi/v1/HeaderInterceptor.java‎

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -284,8 +284,8 @@ private Map<String, String> getBuiltInMetricAttributes(String key, DatabaseName
284284
() -> {
285285
Map<String, String> attributes = new HashMap<>();
286286
attributes.put(BuiltInMetricsConstant.DATABASE_KEY.getKey(), databaseName.getDatabase());
287-
attributes.put(
288-
BuiltInMetricsConstant.INSTANCE_ID_KEY.getKey(), databaseName.getInstance());
287+
// attributes.put(
288+
// BuiltInMetricsConstant.INSTANCE_ID_KEY.getKey(), databaseName.getInstance());
289289
attributes.put(
290290
BuiltInMetricsConstant.DIRECT_PATH_ENABLED_KEY.getKey(),
291291
String.valueOf(GapicSpannerRpc.DIRECTPATH_CHANNEL_CREATED));

‎google-cloud-spanner/src/main/java/com/google/cloud/spanner/testing/RemoteSpannerHelper.java‎

Lines changed: 40 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -19,6 +19,8 @@
1919
import com.google.api.client.util.BackOff;
2020
import com.google.api.client.util.ExponentialBackOff;
2121
import com.google.api.gax.longrunning.OperationFuture;
22+
import com.google.cloud.opentelemetry.trace.TraceConfiguration;
23+
import com.google.cloud.opentelemetry.trace.TraceExporter;
2224
import com.google.cloud.spanner.BatchClient;
2325
import com.google.cloud.spanner.Database;
2426
import com.google.cloud.spanner.DatabaseClient;
@@ -32,6 +34,12 @@
3234
import com.google.cloud.spanner.SpannerOptions;
3335
import com.google.common.collect.Iterables;
3436
import com.google.spanner.admin.database.v1.CreateDatabaseMetadata;
37+
import io.opentelemetry.api.OpenTelemetry;
38+
import io.opentelemetry.sdk.OpenTelemetrySdk;
39+
import io.opentelemetry.sdk.resources.Resource;
40+
import io.opentelemetry.sdk.trace.SdkTracerProvider;
41+
import io.opentelemetry.sdk.trace.export.BatchSpanProcessor;
42+
import io.opentelemetry.sdk.trace.samplers.Sampler;
3543
import java.util.ArrayList;
3644
import java.util.Arrays;
3745
import java.util.Collections;
@@ -220,6 +228,8 @@ public void cleanUp() {
220228
* using this will be created in the given instance.
221229
*/
222230
public static RemoteSpannerHelper create(InstanceId instanceId) {
231+
SpannerOptions.enableOpenTelemetryTraces();
232+
// createTraceExporter(instanceId.getProject());
223233
SpannerOptions options =
224234
SpannerOptions.newBuilder()
225235
.setProjectId(instanceId.getProject())
@@ -238,4 +248,34 @@ public static RemoteSpannerHelper create(SpannerOptions options, InstanceId inst
238248
Spanner client = options.getService();
239249
return new RemoteSpannerHelper(options, instanceId, client);
240250
}
251+
252+
public static OpenTelemetry createTraceExporter() {
253+
// Tried with separate OT object for Traces. In this case exemplars are coming up but not linked
254+
// to correct trace id.
255+
Resource resource =
256+
Resource.getDefault().merge(Resource.builder().put("service.name", "surbhi").build());
257+
258+
io.opentelemetry.sdk.trace.export.SpanExporter traceExporter =
259+
TraceExporter.createWithConfiguration(TraceConfiguration.builder().build());
260+
261+
// Using a batch span processor
262+
// You can use `.setScheduleDelay()`, `.setExporterTimeout()`,
263+
// `.setMaxQueueSize`(), and `.setMaxExportBatchSize()` to further customize.
264+
BatchSpanProcessor otlpGrpcSpanProcessor = BatchSpanProcessor.builder(traceExporter).build();
265+
266+
// Create a new tracer provider
267+
SdkTracerProvider sdkTracerProvider =
268+
SdkTracerProvider.builder()
269+
// Use Otlp exporter or any other exporter of your choice.
270+
.addSpanProcessor(otlpGrpcSpanProcessor)
271+
.setResource(resource)
272+
.setSampler(Sampler.traceIdRatioBased(1))
273+
.build();
274+
275+
// Export to a collector that is expecting OTLP using gRPC.
276+
OpenTelemetry openTelemetry =
277+
OpenTelemetrySdk.builder().setTracerProvider(sdkTracerProvider).build();
278+
279+
return openTelemetry;
280+
}
241281
}

‎google-cloud-spanner/src/test/java/com/google/cloud/spanner/GceTestEnvConfig.java‎

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -21,6 +21,7 @@
2121
import com.google.api.gax.grpc.InstantiatingGrpcChannelProvider;
2222
import com.google.auth.oauth2.GoogleCredentials;
2323
import com.google.cloud.spanner.spi.v1.SpannerInterceptorProvider;
24+
import com.google.cloud.spanner.testing.RemoteSpannerHelper;
2425
import io.grpc.CallOptions;
2526
import io.grpc.Channel;
2627
import io.grpc.ClientCall;
@@ -90,6 +91,7 @@ public GceTestEnvConfig() {
9091
interceptorProvider.with(new DirectPathAddressCheckInterceptor(directPathTestScenario));
9192
}
9293
builder.setInterceptorProvider(interceptorProvider);
94+
builder.setOpenTelemetry(RemoteSpannerHelper.createTraceExporter());
9395
// DirectPath tests need to set a custom endpoint to the ChannelProvider
9496
InstantiatingGrpcChannelProvider.Builder customChannelProviderBuilder =
9597
InstantiatingGrpcChannelProvider.newBuilder();

0 commit comments

Comments
 (0)