Skip to content

Commit 0cc3997

Browse files
igorbernstein2igorbernstein2
authored
chore: export internal DirectAccess gRPC metrics to the new bigtable_client monitored resource (#2545)
This will export a relevant subset of the metrics described in https://github.com/grpc/proposal/blob/master/A78-grpc-metrics-wrr-pf-xds.md as internal metrics to help rollout of DirectAccess Change-Id: Ifddadb84d091f0f904f8f0f6e8da52de552e9757
1 parent 522fb8c commit 0cc3997

File tree

5 files changed

+130
-25
lines changed

5 files changed

+130
-25
lines changed

‎google-cloud-bigtable/pom.xml

Lines changed: 4 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -202,6 +202,10 @@
202202
<groupId>io.grpc</groupId>
203203
<artifactId>grpc-protobuf</artifactId>
204204
</dependency>
205+
<dependency>
206+
<groupId>io.grpc</groupId>
207+
<artifactId>grpc-opentelemetry</artifactId>
208+
</dependency>
205209
<dependency>
206210
<groupId>org.threeten</groupId>
207211
<artifactId>threetenbp</artifactId>

‎google-cloud-bigtable/src/main/java/com/google/cloud/bigtable/data/v2/stub/BigtableClientContext.java

Lines changed: 32 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -26,12 +26,14 @@
2626
import com.google.auth.oauth2.ServiceAccountJwtAccessCredentials;
2727
import com.google.cloud.bigtable.data.v2.BigtableDataSettings;
2828
import com.google.cloud.bigtable.data.v2.internal.JwtCredentialsWithAudience;
29+
import com.google.cloud.bigtable.data.v2.stub.metrics.BuiltinMetricsConstants;
2930
import com.google.cloud.bigtable.data.v2.stub.metrics.CustomOpenTelemetryMetricsProvider;
3031
import com.google.cloud.bigtable.data.v2.stub.metrics.DefaultMetricsProvider;
3132
import com.google.cloud.bigtable.data.v2.stub.metrics.ErrorCountPerConnectionMetricTracker;
3233
import com.google.cloud.bigtable.data.v2.stub.metrics.MetricsProvider;
3334
import com.google.cloud.bigtable.data.v2.stub.metrics.NoopMetricsProvider;
3435
import io.grpc.ManagedChannelBuilder;
36+
import io.grpc.opentelemetry.GrpcOpenTelemetry;
3537
import io.opentelemetry.api.OpenTelemetry;
3638
import io.opentelemetry.sdk.OpenTelemetrySdk;
3739
import java.io.IOException;
@@ -100,6 +102,9 @@ public static BigtableClientContext create(EnhancedBigtableStubSettings settings
100102
// a configurable transport provider + otel
101103
errorCountPerConnectionMetricTracker =
102104
setupPerConnectionErrorTracer(builder, transportProvider, internalOtel);
105+
106+
// Configure grpc metrics
107+
configureGrpcOtel(transportProvider, internalOtel);
103108
}
104109
}
105110

@@ -133,6 +138,33 @@ public static BigtableClientContext create(EnhancedBigtableStubSettings settings
133138
return new BigtableClientContext(clientContext, openTelemetry, internalOtel);
134139
}
135140

141+
private static void configureGrpcOtel(
142+
InstantiatingGrpcChannelProvider.Builder transportProvider, OpenTelemetrySdk otel) {
143+
144+
GrpcOpenTelemetry grpcOtel =
145+
GrpcOpenTelemetry.newBuilder()
146+
.sdk(otel)
147+
.addOptionalLabel("grpc.lb.locality")
148+
// Disable default grpc metrics
149+
.disableAllMetrics()
150+
// Enable specific grpc metrics
151+
.enableMetrics(BuiltinMetricsConstants.GRPC_METRICS.keySet())
152+
.build();
153+
154+
@SuppressWarnings("rawtypes")
155+
ApiFunction<ManagedChannelBuilder, ManagedChannelBuilder> oldConfigurator =
156+
transportProvider.getChannelConfigurator();
157+
158+
transportProvider.setChannelConfigurator(
159+
b -> {
160+
if (oldConfigurator != null) {
161+
b = oldConfigurator.apply(b);
162+
}
163+
grpcOtel.configureChannelBuilder(b);
164+
return b;
165+
});
166+
}
167+
136168
private BigtableClientContext(
137169
ClientContext clientContext,
138170
OpenTelemetry openTelemetry,

‎google-cloud-bigtable/src/main/java/com/google/cloud/bigtable/data/v2/stub/metrics/BigtableCloudMonitoringExporter.java

Lines changed: 3 additions & 9 deletions
Original file line numberDiff line numberDiff line change
@@ -172,8 +172,10 @@ private CompletableResultCode doExport(Collection<MetricData> metricData) {
172172

173173
// Skips exporting if there's none
174174
if (bigtableTimeSeries.isEmpty()) {
175+
System.out.println("skipping empty metrics: " + this.exporterName);
175176
return CompletableResultCode.ofSuccess();
176177
}
178+
System.out.println(bigtableTimeSeries);
177179

178180
CompletableResultCode exportCode = new CompletableResultCode();
179181
bigtableTimeSeries.forEach(
@@ -335,18 +337,10 @@ public Map<ProjectName, List<TimeSeries>> convert(Collection<MetricData> metricD
335337
return ImmutableMap.of();
336338
}
337339

338-
List<MetricData> relevantData =
339-
metricData.stream()
340-
.filter(md -> APPLICATION_METRICS.contains(md.getName()))
341-
.collect(Collectors.toList());
342-
if (relevantData.isEmpty()) {
343-
return ImmutableMap.of();
344-
}
345-
346340
return ImmutableMap.of(
347341
ProjectName.of(monitoredResource.getLabelsOrThrow(APPLICATION_RESOURCE_PROJECT_ID)),
348342
BigtableExporterUtils.convertToApplicationResourceTimeSeries(
349-
relevantData, monitoredResource));
343+
metricData, monitoredResource));
350344
}
351345
}
352346
}

‎google-cloud-bigtable/src/main/java/com/google/cloud/bigtable/data/v2/stub/metrics/BigtableExporterUtils.java

Lines changed: 57 additions & 14 deletions
Original file line numberDiff line numberDiff line change
@@ -28,7 +28,9 @@
2828
import static com.google.cloud.bigtable.data.v2.stub.metrics.BuiltinMetricsConstants.BIGTABLE_PROJECT_ID_KEY;
2929
import static com.google.cloud.bigtable.data.v2.stub.metrics.BuiltinMetricsConstants.CLIENT_UID_KEY;
3030
import static com.google.cloud.bigtable.data.v2.stub.metrics.BuiltinMetricsConstants.CLUSTER_ID_KEY;
31+
import static com.google.cloud.bigtable.data.v2.stub.metrics.BuiltinMetricsConstants.GRPC_METRICS;
3132
import static com.google.cloud.bigtable.data.v2.stub.metrics.BuiltinMetricsConstants.INSTANCE_ID_KEY;
33+
import static com.google.cloud.bigtable.data.v2.stub.metrics.BuiltinMetricsConstants.INTERNAL_METRICS;
3234
import static com.google.cloud.bigtable.data.v2.stub.metrics.BuiltinMetricsConstants.METER_NAME;
3335
import static com.google.cloud.bigtable.data.v2.stub.metrics.BuiltinMetricsConstants.TABLE_ID_KEY;
3436
import static com.google.cloud.bigtable.data.v2.stub.metrics.BuiltinMetricsConstants.ZONE_ID_KEY;
@@ -74,6 +76,7 @@
7476
import java.util.List;
7577
import java.util.Map;
7678
import java.util.Objects;
79+
import java.util.Optional;
7780
import java.util.Set;
7881
import java.util.UUID;
7982
import java.util.logging.Level;
@@ -164,16 +167,12 @@ static List<TimeSeries> convertToApplicationResourceTimeSeries(
164167
"convert application metrics is called when the supported resource is not detected");
165168
List<TimeSeries> allTimeSeries = new ArrayList<>();
166169
for (MetricData metricData : collection) {
167-
if (!metricData.getInstrumentationScopeInfo().getName().equals(METER_NAME)) {
168-
// Filter out metric data for instruments that are not part of the bigtable builtin metrics
169-
continue;
170-
}
171170
metricData.getData().getPoints().stream()
172171
.map(
173172
pointData ->
174-
convertPointToApplicationResourceTimeSeries(
175-
metricData, pointData, applicationResource))
176-
.forEach(allTimeSeries::add);
173+
createInternalMetricsTimeSeries(metricData, pointData, applicationResource))
174+
.filter(Optional::isPresent)
175+
.forEach(ts -> ts.ifPresent(allTimeSeries::add));
177176
}
178177
return allTimeSeries;
179178
}
@@ -290,19 +289,28 @@ private static TimeSeries convertPointToBigtableTimeSeries(
290289
return builder.build();
291290
}
292291

293-
private static TimeSeries convertPointToApplicationResourceTimeSeries(
292+
private static Optional<TimeSeries> createInternalMetricsTimeSeries(
294293
MetricData metricData, PointData pointData, MonitoredResource applicationResource) {
295294
TimeSeries.Builder builder =
296295
TimeSeries.newBuilder()
297296
.setMetricKind(convertMetricKind(metricData))
298297
.setValueType(convertValueType(metricData.getType()))
299298
.setResource(applicationResource);
300299

301-
Metric.Builder metricBuilder = Metric.newBuilder().setType(metricData.getName());
302-
303-
Attributes attributes = pointData.getAttributes();
304-
for (AttributeKey<?> key : attributes.asMap().keySet()) {
305-
metricBuilder.putLabels(key.getKey(), String.valueOf(attributes.get(key)));
300+
final Metric.Builder metricBuilder;
301+
// TODO: clean this up
302+
// Internal metrics are based on views that include the metric prefix
303+
// gRPC metrics dont have views and are dot encoded
304+
// To unify these:
305+
// - the useless views should be removed
306+
// - internal metrics should use relative metric names w/o the prefix
307+
if (INTERNAL_METRICS.contains(metricData.getName())) {
308+
metricBuilder = newApplicationMetricBuilder(metricData.getName(), pointData.getAttributes());
309+
} else if (GRPC_METRICS.containsKey(metricData.getName())) {
310+
metricBuilder = newGrpcMetricBuilder(metricData.getName(), pointData.getAttributes());
311+
} else {
312+
logger.fine("Skipping unexpected internal metric: " + metricData.getName());
313+
return Optional.empty();
306314
}
307315

308316
builder.setMetric(metricBuilder.build());
@@ -314,7 +322,42 @@ private static TimeSeries convertPointToApplicationResourceTimeSeries(
314322
.build();
315323

316324
builder.addPoints(createPoint(metricData.getType(), pointData, timeInterval));
317-
return builder.build();
325+
return Optional.of(builder.build());
326+
}
327+
328+
private static Metric.Builder newApplicationMetricBuilder(
329+
String metricName, Attributes attributes) {
330+
// TODO: unify handling of metric prefixes
331+
Metric.Builder metricBuilder = Metric.newBuilder().setType(metricName);
332+
for (Map.Entry<AttributeKey<?>, Object> e : attributes.asMap().entrySet()) {
333+
metricBuilder.putLabels(e.getKey().getKey(), String.valueOf(e.getValue()));
334+
}
335+
return metricBuilder;
336+
}
337+
338+
private static Metric.Builder newGrpcMetricBuilder(String grpcMetricName, Attributes attributes) {
339+
Set<String> allowedAttrs = GRPC_METRICS.get(grpcMetricName);
340+
341+
Metric.Builder metricBuilder =
342+
Metric.newBuilder()
343+
.setType("bigtable.googleapis.com/internal/client/" + grpcMetricName.replace('.', '/'));
344+
for (Map.Entry<AttributeKey<?>, Object> e : attributes.asMap().entrySet()) {
345+
String attrKey = e.getKey().getKey();
346+
Object attrValue = e.getValue();
347+
348+
// gRPC metrics are experimental and can change attribute names, to avoid incompatibility with
349+
// the predefined
350+
// metric schemas in stackdriver, filter out unknown keys
351+
if (!allowedAttrs.contains(attrKey)) {
352+
continue;
353+
}
354+
// translate grpc key format to be compatible with cloud monitoring:
355+
// grpc.xds_client.server_failure -> grpc_xds_client_server_failure
356+
String normalizedKey = attrKey.replace('.', '_');
357+
metricBuilder.putLabels(normalizedKey, String.valueOf(attrValue));
358+
}
359+
360+
return metricBuilder;
318361
}
319362

320363
private static MetricKind convertMetricKind(MetricData metricData) {

‎google-cloud-bigtable/src/main/java/com/google/cloud/bigtable/data/v2/stub/metrics/BuiltinMetricsConstants.java

Lines changed: 34 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -50,6 +50,8 @@ public class BuiltinMetricsConstants {
5050
static final AttributeKey<String> STATUS_KEY = AttributeKey.stringKey("status");
5151
static final AttributeKey<String> CLIENT_UID_KEY = AttributeKey.stringKey("client_uid");
5252

53+
public static final String METER_NAME = "bigtable.googleapis.com/internal/client/";
54+
5355
// Metric names
5456
public static final String OPERATION_LATENCIES_NAME = "operation_latencies";
5557
public static final String ATTEMPT_LATENCIES_NAME = "attempt_latencies";
@@ -62,6 +64,38 @@ public class BuiltinMetricsConstants {
6264
static final String CLIENT_BLOCKING_LATENCIES_NAME = "throttling_latencies";
6365
static final String PER_CONNECTION_ERROR_COUNT_NAME = "per_connection_error_count";
6466

67+
public static final Map<String, Set<String>> GRPC_METRICS =
68+
ImmutableMap.<String, Set<String>>builder()
69+
.put(
70+
"grpc.client.attempt.duration",
71+
ImmutableSet.of("grpc.lb.locality", "grpc.method", "grpc.target", "grpc.status"))
72+
.put(
73+
"grpc.lb.rls.default_target_picks",
74+
ImmutableSet.of("grpc.lb.rls.data_plane_target", "grpc.lb.pick_result"))
75+
.put(
76+
"grpc.lb.rls.target_picks",
77+
ImmutableSet.of(
78+
"grpc.target",
79+
"grpc.lb.rls.server_target",
80+
"grpc.lb.rls.data_plane_target",
81+
"grpc.lb.pick_result"))
82+
.put(
83+
"grpc.lb.rls.failed_picks",
84+
ImmutableSet.of("grpc.target", "grpc.lb.rls.server_target"))
85+
// TODO: "grpc.xds_client.connected"
86+
.put("grpc.xds_client.server_failure", ImmutableSet.of("grpc.target", "grpc.xds.server"))
87+
// TODO: "grpc.xds_client.resource_updates_valid",
88+
.put(
89+
"grpc.xds_client.resource_updates_invalid",
90+
ImmutableSet.of("grpc.target", "grpc.xds.server", "grpc.xds.resource_type"))
91+
// TODO: "grpc.xds_client.resources"
92+
.build();
93+
94+
public static final Set<String> INTERNAL_METRICS =
95+
ImmutableSet.of(PER_CONNECTION_ERROR_COUNT_NAME).stream()
96+
.map(m -> METER_NAME + m)
97+
.collect(ImmutableSet.toImmutableSet());
98+
6599
// Buckets under 100,000 are identical to buckets for server side metrics handler_latencies.
66100
// Extending client side bucket to up to 3,200,000.
67101
private static final Aggregation AGGREGATION_WITH_MILLIS_HISTOGRAM =
@@ -97,8 +131,6 @@ public class BuiltinMetricsConstants {
97131
500_000.0,
98132
1_000_000.0));
99133

100-
public static final String METER_NAME = "bigtable.googleapis.com/internal/client/";
101-
102134
static final Set<AttributeKey> COMMON_ATTRIBUTES =
103135
ImmutableSet.of(
104136
BIGTABLE_PROJECT_ID_KEY,

0 commit comments

Comments
 (0)