Skip to content

Commit 522fb8c

Browse files
igorbernstein2igorbernstein2
authored
chore: migrate internal metrics to use bigtable_client monitored resource (#2544)
* chore: migrate internal metrics to use bigtable_client monitored resource This unifies the k8s_container & gce_instance monitored resources. In addition it adds some client specific attributes on the new monitored resource to make it easier to debug customer issues Change-Id: Ic0a3b2efeb486a82dab53028ee524fe473a619ad * fix test Change-Id: I3ed6bd6f3dd678ed474b8a9a5daf3d68626b7569
1 parent 4a51556 commit 522fb8c

File tree

6 files changed

+111
-91
lines changed

6 files changed

+111
-91
lines changed

‎google-cloud-bigtable/src/main/java/com/google/cloud/bigtable/data/v2/stub/BigtableClientContext.java‎

Lines changed: 1 addition & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -94,9 +94,7 @@ public static BigtableClientContext create(EnhancedBigtableStubSettings settings
9494
// no reason to build the internal OtelProvider
9595
if (transportProvider != null) {
9696
internalOtel =
97-
settings
98-
.getInternalMetricsProvider()
99-
.createOtelProvider(credentials, settings.getMetricsEndpoint());
97+
settings.getInternalMetricsProvider().createOtelProvider(settings, credentials);
10098
if (internalOtel != null) {
10199
// Set up per connection error count tracker if all dependencies are met:
102100
// a configurable transport provider + otel

‎google-cloud-bigtable/src/main/java/com/google/cloud/bigtable/data/v2/stub/EnhancedBigtableStubSettings.java‎

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -1346,8 +1346,8 @@ public String toString() {
13461346
@FunctionalInterface
13471347
public interface InternalMetricsProvider {
13481348
@Nullable
1349-
OpenTelemetrySdk createOtelProvider(Credentials creds, @Nullable String endpoint)
1350-
throws IOException;
1349+
OpenTelemetrySdk createOtelProvider(
1350+
EnhancedBigtableStubSettings userSettings, Credentials creds) throws IOException;
13511351
}
13521352

13531353
private static final InternalMetricsProvider DEFAULT_INTERNAL_OTEL_PROVIDER =

‎google-cloud-bigtable/src/main/java/com/google/cloud/bigtable/data/v2/stub/metrics/BigtableCloudMonitoringExporter.java‎

Lines changed: 1 addition & 7 deletions
Original file line numberDiff line numberDiff line change
@@ -322,16 +322,10 @@ static class InternalTimeSeriesConverter implements TimeSeriesConverter {
322322
.map(m -> METER_NAME + m)
323323
.collect(ImmutableList.toImmutableList());
324324

325-
private final String taskId;
326325
private final Supplier<MonitoredResource> monitoredResource;
327326

328327
InternalTimeSeriesConverter(Supplier<MonitoredResource> monitoredResource) {
329-
this(monitoredResource, BigtableExporterUtils.DEFAULT_TASK_VALUE.get());
330-
}
331-
332-
InternalTimeSeriesConverter(Supplier<MonitoredResource> monitoredResource, String taskId) {
333328
this.monitoredResource = monitoredResource;
334-
this.taskId = taskId;
335329
}
336330

337331
@Override
@@ -352,7 +346,7 @@ public Map<ProjectName, List<TimeSeries>> convert(Collection<MetricData> metricD
352346
return ImmutableMap.of(
353347
ProjectName.of(monitoredResource.getLabelsOrThrow(APPLICATION_RESOURCE_PROJECT_ID)),
354348
BigtableExporterUtils.convertToApplicationResourceTimeSeries(
355-
relevantData, taskId, monitoredResource));
349+
relevantData, monitoredResource));
356350
}
357351
}
358352
}

‎google-cloud-bigtable/src/main/java/com/google/cloud/bigtable/data/v2/stub/metrics/BigtableExporterUtils.java‎

Lines changed: 71 additions & 59 deletions
Original file line numberDiff line numberDiff line change
@@ -36,13 +36,16 @@
3636
import com.google.api.Distribution;
3737
import com.google.api.Metric;
3838
import com.google.api.MonitoredResource;
39+
import com.google.cloud.bigtable.Version;
40+
import com.google.cloud.bigtable.data.v2.stub.EnhancedBigtableStubSettings;
3941
import com.google.cloud.opentelemetry.detection.AttributeKeys;
4042
import com.google.cloud.opentelemetry.detection.DetectedPlatform;
4143
import com.google.cloud.opentelemetry.detection.GCPPlatformDetector;
42-
import com.google.common.base.MoreObjects;
4344
import com.google.common.base.Preconditions;
4445
import com.google.common.base.Supplier;
4546
import com.google.common.base.Suppliers;
47+
import com.google.common.collect.ImmutableList;
48+
import com.google.common.collect.ImmutableMap;
4649
import com.google.common.collect.ImmutableSet;
4750
import com.google.monitoring.v3.Point;
4851
import com.google.monitoring.v3.ProjectName;
@@ -65,18 +68,22 @@
6568
import java.net.InetAddress;
6669
import java.net.UnknownHostException;
6770
import java.util.ArrayList;
71+
import java.util.Arrays;
6872
import java.util.Collection;
6973
import java.util.HashMap;
7074
import java.util.List;
7175
import java.util.Map;
76+
import java.util.Objects;
7277
import java.util.Set;
7378
import java.util.UUID;
7479
import java.util.logging.Level;
7580
import java.util.logging.Logger;
81+
import java.util.stream.Collectors;
7682
import javax.annotation.Nullable;
7783

7884
/** Utils to convert OpenTelemetry types to Google Cloud Monitoring types. */
7985
class BigtableExporterUtils {
86+
private static final String CLIENT_NAME = "java-bigtable/" + Version.VERSION;
8087

8188
private static final Logger logger = Logger.getLogger(BigtableExporterUtils.class.getName());
8289

@@ -87,6 +94,11 @@ class BigtableExporterUtils {
8794
ImmutableSet.of(
8895
BIGTABLE_PROJECT_ID_KEY, INSTANCE_ID_KEY, TABLE_ID_KEY, CLUSTER_ID_KEY, ZONE_ID_KEY);
8996

97+
private static final Map<GCPPlatformDetector.SupportedPlatform, String> SUPPORTED_PLATFORM_MAP =
98+
ImmutableMap.of(
99+
GCPPlatformDetector.SupportedPlatform.GOOGLE_COMPUTE_ENGINE, "gcp_compute_engine",
100+
GCPPlatformDetector.SupportedPlatform.GOOGLE_KUBERNETES_ENGINE, "gcp_kubernetes_engine");
101+
90102
private BigtableExporterUtils() {}
91103

92104
/**
@@ -146,7 +158,7 @@ static Map<ProjectName, List<TimeSeries>> convertToBigtableTimeSeries(
146158
}
147159

148160
static List<TimeSeries> convertToApplicationResourceTimeSeries(
149-
Collection<MetricData> collection, String taskId, MonitoredResource applicationResource) {
161+
Collection<MetricData> collection, MonitoredResource applicationResource) {
150162
Preconditions.checkNotNull(
151163
applicationResource,
152164
"convert application metrics is called when the supported resource is not detected");
@@ -160,16 +172,18 @@ static List<TimeSeries> convertToApplicationResourceTimeSeries(
160172
.map(
161173
pointData ->
162174
convertPointToApplicationResourceTimeSeries(
163-
metricData, pointData, taskId, applicationResource))
175+
metricData, pointData, applicationResource))
164176
.forEach(allTimeSeries::add);
165177
}
166178
return allTimeSeries;
167179
}
168180

169181
@Nullable
170-
static MonitoredResource detectResourceSafe() {
182+
static MonitoredResource createInternalMonitoredResource(EnhancedBigtableStubSettings settings) {
171183
try {
172-
return detectResource();
184+
MonitoredResource monitoredResource = detectResource(settings);
185+
logger.log(Level.FINE, "Internal metrics monitored resource: %s", monitoredResource);
186+
return monitoredResource;
173187
} catch (Exception e) {
174188
logger.log(
175189
Level.WARNING,
@@ -180,64 +194,66 @@ static MonitoredResource detectResourceSafe() {
180194
}
181195

182196
@Nullable
183-
private static MonitoredResource detectResource() {
197+
private static MonitoredResource detectResource(EnhancedBigtableStubSettings settings) {
184198
GCPPlatformDetector detector = GCPPlatformDetector.DEFAULT_INSTANCE;
185199
DetectedPlatform detectedPlatform = detector.detectPlatform();
186-
MonitoredResource monitoredResource = null;
187-
try {
188-
switch (detectedPlatform.getSupportedPlatform()) {
189-
case GOOGLE_COMPUTE_ENGINE:
190-
monitoredResource =
191-
createGceMonitoredResource(
192-
detectedPlatform.getProjectId(), detectedPlatform.getAttributes());
193-
break;
194-
case GOOGLE_KUBERNETES_ENGINE:
195-
monitoredResource =
196-
createGkeMonitoredResource(
197-
detectedPlatform.getProjectId(), detectedPlatform.getAttributes());
198-
break;
199-
}
200-
} catch (IllegalStateException e) {
201-
logger.log(
202-
Level.WARNING,
203-
"Failed to create monitored resource for " + detectedPlatform.getSupportedPlatform(),
204-
e);
200+
201+
@Nullable
202+
String cloud_platform = SUPPORTED_PLATFORM_MAP.get(detectedPlatform.getSupportedPlatform());
203+
if (cloud_platform == null) {
204+
return null;
205205
}
206-
return monitoredResource;
207-
}
208206

209-
private static MonitoredResource createGceMonitoredResource(
210-
String projectId, Map<String, String> attributes) {
211-
return MonitoredResource.newBuilder()
212-
.setType("gce_instance")
213-
.putLabels("project_id", projectId)
214-
.putLabels("instance_id", getAttribute(attributes, AttributeKeys.GCE_INSTANCE_ID))
215-
.putLabels("zone", getAttribute(attributes, AttributeKeys.GCE_AVAILABILITY_ZONE))
216-
.build();
217-
}
207+
Map<String, String> attrs = detectedPlatform.getAttributes();
208+
ImmutableList<String> locationKeys =
209+
ImmutableList.of(
210+
AttributeKeys.GCE_CLOUD_REGION,
211+
AttributeKeys.GCE_AVAILABILITY_ZONE,
212+
AttributeKeys.GKE_LOCATION_TYPE_REGION,
213+
AttributeKeys.GKE_CLUSTER_LOCATION);
214+
215+
String region =
216+
locationKeys.stream().map(attrs::get).filter(Objects::nonNull).findFirst().orElse("global");
217+
218+
// Deal with possibility of a zone. Zones are of the form us-east1-c, but we want a region
219+
// which, which is us-east1.
220+
region = Arrays.stream(region.split("-")).limit(2).collect(Collectors.joining("-"));
221+
222+
String hostname = attrs.get(AttributeKeys.GCE_INSTANCE_HOSTNAME);
223+
// if (hostname == null) {
224+
// hostname = attrs.get(AttributeKeys.SERVERLESS_COMPUTE_NAME);
225+
// }
226+
// if (hostname == null) {
227+
// hostname = attrs.get(AttributeKeys.GAE_MODULE_NAME);
228+
// }
229+
if (hostname == null) {
230+
hostname = System.getenv("HOSTNAME");
231+
}
232+
if (hostname == null) {
233+
try {
234+
hostname = InetAddress.getLocalHost().getHostName();
235+
} catch (UnknownHostException ignored) {
236+
}
237+
}
238+
if (hostname == null) {
239+
hostname = "";
240+
}
218241

219-
private static MonitoredResource createGkeMonitoredResource(
220-
String projectId, Map<String, String> attributes) {
221242
return MonitoredResource.newBuilder()
222-
.setType("k8s_container")
223-
.putLabels("project_id", projectId)
224-
.putLabels("location", getAttribute(attributes, AttributeKeys.GKE_CLUSTER_LOCATION))
225-
.putLabels("cluster_name", getAttribute(attributes, AttributeKeys.GKE_CLUSTER_NAME))
226-
.putLabels("namespace_name", MoreObjects.firstNonNull(System.getenv("NAMESPACE"), ""))
227-
.putLabels("pod_name", MoreObjects.firstNonNull(System.getenv("HOSTNAME"), ""))
228-
.putLabels("container_name", MoreObjects.firstNonNull(System.getenv("CONTAINER_NAME"), ""))
243+
.setType("bigtable_client")
244+
.putLabels("project_id", settings.getProjectId())
245+
.putLabels("instance", settings.getInstanceId())
246+
.putLabels("app_profile", settings.getAppProfileId())
247+
.putLabels("client_project", detectedPlatform.getProjectId())
248+
.putLabels("region", region)
249+
.putLabels("cloud_platform", cloud_platform)
250+
.putLabels("host_id", attrs.get(AttributeKeys.GKE_HOST_ID))
251+
.putLabels("host_name", hostname)
252+
.putLabels("client_name", CLIENT_NAME)
253+
.putLabels("uuid", DEFAULT_TASK_VALUE.get())
229254
.build();
230255
}
231256

232-
private static String getAttribute(Map<String, String> attributes, String key) {
233-
String value = attributes.get(key);
234-
if (value == null) {
235-
throw new IllegalStateException(
236-
"Required attribute " + key + " does not exist in the attributes map " + attributes);
237-
}
238-
return value;
239-
}
240-
241257
private static TimeSeries convertPointToBigtableTimeSeries(
242258
MetricData metricData, PointData pointData, String taskId) {
243259
TimeSeries.Builder builder =
@@ -275,10 +291,7 @@ private static TimeSeries convertPointToBigtableTimeSeries(
275291
}
276292

277293
private static TimeSeries convertPointToApplicationResourceTimeSeries(
278-
MetricData metricData,
279-
PointData pointData,
280-
String taskId,
281-
MonitoredResource applicationResource) {
294+
MetricData metricData, PointData pointData, MonitoredResource applicationResource) {
282295
TimeSeries.Builder builder =
283296
TimeSeries.newBuilder()
284297
.setMetricKind(convertMetricKind(metricData))
@@ -292,7 +305,6 @@ private static TimeSeries convertPointToApplicationResourceTimeSeries(
292305
metricBuilder.putLabels(key.getKey(), String.valueOf(attributes.get(key)));
293306
}
294307

295-
metricBuilder.putLabels(CLIENT_UID_KEY.getKey(), taskId);
296308
builder.setMetric(metricBuilder.build());
297309

298310
TimeInterval timeInterval =

‎google-cloud-bigtable/src/main/java/com/google/cloud/bigtable/data/v2/stub/metrics/Util.java‎

Lines changed: 5 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -34,6 +34,7 @@
3434
import com.google.bigtable.v2.ResponseParams;
3535
import com.google.bigtable.v2.SampleRowKeysRequest;
3636
import com.google.bigtable.v2.TableName;
37+
import com.google.cloud.bigtable.data.v2.stub.EnhancedBigtableStubSettings;
3738
import com.google.common.base.Suppliers;
3839
import com.google.common.collect.ImmutableMap;
3940
import com.google.protobuf.InvalidProtocolBufferException;
@@ -248,7 +249,7 @@ static GrpcCallContext injectBigtableStreamTracer(
248249
}
249250

250251
public static OpenTelemetrySdk newInternalOpentelemetry(
251-
Credentials credentials, @Nullable String metricsEndpoint) throws IOException {
252+
EnhancedBigtableStubSettings settings, Credentials credentials) throws IOException {
252253
SdkMeterProviderBuilder meterProviderBuilder = SdkMeterProvider.builder();
253254

254255
for (Map.Entry<InstrumentSelector, View> e :
@@ -261,9 +262,10 @@ public static OpenTelemetrySdk newInternalOpentelemetry(
261262
BigtableCloudMonitoringExporter.create(
262263
"application metrics",
263264
credentials,
264-
metricsEndpoint,
265+
settings.getMetricsEndpoint(),
265266
new BigtableCloudMonitoringExporter.InternalTimeSeriesConverter(
266-
Suppliers.memoize(BigtableExporterUtils::detectResourceSafe)))));
267+
Suppliers.memoize(
268+
() -> BigtableExporterUtils.createInternalMonitoredResource(settings))))));
267269
return OpenTelemetrySdk.builder().setMeterProvider(meterProviderBuilder.build()).build();
268270
}
269271
}

‎google-cloud-bigtable/src/test/java/com/google/cloud/bigtable/data/v2/stub/metrics/BigtableCloudMonitoringExporterTest.java‎

Lines changed: 31 additions & 17 deletions
Original file line numberDiff line numberDiff line change
@@ -315,11 +315,18 @@ public void testTimeSeriesForMetricWithGceOrGkeResource() {
315315
new BigtableCloudMonitoringExporter.InternalTimeSeriesConverter(
316316
Suppliers.ofInstance(
317317
MonitoredResource.newBuilder()
318-
.setType("gce-instance")
319-
.putLabels("some-gce-key", "some-gce-value")
318+
.setType("bigtable_client")
320319
.putLabels("project_id", gceProjectId)
321-
.build()),
322-
taskId));
320+
.putLabels("instance", "resource-instance")
321+
.putLabels("app_profile", "resource-app-profile")
322+
.putLabels("client_project", "client-project")
323+
.putLabels("region", "cleint-region")
324+
.putLabels("cloud_platform", "gce_instance")
325+
.putLabels("host_id", "1234567890")
326+
.putLabels("host_name", "harold")
327+
.putLabels("client_name", "java/1234")
328+
.putLabels("uuid", "something")
329+
.build())));
323330
ArgumentCaptor<CreateTimeSeriesRequest> argumentCaptor =
324331
ArgumentCaptor.forClass(CreateTimeSeriesRequest.class);
325332

@@ -371,21 +378,28 @@ public void testTimeSeriesForMetricWithGceOrGkeResource() {
371378
com.google.monitoring.v3.TimeSeries timeSeries = request.getTimeSeriesList().get(0);
372379

373380
assertThat(timeSeries.getResource().getLabelsMap())
374-
.containsExactly("some-gce-key", "some-gce-value", "project_id", gceProjectId);
381+
.isEqualTo(
382+
ImmutableMap.<String, String>builder()
383+
.put("project_id", gceProjectId)
384+
.put("instance", "resource-instance")
385+
.put("app_profile", "resource-app-profile")
386+
.put("client_project", "client-project")
387+
.put("region", "cleint-region")
388+
.put("cloud_platform", "gce_instance")
389+
.put("host_id", "1234567890")
390+
.put("host_name", "harold")
391+
.put("client_name", "java/1234")
392+
.put("uuid", "something")
393+
.build());
375394

376-
assertThat(timeSeries.getMetric().getLabelsMap()).hasSize(5);
377395
assertThat(timeSeries.getMetric().getLabelsMap())
378-
.containsAtLeast(
379-
BIGTABLE_PROJECT_ID_KEY.getKey(),
380-
projectId,
381-
INSTANCE_ID_KEY.getKey(),
382-
instanceId,
383-
APP_PROFILE_KEY.getKey(),
384-
appProfileId,
385-
CLIENT_NAME_KEY.getKey(),
386-
clientName,
387-
CLIENT_UID_KEY.getKey(),
388-
taskId);
396+
.isEqualTo(
397+
ImmutableMap.builder()
398+
.put(BIGTABLE_PROJECT_ID_KEY.getKey(), projectId)
399+
.put(INSTANCE_ID_KEY.getKey(), instanceId)
400+
.put(APP_PROFILE_KEY.getKey(), appProfileId)
401+
.put(CLIENT_NAME_KEY.getKey(), clientName)
402+
.build());
389403
}
390404

391405
@Test

0 commit comments

Comments
 (0)