diff --git a/google-cloud-spanner/pom.xml b/google-cloud-spanner/pom.xml index b6fde80b058..5e7328d9634 100644 --- a/google-cloud-spanner/pom.xml +++ b/google-cloud-spanner/pom.xml @@ -371,7 +371,7 @@ junit test - + com.google.api.grpc diff --git a/google-cloud-spanner/src/main/java/com/google/cloud/spanner/BuiltInMetricsConstant.java b/google-cloud-spanner/src/main/java/com/google/cloud/spanner/BuiltInMetricsConstant.java index 4f8b091d550..4adf53d7e40 100644 --- a/google-cloud-spanner/src/main/java/com/google/cloud/spanner/BuiltInMetricsConstant.java +++ b/google-cloud-spanner/src/main/java/com/google/cloud/spanner/BuiltInMetricsConstant.java @@ -34,9 +34,9 @@ public class BuiltInMetricsConstant { public static final String METER_NAME = "spanner.googleapis.com/internal/client"; - public static final String GAX_METER_NAME = OpenTelemetryMetricsRecorder.GAX_METER_NAME; - + static final String SPANNER_METER_NAME = "spanner-java"; + static final String GFE_LATENCIES_NAME = "gfe_latencies"; static final String OPERATION_LATENCIES_NAME = "operation_latencies"; static final String ATTEMPT_LATENCIES_NAME = "attempt_latencies"; static final String OPERATION_LATENCY_NAME = "operation_latency"; @@ -49,7 +49,8 @@ public class BuiltInMetricsConstant { OPERATION_LATENCIES_NAME, ATTEMPT_LATENCIES_NAME, OPERATION_COUNT_NAME, - ATTEMPT_COUNT_NAME) + ATTEMPT_COUNT_NAME, + GFE_LATENCIES_NAME) .stream() .map(m -> METER_NAME + '/' + m) .collect(Collectors.toSet()); @@ -114,6 +115,7 @@ static Map getAllViews() { ImmutableMap.Builder views = ImmutableMap.builder(); defineView( views, + BuiltInMetricsConstant.GAX_METER_NAME, BuiltInMetricsConstant.OPERATION_LATENCY_NAME, BuiltInMetricsConstant.OPERATION_LATENCIES_NAME, BuiltInMetricsConstant.AGGREGATION_WITH_MILLIS_HISTOGRAM, @@ -121,6 +123,7 @@ static Map getAllViews() { "ms"); defineView( views, + BuiltInMetricsConstant.GAX_METER_NAME, BuiltInMetricsConstant.ATTEMPT_LATENCY_NAME, BuiltInMetricsConstant.ATTEMPT_LATENCIES_NAME, BuiltInMetricsConstant.AGGREGATION_WITH_MILLIS_HISTOGRAM, @@ -128,6 +131,15 @@ static Map getAllViews() { "ms"); defineView( views, + BuiltInMetricsConstant.SPANNER_METER_NAME, + BuiltInMetricsConstant.GFE_LATENCIES_NAME, + BuiltInMetricsConstant.GFE_LATENCIES_NAME, + BuiltInMetricsConstant.AGGREGATION_WITH_MILLIS_HISTOGRAM, + InstrumentType.HISTOGRAM, + "ms"); + defineView( + views, + BuiltInMetricsConstant.GAX_METER_NAME, BuiltInMetricsConstant.OPERATION_COUNT_NAME, BuiltInMetricsConstant.OPERATION_COUNT_NAME, Aggregation.sum(), @@ -135,6 +147,7 @@ static Map getAllViews() { "1"); defineView( views, + BuiltInMetricsConstant.GAX_METER_NAME, BuiltInMetricsConstant.ATTEMPT_COUNT_NAME, BuiltInMetricsConstant.ATTEMPT_COUNT_NAME, Aggregation.sum(), @@ -145,6 +158,7 @@ static Map getAllViews() { private static void defineView( ImmutableMap.Builder viewMap, + String meterName, String metricName, String metricViewName, Aggregation aggregation, @@ -153,7 +167,7 @@ private static void defineView( InstrumentSelector selector = InstrumentSelector.builder() .setName(BuiltInMetricsConstant.METER_NAME + '/' + metricName) - .setMeterName(BuiltInMetricsConstant.GAX_METER_NAME) + .setMeterName(meterName) .setType(type) .setUnit(unit) .build(); diff --git a/google-cloud-spanner/src/main/java/com/google/cloud/spanner/BuiltInOpenTelemetryMetricsProvider.java b/google-cloud-spanner/src/main/java/com/google/cloud/spanner/BuiltInMetricsProvider.java similarity index 95% rename from google-cloud-spanner/src/main/java/com/google/cloud/spanner/BuiltInOpenTelemetryMetricsProvider.java rename to google-cloud-spanner/src/main/java/com/google/cloud/spanner/BuiltInMetricsProvider.java index 4aeb98987d1..f624f310f77 100644 --- a/google-cloud-spanner/src/main/java/com/google/cloud/spanner/BuiltInOpenTelemetryMetricsProvider.java +++ b/google-cloud-spanner/src/main/java/com/google/cloud/spanner/BuiltInMetricsProvider.java @@ -46,25 +46,24 @@ import java.util.logging.Logger; import javax.annotation.Nullable; -final class BuiltInOpenTelemetryMetricsProvider { +final class BuiltInMetricsProvider { - static BuiltInOpenTelemetryMetricsProvider INSTANCE = new BuiltInOpenTelemetryMetricsProvider(); + static BuiltInMetricsProvider INSTANCE = new BuiltInMetricsProvider(); - private static final Logger logger = - Logger.getLogger(BuiltInOpenTelemetryMetricsProvider.class.getName()); + private static final Logger logger = Logger.getLogger(BuiltInMetricsProvider.class.getName()); private static String taskId; private OpenTelemetry openTelemetry; - private BuiltInOpenTelemetryMetricsProvider() {} + private BuiltInMetricsProvider() {} OpenTelemetry getOrCreateOpenTelemetry( String projectId, @Nullable Credentials credentials, @Nullable String monitoringHost) { try { if (this.openTelemetry == null) { SdkMeterProviderBuilder sdkMeterProviderBuilder = SdkMeterProvider.builder(); - BuiltInOpenTelemetryMetricsView.registerBuiltinMetrics( + BuiltInMetricsView.registerBuiltinMetrics( SpannerCloudMonitoringExporter.create(projectId, credentials, monitoringHost), sdkMeterProviderBuilder); SdkMeterProvider sdkMeterProvider = sdkMeterProviderBuilder.build(); diff --git a/google-cloud-spanner/src/main/java/com/google/cloud/spanner/BuiltInMetricsRecorder.java b/google-cloud-spanner/src/main/java/com/google/cloud/spanner/BuiltInMetricsRecorder.java new file mode 100644 index 00000000000..a12da470b61 --- /dev/null +++ b/google-cloud-spanner/src/main/java/com/google/cloud/spanner/BuiltInMetricsRecorder.java @@ -0,0 +1,82 @@ +/* + * Copyright 2025 Google LLC + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.google.cloud.spanner; + +import com.google.api.gax.core.GaxProperties; +import com.google.api.gax.tracing.OpenTelemetryMetricsRecorder; +import com.google.common.base.Preconditions; +import io.opentelemetry.api.OpenTelemetry; +import io.opentelemetry.api.common.Attributes; +import io.opentelemetry.api.common.AttributesBuilder; +import io.opentelemetry.api.metrics.DoubleHistogram; +import io.opentelemetry.api.metrics.Meter; +import java.util.Map; + +/** + * Implementation for recording built in metrics. + * + *

This class extends the {@link OpenTelemetryMetricsRecorder} which implements the * + * measurements related to the lifecyle of an RPC. + */ +class BuiltInMetricsRecorder extends OpenTelemetryMetricsRecorder { + + private final DoubleHistogram gfeLatencyRecorder; + + /** + * Creates the following instruments for the following metrics: + * + *

    + *
  • GFE Latency: Histogram + *
+ * + * @param openTelemetry OpenTelemetry instance + * @param serviceName Service Name + */ + BuiltInMetricsRecorder(OpenTelemetry openTelemetry, String serviceName) { + super(openTelemetry, serviceName); + Meter meter = + openTelemetry + .meterBuilder(BuiltInMetricsConstant.SPANNER_METER_NAME) + .setInstrumentationVersion(GaxProperties.getLibraryVersion(getClass())) + .build(); + this.gfeLatencyRecorder = + meter + .histogramBuilder(serviceName + '/' + BuiltInMetricsConstant.GFE_LATENCIES_NAME) + .setDescription( + "Latency between Google's network receiving an RPC and reading back the first byte of the response") + .setUnit("ms") + .build(); + } + + /** + * Record the latency between Google's network receiving an RPC and reading back the first byte of + * the response. Data is stored in a Histogram. + * + * @param gfeLatency Attempt Latency in ms + * @param attributes Map of the attributes to store + */ + void recordGFELatency(double gfeLatency, Map attributes) { + gfeLatencyRecorder.record(gfeLatency, toOtelAttributes(attributes)); + } + + Attributes toOtelAttributes(Map attributes) { + Preconditions.checkNotNull(attributes, "Attributes map cannot be null"); + AttributesBuilder attributesBuilder = Attributes.builder(); + attributes.forEach(attributesBuilder::put); + return attributesBuilder.build(); + } +} diff --git a/google-cloud-spanner/src/main/java/com/google/cloud/spanner/BuiltInMetricsTracer.java b/google-cloud-spanner/src/main/java/com/google/cloud/spanner/BuiltInMetricsTracer.java new file mode 100644 index 00000000000..6faff5ad6d7 --- /dev/null +++ b/google-cloud-spanner/src/main/java/com/google/cloud/spanner/BuiltInMetricsTracer.java @@ -0,0 +1,156 @@ +/* + * Copyright 2024 Google LLC + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.google.cloud.spanner; + +import com.google.api.gax.rpc.ApiException; +import com.google.api.gax.rpc.StatusCode; +import com.google.api.gax.tracing.ApiTracer; +import com.google.api.gax.tracing.MethodName; +import com.google.api.gax.tracing.MetricsTracer; +import java.util.HashMap; +import java.util.Map; +import java.util.concurrent.CancellationException; +import javax.annotation.Nullable; + +/** + * Implements built-in metrics tracer. + * + *

This class extends the {@link MetricsTracer} which computes generic metrics that can be + * observed in the lifecycle of an RPC operation. + */ +class BuiltInMetricsTracer extends MetricsTracer implements ApiTracer { + + private final BuiltInMetricsRecorder builtInOpenTelemetryMetricsRecorder; + // These are RPC specific attributes and pertain to a specific API Trace + private final Map attributes = new HashMap<>(); + + private Long gfeLatency = null; + + BuiltInMetricsTracer( + MethodName methodName, BuiltInMetricsRecorder builtInOpenTelemetryMetricsRecorder) { + super(methodName, builtInOpenTelemetryMetricsRecorder); + this.builtInOpenTelemetryMetricsRecorder = builtInOpenTelemetryMetricsRecorder; + this.attributes.put(METHOD_ATTRIBUTE, methodName.toString()); + } + + /** + * Adds an annotation that the attempt succeeded. Successful attempt add "OK" value to the status + * attribute key. + */ + @Override + public void attemptSucceeded() { + super.attemptSucceeded(); + if (gfeLatency != null) { + attributes.put(STATUS_ATTRIBUTE, StatusCode.Code.OK.toString()); + builtInOpenTelemetryMetricsRecorder.recordGFELatency(gfeLatency, attributes); + } + } + + /** + * Add an annotation that the attempt was cancelled by the user. Cancelled attempt add "CANCELLED" + * to the status attribute key. + */ + @Override + public void attemptCancelled() { + super.attemptCancelled(); + if (gfeLatency != null) { + attributes.put(STATUS_ATTRIBUTE, StatusCode.Code.CANCELLED.toString()); + builtInOpenTelemetryMetricsRecorder.recordGFELatency(gfeLatency, attributes); + } + } + + /** + * Adds an annotation that the attempt failed, but another attempt will be made after the delay. + * + * @param error the error that caused the attempt to fail. + * @param delay the amount of time to wait before the next attempt will start. + *

Failed attempt extracts the error from the throwable and adds it to the status attribute + * key. + */ + @Override + public void attemptFailedDuration(Throwable error, java.time.Duration delay) { + super.attemptFailedDuration(error, delay); + if (gfeLatency != null) { + attributes.put(STATUS_ATTRIBUTE, extractStatus(error)); + builtInOpenTelemetryMetricsRecorder.recordGFELatency(gfeLatency, attributes); + } + } + + /** + * Adds an annotation that the attempt failed and that no further attempts will be made because + * retry limits have been reached. This extracts the error from the throwable and adds it to the + * status attribute key. + * + * @param error the last error received before retries were exhausted. + */ + @Override + public void attemptFailedRetriesExhausted(Throwable error) { + super.attemptFailedRetriesExhausted(error); + if (gfeLatency != null) { + attributes.put(STATUS_ATTRIBUTE, extractStatus(error)); + builtInOpenTelemetryMetricsRecorder.recordGFELatency(gfeLatency, attributes); + } + } + + /** + * Adds an annotation that the attempt failed and that no further attempts will be made because + * the last error was not retryable. This extracts the error from the throwable and adds it to the + * status attribute key. + * + * @param error the error that caused the final attempt to fail. + */ + @Override + public void attemptPermanentFailure(Throwable error) { + super.attemptPermanentFailure(error); + if (gfeLatency != null) { + attributes.put(STATUS_ATTRIBUTE, extractStatus(error)); + builtInOpenTelemetryMetricsRecorder.recordGFELatency(gfeLatency, attributes); + } + } + + void recordGFELatency(Long gfeLatency) { + this.gfeLatency = gfeLatency; + } + + @Override + public void addAttributes(Map attributes) { + super.addAttributes(attributes); + this.attributes.putAll(attributes); + }; + + @Override + public void addAttributes(String key, String value) { + super.addAttributes(key, value); + this.attributes.put(key, value); + } + + private static String extractStatus(@Nullable Throwable error) { + final String statusString; + + if (error == null) { + return StatusCode.Code.OK.toString(); + } else if (error instanceof CancellationException) { + statusString = StatusCode.Code.CANCELLED.toString(); + } else if (error instanceof ApiException) { + statusString = ((ApiException) error).getStatusCode().getCode().toString(); + } else { + statusString = StatusCode.Code.UNKNOWN.toString(); + } + + return statusString; + } +} diff --git a/google-cloud-spanner/src/main/java/com/google/cloud/spanner/BuiltInMetricsTracerFactory.java b/google-cloud-spanner/src/main/java/com/google/cloud/spanner/BuiltInMetricsTracerFactory.java new file mode 100644 index 00000000000..42c19dd72a0 --- /dev/null +++ b/google-cloud-spanner/src/main/java/com/google/cloud/spanner/BuiltInMetricsTracerFactory.java @@ -0,0 +1,62 @@ +/* + * Copyright 2024 Google LLC + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.google.cloud.spanner; + +import com.google.api.gax.tracing.ApiTracer; +import com.google.api.gax.tracing.ApiTracerFactory; +import com.google.api.gax.tracing.MethodName; +import com.google.api.gax.tracing.MetricsTracer; +import com.google.api.gax.tracing.MetricsTracerFactory; +import com.google.api.gax.tracing.SpanName; +import com.google.common.collect.ImmutableMap; +import java.util.Map; + +/** + * A {@link ApiTracerFactory} to build instances of {@link MetricsTracer}. + * + *

This class extends the {@link MetricsTracerFactory} which wraps the {@link + * BuiltInMetricsRecorder} and pass it to {@link BuiltInMetricsTracer}. It will be * used to record + * metrics in {@link BuiltInMetricsTracer}. + * + *

This class is expected to be initialized once during client initialization. + */ +class BuiltInMetricsTracerFactory extends MetricsTracerFactory { + + protected BuiltInMetricsRecorder builtInMetricsRecorder; + private final Map attributes; + + /** + * Pass in a Map of client level attributes which will be added to every single MetricsTracer + * created from the ApiTracerFactory. + */ + public BuiltInMetricsTracerFactory( + BuiltInMetricsRecorder builtInMetricsRecorder, Map attributes) { + super(builtInMetricsRecorder, attributes); + this.builtInMetricsRecorder = builtInMetricsRecorder; + this.attributes = ImmutableMap.copyOf(attributes); + } + + @Override + public ApiTracer newTracer(ApiTracer parent, SpanName spanName, OperationType operationType) { + BuiltInMetricsTracer metricsTracer = + new BuiltInMetricsTracer( + MethodName.of(spanName.getClientName(), spanName.getMethodName()), + builtInMetricsRecorder); + metricsTracer.addAttributes(attributes); + return metricsTracer; + } +} diff --git a/google-cloud-spanner/src/main/java/com/google/cloud/spanner/BuiltInOpenTelemetryMetricsView.java b/google-cloud-spanner/src/main/java/com/google/cloud/spanner/BuiltInMetricsView.java similarity index 93% rename from google-cloud-spanner/src/main/java/com/google/cloud/spanner/BuiltInOpenTelemetryMetricsView.java rename to google-cloud-spanner/src/main/java/com/google/cloud/spanner/BuiltInMetricsView.java index 4a09c0d856a..e72eeb9425a 100644 --- a/google-cloud-spanner/src/main/java/com/google/cloud/spanner/BuiltInOpenTelemetryMetricsView.java +++ b/google-cloud-spanner/src/main/java/com/google/cloud/spanner/BuiltInMetricsView.java @@ -20,9 +20,9 @@ import io.opentelemetry.sdk.metrics.export.MetricExporter; import io.opentelemetry.sdk.metrics.export.PeriodicMetricReader; -class BuiltInOpenTelemetryMetricsView { +class BuiltInMetricsView { - private BuiltInOpenTelemetryMetricsView() {} + private BuiltInMetricsView() {} /** Register built-in metrics on the {@link SdkMeterProviderBuilder} with credentials. */ static void registerBuiltinMetrics( diff --git a/google-cloud-spanner/src/main/java/com/google/cloud/spanner/CompositeTracer.java b/google-cloud-spanner/src/main/java/com/google/cloud/spanner/CompositeTracer.java index 60d7081cc1e..5268e9046f8 100644 --- a/google-cloud-spanner/src/main/java/com/google/cloud/spanner/CompositeTracer.java +++ b/google-cloud-spanner/src/main/java/com/google/cloud/spanner/CompositeTracer.java @@ -190,4 +190,12 @@ public void addAttributes(Map attributes) { } } } + + public void recordGFELatency(Long gfeLatency) { + for (ApiTracer child : children) { + if (child instanceof BuiltInMetricsTracer) { + ((BuiltInMetricsTracer) child).recordGFELatency(gfeLatency); + } + } + } } diff --git a/google-cloud-spanner/src/main/java/com/google/cloud/spanner/SpannerCloudMonitoringExporterUtils.java b/google-cloud-spanner/src/main/java/com/google/cloud/spanner/SpannerCloudMonitoringExporterUtils.java index 21fcba8194d..620430b87df 100644 --- a/google-cloud-spanner/src/main/java/com/google/cloud/spanner/SpannerCloudMonitoringExporterUtils.java +++ b/google-cloud-spanner/src/main/java/com/google/cloud/spanner/SpannerCloudMonitoringExporterUtils.java @@ -25,6 +25,7 @@ import static com.google.cloud.spanner.BuiltInMetricsConstant.GAX_METER_NAME; import static com.google.cloud.spanner.BuiltInMetricsConstant.INSTANCE_ID_KEY; import static com.google.cloud.spanner.BuiltInMetricsConstant.PROJECT_ID_KEY; +import static com.google.cloud.spanner.BuiltInMetricsConstant.SPANNER_METER_NAME; import static com.google.cloud.spanner.BuiltInMetricsConstant.SPANNER_PROMOTED_RESOURCE_LABELS; import static com.google.cloud.spanner.BuiltInMetricsConstant.SPANNER_RESOURCE_TYPE; @@ -75,8 +76,9 @@ static List convertToSpannerTimeSeries(List collection) List allTimeSeries = new ArrayList<>(); for (MetricData metricData : collection) { - // Get common metrics data from GAX library - if (!metricData.getInstrumentationScopeInfo().getName().equals(GAX_METER_NAME)) { + // Get metrics data from GAX library and Spanner library + if (!(metricData.getInstrumentationScopeInfo().getName().equals(GAX_METER_NAME) + || metricData.getInstrumentationScopeInfo().getName().equals(SPANNER_METER_NAME))) { // Filter out metric data for instruments that are not part of the spanner metrics list continue; } diff --git a/google-cloud-spanner/src/main/java/com/google/cloud/spanner/SpannerOptions.java b/google-cloud-spanner/src/main/java/com/google/cloud/spanner/SpannerOptions.java index 42fc0c2d0bd..5b63ff4fe44 100644 --- a/google-cloud-spanner/src/main/java/com/google/cloud/spanner/SpannerOptions.java +++ b/google-cloud-spanner/src/main/java/com/google/cloud/spanner/SpannerOptions.java @@ -33,8 +33,6 @@ import com.google.api.gax.rpc.TransportChannelProvider; import com.google.api.gax.tracing.ApiTracerFactory; import com.google.api.gax.tracing.BaseApiTracerFactory; -import com.google.api.gax.tracing.MetricsTracerFactory; -import com.google.api.gax.tracing.OpenTelemetryMetricsRecorder; import com.google.api.gax.tracing.OpencensusTracerFactory; import com.google.cloud.NoCredentials; import com.google.cloud.ServiceDefaults; @@ -144,8 +142,7 @@ public class SpannerOptions extends ServiceOptions { private final boolean autoThrottleAdministrativeRequests; private final RetrySettings retryAdministrativeRequestsSettings; private final boolean trackTransactionStarter; - private final BuiltInOpenTelemetryMetricsProvider builtInOpenTelemetryMetricsProvider = - BuiltInOpenTelemetryMetricsProvider.INSTANCE; + private final BuiltInMetricsProvider builtInMetricsProvider = BuiltInMetricsProvider.INSTANCE; /** * These are the default {@link QueryOptions} defined by the user on this {@link SpannerOptions}. */ @@ -1910,13 +1907,13 @@ private ApiTracerFactory getDefaultApiTracerFactory() { private ApiTracerFactory createMetricsApiTracerFactory() { OpenTelemetry openTelemetry = - this.builtInOpenTelemetryMetricsProvider.getOrCreateOpenTelemetry( + this.builtInMetricsProvider.getOrCreateOpenTelemetry( this.getProjectId(), getCredentials(), this.monitoringHost); return openTelemetry != null - ? new MetricsTracerFactory( - new OpenTelemetryMetricsRecorder(openTelemetry, BuiltInMetricsConstant.METER_NAME), - builtInOpenTelemetryMetricsProvider.createClientAttributes( + ? new BuiltInMetricsTracerFactory( + new BuiltInMetricsRecorder(openTelemetry, BuiltInMetricsConstant.METER_NAME), + builtInMetricsProvider.createClientAttributes( this.getProjectId(), "spanner-java/" + GaxProperties.getLibraryVersion(getClass()))) : null; } diff --git a/google-cloud-spanner/src/main/java/com/google/cloud/spanner/spi/v1/HeaderInterceptor.java b/google-cloud-spanner/src/main/java/com/google/cloud/spanner/spi/v1/HeaderInterceptor.java index e4eec68b278..dba3b38e92f 100644 --- a/google-cloud-spanner/src/main/java/com/google/cloud/spanner/spi/v1/HeaderInterceptor.java +++ b/google-cloud-spanner/src/main/java/com/google/cloud/spanner/spi/v1/HeaderInterceptor.java @@ -71,9 +71,11 @@ class HeaderInterceptor implements ClientInterceptor { DatabaseName.of("undefined-project", "undefined-instance", "undefined-database"); private static final Metadata.Key SERVER_TIMING_HEADER_KEY = Metadata.Key.of("server-timing", Metadata.ASCII_STRING_MARSHALLER); - private static final String SERVER_TIMING_HEADER_PREFIX = "gfet4t7; dur="; + private static final String GFE_TIMING_HEADER = "gfet4t7"; private static final Metadata.Key GOOGLE_CLOUD_RESOURCE_PREFIX_KEY = Metadata.Key.of("google-cloud-resource-prefix", Metadata.ASCII_STRING_MARSHALLER); + private static final Pattern SERVER_TIMING_PATTERN = + Pattern.compile("(?[a-zA-Z0-9_-]+);\\s*dur=(?\\d+)"); private static final Pattern GOOGLE_CLOUD_RESOURCE_PREFIX_PATTERN = Pattern.compile( ".*projects/(?\\p{ASCII}[^/]*)(/instances/(?\\p{ASCII}[^/]*))?(/databases/(?\\p{ASCII}[^/]*))?"); @@ -128,7 +130,7 @@ public void onHeaders(Metadata metadata) { Boolean isDirectPathUsed = isDirectPathUsed(getAttributes().get(Grpc.TRANSPORT_ATTR_REMOTE_ADDR)); addDirectPathUsedAttribute(compositeTracer, isDirectPathUsed); - processHeader(metadata, tagContext, attributes, span); + processHeader(metadata, tagContext, attributes, span, compositeTracer); super.onHeaders(metadata); } }, @@ -142,29 +144,61 @@ public void onHeaders(Metadata metadata) { } private void processHeader( - Metadata metadata, TagContext tagContext, Attributes attributes, Span span) { + Metadata metadata, + TagContext tagContext, + Attributes attributes, + Span span, + CompositeTracer compositeTracer) { MeasureMap measureMap = STATS_RECORDER.newMeasureMap(); String serverTiming = metadata.get(SERVER_TIMING_HEADER_KEY); - if (serverTiming != null && serverTiming.startsWith(SERVER_TIMING_HEADER_PREFIX)) { - try { - long latency = Long.parseLong(serverTiming.substring(SERVER_TIMING_HEADER_PREFIX.length())); - measureMap.put(SPANNER_GFE_LATENCY, latency); + try { + // Previous implementation parsed the GFE latency directly using: + // long latency = Long.parseLong(serverTiming.substring("gfet4t7; dur=".length())); + // This approach assumed the serverTiming header contained exactly one metric "gfet4t7". + // If additional metrics were introduced in the header, older versions of the library + // would fail to parse it correctly. To make the parsing more robust, the logic has been + // updated to handle multiple metrics gracefully. + + Map serverTimingMetrics = parseServerTimingHeader(serverTiming); + if (serverTimingMetrics.containsKey(GFE_TIMING_HEADER)) { + long gfeLatency = serverTimingMetrics.get(GFE_TIMING_HEADER); + + measureMap.put(SPANNER_GFE_LATENCY, gfeLatency); measureMap.put(SPANNER_GFE_HEADER_MISSING_COUNT, 0L); measureMap.record(tagContext); - spannerRpcMetrics.recordGfeLatency(latency, attributes); + spannerRpcMetrics.recordGfeLatency(gfeLatency, attributes); spannerRpcMetrics.recordGfeHeaderMissingCount(0L, attributes); + if (compositeTracer != null) { + compositeTracer.recordGFELatency(gfeLatency); + } if (span != null) { - span.setAttribute("gfe_latency", String.valueOf(latency)); + span.setAttribute("gfe_latency", String.valueOf(gfeLatency)); + } + } else { + measureMap.put(SPANNER_GFE_HEADER_MISSING_COUNT, 1L).record(tagContext); + spannerRpcMetrics.recordGfeHeaderMissingCount(1L, attributes); + } + } catch (NumberFormatException e) { + LOGGER.log(LEVEL, "Invalid server-timing object in header: {}", serverTiming); + } + } + + private Map parseServerTimingHeader(String serverTiming) { + Map serverTimingMetrics = new HashMap<>(); + if (serverTiming != null) { + Matcher matcher = SERVER_TIMING_PATTERN.matcher(serverTiming); + while (matcher.find()) { + String metricName = matcher.group("metricName"); + String durationStr = matcher.group("duration"); + + if (metricName != null && durationStr != null) { + serverTimingMetrics.put(metricName, Long.valueOf(durationStr)); } - } catch (NumberFormatException e) { - LOGGER.log(LEVEL, "Invalid server-timing object in header: {}", serverTiming); } - } else { - spannerRpcMetrics.recordGfeHeaderMissingCount(1L, attributes); - measureMap.put(SPANNER_GFE_HEADER_MISSING_COUNT, 1L).record(tagContext); } + return serverTimingMetrics; } private DatabaseName extractDatabaseName(Metadata headers) throws ExecutionException { diff --git a/google-cloud-spanner/src/test/java/com/google/cloud/spanner/AbstractNettyMockServerTest.java b/google-cloud-spanner/src/test/java/com/google/cloud/spanner/AbstractNettyMockServerTest.java new file mode 100644 index 00000000000..8e8da054b08 --- /dev/null +++ b/google-cloud-spanner/src/test/java/com/google/cloud/spanner/AbstractNettyMockServerTest.java @@ -0,0 +1,114 @@ +/* + * Copyright 2023 Google LLC + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.google.cloud.spanner; + +import com.google.api.gax.grpc.testing.LocalChannelProvider; +import com.google.cloud.NoCredentials; +import io.grpc.ForwardingServerCall; +import io.grpc.ManagedChannelBuilder; +import io.grpc.Metadata; +import io.grpc.Server; +import io.grpc.ServerCall; +import io.grpc.ServerCallHandler; +import io.grpc.ServerInterceptor; +import io.grpc.netty.shaded.io.grpc.netty.NettyServerBuilder; +import java.io.IOException; +import java.net.InetSocketAddress; +import java.util.Random; +import java.util.concurrent.ExecutorService; +import java.util.concurrent.Executors; +import java.util.concurrent.atomic.AtomicInteger; +import org.junit.After; +import org.junit.AfterClass; +import org.junit.Before; +import org.junit.BeforeClass; + +abstract class AbstractNettyMockServerTest { + protected static MockSpannerServiceImpl mockSpanner; + + protected static Server server; + protected static InetSocketAddress address; + static ExecutorService executor; + protected static LocalChannelProvider channelProvider; + protected static AtomicInteger fakeServerTiming = + new AtomicInteger(new Random().nextInt(1000) + 1); + + protected Spanner spanner; + + @BeforeClass + public static void startMockServer() throws IOException { + mockSpanner = new MockSpannerServiceImpl(); + mockSpanner.setAbortProbability(0.0D); // We don't want any unpredictable aborted transactions. + + address = new InetSocketAddress("localhost", 0); + server = + NettyServerBuilder.forAddress(address) + .addService(mockSpanner) + .intercept( + new ServerInterceptor() { + @Override + public ServerCall.Listener interceptCall( + ServerCall serverCall, + Metadata headers, + ServerCallHandler serverCallHandler) { + return serverCallHandler.startCall( + new ForwardingServerCall.SimpleForwardingServerCall( + serverCall) { + @Override + public void sendHeaders(Metadata headers) { + headers.put( + Metadata.Key.of("server-timing", Metadata.ASCII_STRING_MARSHALLER), + String.format("gfet4t7; dur=%d", fakeServerTiming.get())); + super.sendHeaders(headers); + } + }, + headers); + } + }) + .build() + .start(); + executor = Executors.newSingleThreadExecutor(); + } + + @AfterClass + public static void stopMockServer() throws InterruptedException { + server.shutdown(); + server.awaitTermination(); + executor.shutdown(); + } + + @Before + public void createSpannerInstance() { + String endpoint = address.getHostString() + ":" + server.getPort(); + spanner = + SpannerOptions.newBuilder() + .setProjectId("test-project") + .setChannelConfigurator(ManagedChannelBuilder::usePlaintext) + .setHost("http://" + endpoint) + .setCredentials(NoCredentials.getInstance()) + .setSessionPoolOption(SessionPoolOptions.newBuilder().setFailOnSessionLeak().build()) + .build() + .getService(); + } + + @After + public void cleanup() { + spanner.close(); + mockSpanner.reset(); + mockSpanner.removeAllExecutionTimes(); + } +} diff --git a/google-cloud-spanner/src/test/java/com/google/cloud/spanner/BuiltInOpenTelemetryMetricsProviderTest.java b/google-cloud-spanner/src/test/java/com/google/cloud/spanner/BuiltInOpenTelemetryMetricsProviderTest.java index 43fe97113d0..73185177de1 100644 --- a/google-cloud-spanner/src/test/java/com/google/cloud/spanner/BuiltInOpenTelemetryMetricsProviderTest.java +++ b/google-cloud-spanner/src/test/java/com/google/cloud/spanner/BuiltInOpenTelemetryMetricsProviderTest.java @@ -29,31 +29,31 @@ public class BuiltInOpenTelemetryMetricsProviderTest { @Test public void testGenerateClientHashWithSimpleUid() { String clientUid = "testClient"; - verifyHash(BuiltInOpenTelemetryMetricsProvider.generateClientHash(clientUid)); + verifyHash(BuiltInMetricsProvider.generateClientHash(clientUid)); } @Test public void testGenerateClientHashWithEmptyUid() { String clientUid = ""; - verifyHash(BuiltInOpenTelemetryMetricsProvider.generateClientHash(clientUid)); + verifyHash(BuiltInMetricsProvider.generateClientHash(clientUid)); } @Test public void testGenerateClientHashWithNullUid() { String clientUid = null; - verifyHash(BuiltInOpenTelemetryMetricsProvider.generateClientHash(clientUid)); + verifyHash(BuiltInMetricsProvider.generateClientHash(clientUid)); } @Test public void testGenerateClientHashWithLongUid() { String clientUid = "aVeryLongUniqueClientIdentifierThatIsUnusuallyLong"; - verifyHash(BuiltInOpenTelemetryMetricsProvider.generateClientHash(clientUid)); + verifyHash(BuiltInMetricsProvider.generateClientHash(clientUid)); } @Test public void testGenerateClientHashWithSpecialCharacters() { String clientUid = "273d60f2-5604-42f1-b687-f5f1b975fd07@2316645@test#"; - verifyHash(BuiltInOpenTelemetryMetricsProvider.generateClientHash(clientUid)); + verifyHash(BuiltInMetricsProvider.generateClientHash(clientUid)); } private void verifyHash(String hash) { diff --git a/google-cloud-spanner/src/test/java/com/google/cloud/spanner/OpenTelemetryBuiltInMetricsTracerTest.java b/google-cloud-spanner/src/test/java/com/google/cloud/spanner/OpenTelemetryBuiltInMetricsTracerTest.java index 1b6d99260fe..f0c13b0f389 100644 --- a/google-cloud-spanner/src/test/java/com/google/cloud/spanner/OpenTelemetryBuiltInMetricsTracerTest.java +++ b/google-cloud-spanner/src/test/java/com/google/cloud/spanner/OpenTelemetryBuiltInMetricsTracerTest.java @@ -60,7 +60,7 @@ import org.junit.runners.JUnit4; @RunWith(JUnit4.class) -public class OpenTelemetryBuiltInMetricsTracerTest extends AbstractMockServerTest { +public class OpenTelemetryBuiltInMetricsTracerTest extends AbstractNettyMockServerTest { private static final Statement SELECT_RANDOM = Statement.of("SELECT * FROM random"); @@ -71,7 +71,8 @@ public class OpenTelemetryBuiltInMetricsTracerTest extends AbstractMockServerTes private static Map attributes; - private static Attributes expectedBaseAttributes; + private static Attributes expectedCommonBaseAttributes; + private static Attributes expectedCommonRequestAttributes; private static final long MIN_LATENCY = 0; @@ -81,7 +82,7 @@ public class OpenTelemetryBuiltInMetricsTracerTest extends AbstractMockServerTes public static void setup() { metricReader = InMemoryMetricReader.create(); - BuiltInOpenTelemetryMetricsProvider provider = BuiltInOpenTelemetryMetricsProvider.INSTANCE; + BuiltInMetricsProvider provider = BuiltInMetricsProvider.INSTANCE; SdkMeterProviderBuilder meterProvider = SdkMeterProvider.builder().registerMetricReader(metricReader); @@ -92,17 +93,23 @@ public static void setup() { openTelemetry = OpenTelemetrySdk.builder().setMeterProvider(meterProvider.build()).build(); attributes = provider.createClientAttributes("test-project", client_name); - expectedBaseAttributes = + expectedCommonBaseAttributes = Attributes.builder() .put(BuiltInMetricsConstant.PROJECT_ID_KEY, "test-project") .put(BuiltInMetricsConstant.INSTANCE_CONFIG_ID_KEY, "unknown") .put( BuiltInMetricsConstant.LOCATION_ID_KEY, - BuiltInOpenTelemetryMetricsProvider.detectClientLocation()) + BuiltInMetricsProvider.detectClientLocation()) .put(BuiltInMetricsConstant.CLIENT_NAME_KEY, client_name) .put(BuiltInMetricsConstant.CLIENT_UID_KEY, attributes.get("client_uid")) .put(BuiltInMetricsConstant.CLIENT_HASH_KEY, attributes.get("client_hash")) + .put(BuiltInMetricsConstant.INSTANCE_ID_KEY, "i") + .put(BuiltInMetricsConstant.DATABASE_KEY, "d") + .put(BuiltInMetricsConstant.DIRECT_PATH_ENABLED_KEY, "false") .build(); + + expectedCommonRequestAttributes = + Attributes.builder().put(BuiltInMetricsConstant.DIRECT_PATH_USED_KEY, "false").build(); } @BeforeClass @@ -122,8 +129,8 @@ public void createSpannerInstance() { SpannerOptions.Builder builder = SpannerOptions.newBuilder(); ApiTracerFactory metricsTracerFactory = - new MetricsTracerFactory( - new OpenTelemetryMetricsRecorder(openTelemetry, BuiltInMetricsConstant.METER_NAME), + new BuiltInMetricsTracerFactory( + new BuiltInMetricsRecorder(openTelemetry, BuiltInMetricsConstant.METER_NAME), attributes); // Set a quick polling algorithm to prevent this from slowing down the test unnecessarily. builder @@ -137,10 +144,12 @@ public void createSpannerInstance() { .setRetryDelayMultiplier(1.0) .setTotalTimeoutDuration(Duration.ofMinutes(10L)) .build())); + String endpoint = address.getHostString() + ":" + server.getPort(); spanner = - builder + SpannerOptions.newBuilder() .setProjectId("test-project") - .setChannelProvider(channelProvider) + .setChannelConfigurator(ManagedChannelBuilder::usePlaintext) + .setHost("http://" + endpoint) .setCredentials(NoCredentials.getInstance()) .setSessionPoolOption( SessionPoolOptions.newBuilder() @@ -167,8 +176,9 @@ public void testMetricsSingleUseQuery() { long elapsed = stopwatch.elapsed(TimeUnit.MILLISECONDS); Attributes expectedAttributes = - expectedBaseAttributes + expectedCommonBaseAttributes .toBuilder() + .putAll(expectedCommonRequestAttributes) .put(BuiltInMetricsConstant.STATUS_KEY, "OK") .put(BuiltInMetricsConstant.METHOD_KEY, "Spanner.ExecuteStreamingSql") .build(); @@ -194,6 +204,11 @@ public void testMetricsSingleUseQuery() { getMetricData(metricReader, BuiltInMetricsConstant.ATTEMPT_COUNT_NAME); assertNotNull(attemptCountMetricData); assertThat(getAggregatedValue(attemptCountMetricData, expectedAttributes)).isEqualTo(1); + + MetricData gfeLatencyMetricData = + getMetricData(metricReader, BuiltInMetricsConstant.GFE_LATENCIES_NAME); + long gfeLatencyValue = getAggregatedValue(gfeLatencyMetricData, expectedAttributes); + assertEquals(fakeServerTiming.get(), gfeLatencyValue, 0); } @Test @@ -210,14 +225,15 @@ public void testMetricsWithGaxRetryUnaryRpc() { stopwatch.elapsed(TimeUnit.MILLISECONDS); Attributes expectedAttributesBeginTransactionOK = - expectedBaseAttributes + expectedCommonBaseAttributes .toBuilder() + .putAll(expectedCommonRequestAttributes) .put(BuiltInMetricsConstant.STATUS_KEY, "OK") .put(BuiltInMetricsConstant.METHOD_KEY, "Spanner.BeginTransaction") .build(); Attributes expectedAttributesBeginTransactionFailed = - expectedBaseAttributes + expectedCommonBaseAttributes .toBuilder() .put(BuiltInMetricsConstant.STATUS_KEY, "UNAVAILABLE") .put(BuiltInMetricsConstant.METHOD_KEY, "Spanner.BeginTransaction") @@ -289,7 +305,7 @@ public void testNoNetworkConnection() { .setApiTracerFactory(metricsTracerFactory) .build() .getService(); - String instance = "test-instance"; + String instance = "i"; DatabaseClient client = spanner.getDatabaseClient(DatabaseId.of("test-project", instance, "d")); // Using this client will return UNAVAILABLE, as the server is not reachable and we have @@ -300,29 +316,24 @@ public void testNoNetworkConnection() { assertEquals(ErrorCode.UNAVAILABLE, exception.getErrorCode()); Attributes expectedAttributesCreateSessionOK = - expectedBaseAttributes + expectedCommonBaseAttributes .toBuilder() + .putAll(expectedCommonRequestAttributes) .put(BuiltInMetricsConstant.STATUS_KEY, "OK") .put(BuiltInMetricsConstant.METHOD_KEY, "Spanner.CreateSession") // Include the additional attributes that are added by the HeaderInterceptor in the // filter. Note that the DIRECT_PATH_USED attribute is not added, as the request never // leaves the client. - .put(BuiltInMetricsConstant.INSTANCE_ID_KEY, instance) - .put(BuiltInMetricsConstant.DATABASE_KEY, "d") - .put(BuiltInMetricsConstant.DIRECT_PATH_ENABLED_KEY, "false") .build(); Attributes expectedAttributesCreateSessionFailed = - expectedBaseAttributes + expectedCommonBaseAttributes .toBuilder() .put(BuiltInMetricsConstant.STATUS_KEY, "UNAVAILABLE") .put(BuiltInMetricsConstant.METHOD_KEY, "Spanner.CreateSession") // Include the additional attributes that are added by the HeaderInterceptor in the // filter. Note that the DIRECT_PATH_USED attribute is not added, as the request never // leaves the client. - .put(BuiltInMetricsConstant.INSTANCE_ID_KEY, instance) - .put(BuiltInMetricsConstant.DATABASE_KEY, "d") - .put(BuiltInMetricsConstant.DIRECT_PATH_ENABLED_KEY, "false") .build(); MetricData attemptCountMetricData = @@ -332,8 +343,6 @@ public void testNoNetworkConnection() { // Attempt count should have a failed metric point for CreateSession. assertEquals( 1, getAggregatedValue(attemptCountMetricData, expectedAttributesCreateSessionFailed)); - // There should be no OK metric points for CreateSession. - assertEquals(0, getAggregatedValue(attemptCountMetricData, expectedAttributesCreateSessionOK)); } private MetricData getMetricData(InMemoryMetricReader reader, String metricName) { diff --git a/google-cloud-spanner/src/test/java/com/google/cloud/spanner/it/ITBuiltInMetricsTest.java b/google-cloud-spanner/src/test/java/com/google/cloud/spanner/it/ITBuiltInMetricsTest.java index 258c1230709..5bf8e42ccb6 100644 --- a/google-cloud-spanner/src/test/java/com/google/cloud/spanner/it/ITBuiltInMetricsTest.java +++ b/google-cloud-spanner/src/test/java/com/google/cloud/spanner/it/ITBuiltInMetricsTest.java @@ -82,10 +82,14 @@ public void testBuiltinMetricsWithDefaultOTEL() throws Exception { String metricFilter = String.format( - "metric.type=\"spanner.googleapis.com/client/%s\" " - + "AND resource.labels.instance=\"%s\" AND metric.labels.method=\"Spanner.ExecuteStreamingSql\"" + "metric.type=\"spanner.googleapis.com/client/%s\"" + + " AND resource.type=\"spanner_instance\"" + + " AND metric.labels.method=\"Spanner.Commit\"" + + " AND resource.labels.instance_id=\"%s\"" + " AND metric.labels.database=\"%s\"", - "operation_latencies", env.getTestHelper().getInstanceId(), db.getId()); + "operation_latencies", + db.getId().getInstanceId().getInstance(), + db.getId().getDatabase()); ListTimeSeriesRequest.Builder requestBuilder = ListTimeSeriesRequest.newBuilder()