From 29a4f7c854ffe8b6369bcdc897836b59803bdee3 Mon Sep 17 00:00:00 2001 From: surbhigarg92 Date: Thu, 8 May 2025 14:54:12 +0530 Subject: [PATCH 1/7] feat: enable AFE and gRPC metrics for DP --- .../java/com/google/cloud/spanner/SpannerOptions.java | 3 ++- .../google/cloud/spanner/spi/v1/GapicSpannerRpc.java | 10 +++++++--- .../google/cloud/spanner/spi/v1/HeaderInterceptor.java | 10 ++++++---- 3 files changed, 15 insertions(+), 8 deletions(-) diff --git a/google-cloud-spanner/src/main/java/com/google/cloud/spanner/SpannerOptions.java b/google-cloud-spanner/src/main/java/com/google/cloud/spanner/SpannerOptions.java index 0995c478427..ba9766a21ff 100644 --- a/google-cloud-spanner/src/main/java/com/google/cloud/spanner/SpannerOptions.java +++ b/google-cloud-spanner/src/main/java/com/google/cloud/spanner/SpannerOptions.java @@ -1994,7 +1994,8 @@ public ApiTracerFactory getApiTracerFactory() { } public void enablegRPCMetrics(InstantiatingGrpcChannelProvider.Builder channelProviderBuilder) { - if (SpannerOptions.environment.isEnableGRPCBuiltInMetrics()) { + if (GapicSpannerRpc.isEnableDirectPathXdsEnv() + || SpannerOptions.environment.isEnableGRPCBuiltInMetrics()) { this.builtInMetricsProvider.enableGrpcMetrics( channelProviderBuilder, this.getProjectId(), getCredentials(), this.monitoringHost); } diff --git a/google-cloud-spanner/src/main/java/com/google/cloud/spanner/spi/v1/GapicSpannerRpc.java b/google-cloud-spanner/src/main/java/com/google/cloud/spanner/spi/v1/GapicSpannerRpc.java index fa5719c95c4..309e6edc181 100644 --- a/google-cloud-spanner/src/main/java/com/google/cloud/spanner/spi/v1/GapicSpannerRpc.java +++ b/google-cloud-spanner/src/main/java/com/google/cloud/spanner/spi/v1/GapicSpannerRpc.java @@ -367,8 +367,7 @@ public GapicSpannerRpc(final SpannerOptions options) { .withEncoding(compressorName)) .setHeaderProvider(headerProviderWithUserAgent) .setAllowNonDefaultServiceAccount(true); - String directPathXdsEnv = System.getenv("GOOGLE_SPANNER_ENABLE_DIRECT_ACCESS"); - boolean isAttemptDirectPathXds = Boolean.parseBoolean(directPathXdsEnv); + boolean isAttemptDirectPathXds = isEnableDirectPathXdsEnv(); if (isAttemptDirectPathXds) { defaultChannelProviderBuilder.setAttemptDirectPath(true); defaultChannelProviderBuilder.setAttemptDirectPathXds(); @@ -678,7 +677,12 @@ private static boolean isEmulatorEnabled(SpannerOptions options, String emulator } public static boolean isEnableAFEServerTiming() { - return "false".equalsIgnoreCase(System.getenv("SPANNER_DISABLE_AFE_SERVER_TIMING")); + return isEnableDirectPathXdsEnv() + || "false".equalsIgnoreCase(System.getenv("SPANNER_DISABLE_AFE_SERVER_TIMING")); + } + + public static boolean isEnableDirectPathXdsEnv() { + return Boolean.parseBoolean(System.getenv("GOOGLE_SPANNER_ENABLE_DIRECT_ACCESS")); } private static final RetrySettings ADMIN_REQUESTS_LIMIT_EXCEEDED_RETRY_SETTINGS = diff --git a/google-cloud-spanner/src/main/java/com/google/cloud/spanner/spi/v1/HeaderInterceptor.java b/google-cloud-spanner/src/main/java/com/google/cloud/spanner/spi/v1/HeaderInterceptor.java index 32e6f2c3b96..e3bc848dc1e 100644 --- a/google-cloud-spanner/src/main/java/com/google/cloud/spanner/spi/v1/HeaderInterceptor.java +++ b/google-cloud-spanner/src/main/java/com/google/cloud/spanner/spi/v1/HeaderInterceptor.java @@ -133,7 +133,8 @@ public void onHeaders(Metadata metadata) { Boolean isDirectPathUsed = isDirectPathUsed(getAttributes().get(Grpc.TRANSPORT_ATTR_REMOTE_ADDR)); addDirectPathUsedAttribute(compositeTracer, isDirectPathUsed); - processHeader(metadata, tagContext, attributes, span, compositeTracer); + processHeader( + metadata, tagContext, attributes, span, compositeTracer, isDirectPathUsed); super.onHeaders(metadata); } }, @@ -151,7 +152,8 @@ private void processHeader( TagContext tagContext, Attributes attributes, Span span, - CompositeTracer compositeTracer) { + CompositeTracer compositeTracer, + boolean isDirectPathUsed) { MeasureMap measureMap = STATS_RECORDER.newMeasureMap(); String serverTiming = metadata.get(SERVER_TIMING_HEADER_KEY); try { @@ -172,7 +174,7 @@ private void processHeader( spannerRpcMetrics.recordGfeLatency((long) gfeLatency, attributes); spannerRpcMetrics.recordGfeHeaderMissingCount(0L, attributes); - if (compositeTracer != null) { + if (compositeTracer != null && !isDirectPathUsed) { compositeTracer.recordGFELatency(gfeLatency); } if (span != null) { @@ -181,7 +183,7 @@ private void processHeader( } else { measureMap.put(SPANNER_GFE_HEADER_MISSING_COUNT, 1L).record(tagContext); spannerRpcMetrics.recordGfeHeaderMissingCount(1L, attributes); - if (compositeTracer != null) { + if (compositeTracer != null && !isDirectPathUsed) { compositeTracer.recordGfeHeaderMissingCount(1L); } } From 4a324b14cd0c198580c9814d655397308d281668 Mon Sep 17 00:00:00 2001 From: surbhigarg92 Date: Thu, 8 May 2025 16:25:50 +0530 Subject: [PATCH 2/7] check afe metrics when DP is enabled --- ...OpenTelemetryBuiltInMetricsTracerTest.java | 54 ++++++++++++------- 1 file changed, 34 insertions(+), 20 deletions(-) diff --git a/google-cloud-spanner/src/test/java/com/google/cloud/spanner/OpenTelemetryBuiltInMetricsTracerTest.java b/google-cloud-spanner/src/test/java/com/google/cloud/spanner/OpenTelemetryBuiltInMetricsTracerTest.java index acc94ea56e5..cf4a3202569 100644 --- a/google-cloud-spanner/src/test/java/com/google/cloud/spanner/OpenTelemetryBuiltInMetricsTracerTest.java +++ b/google-cloud-spanner/src/test/java/com/google/cloud/spanner/OpenTelemetryBuiltInMetricsTracerTest.java @@ -34,6 +34,7 @@ import com.google.cloud.spanner.MockSpannerServiceImpl.SimulatedExecutionTime; import com.google.cloud.spanner.MockSpannerServiceImpl.StatementResult; import com.google.cloud.spanner.connection.RandomResultSetGenerator; +import com.google.cloud.spanner.spi.v1.GapicSpannerRpc; import com.google.common.base.Stopwatch; import com.google.common.collect.ImmutableList; import com.google.common.collect.Range; @@ -190,16 +191,23 @@ public void testMetricsSingleUseQuery() { assertNotNull(attemptCountMetricData); assertThat(getAggregatedValue(attemptCountMetricData, expectedAttributes)).isEqualTo(1); - MetricData gfeLatencyMetricData = - getMetricData(metricReader, BuiltInMetricsConstant.GFE_LATENCIES_NAME); - double gfeLatencyValue = getAggregatedValue(gfeLatencyMetricData, expectedAttributes); - assertEquals(fakeServerTiming.get(), gfeLatencyValue, 0); - - assertFalse( + assertFalse( checkIfMetricExists(metricReader, BuiltInMetricsConstant.GFE_CONNECTIVITY_ERROR_NAME)); - assertFalse(checkIfMetricExists(metricReader, BuiltInMetricsConstant.AFE_LATENCIES_NAME)); assertFalse( checkIfMetricExists(metricReader, BuiltInMetricsConstant.AFE_CONNECTIVITY_ERROR_NAME)); + if (GapicSpannerRpc.isEnableDirectPathXdsEnv()) { + // AFE metrics are enabled for DirectPath. + MetricData afeLatencyMetricData = + getMetricData(metricReader, BuiltInMetricsConstant.AFE_LATENCIES_NAME); + double afeLatencyValue = getAggregatedValue(afeLatencyMetricData, expectedAttributes); + assertEquals(fakeAFEServerTiming.get(), afeLatencyValue, 0); + } else { + MetricData gfeLatencyMetricData = + getMetricData(metricReader, BuiltInMetricsConstant.GFE_LATENCIES_NAME); + double gfeLatencyValue = getAggregatedValue(gfeLatencyMetricData, expectedAttributes); + assertEquals(fakeServerTiming.get(), gfeLatencyValue, 0); + assertFalse(checkIfMetricExists(metricReader, BuiltInMetricsConstant.AFE_LATENCIES_NAME)); + } } private boolean isJava8() { @@ -261,20 +269,19 @@ public void testMetricsSingleUseQueryWithAfeEnabled() throws Exception { assertNotNull(attemptCountMetricData); assertThat(getAggregatedValue(attemptCountMetricData, expectedAttributes)).isEqualTo(1); - MetricData gfeLatencyMetricData = - getMetricData(metricReader, BuiltInMetricsConstant.GFE_LATENCIES_NAME); - double gfeLatencyValue = getAggregatedValue(gfeLatencyMetricData, expectedAttributes); - assertEquals(fakeServerTiming.get(), gfeLatencyValue, 0); - assertFalse( checkIfMetricExists(metricReader, BuiltInMetricsConstant.GFE_CONNECTIVITY_ERROR_NAME)); - + assertFalse( + checkIfMetricExists(metricReader, BuiltInMetricsConstant.AFE_CONNECTIVITY_ERROR_NAME)); MetricData afeLatencyMetricData = getMetricData(metricReader, BuiltInMetricsConstant.AFE_LATENCIES_NAME); double afeLatencyValue = getAggregatedValue(afeLatencyMetricData, expectedAttributes); assertEquals(fakeAFEServerTiming.get(), afeLatencyValue, 0); - assertFalse( - checkIfMetricExists(metricReader, BuiltInMetricsConstant.AFE_CONNECTIVITY_ERROR_NAME)); + + MetricData gfeLatencyMetricData = + getMetricData(metricReader, BuiltInMetricsConstant.GFE_LATENCIES_NAME); + double gfeLatencyValue = getAggregatedValue(gfeLatencyMetricData, expectedAttributes); + assertEquals(fakeServerTiming.get(), gfeLatencyValue, 0); } finally { writeableEnvironmentVariables.remove("GOOGLE_CLOUD_SPANNER_MULTIPLEXED_SESSIONS"); } @@ -445,13 +452,20 @@ public void testNoServerTimingHeader() throws IOException, InterruptedException .put(BuiltInMetricsConstant.METHOD_KEY, "Spanner.ExecuteSql") .build(); - MetricData gfeConnectivityMetricData = - getMetricData(metricReader, BuiltInMetricsConstant.GFE_CONNECTIVITY_ERROR_NAME); - assertThat(getAggregatedValue(gfeConnectivityMetricData, expectedAttributes)).isEqualTo(1); assertFalse(checkIfMetricExists(metricReader, BuiltInMetricsConstant.AFE_LATENCIES_NAME)); assertFalse(checkIfMetricExists(metricReader, BuiltInMetricsConstant.GFE_LATENCIES_NAME)); - assertFalse( - checkIfMetricExists(metricReader, BuiltInMetricsConstant.AFE_CONNECTIVITY_ERROR_NAME)); + if (GapicSpannerRpc.isEnableDirectPathXdsEnv()) { + MetricData afeConnectivityMetricData = + getMetricData(metricReader, BuiltInMetricsConstant.AFE_CONNECTIVITY_ERROR_NAME); + assertThat(getAggregatedValue(afeConnectivityMetricData, expectedAttributes)).isEqualTo(1); + } else { + MetricData gfeConnectivityMetricData = + getMetricData(metricReader, BuiltInMetricsConstant.GFE_CONNECTIVITY_ERROR_NAME); + assertThat(getAggregatedValue(gfeConnectivityMetricData, expectedAttributes)).isEqualTo(1); + assertFalse( + checkIfMetricExists(metricReader, BuiltInMetricsConstant.AFE_CONNECTIVITY_ERROR_NAME)); + } + spannerNoHeader.close(); serverNoHeader.shutdown(); serverNoHeader.awaitTermination(); From 113c2121d2ae4cd4ac915d01b4cc33a871e6c97c Mon Sep 17 00:00:00 2001 From: surbhigarg92 Date: Thu, 8 May 2025 16:25:50 +0530 Subject: [PATCH 3/7] check afe metrics when DP is enabled --- .../cloud/spanner/OpenTelemetryBuiltInMetricsTracerTest.java | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/google-cloud-spanner/src/test/java/com/google/cloud/spanner/OpenTelemetryBuiltInMetricsTracerTest.java b/google-cloud-spanner/src/test/java/com/google/cloud/spanner/OpenTelemetryBuiltInMetricsTracerTest.java index cf4a3202569..4c96c72bb21 100644 --- a/google-cloud-spanner/src/test/java/com/google/cloud/spanner/OpenTelemetryBuiltInMetricsTracerTest.java +++ b/google-cloud-spanner/src/test/java/com/google/cloud/spanner/OpenTelemetryBuiltInMetricsTracerTest.java @@ -269,6 +269,11 @@ public void testMetricsSingleUseQueryWithAfeEnabled() throws Exception { assertNotNull(attemptCountMetricData); assertThat(getAggregatedValue(attemptCountMetricData, expectedAttributes)).isEqualTo(1); + MetricData gfeLatencyMetricData = + getMetricData(metricReader, BuiltInMetricsConstant.GFE_LATENCIES_NAME); + double gfeLatencyValue = getAggregatedValue(gfeLatencyMetricData, expectedAttributes); + assertEquals(fakeServerTiming.get(), gfeLatencyValue, 0); + assertFalse( checkIfMetricExists(metricReader, BuiltInMetricsConstant.GFE_CONNECTIVITY_ERROR_NAME)); assertFalse( From 479b6d5aabe06cb4a52c4dd50881785bbfd39ed5 Mon Sep 17 00:00:00 2001 From: surbhigarg92 Date: Fri, 9 May 2025 11:16:59 +0530 Subject: [PATCH 4/7] mergeconflict --- .../cloud/spanner/OpenTelemetryBuiltInMetricsTracerTest.java | 5 ----- 1 file changed, 5 deletions(-) diff --git a/google-cloud-spanner/src/test/java/com/google/cloud/spanner/OpenTelemetryBuiltInMetricsTracerTest.java b/google-cloud-spanner/src/test/java/com/google/cloud/spanner/OpenTelemetryBuiltInMetricsTracerTest.java index 4c96c72bb21..cf4a3202569 100644 --- a/google-cloud-spanner/src/test/java/com/google/cloud/spanner/OpenTelemetryBuiltInMetricsTracerTest.java +++ b/google-cloud-spanner/src/test/java/com/google/cloud/spanner/OpenTelemetryBuiltInMetricsTracerTest.java @@ -269,11 +269,6 @@ public void testMetricsSingleUseQueryWithAfeEnabled() throws Exception { assertNotNull(attemptCountMetricData); assertThat(getAggregatedValue(attemptCountMetricData, expectedAttributes)).isEqualTo(1); - MetricData gfeLatencyMetricData = - getMetricData(metricReader, BuiltInMetricsConstant.GFE_LATENCIES_NAME); - double gfeLatencyValue = getAggregatedValue(gfeLatencyMetricData, expectedAttributes); - assertEquals(fakeServerTiming.get(), gfeLatencyValue, 0); - assertFalse( checkIfMetricExists(metricReader, BuiltInMetricsConstant.GFE_CONNECTIVITY_ERROR_NAME)); assertFalse( From 4221721be3425d65fe74ef5b82059980abe2f9d1 Mon Sep 17 00:00:00 2001 From: cloud-java-bot Date: Fri, 9 May 2025 05:51:58 +0000 Subject: [PATCH 5/7] chore: generate libraries at Fri May 9 05:49:15 UTC 2025 --- .../cloud/spanner/OpenTelemetryBuiltInMetricsTracerTest.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/google-cloud-spanner/src/test/java/com/google/cloud/spanner/OpenTelemetryBuiltInMetricsTracerTest.java b/google-cloud-spanner/src/test/java/com/google/cloud/spanner/OpenTelemetryBuiltInMetricsTracerTest.java index cf4a3202569..9a64c13adeb 100644 --- a/google-cloud-spanner/src/test/java/com/google/cloud/spanner/OpenTelemetryBuiltInMetricsTracerTest.java +++ b/google-cloud-spanner/src/test/java/com/google/cloud/spanner/OpenTelemetryBuiltInMetricsTracerTest.java @@ -191,7 +191,7 @@ public void testMetricsSingleUseQuery() { assertNotNull(attemptCountMetricData); assertThat(getAggregatedValue(attemptCountMetricData, expectedAttributes)).isEqualTo(1); - assertFalse( + assertFalse( checkIfMetricExists(metricReader, BuiltInMetricsConstant.GFE_CONNECTIVITY_ERROR_NAME)); assertFalse( checkIfMetricExists(metricReader, BuiltInMetricsConstant.AFE_CONNECTIVITY_ERROR_NAME)); From c2e6c0f03898c47cd9ae86614024bbae8ca15fbb Mon Sep 17 00:00:00 2001 From: surbhigarg92 Date: Fri, 9 May 2025 12:11:13 +0530 Subject: [PATCH 6/7] review comments --- .../com/google/cloud/spanner/SpannerOptions.java | 15 +++++++++++---- .../cloud/spanner/spi/v1/GapicSpannerRpc.java | 11 +++++++++-- 2 files changed, 20 insertions(+), 6 deletions(-) diff --git a/google-cloud-spanner/src/main/java/com/google/cloud/spanner/SpannerOptions.java b/google-cloud-spanner/src/main/java/com/google/cloud/spanner/SpannerOptions.java index ba9766a21ff..a25e8bfa997 100644 --- a/google-cloud-spanner/src/main/java/com/google/cloud/spanner/SpannerOptions.java +++ b/google-cloud-spanner/src/main/java/com/google/cloud/spanner/SpannerOptions.java @@ -923,8 +923,16 @@ public boolean isEnableBuiltInMetrics() { @Override public boolean isEnableGRPCBuiltInMetrics() { - return "false" - .equalsIgnoreCase(System.getenv(SPANNER_DISABLE_DIRECT_ACCESS_GRPC_BUILTIN_METRICS)); + // Enable gRPC built-in metrics if: + // 1. The env var SPANNER_DISABLE_DIRECT_ACCESS_GRPC_BUILTIN_METRICS is explicitly set to + // "false", OR + // 2. DirectPath is enabled AND the env var is not set to "true" + // This allows metrics to be enabled by default when DirectPath is on, unless explicitly + // disabled via env. + String grpcDisableEnv = System.getenv("SPANNER_DISABLE_DIRECT_ACCESS_GRPC_BUILTIN_METRICS"); + boolean isDirectPathEnabled = GapicSpannerRpc.isEnableDirectPathXdsEnv(); + return ("false".equalsIgnoreCase(grpcDisableEnv)) + || (isDirectPathEnabled && !"true".equalsIgnoreCase(grpcDisableEnv)); } @Override @@ -1994,8 +2002,7 @@ public ApiTracerFactory getApiTracerFactory() { } public void enablegRPCMetrics(InstantiatingGrpcChannelProvider.Builder channelProviderBuilder) { - if (GapicSpannerRpc.isEnableDirectPathXdsEnv() - || SpannerOptions.environment.isEnableGRPCBuiltInMetrics()) { + if (SpannerOptions.environment.isEnableGRPCBuiltInMetrics()) { this.builtInMetricsProvider.enableGrpcMetrics( channelProviderBuilder, this.getProjectId(), getCredentials(), this.monitoringHost); } diff --git a/google-cloud-spanner/src/main/java/com/google/cloud/spanner/spi/v1/GapicSpannerRpc.java b/google-cloud-spanner/src/main/java/com/google/cloud/spanner/spi/v1/GapicSpannerRpc.java index 309e6edc181..c43bbe1f11b 100644 --- a/google-cloud-spanner/src/main/java/com/google/cloud/spanner/spi/v1/GapicSpannerRpc.java +++ b/google-cloud-spanner/src/main/java/com/google/cloud/spanner/spi/v1/GapicSpannerRpc.java @@ -677,8 +677,15 @@ private static boolean isEmulatorEnabled(SpannerOptions options, String emulator } public static boolean isEnableAFEServerTiming() { - return isEnableDirectPathXdsEnv() - || "false".equalsIgnoreCase(System.getenv("SPANNER_DISABLE_AFE_SERVER_TIMING")); + // Enable AFE metrics and add AFE header if: + // 1. The env var SPANNER_DISABLE_AFE_SERVER_TIMING is explicitly set to "false", OR + // 2. DirectPath is enabled AND the env var is not set to "true" + // This allows metrics to be enabled by default when DirectPath is on, unless explicitly + // disabled via env. + String afeDisableEnv = System.getenv("SPANNER_DISABLE_AFE_SERVER_TIMING"); + boolean isDirectPathEnabled = isEnableDirectPathXdsEnv(); + return ("false".equalsIgnoreCase(afeDisableEnv)) + || (isDirectPathEnabled && !"true".equalsIgnoreCase(afeDisableEnv)); } public static boolean isEnableDirectPathXdsEnv() { From 0bb95fa8fba96bc5bcf361e4574582c697fae385 Mon Sep 17 00:00:00 2001 From: surbhigarg92 Date: Fri, 9 May 2025 12:17:33 +0530 Subject: [PATCH 7/7] roundofferrorfordouble --- .../spanner/OpenTelemetryBuiltInMetricsTracerTest.java | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/google-cloud-spanner/src/test/java/com/google/cloud/spanner/OpenTelemetryBuiltInMetricsTracerTest.java b/google-cloud-spanner/src/test/java/com/google/cloud/spanner/OpenTelemetryBuiltInMetricsTracerTest.java index 9a64c13adeb..55c5cf47714 100644 --- a/google-cloud-spanner/src/test/java/com/google/cloud/spanner/OpenTelemetryBuiltInMetricsTracerTest.java +++ b/google-cloud-spanner/src/test/java/com/google/cloud/spanner/OpenTelemetryBuiltInMetricsTracerTest.java @@ -200,12 +200,12 @@ public void testMetricsSingleUseQuery() { MetricData afeLatencyMetricData = getMetricData(metricReader, BuiltInMetricsConstant.AFE_LATENCIES_NAME); double afeLatencyValue = getAggregatedValue(afeLatencyMetricData, expectedAttributes); - assertEquals(fakeAFEServerTiming.get(), afeLatencyValue, 0); + assertEquals(fakeAFEServerTiming.get(), afeLatencyValue, 1e-6); } else { MetricData gfeLatencyMetricData = getMetricData(metricReader, BuiltInMetricsConstant.GFE_LATENCIES_NAME); double gfeLatencyValue = getAggregatedValue(gfeLatencyMetricData, expectedAttributes); - assertEquals(fakeServerTiming.get(), gfeLatencyValue, 0); + assertEquals(fakeServerTiming.get(), gfeLatencyValue, 1e-6); assertFalse(checkIfMetricExists(metricReader, BuiltInMetricsConstant.AFE_LATENCIES_NAME)); } } @@ -276,12 +276,12 @@ public void testMetricsSingleUseQueryWithAfeEnabled() throws Exception { MetricData afeLatencyMetricData = getMetricData(metricReader, BuiltInMetricsConstant.AFE_LATENCIES_NAME); double afeLatencyValue = getAggregatedValue(afeLatencyMetricData, expectedAttributes); - assertEquals(fakeAFEServerTiming.get(), afeLatencyValue, 0); + assertEquals(fakeAFEServerTiming.get(), afeLatencyValue, 1e-6); MetricData gfeLatencyMetricData = getMetricData(metricReader, BuiltInMetricsConstant.GFE_LATENCIES_NAME); double gfeLatencyValue = getAggregatedValue(gfeLatencyMetricData, expectedAttributes); - assertEquals(fakeServerTiming.get(), gfeLatencyValue, 0); + assertEquals(fakeServerTiming.get(), gfeLatencyValue, 1e-6); } finally { writeableEnvironmentVariables.remove("GOOGLE_CLOUD_SPANNER_MULTIPLEXED_SESSIONS"); }