diff --git a/.github/workflows/update_generation_config.yaml b/.github/workflows/update_generation_config.yaml
index 59e39834dd..a7e14bb483 100644
--- a/.github/workflows/update_generation_config.yaml
+++ b/.github/workflows/update_generation_config.yaml
@@ -26,7 +26,7 @@ jobs:
# the branch into which the pull request is merged
base_branch: main
steps:
- - uses: actions/checkout@v5
+ - uses: actions/checkout@v4
with:
fetch-depth: 0
token: ${{ secrets.CLOUD_JAVA_BOT_TOKEN }}
diff --git a/google-cloud-spanner/pom.xml b/google-cloud-spanner/pom.xml
index 965991b346..f7d78e5e36 100644
--- a/google-cloud-spanner/pom.xml
+++ b/google-cloud-spanner/pom.xml
@@ -276,6 +276,12 @@
proto-google-cloud-monitoring-v3
3.76.0
+
+ com.google.api.grpc
+ grpc-google-cloud-monitoring-v3
+ 3.63.0
+ test
+
com.google.auth
google-auth-library-oauth2-http
@@ -522,7 +528,11 @@
-classpath
org.openjdk.jmh.Main
- ${benchmark.name}
+ ${benchmark.name}
+ -rf
+ JSON
+ -rff
+ jmh-results.json
@@ -544,6 +554,21 @@
+
+ validate-benchmark
+
+
+
+ org.codehaus.mojo
+ exec-maven-plugin
+
+ com.google.cloud.spanner.benchmarking.BenchmarkValidator
+ test
+
+
+
+
+
slow-tests
diff --git a/google-cloud-spanner/src/main/java/com/google/cloud/spanner/SpannerCloudMonitoringExporter.java b/google-cloud-spanner/src/main/java/com/google/cloud/spanner/SpannerCloudMonitoringExporter.java
index 40202a0eef..bedf660007 100644
--- a/google-cloud-spanner/src/main/java/com/google/cloud/spanner/SpannerCloudMonitoringExporter.java
+++ b/google-cloud-spanner/src/main/java/com/google/cloud/spanner/SpannerCloudMonitoringExporter.java
@@ -22,8 +22,10 @@
import com.google.api.gax.core.CredentialsProvider;
import com.google.api.gax.core.FixedCredentialsProvider;
import com.google.api.gax.core.NoCredentialsProvider;
+import com.google.api.gax.grpc.InstantiatingGrpcChannelProvider;
import com.google.api.gax.rpc.PermissionDeniedException;
import com.google.auth.Credentials;
+import com.google.cloud.NoCredentials;
import com.google.cloud.monitoring.v3.MetricServiceClient;
import com.google.cloud.monitoring.v3.MetricServiceSettings;
import com.google.common.annotations.VisibleForTesting;
@@ -34,6 +36,7 @@
import com.google.monitoring.v3.ProjectName;
import com.google.monitoring.v3.TimeSeries;
import com.google.protobuf.Empty;
+import io.grpc.ManagedChannelBuilder;
import io.opentelemetry.sdk.common.CompletableResultCode;
import io.opentelemetry.sdk.metrics.InstrumentType;
import io.opentelemetry.sdk.metrics.data.AggregationTemporality;
@@ -79,7 +82,7 @@ static SpannerCloudMonitoringExporter create(
throws IOException {
MetricServiceSettings.Builder settingsBuilder = MetricServiceSettings.newBuilder();
CredentialsProvider credentialsProvider;
- if (credentials == null) {
+ if (credentials == null || credentials instanceof NoCredentials) {
credentialsProvider = NoCredentialsProvider.create();
} else {
credentialsProvider = FixedCredentialsProvider.create(credentials);
@@ -92,6 +95,19 @@ static SpannerCloudMonitoringExporter create(
settingsBuilder.setUniverseDomain(universeDomain);
}
+ if (System.getProperty("jmh.monitoring-server-port") != null) {
+ settingsBuilder.setTransportChannelProvider(
+ InstantiatingGrpcChannelProvider.newBuilder()
+ .setCredentials(NoCredentials.getInstance())
+ .setChannelConfigurator(
+ managedChannelBuilder ->
+ ManagedChannelBuilder.forAddress(
+ "0.0.0.0",
+ Integer.parseInt(System.getProperty("jmh.monitoring-server-port")))
+ .usePlaintext())
+ .build());
+ }
+
Duration timeout = Duration.ofMinutes(1);
// TODO: createServiceTimeSeries needs special handling if the request failed. Leaving
// it as not retried for now.
diff --git a/google-cloud-spanner/src/main/java/com/google/cloud/spanner/SpannerOptions.java b/google-cloud-spanner/src/main/java/com/google/cloud/spanner/SpannerOptions.java
index af4ed58bad..765114dc68 100644
--- a/google-cloud-spanner/src/main/java/com/google/cloud/spanner/SpannerOptions.java
+++ b/google-cloud-spanner/src/main/java/com/google/cloud/spanner/SpannerOptions.java
@@ -2013,6 +2013,11 @@ public CallCredentialsProvider getCallCredentialsProvider() {
}
private boolean usesNoCredentials() {
+ // When JMH is enabled, we need to enable built-in metrics
+ if (System.getProperty("jmh.enabled") != null
+ && System.getProperty("jmh.enabled").equals("true")) {
+ return false;
+ }
return Objects.equals(getCredentials(), NoCredentials.getInstance());
}
diff --git a/google-cloud-spanner/src/test/java/com/google/cloud/spanner/benchmarking/BenchmarkValidator.java b/google-cloud-spanner/src/test/java/com/google/cloud/spanner/benchmarking/BenchmarkValidator.java
new file mode 100644
index 0000000000..225197af6c
--- /dev/null
+++ b/google-cloud-spanner/src/test/java/com/google/cloud/spanner/benchmarking/BenchmarkValidator.java
@@ -0,0 +1,156 @@
+/*
+ * Copyright 2025 Google LLC
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * https://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package com.google.cloud.spanner.benchmarking;
+
+import com.google.cloud.spanner.benchmarking.BenchmarkValidator.BaselineResult.BenchmarkResult;
+import com.google.cloud.spanner.benchmarking.BenchmarkValidator.BaselineResult.BenchmarkResult.Percentile;
+import com.google.gson.Gson;
+import com.google.gson.reflect.TypeToken;
+import java.io.File;
+import java.io.IOException;
+import java.net.URL;
+import java.nio.file.Files;
+import java.nio.file.Paths;
+import java.util.ArrayList;
+import java.util.List;
+import java.util.Map;
+
+public class BenchmarkValidator {
+
+ private final BaselineResult expectedResults;
+ private final List actualResults;
+
+ public BenchmarkValidator(String baselineFile, String actualFile) {
+ Gson gson = new Gson();
+ // Load expected result JSON from resource folder
+ this.expectedResults = gson.fromJson(loadJsonFromResources(baselineFile), BaselineResult.class);
+ // Load the actual result from current benchmarking run
+ this.actualResults =
+ gson.fromJson(
+ loadJsonFromFile(actualFile),
+ new TypeToken>() {}.getType());
+ }
+
+ void validate() {
+ // Validating the resultant percentile against expected percentile with allowed threshold
+ for (ActualBenchmarkResult actualResult : actualResults) {
+ BenchmarkResult expectResult = expectedResults.benchmarkResultMap.get(actualResult.benchmark);
+ if (expectResult == null) {
+ throw new ValidationException(
+ "Missing expected benchmark configuration for actual benchmarking");
+ }
+ Map actualPercentilesMap = actualResult.primaryMetric.scorePercentiles;
+ // We will only be comparing the percentiles(p50, p90, p90) which are configured in the
+ // expected percentiles. This allows some checks to be disabled if required.
+ for (Percentile expectedPercentile : expectResult.scorePercentiles) {
+ String percentile = expectedPercentile.percentile;
+ double difference =
+ calculatePercentageDifference(
+ expectedPercentile.baseline, actualPercentilesMap.get(percentile));
+ // if an absolute different in percentage is greater than allowed difference
+ // Then we are throwing validation error
+ if (Math.abs(Math.ceil(difference)) > expectedPercentile.difference) {
+ throw new ValidationException(
+ String.format(
+ "[%s][%s] Expected percentile %s[+/-%s] but got %s",
+ actualResult.benchmark,
+ percentile,
+ expectedPercentile.baseline,
+ expectedPercentile.difference,
+ actualPercentilesMap.get(percentile)));
+ }
+ }
+ }
+ }
+
+ public static double calculatePercentageDifference(double base, double compareWith) {
+ if (base == 0) {
+ return 0.0;
+ }
+ return ((compareWith - base) / base) * 100;
+ }
+
+ private String loadJsonFromFile(String file) {
+ try {
+ return new String(Files.readAllBytes(Paths.get(file)));
+ } catch (IOException e) {
+ throw new ValidationException("Failed to read file: " + file, e);
+ }
+ }
+
+ private String loadJsonFromResources(String baselineFile) {
+ URL resourceUrl = getClass().getClassLoader().getResource(baselineFile);
+ if (resourceUrl == null) {
+ throw new ValidationException("File not found: " + baselineFile);
+ }
+ File file = new File(resourceUrl.getFile());
+ return loadJsonFromFile(file.getAbsolutePath());
+ }
+
+ static class ActualBenchmarkResult {
+ String benchmark;
+ PrimaryMetric primaryMetric;
+
+ static class PrimaryMetric {
+ Map scorePercentiles;
+ }
+ }
+
+ static class BaselineResult {
+ Map benchmarkResultMap;
+
+ static class BenchmarkResult {
+ List scorePercentiles;
+
+ static class Percentile {
+ String percentile;
+ Double baseline;
+ Double difference;
+ }
+ }
+ }
+
+ static class ValidationException extends RuntimeException {
+ ValidationException(String message) {
+ super(message);
+ }
+
+ ValidationException(String message, Throwable cause) {
+ super(message, cause);
+ }
+ }
+
+ private static String parseCommandLineArgs(String[] args, String key) {
+ if (args == null) {
+ return "";
+ }
+ for (String arg : args) {
+ if (arg.startsWith("--" + key)) {
+ String[] splits = arg.split("=");
+ if (splits.length == 2) {
+ return splits[1].trim();
+ }
+ }
+ }
+ return "";
+ }
+
+ public static void main(String[] args) {
+ String actualFile = parseCommandLineArgs(args, "file");
+ new BenchmarkValidator("com/google/cloud/spanner/jmh/jmh-baseline.json", actualFile).validate();
+ }
+}
diff --git a/google-cloud-spanner/src/test/java/com/google/cloud/spanner/benchmarking/MonitoringServiceImpl.java b/google-cloud-spanner/src/test/java/com/google/cloud/spanner/benchmarking/MonitoringServiceImpl.java
new file mode 100644
index 0000000000..aaa7387612
--- /dev/null
+++ b/google-cloud-spanner/src/test/java/com/google/cloud/spanner/benchmarking/MonitoringServiceImpl.java
@@ -0,0 +1,39 @@
+/*
+ * Copyright 2025 Google LLC
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * https://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package com.google.cloud.spanner.benchmarking;
+
+import com.google.monitoring.v3.CreateTimeSeriesRequest;
+import com.google.monitoring.v3.MetricServiceGrpc.MetricServiceImplBase;
+import com.google.protobuf.Empty;
+import io.grpc.Status;
+import io.grpc.stub.StreamObserver;
+
+class MonitoringServiceImpl extends MetricServiceImplBase {
+
+ @Override
+ public void createServiceTimeSeries(
+ CreateTimeSeriesRequest request, StreamObserver responseObserver) {
+ try {
+ Thread.sleep(100);
+ responseObserver.onNext(Empty.getDefaultInstance());
+ responseObserver.onCompleted();
+ } catch (InterruptedException e) {
+ responseObserver.onError(
+ Status.CANCELLED.withCause(e).withDescription(e.getMessage()).asException());
+ }
+ }
+}
diff --git a/google-cloud-spanner/src/test/java/com/google/cloud/spanner/benchmarking/ReadBenchmark.java b/google-cloud-spanner/src/test/java/com/google/cloud/spanner/benchmarking/ReadBenchmark.java
new file mode 100644
index 0000000000..eed461fc89
--- /dev/null
+++ b/google-cloud-spanner/src/test/java/com/google/cloud/spanner/benchmarking/ReadBenchmark.java
@@ -0,0 +1,228 @@
+/*
+ * Copyright 2025 Google LLC
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * https://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package com.google.cloud.spanner.benchmarking;
+
+import com.google.cloud.NoCredentials;
+import com.google.cloud.spanner.DatabaseClient;
+import com.google.cloud.spanner.DatabaseId;
+import com.google.cloud.spanner.Key;
+import com.google.cloud.spanner.KeySet;
+import com.google.cloud.spanner.MockSpannerServiceImpl;
+import com.google.cloud.spanner.MockSpannerServiceImpl.StatementResult;
+import com.google.cloud.spanner.ReadContext;
+import com.google.cloud.spanner.ResultSet;
+import com.google.cloud.spanner.Spanner;
+import com.google.cloud.spanner.SpannerOptions;
+import com.google.cloud.spanner.Statement;
+import com.google.protobuf.ListValue;
+import com.google.spanner.v1.ResultSetMetadata;
+import com.google.spanner.v1.StructType;
+import com.google.spanner.v1.StructType.Field;
+import com.google.spanner.v1.TypeCode;
+import io.grpc.ManagedChannelBuilder;
+import io.grpc.Server;
+import io.grpc.ServerBuilder;
+import java.io.IOException;
+import java.util.Arrays;
+import java.util.List;
+import java.util.concurrent.ExecutorService;
+import java.util.concurrent.Executors;
+import java.util.concurrent.TimeUnit;
+import org.openjdk.jmh.annotations.Benchmark;
+import org.openjdk.jmh.annotations.BenchmarkMode;
+import org.openjdk.jmh.annotations.Fork;
+import org.openjdk.jmh.annotations.Level;
+import org.openjdk.jmh.annotations.Measurement;
+import org.openjdk.jmh.annotations.Mode;
+import org.openjdk.jmh.annotations.OutputTimeUnit;
+import org.openjdk.jmh.annotations.Scope;
+import org.openjdk.jmh.annotations.Setup;
+import org.openjdk.jmh.annotations.State;
+import org.openjdk.jmh.annotations.TearDown;
+import org.openjdk.jmh.annotations.Threads;
+import org.openjdk.jmh.annotations.Timeout;
+import org.openjdk.jmh.annotations.Warmup;
+import org.openjdk.jmh.infra.Blackhole;
+import org.openjdk.jmh.results.format.ResultFormatType;
+import org.openjdk.jmh.runner.Runner;
+import org.openjdk.jmh.runner.RunnerException;
+import org.openjdk.jmh.runner.options.Options;
+import org.openjdk.jmh.runner.options.OptionsBuilder;
+
+@BenchmarkMode(Mode.SampleTime)
+@OutputTimeUnit(TimeUnit.MICROSECONDS)
+@Threads(10)
+@Fork(1)
+public class ReadBenchmark {
+
+ @State(Scope.Benchmark)
+ public static class BenchmarkState {
+
+ // Spanner state
+ Spanner spanner;
+ DatabaseClient databaseClient;
+
+ // gRPC server
+ Server gRPCServer;
+ Server gRPCMonitoringServer;
+
+ // Executors for handling parallel requests by gRPC server
+ ExecutorService gRPCServerExecutor;
+
+ // Table
+ List columns = Arrays.asList("id", "name");
+ String selectQuery = "SELECT * FROM [TABLE] WHERE ID = 1";
+
+ @Setup(Level.Trial)
+ public void setup() throws IOException {
+ // Enable JMH system property
+ System.setProperty("jmh.enabled", "true");
+
+ // Initializing mock spanner service
+ MockSpannerServiceImpl mockSpannerService = new MockSpannerServiceImpl();
+ mockSpannerService.setAbortProbability(0.0D);
+
+ // Initializing mock monitoring service
+ MonitoringServiceImpl mockMonitoringService = new MonitoringServiceImpl();
+
+ // Create a thread pool to handle concurrent requests
+ gRPCServerExecutor = Executors.newFixedThreadPool(Runtime.getRuntime().availableProcessors());
+
+ // Creating Spanner Inprocess gRPC server
+ gRPCServer =
+ ServerBuilder.forPort(0)
+ .addService(mockSpannerService)
+ .executor(gRPCServerExecutor)
+ .build()
+ .start();
+
+ registerMocks(mockSpannerService);
+
+ // Creating Monitoring Inprocess gRPC server
+ gRPCMonitoringServer =
+ ServerBuilder.forPort(0).addService(mockMonitoringService).build().start();
+
+ // Set the monitoring host port for exporter to forward requests to local netty gRPC server
+ System.setProperty(
+ "jmh.monitoring-server-port", String.valueOf(gRPCMonitoringServer.getPort()));
+
+ spanner =
+ SpannerOptions.newBuilder()
+ .setProjectId("[PROJECT]")
+ .setCredentials(NoCredentials.getInstance())
+ .setChannelConfigurator(
+ managedChannelBuilder ->
+ ManagedChannelBuilder.forAddress("0.0.0.0", gRPCServer.getPort())
+ .usePlaintext())
+ .build()
+ .getService();
+ databaseClient =
+ spanner.getDatabaseClient(DatabaseId.of("[PROJECT]", "[INSTANCE_ID]", "[DATABASE_ID]"));
+ }
+
+ private void registerMocks(MockSpannerServiceImpl mockSpannerService) {
+ ResultSetMetadata selectMetadata =
+ ResultSetMetadata.newBuilder()
+ .setRowType(
+ StructType.newBuilder()
+ .addFields(
+ Field.newBuilder()
+ .setName("id")
+ .setType(
+ com.google.spanner.v1.Type.newBuilder()
+ .setCode(TypeCode.INT64)
+ .build())
+ .build())
+ .addFields(
+ Field.newBuilder()
+ .setName("name")
+ .setType(
+ com.google.spanner.v1.Type.newBuilder()
+ .setCode(TypeCode.STRING)
+ .build())
+ .build())
+ .build())
+ .build();
+ com.google.spanner.v1.ResultSet selectResultSet =
+ com.google.spanner.v1.ResultSet.newBuilder()
+ .addRows(
+ ListValue.newBuilder()
+ .addValues(com.google.protobuf.Value.newBuilder().setStringValue("1").build())
+ .addValues(
+ com.google.protobuf.Value.newBuilder().setStringValue("[NAME]").build())
+ .build())
+ .setMetadata(selectMetadata)
+ .build();
+ mockSpannerService.putStatementResult(
+ StatementResult.read(
+ "[TABLE]", KeySet.singleKey(Key.of()), this.columns, selectResultSet));
+ mockSpannerService.putStatementResult(
+ StatementResult.query(Statement.of(this.selectQuery), selectResultSet));
+ }
+
+ @TearDown(Level.Trial)
+ public void tearDown() throws InterruptedException {
+ spanner.close();
+ gRPCServer.shutdown();
+ gRPCServerExecutor.shutdown();
+
+ // awaiting termination for servers and executors
+ gRPCServer.awaitTermination(10, TimeUnit.SECONDS);
+ gRPCServerExecutor.awaitTermination(10, TimeUnit.SECONDS);
+ }
+ }
+
+ @Benchmark
+ @Warmup(time = 5, timeUnit = TimeUnit.MINUTES, iterations = 1)
+ @Measurement(time = 15, timeUnit = TimeUnit.MINUTES, iterations = 1)
+ @Timeout(time = 30, timeUnit = TimeUnit.MINUTES)
+ public void readBenchmark(BenchmarkState benchmarkState, Blackhole blackhole) {
+ try (ReadContext readContext = benchmarkState.databaseClient.singleUse()) {
+ try (ResultSet resultSet =
+ readContext.read("[TABLE]", KeySet.singleKey(Key.of("2")), benchmarkState.columns)) {
+ while (resultSet.next()) {
+ blackhole.consume(resultSet.getLong("id"));
+ }
+ }
+ }
+ }
+
+ @Benchmark
+ @Warmup(time = 5, timeUnit = TimeUnit.MINUTES, iterations = 1)
+ @Measurement(time = 15, timeUnit = TimeUnit.MINUTES, iterations = 1)
+ @Timeout(time = 30, timeUnit = TimeUnit.MINUTES)
+ public void queryBenchmark(BenchmarkState benchmarkState, Blackhole blackhole) {
+ try (ReadContext readContext = benchmarkState.databaseClient.singleUse()) {
+ try (ResultSet resultSet =
+ readContext.executeQuery(Statement.of(benchmarkState.selectQuery))) {
+ while (resultSet.next()) {
+ blackhole.consume(resultSet.getLong("id"));
+ }
+ }
+ }
+ }
+
+ public static void main(String[] args) throws RunnerException {
+ Options opt =
+ new OptionsBuilder()
+ .include(ReadBenchmark.class.getSimpleName())
+ .result("jmh-result.json")
+ .resultFormat(ResultFormatType.JSON)
+ .build();
+ new Runner(opt).run();
+ }
+}
diff --git a/google-cloud-spanner/src/test/resources/com/google/cloud/spanner/jmh/jmh-baseline.json b/google-cloud-spanner/src/test/resources/com/google/cloud/spanner/jmh/jmh-baseline.json
new file mode 100644
index 0000000000..7753f173ee
--- /dev/null
+++ b/google-cloud-spanner/src/test/resources/com/google/cloud/spanner/jmh/jmh-baseline.json
@@ -0,0 +1,22 @@
+{
+ "benchmarkResultMap": {
+ "com.google.cloud.spanner.benchmarking.ReadBenchmark.queryBenchmark": {
+ "scorePercentiles": [
+ {
+ "percentile": "50.0",
+ "baseline": "450",
+ "difference": "15"
+ }
+ ]
+ },
+ "com.google.cloud.spanner.benchmarking.ReadBenchmark.readBenchmark": {
+ "scorePercentiles": [
+ {
+ "percentile": "50.0",
+ "baseline": "450",
+ "difference": "15"
+ }
+ ]
+ }
+ }
+}
\ No newline at end of file