diff --git a/.github/workflows/update_generation_config.yaml b/.github/workflows/update_generation_config.yaml index 59e39834dd..a7e14bb483 100644 --- a/.github/workflows/update_generation_config.yaml +++ b/.github/workflows/update_generation_config.yaml @@ -26,7 +26,7 @@ jobs: # the branch into which the pull request is merged base_branch: main steps: - - uses: actions/checkout@v5 + - uses: actions/checkout@v4 with: fetch-depth: 0 token: ${{ secrets.CLOUD_JAVA_BOT_TOKEN }} diff --git a/google-cloud-spanner/pom.xml b/google-cloud-spanner/pom.xml index 965991b346..f7d78e5e36 100644 --- a/google-cloud-spanner/pom.xml +++ b/google-cloud-spanner/pom.xml @@ -276,6 +276,12 @@ proto-google-cloud-monitoring-v3 3.76.0 + + com.google.api.grpc + grpc-google-cloud-monitoring-v3 + 3.63.0 + test + com.google.auth google-auth-library-oauth2-http @@ -522,7 +528,11 @@ -classpath org.openjdk.jmh.Main - ${benchmark.name} + ${benchmark.name} + -rf + JSON + -rff + jmh-results.json @@ -544,6 +554,21 @@ + + validate-benchmark + + + + org.codehaus.mojo + exec-maven-plugin + + com.google.cloud.spanner.benchmarking.BenchmarkValidator + test + + + + + slow-tests diff --git a/google-cloud-spanner/src/main/java/com/google/cloud/spanner/SpannerCloudMonitoringExporter.java b/google-cloud-spanner/src/main/java/com/google/cloud/spanner/SpannerCloudMonitoringExporter.java index 40202a0eef..bedf660007 100644 --- a/google-cloud-spanner/src/main/java/com/google/cloud/spanner/SpannerCloudMonitoringExporter.java +++ b/google-cloud-spanner/src/main/java/com/google/cloud/spanner/SpannerCloudMonitoringExporter.java @@ -22,8 +22,10 @@ import com.google.api.gax.core.CredentialsProvider; import com.google.api.gax.core.FixedCredentialsProvider; import com.google.api.gax.core.NoCredentialsProvider; +import com.google.api.gax.grpc.InstantiatingGrpcChannelProvider; import com.google.api.gax.rpc.PermissionDeniedException; import com.google.auth.Credentials; +import com.google.cloud.NoCredentials; import com.google.cloud.monitoring.v3.MetricServiceClient; import com.google.cloud.monitoring.v3.MetricServiceSettings; import com.google.common.annotations.VisibleForTesting; @@ -34,6 +36,7 @@ import com.google.monitoring.v3.ProjectName; import com.google.monitoring.v3.TimeSeries; import com.google.protobuf.Empty; +import io.grpc.ManagedChannelBuilder; import io.opentelemetry.sdk.common.CompletableResultCode; import io.opentelemetry.sdk.metrics.InstrumentType; import io.opentelemetry.sdk.metrics.data.AggregationTemporality; @@ -79,7 +82,7 @@ static SpannerCloudMonitoringExporter create( throws IOException { MetricServiceSettings.Builder settingsBuilder = MetricServiceSettings.newBuilder(); CredentialsProvider credentialsProvider; - if (credentials == null) { + if (credentials == null || credentials instanceof NoCredentials) { credentialsProvider = NoCredentialsProvider.create(); } else { credentialsProvider = FixedCredentialsProvider.create(credentials); @@ -92,6 +95,19 @@ static SpannerCloudMonitoringExporter create( settingsBuilder.setUniverseDomain(universeDomain); } + if (System.getProperty("jmh.monitoring-server-port") != null) { + settingsBuilder.setTransportChannelProvider( + InstantiatingGrpcChannelProvider.newBuilder() + .setCredentials(NoCredentials.getInstance()) + .setChannelConfigurator( + managedChannelBuilder -> + ManagedChannelBuilder.forAddress( + "0.0.0.0", + Integer.parseInt(System.getProperty("jmh.monitoring-server-port"))) + .usePlaintext()) + .build()); + } + Duration timeout = Duration.ofMinutes(1); // TODO: createServiceTimeSeries needs special handling if the request failed. Leaving // it as not retried for now. diff --git a/google-cloud-spanner/src/main/java/com/google/cloud/spanner/SpannerOptions.java b/google-cloud-spanner/src/main/java/com/google/cloud/spanner/SpannerOptions.java index af4ed58bad..765114dc68 100644 --- a/google-cloud-spanner/src/main/java/com/google/cloud/spanner/SpannerOptions.java +++ b/google-cloud-spanner/src/main/java/com/google/cloud/spanner/SpannerOptions.java @@ -2013,6 +2013,11 @@ public CallCredentialsProvider getCallCredentialsProvider() { } private boolean usesNoCredentials() { + // When JMH is enabled, we need to enable built-in metrics + if (System.getProperty("jmh.enabled") != null + && System.getProperty("jmh.enabled").equals("true")) { + return false; + } return Objects.equals(getCredentials(), NoCredentials.getInstance()); } diff --git a/google-cloud-spanner/src/test/java/com/google/cloud/spanner/benchmarking/BenchmarkValidator.java b/google-cloud-spanner/src/test/java/com/google/cloud/spanner/benchmarking/BenchmarkValidator.java new file mode 100644 index 0000000000..225197af6c --- /dev/null +++ b/google-cloud-spanner/src/test/java/com/google/cloud/spanner/benchmarking/BenchmarkValidator.java @@ -0,0 +1,156 @@ +/* + * Copyright 2025 Google LLC + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * https://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.google.cloud.spanner.benchmarking; + +import com.google.cloud.spanner.benchmarking.BenchmarkValidator.BaselineResult.BenchmarkResult; +import com.google.cloud.spanner.benchmarking.BenchmarkValidator.BaselineResult.BenchmarkResult.Percentile; +import com.google.gson.Gson; +import com.google.gson.reflect.TypeToken; +import java.io.File; +import java.io.IOException; +import java.net.URL; +import java.nio.file.Files; +import java.nio.file.Paths; +import java.util.ArrayList; +import java.util.List; +import java.util.Map; + +public class BenchmarkValidator { + + private final BaselineResult expectedResults; + private final List actualResults; + + public BenchmarkValidator(String baselineFile, String actualFile) { + Gson gson = new Gson(); + // Load expected result JSON from resource folder + this.expectedResults = gson.fromJson(loadJsonFromResources(baselineFile), BaselineResult.class); + // Load the actual result from current benchmarking run + this.actualResults = + gson.fromJson( + loadJsonFromFile(actualFile), + new TypeToken>() {}.getType()); + } + + void validate() { + // Validating the resultant percentile against expected percentile with allowed threshold + for (ActualBenchmarkResult actualResult : actualResults) { + BenchmarkResult expectResult = expectedResults.benchmarkResultMap.get(actualResult.benchmark); + if (expectResult == null) { + throw new ValidationException( + "Missing expected benchmark configuration for actual benchmarking"); + } + Map actualPercentilesMap = actualResult.primaryMetric.scorePercentiles; + // We will only be comparing the percentiles(p50, p90, p90) which are configured in the + // expected percentiles. This allows some checks to be disabled if required. + for (Percentile expectedPercentile : expectResult.scorePercentiles) { + String percentile = expectedPercentile.percentile; + double difference = + calculatePercentageDifference( + expectedPercentile.baseline, actualPercentilesMap.get(percentile)); + // if an absolute different in percentage is greater than allowed difference + // Then we are throwing validation error + if (Math.abs(Math.ceil(difference)) > expectedPercentile.difference) { + throw new ValidationException( + String.format( + "[%s][%s] Expected percentile %s[+/-%s] but got %s", + actualResult.benchmark, + percentile, + expectedPercentile.baseline, + expectedPercentile.difference, + actualPercentilesMap.get(percentile))); + } + } + } + } + + public static double calculatePercentageDifference(double base, double compareWith) { + if (base == 0) { + return 0.0; + } + return ((compareWith - base) / base) * 100; + } + + private String loadJsonFromFile(String file) { + try { + return new String(Files.readAllBytes(Paths.get(file))); + } catch (IOException e) { + throw new ValidationException("Failed to read file: " + file, e); + } + } + + private String loadJsonFromResources(String baselineFile) { + URL resourceUrl = getClass().getClassLoader().getResource(baselineFile); + if (resourceUrl == null) { + throw new ValidationException("File not found: " + baselineFile); + } + File file = new File(resourceUrl.getFile()); + return loadJsonFromFile(file.getAbsolutePath()); + } + + static class ActualBenchmarkResult { + String benchmark; + PrimaryMetric primaryMetric; + + static class PrimaryMetric { + Map scorePercentiles; + } + } + + static class BaselineResult { + Map benchmarkResultMap; + + static class BenchmarkResult { + List scorePercentiles; + + static class Percentile { + String percentile; + Double baseline; + Double difference; + } + } + } + + static class ValidationException extends RuntimeException { + ValidationException(String message) { + super(message); + } + + ValidationException(String message, Throwable cause) { + super(message, cause); + } + } + + private static String parseCommandLineArgs(String[] args, String key) { + if (args == null) { + return ""; + } + for (String arg : args) { + if (arg.startsWith("--" + key)) { + String[] splits = arg.split("="); + if (splits.length == 2) { + return splits[1].trim(); + } + } + } + return ""; + } + + public static void main(String[] args) { + String actualFile = parseCommandLineArgs(args, "file"); + new BenchmarkValidator("com/google/cloud/spanner/jmh/jmh-baseline.json", actualFile).validate(); + } +} diff --git a/google-cloud-spanner/src/test/java/com/google/cloud/spanner/benchmarking/MonitoringServiceImpl.java b/google-cloud-spanner/src/test/java/com/google/cloud/spanner/benchmarking/MonitoringServiceImpl.java new file mode 100644 index 0000000000..aaa7387612 --- /dev/null +++ b/google-cloud-spanner/src/test/java/com/google/cloud/spanner/benchmarking/MonitoringServiceImpl.java @@ -0,0 +1,39 @@ +/* + * Copyright 2025 Google LLC + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * https://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.google.cloud.spanner.benchmarking; + +import com.google.monitoring.v3.CreateTimeSeriesRequest; +import com.google.monitoring.v3.MetricServiceGrpc.MetricServiceImplBase; +import com.google.protobuf.Empty; +import io.grpc.Status; +import io.grpc.stub.StreamObserver; + +class MonitoringServiceImpl extends MetricServiceImplBase { + + @Override + public void createServiceTimeSeries( + CreateTimeSeriesRequest request, StreamObserver responseObserver) { + try { + Thread.sleep(100); + responseObserver.onNext(Empty.getDefaultInstance()); + responseObserver.onCompleted(); + } catch (InterruptedException e) { + responseObserver.onError( + Status.CANCELLED.withCause(e).withDescription(e.getMessage()).asException()); + } + } +} diff --git a/google-cloud-spanner/src/test/java/com/google/cloud/spanner/benchmarking/ReadBenchmark.java b/google-cloud-spanner/src/test/java/com/google/cloud/spanner/benchmarking/ReadBenchmark.java new file mode 100644 index 0000000000..eed461fc89 --- /dev/null +++ b/google-cloud-spanner/src/test/java/com/google/cloud/spanner/benchmarking/ReadBenchmark.java @@ -0,0 +1,228 @@ +/* + * Copyright 2025 Google LLC + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * https://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.google.cloud.spanner.benchmarking; + +import com.google.cloud.NoCredentials; +import com.google.cloud.spanner.DatabaseClient; +import com.google.cloud.spanner.DatabaseId; +import com.google.cloud.spanner.Key; +import com.google.cloud.spanner.KeySet; +import com.google.cloud.spanner.MockSpannerServiceImpl; +import com.google.cloud.spanner.MockSpannerServiceImpl.StatementResult; +import com.google.cloud.spanner.ReadContext; +import com.google.cloud.spanner.ResultSet; +import com.google.cloud.spanner.Spanner; +import com.google.cloud.spanner.SpannerOptions; +import com.google.cloud.spanner.Statement; +import com.google.protobuf.ListValue; +import com.google.spanner.v1.ResultSetMetadata; +import com.google.spanner.v1.StructType; +import com.google.spanner.v1.StructType.Field; +import com.google.spanner.v1.TypeCode; +import io.grpc.ManagedChannelBuilder; +import io.grpc.Server; +import io.grpc.ServerBuilder; +import java.io.IOException; +import java.util.Arrays; +import java.util.List; +import java.util.concurrent.ExecutorService; +import java.util.concurrent.Executors; +import java.util.concurrent.TimeUnit; +import org.openjdk.jmh.annotations.Benchmark; +import org.openjdk.jmh.annotations.BenchmarkMode; +import org.openjdk.jmh.annotations.Fork; +import org.openjdk.jmh.annotations.Level; +import org.openjdk.jmh.annotations.Measurement; +import org.openjdk.jmh.annotations.Mode; +import org.openjdk.jmh.annotations.OutputTimeUnit; +import org.openjdk.jmh.annotations.Scope; +import org.openjdk.jmh.annotations.Setup; +import org.openjdk.jmh.annotations.State; +import org.openjdk.jmh.annotations.TearDown; +import org.openjdk.jmh.annotations.Threads; +import org.openjdk.jmh.annotations.Timeout; +import org.openjdk.jmh.annotations.Warmup; +import org.openjdk.jmh.infra.Blackhole; +import org.openjdk.jmh.results.format.ResultFormatType; +import org.openjdk.jmh.runner.Runner; +import org.openjdk.jmh.runner.RunnerException; +import org.openjdk.jmh.runner.options.Options; +import org.openjdk.jmh.runner.options.OptionsBuilder; + +@BenchmarkMode(Mode.SampleTime) +@OutputTimeUnit(TimeUnit.MICROSECONDS) +@Threads(10) +@Fork(1) +public class ReadBenchmark { + + @State(Scope.Benchmark) + public static class BenchmarkState { + + // Spanner state + Spanner spanner; + DatabaseClient databaseClient; + + // gRPC server + Server gRPCServer; + Server gRPCMonitoringServer; + + // Executors for handling parallel requests by gRPC server + ExecutorService gRPCServerExecutor; + + // Table + List columns = Arrays.asList("id", "name"); + String selectQuery = "SELECT * FROM [TABLE] WHERE ID = 1"; + + @Setup(Level.Trial) + public void setup() throws IOException { + // Enable JMH system property + System.setProperty("jmh.enabled", "true"); + + // Initializing mock spanner service + MockSpannerServiceImpl mockSpannerService = new MockSpannerServiceImpl(); + mockSpannerService.setAbortProbability(0.0D); + + // Initializing mock monitoring service + MonitoringServiceImpl mockMonitoringService = new MonitoringServiceImpl(); + + // Create a thread pool to handle concurrent requests + gRPCServerExecutor = Executors.newFixedThreadPool(Runtime.getRuntime().availableProcessors()); + + // Creating Spanner Inprocess gRPC server + gRPCServer = + ServerBuilder.forPort(0) + .addService(mockSpannerService) + .executor(gRPCServerExecutor) + .build() + .start(); + + registerMocks(mockSpannerService); + + // Creating Monitoring Inprocess gRPC server + gRPCMonitoringServer = + ServerBuilder.forPort(0).addService(mockMonitoringService).build().start(); + + // Set the monitoring host port for exporter to forward requests to local netty gRPC server + System.setProperty( + "jmh.monitoring-server-port", String.valueOf(gRPCMonitoringServer.getPort())); + + spanner = + SpannerOptions.newBuilder() + .setProjectId("[PROJECT]") + .setCredentials(NoCredentials.getInstance()) + .setChannelConfigurator( + managedChannelBuilder -> + ManagedChannelBuilder.forAddress("0.0.0.0", gRPCServer.getPort()) + .usePlaintext()) + .build() + .getService(); + databaseClient = + spanner.getDatabaseClient(DatabaseId.of("[PROJECT]", "[INSTANCE_ID]", "[DATABASE_ID]")); + } + + private void registerMocks(MockSpannerServiceImpl mockSpannerService) { + ResultSetMetadata selectMetadata = + ResultSetMetadata.newBuilder() + .setRowType( + StructType.newBuilder() + .addFields( + Field.newBuilder() + .setName("id") + .setType( + com.google.spanner.v1.Type.newBuilder() + .setCode(TypeCode.INT64) + .build()) + .build()) + .addFields( + Field.newBuilder() + .setName("name") + .setType( + com.google.spanner.v1.Type.newBuilder() + .setCode(TypeCode.STRING) + .build()) + .build()) + .build()) + .build(); + com.google.spanner.v1.ResultSet selectResultSet = + com.google.spanner.v1.ResultSet.newBuilder() + .addRows( + ListValue.newBuilder() + .addValues(com.google.protobuf.Value.newBuilder().setStringValue("1").build()) + .addValues( + com.google.protobuf.Value.newBuilder().setStringValue("[NAME]").build()) + .build()) + .setMetadata(selectMetadata) + .build(); + mockSpannerService.putStatementResult( + StatementResult.read( + "[TABLE]", KeySet.singleKey(Key.of()), this.columns, selectResultSet)); + mockSpannerService.putStatementResult( + StatementResult.query(Statement.of(this.selectQuery), selectResultSet)); + } + + @TearDown(Level.Trial) + public void tearDown() throws InterruptedException { + spanner.close(); + gRPCServer.shutdown(); + gRPCServerExecutor.shutdown(); + + // awaiting termination for servers and executors + gRPCServer.awaitTermination(10, TimeUnit.SECONDS); + gRPCServerExecutor.awaitTermination(10, TimeUnit.SECONDS); + } + } + + @Benchmark + @Warmup(time = 5, timeUnit = TimeUnit.MINUTES, iterations = 1) + @Measurement(time = 15, timeUnit = TimeUnit.MINUTES, iterations = 1) + @Timeout(time = 30, timeUnit = TimeUnit.MINUTES) + public void readBenchmark(BenchmarkState benchmarkState, Blackhole blackhole) { + try (ReadContext readContext = benchmarkState.databaseClient.singleUse()) { + try (ResultSet resultSet = + readContext.read("[TABLE]", KeySet.singleKey(Key.of("2")), benchmarkState.columns)) { + while (resultSet.next()) { + blackhole.consume(resultSet.getLong("id")); + } + } + } + } + + @Benchmark + @Warmup(time = 5, timeUnit = TimeUnit.MINUTES, iterations = 1) + @Measurement(time = 15, timeUnit = TimeUnit.MINUTES, iterations = 1) + @Timeout(time = 30, timeUnit = TimeUnit.MINUTES) + public void queryBenchmark(BenchmarkState benchmarkState, Blackhole blackhole) { + try (ReadContext readContext = benchmarkState.databaseClient.singleUse()) { + try (ResultSet resultSet = + readContext.executeQuery(Statement.of(benchmarkState.selectQuery))) { + while (resultSet.next()) { + blackhole.consume(resultSet.getLong("id")); + } + } + } + } + + public static void main(String[] args) throws RunnerException { + Options opt = + new OptionsBuilder() + .include(ReadBenchmark.class.getSimpleName()) + .result("jmh-result.json") + .resultFormat(ResultFormatType.JSON) + .build(); + new Runner(opt).run(); + } +} diff --git a/google-cloud-spanner/src/test/resources/com/google/cloud/spanner/jmh/jmh-baseline.json b/google-cloud-spanner/src/test/resources/com/google/cloud/spanner/jmh/jmh-baseline.json new file mode 100644 index 0000000000..7753f173ee --- /dev/null +++ b/google-cloud-spanner/src/test/resources/com/google/cloud/spanner/jmh/jmh-baseline.json @@ -0,0 +1,22 @@ +{ + "benchmarkResultMap": { + "com.google.cloud.spanner.benchmarking.ReadBenchmark.queryBenchmark": { + "scorePercentiles": [ + { + "percentile": "50.0", + "baseline": "450", + "difference": "15" + } + ] + }, + "com.google.cloud.spanner.benchmarking.ReadBenchmark.readBenchmark": { + "scorePercentiles": [ + { + "percentile": "50.0", + "baseline": "450", + "difference": "15" + } + ] + } + } +} \ No newline at end of file