From f8154ed98d13a63a5d42e869016308022ceef123 Mon Sep 17 00:00:00 2001 From: Sakthivel Subramanian Date: Sat, 20 Sep 2025 11:25:05 +0530 Subject: [PATCH 1/3] ci: Design and implement a comprehensive performance regression presubmit/continuous job --- google-cloud-spanner/pom.xml | 27 ++- .../SpannerCloudMonitoringExporter.java | 11 + .../benchmarking/BenchmarkValidator.java | 153 ++++++++++++ .../benchmarking/MonitoringServiceImpl.java | 37 +++ .../spanner/benchmarking/ReadBenchmark.java | 221 ++++++++++++++++++ .../cloud/spanner/jmh/jmh-baseline.json | 22 ++ 6 files changed, 470 insertions(+), 1 deletion(-) create mode 100644 google-cloud-spanner/src/test/java/com/google/cloud/spanner/benchmarking/BenchmarkValidator.java create mode 100644 google-cloud-spanner/src/test/java/com/google/cloud/spanner/benchmarking/MonitoringServiceImpl.java create mode 100644 google-cloud-spanner/src/test/java/com/google/cloud/spanner/benchmarking/ReadBenchmark.java create mode 100644 google-cloud-spanner/src/test/resources/com/google/cloud/spanner/jmh/jmh-baseline.json diff --git a/google-cloud-spanner/pom.xml b/google-cloud-spanner/pom.xml index 965991b346..f7d78e5e36 100644 --- a/google-cloud-spanner/pom.xml +++ b/google-cloud-spanner/pom.xml @@ -276,6 +276,12 @@ proto-google-cloud-monitoring-v3 3.76.0 + + com.google.api.grpc + grpc-google-cloud-monitoring-v3 + 3.63.0 + test + com.google.auth google-auth-library-oauth2-http @@ -522,7 +528,11 @@ -classpath org.openjdk.jmh.Main - ${benchmark.name} + ${benchmark.name} + -rf + JSON + -rff + jmh-results.json @@ -544,6 +554,21 @@ + + validate-benchmark + + + + org.codehaus.mojo + exec-maven-plugin + + com.google.cloud.spanner.benchmarking.BenchmarkValidator + test + + + + + slow-tests diff --git a/google-cloud-spanner/src/main/java/com/google/cloud/spanner/SpannerCloudMonitoringExporter.java b/google-cloud-spanner/src/main/java/com/google/cloud/spanner/SpannerCloudMonitoringExporter.java index 40202a0eef..621cd7cb38 100644 --- a/google-cloud-spanner/src/main/java/com/google/cloud/spanner/SpannerCloudMonitoringExporter.java +++ b/google-cloud-spanner/src/main/java/com/google/cloud/spanner/SpannerCloudMonitoringExporter.java @@ -22,6 +22,7 @@ import com.google.api.gax.core.CredentialsProvider; import com.google.api.gax.core.FixedCredentialsProvider; import com.google.api.gax.core.NoCredentialsProvider; +import com.google.api.gax.grpc.InstantiatingGrpcChannelProvider; import com.google.api.gax.rpc.PermissionDeniedException; import com.google.auth.Credentials; import com.google.cloud.monitoring.v3.MetricServiceClient; @@ -34,6 +35,7 @@ import com.google.monitoring.v3.ProjectName; import com.google.monitoring.v3.TimeSeries; import com.google.protobuf.Empty; +import io.grpc.inprocess.InProcessChannelBuilder; import io.opentelemetry.sdk.common.CompletableResultCode; import io.opentelemetry.sdk.metrics.InstrumentType; import io.opentelemetry.sdk.metrics.data.AggregationTemporality; @@ -92,6 +94,15 @@ static SpannerCloudMonitoringExporter create( settingsBuilder.setUniverseDomain(universeDomain); } + if (System.getProperty("jmh.monitoring-server") != null) { + settingsBuilder.setTransportChannelProvider( + InstantiatingGrpcChannelProvider.newBuilder() + .setChannelConfigurator( + managedChannelBuilder -> + InProcessChannelBuilder.forName(System.getProperty("jmh.monitoring-server"))) + .build()); + } + Duration timeout = Duration.ofMinutes(1); // TODO: createServiceTimeSeries needs special handling if the request failed. Leaving // it as not retried for now. diff --git a/google-cloud-spanner/src/test/java/com/google/cloud/spanner/benchmarking/BenchmarkValidator.java b/google-cloud-spanner/src/test/java/com/google/cloud/spanner/benchmarking/BenchmarkValidator.java new file mode 100644 index 0000000000..97b7eadb01 --- /dev/null +++ b/google-cloud-spanner/src/test/java/com/google/cloud/spanner/benchmarking/BenchmarkValidator.java @@ -0,0 +1,153 @@ +/* + * Copyright 2025 Google LLC + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * https://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.google.cloud.spanner.benchmarking; + +import com.google.cloud.spanner.benchmarking.BenchmarkValidator.BaselineResult.BenchmarkResult; +import com.google.cloud.spanner.benchmarking.BenchmarkValidator.BaselineResult.BenchmarkResult.Percentile; +import com.google.gson.Gson; +import com.google.gson.reflect.TypeToken; +import java.io.File; +import java.io.IOException; +import java.net.URL; +import java.nio.file.Files; +import java.nio.file.Paths; +import java.util.ArrayList; +import java.util.List; +import java.util.Map; + +public class BenchmarkValidator { + + private final BaselineResult expectedResults; + private final List actualResults; + + public BenchmarkValidator(String baselineFile, String actualFile) { + Gson gson = new Gson(); + // Load expected result JSON from resource folder + this.expectedResults = gson.fromJson(loadJsonFromResources(baselineFile), BaselineResult.class); + // Load the actual result from current benchmarking run + this.actualResults = + gson.fromJson( + loadJsonFromFile(actualFile), + new TypeToken>() {}.getType()); + } + + void validate() { + // Validating the resultant percentile against expected percentile with allowed threshold + for (ActualBenchmarkResult actualResult : actualResults) { + BenchmarkResult expectResult = expectedResults.benchmarkResultMap.get(actualResult.benchmark); + if (expectResult == null) { + throw new ValidationException( + "Missing expected benchmark configuration for actual benchmarking"); + } + Map actualPercentilesMap = actualResult.primaryMetric.scorePercentiles; + // We will only be comparing the percentiles(p50, p90, p90) which are configured in the + // expected + // percentiles. This allows some checks to be disabled if required. + for (Percentile expectedPercentile : expectResult.scorePercentiles) { + String percentile = expectedPercentile.percentile; + double difference = + calculatePercentageDifference( + expectedPercentile.baseline, actualPercentilesMap.get(percentile)); + // if an absolute different in percentage is greater than allowed difference + // Then we are throwing validation error + if (Math.abs(Math.ceil(difference)) > expectedPercentile.difference) { + throw new ValidationException( + String.format( + "[%s][%s] Expected percentile %s[+/-%s] but got %s", + actualResult.benchmark, + percentile, + expectedPercentile.baseline, + expectedPercentile.difference, + actualPercentilesMap.get(percentile))); + } + } + } + } + + public static double calculatePercentageDifference(double base, double compareWith) { + if (base == 0) { + return 0.0; + } + return ((compareWith - base) / base) * 100; + } + + private String loadJsonFromFile(String file) { + try { + return new String(Files.readAllBytes(Paths.get(file))); + } catch (IOException e) { + throw new ValidationException("Failed to read file: " + file, e); + } + } + + private String loadJsonFromResources(String baselineFile) { + URL resourceUrl = getClass().getClassLoader().getResource(baselineFile); + if (resourceUrl == null) { + throw new ValidationException("File not found: " + baselineFile); + } + File file = new File(resourceUrl.getFile()); + return loadJsonFromFile(file.getAbsolutePath()); + } + + static class ActualBenchmarkResult { + String benchmark; + PrimaryMetric primaryMetric; + + static class PrimaryMetric { + Map scorePercentiles; + } + } + + static class BaselineResult { + Map benchmarkResultMap; + + static class BenchmarkResult { + List scorePercentiles; + + static class Percentile { + String percentile; + Double baseline; + Double difference; + } + } + } + + static class ValidationException extends RuntimeException { + ValidationException(String message) { + super(message); + } + + ValidationException(String message, Throwable cause) { + super(message, cause); + } + } + + private static String parseCommandLineArg(String arg) { + if (arg == null || arg.isEmpty()) { + return ""; + } + String[] args = arg.split("="); + if (args.length != 2) { + return ""; + } + return args[1]; + } + + public static void main(String[] args) { + String actualFile = parseCommandLineArg(args[0]); + new BenchmarkValidator("com/google/cloud/spanner/jmh/jmh-baseline.json", actualFile).validate(); + } +} diff --git a/google-cloud-spanner/src/test/java/com/google/cloud/spanner/benchmarking/MonitoringServiceImpl.java b/google-cloud-spanner/src/test/java/com/google/cloud/spanner/benchmarking/MonitoringServiceImpl.java new file mode 100644 index 0000000000..3f0a5710f9 --- /dev/null +++ b/google-cloud-spanner/src/test/java/com/google/cloud/spanner/benchmarking/MonitoringServiceImpl.java @@ -0,0 +1,37 @@ +/* + * Copyright 2025 Google LLC + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * https://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.google.cloud.spanner.benchmarking; + +import com.google.monitoring.v3.CreateTimeSeriesRequest; +import com.google.monitoring.v3.MetricServiceGrpc.MetricServiceImplBase; +import com.google.protobuf.Empty; +import io.grpc.stub.StreamObserver; + +class MonitoringServiceImpl extends MetricServiceImplBase { + + @Override + public void createServiceTimeSeries( + CreateTimeSeriesRequest request, StreamObserver responseObserver) { + try { + Thread.sleep(100); + responseObserver.onNext(Empty.getDefaultInstance()); + responseObserver.onCompleted(); + } catch (InterruptedException e) { + responseObserver.onError(e); + } + } +} diff --git a/google-cloud-spanner/src/test/java/com/google/cloud/spanner/benchmarking/ReadBenchmark.java b/google-cloud-spanner/src/test/java/com/google/cloud/spanner/benchmarking/ReadBenchmark.java new file mode 100644 index 0000000000..99fe2f7b44 --- /dev/null +++ b/google-cloud-spanner/src/test/java/com/google/cloud/spanner/benchmarking/ReadBenchmark.java @@ -0,0 +1,221 @@ +/* + * Copyright 2025 Google LLC + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * https://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.google.cloud.spanner.benchmarking; + +import com.google.cloud.spanner.DatabaseClient; +import com.google.cloud.spanner.DatabaseId; +import com.google.cloud.spanner.Key; +import com.google.cloud.spanner.KeySet; +import com.google.cloud.spanner.MockSpannerServiceImpl; +import com.google.cloud.spanner.MockSpannerServiceImpl.StatementResult; +import com.google.cloud.spanner.ReadContext; +import com.google.cloud.spanner.ResultSet; +import com.google.cloud.spanner.Spanner; +import com.google.cloud.spanner.SpannerOptions; +import com.google.cloud.spanner.Statement; +import com.google.protobuf.ListValue; +import com.google.spanner.v1.ResultSetMetadata; +import com.google.spanner.v1.StructType; +import com.google.spanner.v1.StructType.Field; +import com.google.spanner.v1.TypeCode; +import io.grpc.Server; +import io.grpc.inprocess.InProcessChannelBuilder; +import io.grpc.inprocess.InProcessServerBuilder; +import java.io.IOException; +import java.util.Arrays; +import java.util.List; +import java.util.concurrent.ExecutorService; +import java.util.concurrent.Executors; +import java.util.concurrent.TimeUnit; +import org.openjdk.jmh.annotations.Benchmark; +import org.openjdk.jmh.annotations.BenchmarkMode; +import org.openjdk.jmh.annotations.Fork; +import org.openjdk.jmh.annotations.Level; +import org.openjdk.jmh.annotations.Measurement; +import org.openjdk.jmh.annotations.Mode; +import org.openjdk.jmh.annotations.OutputTimeUnit; +import org.openjdk.jmh.annotations.Scope; +import org.openjdk.jmh.annotations.Setup; +import org.openjdk.jmh.annotations.State; +import org.openjdk.jmh.annotations.TearDown; +import org.openjdk.jmh.annotations.Threads; +import org.openjdk.jmh.annotations.Timeout; +import org.openjdk.jmh.annotations.Warmup; +import org.openjdk.jmh.infra.Blackhole; +import org.openjdk.jmh.results.format.ResultFormatType; +import org.openjdk.jmh.runner.Runner; +import org.openjdk.jmh.runner.RunnerException; +import org.openjdk.jmh.runner.options.Options; +import org.openjdk.jmh.runner.options.OptionsBuilder; + +@BenchmarkMode(Mode.SampleTime) +@OutputTimeUnit(TimeUnit.MICROSECONDS) +@Threads(10) +@Fork(1) +public class ReadBenchmark { + + @State(Scope.Benchmark) + public static class BenchmarkState { + + // Spanner state + Spanner spanner; + DatabaseClient databaseClient; + + // gRPC server + Server gRPCServer; + Server gRPCMonitoringServer; + + // Executors for handling parallel requests by gRPC server + ExecutorService gRPCServerExecutor; + + // Table + List columns = Arrays.asList("id", "name"); + String selectQuery = "SELECT * FROM [TABLE] WHERE ID = 1"; + + @Setup(Level.Trial) + public void setup() throws IOException { + // Initializing mock spanner service + MockSpannerServiceImpl mockSpannerService = new MockSpannerServiceImpl(); + mockSpannerService.setAbortProbability(0.0D); + + // Initializing mock monitoring service + MonitoringServiceImpl mockMonitoringService = new MonitoringServiceImpl(); + + // Create a thread pool to handle concurrent requests + gRPCServerExecutor = Executors.newFixedThreadPool(Runtime.getRuntime().availableProcessors()); + + // Creating Spanner Inprocess gRPC server + String serverName = InProcessServerBuilder.generateName(); + gRPCServer = + InProcessServerBuilder.forName(serverName) + .addService(mockSpannerService) + .executor(gRPCServerExecutor) + .build() + .start(); + + registerMocks(mockSpannerService); + + // Creating Monitoring Inprocess gRPC server + String monitoringServerName = InProcessServerBuilder.generateName(); + gRPCMonitoringServer = + InProcessServerBuilder.forName(monitoringServerName) + .addService(mockMonitoringService) + .build() + .start(); + + // Set the monitoring host for exporter to forward requests to inprocess gRPC server + System.setProperty("jmh.monitoring-server", monitoringServerName); + + spanner = + SpannerOptions.newBuilder() + .setProjectId("[PROJECT]") + .setChannelConfigurator( + managedChannelBuilder -> InProcessChannelBuilder.forName(serverName)) + .build() + .getService(); + databaseClient = + spanner.getDatabaseClient(DatabaseId.of("[PROJECT]", "[INSTANCE_ID]", "[DATABASE_ID]")); + } + + private void registerMocks(MockSpannerServiceImpl mockSpannerService) { + ResultSetMetadata selectMetadata = + ResultSetMetadata.newBuilder() + .setRowType( + StructType.newBuilder() + .addFields( + Field.newBuilder() + .setName("id") + .setType( + com.google.spanner.v1.Type.newBuilder() + .setCode(TypeCode.INT64) + .build()) + .build()) + .addFields( + Field.newBuilder() + .setName("name") + .setType( + com.google.spanner.v1.Type.newBuilder() + .setCode(TypeCode.STRING) + .build()) + .build()) + .build()) + .build(); + com.google.spanner.v1.ResultSet selectResultSet = + com.google.spanner.v1.ResultSet.newBuilder() + .addRows( + ListValue.newBuilder() + .addValues(com.google.protobuf.Value.newBuilder().setStringValue("1").build()) + .addValues( + com.google.protobuf.Value.newBuilder().setStringValue("[NAME]").build()) + .build()) + .setMetadata(selectMetadata) + .build(); + mockSpannerService.putStatementResult( + StatementResult.read( + "[TABLE]", KeySet.singleKey(Key.of()), this.columns, selectResultSet)); + mockSpannerService.putStatementResult( + StatementResult.query(Statement.of(this.selectQuery), selectResultSet)); + } + + @TearDown(Level.Trial) + public void tearDown() { + spanner.close(); + gRPCServer.shutdown(); + gRPCServerExecutor.shutdown(); + } + } + + @Benchmark + @Warmup(time = 5, timeUnit = TimeUnit.MINUTES, iterations = 1) + @Measurement(time = 15, timeUnit = TimeUnit.MINUTES, iterations = 1) + @Timeout(time = 30, timeUnit = TimeUnit.MINUTES) + public void readBenchmark(BenchmarkState benchmarkState, Blackhole blackhole) { + try (ReadContext readContext = benchmarkState.databaseClient.singleUse()) { + try (ResultSet resultSet = + readContext.read("[TABLE]", KeySet.singleKey(Key.of("2")), benchmarkState.columns)) { + while (resultSet.next()) { + blackhole.consume(resultSet.getLong("id")); + } + } + } + } + + @Benchmark + @Warmup(time = 5, timeUnit = TimeUnit.MINUTES, iterations = 1) + @Measurement(time = 15, timeUnit = TimeUnit.MINUTES, iterations = 1) + @Timeout(time = 30, timeUnit = TimeUnit.MINUTES) + public void queryBenchmark(BenchmarkState benchmarkState, Blackhole blackhole) { + try (ReadContext readContext = benchmarkState.databaseClient.singleUse()) { + try (ResultSet resultSet = + readContext.executeQuery(Statement.of(benchmarkState.selectQuery))) { + while (resultSet.next()) { + blackhole.consume(resultSet.getLong("id")); + } + } + } + } + + public static void main(String[] args) throws RunnerException { + Options opt = + new OptionsBuilder() + .include(ReadBenchmark.class.getSimpleName()) + .result("jmh-result.json") + .resultFormat(ResultFormatType.JSON) + .build(); + new Runner(opt).run(); + } +} diff --git a/google-cloud-spanner/src/test/resources/com/google/cloud/spanner/jmh/jmh-baseline.json b/google-cloud-spanner/src/test/resources/com/google/cloud/spanner/jmh/jmh-baseline.json new file mode 100644 index 0000000000..b095d0439b --- /dev/null +++ b/google-cloud-spanner/src/test/resources/com/google/cloud/spanner/jmh/jmh-baseline.json @@ -0,0 +1,22 @@ +{ + "baselineConfigurations": { + "com.google.cloud.spanner.benchmarking.ReadBenchmark.queryBenchmark": { + "scorePercentiles": [ + { + "percentile": "50.0", + "baseline": "120", + "difference": "15" + } + ] + }, + "com.google.cloud.spanner.benchmarking.ReadBenchmark.readBenchmark": { + "scorePercentiles": [ + { + "percentile": "50.0", + "baseline": "120", + "difference": "15" + } + ] + } + } +} \ No newline at end of file From f219b7300c6f88bd8c046b8105a36b7ddbc393fd Mon Sep 17 00:00:00 2001 From: cloud-java-bot Date: Wed, 1 Oct 2025 08:27:53 +0000 Subject: [PATCH 2/3] chore: generate libraries at Wed Oct 1 08:25:18 UTC 2025 --- .github/workflows/update_generation_config.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/update_generation_config.yaml b/.github/workflows/update_generation_config.yaml index 59e39834dd..a7e14bb483 100644 --- a/.github/workflows/update_generation_config.yaml +++ b/.github/workflows/update_generation_config.yaml @@ -26,7 +26,7 @@ jobs: # the branch into which the pull request is merged base_branch: main steps: - - uses: actions/checkout@v5 + - uses: actions/checkout@v4 with: fetch-depth: 0 token: ${{ secrets.CLOUD_JAVA_BOT_TOKEN }} From c635377f279256c2762c7432639ca7bda1efb1a5 Mon Sep 17 00:00:00 2001 From: Sakthivel Subramanian Date: Wed, 1 Oct 2025 16:01:30 +0530 Subject: [PATCH 3/3] Addressed comments --- .../SpannerCloudMonitoringExporter.java | 13 +++++--- .../google/cloud/spanner/SpannerOptions.java | 5 +++ .../benchmarking/BenchmarkValidator.java | 21 +++++++----- .../benchmarking/MonitoringServiceImpl.java | 4 ++- .../spanner/benchmarking/ReadBenchmark.java | 33 +++++++++++-------- .../cloud/spanner/jmh/jmh-baseline.json | 6 ++-- 6 files changed, 52 insertions(+), 30 deletions(-) diff --git a/google-cloud-spanner/src/main/java/com/google/cloud/spanner/SpannerCloudMonitoringExporter.java b/google-cloud-spanner/src/main/java/com/google/cloud/spanner/SpannerCloudMonitoringExporter.java index 621cd7cb38..bedf660007 100644 --- a/google-cloud-spanner/src/main/java/com/google/cloud/spanner/SpannerCloudMonitoringExporter.java +++ b/google-cloud-spanner/src/main/java/com/google/cloud/spanner/SpannerCloudMonitoringExporter.java @@ -25,6 +25,7 @@ import com.google.api.gax.grpc.InstantiatingGrpcChannelProvider; import com.google.api.gax.rpc.PermissionDeniedException; import com.google.auth.Credentials; +import com.google.cloud.NoCredentials; import com.google.cloud.monitoring.v3.MetricServiceClient; import com.google.cloud.monitoring.v3.MetricServiceSettings; import com.google.common.annotations.VisibleForTesting; @@ -35,7 +36,7 @@ import com.google.monitoring.v3.ProjectName; import com.google.monitoring.v3.TimeSeries; import com.google.protobuf.Empty; -import io.grpc.inprocess.InProcessChannelBuilder; +import io.grpc.ManagedChannelBuilder; import io.opentelemetry.sdk.common.CompletableResultCode; import io.opentelemetry.sdk.metrics.InstrumentType; import io.opentelemetry.sdk.metrics.data.AggregationTemporality; @@ -81,7 +82,7 @@ static SpannerCloudMonitoringExporter create( throws IOException { MetricServiceSettings.Builder settingsBuilder = MetricServiceSettings.newBuilder(); CredentialsProvider credentialsProvider; - if (credentials == null) { + if (credentials == null || credentials instanceof NoCredentials) { credentialsProvider = NoCredentialsProvider.create(); } else { credentialsProvider = FixedCredentialsProvider.create(credentials); @@ -94,12 +95,16 @@ static SpannerCloudMonitoringExporter create( settingsBuilder.setUniverseDomain(universeDomain); } - if (System.getProperty("jmh.monitoring-server") != null) { + if (System.getProperty("jmh.monitoring-server-port") != null) { settingsBuilder.setTransportChannelProvider( InstantiatingGrpcChannelProvider.newBuilder() + .setCredentials(NoCredentials.getInstance()) .setChannelConfigurator( managedChannelBuilder -> - InProcessChannelBuilder.forName(System.getProperty("jmh.monitoring-server"))) + ManagedChannelBuilder.forAddress( + "0.0.0.0", + Integer.parseInt(System.getProperty("jmh.monitoring-server-port"))) + .usePlaintext()) .build()); } diff --git a/google-cloud-spanner/src/main/java/com/google/cloud/spanner/SpannerOptions.java b/google-cloud-spanner/src/main/java/com/google/cloud/spanner/SpannerOptions.java index af4ed58bad..765114dc68 100644 --- a/google-cloud-spanner/src/main/java/com/google/cloud/spanner/SpannerOptions.java +++ b/google-cloud-spanner/src/main/java/com/google/cloud/spanner/SpannerOptions.java @@ -2013,6 +2013,11 @@ public CallCredentialsProvider getCallCredentialsProvider() { } private boolean usesNoCredentials() { + // When JMH is enabled, we need to enable built-in metrics + if (System.getProperty("jmh.enabled") != null + && System.getProperty("jmh.enabled").equals("true")) { + return false; + } return Objects.equals(getCredentials(), NoCredentials.getInstance()); } diff --git a/google-cloud-spanner/src/test/java/com/google/cloud/spanner/benchmarking/BenchmarkValidator.java b/google-cloud-spanner/src/test/java/com/google/cloud/spanner/benchmarking/BenchmarkValidator.java index 97b7eadb01..225197af6c 100644 --- a/google-cloud-spanner/src/test/java/com/google/cloud/spanner/benchmarking/BenchmarkValidator.java +++ b/google-cloud-spanner/src/test/java/com/google/cloud/spanner/benchmarking/BenchmarkValidator.java @@ -55,8 +55,7 @@ void validate() { } Map actualPercentilesMap = actualResult.primaryMetric.scorePercentiles; // We will only be comparing the percentiles(p50, p90, p90) which are configured in the - // expected - // percentiles. This allows some checks to be disabled if required. + // expected percentiles. This allows some checks to be disabled if required. for (Percentile expectedPercentile : expectResult.scorePercentiles) { String percentile = expectedPercentile.percentile; double difference = @@ -135,19 +134,23 @@ static class ValidationException extends RuntimeException { } } - private static String parseCommandLineArg(String arg) { - if (arg == null || arg.isEmpty()) { + private static String parseCommandLineArgs(String[] args, String key) { + if (args == null) { return ""; } - String[] args = arg.split("="); - if (args.length != 2) { - return ""; + for (String arg : args) { + if (arg.startsWith("--" + key)) { + String[] splits = arg.split("="); + if (splits.length == 2) { + return splits[1].trim(); + } + } } - return args[1]; + return ""; } public static void main(String[] args) { - String actualFile = parseCommandLineArg(args[0]); + String actualFile = parseCommandLineArgs(args, "file"); new BenchmarkValidator("com/google/cloud/spanner/jmh/jmh-baseline.json", actualFile).validate(); } } diff --git a/google-cloud-spanner/src/test/java/com/google/cloud/spanner/benchmarking/MonitoringServiceImpl.java b/google-cloud-spanner/src/test/java/com/google/cloud/spanner/benchmarking/MonitoringServiceImpl.java index 3f0a5710f9..aaa7387612 100644 --- a/google-cloud-spanner/src/test/java/com/google/cloud/spanner/benchmarking/MonitoringServiceImpl.java +++ b/google-cloud-spanner/src/test/java/com/google/cloud/spanner/benchmarking/MonitoringServiceImpl.java @@ -19,6 +19,7 @@ import com.google.monitoring.v3.CreateTimeSeriesRequest; import com.google.monitoring.v3.MetricServiceGrpc.MetricServiceImplBase; import com.google.protobuf.Empty; +import io.grpc.Status; import io.grpc.stub.StreamObserver; class MonitoringServiceImpl extends MetricServiceImplBase { @@ -31,7 +32,8 @@ public void createServiceTimeSeries( responseObserver.onNext(Empty.getDefaultInstance()); responseObserver.onCompleted(); } catch (InterruptedException e) { - responseObserver.onError(e); + responseObserver.onError( + Status.CANCELLED.withCause(e).withDescription(e.getMessage()).asException()); } } } diff --git a/google-cloud-spanner/src/test/java/com/google/cloud/spanner/benchmarking/ReadBenchmark.java b/google-cloud-spanner/src/test/java/com/google/cloud/spanner/benchmarking/ReadBenchmark.java index 99fe2f7b44..eed461fc89 100644 --- a/google-cloud-spanner/src/test/java/com/google/cloud/spanner/benchmarking/ReadBenchmark.java +++ b/google-cloud-spanner/src/test/java/com/google/cloud/spanner/benchmarking/ReadBenchmark.java @@ -16,6 +16,7 @@ package com.google.cloud.spanner.benchmarking; +import com.google.cloud.NoCredentials; import com.google.cloud.spanner.DatabaseClient; import com.google.cloud.spanner.DatabaseId; import com.google.cloud.spanner.Key; @@ -32,9 +33,9 @@ import com.google.spanner.v1.StructType; import com.google.spanner.v1.StructType.Field; import com.google.spanner.v1.TypeCode; +import io.grpc.ManagedChannelBuilder; import io.grpc.Server; -import io.grpc.inprocess.InProcessChannelBuilder; -import io.grpc.inprocess.InProcessServerBuilder; +import io.grpc.ServerBuilder; import java.io.IOException; import java.util.Arrays; import java.util.List; @@ -88,6 +89,9 @@ public static class BenchmarkState { @Setup(Level.Trial) public void setup() throws IOException { + // Enable JMH system property + System.setProperty("jmh.enabled", "true"); + // Initializing mock spanner service MockSpannerServiceImpl mockSpannerService = new MockSpannerServiceImpl(); mockSpannerService.setAbortProbability(0.0D); @@ -99,9 +103,8 @@ public void setup() throws IOException { gRPCServerExecutor = Executors.newFixedThreadPool(Runtime.getRuntime().availableProcessors()); // Creating Spanner Inprocess gRPC server - String serverName = InProcessServerBuilder.generateName(); gRPCServer = - InProcessServerBuilder.forName(serverName) + ServerBuilder.forPort(0) .addService(mockSpannerService) .executor(gRPCServerExecutor) .build() @@ -110,21 +113,21 @@ public void setup() throws IOException { registerMocks(mockSpannerService); // Creating Monitoring Inprocess gRPC server - String monitoringServerName = InProcessServerBuilder.generateName(); gRPCMonitoringServer = - InProcessServerBuilder.forName(monitoringServerName) - .addService(mockMonitoringService) - .build() - .start(); + ServerBuilder.forPort(0).addService(mockMonitoringService).build().start(); - // Set the monitoring host for exporter to forward requests to inprocess gRPC server - System.setProperty("jmh.monitoring-server", monitoringServerName); + // Set the monitoring host port for exporter to forward requests to local netty gRPC server + System.setProperty( + "jmh.monitoring-server-port", String.valueOf(gRPCMonitoringServer.getPort())); spanner = SpannerOptions.newBuilder() .setProjectId("[PROJECT]") + .setCredentials(NoCredentials.getInstance()) .setChannelConfigurator( - managedChannelBuilder -> InProcessChannelBuilder.forName(serverName)) + managedChannelBuilder -> + ManagedChannelBuilder.forAddress("0.0.0.0", gRPCServer.getPort()) + .usePlaintext()) .build() .getService(); databaseClient = @@ -172,10 +175,14 @@ private void registerMocks(MockSpannerServiceImpl mockSpannerService) { } @TearDown(Level.Trial) - public void tearDown() { + public void tearDown() throws InterruptedException { spanner.close(); gRPCServer.shutdown(); gRPCServerExecutor.shutdown(); + + // awaiting termination for servers and executors + gRPCServer.awaitTermination(10, TimeUnit.SECONDS); + gRPCServerExecutor.awaitTermination(10, TimeUnit.SECONDS); } } diff --git a/google-cloud-spanner/src/test/resources/com/google/cloud/spanner/jmh/jmh-baseline.json b/google-cloud-spanner/src/test/resources/com/google/cloud/spanner/jmh/jmh-baseline.json index b095d0439b..7753f173ee 100644 --- a/google-cloud-spanner/src/test/resources/com/google/cloud/spanner/jmh/jmh-baseline.json +++ b/google-cloud-spanner/src/test/resources/com/google/cloud/spanner/jmh/jmh-baseline.json @@ -1,10 +1,10 @@ { - "baselineConfigurations": { + "benchmarkResultMap": { "com.google.cloud.spanner.benchmarking.ReadBenchmark.queryBenchmark": { "scorePercentiles": [ { "percentile": "50.0", - "baseline": "120", + "baseline": "450", "difference": "15" } ] @@ -13,7 +13,7 @@ "scorePercentiles": [ { "percentile": "50.0", - "baseline": "120", + "baseline": "450", "difference": "15" } ]