ci: Design and implement a comprehensive performance regression presubmit/continuous job

.github/workflows/update_generation_config.yaml

@@ -26,7 +26,7 @@ jobs:
       # the branch into which the pull request is merged
       base_branch: main
     steps:
-    - uses: actions/checkout@v5
+    - uses: actions/checkout@v4
       with:
         fetch-depth: 0
         token: ${{ secrets.CLOUD_JAVA_BOT_TOKEN }}

google-cloud-spanner/pom.xml

@@ -276,6 +276,12 @@
       <artifactId>proto-google-cloud-monitoring-v3</artifactId>
       <version>3.76.0</version>
     </dependency>
+    <dependency>
+      <groupId>com.google.api.grpc</groupId>
+      <artifactId>grpc-google-cloud-monitoring-v3</artifactId>
+      <version>3.63.0</version>
+      <scope>test</scope>
+    </dependency>
     <dependency>
       <groupId>com.google.auth</groupId>
       <artifactId>google-auth-library-oauth2-http</artifactId>
@@ -522,7 +528,11 @@
                       <argument>-classpath</argument>
                       <classpath/>
                       <argument>org.openjdk.jmh.Main</argument>
-                      <argument>${benchmark.name}</argument>
+                        <argument>${benchmark.name}</argument>
+                      <argument>-rf</argument>
+                      <argument>JSON</argument>
+                      <argument>-rff</argument>
+                      <argument>jmh-results.json</argument>
                   </arguments>
                 </configuration>
               </execution>
@@ -544,6 +554,21 @@
         </plugins>
       </build>
     </profile>
+    <profile>
+      <id>validate-benchmark</id>
+      <build>
+        <plugins>
+          <plugin>
+            <groupId>org.codehaus.mojo</groupId>
+            <artifactId>exec-maven-plugin</artifactId>
+            <configuration>
+              <mainClass>com.google.cloud.spanner.benchmarking.BenchmarkValidator</mainClass>
+              <classpathScope>test</classpathScope>
+            </configuration>
+          </plugin>
+        </plugins>
+      </build>
+    </profile>
     <profile>
       <id>slow-tests</id>
       <build>

google-cloud-spanner/src/main/java/com/google/cloud/spanner/SpannerCloudMonitoringExporter.java

@@ -22,8 +22,10 @@
 import com.google.api.gax.core.CredentialsProvider;
 import com.google.api.gax.core.FixedCredentialsProvider;
 import com.google.api.gax.core.NoCredentialsProvider;
+import com.google.api.gax.grpc.InstantiatingGrpcChannelProvider;
 import com.google.api.gax.rpc.PermissionDeniedException;
 import com.google.auth.Credentials;
+import com.google.cloud.NoCredentials;
 import com.google.cloud.monitoring.v3.MetricServiceClient;
 import com.google.cloud.monitoring.v3.MetricServiceSettings;
 import com.google.common.annotations.VisibleForTesting;
@@ -34,6 +36,7 @@
 import com.google.monitoring.v3.ProjectName;
 import com.google.monitoring.v3.TimeSeries;
 import com.google.protobuf.Empty;
+import io.grpc.ManagedChannelBuilder;
 import io.opentelemetry.sdk.common.CompletableResultCode;
 import io.opentelemetry.sdk.metrics.InstrumentType;
 import io.opentelemetry.sdk.metrics.data.AggregationTemporality;
@@ -79,7 +82,7 @@ static SpannerCloudMonitoringExporter create(
       throws IOException {
     MetricServiceSettings.Builder settingsBuilder = MetricServiceSettings.newBuilder();
     CredentialsProvider credentialsProvider;
-    if (credentials == null) {
+    if (credentials == null || credentials instanceof NoCredentials) {
       credentialsProvider = NoCredentialsProvider.create();
     } else {
       credentialsProvider = FixedCredentialsProvider.create(credentials);
@@ -92,6 +95,19 @@ static SpannerCloudMonitoringExporter create(
       settingsBuilder.setUniverseDomain(universeDomain);
     }
 
+    if (System.getProperty("jmh.monitoring-server-port") != null) {
+      settingsBuilder.setTransportChannelProvider(
+          InstantiatingGrpcChannelProvider.newBuilder()
+              .setCredentials(NoCredentials.getInstance())
+              .setChannelConfigurator(
+                  managedChannelBuilder ->
+                      ManagedChannelBuilder.forAddress(
+                              "0.0.0.0",
+                              Integer.parseInt(System.getProperty("jmh.monitoring-server-port")))
+                          .usePlaintext())
+              .build());
+    }
+
     Duration timeout = Duration.ofMinutes(1);
     // TODO: createServiceTimeSeries needs special handling if the request failed. Leaving
     // it as not retried for now.

google-cloud-spanner/src/main/java/com/google/cloud/spanner/SpannerOptions.java

@@ -2013,6 +2013,11 @@ public CallCredentialsProvider getCallCredentialsProvider() {
   }
 
   private boolean usesNoCredentials() {
+    // When JMH is enabled, we need to enable built-in metrics
+    if (System.getProperty("jmh.enabled") != null
+        && System.getProperty("jmh.enabled").equals("true")) {
+      return false;
+    }
     return Objects.equals(getCredentials(), NoCredentials.getInstance());
   }
 

google-cloud-spanner/src/test/java/com/google/cloud/spanner/benchmarking/BenchmarkValidator.java

@@ -0,0 +1,156 @@
+/*
+ * Copyright 2025 Google LLC
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *      https://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package com.google.cloud.spanner.benchmarking;
+
+import com.google.cloud.spanner.benchmarking.BenchmarkValidator.BaselineResult.BenchmarkResult;
+import com.google.cloud.spanner.benchmarking.BenchmarkValidator.BaselineResult.BenchmarkResult.Percentile;
+import com.google.gson.Gson;
+import com.google.gson.reflect.TypeToken;
+import java.io.File;
+import java.io.IOException;
+import java.net.URL;
+import java.nio.file.Files;
+import java.nio.file.Paths;
+import java.util.ArrayList;
+import java.util.List;
+import java.util.Map;
+
+public class BenchmarkValidator {
+
+  private final BaselineResult expectedResults;
+  private final List<ActualBenchmarkResult> actualResults;
+
+  public BenchmarkValidator(String baselineFile, String actualFile) {
+    Gson gson = new Gson();
+    // Load expected result JSON from resource folder
+    this.expectedResults = gson.fromJson(loadJsonFromResources(baselineFile), BaselineResult.class);
+    // Load the actual result from current benchmarking run
+    this.actualResults =
+        gson.fromJson(
+            loadJsonFromFile(actualFile),
+            new TypeToken<ArrayList<ActualBenchmarkResult>>() {}.getType());
+  }
+
+  void validate() {
+    // Validating the resultant percentile against expected percentile with allowed threshold
+    for (ActualBenchmarkResult actualResult : actualResults) {
+      BenchmarkResult expectResult = expectedResults.benchmarkResultMap.get(actualResult.benchmark);
+      if (expectResult == null) {
+        throw new ValidationException(
+            "Missing expected benchmark configuration for actual benchmarking");
+      }
+      Map<String, Double> actualPercentilesMap = actualResult.primaryMetric.scorePercentiles;
+      // We will only be comparing the percentiles(p50, p90, p90) which are configured in the
+      // expected percentiles. This allows some checks to be disabled if required.
+      for (Percentile expectedPercentile : expectResult.scorePercentiles) {
+        String percentile = expectedPercentile.percentile;
+        double difference =
+            calculatePercentageDifference(
+                expectedPercentile.baseline, actualPercentilesMap.get(percentile));
+        // if an absolute different in percentage is greater than allowed difference
+        // Then we are throwing validation error
+        if (Math.abs(Math.ceil(difference)) > expectedPercentile.difference) {
+          throw new ValidationException(
+              String.format(
+                  "[%s][%s] Expected percentile %s[+/-%s] but got %s",
+                  actualResult.benchmark,
+                  percentile,
+                  expectedPercentile.baseline,
+                  expectedPercentile.difference,
+                  actualPercentilesMap.get(percentile)));
+        }
+      }
+    }
+  }
+
+  public static double calculatePercentageDifference(double base, double compareWith) {
+    if (base == 0) {
+      return 0.0;
+    }
+    return ((compareWith - base) / base) * 100;
+  }
+
+  private String loadJsonFromFile(String file) {
+    try {
+      return new String(Files.readAllBytes(Paths.get(file)));
+    } catch (IOException e) {
+      throw new ValidationException("Failed to read file: " + file, e);
+    }
+  }
+
+  private String loadJsonFromResources(String baselineFile) {
+    URL resourceUrl = getClass().getClassLoader().getResource(baselineFile);
+    if (resourceUrl == null) {
+      throw new ValidationException("File not found: " + baselineFile);
+    }
+    File file = new File(resourceUrl.getFile());
+    return loadJsonFromFile(file.getAbsolutePath());
+  }
+
+  static class ActualBenchmarkResult {
+    String benchmark;
+    PrimaryMetric primaryMetric;
+
+    static class PrimaryMetric {
+      Map<String, Double> scorePercentiles;
+    }
+  }
+
+  static class BaselineResult {
+    Map<String, BenchmarkResult> benchmarkResultMap;
+
+    static class BenchmarkResult {
+      List<Percentile> scorePercentiles;
+
+      static class Percentile {
+        String percentile;
+        Double baseline;
+        Double difference;
+      }
+    }
+  }
+
+  static class ValidationException extends RuntimeException {
+    ValidationException(String message) {
+      super(message);
+    }
+
+    ValidationException(String message, Throwable cause) {
+      super(message, cause);
+    }
+  }
+
+  private static String parseCommandLineArgs(String[] args, String key) {
+    if (args == null) {
+      return "";
+    }
+    for (String arg : args) {
+      if (arg.startsWith("--" + key)) {
+        String[] splits = arg.split("=");
+        if (splits.length == 2) {
+          return splits[1].trim();
+        }
+      }
+    }
+    return "";
+  }
+
+  public static void main(String[] args) {
+    String actualFile = parseCommandLineArgs(args, "file");
+    new BenchmarkValidator("com/google/cloud/spanner/jmh/jmh-baseline.json", actualFile).validate();
+  }
+}

google-cloud-spanner/src/test/java/com/google/cloud/spanner/benchmarking/MonitoringServiceImpl.java

@@ -0,0 +1,39 @@
+/*
+ * Copyright 2025 Google LLC
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *      https://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package com.google.cloud.spanner.benchmarking;
+
+import com.google.monitoring.v3.CreateTimeSeriesRequest;
+import com.google.monitoring.v3.MetricServiceGrpc.MetricServiceImplBase;
+import com.google.protobuf.Empty;
+import io.grpc.Status;
+import io.grpc.stub.StreamObserver;
+
+class MonitoringServiceImpl extends MetricServiceImplBase {
+
+  @Override
+  public void createServiceTimeSeries(
+      CreateTimeSeriesRequest request, StreamObserver<Empty> responseObserver) {
+    try {
+      Thread.sleep(100);
+      responseObserver.onNext(Empty.getDefaultInstance());
+      responseObserver.onCompleted();
+    } catch (InterruptedException e) {
+      responseObserver.onError(
+          Status.CANCELLED.withCause(e).withDescription(e.getMessage()).asException());
+    }
+  }
+}