chore: update benchmarking script (#3882)

Author: sakthivelmanii

benchmarks/pom.xml

@@ -34,7 +34,7 @@
     <project.build.sourceEncoding>UTF-8</project.build.sourceEncoding>
     <project.reporting.outputEncoding>UTF-8</project.reporting.outputEncoding>
     <junixsocket.version>2.10.1</junixsocket.version>
-    <opentelemetry.version>1.50.0</opentelemetry.version>
+    <opentelemetry.version>1.47.0</opentelemetry.version>
   </properties>
 
   <dependencies>
@@ -54,7 +54,12 @@
     <dependency>
       <groupId>com.google.cloud.opentelemetry</groupId>
       <artifactId>exporter-metrics</artifactId>
-      <version>0.34.0</version>
+      <version>0.33.0</version>
+    </dependency>
+    <dependency>
+      <groupId>com.google.cloud</groupId>
+      <artifactId>google-cloud-monitoring</artifactId>
+      <version>3.63.0</version>
     </dependency>
     <!-- OpenTelemetry test dependencies -->
     <dependency>
@@ -85,7 +90,7 @@
     <dependency>
       <groupId>io.opentelemetry</groupId>
       <artifactId>opentelemetry-bom</artifactId>
-      <version>1.50.0</version>
+      <version>1.47.0</version>
       <type>pom</type>
       <scope>import</scope>
     </dependency>
@@ -94,11 +99,6 @@
       <artifactId>google-cloud-spanner</artifactId>
       <version>6.93.0</version>
     </dependency>
-    <dependency>
-      <groupId>commons-cli</groupId>
-      <artifactId>commons-cli</artifactId>
-      <version>1.9.0</version>
-    </dependency>
     <dependency>
       <groupId>com.google.auto.value</groupId>
       <artifactId>auto-value-annotations</artifactId>
@@ -140,8 +140,9 @@
         </configuration>
       </plugin>
       <plugin>
-        <groupId>com.coveo</groupId>
+        <groupId>com.spotify.fmt</groupId>
         <artifactId>fmt-maven-plugin</artifactId>
+        <version>2.27</version>
         <executions>
           <execution>
             <goals>

benchmarks/src/main/java/com/google/cloud/spanner/benchmark/AbstractRunner.java

@@ -18,46 +18,80 @@
 
 import java.nio.charset.StandardCharsets;
 import java.time.Duration;
+import java.time.Instant;
+import java.time.temporal.ChronoUnit;
 import java.util.ArrayList;
+import java.util.HashMap;
+import java.util.HashSet;
 import java.util.List;
+import java.util.Map;
+import java.util.Set;
 import java.util.concurrent.ExecutorService;
 import java.util.concurrent.Future;
 import java.util.concurrent.ThreadLocalRandom;
 import java.util.concurrent.TimeUnit;
 import java.util.concurrent.TimeoutException;
-import java.util.concurrent.atomic.AtomicInteger;
 
 abstract class AbstractRunner implements BenchmarkRunner {
-  static final int TOTAL_RECORDS = 1000000;
-  static final String SELECT_QUERY = "SELECT ID FROM FOO WHERE ID = @id";
-  static final String UPDATE_QUERY = "UPDATE FOO SET BAR=1 WHERE ID = @id";
+  static final int TOTAL_RECORDS = 100000;
+  static final String TABLE_NAME = "Employees";
+  static final String SELECT_QUERY = String.format("SELECT ID FROM %s WHERE ID = @id", TABLE_NAME);
+  static final String UPDATE_QUERY =
+      String.format("UPDATE %s SET Name=Google WHERE ID = @id", TABLE_NAME);
   static final String ID_COLUMN_NAME = "id";
-  static final String SERVER_URL = "https://staging-wrenchworks.sandbox.googleapis.com";
+  static final Map<Environment, String> SERVER_URL_MAPPING = new HashMap<>();
 
-  private final AtomicInteger operationCounter = new AtomicInteger();
+  static {
+    SERVER_URL_MAPPING.put(
+        Environment.CLOUD_DEVEL, "https://staging-wrenchworks.sandbox.googleapis.com");
+    SERVER_URL_MAPPING.put(Environment.PROD, "https://spanner.googleapis.com");
+  }
+
+  Map<Integer, TimerConfiguration> timerConfigurations = new HashMap<>();
+  private final Set<Integer> completedClients = new HashSet<>();
+  private final Set<Integer> finishedClients = new HashSet<>();
+
+  protected void initiateTimer(int clientId, String message, Instant endTime) {
+    TimerConfiguration timerConfiguration =
+        timerConfigurations.getOrDefault(clientId, new TimerConfiguration());
+    timerConfiguration.setMessage(message);
+    timerConfiguration.setEndTime(endTime);
+    timerConfigurations.put(clientId, timerConfiguration);
+  }
 
-  protected void incOperations() {
-    operationCounter.incrementAndGet();
+  protected void setBenchmarkingCompleted(int clientId) {
+    this.completedClients.add(clientId);
   }
 
   protected List<Duration> collectResults(
       ExecutorService service,
       List<Future<List<Duration>>> results,
-      int numClients,
-      int numOperations)
+      BenchmarkingConfiguration configuration)
       throws Exception {
-    int totalOperations = numClients * numOperations;
+    while (!(finishedClients.size() == configuration.getNumOfClients()))
+      for (int i = 0; i < configuration.getNumOfClients(); i++) {
+        TimerConfiguration timerConfiguration =
+            timerConfigurations.getOrDefault(i, new TimerConfiguration());
+        long totalSeconds =
+            ChronoUnit.SECONDS.between(Instant.now(), timerConfiguration.getEndTime());
+        if (completedClients.contains(i)) {
+          if (!finishedClients.contains(i)) {
+            System.out.printf("Client %s: Completed", i);
+            finishedClients.add(i);
+          }
+        } else {
+          System.out.printf(
+              "Client %s: %s %s Minutes %s Seconds\r",
+              i + 1, timerConfiguration.getMessage(), totalSeconds / 60, totalSeconds % 60);
+        }
+        //noinspection BusyWait
+        Thread.sleep(1000L);
+      }
     service.shutdown();
-    while (!service.isTerminated()) {
-      //noinspection BusyWait
-      Thread.sleep(1000L);
-      System.out.printf("\r%d/%d", operationCounter.get(), totalOperations);
-    }
-    System.out.println();
     if (!service.awaitTermination(60L, TimeUnit.MINUTES)) {
       throw new TimeoutException();
     }
-    List<Duration> allResults = new ArrayList<>(numClients * numOperations);
+    List<Duration> allResults = new ArrayList<>();
     for (Future<List<Duration>> result : results) {
       allResults.addAll(result.get());
     }
@@ -77,4 +111,25 @@ protected String generateRandomString() {
     ThreadLocalRandom.current().nextBytes(bytes);
     return new String(bytes, StandardCharsets.UTF_8);
   }
+
+  static class TimerConfiguration {
+    private Instant endTime = Instant.now();
+    private String message = "Waiting for benchmarks to start...";
+
+    Instant getEndTime() {
+      return endTime;
+    }
+
+    void setEndTime(Instant endTime) {
+      this.endTime = endTime;
+    }
+
+    String getMessage() {
+      return message;
+    }
+
+    void setMessage(String message) {
+      this.message = message;
+    }
+  }
 }

benchmarks/src/main/java/com/google/cloud/spanner/benchmark/BenchmarkRunner.java

@@ -19,17 +19,18 @@
 import java.time.Duration;
 import java.util.List;
 
-public interface BenchmarkRunner {
+interface BenchmarkRunner {
   enum TransactionType {
-    READ_ONLY_SINGLE_USE,
+    READ_ONLY_SINGLE_USE_READ,
+    READ_ONLY_SINGLE_USE_QUERY,
     READ_ONLY_MULTI_USE,
     READ_WRITE
   }
 
-  List<Duration> execute(
-      TransactionType transactionType,
-      int numClients,
-      int numOperations,
-      int waitMillis,
-      boolean useMultiplexedSession);
+  enum Environment {
+    PROD,
+    CLOUD_DEVEL
+  }
+
+  List<Duration> execute(BenchmarkingConfiguration configuration);
 }

benchmarks/src/main/java/com/google/cloud/spanner/benchmark/BenchmarkingConfiguration.java

@@ -0,0 +1,115 @@
+/*
+ * Copyright 2025 Google LLC
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *      http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package com.google.cloud.spanner.benchmark;
+
+import com.google.cloud.spanner.DatabaseId;
+import com.google.cloud.spanner.benchmark.BenchmarkRunner.Environment;
+import com.google.cloud.spanner.benchmark.BenchmarkRunner.TransactionType;
+
+class BenchmarkingConfiguration {
+
+  private DatabaseId databaseId;
+  private int numOfClients;
+  private int staleness;
+  private int warmupTime;
+  private int executionTime;
+  private int waitBetweenRequests;
+  private boolean useMultiplexSession;
+  private TransactionType transactionType;
+  private Environment environment;
+
+  int getExecutionTime() {
+    return executionTime;
+  }
+
+  BenchmarkingConfiguration setExecutionTime(int executionTime) {
+    this.executionTime = executionTime;
+    return this;
+  }
+
+  DatabaseId getDatabaseId() {
+    return databaseId;
+  }
+
+  BenchmarkingConfiguration setDatabaseId(DatabaseId databaseId) {
+    this.databaseId = databaseId;
+    return this;
+  }
+
+  int getNumOfClients() {
+    return numOfClients;
+  }
+
+  BenchmarkingConfiguration setNumOfClients(int numOfClients) {
+    this.numOfClients = numOfClients;
+    return this;
+  }
+
+  int getStaleness() {
+    return staleness;
+  }
+
+  BenchmarkingConfiguration setStaleness(int staleness) {
+    this.staleness = staleness;
+    return this;
+  }
+
+  int getWarmupTime() {
+    return warmupTime;
+  }
+
+  BenchmarkingConfiguration setWarmupTime(int warmupTime) {
+    this.warmupTime = warmupTime;
+    return this;
+  }
+
+  int getWaitBetweenRequests() {
+    return waitBetweenRequests;
+  }
+
+  BenchmarkingConfiguration setWaitBetweenRequests(int waitBetweenRequests) {
+    this.waitBetweenRequests = waitBetweenRequests;
+    return this;
+  }
+
+  boolean isUseMultiplexSession() {
+    return useMultiplexSession;
+  }
+
+  BenchmarkingConfiguration setUseMultiplexSession(boolean useMultiplexSession) {
+    this.useMultiplexSession = useMultiplexSession;
+    return this;
+  }
+
+  TransactionType getTransactionType() {
+    return transactionType;
+  }
+
+  BenchmarkingConfiguration setTransactionType(TransactionType transactionType) {
+    this.transactionType = transactionType;
+    return this;
+  }
+
+  Environment getEnvironment() {
+    return environment;
+  }
+
+  BenchmarkingConfiguration setEnvironment(Environment environment) {
+    this.environment = environment;
+    return this;
+  }
+}