Skip to content

Commit 9415591

Browse files
sakthivelmaniisakthivelmanii
committed
investigate grpc latency
1 parent e62f5ab commit 9415591

File tree

6 files changed

+323
-97
lines changed

6 files changed

+323
-97
lines changed

‎benchmarks/pom.xml‎

Lines changed: 41 additions & 9 deletions
Original file line numberDiff line numberDiff line change
@@ -34,7 +34,7 @@
3434
<project.build.sourceEncoding>UTF-8</project.build.sourceEncoding>
3535
<project.reporting.outputEncoding>UTF-8</project.reporting.outputEncoding>
3636
<junixsocket.version>2.10.1</junixsocket.version>
37-
<opentelemetry.version>1.45.0</opentelemetry.version>
37+
<opentelemetry.version>1.47.0</opentelemetry.version>
3838
</properties>
3939

4040
<dependencies>
@@ -49,12 +49,12 @@
4949
<dependency>
5050
<groupId>com.google.cloud.opentelemetry</groupId>
5151
<artifactId>exporter-trace</artifactId>
52-
<version>0.33.0</version>
52+
<version>0.32.0</version>
5353
</dependency>
5454
<dependency>
5555
<groupId>com.google.cloud.opentelemetry</groupId>
5656
<artifactId>exporter-metrics</artifactId>
57-
<version>0.33.0</version>
57+
<version>0.32.0</version>
5858
</dependency>
5959
<!-- OpenTelemetry test dependencies -->
6060
<dependency>
@@ -89,10 +89,15 @@
8989
<type>pom</type>
9090
<scope>import</scope>
9191
</dependency>
92+
<dependency>
93+
<groupId>com.google.cloud</groupId>
94+
<artifactId>google-cloud-monitoring</artifactId>
95+
<version>3.57.0</version>
96+
</dependency>
9297
<dependency>
9398
<groupId>com.google.cloud</groupId>
9499
<artifactId>google-cloud-spanner</artifactId>
95-
<version>6.84.0</version>
100+
<version>6.88.1-SNAPSHOT</version>
96101
</dependency>
97102
<dependency>
98103
<groupId>commons-cli</groupId>
@@ -115,11 +120,6 @@
115120
<artifactId>junixsocket-common</artifactId>
116121
<version>${junixsocket.version}</version>
117122
</dependency>
118-
<dependency>
119-
<groupId>commons-cli</groupId>
120-
<artifactId>commons-cli</artifactId>
121-
<version>1.9.0</version>
122-
</dependency>
123123

124124
<dependency>
125125
<groupId>junit</groupId>
@@ -130,6 +130,38 @@
130130
</dependencies>
131131
<build>
132132
<plugins>
133+
<plugin>
134+
<groupId>org.apache.maven.plugins</groupId>
135+
<artifactId>maven-checkstyle-plugin</artifactId>
136+
<configuration>
137+
<skip>true</skip>
138+
</configuration>
139+
</plugin>
140+
<plugin>
141+
<groupId>org.apache.maven.plugins</groupId>
142+
<artifactId>maven-shade-plugin</artifactId>
143+
<version>1.3.1</version>
144+
<executions>
145+
<execution>
146+
<phase>package</phase>
147+
<goals>
148+
<goal>shade</goal>
149+
</goals>
150+
<configuration>
151+
<filters>
152+
<filter>
153+
<artifact>*:*</artifact>
154+
<excludes>
155+
<exclude>META-INF/*.SF</exclude>
156+
<exclude>META-INF/*.DSA</exclude>
157+
<exclude>META-INF/*.RSA</exclude>
158+
</excludes>
159+
</filter>
160+
</filters>
161+
</configuration>
162+
</execution>
163+
</executions>
164+
</plugin>
133165
<plugin>
134166
<groupId>org.codehaus.mojo</groupId>
135167
<artifactId>exec-maven-plugin</artifactId>

‎benchmarks/src/main/java/com/google/cloud/spanner/benchmark/AbstractRunner.java‎

Lines changed: 42 additions & 20 deletions
Original file line numberDiff line numberDiff line change
@@ -16,43 +16,64 @@
1616

1717
package com.google.cloud.spanner.benchmark;
1818

19-
import java.nio.charset.StandardCharsets;
2019
import java.time.Duration;
20+
import java.time.Instant;
21+
import java.time.temporal.ChronoUnit;
2122
import java.util.ArrayList;
2223
import java.util.List;
2324
import java.util.concurrent.ExecutorService;
2425
import java.util.concurrent.Future;
2526
import java.util.concurrent.ThreadLocalRandom;
2627
import java.util.concurrent.TimeUnit;
2728
import java.util.concurrent.TimeoutException;
28-
import java.util.concurrent.atomic.AtomicInteger;
29+
import java.util.concurrent.atomic.AtomicBoolean;
30+
import java.util.concurrent.atomic.AtomicReference;
31+
import java.util.stream.Collectors;
2932

3033
abstract class AbstractRunner implements BenchmarkRunner {
31-
static final int TOTAL_RECORDS = 1000000;
32-
static final String SELECT_QUERY = "SELECT ID FROM FOO WHERE ID = @id";
33-
static final String UPDATE_QUERY = "UPDATE FOO SET BAR=1 WHERE ID = @id";
34+
static final int TOTAL_RECORDS = 100000;
35+
static final int READ_RANGE = 99;
36+
static final String TABLE_NAME = "Employees";
37+
static final String SELECT_QUERY = "SELECT ID FROM Employees WHERE ID=@id";
38+
static final String UPDATE_QUERY = "UPDATE Employees SET NAME=SAKTHI WHERE ID = @id";
3439
static final String ID_COLUMN_NAME = "id";
35-
static final String SERVER_URL = "https://staging-wrenchworks.sandbox.googleapis.com";
40+
static final String SERVER_URL = "https://spanner.googleapis.com";
3641

37-
private final AtomicInteger operationCounter = new AtomicInteger();
42+
private final AtomicReference<Instant> operationEndTime = new AtomicReference<>();
3843

39-
protected void incOperations() {
40-
operationCounter.incrementAndGet();
44+
private final AtomicBoolean operationStarted = new AtomicBoolean();
45+
46+
private final AtomicReference<Instant> warmUpEndTime = new AtomicReference<>();
47+
48+
protected void setOperationEndTime(Instant instant) {
49+
operationEndTime.set(instant);
50+
}
51+
52+
protected void operationStarted(boolean started) {
53+
operationStarted.set(started);
54+
}
55+
56+
protected void setWarmUpEndTime(Instant instant) {
57+
warmUpEndTime.set(instant);
4158
}
4259

43-
protected List<Duration> collectResults(
60+
protected List<Integer> collectResults(
4461
ExecutorService service,
4562
List<Future<List<Duration>>> results,
4663
int numClients,
4764
int numOperations)
4865
throws Exception {
49-
int totalOperations = numClients * numOperations;
5066
service.shutdown();
5167
while (!service.isTerminated()) {
52-
//noinspection BusyWait
5368
Thread.sleep(1000L);
54-
System.out.printf("\r%d/%d", operationCounter.get(), totalOperations);
69+
if (operationStarted.get()) {
70+
//noinspection BusyWait
71+
System.out.printf("\rOperation Ends in %s", showTime(operationEndTime.get()));
72+
} else if (warmUpEndTime.get() != null) {
73+
System.out.printf("\rWarm up ends in %s", showTime(warmUpEndTime.get()));
74+
}
5575
}
76+
System.out.print("\r\r");
5677
System.out.println();
5778
if (!service.awaitTermination(60L, TimeUnit.MINUTES)) {
5879
throw new TimeoutException();
@@ -61,7 +82,14 @@ protected List<Duration> collectResults(
6182
for (Future<List<Duration>> result : results) {
6283
allResults.addAll(result.get());
6384
}
64-
return allResults;
85+
return allResults.stream().map(Duration::getNano).collect(Collectors.toList());
86+
}
87+
88+
private String showTime(Instant endTime) {
89+
long totalSeconds = ChronoUnit.SECONDS.between(Instant.now(), endTime);
90+
long totalMinutes = ChronoUnit.MINUTES.between(Instant.now(), endTime);
91+
92+
return String.format("%s-%s", totalMinutes, totalSeconds - (totalMinutes * 60));
6593
}
6694

6795
protected void randomWait(int waitMillis) throws InterruptedException {
@@ -71,10 +99,4 @@ protected void randomWait(int waitMillis) throws InterruptedException {
7199
int randomMillis = ThreadLocalRandom.current().nextInt(waitMillis * 2);
72100
Thread.sleep(randomMillis);
73101
}
74-
75-
protected String generateRandomString() {
76-
byte[] bytes = new byte[64];
77-
ThreadLocalRandom.current().nextBytes(bytes);
78-
return new String(bytes, StandardCharsets.UTF_8);
79-
}
80102
}

‎benchmarks/src/main/java/com/google/cloud/spanner/benchmark/BenchmarkRunner.java‎

Lines changed: 6 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -16,20 +16,20 @@
1616

1717
package com.google.cloud.spanner.benchmark;
1818

19-
import java.time.Duration;
20-
import java.util.List;
21-
2219
public interface BenchmarkRunner {
2320
enum TransactionType {
2421
READ_ONLY_SINGLE_USE,
2522
READ_ONLY_MULTI_USE,
26-
READ_WRITE
23+
READ_WRITE,
24+
READ_ONLY_STALE_READ
2725
}
2826

29-
List<Duration> execute(
27+
void execute(
3028
TransactionType transactionType,
3129
int numClients,
3230
int numOperations,
3331
int waitMillis,
34-
boolean useMultiplexedSession);
32+
boolean useMultiplexedSession,
33+
int warmUpMinutes,
34+
int staleReadMinutes);
3535
}

‎benchmarks/src/main/java/com/google/cloud/spanner/benchmark/DataGenerator.java‎

Lines changed: 91 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,91 @@
1+
package com.google.cloud.spanner.benchmark;
2+
3+
import com.google.cloud.spanner.DatabaseAdminClient;
4+
import com.google.cloud.spanner.DatabaseClient;
5+
import com.google.cloud.spanner.DatabaseId;
6+
import com.google.cloud.spanner.Mutation;
7+
import com.google.cloud.spanner.SessionPoolOptions;
8+
import com.google.cloud.spanner.SessionPoolOptionsHelper;
9+
import com.google.cloud.spanner.Spanner;
10+
import com.google.cloud.spanner.SpannerOptions;
11+
import java.util.ArrayList;
12+
import java.util.Collections;
13+
import java.util.List;
14+
import java.util.concurrent.ExecutionException;
15+
import java.util.concurrent.TimeUnit;
16+
import java.util.concurrent.TimeoutException;
17+
18+
public class DataGenerator {
19+
20+
private static final String CREATE_TABLE_QUERY =
21+
"CREATE TABLE Employees (ID INT64 NOT NULL, NAME STRING(30)) PRIMARY KEY(ID)";
22+
23+
public static void main(String[] args)
24+
throws ExecutionException, InterruptedException, TimeoutException {
25+
String project = System.getenv("SPANNER_CLIENT_BENCHMARK_GOOGLE_CLOUD_PROJECT");
26+
String instance = System.getenv("SPANNER_CLIENT_BENCHMARK_SPANNER_INSTANCE");
27+
String database = System.getenv("SPANNER_CLIENT_BENCHMARK_SPANNER_DATABASE");
28+
29+
String fullyQualifiedDatabase = null;
30+
if (project != null && instance != null && database != null) {
31+
fullyQualifiedDatabase =
32+
String.format("projects/%s/instances/%s/databases/%s", project, instance, database);
33+
} else {
34+
throw new IllegalArgumentException(
35+
"You must either set all the environment variables SPANNER_CLIENT_BENCHMARK_GOOGLE_CLOUD_PROJECT, SPANNER_CLIENT_BENCHMARK_SPANNER_INSTANCE and SPANNER_CLIENT_BENCHMARK_SPANNER_DATABASE, or specify a value for the command line argument --database");
36+
}
37+
38+
DatabaseId databaseId = DatabaseId.of(fullyQualifiedDatabase);
39+
try (Spanner spanner = getSpannerOptions(databaseId).getService()) {
40+
// Create database and tables
41+
DatabaseAdminClient adminClient = spanner.getDatabaseAdminClient();
42+
createDatabaseAndTables(adminClient, databaseId);
43+
System.out.println("Database and Tables are created");
44+
45+
// Insert Rows into the table
46+
System.out.println("Inserting rows");
47+
DatabaseClient databaseClient = spanner.getDatabaseClient(databaseId);
48+
System.out.println(databaseId);
49+
for (int i = 0; i < 100; i++) {
50+
List<Mutation> mutationList = new ArrayList<>();
51+
for (int j = 0; j < 1000; j++) {
52+
int row = i * 1000 + j;
53+
Mutation mutation =
54+
Mutation.newInsertBuilder("Employees")
55+
.set("ID")
56+
.to(row)
57+
.set("NAME")
58+
.to("RANDOM")
59+
.build();
60+
mutationList.add(mutation);
61+
}
62+
databaseClient.write(mutationList);
63+
}
64+
System.out.println("Inserting rows completed");
65+
}
66+
}
67+
68+
private static void createDatabaseAndTables(
69+
DatabaseAdminClient adminClient, DatabaseId databaseId)
70+
throws ExecutionException, InterruptedException, TimeoutException {
71+
adminClient
72+
.createDatabase(
73+
adminClient.newDatabaseBuilder(databaseId).build(),
74+
Collections.singleton(CREATE_TABLE_QUERY))
75+
.get(60, TimeUnit.SECONDS);
76+
}
77+
78+
private static SpannerOptions getSpannerOptions(DatabaseId databaseId) {
79+
SessionPoolOptions sessionPoolOptions =
80+
SessionPoolOptionsHelper.setUseMultiplexedSession(SessionPoolOptions.newBuilder(), true)
81+
.build();
82+
83+
System.out.println(databaseId.toString());
84+
85+
return SpannerOptions.newBuilder()
86+
.setProjectId(databaseId.getInstanceId().getProject())
87+
.setSessionPoolOption(sessionPoolOptions)
88+
.setHost("https://spanner.googleapis.com")
89+
.build();
90+
}
91+
}

0 commit comments

Comments
 (0)