Skip to content

Commit 4677774

Browse files
mzitnikmzitnik
authored
JDBC perf test with JMH (#2192)
* Disable since it does not compile. (Depend test where changed) * First JDBC V1/V2 Benchmark using JMH perfomce testing * Adding JSON output * Adding a connection only dimemntion * Adding a README.md for JMH
1 parent 65317f4 commit 4677774

File tree

7 files changed

+446
-8
lines changed

7 files changed

+446
-8
lines changed

‎clickhouse-jdbc/pom.xml‎

Lines changed: 114 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -137,6 +137,19 @@
137137
<version>${revision}</version>
138138
<scope>compile</scope>
139139
</dependency>
140+
<dependency>
141+
<groupId>org.openjdk.jmh</groupId>
142+
<artifactId>jmh-core</artifactId>
143+
<version>${jmh.version}</version>
144+
<scope>test</scope>
145+
</dependency>
146+
<dependency>
147+
<groupId>org.openjdk.jmh</groupId>
148+
<artifactId>jmh-generator-annprocess</artifactId>
149+
<version>${jmh.version}</version>
150+
<scope>test</scope>
151+
</dependency>
152+
140153
</dependencies>
141154

142155
<profiles>
@@ -222,8 +235,108 @@
222235
</plugins>
223236
</build>
224237
</profile>
238+
<profile>
239+
<id>performance-testing-jdbc</id>
240+
<activation><activeByDefault>false</activeByDefault></activation>
241+
<dependencies>
242+
<dependency>
243+
<groupId>org.openjdk.jmh</groupId>
244+
<artifactId>jmh-core</artifactId>
245+
<version>${jmh.version}</version>
246+
</dependency>
247+
<dependency>
248+
<groupId>org.openjdk.jmh</groupId>
249+
<artifactId>jmh-generator-annprocess</artifactId>
250+
<version>${jmh.version}</version>
251+
</dependency>
252+
</dependencies>
253+
<build>
254+
<plugins>
255+
<plugin>
256+
<groupId>org.apache.maven.plugins</groupId>
257+
<artifactId>maven-compiler-plugin</artifactId>
258+
<configuration>
259+
<annotationProcessorPaths>
260+
<path>
261+
<groupId>org.projectlombok</groupId>
262+
<artifactId>lombok</artifactId>
263+
<version>1.18.32</version>
264+
</path>
265+
<path>
266+
<groupId>org.openjdk.jmh</groupId>
267+
<artifactId>jmh-generator-annprocess</artifactId>
268+
<version>${jmh.version}</version>
269+
</path>
270+
</annotationProcessorPaths>
271+
<release>8</release>
272+
</configuration>
273+
</plugin>
274+
<plugin>
275+
<groupId>org.apache.maven.plugins</groupId>
276+
<artifactId>maven-shade-plugin</artifactId>
277+
<executions>
278+
<execution>
279+
<phase>package</phase>
280+
<goals>
281+
<goal>shade</goal>
282+
</goals>
283+
<configuration>
284+
<finalName>benchmarks</finalName>
285+
<transformers>
286+
<transformer implementation="org.apache.maven.plugins.shade.resource.ManifestResourceTransformer">
287+
<mainClass>org.openjdk.jmh.Main</mainClass>
288+
</transformer>
289+
</transformers>
290+
</configuration>
291+
</execution>
292+
</executions>
293+
</plugin>
294+
<plugin>
295+
<groupId>org.codehaus.mojo</groupId>
296+
<artifactId>build-helper-maven-plugin</artifactId>
297+
<version>3.6.0</version>
298+
<executions>
299+
<execution>
300+
<id>add-test-source</id>
301+
<phase>generate-test-sources</phase>
302+
<goals>
303+
<goal>add-test-source</goal>
304+
</goals>
305+
<configuration>
306+
<sources>
307+
<source>src/test/perf</source>
308+
</sources>
309+
</configuration>
310+
</execution>
311+
</executions>
312+
</plugin>
313+
<plugin>
314+
<groupId>org.codehaus.mojo</groupId>
315+
<artifactId>exec-maven-plugin</artifactId>
316+
<executions>
317+
<execution>
318+
<id>run-benchmarks</id>
319+
<phase>integration-test</phase>
320+
<goals>
321+
<goal>exec</goal>
322+
</goals>
323+
<configuration>
324+
<classpathScope>test</classpathScope>
325+
<executable>java</executable>
326+
<arguments>
327+
<argument>-classpath</argument>
328+
<classpath />
329+
<argument>org.openjdk.jmh.Main</argument>
330+
<argument>.*</argument>
331+
</arguments>
332+
</configuration>
333+
</execution>
334+
</executions>
335+
</plugin>
336+
</plugins>
337+
</build>
338+
</profile>
225339
</profiles>
226-
227340
<build>
228341
<plugins>
229342
<plugin>

‎clickhouse-jdbc/src/test/perf/README.md‎

Lines changed: 20 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,20 @@
1+
# Performance Testing using JMH for JDBC driver
2+
Testing of Insert, Select & comparing the performance of JDBC V1 & JDBC V2
3+
Insert testing
4+
- batch sizes [100, 1000, 10000, 100000]
5+
- data types [int8, int16, int32, int64, float32, float64, string, boolean] (Will be extended to other data types as well)
6+
7+
Note: string payload size is adjustable ["1024", "2048", "4096", "8192" ]
8+
9+
Select testing query `"SELECT * FROM %s LIMIT %d"`
10+
- limit is adjustable currently use 100
11+
- retrieval of different data types [int8, int16, int32, int64, float32, float64, string, boolean] (Will be extended to other data types as well)
12+
- separated test for connection part and connection with deserialization part
13+
14+
# how to run
15+
mvn clean package -DskipTests=true -P performance-testing-jdbc
16+
17+
# output file
18+
The benchmark generate the output file in jmh-jdbc-results.json
19+
Use https://jmh.morethan.io/ to visualize the results
20+

‎clickhouse-jdbc/src/test/perf/com/clickhouse/jdbc/perf/JDBCBenchmarkBase.java‎

Lines changed: 44 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,44 @@
1+
package com.clickhouse.jdbc.perf;
2+
3+
import com.clickhouse.client.BaseIntegrationTest;
4+
import com.clickhouse.client.ClickHouseNode;
5+
import com.clickhouse.client.ClickHouseProtocol;
6+
import com.clickhouse.client.ClickHouseServerForTest;
7+
import com.clickhouse.jdbc.ClickHouseDriver;
8+
import org.slf4j.Logger;
9+
import org.slf4j.LoggerFactory;
10+
11+
import java.sql.Connection;
12+
import java.sql.SQLException;
13+
import java.util.Properties;
14+
15+
import static com.clickhouse.client.ClickHouseServerForTest.isCloud;
16+
17+
public class JDBCBenchmarkBase {
18+
private static final Logger LOGGER = LoggerFactory.getLogger(JDBCBenchmarkBase.class);
19+
protected static Connection jdbcV1 = null;
20+
protected static Connection jdbcV2 = null;
21+
22+
public void setup() throws SQLException {
23+
BaseIntegrationTest.setupClickHouseContainer();
24+
// TODO: create two connections
25+
Properties properties = new Properties();
26+
properties.put("user", "default");
27+
properties.put("password", "test_default_password");
28+
29+
ClickHouseNode node = ClickHouseServerForTest.getClickHouseNode(ClickHouseProtocol.HTTP, isCloud(), ClickHouseNode.builder().build());
30+
31+
LOGGER.info(String.format("clickhouse endpoint [%s:%s]", node.getHost(), node.getPort()));
32+
jdbcV1 = new ClickHouseDriver().connect(String.format("jdbc:clickhouse://%s:%s?clickhouse.jdbc.v1=true", node.getHost(), node.getPort()), properties);
33+
jdbcV2 = new ClickHouseDriver().connect(String.format("jdbc:clickhouse://%s:%s", node.getHost(), node.getPort()), properties);
34+
35+
}
36+
37+
public void tearDown() throws SQLException {
38+
LOGGER.info("Tearing down JDBC connections and ClickHouse container.");
39+
if (jdbcV1 != null) jdbcV1.close();
40+
if (jdbcV2 != null) jdbcV2.close();
41+
BaseIntegrationTest.teardownClickHouseContainer();
42+
}
43+
44+
}

‎clickhouse-jdbc/src/test/perf/com/clickhouse/jdbc/perf/JDBCBenchmarkRunner.java‎

Lines changed: 39 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,39 @@
1+
package com.clickhouse.jdbc.perf;
2+
3+
import org.openjdk.jmh.annotations.Mode;
4+
import org.openjdk.jmh.profile.GCProfiler;
5+
import org.openjdk.jmh.profile.MemPoolProfiler;
6+
import org.openjdk.jmh.results.format.ResultFormatType;
7+
import org.openjdk.jmh.runner.Runner;
8+
import org.openjdk.jmh.runner.options.Options;
9+
import org.openjdk.jmh.runner.options.OptionsBuilder;
10+
import org.openjdk.jmh.runner.options.TimeValue;
11+
import org.slf4j.Logger;
12+
import org.slf4j.LoggerFactory;
13+
14+
import java.util.concurrent.TimeUnit;
15+
16+
public class JDBCBenchmarkRunner {
17+
private static final Logger LOGGER = LoggerFactory.getLogger(JDBCBenchmarkRunner.class);
18+
19+
public static void main(String[] args) throws Exception {
20+
LOGGER.info("Starting Benchmarks");
21+
Options opt = new OptionsBuilder()
22+
.include(JDBCSelectBenchmark.class.getSimpleName())
23+
.include(JDBCInsertBenchmark.class.getSimpleName())
24+
.forks(1)
25+
.mode(Mode.AverageTime)
26+
.timeUnit(TimeUnit.MILLISECONDS)
27+
.threads(1)
28+
.addProfiler(GCProfiler.class)
29+
.addProfiler(MemPoolProfiler.class)
30+
.warmupIterations(1)
31+
.warmupTime(TimeValue.seconds(5))
32+
.measurementIterations(1)
33+
.measurementTime(TimeValue.seconds(5))
34+
.resultFormat(ResultFormatType.JSON)
35+
.result("jmh-jdbc-results.json")
36+
.build();
37+
new Runner(opt).run();
38+
}
39+
}

‎clickhouse-jdbc/src/test/perf/com/clickhouse/jdbc/perf/JDBCInsertBenchmark.java‎

Lines changed: 112 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,112 @@
1+
package com.clickhouse.jdbc.perf;
2+
3+
import org.apache.commons.lang3.RandomStringUtils;
4+
import org.openjdk.jmh.annotations.Benchmark;
5+
import org.openjdk.jmh.annotations.BenchmarkMode;
6+
import org.openjdk.jmh.annotations.Fork;
7+
import org.openjdk.jmh.annotations.Level;
8+
import org.openjdk.jmh.annotations.Measurement;
9+
import org.openjdk.jmh.annotations.Mode;
10+
import org.openjdk.jmh.annotations.OutputTimeUnit;
11+
import org.openjdk.jmh.annotations.Param;
12+
import org.openjdk.jmh.annotations.Scope;
13+
import org.openjdk.jmh.annotations.Setup;
14+
import org.openjdk.jmh.annotations.State;
15+
import org.openjdk.jmh.annotations.TearDown;
16+
import org.openjdk.jmh.annotations.Threads;
17+
import org.openjdk.jmh.annotations.Warmup;
18+
import org.slf4j.Logger;
19+
import org.slf4j.LoggerFactory;
20+
21+
import java.sql.Connection;
22+
import java.sql.PreparedStatement;
23+
import java.sql.SQLException;
24+
import java.util.HashMap;
25+
import java.util.Map;
26+
import java.util.concurrent.TimeUnit;
27+
@State(Scope.Benchmark)
28+
public class JDBCInsertBenchmark extends JDBCBenchmarkBase {
29+
private static final Logger LOGGER = LoggerFactory.getLogger(JDBCInsertBenchmark.class);
30+
31+
@Setup
32+
public void setup() throws SQLException {
33+
super.setup();
34+
}
35+
36+
@TearDown
37+
public void tearDown() throws SQLException {
38+
super.tearDown();
39+
}
40+
41+
@State(Scope.Thread)
42+
public static class InsertState {
43+
String payload1024 = RandomStringUtils.random(1024, true, true);;
44+
String payload2048 = RandomStringUtils.random(1024, true, true);;
45+
String payload4096 = RandomStringUtils.random(4096, true, true);;
46+
String payload8192 = RandomStringUtils.random(8192, true, true);;
47+
48+
Map payloads = new HashMap<String, String>() {{
49+
put("1024", payload1024);
50+
put("2048", payload2048);
51+
put("4096", payload4096);
52+
put("8192", payload8192);
53+
}};
54+
// @Param({"100", "1000", "10000", "100000"})
55+
@Param({"100000"})
56+
int batchSize;
57+
// @Param({"1024", "2048", "4096", "8192"})
58+
@Param({"8192"})
59+
String payloadSize;
60+
61+
public String getPayload(String payloadSize) {
62+
return (String) payloads.get(payloadSize);
63+
}
64+
@Setup(Level.Invocation)
65+
public void setup() throws SQLException {
66+
LOGGER.info("Setting up insert state");
67+
jdbcV1.createStatement().execute("DROP TABLE IF EXISTS insert_test_jdbcV1");
68+
jdbcV2.createStatement().execute("DROP TABLE IF EXISTS insert_test_jdbcV2");
69+
jdbcV1.createStatement().execute("CREATE TABLE insert_test_jdbcV1 ( `off16` Int16, `str` String, `p_int8` Int8, `p_int16` Int16, `p_int32` Int32, `p_int64` Int64, `p_float32` Float32, `p_float64` Float64, `p_bool` Bool) Engine = Memory");
70+
jdbcV2.createStatement().execute("CREATE TABLE insert_test_jdbcV2 ( `off16` Int16, `str` String, `p_int8` Int8, `p_int16` Int16, `p_int32` Int32, `p_int64` Int64, `p_float32` Float32, `p_float64` Float64, `p_bool` Bool) Engine = Memory");
71+
}
72+
73+
@TearDown(Level.Invocation)
74+
public void tearDown() {
75+
LOGGER.info("Tearing down insert state");
76+
}
77+
}
78+
79+
void insertData(Connection conn, String table, int size, String payload) throws SQLException {
80+
PreparedStatement ps = conn.prepareStatement(String.format("INSERT INTO %s (off16, str, p_int8, p_int16, p_int32, p_int64, p_float32, p_float64, p_bool) VALUES (?,?,?,?,?,?,?,?,?)", table));
81+
for (int i = 0; i < size; i++) {
82+
ps.setShort(1, (short) i);
83+
ps.setString(2, payload);
84+
ps.setByte(3, (byte)i);
85+
ps.setShort(4, (short)i);
86+
ps.setInt(5, i);
87+
ps.setLong(6, (long)i);
88+
ps.setFloat(7, (float)(i*0.1));
89+
ps.setDouble(8, (double)(i*0.1));
90+
ps.setBoolean(9, true);
91+
ps.addBatch();
92+
}
93+
ps.executeBatch();
94+
}
95+
96+
@Benchmark
97+
@Warmup(iterations = 1, time = 5, timeUnit = TimeUnit.SECONDS)
98+
@Measurement(iterations = 5, time = 5, timeUnit = TimeUnit.SECONDS)
99+
public void insertDataV1(InsertState state) throws Exception {
100+
LOGGER.info("(V1) JDBC Insert data benchmark");
101+
insertData(jdbcV1, "insert_test_jdbcV1", state.batchSize, state.getPayload(state.payloadSize));
102+
}
103+
104+
@Benchmark
105+
@Warmup(iterations = 1, time = 5, timeUnit = TimeUnit.SECONDS)
106+
@Measurement(iterations = 5, time = 5, timeUnit = TimeUnit.SECONDS)
107+
public void insertDataV2(InsertState state) throws Exception {
108+
LOGGER.info("(V2) JDBC Insert data benchmark");
109+
insertData(jdbcV2, "insert_test_jdbcV2", state.batchSize, state.getPayload(state.payloadSize));
110+
}
111+
112+
}

0 commit comments

Comments
 (0)