JDBC perf test with JMH

clickhouse-jdbc/pom.xml

@@ -137,6 +137,19 @@
             <version>${revision}</version>
             <scope>compile</scope>
         </dependency>
+        <dependency>
+            <groupId>org.openjdk.jmh</groupId>
+            <artifactId>jmh-core</artifactId>
+            <version>${jmh.version}</version>
+            <scope>test</scope>
+        </dependency>
+        <dependency>
+            <groupId>org.openjdk.jmh</groupId>
+            <artifactId>jmh-generator-annprocess</artifactId>
+            <version>${jmh.version}</version>
+            <scope>test</scope>
+        </dependency>
+
     </dependencies>
 
     <profiles>
@@ -222,8 +235,108 @@
                 </plugins>
             </build>
         </profile>
+        <profile>
+            <id>performance-testing-jdbc</id>
+            <activation><activeByDefault>false</activeByDefault></activation>
+            <dependencies>
+                <dependency>
+                    <groupId>org.openjdk.jmh</groupId>
+                    <artifactId>jmh-core</artifactId>
+                    <version>${jmh.version}</version>
+                </dependency>
+                <dependency>
+                    <groupId>org.openjdk.jmh</groupId>
+                    <artifactId>jmh-generator-annprocess</artifactId>
+                    <version>${jmh.version}</version>
+                </dependency>
+            </dependencies>
+            <build>
+                <plugins>
+                    <plugin>
+                        <groupId>org.apache.maven.plugins</groupId>
+                        <artifactId>maven-compiler-plugin</artifactId>
+                        <configuration>
+                            <annotationProcessorPaths>
+                                <path>
+                                    <groupId>org.projectlombok</groupId>
+                                    <artifactId>lombok</artifactId>
+                                    <version>1.18.32</version>
+                                </path>
+                                <path>
+                                    <groupId>org.openjdk.jmh</groupId>
+                                    <artifactId>jmh-generator-annprocess</artifactId>
+                                    <version>${jmh.version}</version>
+                                </path>
+                            </annotationProcessorPaths>
+                            <release>8</release>
+                        </configuration>
+                    </plugin>
+                    <plugin>
+                        <groupId>org.apache.maven.plugins</groupId>
+                        <artifactId>maven-shade-plugin</artifactId>
+                        <executions>
+                            <execution>
+                                <phase>package</phase>
+                                <goals>
+                                    <goal>shade</goal>
+                                </goals>
+                                <configuration>
+                                    <finalName>benchmarks</finalName>
+                                    <transformers>
+                                        <transformer implementation="org.apache.maven.plugins.shade.resource.ManifestResourceTransformer">
+                                            <mainClass>org.openjdk.jmh.Main</mainClass>
+                                        </transformer>
+                                    </transformers>
+                                </configuration>
+                            </execution>
+                        </executions>
+                    </plugin>
+                    <plugin>
+                        <groupId>org.codehaus.mojo</groupId>
+                        <artifactId>build-helper-maven-plugin</artifactId>
+                        <version>3.6.0</version>
+                        <executions>
+                            <execution>
+                                <id>add-test-source</id>
+                                <phase>generate-test-sources</phase>
+                                <goals>
+                                    <goal>add-test-source</goal>
+                                </goals>
+                                <configuration>
+                                    <sources>
+                                        <source>src/test/perf</source>
+                                    </sources>
+                                </configuration>
+                            </execution>
+                        </executions>
+                    </plugin>
+                    <plugin>
+                        <groupId>org.codehaus.mojo</groupId>
+                        <artifactId>exec-maven-plugin</artifactId>
+                        <executions>
+                            <execution>
+                                <id>run-benchmarks</id>
+                                <phase>integration-test</phase>
+                                <goals>
+                                    <goal>exec</goal>
+                                </goals>
+                                <configuration>
+                                    <classpathScope>test</classpathScope>
+                                    <executable>java</executable>
+                                    <arguments>
+                                        <argument>-classpath</argument>
+                                        <classpath />
+                                        <argument>org.openjdk.jmh.Main</argument>
+                                        <argument>.*</argument>
+                                    </arguments>
+                                </configuration>
+                            </execution>
+                        </executions>
+                    </plugin>
+                </plugins>
+            </build>
+        </profile>
     </profiles>
-
     <build>
         <plugins>
             <plugin>

clickhouse-jdbc/src/test/perf/README.md

@@ -0,0 +1,20 @@
+# Performance Testing using JMH for JDBC driver
+Testing of Insert, Select & comparing the performance of JDBC V1 & JDBC V2
+Insert testing 
+- batch sizes [100, 1000, 10000, 100000]
+- data types [int8, int16, int32, int64, float32, float64, string, boolean] (Will be extended to other data types as well)
+
+Note: string payload size is adjustable ["1024", "2048", "4096", "8192" ] 
+
+Select testing query `"SELECT * FROM %s LIMIT %d"`
+- limit is adjustable currently use 100 
+- retrieval of different data types [int8, int16, int32, int64, float32, float64, string, boolean] (Will be extended to other data types as well)
+- separated test for connection part and connection with deserialization part
+
+# how to run 
+mvn clean package -DskipTests=true -P performance-testing-jdbc
+
+# output file 
+The benchmark generate the output file in jmh-jdbc-results.json 
+Use https://jmh.morethan.io/ to visualize the results
+

clickhouse-jdbc/src/test/perf/com/clickhouse/jdbc/perf/JDBCBenchmarkBase.java

@@ -0,0 +1,44 @@
+package com.clickhouse.jdbc.perf;
+
+import com.clickhouse.client.BaseIntegrationTest;
+import com.clickhouse.client.ClickHouseNode;
+import com.clickhouse.client.ClickHouseProtocol;
+import com.clickhouse.client.ClickHouseServerForTest;
+import com.clickhouse.jdbc.ClickHouseDriver;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import java.sql.Connection;
+import java.sql.SQLException;
+import java.util.Properties;
+
+import static com.clickhouse.client.ClickHouseServerForTest.isCloud;
+
+public class JDBCBenchmarkBase {
+    private static final Logger LOGGER = LoggerFactory.getLogger(JDBCBenchmarkBase.class);
+    protected static Connection jdbcV1 = null;
+    protected static Connection jdbcV2 = null;
+
+    public void setup() throws SQLException {
+        BaseIntegrationTest.setupClickHouseContainer();
+        // TODO: create two connections
+        Properties properties = new Properties();
+        properties.put("user", "default");
+        properties.put("password", "test_default_password");
+
+        ClickHouseNode node = ClickHouseServerForTest.getClickHouseNode(ClickHouseProtocol.HTTP, isCloud(), ClickHouseNode.builder().build());
+
+        LOGGER.info(String.format("clickhouse endpoint [%s:%s]", node.getHost(), node.getPort()));
+        jdbcV1 = new ClickHouseDriver().connect(String.format("jdbc:clickhouse://%s:%s?clickhouse.jdbc.v1=true", node.getHost(), node.getPort()), properties);
+        jdbcV2 = new ClickHouseDriver().connect(String.format("jdbc:clickhouse://%s:%s", node.getHost(), node.getPort()), properties);
+
+    }
+
+    public void tearDown() throws SQLException {
+        LOGGER.info("Tearing down JDBC connections and ClickHouse container.");
+        if (jdbcV1 != null) jdbcV1.close();
+        if (jdbcV2 != null) jdbcV2.close();
+        BaseIntegrationTest.teardownClickHouseContainer();
+    }
+
+}

clickhouse-jdbc/src/test/perf/com/clickhouse/jdbc/perf/JDBCBenchmarkRunner.java

@@ -0,0 +1,39 @@
+package com.clickhouse.jdbc.perf;
+
+import org.openjdk.jmh.annotations.Mode;
+import org.openjdk.jmh.profile.GCProfiler;
+import org.openjdk.jmh.profile.MemPoolProfiler;
+import org.openjdk.jmh.results.format.ResultFormatType;
+import org.openjdk.jmh.runner.Runner;
+import org.openjdk.jmh.runner.options.Options;
+import org.openjdk.jmh.runner.options.OptionsBuilder;
+import org.openjdk.jmh.runner.options.TimeValue;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import java.util.concurrent.TimeUnit;
+
+public class JDBCBenchmarkRunner {
+    private static final Logger LOGGER = LoggerFactory.getLogger(JDBCBenchmarkRunner.class);
+
+    public static void main(String[] args) throws Exception {
+        LOGGER.info("Starting Benchmarks");
+        Options opt = new OptionsBuilder()
+                .include(JDBCSelectBenchmark.class.getSimpleName())
+                .include(JDBCInsertBenchmark.class.getSimpleName())
+                .forks(1)
+                .mode(Mode.AverageTime)
+                .timeUnit(TimeUnit.MILLISECONDS)
+                .threads(1)
+                .addProfiler(GCProfiler.class)
+                .addProfiler(MemPoolProfiler.class)
+                .warmupIterations(1)
+                .warmupTime(TimeValue.seconds(5))
+                .measurementIterations(1)
+                .measurementTime(TimeValue.seconds(5))
+                .resultFormat(ResultFormatType.JSON)
+                .result("jmh-jdbc-results.json")
+                .build();
+        new Runner(opt).run();
+    }
+}

clickhouse-jdbc/src/test/perf/com/clickhouse/jdbc/perf/JDBCInsertBenchmark.java

@@ -0,0 +1,112 @@
+package com.clickhouse.jdbc.perf;
+
+import org.apache.commons.lang3.RandomStringUtils;
+import org.openjdk.jmh.annotations.Benchmark;
+import org.openjdk.jmh.annotations.BenchmarkMode;
+import org.openjdk.jmh.annotations.Fork;
+import org.openjdk.jmh.annotations.Level;
+import org.openjdk.jmh.annotations.Measurement;
+import org.openjdk.jmh.annotations.Mode;
+import org.openjdk.jmh.annotations.OutputTimeUnit;
+import org.openjdk.jmh.annotations.Param;
+import org.openjdk.jmh.annotations.Scope;
+import org.openjdk.jmh.annotations.Setup;
+import org.openjdk.jmh.annotations.State;
+import org.openjdk.jmh.annotations.TearDown;
+import org.openjdk.jmh.annotations.Threads;
+import org.openjdk.jmh.annotations.Warmup;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import java.sql.Connection;
+import java.sql.PreparedStatement;
+import java.sql.SQLException;
+import java.util.HashMap;
+import java.util.Map;
+import java.util.concurrent.TimeUnit;
+@State(Scope.Benchmark)
+public class JDBCInsertBenchmark extends JDBCBenchmarkBase {
+    private static final Logger LOGGER = LoggerFactory.getLogger(JDBCInsertBenchmark.class);
+
+    @Setup
+    public void setup() throws SQLException {
+        super.setup();
+    }
+
+    @TearDown
+    public void tearDown() throws SQLException {
+       super.tearDown();
+    }
+
+    @State(Scope.Thread)
+    public static class InsertState {
+        String payload1024 = RandomStringUtils.random(1024, true, true);;
+        String payload2048 = RandomStringUtils.random(1024, true, true);;
+        String payload4096 = RandomStringUtils.random(4096, true, true);;
+        String payload8192 = RandomStringUtils.random(8192, true, true);;
+
+        Map payloads = new HashMap<String, String>() {{
+            put("1024", payload1024);
+            put("2048", payload2048);
+            put("4096", payload4096);
+            put("8192", payload8192);
+        }};
+//        @Param({"100", "1000", "10000", "100000"})
+        @Param({"100000"})
+        int batchSize;
+//        @Param({"1024", "2048", "4096", "8192"})
+        @Param({"8192"})
+        String payloadSize;
+
+        public String getPayload(String payloadSize) {
+            return (String) payloads.get(payloadSize);
+        }
+        @Setup(Level.Invocation)
+        public void setup() throws SQLException {
+            LOGGER.info("Setting up insert state");
+            jdbcV1.createStatement().execute("DROP TABLE IF EXISTS insert_test_jdbcV1");
+            jdbcV2.createStatement().execute("DROP TABLE IF EXISTS insert_test_jdbcV2");
+            jdbcV1.createStatement().execute("CREATE TABLE insert_test_jdbcV1 ( `off16` Int16, `str` String, `p_int8` Int8, `p_int16` Int16, `p_int32` Int32, `p_int64` Int64, `p_float32` Float32, `p_float64` Float64, `p_bool` Bool) Engine = Memory");
+            jdbcV2.createStatement().execute("CREATE TABLE insert_test_jdbcV2 ( `off16` Int16, `str` String, `p_int8` Int8, `p_int16` Int16, `p_int32` Int32, `p_int64` Int64, `p_float32` Float32, `p_float64` Float64, `p_bool` Bool) Engine = Memory");
+        }
+
+        @TearDown(Level.Invocation)
+        public void tearDown() {
+            LOGGER.info("Tearing down insert state");
+        }
+    }
+
+    void insertData(Connection conn, String table, int size, String payload) throws SQLException {
+        PreparedStatement ps = conn.prepareStatement(String.format("INSERT INTO %s (off16, str, p_int8, p_int16, p_int32, p_int64, p_float32, p_float64, p_bool) VALUES (?,?,?,?,?,?,?,?,?)", table));
+        for (int i = 0; i < size; i++) {
+            ps.setShort(1, (short) i);
+            ps.setString(2, payload);
+            ps.setByte(3, (byte)i);
+            ps.setShort(4, (short)i);
+            ps.setInt(5, i);
+            ps.setLong(6, (long)i);
+            ps.setFloat(7, (float)(i*0.1));
+            ps.setDouble(8, (double)(i*0.1));
+            ps.setBoolean(9, true);
+            ps.addBatch();
+        }
+        ps.executeBatch();
+    }
+
+    @Benchmark
+    @Warmup(iterations = 1, time = 5, timeUnit = TimeUnit.SECONDS)
+    @Measurement(iterations = 5, time = 5, timeUnit = TimeUnit.SECONDS)
+    public void insertDataV1(InsertState state) throws Exception {
+        LOGGER.info("(V1) JDBC Insert data benchmark");
+        insertData(jdbcV1, "insert_test_jdbcV1", state.batchSize, state.getPayload(state.payloadSize));
+    }
+
+    @Benchmark
+    @Warmup(iterations = 1, time = 5, timeUnit = TimeUnit.SECONDS)
+    @Measurement(iterations = 5, time = 5, timeUnit = TimeUnit.SECONDS)
+    public void insertDataV2(InsertState state) throws Exception {
+        LOGGER.info("(V2) JDBC Insert data benchmark");
+        insertData(jdbcV2, "insert_test_jdbcV2", state.batchSize, state.getPayload(state.payloadSize));
+    }
+
+}