Skip to content

Commit e440df5

Browse files
author
Paultagoras
committed
Add rowbinary insert
1 parent 19b7347 commit e440df5

File tree

2 files changed

+85
-24
lines changed

2 files changed

+85
-24
lines changed

‎performance/src/test/com/clickhouse/benchmark/clients/BenchmarkBase.java‎

Lines changed: 14 additions & 17 deletions
Original file line numberDiff line numberDiff line change
@@ -1,6 +1,7 @@
11
package com.clickhouse.benchmark.clients;
22

33
import com.clickhouse.benchmark.BenchmarkRunner;
4+
import com.clickhouse.benchmark.TestEnvironment;
45
import com.clickhouse.benchmark.data.DataSet;
56
import com.clickhouse.benchmark.data.FileDataSet;
67
import com.clickhouse.benchmark.data.SimpleDataSet;
@@ -108,6 +109,8 @@ public void setTableNameEmpty(String tableNameEmpty) {
108109

109110
}
110111

112+
113+
111114
@Setup(Level.Trial)
112115
public void setup(DataState dataState) {
113116
setupEnvironment();
@@ -120,34 +123,28 @@ public void setup(DataState dataState) {
120123
}
121124
initializeTables(dataState);
122125
}
123-
124126
@TearDown(Level.Trial)
125127
public void tearDown() {
126128
cleanupEnvironment();
127129
}
128130

129131

130-
public static void initializeTables(DataState dataState) {
131-
LOGGER.info("Initializing tables: {}, {}", dataState.tableNameFilled, dataState.tableNameEmpty);
132-
LOGGER.debug("Create {}: {}", dataState.tableNameFilled, dataState.dataSet.getCreateTableString(dataState.tableNameFilled));
133-
LOGGER.debug("Create {}: {}", dataState.tableNameEmpty, dataState.dataSet.getCreateTableString(dataState.tableNameEmpty));
134-
runAndSyncQuery(dataState.dataSet.getCreateTableString(dataState.tableNameEmpty), dataState.tableNameEmpty);
135-
runAndSyncQuery(dataState.dataSet.getCreateTableString(dataState.tableNameFilled), dataState.tableNameFilled);
136-
//Truncate tables if they existed
137-
truncateTable(dataState.tableNameEmpty);
138-
truncateTable(dataState.tableNameFilled);
139-
140-
ClickHouseFormat format = dataState.dataSet.getFormat();
141-
LOGGER.debug("Inserting data into table: {}, format: {}", dataState.tableNameFilled, format);
142-
insertData(dataState.tableNameFilled, dataState.dataSet.getInputStream(format), format);//For query testing
132+
private static void initializeTables(DataState dataState) {
133+
initializeTable(dataState.tableNameFilled, dataState.dataSet.getCreateTableString(dataState.tableNameFilled)
134+
, dataState.dataSet.getInputStream(dataState.dataSet.getFormat()), dataState.dataSet.getFormat());
135+
initializeTable(dataState.tableNameEmpty, dataState.dataSet.getCreateTableString(dataState.tableNameEmpty), null, null);
143136
loadClickHouseRecords(dataState);//For insert testing
144137
}
145138

146139

147140

148-
public static void isNotNull(Object obj, boolean doWeCare) {
149-
if (obj == null && doWeCare) {
150-
throw new RuntimeException("Object is null");
141+
public static void initializeTable(String tableName, String createTableString, InputStream dataStream, ClickHouseFormat format) {
142+
LOGGER.info("Initializing table: {}", tableName);
143+
runAndSyncQuery(createTableString, tableName);
144+
truncateTable(tableName);
145+
146+
if (dataStream != null) {
147+
insertData(tableName, dataStream, format);
151148
}
152149
}
153150

‎performance/src/test/com/clickhouse/benchmark/clients/MixedWorkload.java‎

Lines changed: 71 additions & 7 deletions
Original file line numberDiff line numberDiff line change
@@ -5,15 +5,19 @@
55
import com.clickhouse.client.ClickHouseResponse;
66
import com.clickhouse.client.api.Client;
77
import com.clickhouse.client.api.data_formats.ClickHouseBinaryFormatReader;
8+
import com.clickhouse.client.api.data_formats.RowBinaryFormatWriter;
89
import com.clickhouse.client.api.insert.InsertResponse;
910
import com.clickhouse.client.api.insert.InsertSettings;
1011
import com.clickhouse.client.api.query.QueryResponse;
1112
import com.clickhouse.client.config.ClickHouseClientOption;
13+
import com.clickhouse.data.ClickHouseDataProcessor;
1214
import com.clickhouse.data.ClickHouseFormat;
1315
import com.clickhouse.data.ClickHouseRecord;
16+
import com.clickhouse.data.ClickHouseSerializer;
1417
import org.openjdk.jmh.annotations.Benchmark;
1518
import org.openjdk.jmh.annotations.Group;
1619
import org.openjdk.jmh.annotations.Level;
20+
import org.openjdk.jmh.annotations.Param;
1721
import org.openjdk.jmh.annotations.Scope;
1822
import org.openjdk.jmh.annotations.Setup;
1923
import org.openjdk.jmh.annotations.State;
@@ -23,23 +27,25 @@
2327
import org.slf4j.Logger;
2428
import org.slf4j.LoggerFactory;
2529

30+
import java.util.List;
31+
2632
import static com.clickhouse.benchmark.TestEnvironment.getServer;
2733

28-
@Threads(2)
34+
@Threads(3)
2935
@State(Scope.Benchmark)
3036
public class MixedWorkload extends BenchmarkBase {
3137
private static final Logger LOGGER = LoggerFactory.getLogger(MixedWorkload.class);
3238

3339
private ClickHouseClient clientV1Shared;
3440
private Client clientV2Shared;
3541
@Setup(Level.Trial)
36-
public void setUpIteration() {
42+
public void setUpTrial() {
3743
clientV1Shared = getClientV1();
3844
clientV2Shared = getClientV2();
3945
}
4046

4147
@TearDown(Level.Trial)
42-
public void tearDownIteration() {
48+
public void tearDownTrial() {
4349
if (clientV1Shared != null) {
4450
clientV1Shared.close();
4551
clientV1Shared = null;
@@ -50,15 +56,19 @@ public void tearDownIteration() {
5056
}
5157
}
5258

59+
@State(Scope.Thread)
60+
public static class MixedWorkloadState {
61+
@Param({"true", "false"})
62+
public boolean alternate;
63+
}
64+
65+
5366
@TearDown(Level.Iteration)
5467
public void teardownIteration(DataState dataState) {
5568
truncateTable(dataState.tableNameEmpty);
5669
}
5770

58-
// @State(Scope.Thread)
59-
// public static class MixedWorkloadState {
60-
//
61-
// }
71+
6272

6373
@Benchmark
6474
@Group("mixed_v1")
@@ -82,6 +92,33 @@ public void insertV1(DataState dataState) {
8292
}
8393
}
8494

95+
@Benchmark
96+
@Group("mixed_v1")
97+
public void insertV1RowBinary(DataState dataState) {
98+
try {
99+
ClickHouseFormat format = ClickHouseFormat.RowBinary;
100+
try (ClickHouseResponse response = clientV1Shared.read(getServer())
101+
.write()
102+
.option(ClickHouseClientOption.ASYNC, false)
103+
.format(format)
104+
.query(BenchmarkRunner.getInsertQuery(dataState.tableNameEmpty))
105+
.data(out -> {
106+
ClickHouseDataProcessor p = dataState.dataSet.getClickHouseDataProcessor();
107+
ClickHouseSerializer[] serializers = p.getSerializers(clientV1Shared.getConfig(), p.getColumns());
108+
for (ClickHouseRecord record : dataState.dataSet.getClickHouseRecords()) {
109+
for (int i = 0; i < serializers.length; i++) {
110+
serializers[i].serialize(record.getValue(i), out);
111+
}
112+
}
113+
})
114+
.executeAndWait()) {
115+
response.getSummary();
116+
}
117+
} catch ( Exception e) {
118+
LOGGER.error("Error: ", e);
119+
}
120+
}
121+
85122
@Benchmark
86123
@Group("mixed_v1")
87124
public void queryV1(DataState dataState, Blackhole blackhole) {
@@ -104,6 +141,9 @@ public void queryV1(DataState dataState, Blackhole blackhole) {
104141

105142

106143

144+
145+
146+
107147
@Benchmark
108148
@Group("mixed_v2")
109149
public void insertV2(DataState dataState) {
@@ -122,6 +162,30 @@ public void insertV2(DataState dataState) {
122162
}
123163
}
124164

165+
@Benchmark
166+
@Group("mixed_v2")
167+
public void insertV2RowBinary(DataState dataState) {
168+
try {
169+
try (InsertResponse response = clientV2Shared.insert(dataState.tableNameEmpty, out -> {
170+
RowBinaryFormatWriter w = new RowBinaryFormatWriter(out, dataState.dataSet.getSchema(), ClickHouseFormat.RowBinary);
171+
for (List<Object> row : dataState.dataSet.getRowsOrdered()) {
172+
int index = 1;
173+
for (Object value : row) {
174+
w.setValue(index, value);
175+
index++;
176+
}
177+
w.commitRow();
178+
}
179+
out.flush();
180+
181+
}, ClickHouseFormat.RowBinaryWithDefaults, new InsertSettings()).get()) {
182+
response.getWrittenRows();
183+
}
184+
} catch (Exception e) {
185+
LOGGER.error("Error: ", e);
186+
}
187+
}
188+
125189
@Benchmark
126190
@Group("mixed_v2")
127191
public void queryV2(DataState dataState, Blackhole blackhole) {

0 commit comments

Comments
 (0)