Skip to content

Commit 856bc9d

Browse files
chernserchernser
committed
added row binary tests for v2
1 parent e00a973 commit 856bc9d

File tree

4 files changed

+85
-4
lines changed

4 files changed

+85
-4
lines changed

‎performance/src/test/com/clickhouse/benchmark/clients/BenchmarkBase.java‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -14,6 +14,7 @@
1414
import com.clickhouse.client.api.Client;
1515
import com.clickhouse.client.api.enums.Protocol;
1616
import com.clickhouse.client.api.insert.InsertResponse;
17+
import com.clickhouse.client.api.metadata.TableSchema;
1718
import com.clickhouse.data.ClickHouseDataProcessor;
1819
import com.clickhouse.data.ClickHouseFormat;
1920
import com.clickhouse.data.ClickHouseOutputStream;
@@ -126,7 +127,6 @@ public static void loadClickHouseRecords(String tableName, DataSet dataSet) {
126127
ClickHouseOutputStream.of(new ByteArrayOutputStream()), response.getColumns(), Collections.emptyMap());
127128
assert dataProcessor.getColumns() != null;
128129
dataSet.setClickHouseDataProcessor(dataProcessor);
129-
130130
ArrayList<ClickHouseRecord> records = new ArrayList<>();
131131
for (ClickHouseRecord record : response.records()) {
132132
records.add(record);

‎performance/src/test/com/clickhouse/benchmark/clients/InsertClient.java‎

Lines changed: 59 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -8,11 +8,13 @@
88
import com.clickhouse.client.ClickHouseResponseSummary;
99
import com.clickhouse.client.api.Client;
1010
import com.clickhouse.client.api.data_formats.ClickHouseBinaryFormatReader;
11+
import com.clickhouse.client.api.data_formats.RowBinaryFormatWriter;
1112
import com.clickhouse.client.api.enums.Protocol;
1213
import com.clickhouse.client.api.insert.InsertResponse;
1314
import com.clickhouse.client.api.insert.InsertSettings;
1415
import com.clickhouse.client.api.query.QueryResponse;
1516
import com.clickhouse.client.config.ClickHouseClientOption;
17+
import com.clickhouse.data.ClickHouseColumn;
1618
import com.clickhouse.data.ClickHouseDataProcessor;
1719
import com.clickhouse.data.ClickHouseFormat;
1820
import com.clickhouse.data.ClickHouseRecord;
@@ -27,6 +29,8 @@
2729
import org.slf4j.LoggerFactory;
2830

2931
import java.math.BigInteger;
32+
import java.util.List;
33+
import java.util.Map;
3034

3135
import static com.clickhouse.client.ClickHouseServerForTest.isCloud;
3236

@@ -168,4 +172,59 @@ public void insertV1RowBinary(DataState dataState) {
168172
LOGGER.error("Error: ", e);
169173
}
170174
}
175+
176+
@Benchmark
177+
public void insertV2RowBinary(DataState dataState) {
178+
try {
179+
try (InsertResponse response = clientV2.insert(dataState.dataSet.getTableName(), out -> {
180+
RowBinaryFormatWriter w = new RowBinaryFormatWriter(out, dataState.dataSet.getSchema(), ClickHouseFormat.RowBinary);
181+
List<ClickHouseColumn> columns = dataState.dataSet.getSchema().getColumns();
182+
for (Map<String, Object> row : dataState.dataSet.getRows()) {
183+
for (ClickHouseColumn column : columns) {
184+
w.setValue(column.getColumnName(),row.get(column.getColumnName()));
185+
}
186+
w.commitRow();
187+
}
188+
out.flush();
189+
190+
}, ClickHouseFormat.RowBinaryWithDefaults, new InsertSettings()).get()) {
191+
if (response.getWrittenRows() <= 0) {
192+
throw new RuntimeException("Rows written: " + response.getWrittenRows());
193+
}
194+
}
195+
} catch (Exception e) {
196+
LOGGER.error("Error: ", e);
197+
}
198+
}
199+
200+
@Benchmark
201+
public void insertV1WithV2RowBinaryWriter(DataState dataState) {
202+
try {
203+
ClickHouseFormat format = ClickHouseFormat.RowBinary;
204+
try (ClickHouseResponse response = clientV1.read(getServer())
205+
.write()
206+
.option(ClickHouseClientOption.ASYNC, false)
207+
.format(format)
208+
.query("INSERT INTO `" + DB_NAME + "`.`" + dataState.dataSet.getTableName() + "`")
209+
.data(out -> {
210+
RowBinaryFormatWriter w = new RowBinaryFormatWriter(out, dataState.dataSet.getSchema(), ClickHouseFormat.RowBinary);
211+
List<ClickHouseColumn> columns = dataState.dataSet.getSchema().getColumns();
212+
for (Map<String, Object> row : dataState.dataSet.getRows()) {
213+
for (ClickHouseColumn column : columns) {
214+
w.setValue(column.getColumnName(),row.get(column.getColumnName()));
215+
}
216+
w.commitRow();
217+
}
218+
out.flush();
219+
})
220+
.executeAndWait()) {
221+
ClickHouseResponseSummary summary = response.getSummary();
222+
if (summary.getWrittenRows() <= 0) {
223+
throw new RuntimeException("Rows written: " + summary.getWrittenRows());
224+
}
225+
}
226+
} catch ( Exception e) {
227+
LOGGER.error("Error: ", e);
228+
}
229+
}
171230
}

‎performance/src/test/com/clickhouse/benchmark/data/FileDataSet.java‎

Lines changed: 24 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -1,17 +1,21 @@
11
package com.clickhouse.benchmark.data;
22

33
import com.clickhouse.client.api.metadata.TableSchema;
4+
import com.clickhouse.data.ClickHouseColumn;
45
import com.clickhouse.data.ClickHouseDataProcessor;
56
import com.clickhouse.data.ClickHouseFormat;
67
import com.clickhouse.data.ClickHouseRecord;
8+
import com.clickhouse.data.ClickHouseValue;
79
import org.slf4j.Logger;
810
import org.slf4j.LoggerFactory;
11+
import org.testcontainers.shaded.com.google.common.collect.Table;
912

1013
import java.io.BufferedReader;
1114
import java.io.File;
1215
import java.util.ArrayList;
1316
import java.util.Collections;
1417
import java.util.HashMap;
18+
import java.util.Iterator;
1519
import java.util.List;
1620
import java.util.Map;
1721

@@ -26,6 +30,8 @@ public class FileDataSet implements DataSet{
2630

2731
private List<byte[]> lines =null;
2832

33+
private List<Map<String, Object>> data;
34+
2935
public FileDataSet(String filePath) {
3036
File srcFile = new File(filePath);
3137

@@ -98,9 +104,10 @@ public String getTrucateTableString() {
98104
return "TRUNCATE TABLE " + getTableName();
99105
}
100106

107+
private TableSchema tableSchema = new TableSchema();
101108
@Override
102109
public TableSchema getSchema() {
103-
return new TableSchema(); //TODO: parse from create statement or get by client
110+
return tableSchema;
104111
}
105112

106113
@Override
@@ -115,7 +122,7 @@ public List<byte[]> getBytesList(ClickHouseFormat format) {
115122

116123
@Override
117124
public List<Map<String, Object>> getRows() {
118-
return Collections.emptyList();
125+
return data;
119126
}
120127

121128
@Override
@@ -133,6 +140,18 @@ public List<ClickHouseRecord> getClickHouseRecords() {
133140
@Override
134141
public void setClickHouseRecords(List<ClickHouseRecord> records) {
135142
this.clickHouseRecords = records;
143+
List<ClickHouseColumn> columns = tableSchema.getColumns();
144+
data = new ArrayList<>(records.size());
145+
for (ClickHouseRecord record : records) {
146+
Iterator<ClickHouseValue> vIter = record.iterator();
147+
int i = 0;
148+
Map<String, Object> row = new HashMap<>();
149+
while (vIter.hasNext()) {
150+
ClickHouseValue v = vIter.next();
151+
row.put(columns.get(i++).getColumnName(), v.asObject());
152+
}
153+
data.add(row);
154+
}
136155
}
137156

138157
private ClickHouseDataProcessor dataProcessor;
@@ -145,6 +164,9 @@ public ClickHouseDataProcessor getClickHouseDataProcessor() {
145164
@Override
146165
public void setClickHouseDataProcessor(ClickHouseDataProcessor dataProcessor) {
147166
this.dataProcessor = dataProcessor;
167+
for (ClickHouseColumn column : dataProcessor.getColumns()) {
168+
tableSchema.addColumn(column.getColumnName(), column.getOriginalTypeName());
169+
}
148170
}
149171

150172
@Override

‎performance/src/test/com/clickhouse/benchmark/data/SimpleDataSet.java‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -28,7 +28,7 @@ public class SimpleDataSet implements DataSet {
2828

2929
public SimpleDataSet() {
3030
tableName = name + "_dataset_" + UUID.randomUUID().toString().replaceAll("-", "");
31-
size = 10;
31+
size = 100000;
3232

3333
data = new ArrayList<>(size);
3434
for (int i = 0; i < size; i++) {

0 commit comments

Comments
 (0)