Skip to content

Commit 28ebc30

Browse files
author
Paultagoras
committed
Adjusting sample data generation
1 parent a62441f commit 28ebc30

File tree

3 files changed

+53
-20
lines changed

3 files changed

+53
-20
lines changed

‎performance/src/test/com/clickhouse/benchmark/BenchmarkRunner.java‎

Lines changed: 4 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -20,11 +20,14 @@
2020
import org.slf4j.LoggerFactory;
2121

2222
import java.io.InputStream;
23-
import java.util.List;
23+
import java.time.format.DateTimeFormatter;
24+
import java.time.format.DateTimeFormatterBuilder;
2425
import java.util.concurrent.TimeUnit;
2526

2627

2728
public class BenchmarkRunner {
29+
public static final DateTimeFormatter DATETIME_FORMATTER = new DateTimeFormatterBuilder()
30+
.appendPattern("yyyy-MM-dd HH:mm:ss").toFormatter();
2831
private static final Logger LOGGER = LoggerFactory.getLogger(BenchmarkRunner.class);
2932
public static final String DB_NAME = "benchmarks";
3033
public static final long SMALL_SIZE = 500;

‎performance/src/test/com/clickhouse/benchmark/clients/ClientV2.java‎

Lines changed: 2 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -33,6 +33,7 @@ public void setUp() {
3333
.compressClientRequest(true)
3434
.setMaxRetries(0)
3535
.useHttpCompression(true)
36+
.setDefaultDatabase(BenchmarkRunner.DB_NAME)
3637
.build();
3738
}
3839

@@ -48,7 +49,7 @@ public void tearDown() {
4849
@Benchmark
4950
public void query(DataSet dataSet) {
5051
try {
51-
try(QueryResponse response = client.query("SELECT * FROM `" + BenchmarkRunner.DB_NAME + "`.`" + dataSet.tableName + "`").get()) {
52+
try(QueryResponse response = client.query("SELECT * FROM `" + dataSet.tableName + "`").get()) {
5253
ClickHouseBinaryFormatReader reader = client.newBinaryFormatReader(response);
5354
while (reader.next() != null) {//Compiler optimization avoidance
5455
notNull(reader.readValue(1));

‎performance/src/test/com/clickhouse/benchmark/data/DataSet.java‎

Lines changed: 47 additions & 18 deletions
Original file line numberDiff line numberDiff line change
@@ -1,30 +1,27 @@
11
package com.clickhouse.benchmark.data;
22

33
import com.clickhouse.benchmark.BenchmarkRunner;
4-
import com.clickhouse.client.api.Client;
5-
import com.clickhouse.client.api.data_formats.RowBinaryFormatReader;
6-
import com.clickhouse.client.api.data_formats.RowBinaryFormatWriter;
74
import com.clickhouse.client.api.metadata.TableSchema;
8-
import com.clickhouse.client.api.query.QueryResponse;
95
import com.clickhouse.data.ClickHouseFormat;
106
import org.openjdk.jmh.annotations.Level;
117
import org.openjdk.jmh.annotations.Param;
128
import org.openjdk.jmh.annotations.Scope;
139
import org.openjdk.jmh.annotations.Setup;
1410
import org.openjdk.jmh.annotations.State;
1511
import org.openjdk.jmh.annotations.TearDown;
12+
import org.slf4j.Logger;
13+
import org.slf4j.LoggerFactory;
1614

1715
import java.io.ByteArrayInputStream;
18-
import java.io.ByteArrayOutputStream;
1916
import java.io.IOException;
2017
import java.io.InputStream;
2118
import java.time.ZonedDateTime;
22-
import java.util.Map;
2319
import java.util.UUID;
2420
import java.util.concurrent.ThreadLocalRandom;
2521

2622
@State(Scope.Benchmark)
2723
public class DataSet {
24+
private static final Logger LOGGER = LoggerFactory.getLogger(DataSet.class);
2825
public final String name = "sample_data_set";
2926
public final String tableName = name + "_" + UUID.randomUUID().toString().replaceAll("-", "");
3027
@Param({"10000"})
@@ -47,18 +44,34 @@ public void setup() {
4744
data = generateData(size);
4845
BenchmarkRunner.insertData(tableName, new ByteArrayInputStream(generateData(size)), getFormat());
4946
} catch (Exception e) {
50-
throw new RuntimeException("Error generating data", e);
47+
LOGGER.error("Error while creating table or inserting data.", e);
48+
throw new RuntimeException("Error while creating table or inserting data.", e);
5149
}
5250
}
53-
private byte[] generateData(int size) throws IOException {
54-
ByteArrayOutputStream outputStream = new ByteArrayOutputStream();
55-
RowBinaryFormatWriter writer = new RowBinaryFormatWriter(outputStream, getSchema(), getFormat());
51+
private byte[] generateData(int size) {
52+
//Generate JSON sample data
53+
StringBuilder builder = new StringBuilder();
5654
for (int i = 0; i < size; i++) {
57-
writer.setValue("id", rowCounter++);
58-
writer.setValue("sample_int", ThreadLocalRandom.current().nextInt());
59-
writer.commitRow();
55+
builder.append("{\"trip_id\":").append(rowCounter)
56+
.append(",\"dropoff_datetime\":\"").append(BenchmarkRunner.DATETIME_FORMATTER.format(ZonedDateTime.now())).append("\"")
57+
.append(",\"pickup_longitude\":").append(ThreadLocalRandom.current().nextDouble(-180, 180))
58+
.append(",\"pickup_latitude\":").append(ThreadLocalRandom.current().nextDouble(-90, 90))
59+
.append(",\"dropoff_longitude\":").append(ThreadLocalRandom.current().nextDouble(-180, 180))
60+
.append(",\"dropoff_latitude\":").append(ThreadLocalRandom.current().nextDouble(-90, 90))
61+
.append(",\"passenger_count\":").append(ThreadLocalRandom.current().nextInt(1, 5))
62+
.append(",\"trip_distance\":").append(ThreadLocalRandom.current().nextDouble(0, 100))
63+
.append(",\"fare_amount\":").append(ThreadLocalRandom.current().nextDouble(0, 100))
64+
.append(",\"extra\":").append(ThreadLocalRandom.current().nextDouble(0, 100))
65+
.append(",\"tip_amount\":").append(ThreadLocalRandom.current().nextDouble(0, 100))
66+
.append(",\"tolls_amount\":").append(ThreadLocalRandom.current().nextDouble(0, 100))
67+
.append(",\"total_amount\":").append(ThreadLocalRandom.current().nextDouble(0, 100))
68+
.append(",\"payment_type\":\"CSH\",\"pickup_ntaname\":\"NTA1\",\"dropoff_ntaname\":\"NTA2\"}");
69+
if (i < size - 1) {
70+
builder.append("\n");
71+
}
72+
rowCounter++;
6073
}
61-
return outputStream.toByteArray();
74+
return builder.toString().getBytes();
6275
}
6376

6477
@TearDown(Level.Trial)
@@ -82,18 +95,34 @@ public void teardown() {
8295

8396
public String getCreateTableString() {
8497
return "CREATE TABLE " + tableName + " (\n" +
85-
" id UInt32,\n" +
86-
" sample_int Int32\n" +
98+
" trip_id UInt32,\n" +
99+
" pickup_datetime DateTime DEFAULT now(),\n" +
100+
" dropoff_datetime DateTime,\n" +
101+
" pickup_longitude Nullable(Float64),\n" +
102+
" pickup_latitude Nullable(Float64),\n" +
103+
" dropoff_longitude Nullable(Float64),\n" +
104+
" dropoff_latitude Nullable(Float64),\n" +
105+
" passenger_count UInt8,\n" +
106+
" trip_distance Float32,\n" +
107+
" fare_amount Float32,\n" +
108+
" extra Float32,\n" +
109+
" tip_amount Float32,\n" +
110+
" tolls_amount Float32,\n" +
111+
" total_amount Float32,\n" +
112+
" payment_type Enum('CSH' = 1, 'CRE' = 2, 'NOC' = 3, 'DIS' = 4, 'UNK' = 5),\n" +
113+
" pickup_ntaname LowCardinality(String),\n" +
114+
" dropoff_ntaname LowCardinality(String)\n" +
87115
")\n" +
88-
"ENGINE = Memory;";
116+
"ENGINE = MergeTree\n" +
117+
"PRIMARY KEY (pickup_datetime, dropoff_datetime);";
89118
}
90119

91120
public TableSchema getSchema() {
92121
return schema;
93122
}
94123

95124
public ClickHouseFormat getFormat() {
96-
return ClickHouseFormat.RowBinaryWithNamesAndTypes;
125+
return ClickHouseFormat.JSONEachRow;
97126
}
98127
public InputStream getInputStream() {
99128
return new ByteArrayInputStream(data);

0 commit comments

Comments
 (0)