Skip to content

Commit 16d973d

Browse files
committed
made runned configurable thru arguments
1 parent c3f7f44 commit 16d973d

File tree

11 files changed

+434
-68
lines changed

11 files changed

+434
-68
lines changed

performance/README.md

Lines changed: 24 additions & 7 deletions
Original file line numberDiff line numberDiff line change
@@ -1,16 +1,33 @@
11

2+
## JMH Benchmarks
23

3-
## DataSetGenerator
44

5-
```shell
6-
mvn exec:java -Dscope=test -Dexec.mainClass="com.clickhouse.com.clickhouse.benchmark.data.ClickHouseDataTypesShort" -input <table_fields.sql> -rows <number_of_rows>
7-
```
5+
### Dependencies
6+
87

98

10-
## Performance Test
9+
### How to Run
10+
11+
With default settings :
12+
```shell
13+
mvn compile exec:exec
14+
```
1115

12-
with custom dataset
16+
With custom measurement iterations:
1317
```shell
14-
mvn test-compile exec:java -Dexec.classpathScope=test -Dexec.mainClass="com.clickhouse.com.clickhouse.benchmark.BenchmarkRunner" -Dexec.args="--dataset=file://dataset_1741150759025.csv"
18+
mvn compile exec:exec -Dexec.args="-classpath %classpath com.clickhouse.benchmark.BenchmarkRunner -m 3"
1519
```
1620

21+
Other options:
22+
- "-m" - number of measurement iterations
23+
- "-t" - time in seconds per iteration
24+
- "-b" - benchmark mask coma separated. Ex.: `-b writer,reader,i`. Default : `-b i,q`
25+
- "i" - InsertClient - insert operation benchmarks
26+
- "q" - QueryClient - query operation benchmarks
27+
- "ci" - ConcurrentInsertClient - concurrent version of insert benchmarks
28+
- "cq" - ConcurrentQueryClient - concurrent version of query benchmarks
29+
- "lz" - Compression - compression related benchmarks
30+
- "writer" - Serializer - serialization only logic benchmarks
31+
- "reader" - DeSerilalizer - deserialization only logic benchmarks
32+
- "mixed" - MixedWorkload
33+

performance/pom.xml

Lines changed: 13 additions & 18 deletions
Original file line numberDiff line numberDiff line change
@@ -45,6 +45,9 @@
4545
<versions-plugin.version>2.16.0</versions-plugin.version>
4646
<resource-plugin.version>3.3.1</resource-plugin.version>
4747

48+
<jmh.measure-iter>10</jmh.measure-iter>
49+
<jmh.measure-time>10</jmh.measure-time>
50+
4851
</properties>
4952

5053
<dependencies>
@@ -161,24 +164,16 @@
161164
<plugin>
162165
<groupId>org.codehaus.mojo</groupId>
163166
<artifactId>exec-maven-plugin</artifactId>
164-
<executions>
165-
<execution>
166-
<id>run-benchmarks</id>
167-
<goals>
168-
<goal>exec</goal>
169-
</goals>
170-
<configuration>
171-
<classpathScope>runtime</classpathScope>
172-
<executable>java</executable>
173-
<arguments>
174-
<argument>-classpath</argument>
175-
<classpath />
176-
<argument>com.clickhouse.benchmark.BenchmarkRunner</argument>
177-
<argument>.*</argument>
178-
</arguments>
179-
</configuration>
180-
</execution>
181-
</executions>
167+
<configuration>
168+
<executable>java</executable>
169+
<arguments>
170+
<argument>-classpath</argument>
171+
<classpath/>
172+
<argument>com.clickhouse.benchmark.BenchmarkRunner</argument>
173+
<argument>-m ${jmh.measure-iter}</argument>
174+
<argument>-t ${jmh.measure-time}</argument>
175+
</arguments>
176+
</configuration>
182177
</plugin>
183178
</plugins>
184179
</build>

performance/src/main/java/com/clickhouse/benchmark/BenchmarkRunner.java

Lines changed: 46 additions & 31 deletions
Original file line numberDiff line numberDiff line change
@@ -12,6 +12,7 @@
1212
import org.slf4j.Logger;
1313
import org.slf4j.LoggerFactory;
1414

15+
import java.util.*;
1516
import java.util.concurrent.TimeUnit;
1617

1718
import static com.clickhouse.benchmark.TestEnvironment.isCloud;
@@ -22,8 +23,20 @@ public class BenchmarkRunner {
2223

2324
public static void main(String[] args) throws Exception {
2425
LOGGER.info("Starting Benchmarks");
25-
int measurementIterations = Integer.getInteger("m", 10);
26-
int measurementTime = Integer.getInteger("t", isCloud() ? 30 : 10);
26+
Map<String, String> options = parseArgs(args);
27+
System.out.println("Start Benchmarks with options: " + options);
28+
final String env = isCloud() ? "cloud" : "local";
29+
final long time = System.currentTimeMillis();
30+
31+
final int measurementIterations = Integer.parseInt(options.getOrDefault("-m", "10"));
32+
final int measurementTime = Integer.parseInt(options.getOrDefault("-t", "" + (isCloud() ? 30 : 10)));
33+
final String resultFile = String.format("jmh-results-%s-%s.json", env, time);
34+
final String outputFile = String.format("jmh-results-%s-%s.out", env, time);
35+
36+
System.out.println("Measurement iterations: " + measurementIterations);
37+
System.out.println("Measurement time: " + measurementTime + "s");
38+
System.out.println("Env: " + env);
39+
System.out.println("Epoch Time: " + time);
2740

2841
ChainedOptionsBuilder optBuilder = new OptionsBuilder()
2942
.forks(1) // must be a fork. No fork only for debugging
@@ -37,43 +50,45 @@ public static void main(String[] args) throws Exception {
3750
.jvmArgs("-Xms8g", "-Xmx8g")
3851
.measurementTime(TimeValue.seconds(measurementTime))
3952
.resultFormat(ResultFormatType.JSON)
40-
// .output(String.format("jmh-results-%s-%s.out", isCloud() ? "cloud" : "local", System.currentTimeMillis()))
41-
.result(String.format("jmh-results-%s-%s.json", isCloud() ? "cloud" : "local", System.currentTimeMillis()));
53+
.output(outputFile)
54+
.result(resultFile);
4255

43-
String testMask = System.getProperty("b", "q,i");
56+
String testMask = options.getOrDefault("-b", "q,i");
4457
String[] testMaskParts = testMask.split(",");
4558

46-
// .include(QueryClient.class.getSimpleName())
47-
// .include(InsertClient.class.getSimpleName())
48-
// .include(ConcurrentInsertClient.class.getSimpleName())
49-
// .include(ConcurrentQueryClient.class.getSimpleName())
50-
// .include(Compression.class.getSimpleName())
51-
// .include(Serializers.class.getSimpleName())
52-
// .include(Deserializers.class.getSimpleName())
53-
// .include(MixedWorkload.class.getSimpleName())
54-
59+
SortedSet<String> benchmarks = new TreeSet<>();
60+
for (String p : testMaskParts) {
61+
String benchmark = BENCHMARK_FLAGS.get(p);
62+
if (benchmark != null) {
63+
optBuilder.include(benchmark);
64+
benchmarks.add(benchmark);
65+
}
66+
}
5567

68+
System.out.println("Running benchmarks: " + benchmarks);
5669
new Runner(optBuilder.build()).run();
5770
}
5871

59-
private enum Benchmarks {
60-
61-
QUERY_CLIENT(QueryClient.class, "q");
62-
63-
private Class<QueryClient> queryClientClass;
64-
private String mask;
65-
66-
Benchmarks(Class<QueryClient> queryClientClass, String mask) {
67-
this.queryClientClass = queryClientClass;
68-
this.mask = mask;
69-
}
70-
71-
String benchmarkName() {
72-
return queryClientClass.getName();
73-
}
72+
private static final Map<String, String> BENCHMARK_FLAGS = buildBenchmarkFlags();
73+
74+
private static Map<String, String> buildBenchmarkFlags() {
75+
HashMap<String, String> map = new HashMap<>();
76+
map.put("q", QueryClient.class.getName());
77+
map.put("i", InsertClient.class.getName());
78+
map.put("cq", ConcurrentQueryClient.class.getName());
79+
map.put("ci", ConcurrentInsertClient.class.getName());
80+
map.put("lz", Compression.class.getName());
81+
map.put("reader", Deserializers.class.getName());
82+
map.put("writer", Serializers.class.getName());
83+
map.put("mixed", MixedWorkload.class.getName());
84+
return map;
85+
}
7486

75-
String mask() {
76-
return mask;
87+
private static Map<String, String> parseArgs(String[] args) {
88+
Map<String, String> options = new HashMap<>();
89+
for (int i = 0; i < args.length; i+=2) {
90+
options.put(args[i], args[i+1]);
7791
}
92+
return options;
7893
}
7994
}

performance/src/main/java/com/clickhouse/benchmark/clients/BenchmarkBase.java

Lines changed: 0 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -5,7 +5,6 @@
55
import com.clickhouse.benchmark.data.SimpleDataSet;
66
import com.clickhouse.benchmark.data.SyntheticDataSet;
77
import com.clickhouse.client.*;
8-
import com.clickhouse.client.ClickHouseServerForTest;
98
import com.clickhouse.client.api.Client;
109
import com.clickhouse.client.api.enums.Protocol;
1110
import com.clickhouse.client.api.insert.InsertResponse;
Lines changed: 171 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,171 @@
1+
package com.clickhouse.benchmark.clients;
2+
3+
import com.clickhouse.benchmark.data.SyntheticDataSet;
4+
import com.clickhouse.client.api.Client;
5+
import com.clickhouse.client.api.data_formats.internal.BinaryStreamReader;
6+
import com.clickhouse.client.api.data_formats.internal.SerializerUtils;
7+
import com.clickhouse.client.api.query.QueryResponse;
8+
import com.clickhouse.client.api.query.QuerySettings;
9+
import com.clickhouse.data.ClickHouseColumn;
10+
import com.clickhouse.data.ClickHouseFormat;
11+
import com.clickhouse.data.ClickHouseInputStream;
12+
import com.clickhouse.data.ClickHouseOutputStream;
13+
import com.clickhouse.data.format.BinaryDataProcessor;
14+
import com.clickhouse.data.format.BinaryStreamUtils;
15+
import com.clickhouse.data.value.ClickHouseDateTimeValue;
16+
import org.openjdk.jmh.annotations.Benchmark;
17+
import org.openjdk.jmh.annotations.Level;
18+
import org.openjdk.jmh.annotations.Setup;
19+
import org.openjdk.jmh.infra.Blackhole;
20+
import org.slf4j.Logger;
21+
import org.slf4j.LoggerFactory;
22+
23+
import java.io.EOFException;
24+
import java.io.IOException;
25+
import java.io.OutputStream;
26+
import java.nio.ByteBuffer;
27+
import java.time.LocalDateTime;
28+
import java.util.TimeZone;
29+
30+
public class DataTypes extends BenchmarkBase {
31+
32+
private static final Logger LOGGER = LoggerFactory.getLogger(DataTypes.class);
33+
34+
@Setup(Level.Iteration)
35+
public void setUpIteration(DataState dataState) {
36+
super.setUpIteration();
37+
38+
try (Client c = getClientV2(); QueryResponse r = c.query("SELECT * FROM " + dataState.tableNameFilled, new QuerySettings()
39+
.setFormat(ClickHouseFormat.RowBinaryWithNamesAndTypes)).get()) {
40+
dataState.datasetAsRowBinaryWithNamesAndTypes = ByteBuffer.wrap(r.getInputStream().readAllBytes());
41+
LOGGER.info("Loaded {} from dataset", dataState.datasetAsRowBinaryWithNamesAndTypes.capacity());
42+
} catch (Exception e) {
43+
LOGGER.error("Failed to init data for components benchmark", e);
44+
}
45+
46+
if (dataState.syntheticDataSet != null) {
47+
dataState.syntheticDataSet = new SyntheticDataSet(dataState.limit);
48+
}
49+
}
50+
51+
@Benchmark
52+
public void readDateTimeV1(DataState dataState, Blackhole blackhole) {
53+
ClickHouseInputStream input = ClickHouseInputStream.of(dataState.syntheticDataSet.getDateTimeValuesRowBinaryStream());
54+
BinaryDataProcessor.DateTime64SerDe serDe = new BinaryDataProcessor.DateTime64SerDe(3, TimeZone.getTimeZone("UTC"));
55+
56+
ClickHouseDateTimeValue valueHolder = ClickHouseDateTimeValue.ofNull(3, TimeZone.getTimeZone("UTC"));
57+
58+
int valueCount = 0;
59+
while (valueCount <= dataState.limit) {
60+
try {
61+
serDe.deserialize(valueHolder, input);
62+
blackhole.consume(valueHolder);
63+
valueCount++;
64+
} catch (IOException ex) {
65+
if (valueCount < dataState.limit) {
66+
throw new RuntimeException("Failed to read all values", ex);
67+
}
68+
break;
69+
}
70+
}
71+
}
72+
73+
@Benchmark
74+
public void readDateTimeV2(DataState dataState, Blackhole blackhole) {
75+
ClickHouseInputStream input = ClickHouseInputStream.of(dataState.syntheticDataSet.getDateTimeValuesRowBinaryStream());
76+
77+
byte[] buffer = new byte[8];
78+
TimeZone zoneId = TimeZone.getTimeZone("UTC");
79+
80+
int valueCount = 0;
81+
while (valueCount <= dataState.limit) {
82+
try {
83+
blackhole.consume(BinaryStreamReader.readDateTime64(input, buffer, 3, zoneId));
84+
valueCount++;
85+
} catch (EOFException ex) {
86+
if (valueCount < dataState.limit) {
87+
throw new RuntimeException("Failed to read all values", ex);
88+
}
89+
break;
90+
} catch (IOException ex) {
91+
throw new RuntimeException("Failed to read all values", ex);
92+
}
93+
}
94+
}
95+
96+
97+
@Benchmark
98+
public void DateTimeSerializerV1(DataState dataState, Blackhole blackhole) {
99+
OutputStream empty = new BlackholeOutputStream(blackhole);
100+
BinaryDataProcessor.DateTime64SerDe serDe =
101+
new BinaryDataProcessor.DateTime64SerDe(3, TimeZone.getTimeZone("UTC"));
102+
103+
ClickHouseOutputStream chos = ClickHouseOutputStream.of(empty);
104+
TimeZone tz = TimeZone.getTimeZone("UTC");
105+
106+
for (LocalDateTime dateTime : dataState.syntheticDataSet.getDateTimeValues()) {
107+
try {
108+
BinaryStreamUtils.writeDateTime64(chos, dateTime, 3, tz);
109+
// App should wrap a value with a value object if it wants to use a data processor
110+
// serDe.serialize(ClickHouseDateTimeValue.of(dateTime, 3, tz) , chos);
111+
} catch (Exception e) {
112+
LOGGER.error("Error: ", e);
113+
}
114+
}
115+
try {
116+
chos.flush();
117+
} catch (Exception e) {
118+
LOGGER.error("Error: ", e);
119+
}
120+
}
121+
122+
@Benchmark
123+
public void DateTimeSerializerV2(DataState dataState, Blackhole blackhole) {
124+
OutputStream empty = new BlackholeOutputStream(blackhole);
125+
ClickHouseColumn column = ClickHouseColumn.of("a", "DateTime64(3, 'UTC')");
126+
127+
for (LocalDateTime dateTime : dataState.syntheticDataSet.getDateTimeValues()) {
128+
try {
129+
SerializerUtils.serializeData(empty, dateTime, column);
130+
} catch (Exception e) {
131+
LOGGER.error("Error: ", e);
132+
}
133+
}
134+
}
135+
136+
private static class BlackholeOutputStream extends OutputStream {
137+
138+
private final Blackhole blackhole;
139+
public long count = 0;
140+
141+
public BlackholeOutputStream(Blackhole blackhole) {
142+
this.blackhole = blackhole;
143+
}
144+
145+
@Override
146+
public void write(int b) {
147+
blackhole.consume(b);
148+
count++;
149+
}
150+
151+
@Override
152+
public void write(byte[] b) {
153+
write(b, 0, b.length);
154+
}
155+
156+
@Override
157+
public void write(byte[] b, int off, int len) {
158+
blackhole.consume(b);
159+
count += len;
160+
}
161+
162+
@Override
163+
public void flush() {
164+
165+
}
166+
167+
@Override
168+
public void close() {
169+
}
170+
}
171+
}

0 commit comments

Comments
 (0)