Skip to content

Commit 35359bb

Browse files
chernserchernser
committed
added example for writing/reading protobuf
1 parent d1356d8 commit 35359bb

File tree

7 files changed

+1371
-2
lines changed

7 files changed

+1371
-2
lines changed

‎examples/client/README.md‎

Lines changed: 44 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -15,11 +15,53 @@ mvn clean compile
1515

1616
To run:
1717
```shell
18-
mvn exec:java -Dexec.mainClass="com.clickhouse.examples.jdbc.Main"
18+
mvn exec:java -Dexec.mainClass="com.clickhouse.examples.jdbc.Main"
1919
```
2020

2121
Addition options can be passed to the application:
2222
- `-DchPort` - ClickHouse server port (default: 8123)
2323
- `-DchUser` - ClickHouse user name (default: default)
2424
- `-DchPassword` - ClickHouse user password (default: empty)
25-
- `-DchDatabase` - ClickHouse database name (default: default)
25+
- `-DchDatabase` - ClickHouse database name (default: default)
26+
27+
28+
## Protobuf Example
29+
30+
Class `com.clickhouse.examples.formats.ProtobufMain` demonstrates how to use Protobuf with ClickHouse.
31+
32+
### Setup
33+
34+
First of all new table should be created in a DB:
35+
```sql
36+
CREATE TABLE ui_events
37+
(
38+
`url` String,
39+
`user_id` String,
40+
`session_id` String,
41+
`timestamp` Int64,
42+
`duration` Int64 DEFAULT 0,
43+
`event` String,
44+
)
45+
ENGINE = MergeTree
46+
ORDER BY timestamp;
47+
```
48+
49+
To regenerate Java classes from Protobuf file:
50+
```shell
51+
protoc --java_out=./src/main/java/ ./src/proto/ui_stats_event.proto
52+
```
53+
54+
To compile:
55+
```shell
56+
mvn clean compile
57+
```
58+
59+
To run:
60+
```shell
61+
mvn exec:java -Dexec.mainClass="com.clickhouse.examples.formats.ProtobufMain"
62+
```
63+
64+
To run with a snapshot version of ClickHouse Java client (replace `0.6.0` with the actual version and use repository `https://oss.sonatype.org/content/repositories/snapshots`)
65+
```shell
66+
mvn exec:java -Dclickhouse-java.version=0.6.0-SNAPSHOT -Dexec.mainClass="com.clickhouse.examples.formats.ProtobufMain"
67+
```

‎examples/client/pom.xml‎

Lines changed: 17 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -67,6 +67,9 @@
6767
<project.reporting.outputEncoding>UTF-8</project.reporting.outputEncoding>
6868

6969
<clickhouse-java.version>0.6.0</clickhouse-java.version>
70+
<!-- Nightly snapshot version from https://s01.oss.sonatype.org/content/repositories/snapshots/ or latest from local -->
71+
<!-- <clickhouse-java.version>0.6.0-SNAPSHOT</clickhouse-java.version>-->
72+
7073
<apache-httpclient.version>5.2.1</apache-httpclient.version>
7174

7275
<compiler-plugin.version>3.8.1</compiler-plugin.version>
@@ -98,6 +101,20 @@
98101
<artifactId>httpclient5</artifactId>
99102
<version>${apache-httpclient.version}</version>
100103
</dependency>
104+
105+
<!-- Required for Protobuf example -->
106+
<dependency>
107+
<groupId>com.google.protobuf</groupId>
108+
<artifactId>protobuf-java</artifactId>
109+
<version>4.27.1</version>
110+
</dependency>
111+
112+
<dependency>
113+
<groupId>com.google.protobuf</groupId>
114+
<artifactId>protobuf-java-util</artifactId>
115+
<version>4.27.1</version>
116+
</dependency>
117+
101118
</dependencies>
102119

103120
<build>

‎examples/client/src/main/java/com/clickhouse/examples/formats/ProtobufMain.java‎

Lines changed: 118 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,118 @@
1+
package com.clickhouse.examples.formats;
2+
3+
import com.clickhouse.client.ClickHouseClient;
4+
import com.clickhouse.client.ClickHouseConfig;
5+
import com.clickhouse.client.ClickHouseNode;
6+
import com.clickhouse.client.ClickHouseNodeSelector;
7+
import com.clickhouse.client.ClickHouseProtocol;
8+
import com.clickhouse.client.ClickHouseRequest;
9+
import com.clickhouse.client.ClickHouseResponse;
10+
import com.clickhouse.client.http.config.ClickHouseHttpOption;
11+
import com.clickhouse.config.ClickHouseOption;
12+
import com.clickhouse.data.ClickHouseFormat;
13+
import com.clickhouse.data.ClickHouseInputStream;
14+
import com.clickhouse.examples.protos.UIEvent;
15+
import com.google.protobuf.Message;
16+
17+
18+
import java.io.Serializable;
19+
import java.util.ArrayList;
20+
import java.util.HashMap;
21+
import java.util.List;
22+
import java.util.Map;
23+
import java.util.logging.Level;
24+
import java.util.logging.Logger;
25+
26+
27+
public class ProtobufMain {
28+
29+
private static final Logger log = Logger.getLogger(ProtobufMain.class.getName());
30+
private final String host = "localhost";
31+
private final int port = 8123;
32+
33+
public void write(List<Message> messages) {
34+
try (ClickHouseClient client = getClient()) {
35+
ClickHouseRequest.Mutation mutation =
36+
client.write(ClickHouseNode.builder().host(host).port(ClickHouseProtocol.HTTP, port).build());
37+
38+
mutation.table("default.ui_events");
39+
mutation.format(ClickHouseFormat.Protobuf);
40+
ClickHouseResponse response = mutation.data((out) -> {
41+
for (Message message : messages) {
42+
// it is important to write the size of the message before the message itself
43+
out.writeVarInt(message.getSerializedSize());
44+
message.writeTo(out);
45+
}
46+
}).executeAndWait();
47+
48+
49+
log.info("Response: " + response.getSummary());
50+
} catch (Exception e) {
51+
log.log(Level.SEVERE, "Failed to write data", e);
52+
throw new RuntimeException(e);
53+
}
54+
}
55+
56+
public List<Message> read() {
57+
try (ClickHouseClient client = getClient()) {
58+
ClickHouseRequest request =
59+
client.read(ClickHouseNode.builder().host(host).port(ClickHouseProtocol.HTTP, port).build());
60+
61+
request.table("default.ui_events");
62+
request.format(ClickHouseFormat.Protobuf);
63+
request.set("format_protobuf_use_autogenerated_schema", 1); // use the schema generated by ClickHouse
64+
65+
ClickHouseResponse response = request.executeAndWait();
66+
List<Message> messages = new ArrayList<>();
67+
ClickHouseInputStream inputStream = response.getInputStream();
68+
while (inputStream.available() > 0) {
69+
messages.add(UIEvent.parseDelimitedFrom(inputStream));
70+
}
71+
72+
return messages;
73+
} catch (Exception e) {
74+
log.log(Level.SEVERE, "Failed to write data", e);
75+
throw new RuntimeException(e);
76+
}
77+
}
78+
79+
80+
public static void main(String[] args) {
81+
82+
List<Message> messages = new ArrayList<>();
83+
84+
messages.add(UIEvent.newBuilder()
85+
.setUrl("http://example.com")
86+
.setUserId("user1")
87+
.setSessionId("session1")
88+
.setTimestamp(System.currentTimeMillis())
89+
.setEvent("visit")
90+
.build());
91+
92+
messages.add(UIEvent.newBuilder()
93+
.setUrl("http://example.com")
94+
.setUserId("user1")
95+
.setSessionId("session1")
96+
.setTimestamp(System.currentTimeMillis())
97+
.setDuration(1000)
98+
.setEvent("leave")
99+
.build());
100+
101+
102+
ProtobufMain writer = new ProtobufMain();
103+
writer.write(messages);
104+
105+
List<Message> storedMessages = writer.read();
106+
for (Message message : storedMessages) {
107+
log.info("Message: " + message);
108+
}
109+
}
110+
111+
protected ClickHouseClient getClient() {
112+
Map<ClickHouseOption, Serializable> map = new HashMap<>();
113+
map.put(ClickHouseHttpOption.CUSTOM_PARAMS, "format_protobuf_use_autogenerated_schema=1");
114+
return ClickHouseClient.builder().config(new ClickHouseConfig(map))
115+
.nodeSelector(ClickHouseNodeSelector.of(ClickHouseProtocol.HTTP))
116+
.build();
117+
}
118+
}

0 commit comments

Comments
 (0)