Skip to content

Commit b38866e

Browse files
chernserchernser
authored
Merge pull request #1685 from ClickHouse/feat_client_protobuf_example
Example of using Protobuf format with the client
2 parents c3029d4 + 999c070 commit b38866e

File tree

7 files changed

+1390
-2
lines changed

7 files changed

+1390
-2
lines changed

‎examples/client/README.md‎

Lines changed: 52 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -15,11 +15,61 @@ mvn clean compile
1515

1616
To run:
1717
```shell
18-
mvn exec:java -Dexec.mainClass="com.clickhouse.examples.jdbc.Main"
18+
mvn exec:java -Dexec.mainClass="com.clickhouse.examples.jdbc.Main"
1919
```
2020

2121
Addition options can be passed to the application:
2222
- `-DchPort` - ClickHouse server port (default: 8123)
2323
- `-DchUser` - ClickHouse user name (default: default)
2424
- `-DchPassword` - ClickHouse user password (default: empty)
25-
- `-DchDatabase` - ClickHouse database name (default: default)
25+
- `-DchDatabase` - ClickHouse database name (default: default)
26+
27+
28+
## Protobuf Example
29+
30+
Class `com.clickhouse.examples.formats.ProtobufMain` demonstrates how to use Protobuf with ClickHouse.
31+
32+
### Setup
33+
34+
First of all new table should be created in a DB:
35+
```sql
36+
CREATE TABLE ui_events
37+
(
38+
`url` String,
39+
`user_id` String,
40+
`session_id` String,
41+
`timestamp` Int64,
42+
`duration` Int64 DEFAULT 0,
43+
`event` String,
44+
)
45+
ENGINE = MergeTree
46+
ORDER BY timestamp;
47+
```
48+
49+
To regenerate Java classes from Protobuf file:
50+
```shell
51+
protoc --java_out=./src/main/java/ ./src/proto/ui_stats_event.proto
52+
```
53+
54+
To compile:
55+
```shell
56+
mvn clean compile
57+
```
58+
59+
To run:
60+
```shell
61+
mvn exec:java -Dexec.mainClass="com.clickhouse.examples.formats.ProtobufMain"
62+
```
63+
64+
To run with a snapshot version of ClickHouse Java client (replace `0.6.0` with the actual version and use repository `https://oss.sonatype.org/content/repositories/snapshots`)
65+
```shell
66+
mvn exec:java -Dclickhouse-java.version=0.6.0-SNAPSHOT -Dexec.mainClass="com.clickhouse.examples.formats.ProtobufMain"
67+
```
68+
69+
70+
Addition options can be passed to the application:
71+
- `-DchPort` - ClickHouse server port (default: 8123)
72+
- `-DchSsl` - Set to `true` to use https (default: false)
73+
- `-DchUser` - ClickHouse user name (default: default)
74+
- `-DchPassword` - ClickHouse user password (default: empty)
75+
- `-DchDatabase` - ClickHouse database name (default: default)

‎examples/client/pom.xml‎

Lines changed: 17 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -67,6 +67,9 @@
6767
<project.reporting.outputEncoding>UTF-8</project.reporting.outputEncoding>
6868

6969
<clickhouse-java.version>0.6.0</clickhouse-java.version>
70+
<!-- Nightly snapshot version from https://s01.oss.sonatype.org/content/repositories/snapshots/ or latest from local -->
71+
<!-- <clickhouse-java.version>0.6.0-SNAPSHOT</clickhouse-java.version>-->
72+
7073
<apache-httpclient.version>5.2.1</apache-httpclient.version>
7174

7275
<compiler-plugin.version>3.8.1</compiler-plugin.version>
@@ -98,6 +101,20 @@
98101
<artifactId>httpclient5</artifactId>
99102
<version>${apache-httpclient.version}</version>
100103
</dependency>
104+
105+
<!-- Required for Protobuf example -->
106+
<dependency>
107+
<groupId>com.google.protobuf</groupId>
108+
<artifactId>protobuf-java</artifactId>
109+
<version>4.27.1</version>
110+
</dependency>
111+
112+
<dependency>
113+
<groupId>com.google.protobuf</groupId>
114+
<artifactId>protobuf-java-util</artifactId>
115+
<version>4.27.1</version>
116+
</dependency>
117+
101118
</dependencies>
102119

103120
<build>

‎examples/client/src/main/java/com/clickhouse/examples/formats/ProtobufMain.java‎

Lines changed: 129 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,129 @@
1+
package com.clickhouse.examples.formats;
2+
3+
import com.clickhouse.client.ClickHouseClient;
4+
import com.clickhouse.client.ClickHouseConfig;
5+
import com.clickhouse.client.ClickHouseCredentials;
6+
import com.clickhouse.client.ClickHouseException;
7+
import com.clickhouse.client.ClickHouseNode;
8+
import com.clickhouse.client.ClickHouseNodeSelector;
9+
import com.clickhouse.client.ClickHouseProtocol;
10+
import com.clickhouse.client.ClickHouseRequest;
11+
import com.clickhouse.client.ClickHouseResponse;
12+
import com.clickhouse.client.config.ClickHouseClientOption;
13+
import com.clickhouse.client.http.config.ClickHouseHttpOption;
14+
import com.clickhouse.config.ClickHouseOption;
15+
import com.clickhouse.data.ClickHouseFormat;
16+
import com.clickhouse.data.ClickHouseInputStream;
17+
import com.clickhouse.examples.protos.UIEvent;
18+
import com.google.protobuf.Message;
19+
20+
21+
import java.io.Serializable;
22+
import java.util.ArrayList;
23+
import java.util.HashMap;
24+
import java.util.List;
25+
import java.util.Map;
26+
import java.util.logging.Level;
27+
import java.util.logging.Logger;
28+
29+
30+
public class ProtobufMain {
31+
32+
private static final Logger log = Logger.getLogger(ProtobufMain.class.getName());
33+
34+
public void write(List<Message> messages) {
35+
try (ClickHouseClient client = getClient()) {
36+
ClickHouseRequest.Mutation mutation = client.write(getServer());
37+
mutation.table("default.ui_events");
38+
mutation.format(ClickHouseFormat.Protobuf);
39+
ClickHouseResponse response = mutation.data((out) -> {
40+
for (Message message : messages) {
41+
// it is important to write the size of the message before the message itself
42+
out.writeVarInt(message.getSerializedSize());
43+
message.writeTo(out);
44+
}
45+
}).executeAndWait();
46+
47+
48+
log.info("Response: " + response.getSummary());
49+
} catch (Exception e) {
50+
log.log(Level.SEVERE, "Failed to write data", e);
51+
throw new RuntimeException(e);
52+
}
53+
}
54+
55+
public List<Message> read() {
56+
try (ClickHouseClient client = getClient()) {
57+
ClickHouseRequest request = client.read(getServer());
58+
59+
request.table("default.ui_events");
60+
request.format(ClickHouseFormat.Protobuf);
61+
request.set("format_protobuf_use_autogenerated_schema", 1); // use the schema generated by ClickHouse
62+
63+
ClickHouseResponse response = request.executeAndWait();
64+
List<Message> messages = new ArrayList<>();
65+
ClickHouseInputStream inputStream = response.getInputStream();
66+
while (inputStream.available() > 0) {
67+
messages.add(UIEvent.parseDelimitedFrom(inputStream));
68+
}
69+
70+
return messages;
71+
} catch (ClickHouseException e) {
72+
log.log(Level.SEVERE, "Failed to read data from server: " + getClient().getConfig(), e);
73+
throw new RuntimeException(e);
74+
} catch (Exception e) {
75+
log.log(Level.SEVERE, "Failed to write data", e);
76+
throw new RuntimeException(e);
77+
}
78+
}
79+
80+
81+
public static void main(String[] args) {
82+
83+
List<Message> messages = new ArrayList<>();
84+
85+
messages.add(UIEvent.newBuilder()
86+
.setUrl("http://example.com")
87+
.setUserId("user1")
88+
.setSessionId("session1")
89+
.setTimestamp(System.currentTimeMillis())
90+
.setEvent("visit")
91+
.build());
92+
93+
messages.add(UIEvent.newBuilder()
94+
.setUrl("http://example.com")
95+
.setUserId("user1")
96+
.setSessionId("session1")
97+
.setTimestamp(System.currentTimeMillis())
98+
.setDuration(1000)
99+
.setEvent("leave")
100+
.build());
101+
102+
103+
ProtobufMain writer = new ProtobufMain();
104+
writer.write(messages);
105+
106+
List<Message> storedMessages = writer.read();
107+
for (Message message : storedMessages) {
108+
log.info("Message: " + message);
109+
}
110+
}
111+
112+
protected ClickHouseClient getClient() {
113+
Map<ClickHouseOption, Serializable> map = new HashMap<>();
114+
map.put(ClickHouseHttpOption.CUSTOM_PARAMS, "format_protobuf_use_autogenerated_schema=1");
115+
return ClickHouseClient.builder().config(new ClickHouseConfig(map))
116+
.nodeSelector(ClickHouseNodeSelector.of(ClickHouseProtocol.HTTP))
117+
.build();
118+
}
119+
120+
protected ClickHouseNode getServer() {
121+
return ClickHouseNode.builder()
122+
.host(System.getProperty("chHost", "localhost"))
123+
.addOption(ClickHouseClientOption.SSL.getKey(), String.valueOf(Boolean.getBoolean("chSsl")))
124+
.port(ClickHouseProtocol.HTTP, Integer.getInteger("chPort", 8123))
125+
.credentials(ClickHouseCredentials.fromUserAndPassword(
126+
System.getProperty("chUser", "default"), System.getProperty("chPassword", "")))
127+
.build();
128+
}
129+
}

0 commit comments

Comments
 (0)