Skip to content

Commit fb18dc8

Browse files
chernserchernser
committed
implemented simple JSON support + example
1 parent 12191bd commit fb18dc8

File tree

8 files changed

+239
-11
lines changed

8 files changed

+239
-11
lines changed

‎client-v2/src/main/java/com/clickhouse/client/api/ClientSettings.java‎

Lines changed: 12 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -39,4 +39,16 @@ public static List<String> valuesFromCommaSeparated(String value) {
3939
public static final String SETTING_LOG_COMMENT = SERVER_SETTING_PREFIX + "log_comment";
4040

4141
public static final String HTTP_USE_BASIC_AUTH = "http_use_basic_auth";
42+
43+
// -- Experimental features --
44+
45+
/**
46+
* Server will expect a string in JSON format and parse it into a JSON object.
47+
*/
48+
public static final String INPUT_FORMAT_BINARY_READ_JSON_AS_STRING = "input_format_binary_read_json_as_string";
49+
50+
/**
51+
* Server will return a JSON object as a string.
52+
*/
53+
public static final String OUTPUT_FORMAT_BINARY_WRITE_JSON_AS_STRING = "output_format_binary_write_json_as_string";
4254
}

‎client-v2/src/main/java/com/clickhouse/client/api/data_formats/internal/SerializerUtils.java‎

Lines changed: 11 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -205,11 +205,22 @@ private static void serializePrimitiveData(OutputStream stream, Object value, Cl
205205
case IPv6:
206206
BinaryStreamUtils.writeInet6Address(stream, (Inet6Address) value);
207207
break;
208+
case JSON:
209+
serializeJSON(stream, value);
210+
break;
208211
default:
209212
throw new UnsupportedOperationException("Unsupported data type: " + column.getDataType());
210213
}
211214
}
212215

216+
private static void serializeJSON(OutputStream stream, Object value) throws IOException {
217+
if (value instanceof String) {
218+
BinaryStreamUtils.writeString(stream, (String)value);
219+
} else {
220+
throw new UnsupportedOperationException("Serialization of Java object to JSON is not supported yet.");
221+
}
222+
}
223+
213224
private static void serializeAggregateFunction(OutputStream stream, Object value, ClickHouseColumn column) throws IOException {
214225
if (column.getAggregateFunction() == ClickHouseAggregateFunction.groupBitmap) {
215226
if (value == null) {

‎client-v2/src/test/java/com/clickhouse/client/insert/InsertTests.java‎

Lines changed: 46 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -9,7 +9,9 @@
99
import com.clickhouse.client.ClickHouseProtocol;
1010
import com.clickhouse.client.api.Client;
1111
import com.clickhouse.client.api.ClientException;
12+
import com.clickhouse.client.api.ClientSettings;
1213
import com.clickhouse.client.api.command.CommandResponse;
14+
import com.clickhouse.client.api.command.CommandSettings;
1315
import com.clickhouse.client.api.data_formats.ClickHouseBinaryFormatReader;
1416
import com.clickhouse.client.api.enums.Protocol;
1517
import com.clickhouse.client.api.insert.InsertResponse;
@@ -19,19 +21,25 @@
1921
import com.clickhouse.client.api.metrics.ServerMetrics;
2022
import com.clickhouse.client.api.query.GenericRecord;
2123
import com.clickhouse.client.api.query.QueryResponse;
24+
import com.clickhouse.client.api.query.QuerySettings;
2225
import com.clickhouse.data.ClickHouseFormat;
26+
import com.clickhouse.data.ClickHouseVersion;
27+
import org.apache.commons.lang3.StringEscapeUtils;
2328
import org.testcontainers.shaded.org.apache.commons.lang3.RandomStringUtils;
2429
import org.testng.Assert;
2530
import org.testng.annotations.AfterMethod;
2631
import org.testng.annotations.BeforeMethod;
2732
import org.testng.annotations.DataProvider;
2833
import org.testng.annotations.Test;
2934

35+
import java.io.BufferedReader;
3036
import java.io.ByteArrayInputStream;
3137
import java.io.ByteArrayOutputStream;
3238
import java.io.IOException;
39+
import java.io.InputStreamReader;
3340
import java.io.PrintWriter;
3441
import java.util.ArrayList;
42+
import java.util.Arrays;
3543
import java.util.Collections;
3644
import java.util.List;
3745
import java.util.UUID;
@@ -121,6 +129,44 @@ public void insertSimplePOJOs() throws Exception {
121129
assertEquals(response.getQueryId(), uuid);
122130
}
123131

132+
133+
@Test(groups = { "integration" }, enabled = true)
134+
public void insertPOJOWithJSON() throws Exception {
135+
List<GenericRecord> serverVersion = client.queryAll("SELECT version()");
136+
if (ClickHouseVersion.of(serverVersion.get(0).getString(1)).check("(,24.8]")) {
137+
System.out.println("Test is skipped: feature is supported since 24.8");
138+
return;
139+
}
140+
141+
final String tableName = "pojo_with_json_table";
142+
final String createSQL = PojoWithJSON.createTable(tableName);
143+
final String originalJsonStr = "{\"a\":{\"b\":\"42\"},\"c\":[\"1\",\"2\",\"3\"]}";
144+
145+
146+
CommandSettings commandSettings = new CommandSettings();
147+
commandSettings.serverSetting("allow_experimental_json_type", "1");
148+
client.execute("DROP TABLE IF EXISTS " + tableName, commandSettings).get(1, TimeUnit.SECONDS);
149+
client.execute(createSQL, commandSettings).get(1, TimeUnit.SECONDS);
150+
151+
client.register(PojoWithJSON.class, client.getTableSchema(tableName, "default"));
152+
PojoWithJSON pojo = new PojoWithJSON();
153+
pojo.setEventPayload(originalJsonStr);
154+
List<Object> data = Arrays.asList(pojo);
155+
156+
InsertSettings insertSettings = new InsertSettings()
157+
.serverSetting(ClientSettings.INPUT_FORMAT_BINARY_READ_JSON_AS_STRING, "1");
158+
InsertResponse response = client.insert(tableName, data, insertSettings).get(30, TimeUnit.SECONDS);
159+
assertEquals(response.getWrittenRows(), 1);
160+
161+
QuerySettings settings = new QuerySettings()
162+
.setFormat(ClickHouseFormat.CSV);
163+
try (QueryResponse resp = client.query("SELECT * FROM " + tableName, settings).get(1, TimeUnit.SECONDS)) {
164+
BufferedReader reader = new BufferedReader(new InputStreamReader(resp.getInputStream()));
165+
String jsonStr = StringEscapeUtils.unescapeCsv(reader.lines().findFirst().get());
166+
Assert.assertEquals(jsonStr, originalJsonStr);
167+
}
168+
}
169+
124170
@Test(groups = { "integration" }, enabled = true)
125171
public void insertPOJOAndReadBack() throws Exception {
126172
final String tableName = "single_pojo_table";

‎client-v2/src/test/java/com/clickhouse/client/insert/PojoWithJSON.java‎

Lines changed: 41 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,41 @@
1+
package com.clickhouse.client.insert;
2+
3+
import java.util.Objects;
4+
5+
public class PojoWithJSON {
6+
7+
// This field is a string representation of a JSON object
8+
private String eventPayload;
9+
10+
public String getEventPayload() {
11+
return eventPayload;
12+
}
13+
14+
public void setEventPayload(String eventPayload) {
15+
this.eventPayload = eventPayload;
16+
}
17+
18+
@Override
19+
public boolean equals(Object o) {
20+
if (this == o) return true;
21+
if (o == null || getClass() != o.getClass()) return false;
22+
PojoWithJSON that = (PojoWithJSON) o;
23+
return Objects.equals(eventPayload, that.eventPayload);
24+
}
25+
26+
@Override
27+
public int hashCode() {
28+
return Objects.hash(eventPayload);
29+
}
30+
31+
@Override
32+
public String toString() {
33+
return "PojoWithJSON{" +
34+
"eventPayload='" + eventPayload + '\'' +
35+
'}';
36+
}
37+
38+
public static String createTable(String tableName) {
39+
return "CREATE TABLE " + tableName + " (eventPayload JSON) ENGINE = MergeTree() ORDER BY tuple()";
40+
}
41+
}

‎client-v2/src/test/java/com/clickhouse/client/query/QueryTests.java‎

Lines changed: 2 additions & 11 deletions
Original file line numberDiff line numberDiff line change
@@ -18,7 +18,6 @@
1818
import com.clickhouse.client.api.command.CommandResponse;
1919
import com.clickhouse.client.api.command.CommandSettings;
2020
import com.clickhouse.client.api.data_formats.ClickHouseBinaryFormatReader;
21-
import com.clickhouse.client.api.data_formats.internal.BinaryStreamReader;
2221
import com.clickhouse.client.api.enums.Protocol;
2322
import com.clickhouse.client.api.insert.InsertResponse;
2423
import com.clickhouse.client.api.insert.InsertSettings;
@@ -31,8 +30,6 @@
3130
import com.clickhouse.client.api.query.QueryResponse;
3231
import com.clickhouse.client.api.query.QuerySettings;
3332
import com.clickhouse.client.api.query.Records;
34-
import com.clickhouse.client.http.config.HttpConnectionProvider;
35-
import com.clickhouse.client.insert.SamplePOJO;
3633
import com.clickhouse.data.ClickHouseDataType;
3734
import com.clickhouse.data.ClickHouseFormat;
3835
import com.clickhouse.data.ClickHouseVersion;
@@ -59,15 +56,13 @@
5956
import java.net.Inet4Address;
6057
import java.net.Inet6Address;
6158
import java.net.InetAddress;
62-
import java.nio.file.Files;
6359
import java.time.LocalDate;
6460
import java.time.LocalDateTime;
6561
import java.time.ZoneId;
6662
import java.time.ZonedDateTime;
6763
import java.util.ArrayList;
6864
import java.util.Arrays;
6965
import java.util.Collections;
70-
import java.util.Comparator;
7166
import java.util.HashMap;
7267
import java.util.HashSet;
7368
import java.util.Iterator;
@@ -76,7 +71,6 @@
7671
import java.util.Random;
7772
import java.util.Set;
7873
import java.util.UUID;
79-
import java.util.concurrent.CompletableFuture;
8074
import java.util.concurrent.CountDownLatch;
8175
import java.util.concurrent.ExecutionException;
8276
import java.util.concurrent.ExecutorService;
@@ -1808,17 +1802,14 @@ public void testReadingJSONValues() throws Exception {
18081802
client.execute("INSERT INTO test_json_values VALUES ('{\"a\" : {\"b\" : 42}, \"c\" : [1, 2, 3]}')", commandSettings).get(1, TimeUnit.SECONDS);
18091803

18101804

1811-
QuerySettings settings = new QuerySettings()
1812-
.serverSetting("allow_experimental_json_type", "1")
1813-
.setFormat(ClickHouseFormat.CSV);
1805+
QuerySettings settings = new QuerySettings().setFormat(ClickHouseFormat.CSV);
18141806
try (QueryResponse resp = client.query("SELECT json FROM test_json_values", settings).get(1, TimeUnit.SECONDS)) {
18151807
BufferedReader reader = new BufferedReader(new InputStreamReader(resp.getInputStream()));
18161808
Assert.assertEquals(StringEscapeUtils.unescapeCsv(reader.lines().findFirst().get()), "{\"a\":{\"b\":\"42\"},\"c\":[\"1\",\"2\",\"3\"]}");
18171809
}
18181810

18191811
settings = new QuerySettings()
1820-
.serverSetting("allow_experimental_json_type", "1")
1821-
.serverSetting("output_format_binary_write_json_as_string", "1");
1812+
.serverSetting(ClientSettings.OUTPUT_FORMAT_BINARY_WRITE_JSON_AS_STRING, "1");
18221813
try (QueryResponse resp = client.query("SELECT json FROM test_json_values", settings).get(1, TimeUnit.SECONDS)) {
18231814
ClickHouseBinaryFormatReader reader = client.newBinaryFormatReader(resp);
18241815
Assert.assertNotNull(reader.next());

‎examples/client-v2/src/main/java/com/clickhouse/examples/client_v2/ExperimentalJSONExample.java‎

Lines changed: 81 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,81 @@
1+
package com.clickhouse.examples.client_v2;
2+
3+
import com.clickhouse.client.api.Client;
4+
import com.clickhouse.client.api.ClientSettings;
5+
import com.clickhouse.client.api.command.CommandSettings;
6+
import com.clickhouse.client.api.data_formats.ClickHouseBinaryFormatReader;
7+
import com.clickhouse.client.api.insert.InsertResponse;
8+
import com.clickhouse.client.api.insert.InsertSettings;
9+
import com.clickhouse.client.api.query.QueryResponse;
10+
import com.clickhouse.examples.client_v2.data.PojoWithJSON;
11+
import lombok.extern.slf4j.Slf4j;
12+
13+
import java.util.Arrays;
14+
import java.util.List;
15+
import java.util.concurrent.TimeUnit;
16+
17+
@Slf4j
18+
public class ExperimentalJSONExample {
19+
20+
Client client;
21+
22+
public ExperimentalJSONExample(String endpoint, String user, String password, String database) {
23+
// Create a lightweight object to interact with ClickHouse server
24+
Client.Builder clientBuilder = new Client.Builder()
25+
.addEndpoint(endpoint)
26+
.setUsername(user)
27+
.setPassword(password)
28+
.compressServerResponse(true)
29+
// allow experimental JSON type
30+
.serverSetting("allow_experimental_json_type", "1")
31+
// allow JSON transcoding as a string
32+
.serverSetting(ClientSettings.INPUT_FORMAT_BINARY_READ_JSON_AS_STRING, "1")
33+
.serverSetting(ClientSettings.OUTPUT_FORMAT_BINARY_WRITE_JSON_AS_STRING, "1")
34+
.setDefaultDatabase(database);
35+
36+
this.client = clientBuilder.build();
37+
}
38+
39+
final String tableName = "pojo_with_json_table";
40+
final String createSQL = PojoWithJSON.createTable(tableName);
41+
final String originalJsonStr = "{\"a\":{\"b\":\"42\"},\"c\":[\"1\",\"2\",\"3\"]}";
42+
43+
44+
public void writeData() {
45+
CommandSettings commandSettings = new CommandSettings();
46+
commandSettings.serverSetting("allow_experimental_json_type", "1");
47+
48+
try {
49+
client.execute("DROP TABLE IF EXISTS " + tableName, commandSettings).get(1, TimeUnit.SECONDS);
50+
client.execute(createSQL, commandSettings).get(1, TimeUnit.SECONDS);
51+
} catch (Exception e) {
52+
throw new RuntimeException(e);
53+
}
54+
55+
client.register(PojoWithJSON.class, client.getTableSchema(tableName, "default"));
56+
PojoWithJSON pojo = new PojoWithJSON();
57+
pojo.setEventPayload(originalJsonStr);
58+
List<Object> data = Arrays.asList(pojo);
59+
60+
InsertSettings insertSettings = new InsertSettings()
61+
.serverSetting(ClientSettings.INPUT_FORMAT_BINARY_READ_JSON_AS_STRING, "1");
62+
try (InsertResponse response = client.insert(tableName, data, insertSettings).get(30, TimeUnit.SECONDS)) {
63+
log.info("Data write metrics: {}", response.getMetrics());
64+
} catch (Exception e) {
65+
throw new RuntimeException(e);
66+
}
67+
68+
}
69+
70+
public void readData() {
71+
try (QueryResponse resp = client.query("SELECT * FROM " + tableName).get(1, TimeUnit.SECONDS)) {
72+
ClickHouseBinaryFormatReader reader = client.newBinaryFormatReader(resp);
73+
assert reader.next() != null;
74+
String jsonStr = reader.getString(1);
75+
log.info("Read JSON string: {}", jsonStr);
76+
} catch (Exception e) {
77+
throw new RuntimeException(e);
78+
}
79+
}
80+
81+
}

‎examples/client-v2/src/main/java/com/clickhouse/examples/client_v2/Main.java‎

Lines changed: 5 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -59,6 +59,11 @@ public static void main(String[] args) {
5959

6060
pojoWriter.printLastEvents();
6161

62+
// Insert data using POJO with JSON
63+
ExperimentalJSONExample jsonExample = new ExperimentalJSONExample(endpoint, user, password, database);
64+
jsonExample.writeData();
65+
jsonExample.readData();
66+
6267
log.info("Done");
6368
Runtime.getRuntime().exit(0);
6469
}

‎examples/client-v2/src/main/java/com/clickhouse/examples/client_v2/data/PojoWithJSON.java‎

Lines changed: 41 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,41 @@
1+
package com.clickhouse.examples.client_v2.data;
2+
3+
import java.util.Objects;
4+
5+
public class PojoWithJSON {
6+
7+
// This field is a string representation of a JSON object
8+
private String eventPayload;
9+
10+
public String getEventPayload() {
11+
return eventPayload;
12+
}
13+
14+
public void setEventPayload(String eventPayload) {
15+
this.eventPayload = eventPayload;
16+
}
17+
18+
@Override
19+
public boolean equals(Object o) {
20+
if (this == o) return true;
21+
if (o == null || getClass() != o.getClass()) return false;
22+
PojoWithJSON that = (PojoWithJSON) o;
23+
return Objects.equals(eventPayload, that.eventPayload);
24+
}
25+
26+
@Override
27+
public int hashCode() {
28+
return Objects.hash(eventPayload);
29+
}
30+
31+
@Override
32+
public String toString() {
33+
return "PojoWithJSON{" +
34+
"eventPayload='" + eventPayload + '\'' +
35+
'}';
36+
}
37+
38+
public static String createTable(String tableName) {
39+
return "CREATE TABLE " + tableName + " (eventPayload JSON) ENGINE = MergeTree() ORDER BY tuple()";
40+
}
41+
}

0 commit comments

Comments
 (0)