Skip to content

Commit 7ac7a55

Browse files
chernserchernser
committed
Merge branch 'main' into jdbc_fix_metadata_resultset
2 parents 68c7fb3 + 3d12e6e commit 7ac7a55

File tree

22 files changed

+738
-246
lines changed

22 files changed

+738
-246
lines changed

‎.github/workflows/benchmarks.yml‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -24,7 +24,7 @@ jobs:
2424
if: ${{ startsWith(github.repository, 'ClickHouse/') }}
2525
name: "Mininal JMH Benchmarks"
2626
runs-on: "ubuntu-latest"
27-
timeout-minutes: 20
27+
timeout-minutes: 30
2828
steps:
2929
- name: Check out Git repository
3030
uses: actions/checkout@v4

‎clickhouse-data/src/main/java/com/clickhouse/data/ClickHouseColumn.java‎

Lines changed: 72 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -42,7 +42,6 @@
4242
import java.time.OffsetDateTime;
4343
import java.util.ArrayList;
4444
import java.util.Arrays;
45-
import java.util.Collection;
4645
import java.util.Collections;
4746
import java.util.Comparator;
4847
import java.util.HashMap;
@@ -52,6 +51,7 @@
5251
import java.util.Objects;
5352
import java.util.Set;
5453
import java.util.TimeZone;
54+
import java.util.stream.Collectors;
5555

5656
/**
5757
* This class represents a column defined in database.
@@ -73,6 +73,7 @@ public final class ClickHouseColumn implements Serializable {
7373
private static final String KEYWORD_MAP = ClickHouseDataType.Map.name();
7474
private static final String KEYWORD_NESTED = ClickHouseDataType.Nested.name();
7575
private static final String KEYWORD_VARIANT = ClickHouseDataType.Variant.name();
76+
private static final String KEYWORD_JSON = ClickHouseDataType.JSON.name();
7677

7778
private int columnCount;
7879
private int columnIndex;
@@ -91,6 +92,7 @@ public final class ClickHouseColumn implements Serializable {
9192
private List<ClickHouseColumn> nested;
9293
private List<String> parameters;
9394
private ClickHouseEnum enumConstants;
95+
private Map<String, ClickHouseColumn> jsonPredefinedPaths;
9496
private ValueFunction valueFunction;
9597

9698
private int arrayLevel;
@@ -506,6 +508,23 @@ protected static int readColumn(String args, int startIndex, int len, String nam
506508
}
507509
}
508510
}
511+
} else if (args.startsWith(KEYWORD_JSON, i)) {
512+
int index = args.indexOf('(', i + KEYWORD_JSON.length());
513+
if (index > i) {
514+
i = ClickHouseUtils.skipBrackets(args, index, len, '(');
515+
String originalTypeName = args.substring(startIndex, i);
516+
List<ClickHouseColumn> nestedColumns = new ArrayList<>();
517+
518+
List<String> parameters = new ArrayList<>();
519+
parseJSONColumn(args.substring(index + 1, i - 1), nestedColumns, parameters);
520+
nestedColumns.sort(Comparator.comparing(o -> o.getDataType().name()));
521+
column = new ClickHouseColumn(ClickHouseDataType.JSON, name, originalTypeName, nullable, lowCardinality,
522+
parameters, nestedColumns);
523+
column.jsonPredefinedPaths = nestedColumns.stream().collect(Collectors.toMap(ClickHouseColumn::getColumnName,
524+
c -> c));
525+
fixedLength = false;
526+
estimatedLength++;
527+
}
509528
}
510529

511530
if (column == null) {
@@ -660,6 +679,54 @@ public static List<ClickHouseColumn> parse(String args) {
660679
return Collections.unmodifiableList(c);
661680
}
662681

682+
public static final String JSON_MAX_PATHS_PARAM = "max_dynamic_paths";
683+
public static final String JSON_MAX_DYN_TYPES_PARAM = "max_dynamic_types";
684+
public static final String JSON_SKIP_MARKER = "SKIP";
685+
686+
public static void parseJSONColumn(String args, List<ClickHouseColumn> nestedColumns, List<String> parameters) {
687+
if (args == null || args.isEmpty()) {
688+
return;
689+
}
690+
691+
String name = null;
692+
ClickHouseColumn column = null;
693+
StringBuilder builder = new StringBuilder();
694+
int i =0;
695+
int len = args.length();
696+
while (i < len) {
697+
char ch = args.charAt(i);
698+
if (Character.isWhitespace(ch)) {
699+
i++;
700+
continue;
701+
}
702+
703+
if (name == null) { // column name
704+
i = ClickHouseUtils.readNameOrQuotedString(args, i, len, builder) - 1;
705+
name = builder.toString();
706+
if (name.startsWith(JSON_SKIP_MARKER)) {
707+
name = null; // skip parameters
708+
i = ClickHouseUtils.skipContentsUntil(args, i, len, ',') - 1;
709+
} else if ( name.startsWith(JSON_MAX_PATHS_PARAM) || name.startsWith(JSON_MAX_DYN_TYPES_PARAM)) {
710+
parameters.add(name);
711+
name = null;
712+
i = ClickHouseUtils.skipContentsUntil(args, i, len, ',') - 1;
713+
}
714+
builder.setLength(0);
715+
} else if (column == null) { // now type
716+
LinkedList<ClickHouseColumn> colList = new LinkedList<>();
717+
i = readColumn(args, i, len, name, colList) - 1;
718+
column = colList.getFirst();
719+
nestedColumns.add(column);
720+
} else { // prepare for next column
721+
i = ClickHouseUtils.skipContentsUntil(args, i, len, ',') - 1;
722+
name = null;
723+
column = null;
724+
}
725+
726+
i++;
727+
}
728+
}
729+
663730
public ClickHouseColumn(ClickHouseDataType dataType, String columnName, String originalTypeName, boolean nullable,
664731
boolean lowCardinality, List<String> parameters, List<ClickHouseColumn> nestedColumns) {
665732
this(dataType, columnName, originalTypeName, nullable, lowCardinality, parameters, nestedColumns, ClickHouseEnum.EMPTY);
@@ -968,6 +1035,10 @@ public ClickHouseAggregateFunction getAggregateFunction() {
9681035
return aggFuncType;
9691036
}
9701037

1038+
public Map<String, ClickHouseColumn> getJsonPredefinedPaths() {
1039+
return jsonPredefinedPaths;
1040+
}
1041+
9711042
public ClickHouseArraySequence newArrayValue(ClickHouseDataConfig config) {
9721043
int level = arrayLevel;
9731044
ClickHouseArraySequence value;

‎clickhouse-data/src/main/java/com/clickhouse/data/ClickHouseUtils.java‎

Lines changed: 0 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -51,7 +51,6 @@
5151
import java.util.function.Supplier;
5252
import java.util.function.UnaryOperator;
5353

54-
@Deprecated
5554
public final class ClickHouseUtils {
5655
private static final boolean IS_UNIX;
5756
private static final boolean IS_WINDOWS;

‎clickhouse-data/src/test/java/com/clickhouse/data/ClickHouseColumnTest.java‎

Lines changed: 24 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -1,9 +1,11 @@
11
package com.clickhouse.data;
22

33
import java.math.BigInteger;
4+
import java.util.Arrays;
45
import java.util.Collections;
56
import java.util.LinkedList;
67
import java.util.List;
8+
import java.util.Map;
79

810
import org.testng.Assert;
911
import org.testng.annotations.DataProvider;
@@ -441,4 +443,26 @@ public boolean isWidenUnsignedTypes() {
441443
}
442444
}
443445
}
446+
447+
@Test(groups = {"unit"}, dataProvider = "testJSONBinaryFormat_dp")
448+
public void testJSONBinaryFormat(String jsonDef, int params, List<String> predefinedPaths) throws Exception {
449+
ClickHouseColumn column = ClickHouseColumn.of("v", jsonDef);
450+
Assert.assertEquals(column.getNestedColumns().size(), predefinedPaths.size(), "predefined paths count mismatch");
451+
Assert.assertEquals(column.getParameters().size(), params, "parameters count mismatch");
452+
}
453+
454+
@DataProvider
455+
public Object[][] testJSONBinaryFormat_dp() {
456+
457+
return new Object[][] {
458+
{"JSON", 0, Collections.emptyList()},
459+
{"JSON()", 0, Collections.emptyList()},
460+
{"JSON(stat.name String, count Int32)", 0, Arrays.asList("stat.name", "count")},
461+
{"JSON(stat.name String, `comments` String)", 0, Arrays.asList("stat.name", "comments")},
462+
{"JSON(max_dynamic_paths=3, stat.name String, count Int8, SKIP alt_count)", 1, Arrays.asList("stat.name", "count")},
463+
{"JSON(max_dynamic_paths=3, stat.name String, SKIP REGEXP '^-.*')", 1, Arrays.asList("stat.name")},
464+
{"JSON(max_dynamic_paths=3,SKIP REGEXP '^-.*',SKIP ff, flags Array(Array(Array(Int8))), SKIP alt_count)", 1, Arrays.asList("flags")},
465+
{"JSON(max_dynamic_types=3,max_dynamic_paths=3, SKIP REGEXP '^-.*',SKIP ff, flags Array(Array(Array(Int8))), SKIP alt_count)", 2, Arrays.asList("flags")},
466+
};
467+
}
444468
}

‎clickhouse-jdbc/pom.xml‎

Lines changed: 13 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -238,7 +238,7 @@
238238
<path>
239239
<groupId>org.projectlombok</groupId>
240240
<artifactId>lombok</artifactId>
241-
<version>1.18.32</version>
241+
<version>1.18.38</version>
242242
</path>
243243
<path>
244244
<groupId>org.openjdk.jmh</groupId>
@@ -363,17 +363,29 @@
363363
<groupId>org.apache.maven.plugins</groupId>
364364
<artifactId>maven-assembly-plugin</artifactId>
365365
<configuration>
366+
<finalName>${project.artifactId}-${project.version}-all-dependencies</finalName>
367+
<appendAssemblyId>false</appendAssemblyId>
366368
<descriptorRefs>
367369
<descriptorRef>jar-with-dependencies</descriptorRef>
368370
</descriptorRefs>
369371
<archive>
372+
370373
<addMavenDescriptor>false</addMavenDescriptor>
371374
<manifest>
372375
<addDefaultImplementationEntries>true</addDefaultImplementationEntries>
373376
<addDefaultSpecificationEntries>true</addDefaultSpecificationEntries>
374377
</manifest>
375378
</archive>
376379
</configuration>
380+
<executions>
381+
<execution>
382+
<id>make-assembly</id>
383+
<phase>package</phase>
384+
<goals>
385+
<goal>single</goal>
386+
</goals>
387+
</execution>
388+
</executions>
377389
</plugin>
378390
<plugin>
379391
<groupId>org.apache.maven.plugins</groupId>

‎clickhouse-jdbc/src/main/java/com/clickhouse/jdbc/ClickHouseDriver.java‎

Lines changed: 7 additions & 8 deletions
Original file line numberDiff line numberDiff line change
@@ -20,8 +20,7 @@ public class ClickHouseDriver implements java.sql.Driver {
2020
}
2121

2222
public ClickHouseDriver() {
23-
// log.debug("Creating a new instance of the 'proxy' ClickHouseDriver");
24-
log.info("ClickHouse JDBC driver version: {}", ClickHouseDriver.class.getPackage().getImplementationVersion());
23+
log.debug("ClickHouse JDBC driver version: {}", ClickHouseDriver.class.getPackage().getImplementationVersion());
2524
urlFlagSent = false;
2625
this.driver = getDriver(null);
2726
}
@@ -52,21 +51,21 @@ public boolean isV2(String url) {
5251
log.debug("Checking if V1 driver is requested. V2 is the default driver.");
5352
boolean v1Flag = Boolean.parseBoolean(System.getProperty("clickhouse.jdbc.v1", "false"));
5453
if (v1Flag) {
55-
log.info("V1 driver is requested through system property.");
54+
log.debug("V1 driver is requested through system property.");
5655
return false;
5756
}
5857

5958
if (url != null && url.contains("clickhouse.jdbc.v")) {
6059
urlFlagSent = true;
6160

6261
if (url.contains("clickhouse.jdbc.v1=true")) {
63-
log.info("V1 driver is requested through URL.");
62+
log.debug("V1 driver is requested through URL.");
6463
return false;
6564
} if (url.contains("clickhouse.jdbc.v2=false")) {
66-
log.info("V1 driver is requested through URL.");
65+
log.debug("V1 driver is requested through URL.");
6766
return false;
6867
} else {
69-
log.info("V2 driver is requested through URL.");
68+
log.debug("V2 driver is requested through URL.");
7069
return true;
7170
}
7271
}
@@ -81,10 +80,10 @@ private java.sql.Driver getDriver(String url) {
8180
}
8281

8382
if (isV2(url)) {
84-
log.info("v2 driver");
83+
log.debug("v2 driver");
8584
driver = new com.clickhouse.jdbc.Driver();
8685
} else {
87-
log.info("v1 driver");
86+
log.debug("v1 driver");
8887
driver = new DriverV1();
8988
}
9089

‎clickhouse-jdbc/src/main/java/com/clickhouse/jdbc/DriverV1.java‎

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -109,7 +109,7 @@ public static String getFrameworksDetected() {
109109

110110
public static void load() {
111111
try {
112-
log.info("Registering ClickHouse JDBC driver v1 ({})", driverVersion);
112+
log.debug("Registering ClickHouse JDBC driver v1 ({})", driverVersion);
113113
DriverManager.registerDriver(new DriverV1());
114114
} catch (SQLException e) {
115115
throw new IllegalStateException(e);
@@ -120,7 +120,7 @@ public static void load() {
120120

121121
public static void unload() {
122122
try {
123-
log.info("Unregistering ClickHouse JDBC driver v1 ({})", driverVersion);
123+
log.debug("Unregistering ClickHouse JDBC driver v1 ({})", driverVersion);
124124
DriverManager.deregisterDriver(new DriverV1());
125125
} catch (SQLException e) {
126126
throw new IllegalStateException(e);

‎client-v2/pom.xml‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -153,7 +153,7 @@
153153
<path>
154154
<groupId>org.projectlombok</groupId>
155155
<artifactId>lombok</artifactId>
156-
<version>1.18.32</version>
156+
<version>1.18.38</version>
157157
</path>
158158
</annotationProcessorPaths>
159159
<release>8</release>

‎client-v2/src/main/java/com/clickhouse/client/api/data_formats/internal/BinaryStreamReader.java‎

Lines changed: 7 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -226,7 +226,7 @@ public <T> T readValue(ClickHouseColumn column, Class<?> typeHint) throws IOExce
226226
if (jsonAsString) {
227227
return (T) readString(input);
228228
} else {
229-
return (T) readJsonData(input);
229+
return (T) readJsonData(input, actualColumn);
230230
}
231231
// case Object: // deprecated https://clickhouse.com/docs/en/sql-reference/data-types/object-data-type
232232
case Array:
@@ -1192,16 +1192,20 @@ private ClickHouseColumn readDynamicData() throws IOException {
11921192

11931193
private static final ClickHouseColumn JSON_PLACEHOLDER_COL = ClickHouseColumn.parse("v Dynamic").get(0);
11941194

1195-
private Map<String, Object> readJsonData(InputStream input) throws IOException {
1195+
private Map<String, Object> readJsonData(InputStream input, ClickHouseColumn column) throws IOException {
11961196
int numOfPaths = readVarInt(input);
11971197
if (numOfPaths == 0) {
11981198
return Collections.emptyMap();
11991199
}
12001200

12011201
Map<String, Object> obj = new HashMap<>();
1202+
1203+
final Map<String, ClickHouseColumn> predefinedColumns = column.getJsonPredefinedPaths();
12021204
for (int i = 0; i < numOfPaths; i++) {
12031205
String path = readString(input);
1204-
Object value = readValue(JSON_PLACEHOLDER_COL);
1206+
ClickHouseColumn dataColumn = predefinedColumns == null? JSON_PLACEHOLDER_COL :
1207+
predefinedColumns.getOrDefault(path, JSON_PLACEHOLDER_COL);
1208+
Object value = readValue(dataColumn);
12051209
obj.put(path, value);
12061210
}
12071211
return obj;

‎client-v2/src/test/java/com/clickhouse/client/datatypes/DataTypeTests.java‎

Lines changed: 42 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -7,11 +7,14 @@
77
import com.clickhouse.client.api.Client;
88
import com.clickhouse.client.api.DataTypeUtils;
99
import com.clickhouse.client.api.command.CommandSettings;
10+
import com.clickhouse.client.api.data_formats.ClickHouseBinaryFormatReader;
11+
import com.clickhouse.client.api.data_formats.internal.SerializerUtils;
1012
import com.clickhouse.client.api.enums.Protocol;
1113
import com.clickhouse.client.api.insert.InsertSettings;
1214
import com.clickhouse.client.api.metadata.TableSchema;
1315
import com.clickhouse.client.api.query.GenericRecord;
1416
import com.clickhouse.client.api.query.QueryResponse;
17+
import com.clickhouse.client.api.sql.SQLUtils;
1518
import com.clickhouse.data.ClickHouseDataType;
1619
import com.clickhouse.data.ClickHouseVersion;
1720
import lombok.AllArgsConstructor;
@@ -874,6 +877,45 @@ private void testVariantWith(String withWhat, String[] fields, Object[] values,
874877
}
875878
}
876879

880+
@Test(groups = {"integration"}, dataProvider = "testJSONBinaryFormat_dp")
881+
public void testJSONBinaryFormat(String jsonDef) throws Exception {
882+
if (isVersionMatch("(,24.8]")) {
883+
return;
884+
}
885+
886+
final String table = "test_json_binary_format";
887+
final String jsonCol = "value " + jsonDef;
888+
final String jsonValue = "{\"count\": 1000, \"stat\": {\"float\": 0.999, \"name\": \"temp\" }}";
889+
890+
client.execute("DROP TABLE IF EXISTS " + table).get().close();
891+
client.execute(tableDefinition(table, jsonCol),
892+
(CommandSettings) new CommandSettings()
893+
.serverSetting("enable_json_type", "1")
894+
.serverSetting("allow_experimental_json_type", "1")).get().close();
895+
client.execute("INSERT INTO " + table + " VALUES (" + SQLUtils.enquoteLiteral(jsonValue) + ")").get().close();
896+
897+
try (QueryResponse queryResponse = client.query("SELECT * FROM " + table + " LIMIT 1").get()) {
898+
ClickHouseBinaryFormatReader reader = client.newBinaryFormatReader(queryResponse);
899+
Map<String, Object> row = reader.next();
900+
Object value = row.get("value");
901+
Assert.assertNotNull(value);
902+
}
903+
}
904+
905+
@DataProvider
906+
public Object[][] testJSONBinaryFormat_dp() {
907+
908+
return new Object[][] {
909+
{"JSON"},
910+
{"JSON()"},
911+
{"JSON(stat.name String, count Int32)"},
912+
{"JSON(stat.name String, `comments` String)"},
913+
{"JSON(max_dynamic_paths=3, stat.name String, SKIP alt_count)"},
914+
{"JSON(max_dynamic_paths=3, stat.name String, SKIP REGEXP '^-.*')"},
915+
{"JSON(max_dynamic_paths=3,SKIP REGEXP '^-.*',SKIP ff, flags Array(Array(Array(Int8))), SKIP alt_count)"},
916+
};
917+
}
918+
877919
public static String tableDefinition(String table, String... columns) {
878920
StringBuilder sb = new StringBuilder();
879921
sb.append("CREATE TABLE " + table + " ( ");

0 commit comments

Comments
 (0)