Skip to content

Commit eb27cad

Browse files
chernserchernser
authored
Merge pull request #2130 from ClickHouse/v2_support_dynamic_and_json
[client-v2] Support Dynamic and JSON Data Types
2 parents 7b7749f + edc87a1 commit eb27cad

File tree

22 files changed

+1490
-256
lines changed

22 files changed

+1490
-256
lines changed

‎.github/workflows/build.yml‎

Lines changed: 4 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -145,7 +145,7 @@ jobs:
145145
matrix:
146146
# most recent LTS releases as well as latest stable builds
147147
# https://github.com/ClickHouse/ClickHouse/pulls?q=is%3Aopen+is%3Apr+label%3Arelease
148-
clickhouse: ["23.8", "24.3", "24.8", "latest"]
148+
clickhouse: ["24.3", "24.8", "latest"]
149149
project: ["clickhouse-http-client", "client-v2"]
150150
fail-fast: false
151151
timeout-minutes: 15
@@ -268,7 +268,7 @@ jobs:
268268
needs: compile
269269
strategy:
270270
matrix:
271-
clickhouse: ["23.8", "24.3", "24.6", "latest"]
271+
clickhouse: ["24.3", "24.8", "latest"]
272272
# here http, http_client and apache_http_client represent different value of http_connection_provider
273273
# protocol: ["http", "http_client", "apache_http_client"]
274274
protocol: ["apache_http_client"]
@@ -321,7 +321,7 @@ jobs:
321321
uses: actions/upload-artifact@v4
322322
if: failure()
323323
with:
324-
name: result ${{ github.job }}
324+
name: result ${{ github.job }}_${{ matrix.project }}_${{ matrix.clickhouse }}
325325
path: |
326326
**/target/failsafe-reports
327327
**/target/surefire-reports
@@ -331,7 +331,7 @@ jobs:
331331
needs: compile
332332
strategy:
333333
matrix:
334-
clickhouse: ["23.8", "24.3", "24.8", "latest"]
334+
clickhouse: ["24.3", "24.8", "latest"]
335335
# grpc is not fully supported, and http_client and apache_http_client do not work in CI environment(due to limited threads?)
336336
protocol: ["http"]
337337
r2dbc: ["1.0.0.RELEASE", "0.9.1.RELEASE"]

‎clickhouse-client/src/test/java/com/clickhouse/client/ClickHouseServerForTest.java‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -358,6 +358,7 @@ public static void beforeSuite() {
358358
@AfterSuite(groups = {"integration"})
359359
public static void afterSuite() {
360360
if (clickhouseContainer != null) {
361+
// clickhouseContainer.copyFileFromContainer("/var/log/clickhouse-server/clickhouse-server.log", "server-container.log");
361362
clickhouseContainer.stop();
362363
}
363364

‎clickhouse-data/src/main/java/com/clickhouse/data/ClickHouseColumn.java‎

Lines changed: 7 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -640,8 +640,13 @@ public static List<ClickHouseColumn> parse(String args) {
640640
return Collections.unmodifiableList(c);
641641
}
642642

643-
private ClickHouseColumn(ClickHouseDataType dataType, String columnName, String originalTypeName, boolean nullable,
643+
public ClickHouseColumn(ClickHouseDataType dataType, String columnName, String originalTypeName, boolean nullable,
644644
boolean lowCardinality, List<String> parameters, List<ClickHouseColumn> nestedColumns) {
645+
this(dataType, columnName, originalTypeName, nullable, lowCardinality, parameters, nestedColumns, ClickHouseEnum.EMPTY);
646+
}
647+
648+
public ClickHouseColumn(ClickHouseDataType dataType, String columnName, String originalTypeName, boolean nullable,
649+
boolean lowCardinality, List<String> parameters, List<ClickHouseColumn> nestedColumns, ClickHouseEnum enumConstants) {
645650
this.aggFuncType = null;
646651
this.dataType = ClickHouseChecker.nonNull(dataType, "dataType");
647652

@@ -671,6 +676,7 @@ private ClickHouseColumn(ClickHouseDataType dataType, String columnName, String
671676

672677
this.fixedByteLength = false;
673678
this.estimatedByteLength = 0;
679+
this.enumConstants = enumConstants;
674680
}
675681

676682
/**

‎clickhouse-data/src/main/java/com/clickhouse/data/ClickHouseDataType.java‎

Lines changed: 190 additions & 59 deletions
Large diffs are not rendered by default.

‎clickhouse-data/src/main/java/com/clickhouse/data/ClickHouseEnum.java‎

Lines changed: 23 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -51,7 +51,7 @@ protected ClickHouseEnum(Collection<String> params) {
5151
}
5252
}
5353

54-
protected ClickHouseEnum(String[] names, int[] values) {
54+
public ClickHouseEnum(String[] names, int[] values) {
5555
if (names == null || values == null) {
5656
throw new IllegalArgumentException("Non-null names and values are required");
5757
} else if (names.length != values.length) {
@@ -93,6 +93,16 @@ public String name(int value) {
9393
throw new IllegalArgumentException("Unknown enum value: " + value);
9494
}
9595

96+
public String nameNullable(int value) {
97+
for (int i = 0; i < size; i++) {
98+
if (values[i] == value) {
99+
return names[i];
100+
}
101+
}
102+
103+
return null;
104+
}
105+
96106
public int value(String name) {
97107
for (int i = 0; i < size; i++) {
98108
if (names[i].equals(name)) {
@@ -135,4 +145,16 @@ public String toSqlException() {
135145
}
136146
return builder.toString();
137147
}
148+
149+
public int size() {
150+
return size;
151+
}
152+
153+
public String[] getNames() {
154+
return names;
155+
}
156+
157+
public int[] getValues() {
158+
return values;
159+
}
138160
}

‎client-v2/src/main/java/com/clickhouse/client/api/data_formats/ClickHouseBinaryFormatReader.java‎

Lines changed: 5 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -12,6 +12,7 @@
1212
import java.net.Inet4Address;
1313
import java.net.Inet6Address;
1414
import java.time.*;
15+
import java.time.temporal.TemporalAmount;
1516
import java.util.List;
1617
import java.util.Map;
1718
import java.util.UUID;
@@ -540,4 +541,8 @@ public interface ClickHouseBinaryFormatReader extends AutoCloseable {
540541
ClickHouseBitmap getClickHouseBitmap(String colName);
541542

542543
ClickHouseBitmap getClickHouseBitmap(int index);
544+
545+
TemporalAmount getTemporalAmount(int index);
546+
547+
TemporalAmount getTemporalAmount(String colName);
543548
}

‎client-v2/src/main/java/com/clickhouse/client/api/data_formats/RowBinaryFormatSerializer.java‎

Lines changed: 4 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -207,6 +207,8 @@ public static boolean writeValuePreamble(OutputStream out, boolean defaultsSuppo
207207
return false;//And we're done
208208
} else if (dataType == ClickHouseDataType.Array) {//If the column is an array
209209
SerializerUtils.writeNonNull(out);//Then we send nonNull
210+
} else if (dataType == ClickHouseDataType.Dynamic) {
211+
// do nothing
210212
} else {
211213
throw new IllegalArgumentException(String.format("An attempt to write null into not nullable column '%s'", column));
212214
}
@@ -221,6 +223,8 @@ public static boolean writeValuePreamble(OutputStream out, boolean defaultsSuppo
221223
} else if (value == null) {
222224
if (dataType == ClickHouseDataType.Array) {
223225
SerializerUtils.writeNonNull(out);
226+
} else if (dataType == ClickHouseDataType.Dynamic) {
227+
// do nothing
224228
} else {
225229
throw new IllegalArgumentException(String.format("An attempt to write null into not nullable column '%s'", column));
226230
}

‎client-v2/src/main/java/com/clickhouse/client/api/data_formats/internal/AbstractBinaryFormatReader.java‎

Lines changed: 24 additions & 38 deletions
Original file line numberDiff line numberDiff line change
@@ -30,6 +30,7 @@
3030
import java.time.*;
3131
import java.time.format.DateTimeFormatter;
3232
import java.time.temporal.ChronoUnit;
33+
import java.time.temporal.TemporalAmount;
3334
import java.util.*;
3435
import java.util.concurrent.ConcurrentHashMap;
3536
import java.util.concurrent.atomic.AtomicBoolean;
@@ -247,6 +248,7 @@ protected void setSchema(TableSchema schema) {
247248
case Enum8:
248249
case Enum16:
249250
case Variant:
251+
case Dynamic:
250252
this.convertions[i] = NumberConverter.NUMBER_CONVERTERS;
251253
break;
252254
default:
@@ -285,9 +287,11 @@ public static String readAsString(Object value, ClickHouseColumn column) {
285287
return zdt.format(com.clickhouse.client.api.DataTypeUtils.DATE_FORMATTER);
286288
}
287289
return value.toString();
290+
} else if (value instanceof BinaryStreamReader.EnumValue) {
291+
return ((BinaryStreamReader.EnumValue)value).name;
288292
} else if (value instanceof Number ) {
289293
ClickHouseDataType dataType = column.getDataType();
290-
int num = ((Number)value).intValue();
294+
int num = ((Number) value).intValue();
291295
if (column.getDataType() == ClickHouseDataType.Variant) {
292296
for (ClickHouseColumn c : column.getNestedColumns()) {
293297
// TODO: will work only if single enum listed as variant
@@ -411,38 +415,13 @@ public ZonedDateTime getZonedDateTime(String colName) {
411415

412416
@Override
413417
public Duration getDuration(String colName) {
414-
int colIndex = schema.nameToIndex(colName);
415-
ClickHouseColumn column = schema.getColumns().get(colIndex);
416-
BigInteger value = readValue(colName);
417-
try {
418-
switch (column.getDataType()) {
419-
case IntervalYear:
420-
return Duration.of(value.longValue(), java.time.temporal.ChronoUnit.YEARS);
421-
case IntervalQuarter:
422-
return Duration.of(value.longValue() * 3, java.time.temporal.ChronoUnit.MONTHS);
423-
case IntervalMonth:
424-
return Duration.of(value.longValue(), java.time.temporal.ChronoUnit.MONTHS);
425-
case IntervalWeek:
426-
return Duration.of(value.longValue(), ChronoUnit.WEEKS);
427-
case IntervalDay:
428-
return Duration.of(value.longValue(), java.time.temporal.ChronoUnit.DAYS);
429-
case IntervalHour:
430-
return Duration.of(value.longValue(), java.time.temporal.ChronoUnit.HOURS);
431-
case IntervalMinute:
432-
return Duration.of(value.longValue(), java.time.temporal.ChronoUnit.MINUTES);
433-
case IntervalSecond:
434-
return Duration.of(value.longValue(), java.time.temporal.ChronoUnit.SECONDS);
435-
case IntervalMicrosecond:
436-
return Duration.of(value.longValue(), java.time.temporal.ChronoUnit.MICROS);
437-
case IntervalMillisecond:
438-
return Duration.of(value.longValue(), java.time.temporal.ChronoUnit.MILLIS);
439-
case IntervalNanosecond:
440-
return Duration.of(value.longValue(), java.time.temporal.ChronoUnit.NANOS);
441-
}
442-
} catch (ArithmeticException e) {
443-
throw new ClientException("Stored value is bigger then Long.MAX_VALUE and it cannot be converted to Duration without information loss", e);
444-
}
445-
throw new ClientException("Column of type " + column.getDataType() + " cannot be converted to Duration");
418+
TemporalAmount temporalAmount = getTemporalAmount(colName);
419+
return Duration.from(temporalAmount);
420+
}
421+
422+
@Override
423+
public TemporalAmount getTemporalAmount(String colName) {
424+
return readValue(colName);
446425
}
447426

448427
@Override
@@ -603,7 +582,12 @@ public ZonedDateTime getZonedDateTime(int index) {
603582

604583
@Override
605584
public Duration getDuration(int index) {
606-
return readValue(index);
585+
return getDuration(schema.columnIndexToName(index));
586+
}
587+
588+
@Override
589+
public TemporalAmount getTemporalAmount(int index) {
590+
return getTemporalAmount(schema.columnIndexToName(index));
607591
}
608592

609593
@Override
@@ -688,22 +672,24 @@ public Object[] getTuple(String colName) {
688672

689673
@Override
690674
public byte getEnum8(String colName) {
691-
return readValue(colName);
675+
BinaryStreamReader.EnumValue enumValue = readValue(colName);
676+
return enumValue.byteValue();
692677
}
693678

694679
@Override
695680
public byte getEnum8(int index) {
696-
return readValue(index);
681+
return getEnum8(schema.columnIndexToName(index));
697682
}
698683

699684
@Override
700685
public short getEnum16(String colName) {
701-
return readValue(colName);
686+
BinaryStreamReader.EnumValue enumValue = readValue(colName);
687+
return enumValue.shortValue();
702688
}
703689

704690
@Override
705691
public short getEnum16(int index) {
706-
return readValue(index);
692+
return getEnum16(schema.columnIndexToName(index));
707693
}
708694

709695
@Override

‎client-v2/src/main/java/com/clickhouse/client/api/data_formats/internal/BinaryReaderBackedRecord.java‎

Lines changed: 11 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -11,6 +11,7 @@
1111
import java.net.Inet4Address;
1212
import java.net.Inet6Address;
1313
import java.time.*;
14+
import java.time.temporal.TemporalAmount;
1415
import java.util.List;
1516
import java.util.Map;
1617
import java.util.UUID;
@@ -89,6 +90,11 @@ public Duration getDuration(String colName) {
8990
return reader.getDuration(colName);
9091
}
9192

93+
@Override
94+
public TemporalAmount getTemporalAmount(String colName) {
95+
return reader.getTemporalAmount(colName);
96+
}
97+
9298
@Override
9399
public Inet4Address getInet4Address(String colName) {
94100
return reader.getInet4Address(colName);
@@ -234,6 +240,11 @@ public Duration getDuration(int index) {
234240
return reader.getDuration(index);
235241
}
236242

243+
@Override
244+
public TemporalAmount getTemporalAmount(int index) {
245+
return reader.getTemporalAmount(index);
246+
}
247+
237248
@Override
238249
public Inet4Address getInet4Address(int index) {
239250
return reader.getInet4Address(index);

0 commit comments

Comments
 (0)