Skip to content

Commit 23d6a93

Browse files
chernserchernser
committed
draft implementation of Variant in v2
1 parent 5dc223a commit 23d6a93

File tree

11 files changed

+304
-2
lines changed

11 files changed

+304
-2
lines changed

‎clickhouse-data/src/main/java/com/clickhouse/data/ClickHouseColumn.java‎

Lines changed: 16 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -37,13 +37,16 @@
3737

3838
import java.io.Serializable;
3939
import java.lang.reflect.Array;
40+
import java.math.BigInteger;
4041
import java.time.OffsetDateTime;
4142
import java.util.ArrayList;
4243
import java.util.Arrays;
4344
import java.util.Collections;
4445
import java.util.Comparator;
46+
import java.util.HashMap;
4547
import java.util.LinkedList;
4648
import java.util.List;
49+
import java.util.Map;
4750
import java.util.Objects;
4851
import java.util.TimeZone;
4952

@@ -94,6 +97,8 @@ public final class ClickHouseColumn implements Serializable {
9497

9598
private ClickHouseValue template;
9699

100+
private Map<Class<?>, Integer> classToVariantOrdNumMap;
101+
97102
private static ClickHouseColumn update(ClickHouseColumn column) {
98103
column.enumConstants = ClickHouseEnum.EMPTY;
99104
int size = column.parameters.size();
@@ -423,9 +428,14 @@ protected static int readColumn(String args, int startIndex, int len, String nam
423428
if (nestedColumns.isEmpty()) {
424429
throw new IllegalArgumentException("Tuple should have at least one nested column");
425430
}
431+
432+
List<ClickHouseDataType> variantDataTypes = new ArrayList<>();
426433
if (matchedKeyword.equals(KEYWORD_VARIANT)) {
427434
nestedColumns.sort(Comparator.comparing(o -> o.getDataType().name()));
428-
nestedColumns.forEach(c -> c.columnName = "v." + c.getDataType().name());
435+
nestedColumns.forEach(c -> {
436+
c.columnName = "v." + c.getDataType().name();
437+
variantDataTypes.add(c.dataType);
438+
});
429439
}
430440
column = new ClickHouseColumn(ClickHouseDataType.valueOf(matchedKeyword), name,
431441
args.substring(startIndex, endIndex + 1), nullable, lowCardinality, null, nestedColumns);
@@ -435,6 +445,7 @@ protected static int readColumn(String args, int startIndex, int len, String nam
435445
fixedLength = false;
436446
}
437447
}
448+
column.classToVariantOrdNumMap = ClickHouseDataType.buildVariantMapping(variantDataTypes);
438449
}
439450

440451
if (column == null) {
@@ -638,6 +649,10 @@ public boolean isAggregateFunction() {
638649

639650
}
640651

652+
public int getVariantOrdNum(Object value) {
653+
return classToVariantOrdNumMap.getOrDefault(value.getClass(), -1);
654+
}
655+
641656
public boolean isArray() {
642657
return dataType == ClickHouseDataType.Array;
643658
}

‎clickhouse-data/src/main/java/com/clickhouse/data/ClickHouseDataType.java‎

Lines changed: 73 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -8,12 +8,16 @@
88
import java.time.LocalDateTime;
99
import java.util.Arrays;
1010
import java.util.Collections;
11+
import java.util.Comparator;
12+
import java.util.EnumSet;
1113
import java.util.HashMap;
14+
import java.util.HashSet;
1215
import java.util.LinkedList;
1316
import java.util.List;
1417
import java.util.Locale;
1518
import java.util.Map;
1619
import java.util.Set;
20+
import java.util.TreeMap;
1721
import java.util.TreeSet;
1822
import java.util.UUID;
1923

@@ -102,7 +106,75 @@ public enum ClickHouseDataType {
102106
SimpleAggregateFunction(String.class, true, true, false, 0, 0, 0, 0, 0, false),
103107
// implementation-defined intermediate state
104108
AggregateFunction(String.class, true, true, false, 0, 0, 0, 0, 0, true),
105-
Variant(List.class, true, true, false, 0, 0, 0, 0, 0, true);
109+
Variant(List.class, true, true, false, 0, 0, 0, 0, 0, true),
110+
111+
;
112+
113+
public static final List<ClickHouseDataType> ORDERED_BY_RANGE_INT_TYPES =
114+
Collections.unmodifiableList(Arrays.asList(
115+
Int8, UInt8, Int16, UInt16, Int32, UInt32, Int64, UInt64, Int128, UInt128, Int256, UInt256
116+
));
117+
118+
public static final List<ClickHouseDataType> ORDERED_BY_RANGE_DECIMAL_TYPES =
119+
Collections.unmodifiableList(Arrays.asList(
120+
Float32, Float64, Decimal32, Decimal64, Decimal128, Decimal256, Decimal
121+
));
122+
123+
public static Map<Class<?>, Integer> buildVariantMapping(List<ClickHouseDataType> variantDataTypes) {
124+
Map<Class<?>, Integer> variantMapping = new HashMap<>();
125+
126+
TreeMap<ClickHouseDataType, Integer> intTypesMappings = new TreeMap<>(Comparator.comparingInt(ORDERED_BY_RANGE_INT_TYPES::indexOf));
127+
TreeMap<ClickHouseDataType, Integer> decTypesMappings = new TreeMap<>(Comparator.comparingInt(ORDERED_BY_RANGE_DECIMAL_TYPES::indexOf));
128+
129+
for (int ordNum = 0; ordNum < variantDataTypes.size(); ordNum++) {
130+
ClickHouseDataType dataType = variantDataTypes.get(ordNum);
131+
Set<Class<?>> classSet = DATA_TYPE_TO_CLASS.get(dataType);
132+
133+
final int finalOrdNum = ordNum;
134+
if (classSet != null) {
135+
if (ORDERED_BY_RANGE_INT_TYPES.contains(dataType)) {
136+
intTypesMappings.put(dataType, ordNum);
137+
} else if (ORDERED_BY_RANGE_DECIMAL_TYPES.contains(dataType)) {
138+
decTypesMappings.put(dataType, ordNum);
139+
} else {
140+
classSet.forEach(c -> variantMapping.put(c, finalOrdNum));
141+
}
142+
}
143+
}
144+
145+
// add numbers mappings
146+
for (java.util.Map.Entry<ClickHouseDataType, Integer> entry : intTypesMappings.entrySet()) {
147+
DATA_TYPE_TO_CLASS.get(entry.getKey()).forEach(c -> variantMapping.put(c, entry.getValue()));
148+
}
149+
for (java.util.Map.Entry<ClickHouseDataType, Integer> entry : decTypesMappings.entrySet()) {
150+
DATA_TYPE_TO_CLASS.get(entry.getKey()).forEach(c -> variantMapping.put(c, entry.getValue()));
151+
}
152+
153+
154+
return variantMapping;
155+
}
156+
157+
static final Map<ClickHouseDataType, Set<Class<?>>> DATA_TYPE_TO_CLASS = dataTypeClassMap();
158+
static Map<ClickHouseDataType, Set<Class<?>>> dataTypeClassMap() {
159+
Map<ClickHouseDataType, Set<Class<?>>> map = new HashMap<>();
160+
161+
map.put(UInt256, Collections.unmodifiableSet(new HashSet<>(Arrays.asList(byte.class, Byte.class, short.class, Short.class, int.class, Integer.class, long.class, Long.class, BigInteger.class))));
162+
map.put(Int256, Collections.unmodifiableSet(new HashSet<>(Arrays.asList(byte.class, Byte.class, short.class, Short.class, int.class, Integer.class, long.class, Long.class, BigInteger.class))));
163+
map.put(UInt128, Collections.unmodifiableSet(new HashSet<>(Arrays.asList(byte.class, Byte.class, short.class, Short.class, int.class, Integer.class, long.class, Long.class, BigInteger.class))));
164+
map.put(Int128, Collections.unmodifiableSet(new HashSet<>(Arrays.asList(byte.class, Byte.class, short.class, Short.class, int.class, Integer.class, long.class, Long.class, BigInteger.class))));
165+
map.put(UInt64, Collections.unmodifiableSet(new HashSet<>(Arrays.asList(byte.class, Byte.class, short.class, Short.class, int.class, Integer.class, long.class, Long.class))));
166+
map.put(Int64, Collections.unmodifiableSet(new HashSet<>(Arrays.asList(byte.class, Byte.class, short.class, Short.class, int.class, Integer.class, long.class, Long.class))));
167+
map.put(UInt32, Collections.unmodifiableSet(new HashSet<>(Arrays.asList(byte.class, Byte.class, short.class, Short.class, int.class, Integer.class ))));
168+
map.put(Int32, Collections.unmodifiableSet(new HashSet<>(Arrays.asList(byte.class, Byte.class, short.class, Short.class, int.class, Integer.class))));
169+
map.put(UInt16, Collections.unmodifiableSet(new HashSet<>(Arrays.asList(byte.class, Byte.class, short.class, Short.class))));
170+
map.put(Int16, Collections.unmodifiableSet(new HashSet<>(Arrays.asList(byte.class, Byte.class, short.class, Short.class))));
171+
map.put(UInt8, Collections.unmodifiableSet(new HashSet<>(Arrays.asList(byte.class, Byte.class))));
172+
map.put(Int8, Collections.unmodifiableSet(new HashSet<>(Arrays.asList(byte.class, Byte.class))));
173+
174+
map.put(String, Collections.unmodifiableSet(new HashSet<>(Arrays.asList(String.class))));
175+
176+
return map;
177+
}
106178

107179

108180
/**

‎clickhouse-data/src/main/java/com/clickhouse/data/format/ClickHouseRowBinaryProcessor.java‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -301,6 +301,7 @@ public VariantSerializer(ClickHouseDataConfig config, ClickHouseColumn column,
301301
@Override
302302
public void serialize(ClickHouseValue value, ClickHouseOutputStream output) throws IOException {
303303
List<Object> tupleValues = value.asTuple();
304+
// TODO: variant index
304305
for (int i = 0, len = serializers.length; i < len; i++) {
305306
serializers[i].serialize(values[i].update(tupleValues.get(i)), output);
306307
}

‎client-v2/pom.xml‎

Lines changed: 6 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -135,6 +135,12 @@
135135
<scope>test</scope>
136136
</dependency>
137137

138+
<dependency>
139+
<groupId>org.projectlombok</groupId>
140+
<artifactId>lombok</artifactId>
141+
<version>1.18.36</version>
142+
<scope>test</scope>
143+
</dependency>
138144
</dependencies>
139145

140146
<build>

‎client-v2/src/main/java/com/clickhouse/client/api/data_formats/internal/AbstractBinaryFormatReader.java‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -251,6 +251,7 @@ protected void setSchema(TableSchema schema) {
251251
case String:
252252
case Enum8:
253253
case Enum16:
254+
case Variant:
254255
this.convertions[i] = NumberConverter.NUMBER_CONVERTERS;
255256
break;
256257
default:

‎client-v2/src/main/java/com/clickhouse/client/api/data_formats/internal/BinaryStreamReader.java‎

Lines changed: 7 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -214,6 +214,8 @@ public <T> T readValue(ClickHouseColumn column, Class<?> typeHint) throws IOExce
214214
return (T) readValue(column.getNestedColumns().get(0));
215215
case AggregateFunction:
216216
return (T) readBitmap( column);
217+
case Variant:
218+
return (T) readVariant(column);
217219
default:
218220
throw new IllegalArgumentException("Unsupported data type: " + column.getDataType());
219221
}
@@ -675,6 +677,11 @@ public Object[] readTuple(ClickHouseColumn column) throws IOException {
675677
return tuple;
676678
}
677679

680+
public Object readVariant(ClickHouseColumn column) throws IOException {
681+
int ordNum = readByte();
682+
return readValue(column.getNestedColumns().get(ordNum));
683+
}
684+
678685
/**
679686
* Reads a GEO point as an array of two doubles what represents coordinates (X, Y).
680687
* @return X, Y coordinates

‎client-v2/src/main/java/com/clickhouse/client/api/data_formats/internal/SerializerUtils.java‎

Lines changed: 35 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -28,6 +28,7 @@
2828
import java.time.ZoneId;
2929
import java.time.ZonedDateTime;
3030
import java.util.Arrays;
31+
import java.util.Collection;
3132
import java.util.HashMap;
3233
import java.util.HashSet;
3334
import java.util.List;
@@ -219,6 +220,15 @@ private static void serializePrimitiveData(OutputStream stream, Object value, Cl
219220
case JSON:
220221
serializeJSON(stream, value);
221222
break;
223+
case Tuple:
224+
serializeTuple(stream, column, (Object[]) value);
225+
break;
226+
case Point:
227+
serializeTuple(stream, GEO_POINT_TUPLE, (Object[]) value);
228+
break;
229+
case Variant:
230+
serializerVariant(stream, column, value);
231+
break;
222232
default:
223233
throw new UnsupportedOperationException("Unsupported data type: " + column.getDataType());
224234
}
@@ -232,6 +242,31 @@ private static void serializeJSON(OutputStream stream, Object value) throws IOEx
232242
}
233243
}
234244

245+
private static void serializeTuple(OutputStream out, ClickHouseColumn column, Object[] tupleValues) throws IOException {
246+
if (column.getNestedColumns().size() != tupleValues.length) {
247+
throw new IllegalArgumentException("Column " + column.getColumnName() + " defines as Tuple with "
248+
+ column.getNestedColumns().size() +" elements, but only " + tupleValues.length + " provided");
249+
}
250+
251+
List<ClickHouseColumn> nested = column.getNestedColumns();
252+
for (int i = 0; i < nested.size() ; i++) {
253+
serializeData(out, tupleValues[i], nested.get(i));
254+
}
255+
}
256+
257+
258+
private static void serializerVariant(OutputStream out, ClickHouseColumn column, Object value) throws IOException {
259+
int typeOrdNum = column.getVariantOrdNum(value);
260+
if (typeOrdNum != -1) {
261+
BinaryStreamUtils.writeUnsignedInt8(out, typeOrdNum);
262+
serializeData(out, value, column.getNestedColumns().get(typeOrdNum));
263+
} else {
264+
throw new IllegalArgumentException("Cannot write value of class " + value.getClass() + " into column with variant type " + column.getOriginalTypeName());
265+
}
266+
}
267+
268+
private static final ClickHouseColumn GEO_POINT_TUPLE = ClickHouseColumn.parse("geopoint Tuple(Float64, Float64)").get(0);
269+
235270
private static void serializeAggregateFunction(OutputStream stream, Object value, ClickHouseColumn column) throws IOException {
236271
if (column.getAggregateFunction() == ClickHouseAggregateFunction.groupBitmap) {
237272
if (value == null) {

‎client-v2/src/test/java/com/clickhouse/client/datatypes/DataTypeTests.java‎

Lines changed: 114 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,114 @@
1+
package com.clickhouse.client.datatypes;
2+
3+
import com.clickhouse.client.BaseIntegrationTest;
4+
import com.clickhouse.client.ClickHouseNode;
5+
import com.clickhouse.client.ClickHouseProtocol;
6+
import com.clickhouse.client.api.Client;
7+
import com.clickhouse.client.api.command.CommandSettings;
8+
import com.clickhouse.client.api.enums.Protocol;
9+
import com.clickhouse.client.api.insert.InsertSettings;
10+
import com.clickhouse.client.api.query.GenericRecord;
11+
import com.clickhouse.client.api.query.QueryResponse;
12+
import org.testng.Assert;
13+
import org.testng.annotations.AfterMethod;
14+
import org.testng.annotations.BeforeMethod;
15+
import org.testng.annotations.Test;
16+
17+
import java.io.IOException;
18+
import java.util.Arrays;
19+
import java.util.List;
20+
import java.util.Map;
21+
22+
public class DataTypeTests extends BaseIntegrationTest {
23+
24+
private Client client;
25+
private InsertSettings settings;
26+
27+
private boolean useClientCompression = false;
28+
29+
private boolean useHttpCompression = false;
30+
31+
private static final int EXECUTE_CMD_TIMEOUT = 10; // seconds
32+
33+
public DataTypeTests(boolean useClientCompression, boolean useHttpCompression) {
34+
this.useClientCompression = useClientCompression;
35+
this.useHttpCompression = useHttpCompression;
36+
}
37+
38+
public DataTypeTests() {
39+
this(false, false);
40+
}
41+
42+
@BeforeMethod(groups = {"integration"})
43+
public void setUp() throws IOException {
44+
ClickHouseNode node = getServer(ClickHouseProtocol.HTTP);
45+
client = new Client.Builder()
46+
.addEndpoint(Protocol.HTTP, node.getHost(), node.getPort(), false)
47+
.setUsername("default")
48+
.setPassword("")
49+
.useNewImplementation(System.getProperty("client.tests.useNewImplementation", "true").equals("true"))
50+
.compressClientRequest(useClientCompression)
51+
.useHttpCompression(useHttpCompression)
52+
.build();
53+
}
54+
55+
@AfterMethod(groups = { "integration" })
56+
public void tearDown() {
57+
client.close();
58+
}
59+
60+
61+
@Test
62+
public void testNestedDataTypes() throws Exception {
63+
final String table = "test_nested_types";
64+
String tblCreateSQL = NestedTypesDTO.tblCreateSQL(table);
65+
client.execute("DROP TABLE IF EXISTS " + table).get();
66+
client.execute(tblCreateSQL);
67+
68+
client.register(NestedTypesDTO.class, client.getTableSchema(table));
69+
70+
List<NestedTypesDTO> data =
71+
Arrays.asList(new NestedTypesDTO(0, new Object[] {(short)127, "test 1"}, new Double[] {0.3d, 0.4d} ));
72+
client.insert(table, data);
73+
74+
List<GenericRecord> rows = client.queryAll("SELECT * FROM " + table);
75+
for (GenericRecord row : rows) {
76+
NestedTypesDTO dto = data.get(row.getInteger("rowId"));
77+
Assert.assertEquals(row.getTuple("tuple1"), dto.getTuple1());
78+
Assert.assertEquals(row.getGeoPoint("point1").getValue(), dto.getPoint1());
79+
}
80+
81+
}
82+
83+
@Test
84+
public void testVariantDataTypeWithPrimitives() throws Exception {
85+
final String table = "test_variant_primitives";
86+
String tblCreateSQL = VariantDTO.tblCreateSQL(table);
87+
client.execute("DROP TABLE IF EXISTS " + table).get();
88+
client.execute(tblCreateSQL, (CommandSettings) new CommandSettings().serverSetting("enable_variant_type", "1"));
89+
90+
client.register(VariantDTO.class, client.getTableSchema(table));
91+
92+
// List<VariantDTO> data = Arrays.asList(new VariantDTO(1, (short)200), new VariantDTO(2, (byte)127), new VariantDTO(3, "test ☺"));
93+
List<VariantDTO> data = Arrays.asList(new VariantDTO(1, (short)200), new VariantDTO(2, (byte)127));
94+
client.insert(table, data);
95+
96+
List<GenericRecord> rows = client.queryAll("SELECT * FROM " + table);
97+
for (GenericRecord row : rows) {
98+
System.out.println(row.getInteger("rowId") + " " + row.getInteger("a"));
99+
}
100+
}
101+
102+
103+
public static String tableDefinition(String table, String... columns) {
104+
StringBuilder sb = new StringBuilder();
105+
sb.append("CREATE TABLE " + table + " ( ");
106+
Arrays.stream(columns).forEach(s -> {
107+
sb.append(s).append(", ");
108+
});
109+
sb.setLength(sb.length() - 2);
110+
sb.append(") Engine = MergeTree ORDER BY ()");
111+
return sb.toString();
112+
}
113+
114+
}

0 commit comments

Comments
 (0)