Skip to content

Commit 2ae4d27

Browse files
chernserchernser
committed
implemented dynamic basic types
1 parent 4f8de9d commit 2ae4d27

File tree

6 files changed

+331
-7
lines changed

6 files changed

+331
-7
lines changed

‎clickhouse-client/src/test/java/com/clickhouse/client/ClickHouseServerForTest.java‎

Lines changed: 4 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -250,7 +250,9 @@ public static ClickHouseNode getClickHouseNode(ClickHouseProtocol protocol, bool
250250
}
251251
}
252252

253-
return ClickHouseNode.builder(template).address(protocol, new InetSocketAddress(host, port)).build();
253+
return ClickHouseNode.builder(template).address(protocol, new InetSocketAddress(host, port))
254+
.credentials(new ClickHouseCredentials("default", getPassword()))
255+
.build();
254256
}
255257

256258
public static ClickHouseNode getClickHouseNode(ClickHouseProtocol protocol, int port) {
@@ -314,7 +316,7 @@ public static String getPassword() {
314316
if (isCloud) {
315317
return System.getenv("CLICKHOUSE_CLOUD_PASSWORD");
316318
} else {
317-
return "";
319+
return "test_default_password";
318320
}
319321
}
320322

‎clickhouse-client/src/test/resources/containers/clickhouse-server/users.d/users.xml‎

Lines changed: 9 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -10,6 +10,15 @@
1010
</managed2>
1111
</profiles>
1212
<users>
13+
<default>
14+
<access_management>0</access_management>
15+
<profile>default</profile>
16+
<networks>
17+
<ip>::/0</ip>
18+
</networks>
19+
<password>test_default_password</password>
20+
<quota>default</quota>
21+
</default>
1322
<dba>
1423
<access_management>1</access_management>
1524
<profile>default</profile>

‎clickhouse-data/src/main/java/com/clickhouse/data/ClickHouseDataType.java‎

Lines changed: 7 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -228,6 +228,8 @@ private static Set<Class<?>> setOf(Class<?>... args) {
228228

229229
public static final byte SET_BIN_TAG = 0x21;
230230

231+
public static final byte CUSTOM_TYPE_BIN_TAG = 0x2C;
232+
231233
public enum IntervalKindBinTag {
232234
Nanosecond(IntervalNanosecond, 0x00),
233235
Microsecond(IntervalMicrosecond, 0x01),
@@ -284,6 +286,8 @@ public byte getTag() {
284286

285287
public static final Map<Byte, ClickHouseDataType> intervalKind2Type;
286288

289+
public static final Map<ClickHouseDataType, Byte> intervalType2Kind;
290+
287291
static {
288292
Set<String> set = new TreeSet<>();
289293
Map<String, ClickHouseDataType> map = new HashMap<>();
@@ -320,10 +324,13 @@ public byte getTag() {
320324
binTag2Type = Collections.unmodifiableMap(tmpbinTag2Type);
321325

322326
Map<Byte, ClickHouseDataType> tmpIntervalKind2Type = new HashMap<>();
327+
Map<ClickHouseDataType, Byte > tmpIntervalType2Kind = new HashMap<>();
323328
for (IntervalKindBinTag kind : IntervalKindBinTag.values()) {
324329
tmpIntervalKind2Type.put(kind.getTag(), kind.getIntervalType());
330+
tmpIntervalType2Kind.put(kind.getIntervalType(), kind.tag);
325331
}
326332
intervalKind2Type = Collections.unmodifiableMap(tmpIntervalKind2Type);
333+
intervalType2Kind = Collections.unmodifiableMap(tmpIntervalType2Kind);
327334
}
328335

329336
/**

‎client-v2/src/main/java/com/clickhouse/client/api/data_formats/internal/BinaryStreamReader.java‎

Lines changed: 4 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -1007,7 +1007,11 @@ private ClickHouseDataType readDynamicData() throws IOException {
10071007
if (type == null) {
10081008
throw new ClientException("Unsupported interval kind: " + intervalKind);
10091009
}
1010+
10101011
} else {
1012+
if (tag == ClickHouseDataType.String.getBinTag()) {
1013+
System.out.println("String");
1014+
}
10111015
type = ClickHouseDataType.binTag2Type.get(tag);
10121016
if (type == null) {
10131017
throw new ClientException("Unsupported data type with tag " + tag);

‎client-v2/src/main/java/com/clickhouse/client/api/data_formats/internal/SerializerUtils.java‎

Lines changed: 210 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -2,8 +2,6 @@
22

33
import com.clickhouse.client.api.Client;
44
import com.clickhouse.client.api.ClientException;
5-
import com.clickhouse.client.api.data_formats.RowBinaryFormatSerializer;
6-
import com.clickhouse.client.api.data_formats.RowBinaryFormatWriter;
75
import com.clickhouse.client.api.query.POJOSetter;
86
import com.clickhouse.data.ClickHouseAggregateFunction;
97
import com.clickhouse.data.ClickHouseColumn;
@@ -30,17 +28,21 @@
3028
import java.net.Inet4Address;
3129
import java.net.Inet6Address;
3230
import java.sql.Timestamp;
31+
import java.time.Instant;
3332
import java.time.LocalDate;
3433
import java.time.LocalDateTime;
3534
import java.time.ZoneId;
3635
import java.time.ZonedDateTime;
3736
import java.util.Arrays;
37+
import java.util.Collections;
38+
import java.util.HashMap;
3839
import java.util.HashSet;
3940
import java.util.List;
4041
import java.util.Map;
4142
import java.util.Objects;
4243
import java.util.Set;
4344
import java.util.StringTokenizer;
45+
import java.util.TimeZone;
4446
import java.util.UUID;
4547
import java.util.stream.Collectors;
4648

@@ -90,14 +92,219 @@ public static void serializeData(OutputStream stream, Object value, ClickHouseCo
9092
value = value instanceof ClickHouseGeoMultiPolygonValue ? ((ClickHouseGeoMultiPolygonValue)value).getValue() : value;
9193
serializeArrayData(stream, value, GEO_MULTI_POLYGON_ARRAY);
9294
break;
95+
case Dynamic:
96+
ClickHouseColumn typeColumn = valueToColumnForDynamicType(value);
97+
writeDynamicTypeTag(stream, typeColumn, value);
98+
serializeData(stream, value, typeColumn);
99+
break;
93100
default:
94101
serializePrimitiveData(stream, value, column);
95102
break;
96103

97104
}
98105
}
99106

100-
private static void serializeArrayData(OutputStream stream, Object value, ClickHouseColumn column) throws IOException {
107+
private static final Map<Class<?>, ClickHouseColumn> PREDEFINED_TYPE_COLUMNS = getPredefinedTypeColumnsMap();
108+
109+
private static Map<Class<?>, ClickHouseColumn> getPredefinedTypeColumnsMap() {
110+
HashMap<Class<?>, ClickHouseColumn> map = new HashMap<>();
111+
map.put(Void.class, ClickHouseColumn.of("v", "Nothing"));
112+
map.put(Boolean.class, ClickHouseColumn.of("v", "Bool"));
113+
map.put(Byte.class, ClickHouseColumn.of("v", "Int8"));
114+
map.put(Short.class, ClickHouseColumn.of("v", "Int16"));
115+
map.put(Integer.class, ClickHouseColumn.of("v", "Int32"));
116+
map.put(Long.class, ClickHouseColumn.of("v", "Int64"));
117+
map.put(BigInteger.class, ClickHouseColumn.of("v", "Int256"));
118+
map.put(Float.class, ClickHouseColumn.of("v", "Float32"));
119+
map.put(Double.class, ClickHouseColumn.of("v", "Float64"));
120+
map.put(LocalDate.class, ClickHouseColumn.of("v", "Date"));
121+
map.put(UUID.class, ClickHouseColumn.of("v", "UUID"));
122+
map.put(Inet4Address.class, ClickHouseColumn.of("v", "IPv4"));
123+
map.put(Inet6Address.class, ClickHouseColumn.of("v", "IPv6"));
124+
map.put(String.class, ClickHouseColumn.of("v", "String"));
125+
map.put(LocalDateTime.class, ClickHouseColumn.of("v", "DateTime"));
126+
127+
map.put(boolean[].class, ClickHouseColumn.of("v", "Array(Bool)"));
128+
map.put(boolean[][].class, ClickHouseColumn.of("v", "Array(Array(Bool))"));
129+
map.put(boolean[][][].class, ClickHouseColumn.of("v", "Array(Array(Array(Bool)))"));
130+
131+
map.put(byte[].class, ClickHouseColumn.of("v", "Array(Int8)"));
132+
map.put(byte[][].class, ClickHouseColumn.of("v", "Array(Array(Int8))"));
133+
map.put(byte[][][].class, ClickHouseColumn.of("v", "Array(Array(Array(Int8)))"));
134+
135+
map.put(short[].class, ClickHouseColumn.of("v", "Array(Int16)"));
136+
map.put(short[][].class, ClickHouseColumn.of("v", "Array(Array(Int16))"));
137+
map.put(short[][][].class, ClickHouseColumn.of("v", "Array(Array(Array(Int16)))"));
138+
139+
map.put(int[].class, ClickHouseColumn.of("v", "Array(Int32)"));
140+
map.put(int[][].class, ClickHouseColumn.of("v", "Array(Array(Int32))"));
141+
map.put(int[][][].class, ClickHouseColumn.of("v", "Array(Array(Array(Int32)))"));
142+
143+
map.put(long[].class, ClickHouseColumn.of("v", "Array(Int64)"));
144+
map.put(long[][].class, ClickHouseColumn.of("v", "Array(Array(Int64))"));
145+
map.put(long[][][].class, ClickHouseColumn.of("v", "Array(Array(Array(Int64)))"));
146+
147+
map.put(float[].class, ClickHouseColumn.of("v", "Array(Float32)"));
148+
map.put(float[][].class, ClickHouseColumn.of("v", "Array(Array(Float32))"));
149+
map.put(float[][][].class, ClickHouseColumn.of("v", "Array(Array(Array(Float32)))"));
150+
151+
map.put(double[].class, ClickHouseColumn.of("v", "Array(Float64)"));
152+
map.put(double[][].class, ClickHouseColumn.of("v", "Array(Array(Float64))"));
153+
map.put(double[][][].class, ClickHouseColumn.of("v", "Array(Array(Array(Float64)))"));
154+
155+
return Collections.unmodifiableMap(map);
156+
}
157+
158+
public static ClickHouseColumn valueToColumnForDynamicType(Object value) {
159+
ClickHouseColumn column;
160+
if (value instanceof ZonedDateTime) {
161+
ZonedDateTime dt = (ZonedDateTime) value;
162+
column = ClickHouseColumn.of("v", "DateTime(" + dt.getZone().getId() + ")");
163+
} else if (value instanceof BigDecimal) {
164+
BigDecimal d = (BigDecimal) value;
165+
column = ClickHouseColumn.of("v", "Decimal256(" + d.precision() + ", " + d.scale() + ")");
166+
} else if (value instanceof Map<?,?>) {
167+
Map<?, ?> map = (Map<?, ?>) value;
168+
// TODO: handle empty map?
169+
Map.Entry<?, ?> entry = map.entrySet().iterator().next();
170+
ClickHouseColumn keyInfo = valueToColumnForDynamicType(entry.getKey());
171+
ClickHouseColumn valueInfo = valueToColumnForDynamicType(entry.getValue());
172+
column = ClickHouseColumn.of("v", "Map(" + keyInfo.getOriginalTypeName() + ", " + valueInfo.getOriginalTypeName() + ")");
173+
} else if (value instanceof List<?>) {
174+
List<?> list = (List<?>) value;
175+
StringBuilder type = new StringBuilder("Array()");
176+
int insertPos = type.length() - 2;
177+
while (!list.isEmpty() && list.get(0) instanceof List<?>) {
178+
type.insert(insertPos, "Array()");
179+
insertPos += 6; // add len of 'Array(' string
180+
list = (List<?>) list.get(0);
181+
}
182+
if (list.isEmpty()) {
183+
type.insert(insertPos, "Nothing");
184+
column = ClickHouseColumn.of("v", type.toString());
185+
} else {
186+
ClickHouseColumn arrayBaseColumn = PREDEFINED_TYPE_COLUMNS.get(list.get(0));
187+
if (arrayBaseColumn != null) {
188+
type.insert(insertPos, arrayBaseColumn.getOriginalTypeName());
189+
column = ClickHouseColumn.of("v", type.toString());
190+
} else {
191+
column = null;
192+
}
193+
}
194+
} else if (value == null) {
195+
column = PREDEFINED_TYPE_COLUMNS.get(Void.class);
196+
} else {
197+
column = PREDEFINED_TYPE_COLUMNS.get(value.getClass());
198+
}
199+
200+
if (column == null) {
201+
throw new ClientException("Unable to serialize value of " + value.getClass() + " because not supported yet");
202+
}
203+
204+
return column;
205+
}
206+
207+
public static void writeDynamicTypeTag(OutputStream stream, ClickHouseColumn typeColumn, Object value)
208+
throws IOException {
209+
210+
ClickHouseDataType dt = typeColumn.getDataType();
211+
byte binTag = dt.getBinTag();
212+
if (binTag == -1) {
213+
throw new ClientException("Type " + dt.name() +" serialization is not supported for Dynamic column");
214+
}
215+
216+
if (typeColumn.isNullable()) {
217+
stream.write(ClickHouseDataType.NULLABLE_BIN_TAG);
218+
}
219+
if (typeColumn.isLowCardinality()) {
220+
stream.write(ClickHouseDataType.LOW_CARDINALITY_BIN_TAG);
221+
}
222+
223+
switch (dt) {
224+
case FixedString:
225+
stream.write(binTag);
226+
writeVarInt(stream, typeColumn.getEstimatedLength());
227+
break;
228+
case Enum8:
229+
stream.write(binTag);
230+
/// 0x17<var_uint_number_of_elements><var_uint_name_size_1><name_data_1><int8_value_1>...<var_uint_name_size_N><name_data_N><int8_value_N>
231+
break;
232+
case Enum16:
233+
stream.write(binTag);
234+
//0x18<var_uint_number_of_elements><var_uint_name_size_1><name_data_1><int16_little_endian_value_1>...><var_uint_name_size_N><name_data_N><int16_little_endian_value_N>
235+
break;
236+
case Decimal:
237+
case Decimal32:
238+
case Decimal64:
239+
case Decimal128:
240+
case Decimal256:
241+
stream.write(binTag);
242+
//<tag><uint8_precision><uint8_scale>
243+
break;
244+
245+
case IntervalNanosecond:
246+
case IntervalMillisecond:
247+
case IntervalSecond:
248+
case IntervalMinute:
249+
case IntervalHour:
250+
case IntervalDay:
251+
case IntervalWeek:
252+
case IntervalMonth:
253+
case IntervalQuarter:
254+
case IntervalYear:
255+
stream.write(binTag);
256+
stream.write(ClickHouseDataType.IntervalKindBinTag.Day.getTag());
257+
break;
258+
case DateTime32:
259+
stream.write(binTag);
260+
BinaryStreamUtils.writeString(stream, typeColumn.getTimeZoneOrDefault(TimeZone.getDefault()).getID());
261+
break;
262+
case DateTime64:
263+
break;
264+
case Array:
265+
stream.write(binTag);
266+
// elements 0x1E<nested_type_encoding>
267+
break;
268+
case Map:
269+
stream.write(binTag);
270+
///0x0F<var_uint_size><key_encoding_1><value_encoding_1>...<key_encoding_N><value_encoding_N>
271+
break;
272+
case Tuple:
273+
// Tuple(T1, ..., TN)
274+
// 0x1F<var_uint_number_of_elements><nested_type_encoding_1>...<nested_type_encoding_N>
275+
stream.write(0x1F);
276+
// or
277+
// Tuple(name1 T1, ..., nameN TN)
278+
// 0x20<var_uint_number_of_elements><var_uint_name_size_1><name_data_1><nested_type_encoding_1>...<var_uint_name_size_N><name_data_N><nested_type_encoding_N>
279+
stream.write(0x20);
280+
break;
281+
case Point:
282+
case Polygon:
283+
case Ring:
284+
case MultiPolygon:
285+
stream.write(ClickHouseDataType.CUSTOM_TYPE_BIN_TAG);
286+
break;
287+
case Variant:
288+
stream.write(binTag);
289+
break;
290+
case Dynamic:
291+
stream.write(binTag);
292+
break;
293+
case JSON:
294+
stream.write(binTag);
295+
break;
296+
case SimpleAggregateFunction:
297+
stream.write(binTag);
298+
break;
299+
case AggregateFunction:
300+
stream.write(binTag);
301+
break;
302+
default:
303+
stream.write(binTag);
304+
}
305+
}
306+
307+
public static void serializeArrayData(OutputStream stream, Object value, ClickHouseColumn column) throws IOException {
101308

102309
if (value instanceof List<?>) {
103310
//Serialize the array to the stream

0 commit comments

Comments
 (0)