Skip to content

Commit dbee722

Browse files
chernserchernser
committed
fixed arrays
1 parent 629c0be commit dbee722

File tree

3 files changed

+84
-106
lines changed

3 files changed

+84
-106
lines changed

‎client-v2/src/main/java/com/clickhouse/client/api/data_formats/internal/BinaryStreamReader.java‎

Lines changed: 7 additions & 33 deletions
Original file line numberDiff line numberDiff line change
@@ -527,36 +527,7 @@ public static byte[] readNBytesLE(InputStream input, byte[] buffer, int offset,
527527

528528
return bytes;
529529
}
530-
531-
public ArrayValue readArrayStack(int levels, ClickHouseColumn baseElementColumn) throws IOException {
532-
533-
Stack<ArrayValue> arrays = new Stack<>();
534-
int level = levels;
535-
ArrayValue array = null;
536-
while (level <= levels) {
537-
if (level != 1 && arrays.size() < level) {
538-
int len = readVarInt(input);
539-
arrays.push(new ArrayValue(ArrayValue.class, len));
540-
level--;
541-
} else if (level == 1) {
542-
int len = readVarInt(input);
543-
array = readArrayItem(baseElementColumn, len);
544-
level++;
545-
} else if (array !=null) { // some array read completely
546-
ArrayValue tmp = arrays.pop();
547-
if (tmp.append(array)) { // array filled
548-
array = tmp;
549-
level++;
550-
} else {
551-
array = null;
552-
level--;
553-
}
554-
}
555-
}
556-
557-
return array;
558-
}
559-
530+
560531
/**
561532
* Reads a array into an ArrayValue object.
562533
* @param column - column information
@@ -1070,8 +1041,8 @@ private ClickHouseColumn readDynamicData() throws IOException {
10701041
return ClickHouseColumn.of("v", "Map(" + keyInfo.getOriginalTypeName() + "," + valueInfo.getOriginalTypeName() + ")");
10711042
} else if (tag == ClickHouseDataType.Enum8.getBinTag() || tag == ClickHouseDataType.Enum16.getBinTag()) {
10721043
int constants = readVarInt(input);
1073-
int []values = new int[constants];
1074-
String []names = new String[constants];
1044+
int[] values = new int[constants];
1045+
String[] names = new String[constants];
10751046
ClickHouseDataType enumType = constants > 127 ? ClickHouseDataType.Enum16 : ClickHouseDataType.Enum8;
10761047
for (int i = 0; i < constants; i++) {
10771048
names[i] = readString(input);
@@ -1081,8 +1052,11 @@ private ClickHouseColumn readDynamicData() throws IOException {
10811052
values[i] = readUnsignedShortLE();
10821053
}
10831054
}
1084-
return new ClickHouseColumn(enumType, "v", enumType.name(), false, false, Collections.emptyList(), Collections.emptyList(),
1055+
return new ClickHouseColumn(enumType, "v", enumType.name(), false, false, Collections.emptyList(), Collections.emptyList(),
10851056
new ClickHouseEnum(names, values));
1057+
} else if (tag == ClickHouseDataType.NULLABLE_BIN_TAG) {
1058+
ClickHouseColumn column = readDynamicData();
1059+
return ClickHouseColumn.of("v", "Nullable(" + column.getOriginalTypeName() + ")");
10861060
} else {
10871061
type = ClickHouseDataType.binTag2Type.get(tag);
10881062
if (type == null) {

‎client-v2/src/main/java/com/clickhouse/client/api/data_formats/internal/SerializerUtils.java‎

Lines changed: 63 additions & 50 deletions
Original file line numberDiff line numberDiff line change
@@ -38,6 +38,7 @@
3838
import java.util.Map;
3939
import java.util.Objects;
4040
import java.util.Set;
41+
import java.util.Stack;
4142
import java.util.StringTokenizer;
4243
import java.util.TimeZone;
4344
import java.util.UUID;
@@ -154,10 +155,6 @@ private static Map<Class<?>, ClickHouseColumn> getPredefinedTypeColumnsMap() {
154155
map.put(double[][].class, ClickHouseColumn.of("v", "Array(Array(Float64))"));
155156
map.put(double[][][].class, ClickHouseColumn.of("v", "Array(Array(Array(Float64)))"));
156157

157-
map.put(String[].class, ClickHouseColumn.of("v", "Array(String)"));
158-
map.put(String[][].class, ClickHouseColumn.of("v", "Array(Array(String))"));
159-
map.put(String[][][].class, ClickHouseColumn.of("v", "Array(Array(Array(String)))"));
160-
161158
return Collections.unmodifiableMap(map);
162159
}
163160

@@ -194,7 +191,7 @@ public static ClickHouseColumn valueToColumnForDynamicType(Object value) {
194191
column = ClickHouseColumn.of("v", "Map(" + keyInfo.getOriginalTypeName() + ", " + valueInfo.getOriginalTypeName() + ")");
195192
} else if (value instanceof Enum<?>) {
196193
column = enumValue2Column((Enum)value);
197-
} else if (value instanceof List<?>) {
194+
} else if (value instanceof List<?> || (value !=null && value.getClass().isArray())) {
198195
column = listValue2Column(value);
199196
} else if (value == null) {
200197
column = PREDEFINED_TYPE_COLUMNS.get(Void.class);
@@ -219,28 +216,57 @@ public static ClickHouseColumn valueToColumnForDynamicType(Object value) {
219216
// In this case we need to find max depth.
220217

221218
private static ClickHouseColumn listValue2Column(Object value) {
222-
ClickHouseColumn column;
223-
if (value instanceof List<?>) {
224-
List<?> list = (List<?>) value;
225-
StringBuilder type = new StringBuilder("Array()");
226-
int insertPos = type.length() - 1;
227-
while (!list.isEmpty() && list.get(0) instanceof List<?>) {
228-
type.insert(insertPos, "Array()");
229-
insertPos += 6; // add len of 'Array(' string
230-
list = (List<?>) list.get(0);
231-
}
232-
if (list.isEmpty() || list.get(0) == null) {
233-
type.insert(insertPos, "Nothing");
234-
column = ClickHouseColumn.of("v", type.toString());
235-
} else {
236-
ClickHouseColumn arrayBaseColumn = PREDEFINED_TYPE_COLUMNS.get(list.get(0).getClass());
219+
220+
ClickHouseColumn column = PREDEFINED_TYPE_COLUMNS.get(value.getClass());
221+
if (column != null) {
222+
return column;
223+
}
224+
225+
if (value instanceof List<?> || (value.getClass().isArray())) {
226+
Stack<Object[]> arrays = new Stack<>();
227+
arrays.push(new Object[]{value, 1});
228+
int maxDepth = 0;
229+
boolean hasNulls = false;
230+
ClickHouseColumn arrayBaseColumn = null;
231+
StringBuilder typeStr = new StringBuilder();
232+
int insertPos = 0;
233+
while (!arrays.isEmpty()) {
234+
Object[] arr = arrays.pop();
235+
int depth = (Integer) arr[1];
236+
if (depth > maxDepth) {
237+
maxDepth = depth;
238+
typeStr.insert(insertPos, "Array()");
239+
insertPos += 6;
240+
}
241+
242+
boolean isArray = arr[0].getClass().isArray();
243+
List<?> list = isArray ? null : ((List<?>) arr[0]);
244+
int len = isArray ? Array.getLength(arr[0]) : list.size();
245+
for (int i = 0; i < len; i++) {
246+
Object item = isArray ? Array.get(arr[0], i) : list.get(i);
247+
if (!hasNulls && item == null) {
248+
hasNulls = true;
249+
} else if (item != null && (item instanceof List<?> || item.getClass().isArray())) {
250+
arrays.push(new Object[]{item, depth + 1});
251+
} else if (arrayBaseColumn == null && item != null) {
252+
arrayBaseColumn = PREDEFINED_TYPE_COLUMNS.get(item.getClass());
253+
if (arrayBaseColumn == null) {
254+
throw new ClientException("Cannot serialize " + item.getClass() + " as array element");
255+
}
256+
}
257+
}
258+
237259
if (arrayBaseColumn != null) {
238-
type.insert(insertPos, arrayBaseColumn.getOriginalTypeName());
239-
column = ClickHouseColumn.of("v", type.toString());
240-
} else {
241-
column = null;
260+
if (hasNulls) {
261+
typeStr.insert(insertPos, "Nullable()");
262+
insertPos += 9;
263+
}
264+
typeStr.insert(insertPos, arrayBaseColumn.getOriginalTypeName());
265+
break;
242266
}
243267
}
268+
269+
column = ClickHouseColumn.of("v", typeStr.toString());
244270
} else {
245271
column = null;
246272
}
@@ -370,27 +396,20 @@ public static void writeDynamicTypeTag(OutputStream stream, ClickHouseColumn typ
370396
}
371397

372398
public static void serializeArrayData(OutputStream stream, Object value, ClickHouseColumn column) throws IOException {
399+
if (value == null) {
400+
writeVarInt(stream, 0);
401+
return;
402+
}
373403

374-
if (value instanceof List<?>) {
375-
//Serialize the array to the stream
376-
//The array is a list of values
377-
List<?> values = (List<?>) value;
378-
writeVarInt(stream, values.size());
379-
for (Object val : values) {
380-
if (column.getArrayBaseColumn().isNullable()) {
381-
if (val == null) {
382-
writeNull(stream);
383-
continue;
384-
}
385-
writeNonNull(stream);
386-
}
387-
serializeData(stream, val, column.getNestedColumns().get(0));
388-
}
389-
} else if (value.getClass().isArray()) {
390-
writeVarInt(stream, Array.getLength(value));
391-
for (int i = 0; i < Array.getLength(value); i++) {
392-
Object val = Array.get(value, i);
393-
if (column.getArrayBaseColumn().isNullable()) {
404+
boolean isArray = value.getClass().isArray();
405+
if (value instanceof List<?> || isArray) {
406+
List<?> list = isArray ? null : (List<?>)value;
407+
int len = isArray ? Array.getLength(value) : list.size();
408+
409+
writeVarInt(stream, len);
410+
for (int i = 0; i < len; i++) {
411+
Object val = isArray? Array.get(value, i) : list.get(i);
412+
if (column.getArrayNestedLevel() == 1 && column.getArrayBaseColumn().isNullable()) {
394413
if (val == null) {
395414
writeNull(stream);
396415
continue;
@@ -515,12 +534,6 @@ private static void serializePrimitiveData(OutputStream stream, Object value, Cl
515534
case UUID:
516535
BinaryStreamUtils.writeUuid(stream, (UUID) value);
517536
break;
518-
// case Enum8:
519-
// BinaryStreamUtils.writeEnum8(stream, (Byte) value);
520-
// break;
521-
// case Enum16:
522-
// BinaryStreamUtils.writeEnum16(stream, convertToInteger(value));
523-
// break;
524537
case Enum8:
525538
case Enum16:
526539
serializeEnumData(stream, column, value);

‎client-v2/src/test/java/com/clickhouse/client/datatypes/DataTypeTests.java‎

Lines changed: 14 additions & 23 deletions
Original file line numberDiff line numberDiff line change
@@ -27,6 +27,7 @@
2727
import java.time.temporal.ChronoUnit;
2828
import java.util.ArrayList;
2929
import java.util.Arrays;
30+
import java.util.Collections;
3031
import java.util.HashMap;
3132
import java.util.List;
3233
import java.util.Map;
@@ -389,13 +390,6 @@ public void testDynamicWithPrimitives() throws Exception {
389390
int rowId = 0;
390391
for (ClickHouseDataType dataType : ClickHouseDataType.values()) {
391392
switch (dataType) {
392-
case Date:
393-
case Date32:
394-
case DateTime:
395-
case DateTime32:
396-
case DateTime64:
397-
// requires fix
398-
continue;
399393
case Array:
400394
case Map:
401395
case AggregateFunction:
@@ -481,36 +475,33 @@ public void testDynamicWithArrays() throws Exception {
481475
testDynamicWith("arrays",
482476
new Object[]{
483477
"a,b",
484-
new String[]{"a", "b"},
485-
Arrays.asList("c", "d")
478+
new String[]{"a", null, "b"},
479+
Arrays.asList("c", "d"),
480+
new Integer[]{1, null, 2, null, 3}
481+
486482
},
487483
new String[]{
488484
"a,b",
489-
"[a, b]",
490-
"[c, d]"
485+
"[a, null, b]",
486+
"[c, d]",
487+
"[1, null, 2, null, 3]"
491488
});
492489
testDynamicWith("arrays",
493490
new Object[]{
494491
new int[]{1, 2},
495492
new String[]{"a", "b"},
496-
Arrays.asList("c", "d")
493+
Arrays.asList("c", "d"),
494+
Arrays.asList(Arrays.asList(1, 2), Arrays.asList(3, 4)),
495+
Arrays.asList(Arrays.asList(1, 2), Collections.emptyList()),
496+
Arrays.asList(Arrays.asList(1, 2), null, Arrays.asList(3, 4))
497497
},
498498
new String[]{
499499
"[1, 2]",
500500
"[a, b]",
501501
"[c, d]",
502-
});
503-
504-
testDynamicWith("arrays",
505-
new Object[]{
506-
new int[][]{ new int[] {1, 2}, new int[] { 3, 4}},
507-
new String[][]{new String[]{"a", "b"}, new String[]{"c", "d"}},
508-
Arrays.asList(Arrays.asList("e", "f"), Arrays.asList("j", "h"))
509-
},
510-
new String[]{
511502
"[[1, 2], [3, 4]]",
512-
"[[a, b], [c, d]]",
513-
"[[e, f], [j, h]]",
503+
"[[1, 2], []]",
504+
"[[1, 2], [], [3, 4]]"
514505
});
515506
}
516507

0 commit comments

Comments
 (0)