Skip to content

Commit 14fbca6

Browse files
chernserchernser
committed
implemented support of arrays with different depth
1 parent 004d604 commit 14fbca6

File tree

9 files changed

+150
-30
lines changed

9 files changed

+150
-30
lines changed

‎clickhouse-data/src/main/java/com/clickhouse/data/ClickHouseColumn.java‎

Lines changed: 36 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -41,13 +41,15 @@
4141
import java.time.OffsetDateTime;
4242
import java.util.ArrayList;
4343
import java.util.Arrays;
44+
import java.util.Collection;
4445
import java.util.Collections;
4546
import java.util.Comparator;
4647
import java.util.HashMap;
4748
import java.util.LinkedList;
4849
import java.util.List;
4950
import java.util.Map;
5051
import java.util.Objects;
52+
import java.util.Set;
5153
import java.util.TimeZone;
5254

5355
/**
@@ -99,6 +101,8 @@ public final class ClickHouseColumn implements Serializable {
99101

100102
private Map<Class<?>, Integer> classToVariantOrdNumMap;
101103

104+
private Map<Class<?>, Integer> arrayToVariantOrdNumMap;
105+
102106
private static ClickHouseColumn update(ClickHouseColumn column) {
103107
column.enumConstants = ClickHouseEnum.EMPTY;
104108
int size = column.parameters.size();
@@ -446,6 +450,21 @@ protected static int readColumn(String args, int startIndex, int len, String nam
446450
}
447451
}
448452
column.classToVariantOrdNumMap = ClickHouseDataType.buildVariantMapping(variantDataTypes);
453+
454+
for (int ordNum = 0; ordNum < nestedColumns.size(); ordNum++) {
455+
ClickHouseColumn nestedColumn = nestedColumns.get(ordNum);
456+
if (nestedColumn.getDataType() == ClickHouseDataType.Array) {
457+
Set<Class<?>> classSet = ClickHouseDataType.DATA_TYPE_TO_CLASS.get(nestedColumn.arrayBaseColumn.dataType);
458+
if (classSet != null) {
459+
if (column.arrayToVariantOrdNumMap == null) {
460+
column.arrayToVariantOrdNumMap = new HashMap<>();
461+
}
462+
for (Class<?> c : classSet) {
463+
column.arrayToVariantOrdNumMap.put(c, ordNum);
464+
}
465+
}
466+
}
467+
}
449468
}
450469

451470
if (column == null) {
@@ -650,7 +669,23 @@ public boolean isAggregateFunction() {
650669
}
651670

652671
public int getVariantOrdNum(Object value) {
653-
return classToVariantOrdNumMap.getOrDefault(value.getClass(), -1);
672+
if (value != null && value.getClass().isArray()) {
673+
Class<?> c = value.getClass();
674+
while (c.isArray()) {
675+
c = c.getComponentType();
676+
}
677+
return arrayToVariantOrdNumMap.getOrDefault(c, -1);
678+
} else if (value != null && value instanceof List<?>) {
679+
Object tmpV = ((List)value).get(0);
680+
Class<?> valueClass = tmpV.getClass();
681+
while (tmpV instanceof List<?>) {
682+
tmpV = ((List)tmpV).get(0);
683+
valueClass = tmpV.getClass();
684+
}
685+
return arrayToVariantOrdNumMap.getOrDefault(valueClass, -1);
686+
} else {
687+
return classToVariantOrdNumMap.getOrDefault(value.getClass(), -1);
688+
}
654689
}
655690

656691
public boolean isArray() {

‎clickhouse-data/src/main/java/com/clickhouse/data/ClickHouseDataType.java‎

Lines changed: 3 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -151,15 +151,15 @@ public static Map<Class<?>, Integer> buildVariantMapping(List<ClickHouseDataType
151151
}
152152
}
153153

154-
// add numbers mappings
154+
// add integers
155155
for (java.util.Map.Entry<ClickHouseDataType, Integer> entry : intTypesMappings.entrySet()) {
156156
DATA_TYPE_TO_CLASS.get(entry.getKey()).forEach(c -> variantMapping.put(c, entry.getValue()));
157157
}
158+
// add decimals
158159
for (java.util.Map.Entry<ClickHouseDataType, Integer> entry : decTypesMappings.entrySet()) {
159160
DATA_TYPE_TO_CLASS.get(entry.getKey()).forEach(c -> variantMapping.put(c, entry.getValue()));
160161
}
161162

162-
163163
return variantMapping;
164164
}
165165

@@ -211,6 +211,7 @@ static Map<ClickHouseDataType, Set<Class<?>>> dataTypeClassMap() {
211211

212212
map.put(Enum8, setOf(java.lang.String.class,byte.class, Byte.class, short.class, Short.class, int.class, Integer.class, long.class, Long.class));
213213
map.put(Enum16, setOf(java.lang.String.class,byte.class, Byte.class, short.class, Short.class, int.class, Integer.class, long.class, Long.class));
214+
map.put(Array, setOf(List.class, Object[].class, byte[].class, short[].class, int[].class, long[].class, boolean[].class));
214215
return map;
215216
}
216217

‎client-v2/src/main/java/com/clickhouse/client/api/Client.java‎

Lines changed: 7 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -1807,7 +1807,13 @@ public <T> List<T> queryAll(String sqlQuery, Class<T> clazz, TableSchema schema,
18071807
(RowBinaryWithNamesAndTypesFormatReader) newBinaryFormatReader(response);
18081808

18091809
while (true) {
1810-
Object record = allocator == null ? clazz.getDeclaredConstructor().newInstance() : allocator.get();
1810+
1811+
Object record;
1812+
try {
1813+
record = allocator == null ? clazz.getDeclaredConstructor().newInstance() : allocator.get();
1814+
} catch (NoSuchMethodException e) {
1815+
throw new ClientException("Failed to instantiate DTO to store data: no-args constructor is not defined");
1816+
}
18111817
if (reader.readToPOJO(classDeserializers, record)) {
18121818
records.add((T) record);
18131819
} else {

‎client-v2/src/main/java/com/clickhouse/client/api/data_formats/internal/AbstractBinaryFormatReader.java‎

Lines changed: 3 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -109,7 +109,7 @@ public boolean readToPOJO(Map<String, POJOSetter> deserializers, Object obj ) th
109109
}
110110
throw e;
111111
} catch (Exception e) {
112-
throw new ClientException("Failed to put value of '" + column.getColumnName() + "' into POJO", e);
112+
throw new ClientException("Failed to set value of '" + column.getColumnName(), e);
113113
}
114114
}
115115
return true;
@@ -303,6 +303,8 @@ public static String readAsString(Object value, ClickHouseColumn column) {
303303
} else if (dataType == ClickHouseDataType.Enum8 || dataType == ClickHouseDataType.Enum16) {
304304
return column.getEnumConstants().name(num);
305305
}
306+
} else if (value instanceof BinaryStreamReader.ArrayValue) {
307+
return ((BinaryStreamReader.ArrayValue)value).asList().toString();
306308
}
307309
return value.toString();
308310
}

‎client-v2/src/main/java/com/clickhouse/client/api/data_formats/internal/BinaryStreamReader.java‎

Lines changed: 3 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -247,6 +247,9 @@ private static <T> T convertArray(ArrayValue value, Class<?> typeHint) {
247247
if (typeHint.isAssignableFrom(List.class)) {
248248
return (T) value.asList();
249249
}
250+
if (typeHint.isArray()) {
251+
return (T) value.array;
252+
}
250253

251254
return (T) value;
252255
}

client-v2/src/test/java/com/clickhouse/client/api/DataTypeTests.java renamed to client-v2/src/test/java/com/clickhouse/client/api/DataTypeUtilsTests.java

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -6,7 +6,7 @@
66

77
import static org.testng.AssertJUnit.assertEquals;
88

9-
public class DataTypeTests {
9+
public class DataTypeUtilsTests {
1010

1111

1212
@Test

‎client-v2/src/test/java/com/clickhouse/client/datatypes/DataTypeTests.java‎

Lines changed: 96 additions & 22 deletions
Original file line numberDiff line numberDiff line change
@@ -5,13 +5,16 @@
55
import com.clickhouse.client.ClickHouseProtocol;
66
import com.clickhouse.client.api.Client;
77
import com.clickhouse.client.api.command.CommandSettings;
8+
import com.clickhouse.client.api.data_formats.internal.BinaryStreamReader;
89
import com.clickhouse.client.api.enums.Protocol;
910
import com.clickhouse.client.api.insert.InsertResponse;
1011
import com.clickhouse.client.api.insert.InsertSettings;
12+
import com.clickhouse.client.api.metadata.TableSchema;
1113
import com.clickhouse.client.api.query.GenericRecord;
1214
import com.clickhouse.data.ClickHouseDataType;
1315
import lombok.AllArgsConstructor;
1416
import lombok.Data;
17+
import lombok.NoArgsConstructor;
1518
import org.testng.Assert;
1619
import org.testng.annotations.AfterMethod;
1720
import org.testng.annotations.BeforeMethod;
@@ -25,6 +28,9 @@
2528
import java.util.Arrays;
2629
import java.util.Collection;
2730
import java.util.List;
31+
import java.util.concurrent.atomic.AtomicInteger;
32+
import java.util.function.BiConsumer;
33+
import java.util.function.Consumer;
2834

2935
public class DataTypeTests extends BaseIntegrationTest {
3036

@@ -59,39 +65,83 @@ public void setUp() throws IOException {
5965
.build();
6066
}
6167

62-
@AfterMethod(groups = { "integration" })
68+
@AfterMethod(groups = {"integration"})
6369
public void tearDown() {
6470
client.close();
6571
}
6672

73+
private <T> void writeReadVerify(String table, String tableDef, Class<T> dtoClass, List<T> data,
74+
BiConsumer<List<T>, T> rowVerifier) throws Exception {
75+
client.execute("DROP TABLE IF EXISTS " + table).get();
76+
client.execute(tableDef);
77+
78+
final TableSchema tableSchema = client.getTableSchema(table);
79+
client.register(dtoClass, tableSchema);
80+
client.insert(table, data);
81+
final AtomicInteger rowCount = new AtomicInteger(0);
82+
client.queryAll("SELECT * FROM " + table, dtoClass, tableSchema).forEach(dto -> {
83+
rowVerifier.accept(data, dto);
84+
rowCount.incrementAndGet();
85+
});
86+
87+
Assert.assertEquals(rowCount.get(), data.size());
88+
}
6789

6890
@Test(groups = {"integration"})
6991
public void testNestedDataTypes() throws Exception {
7092
final String table = "test_nested_types";
71-
String tblCreateSQL = NestedTypesDTO.tblCreateSQL(table);
72-
client.execute("DROP TABLE IF EXISTS " + table).get();
73-
client.execute(tblCreateSQL);
93+
writeReadVerify(table,
94+
NestedTypesDTO.tblCreateSQL(table),
95+
NestedTypesDTO.class,
96+
Arrays.asList(new NestedTypesDTO(0, new Object[]{(short) 127, "test 1"}, new double[]{0.3d, 0.4d})),
97+
(data, dto) -> {
98+
NestedTypesDTO dataDto = data.get(dto.getRowId());
99+
Assert.assertEquals(dto.getTuple1(), dataDto.getTuple1());
100+
Assert.assertEquals(dto.getPoint1(), dataDto.getPoint1());
101+
});
102+
}
74103

75-
client.register(NestedTypesDTO.class, client.getTableSchema(table));
104+
@Test(groups = {"integration"})
105+
public void testArrays() throws Exception {
106+
final String table = "test_arrays";
107+
writeReadVerify(table,
108+
DTOForArraysTests.tblCreateSQL(table),
109+
DTOForArraysTests.class,
110+
Arrays.asList(new DTOForArraysTests(
111+
0, Arrays.asList("db", "fast"), new int[]{1, 2, 3}, new String[]{"a", "b", "c"})),
112+
(data, dto) -> {
113+
DTOForArraysTests dataDto = data.get(dto.getRowId());
114+
System.out.println(dto.getWords());
115+
Assert.assertEquals(dto.getWords(), dataDto.getWords());
116+
System.out.println(Arrays.asList(dto.getLetters()));
117+
Assert.assertEquals(dto.getLetters(), dataDto.getLetters());
118+
System.out.println(Arrays.asList(dto.getNumbers()));
119+
Assert.assertEquals(dto.getNumbers(), dataDto.getNumbers());
120+
});
121+
}
76122

77-
List<NestedTypesDTO> data =
78-
Arrays.asList(new NestedTypesDTO(0, new Object[] {(short)127, "test 1"}, new Double[] {0.3d, 0.4d} ));
79-
client.insert(table, data);
123+
@Data
124+
@AllArgsConstructor
125+
@NoArgsConstructor
126+
public static class DTOForArraysTests {
127+
private int rowId;
80128

81-
List<GenericRecord> rows = client.queryAll("SELECT * FROM " + table);
82-
for (GenericRecord row : rows) {
83-
NestedTypesDTO dto = data.get(row.getInteger("rowId"));
84-
Assert.assertEquals(row.getTuple("tuple1"), dto.getTuple1());
85-
Assert.assertEquals(row.getGeoPoint("point1").getValue(), dto.getPoint1());
86-
}
129+
private List<String> words;
130+
131+
private int[] numbers;
87132

133+
private String[] letters;
134+
135+
public static String tblCreateSQL(String table) {
136+
return tableDefinition(table, "rowId Int16", "words Array(String)", "numbers Array(Int32)",
137+
"letters Array(String)");
138+
}
88139
}
89140

90141
@Test(groups = {"integration"})
91142
public void testVariantWithSimpleDataTypes() throws Exception {
92143
final String table = "test_variant_primitives";
93144
final DataTypesTestingPOJO sample = new DataTypesTestingPOJO();
94-
System.out.println("sample: " + sample);
95145

96146
dataTypesLoop:
97147
for (ClickHouseDataType dataType : ClickHouseDataType.values()) {
@@ -168,7 +218,7 @@ public void testVariantWithSimpleDataTypes() throws Exception {
168218
case DateTime:
169219
case DateTime32:
170220
strValue = row.getLocalDateTime("field").truncatedTo(ChronoUnit.SECONDS).toString();
171-
value = ((LocalDateTime)value ).truncatedTo(ChronoUnit.SECONDS).toString();
221+
value = ((LocalDateTime) value).truncatedTo(ChronoUnit.SECONDS).toString();
172222
break;
173223
case Point:
174224
strValue = row.getGeoPoint("field").toString();
@@ -183,7 +233,7 @@ public void testVariantWithSimpleDataTypes() throws Exception {
183233
strValue = row.getGeoMultiPolygon("field").toString();
184234
break;
185235
}
186-
System.out.println("field: " + strValue + " value " + value);
236+
System.out.println("field: " + strValue + " value " + value);
187237
if (value.getClass().isPrimitive()) {
188238
Assert.assertEquals(strValue, String.valueOf(value));
189239
} else {
@@ -222,17 +272,41 @@ public void testVariantWithDecimals() throws Exception {
222272
});
223273
}
224274

225-
@Test(groups = {"integration"}, enabled = false)
275+
@Test(groups = {"integration"})
226276
public void testVariantWithArrays() throws Exception {
227-
// TODO: writing array would need custom serialization logic
228277
testVariantWith("arrays", new String[]{"field Variant(String, Array(String))"},
229278
new Object[]{
230279
"a,b",
231-
new String[]{"a", "b"}
280+
new String[]{"a", "b"},
281+
Arrays.asList("c", "d")
232282
},
233283
new String[]{
234284
"a,b",
235-
"a,b",
285+
"[a, b]",
286+
"[c, d]"
287+
});
288+
testVariantWith("arrays", new String[]{"field Variant(Array(String), Array(Int32))"},
289+
new Object[]{
290+
new int[]{1, 2},
291+
new String[]{"a", "b"},
292+
Arrays.asList("c", "d")
293+
},
294+
new String[]{
295+
"[1, 2]",
296+
"[a, b]",
297+
"[c, d]",
298+
});
299+
300+
testVariantWith("arrays", new String[]{"field Variant(Array(Array(String)), Array(Array(Int32)))"},
301+
new Object[]{
302+
new int[][]{ new int[] {1, 2}, new int[] { 3, 4}},
303+
new String[][]{new String[]{"a", "b"}, new String[]{"c", "d"}},
304+
// Arrays.asList(Arrays.asList("e", "f"), Arrays.asList("j", "h"))
305+
},
306+
new String[]{
307+
"[[1, 2], [3, 4]]",
308+
"[[a, b], [c, d]]",
309+
// "[c, d]",
236310
});
237311
}
238312

@@ -268,7 +342,7 @@ public void testVariantWithTuple() throws Exception {
268342
testVariantWith("arrays", new String[]{"field Variant(String, Tuple(Int32, Float32))"},
269343
new Object[]{
270344
"10,0.34",
271-
new Object[] { 10, 0.34f}
345+
new Object[]{10, 0.34f}
272346
},
273347
new String[]{
274348
"10,0.34",

‎client-v2/src/test/java/com/clickhouse/client/datatypes/NestedTypesDTO.java‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -17,7 +17,7 @@ public class NestedTypesDTO {
1717

1818
private Object[] tuple1;
1919

20-
private Object[] point1;
20+
private double[] point1;
2121

2222
public static String tblCreateSQL(String table) {
2323
return tableDefinition(table,

‎client-v2/src/test/java/com/clickhouse/client/query/QueryTests.java‎

Lines changed: 0 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -1149,7 +1149,6 @@ public void testEnums() {
11491149

11501150
testDataTypes(columns, valueGenerators, verifiers);
11511151
}
1152-
11531152
@Test
11541153
public void testUUID() {
11551154
final List<String> columns = Arrays.asList(

0 commit comments

Comments
 (0)