Skip to content

Commit 004d604

Browse files
chernserchernser
committed
implemented enum support for variant. some fixes
1 parent ab269e0 commit 004d604

File tree

6 files changed

+154
-59
lines changed

6 files changed

+154
-59
lines changed

‎clickhouse-data/src/main/java/com/clickhouse/data/ClickHouseDataType.java‎

Lines changed: 3 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -208,6 +208,9 @@ static Map<ClickHouseDataType, Set<Class<?>>> dataTypeClassMap() {
208208
map.put(DateTime64, setOf(LocalDateTime.class, ZonedDateTime.class));
209209
map.put(DateTime32, setOf(LocalDateTime.class, ZonedDateTime.class));
210210
map.put(DateTime, setOf(LocalDateTime.class, ZonedDateTime.class));
211+
212+
map.put(Enum8, setOf(java.lang.String.class,byte.class, Byte.class, short.class, Short.class, int.class, Integer.class, long.class, Long.class));
213+
map.put(Enum16, setOf(java.lang.String.class,byte.class, Byte.class, short.class, Short.class, int.class, Integer.class, long.class, Long.class));
211214
return map;
212215
}
213216

‎client-v2/src/main/java/com/clickhouse/client/api/data_formats/internal/AbstractBinaryFormatReader.java‎

Lines changed: 21 additions & 7 deletions
Original file line numberDiff line numberDiff line change
@@ -271,23 +271,37 @@ public TableSchema getSchema() {
271271

272272
@Override
273273
public String getString(String colName) {
274-
Object value = readValue(colName);
274+
return readAsString(readValue(colName), schema.getColumnByName(colName));
275+
}
276+
277+
/**
278+
* Converts value in to a string representation. Does some formatting for selected data types
279+
* @return string representation of a value for specified column
280+
*/
281+
public static String readAsString(Object value, ClickHouseColumn column) {
275282
if (value == null) {
276283
return null;
277284
} else if (value instanceof String) {
278285
return (String) value;
279286
} else if (value instanceof ZonedDateTime) {
280-
ClickHouseDataType dataType = schema.getColumnByName(colName).getDataType();
287+
ClickHouseDataType dataType = column.getDataType();
281288
ZonedDateTime zdt = (ZonedDateTime) value;
282289
if (dataType == ClickHouseDataType.Date) {
283290
return zdt.format(com.clickhouse.client.api.DataTypeUtils.DATE_FORMATTER).toString();
284291
}
285292
return value.toString();
286-
} else {
287-
ClickHouseDataType dataType = schema.getColumnByName(colName).getDataType();
288-
if (dataType == ClickHouseDataType.Enum8 || dataType == ClickHouseDataType.Enum16) {
289-
ClickHouseEnum clickHouseEnum = schema.getColumnByName(colName).getEnumConstants();
290-
return clickHouseEnum.name(Integer.parseInt(value.toString()));
293+
} else if (value instanceof Number ) {
294+
ClickHouseDataType dataType = column.getDataType();
295+
int num = ((Number)value).intValue();
296+
if (column.getDataType() == ClickHouseDataType.Variant) {
297+
for (ClickHouseColumn c : column.getNestedColumns()) {
298+
// TODO: will work only if single enum listed as variant
299+
if (c.getDataType() == ClickHouseDataType.Enum8 || c.getDataType() == ClickHouseDataType.Enum16) {
300+
return c.getEnumConstants().name(num);
301+
}
302+
}
303+
} else if (dataType == ClickHouseDataType.Enum8 || dataType == ClickHouseDataType.Enum16) {
304+
return column.getEnumConstants().name(num);
291305
}
292306
}
293307
return value.toString();

‎client-v2/src/main/java/com/clickhouse/client/api/data_formats/internal/MapBackedRecord.java‎

Lines changed: 2 additions & 15 deletions
Original file line numberDiff line numberDiff line change
@@ -47,25 +47,12 @@ public <T> T readValue(String colName) {
4747

4848
@Override
4949
public String getString(String colName) {
50-
Object value = readValue(colName);
51-
if (value == null) {
52-
return null;
53-
} else if (value instanceof String) {
54-
return (String) value;
55-
}
56-
return value.toString();
50+
return AbstractBinaryFormatReader.readAsString(readValue(colName), schema.getColumnByName(colName));
5751
}
5852

5953
@Override
6054
public String getString(int index) {
61-
// TODO: it may be incorrect to call .toString() on some objects
62-
Object value = readValue(index);
63-
if (value == null) {
64-
return null;
65-
} else if (value instanceof String) {
66-
return (String) value;
67-
}
68-
return value.toString();
55+
return getString(schema.columnIndexToName(index));
6956
}
7057

7158
private <T> T readNumberValue(String colName, NumberConverter.NumberType targetType) {

‎client-v2/src/main/java/com/clickhouse/client/api/data_formats/internal/SerializerUtils.java‎

Lines changed: 34 additions & 28 deletions
Original file line numberDiff line numberDiff line change
@@ -2,6 +2,8 @@
22

33
import com.clickhouse.client.api.Client;
44
import com.clickhouse.client.api.ClientException;
5+
import com.clickhouse.client.api.data_formats.RowBinaryFormatSerializer;
6+
import com.clickhouse.client.api.data_formats.RowBinaryFormatWriter;
57
import com.clickhouse.client.api.query.POJOSetter;
68
import com.clickhouse.data.ClickHouseAggregateFunction;
79
import com.clickhouse.data.ClickHouseColumn;
@@ -61,7 +63,7 @@ public static void serializeData(OutputStream stream, Object value, ClickHouseCo
6163
serializeArrayData(stream, value, column);
6264
break;
6365
case Tuple:
64-
serializeTuple(stream, value, column);
66+
serializeTupleData(stream, value, column);
6567
break;
6668
case Map:
6769
serializeMapData(stream, value, column);
@@ -74,7 +76,7 @@ public static void serializeData(OutputStream stream, Object value, ClickHouseCo
7476
break;
7577
case Point:
7678
value = value instanceof ClickHouseGeoPointValue ? ((ClickHouseGeoPointValue)value).getValue() : value;
77-
serializeTuple(stream, value, GEO_POINT_TUPLE);
79+
serializeTupleData(stream, value, GEO_POINT_TUPLE);
7880
break;
7981
case Ring:
8082
value = value instanceof ClickHouseGeoRingValue ? ((ClickHouseGeoRingValue)value).getValue() : value;
@@ -128,22 +130,16 @@ private static void serializeArrayData(OutputStream stream, Object value, ClickH
128130
}
129131
}
130132

131-
private static void serializeTuple(OutputStream stream, Object value, ClickHouseColumn column) throws IOException {
133+
private static void serializeTupleData(OutputStream stream, Object value, ClickHouseColumn column) throws IOException {
132134
//Serialize the tuple to the stream
133135
//The tuple is a list of values
134136
if (value instanceof List) {
135137
List<?> values = (List<?>) value;
136138
for (int i = 0; i < values.size(); i++) {
137139
serializeData(stream, values.get(i), column.getNestedColumns().get(i));
138140
}
139-
}
140-
// else if (value instanceof Object[]) {
141-
// Object[] values = (Object[]) value;
142-
// for (int i = 0; i < values.length; i++) {
143-
// serializeData(stream, values[i], column.getNestedColumns().get(i));
144-
// }
145-
// }
146-
else if (value.getClass().isArray()) {
141+
} else if (value.getClass().isArray()) {
142+
// TODO: this code uses reflection - we might need to measure it and find faster solution.
147143
for (int i = 0; i < Array.getLength(value); i++) {
148144
serializeData(stream, Array.get(value, i), column.getNestedColumns().get(i));
149145
}
@@ -217,7 +213,7 @@ private static void serializePrimitiveData(OutputStream stream, Object value, Cl
217213
case Decimal64:
218214
case Decimal128:
219215
case Decimal256:
220-
BinaryStreamUtils.writeDecimal(stream, (BigDecimal) value, column.getPrecision(), column.getScale());
216+
BinaryStreamUtils.writeDecimal(stream, convertToBigDecimal(value), column.getPrecision(), column.getScale());
221217
break;
222218
case Bool:
223219
BinaryStreamUtils.writeBoolean(stream, (Boolean) value);
@@ -247,11 +243,15 @@ private static void serializePrimitiveData(OutputStream stream, Object value, Cl
247243
case UUID:
248244
BinaryStreamUtils.writeUuid(stream, (UUID) value);
249245
break;
246+
// case Enum8:
247+
// BinaryStreamUtils.writeEnum8(stream, (Byte) value);
248+
// break;
249+
// case Enum16:
250+
// BinaryStreamUtils.writeEnum16(stream, convertToInteger(value));
251+
// break;
250252
case Enum8:
251-
BinaryStreamUtils.writeEnum8(stream, (Byte) value);
252-
break;
253253
case Enum16:
254-
BinaryStreamUtils.writeEnum16(stream, convertToInteger(value));
254+
serializeEnumData(stream, column, value);
255255
break;
256256
case IPv4:
257257
BinaryStreamUtils.writeInet4Address(stream, (Inet4Address) value);
@@ -267,26 +267,32 @@ private static void serializePrimitiveData(OutputStream stream, Object value, Cl
267267
}
268268
}
269269

270+
private static void serializeEnumData(OutputStream stream, ClickHouseColumn column, Object value) throws IOException {
271+
int enumValue = -1;
272+
if (value instanceof String) {
273+
enumValue = column.getEnumConstants().value((String) value);
274+
} else if (value instanceof Number) {
275+
enumValue = ((Number)value).intValue();
276+
} else {
277+
throw new IllegalArgumentException("Cannot write value of class " + value.getClass() + " into column with Enum type " + column.getOriginalTypeName());
278+
}
279+
280+
if (column.getDataType() == ClickHouseDataType.Enum8) {
281+
BinaryStreamUtils.writeInt8(stream, enumValue);
282+
} else if (column.getDataType() == ClickHouseDataType.Enum16) {
283+
BinaryStreamUtils.writeInt16(stream, enumValue);
284+
} else {
285+
throw new ClientException("Bug! serializeEnumData() was called for " + column.getDataType());
286+
}
287+
}
288+
270289
private static void serializeJSON(OutputStream stream, Object value) throws IOException {
271290
if (value instanceof String) {
272291
BinaryStreamUtils.writeString(stream, (String)value);
273292
} else {
274293
throw new UnsupportedOperationException("Serialization of Java object to JSON is not supported yet.");
275294
}
276295
}
277-
//
278-
// private static void serializeTuple(OutputStream out, ClickHouseColumn column, Object[] tupleValues) throws IOException {
279-
// if (column.getNestedColumns().size() != tupleValues.length) {
280-
// throw new IllegalArgumentException("Column " + column.getColumnName() + " defines as Tuple with "
281-
// + column.getNestedColumns().size() +" elements, but only " + tupleValues.length + " provided");
282-
// }
283-
//
284-
// List<ClickHouseColumn> nested = column.getNestedColumns();
285-
// for (int i = 0; i < nested.size() ; i++) {
286-
// serializeData(out, tupleValues[i], nested.get(i));
287-
// }
288-
// }
289-
290296

291297
private static void serializerVariant(OutputStream out, ClickHouseColumn column, Object value) throws IOException {
292298
int typeOrdNum = column.getVariantOrdNum(value);

‎client-v2/src/test/java/com/clickhouse/client/datatypes/DataTypeTests.java‎

Lines changed: 89 additions & 8 deletions
Original file line numberDiff line numberDiff line change
@@ -6,6 +6,7 @@
66
import com.clickhouse.client.api.Client;
77
import com.clickhouse.client.api.command.CommandSettings;
88
import com.clickhouse.client.api.enums.Protocol;
9+
import com.clickhouse.client.api.insert.InsertResponse;
910
import com.clickhouse.client.api.insert.InsertSettings;
1011
import com.clickhouse.client.api.query.GenericRecord;
1112
import com.clickhouse.data.ClickHouseDataType;
@@ -22,6 +23,7 @@
2223
import java.time.temporal.ChronoUnit;
2324
import java.util.ArrayList;
2425
import java.util.Arrays;
26+
import java.util.Collection;
2527
import java.util.List;
2628

2729
public class DataTypeTests extends BaseIntegrationTest {
@@ -63,7 +65,7 @@ public void tearDown() {
6365
}
6466

6567

66-
@Test
68+
@Test(groups = {"integration"})
6769
public void testNestedDataTypes() throws Exception {
6870
final String table = "test_nested_types";
6971
String tblCreateSQL = NestedTypesDTO.tblCreateSQL(table);
@@ -85,7 +87,7 @@ public void testNestedDataTypes() throws Exception {
8587

8688
}
8789

88-
@Test
90+
@Test(groups = {"integration"})
8991
public void testVariantWithSimpleDataTypes() throws Exception {
9092
final String table = "test_variant_primitives";
9193
final DataTypesTestingPOJO sample = new DataTypesTestingPOJO();
@@ -198,24 +200,103 @@ public static class DTOForVariantPrimitivesTests {
198200
private Object field;
199201
}
200202

203+
@Test(groups = {"integration"})
201204
public void testVariantWithDecimals() throws Exception {
202-
205+
testVariantWith("decimals", new String[]{"field Variant(String, Decimal(4, 4))"},
206+
new Object[]{
207+
"10.2",
208+
10.2d, // TODO: when f it gives 10.199
209+
},
210+
new String[]{
211+
"10.2",
212+
"10.2000",
213+
});
214+
testVariantWith("decimal32", new String[]{"field Variant(String, Decimal32(4))"},
215+
new Object[]{
216+
"10.202",
217+
10.1233d,
218+
},
219+
new String[]{
220+
"10.202",
221+
"10.1233",
222+
});
203223
}
204224

205-
public void testVariantWithDateTime() throws Exception {
225+
@Test(groups = {"integration"}, enabled = false)
226+
public void testVariantWithArrays() throws Exception {
227+
// TODO: writing array would need custom serialization logic
228+
testVariantWith("arrays", new String[]{"field Variant(String, Array(String))"},
229+
new Object[]{
230+
"a,b",
231+
new String[]{"a", "b"}
232+
},
233+
new String[]{
234+
"a,b",
235+
"a,b",
236+
});
237+
}
206238

239+
@Test(groups = {"integration"}, enabled = false)
240+
public void testVariantWithMaps() throws Exception {
241+
//TODO: similar to arrays
207242
}
208243

209-
public void testVariantWithNullable() throws Exception {
244+
@Test(groups = {"integration"})
245+
public void testVariantWithEnums() throws Exception {
246+
testVariantWith("enums", new String[]{"field Variant(Bool, Enum('stopped' = 1, 'running' = 2))"},
247+
new Object[]{
248+
"stopped",
249+
1,
250+
"running",
251+
2,
252+
true,
253+
false
254+
},
255+
new String[]{
256+
"stopped",
257+
"stopped",
258+
"running",
259+
"running",
260+
"true",
261+
"false"
262+
});
263+
}
210264

265+
@Test(groups = {"integration"}, enabled = false)
266+
public void testVariantWithTuple() throws Exception {
267+
// TODO: same as array
268+
testVariantWith("arrays", new String[]{"field Variant(String, Tuple(Int32, Float32))"},
269+
new Object[]{
270+
"10,0.34",
271+
new Object[] { 10, 0.34f}
272+
},
273+
new String[]{
274+
"10,0.34",
275+
"(10,0.34)",
276+
});
211277
}
212278

213-
public void testVariantWithArrays() throws Exception {
279+
private void testVariantWith(String withWhat, String[] fields, Object[] values, String[] expectedStrValues) throws Exception {
280+
String table = "test_variant_with_" + withWhat;
281+
String[] actualFields = new String[fields.length + 1];
282+
actualFields[0] = "rowId Int32";
283+
System.arraycopy(fields, 0, actualFields, 1, fields.length);
284+
client.execute("DROP TABLE IF EXISTS " + table).get();
285+
client.execute(tableDefinition(table, actualFields), (CommandSettings) new CommandSettings().serverSetting("enable_variant_type", "1")).get();
214286

215-
}
287+
client.register(DTOForVariantPrimitivesTests.class, client.getTableSchema(table));
216288

217-
public void testVariantWithMaps() throws Exception {
289+
List<DTOForVariantPrimitivesTests> data = new ArrayList<>();
290+
for (int i = 0; i < values.length; i++) {
291+
data.add(new DTOForVariantPrimitivesTests(i, values[i]));
292+
}
293+
client.insert(table, data).get().close();
218294

295+
List<GenericRecord> rows = client.queryAll("SELECT * FROM " + table);
296+
for (GenericRecord row : rows) {
297+
System.out.println("> " + row.getString("field"));
298+
Assert.assertEquals(row.getString("field"), expectedStrValues[row.getInteger("rowId")]);
299+
}
219300
}
220301

221302
public static String tableDefinition(String table, String... columns) {

‎client-v2/src/test/java/com/clickhouse/client/query/QueryTests.java‎

Lines changed: 5 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -1113,7 +1113,6 @@ public void testEnums() {
11131113
"max_enum8 Enum8('value1' = 1, 'value2' = 2, 'value3' = 127)"
11141114
);
11151115

1116-
final UUID providedUUID = UUID.randomUUID();
11171116
final List<Supplier<String>> valueGenerators = Arrays.asList(
11181117
() -> "'value1'",
11191118
() -> "'value3'",
@@ -1126,21 +1125,26 @@ public void testEnums() {
11261125
Assert.assertTrue(r.hasValue("min_enum16"), "No value for column min_enum16 found");
11271126
Assert.assertEquals(r.getEnum16("min_enum16"), (short) -32768);
11281127
Assert.assertEquals(r.getEnum16(1), (short) -32768);
1128+
Assert.assertEquals(r.getString(1), "value1");
1129+
11291130
});
11301131
verifiers.add(r -> {
11311132
Assert.assertTrue(r.hasValue("max_enum16"), "No value for column max_enum16 found");
11321133
Assert.assertEquals(r.getEnum16("max_enum16"), (short) 32767);
11331134
Assert.assertEquals(r.getEnum16(2), (short) 32767);
1135+
Assert.assertEquals(r.getString(2), "value3");
11341136
});
11351137
verifiers.add(r -> {
11361138
Assert.assertTrue(r.hasValue("min_enum8"), "No value for column min_enum8 found");
11371139
Assert.assertEquals(r.getEnum8("min_enum8"), (byte) -128);
11381140
Assert.assertEquals(r.getEnum8(3), (byte) -128);
1141+
Assert.assertEquals(r.getString(3), "value1");
11391142
});
11401143
verifiers.add(r -> {
11411144
Assert.assertTrue(r.hasValue("max_enum8"), "No value for column max_enum8 found");
11421145
Assert.assertEquals(r.getEnum8("max_enum8"), (byte) 127);
11431146
Assert.assertEquals(r.getEnum8(4), (byte) 127);
1147+
Assert.assertEquals(r.getString(4), "value3");
11441148
});
11451149

11461150
testDataTypes(columns, valueGenerators, verifiers);

0 commit comments

Comments
 (0)