Skip to content

Commit 61d2b5f

Browse files
chernserchernser
committed
added data type converter and fixed problem with IP address as string
1 parent b319fd6 commit 61d2b5f

File tree

9 files changed

+258
-74
lines changed

9 files changed

+258
-74
lines changed

‎client-v2/src/main/java/com/clickhouse/client/api/DataTypeUtils.java‎

Lines changed: 13 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -1,5 +1,6 @@
11
package com.clickhouse.client.api;
22

3+
import java.text.SimpleDateFormat;
34
import java.time.Instant;
45
import java.time.ZoneId;
56
import com.clickhouse.client.api.data_formats.internal.BinaryStreamReader;
@@ -39,6 +40,18 @@ public class DataTypeUtils {
3940
.appendFraction(ChronoField.NANO_OF_SECOND, 9, 9, true)
4041
.toFormatter();
4142

43+
public static final DateTimeFormatter TIME_WITH_NANOS_FORMATTER = INSTANT_FORMATTER;
44+
45+
public static final DateTimeFormatter TIME_FORMATTER = DateTimeFormatter.ofPattern("HH:mm:ss");
46+
47+
public static final SimpleDateFormat OLD_DATE_TIME_FORMATTER = new SimpleDateFormat("yyyy-MM-dd HH:mm:ss");
48+
49+
public static final SimpleDateFormat OLD_DATE_FORMATTER = new SimpleDateFormat("yyyy-MM-dd");
50+
51+
public static final SimpleDateFormat OLD_TIME_FORMATTER = new SimpleDateFormat("HH:mm:ss");
52+
53+
public static final SimpleDateFormat OLD_TIME_WITH_NANOS_FORMATTER = new SimpleDateFormat("HH:mm:ss.SSSSSSSSS");
54+
4255
/**
4356
* Formats an {@link Instant} object for use in SQL statements or as query
4457
* parameter.

‎client-v2/src/main/java/com/clickhouse/client/api/data_formats/internal/AbstractBinaryFormatReader.java‎

Lines changed: 5 additions & 46 deletions
Original file line numberDiff line numberDiff line change
@@ -4,6 +4,7 @@
44
import com.clickhouse.client.api.ClientException;
55
import com.clickhouse.client.api.DataTypeUtils;
66
import com.clickhouse.client.api.data_formats.ClickHouseBinaryFormatReader;
7+
import com.clickhouse.client.api.internal.DataTypeConverter;
78
import com.clickhouse.client.api.internal.MapUtils;
89
import com.clickhouse.client.api.internal.ServerSettings;
910
import com.clickhouse.client.api.metadata.NoSuchColumnException;
@@ -61,6 +62,8 @@ public abstract class AbstractBinaryFormatReader implements ClickHouseBinaryForm
6162

6263
protected BinaryStreamReader binaryStreamReader;
6364

65+
protected DataTypeConverter dataTypeConverter;
66+
6467
private TableSchema schema;
6568
private ClickHouseColumn[] columns;
6669
private Map[] convertions;
@@ -84,6 +87,7 @@ protected AbstractBinaryFormatReader(InputStream inputStream, QuerySettings quer
8487
if (schema != null) {
8588
setSchema(schema);
8689
}
90+
this.dataTypeConverter = DataTypeConverter.INSTANCE; // singleton while no need to customize conversion
8791
}
8892

8993
protected Object[] currentRecord;
@@ -326,52 +330,7 @@ public TableSchema getSchema() {
326330

327331
@Override
328332
public String getString(String colName) {
329-
return readAsString(readValue(colName), schema.getColumnByName(colName));
330-
}
331-
332-
/**
333-
* Converts value in to a string representation. Does some formatting for selected data types
334-
* @return string representation of a value for specified column
335-
*/
336-
public static String readAsString(Object value, ClickHouseColumn column) {
337-
if (value == null) {
338-
return null;
339-
} else if (value instanceof String) {
340-
return (String) value;
341-
} else if (value instanceof ZonedDateTime) {
342-
ClickHouseDataType dataType = column.getDataType();
343-
ZonedDateTime zdt = (ZonedDateTime) value;
344-
switch (dataType) { // should not be null
345-
case Date:
346-
case Date32:
347-
return zdt.format(DataTypeUtils.DATE_FORMATTER);
348-
case DateTime:
349-
case DateTime32:
350-
return zdt.format(DataTypeUtils.DATETIME_FORMATTER);
351-
case DateTime64:
352-
return zdt.format(DataTypeUtils.DATETIME_WITH_NANOS_FORMATTER);
353-
default:
354-
return value.toString();
355-
}
356-
} else if (value instanceof BinaryStreamReader.EnumValue) {
357-
return ((BinaryStreamReader.EnumValue)value).name;
358-
} else if (value instanceof Number ) {
359-
ClickHouseDataType dataType = column.getDataType();
360-
int num = ((Number) value).intValue();
361-
if (column.getDataType() == ClickHouseDataType.Variant) {
362-
for (ClickHouseColumn c : column.getNestedColumns()) {
363-
// TODO: will work only if single enum listed as variant
364-
if (c.getDataType() == ClickHouseDataType.Enum8 || c.getDataType() == ClickHouseDataType.Enum16) {
365-
return c.getEnumConstants().name(num);
366-
}
367-
}
368-
} else if (dataType == ClickHouseDataType.Enum8 || dataType == ClickHouseDataType.Enum16) {
369-
return column.getEnumConstants().name(num);
370-
}
371-
} else if (value instanceof BinaryStreamReader.ArrayValue) {
372-
return ((BinaryStreamReader.ArrayValue)value).asList().toString();
373-
}
374-
return value.toString();
333+
return dataTypeConverter.convertToString(readValue(colName), schema.getColumnByName(colName));
375334
}
376335

377336
@Override

‎client-v2/src/main/java/com/clickhouse/client/api/data_formats/internal/MapBackedRecord.java‎

Lines changed: 6 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -2,6 +2,7 @@
22

33
import com.clickhouse.client.api.ClientException;
44
import com.clickhouse.client.api.DataTypeUtils;
5+
import com.clickhouse.client.api.internal.DataTypeConverter;
56
import com.clickhouse.client.api.metadata.TableSchema;
67
import com.clickhouse.client.api.query.GenericRecord;
78
import com.clickhouse.client.api.query.NullValueException;
@@ -24,6 +25,7 @@
2425
import java.time.ZoneOffset;
2526
import java.time.ZonedDateTime;
2627
import java.time.temporal.TemporalAmount;
28+
import java.util.Date;
2729
import java.util.HashMap;
2830
import java.util.List;
2931
import java.util.Map;
@@ -38,10 +40,13 @@ public class MapBackedRecord implements GenericRecord {
3840

3941
private Map[] columnConverters;
4042

43+
private DataTypeConverter dataTypeConverter;
44+
4145
public MapBackedRecord(Map<String, Object> record, Map[] columnConverters, TableSchema schema) {
4246
this.record = new HashMap<>(record);
4347
this.schema = schema;
4448
this.columnConverters = columnConverters;
49+
this.dataTypeConverter = DataTypeConverter.INSTANCE;
4550
}
4651

4752
public <T> T readValue(int colIndex) {
@@ -58,7 +63,7 @@ public <T> T readValue(String colName) {
5863

5964
@Override
6065
public String getString(String colName) {
61-
return AbstractBinaryFormatReader.readAsString(readValue(colName), schema.getColumnByName(colName));
66+
return dataTypeConverter.convertToString(readValue(colName), schema.getColumnByName(colName));
6267
}
6368

6469
@Override

‎client-v2/src/main/java/com/clickhouse/client/api/internal/DataTypeConverter.java‎

Lines changed: 181 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,181 @@
1+
package com.clickhouse.client.api.internal;
2+
3+
import com.clickhouse.client.api.DataTypeUtils;
4+
import com.clickhouse.client.api.data_formats.internal.AbstractBinaryFormatReader;
5+
import com.clickhouse.client.api.data_formats.internal.BinaryStreamReader;
6+
import com.clickhouse.data.ClickHouseColumn;
7+
import com.clickhouse.data.ClickHouseDataType;
8+
9+
import java.net.InetAddress;
10+
import java.time.LocalDate;
11+
import java.time.LocalDateTime;
12+
import java.time.LocalTime;
13+
import java.time.ZonedDateTime;
14+
import java.time.format.DateTimeFormatter;
15+
import java.time.temporal.TemporalAccessor;
16+
import java.util.Arrays;
17+
import java.util.Date;
18+
19+
public class DataTypeConverter {
20+
21+
public static final DataTypeConverter INSTANCE = new DataTypeConverter();
22+
23+
public String convertToString(Object value, ClickHouseColumn column) {
24+
if (value == null) {
25+
return null;
26+
}
27+
28+
switch (column.getDataType()) {
29+
case String:
30+
return stringToString(value, column);
31+
case Date:
32+
case Date32:
33+
return dateToString(value, column);
34+
case Time:
35+
case Time64:
36+
return timeToString(value, column);
37+
case DateTime:
38+
case DateTime32:
39+
case DateTime64:
40+
return dateTimeToString(value, column);
41+
case Enum8:
42+
case Enum16:
43+
case Enum:
44+
return enumToString(value, column);
45+
case IPv4:
46+
case IPv6:
47+
return ipvToString(value, column);
48+
case Array:
49+
return arrayToString(value, column);
50+
case Variant:
51+
case Dynamic:
52+
return variantOrDynamicToString(value, column);
53+
default:
54+
return value.toString();
55+
}
56+
}
57+
58+
public String stringToString(Object bytesOrString, ClickHouseColumn column) {
59+
return bytesOrString instanceof byte[] ? new String((byte[]) bytesOrString) : (String) bytesOrString;
60+
}
61+
62+
public String dateToString(Object value, ClickHouseColumn column) {
63+
DateTimeFormatter formatter = DataTypeUtils.DATE_FORMATTER;
64+
65+
if (value instanceof ZonedDateTime || value instanceof LocalDateTime) {
66+
TemporalAccessor dateTime = (TemporalAccessor) value;
67+
return formatter.format(dateTime);
68+
} else if (value instanceof LocalDate) {
69+
return formatter.format(((LocalDate)value));
70+
} else if (value instanceof Date) {
71+
return DataTypeUtils.OLD_DATE_FORMATTER.format(((Date)value));
72+
}
73+
return value.toString();
74+
}
75+
76+
public String timeToString(Object value, ClickHouseColumn column) {
77+
DateTimeFormatter formatter;
78+
switch (column.getDataType()) {
79+
case Time64:
80+
formatter = DataTypeUtils.TIME_WITH_NANOS_FORMATTER;
81+
break;
82+
default:
83+
formatter = DataTypeUtils.TIME_FORMATTER;
84+
}
85+
86+
if (value instanceof ZonedDateTime || value instanceof LocalDateTime) {
87+
TemporalAccessor dateTime = (TemporalAccessor) value;
88+
return formatter.format(dateTime);
89+
} else if (value instanceof LocalTime) {
90+
return formatter.format(((LocalTime)value));
91+
} else if (value instanceof Date) {
92+
return DataTypeUtils.OLD_TIME_FORMATTER.format(((Date)value));
93+
}
94+
return value.toString();
95+
}
96+
97+
public String dateTimeToString(Object value, ClickHouseColumn column) {
98+
DateTimeFormatter formatter;
99+
switch (column.getDataType()) {
100+
case DateTime64:
101+
formatter = DataTypeUtils.DATETIME_WITH_NANOS_FORMATTER;
102+
break;
103+
default:
104+
formatter = DataTypeUtils.DATETIME_FORMATTER;
105+
}
106+
107+
if (value instanceof ZonedDateTime || value instanceof LocalDateTime) {
108+
TemporalAccessor dateTime = (TemporalAccessor) value;
109+
return formatter.format(dateTime);
110+
} else if (value instanceof LocalDate) {
111+
return formatter.format(((LocalDate)value).atStartOfDay());
112+
} else if (value instanceof LocalTime) {
113+
return formatter.format(((LocalTime)value).atDate(LocalDate.now()));
114+
} else if (value instanceof Date) {
115+
return DataTypeUtils.OLD_DATE_TIME_FORMATTER.format(((Date)value));
116+
}
117+
return value.toString();
118+
}
119+
120+
public String enumToString(Object value, ClickHouseColumn column) {
121+
if (value instanceof BinaryStreamReader.EnumValue) {
122+
return ((BinaryStreamReader.EnumValue)value).name;
123+
} else if (value instanceof Number ) {
124+
int num = ((Number) value).intValue();
125+
switch (column.getDataType()) {
126+
case Variant:
127+
for (ClickHouseColumn c : column.getNestedColumns()) {
128+
// TODO: will work only if single enum listed as variant
129+
if (c.getDataType() == ClickHouseDataType.Enum8 || c.getDataType() == ClickHouseDataType.Enum16) {
130+
return c.getEnumConstants().name(num);
131+
}
132+
}
133+
return String.valueOf(num); // fail-safe
134+
case Enum8:
135+
case Enum16:
136+
case Enum:
137+
return column.getEnumConstants().name(num);
138+
}
139+
}
140+
return value.toString();
141+
}
142+
143+
public String ipvToString(Object value, ClickHouseColumn column) {
144+
if (value instanceof InetAddress) {
145+
return ((InetAddress) value).getHostAddress();
146+
}
147+
return value.toString();
148+
}
149+
150+
public String arrayToString(Object value, ClickHouseColumn column) {
151+
if (value instanceof BinaryStreamReader.ArrayValue) {
152+
return ((BinaryStreamReader.ArrayValue) value).asList().toString();
153+
} else if (value instanceof byte[]) {
154+
return Arrays.toString((byte[]) value);
155+
} else if (value instanceof short[]) {
156+
return Arrays.toString((short[]) value);
157+
} else if (value instanceof int[]) {
158+
return Arrays.toString((int[]) value);
159+
} else if (value instanceof long[]) {
160+
return Arrays.toString((long[]) value);
161+
} else if (value instanceof float[]) {
162+
return Arrays.toString((float[]) value);
163+
} else if (value instanceof double[]) {
164+
return Arrays.toString((double[]) value);
165+
} else if (value instanceof boolean[]) {
166+
return Arrays.toString((boolean[]) value);
167+
} else if (value instanceof char[]) {
168+
return Arrays.toString((char[]) value);
169+
} else if (value instanceof Object[]) {
170+
return Arrays.deepToString((Object[]) value);
171+
}
172+
return value.toString();
173+
}
174+
175+
public String variantOrDynamicToString(Object value, ClickHouseColumn column) {
176+
if (value instanceof BinaryStreamReader.ArrayValue) {
177+
return arrayToString(value, column);
178+
}
179+
return value.toString();
180+
}
181+
}

‎client-v2/src/test/java/com/clickhouse/client/api/internal/DataTypeConverterTest.java‎

Lines changed: 21 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,21 @@
1+
package com.clickhouse.client.api.internal;
2+
3+
import org.testng.annotations.Test;
4+
5+
import static org.testng.Assert.*;
6+
7+
@Test(groups = {"unit"})
8+
public class DataTypeConverterTest {
9+
10+
@Test
11+
public void testDateToString() {
12+
}
13+
14+
@Test
15+
public void testTimeToString() {
16+
}
17+
18+
@Test
19+
public void testDateTimeToString() {
20+
}
21+
}

‎client-v2/src/test/java/com/clickhouse/client/datatypes/DataTypeTests.java‎

Lines changed: 27 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -916,6 +916,33 @@ public Object[][] testJSONBinaryFormat_dp() {
916916
};
917917
}
918918

919+
@Test(groups = {"integration"}, dataProvider = "testDataTypesAsStringDP")
920+
public void testDataTypesAsString(String sql, String[] expectedStrValues) throws Exception {
921+
922+
try (QueryResponse resp = client.query(sql).get()) {
923+
ClickHouseBinaryFormatReader reader = client.newBinaryFormatReader(resp);
924+
reader.next();
925+
for (int i = 0; i < expectedStrValues.length; i++) {
926+
Assert.assertEquals(reader.getString(i + 1), expectedStrValues[i]);
927+
}
928+
}
929+
}
930+
931+
@DataProvider
932+
public static Object[][] testDataTypesAsStringDP() {
933+
return new Object[][] {
934+
{"SELECT '192.168.1.1'::IPv4, '2001:db8::1'::IPv6, '192.168.1.1'::IPv6",
935+
new String[] {"192.168.1.1", "2001:db8:0:0:0:0:0:1", "192.168.1.1"}},
936+
{"SELECT '2024-10-04'::Date32, '2024-10-04 12:34:56'::DateTime32, '2024-10-04 12:34:56.789'::DateTime64(3), " +
937+
" '2024-10-04 12:34:56.789012'::DateTime64(6), '2024-10-04 12:34:56.789012345'::DateTime64(9)",
938+
new String[] {"2024-10-04", "2024-10-04 12:34:56", "2024-10-04 12:34:56.789", "2024-10-04 12:34:56.789012",
939+
"2024-10-04 12:34:56.789012345"}},
940+
{"SELECT 1::Enum16('one' = 1, 'two' = 2)", "one"},
941+
{"SELECT 2::Enum8('one' = 1, 'two' = 2)", "two"},
942+
{"SELECT 3::Enum('one' = 1, 'two' = 2, 'three' = 3)", "three"},
943+
};
944+
}
945+
919946
public static String tableDefinition(String table, String... columns) {
920947
StringBuilder sb = new StringBuilder();
921948
sb.append("CREATE TABLE " + table + " ( ");

‎jdbc-v2/src/test/java/com/clickhouse/jdbc/DataTypeTests.java‎

Lines changed: 3 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -749,11 +749,11 @@ public void testIpAddressTypes() throws SQLException, UnknownHostException {
749749
try (ResultSet rs = stmt.executeQuery("SELECT * FROM test_ips ORDER BY order")) {
750750
assertTrue(rs.next());
751751
assertEquals(rs.getObject("ipv4_ip"), ipv4AddressByIp);
752-
assertEquals(rs.getObject("ipv4_ip", Inet6Address.class).toString(), "/0:0:0:0:0:ffff:5ab0:4b61");
753-
assertEquals(rs.getString("ipv4_ip"), ipv4AddressByIp.toString());
752+
assertEquals(rs.getObject("ipv4_ip", Inet6Address.class).getHostAddress(), "0:0:0:0:0:ffff:5ab0:4b61");
753+
assertEquals(rs.getString("ipv4_ip"), ipv4AddressByIp.getHostAddress());
754754
assertEquals(rs.getObject("ipv4_name"), ipv4AddressByName);
755755
assertEquals(rs.getObject("ipv6"), ipv6Address);
756-
assertEquals(rs.getString("ipv6"), ipv6Address.toString());
756+
assertEquals(rs.getString("ipv6"), ipv6Address.getHostAddress());
757757
assertEquals(rs.getObject("ipv4_as_ipv6"), ipv4AsIpv6);
758758
assertEquals(rs.getObject("ipv4_as_ipv6", Inet4Address.class), ipv4AsIpv6);
759759
assertFalse(rs.next());

0 commit comments

Comments
 (0)