Skip to content

Commit 18a6550

Browse files
chernserchernser
authored
Merge pull request #1826 from ClickHouse/clientv2_primitive_reader
[client-v2] primitive reading into pojo
2 parents 27e2613 + 0de9eb2 commit 18a6550

File tree

7 files changed

+526
-351
lines changed

7 files changed

+526
-351
lines changed

‎client-v2/src/main/java/com/clickhouse/client/api/data_formats/internal/AbstractBinaryFormatReader.java‎

Lines changed: 16 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -76,17 +76,27 @@ protected AbstractBinaryFormatReader(InputStream inputStream, QuerySettings quer
7676

7777
protected AtomicBoolean nextRecordEmpty = new AtomicBoolean(true);
7878

79+
/**
80+
* Reads next record into POJO object using set of serializers.
81+
* There should be a serializer for each column in the record, otherwise it will silently skip a field
82+
* It is done in such a way because it is not the reader concern. Calling code should validate this.
83+
*
84+
* Note: internal API
85+
* @param deserializers
86+
* @param obj
87+
* @return
88+
* @throws IOException
89+
*/
7990
public boolean readToPOJO(Map<String, POJOSetter> deserializers, Object obj ) throws IOException {
8091
boolean firstColumn = true;
8192

8293
for (ClickHouseColumn column : columns) {
8394
try {
84-
Object val = binaryStreamReader.readValue(column);
85-
if (val != null) {
86-
POJOSetter deserializer = deserializers.get(column.getColumnName());
87-
if (deserializer != null) {
88-
deserializer.setValue(obj, val);
89-
}
95+
POJOSetter deserializer = deserializers.get(column.getColumnName());
96+
if (deserializer != null) {
97+
deserializer.setValue(obj, binaryStreamReader, column);
98+
} else {
99+
binaryStreamReader.skipValue(column);
90100
}
91101
firstColumn = false;
92102
} catch (EOFException e) {

‎client-v2/src/main/java/com/clickhouse/client/api/data_formats/internal/BinaryStreamReader.java‎

Lines changed: 286 additions & 88 deletions
Large diffs are not rendered by default.

‎client-v2/src/main/java/com/clickhouse/client/api/internal/SerializerUtils.java‎

Lines changed: 171 additions & 156 deletions
Large diffs are not rendered by default.

‎client-v2/src/main/java/com/clickhouse/client/api/query/POJOSetter.java‎

Lines changed: 4 additions & 33 deletions
Original file line numberDiff line numberDiff line change
@@ -1,6 +1,9 @@
11
package com.clickhouse.client.api.query;
22

33

4+
import com.clickhouse.client.api.data_formats.internal.BinaryStreamReader;
5+
import com.clickhouse.data.ClickHouseColumn;
6+
47
/**
58
* Class used to set value for individual fields in a POJO.
69
* Implementation will have reference to a specific POJO property.
@@ -9,37 +12,5 @@
912
*/
1013
public interface POJOSetter {
1114

12-
default void setValue(Object obj, boolean value) {
13-
throw new UnsupportedOperationException("Unsupported type: boolean");
14-
};
15-
16-
default void setValue(Object obj, byte value) {
17-
throw new UnsupportedOperationException("Unsupported type: byte");
18-
};
19-
20-
default void setValue(Object obj, char value) {
21-
throw new UnsupportedOperationException("Unsupported type: char");
22-
};
23-
24-
default void setValue(Object obj, short value) {
25-
throw new UnsupportedOperationException("Unsupported type: short");
26-
};
27-
28-
default void setValue(Object obj, int value) {
29-
throw new UnsupportedOperationException("Unsupported type: int");
30-
};
31-
32-
default void setValue(Object obj, long value) {
33-
throw new UnsupportedOperationException("Unsupported type: long");
34-
};
35-
36-
default void setValue(Object obj, float value) {
37-
throw new UnsupportedOperationException("Unsupported type: float");
38-
};
39-
40-
default void setValue(Object obj, double value) {
41-
throw new UnsupportedOperationException("Unsupported type: double");
42-
};
43-
44-
void setValue(Object obj, Object value);
15+
void setValue(Object obj, BinaryStreamReader reader, ClickHouseColumn column) throws Exception;
4516
}

‎client-v2/src/test/java/com/clickhouse/client/internal/SerializerUtilsTests.java‎

Lines changed: 12 additions & 46 deletions
Original file line numberDiff line numberDiff line change
@@ -1,62 +1,28 @@
11
package com.clickhouse.client.internal;
22

3-
import com.clickhouse.client.api.internal.SerializerUtils;
3+
import com.clickhouse.client.api.data_formats.internal.BinaryStreamReader;
44
import com.clickhouse.client.api.query.POJOSetter;
5-
import com.clickhouse.client.query.SamplePOJO;
5+
import com.clickhouse.client.query.QuerySamplePOJO;
6+
import com.clickhouse.client.query.SimplePOJO;
67
import com.clickhouse.data.ClickHouseColumn;
7-
import org.testng.Assert;
8-
import org.testng.annotations.Test;
98

10-
import java.lang.reflect.Method;
11-
import java.math.BigInteger;
9+
import java.io.IOException;
1210
import java.time.LocalDateTime;
13-
import java.util.Arrays;
14-
import java.util.HashMap;
15-
import java.util.Map;
16-
import java.util.stream.Collectors;
11+
import java.time.ZonedDateTime;
1712

1813
public class SerializerUtilsTests {
1914

20-
@Test(enabled = false)
21-
public void testDeserialize() throws Exception {
22-
23-
Map<String, POJOSetter> pojoSetterList = new HashMap<>();
24-
for (Method method : SamplePOJOForSerialization.class.getDeclaredMethods()) {
25-
if (method.getName().startsWith("set")) {
26-
pojoSetterList.put(method.getName().substring(3).toLowerCase(),
27-
SerializerUtils.compilePOJOSetter(method, ClickHouseColumn.of(method.getName(),
28-
"String")));
29-
}
30-
}
31-
32-
SamplePOJOForSerialization pojo = new SamplePOJOForSerialization();
33-
pojoSetterList.get("string").setValue(pojo, "John Doe");
34-
pojoSetterList.get("int32").setValue(pojo, Integer.valueOf(30));
35-
pojoSetterList.get("int16").setValue(pojo, 22);
36-
37-
Assert.assertEquals(pojo.getString(), "John Doe");
38-
Assert.assertEquals(pojo.getInt32(), 30);
39-
Assert.assertEquals(pojo.getInt16(), 22);
40-
}
4115

4216
public static class SamplePOJOInt256Setter implements POJOSetter {
4317

44-
45-
46-
47-
/*
48-
public void setValue(java.lang.Object, java.lang.Object);
49-
Code:
50-
0: aload_1
51-
1: checkcast #7 // class com/clickhouse/client/query/SamplePOJO
52-
4: aload_2
53-
5: checkcast #25 // class java/math/BigInteger
54-
8: invokevirtual #27 // Method com/clickhouse/client/query/SamplePOJO.setInt256:(Ljava/math/BigInteger;)V
55-
11: return
56-
*/
5718
@Override
58-
public void setValue(Object obj, Object value) {
59-
Arrays.stream(((Object[]) value)).collect(Collectors.toList());
19+
public void setValue(Object obj, BinaryStreamReader reader, ClickHouseColumn column) throws IOException {
20+
((QuerySamplePOJO)obj).setDateTime(((ZonedDateTime)reader.readValue(column)).toLocalDateTime());
21+
}
22+
23+
public void readValue(Object obj, BinaryStreamReader reader, ClickHouseColumn column) throws IOException {
24+
// ((SamplePOJO)obj).setDateTime(((ZonedDateTime)reader.readValue(column)).toLocalDateTime());
25+
((SimplePOJO)obj).setId(reader.readIntLE());
6026
}
6127
}
6228
}

client-v2/src/test/java/com/clickhouse/client/query/SamplePOJO.java renamed to client-v2/src/test/java/com/clickhouse/client/query/QuerySamplePOJO.java

Lines changed: 33 additions & 16 deletions
Original file line numberDiff line numberDiff line change
@@ -18,7 +18,7 @@
1818
import java.util.Random;
1919
import java.util.UUID;
2020

21-
public class SamplePOJO {
21+
public class QuerySamplePOJO {
2222
private int int8;
2323
private int int8_default;
2424
private int int16;
@@ -35,7 +35,7 @@ public class SamplePOJO {
3535
private int uint8;
3636
private int uint16;
3737
private long uint32;
38-
private long uint64;
38+
private BigInteger uint64;
3939
private BigInteger uint128;
4040
private BigInteger uint256;
4141

@@ -67,12 +67,15 @@ public class SamplePOJO {
6767
private Inet6Address ipv6;
6868

6969
private List<String> array;
70-
private List<Integer> tuple;
70+
private List<?> tuple;
71+
72+
private Object[] tupleArray;
73+
7174
private Map<String, Integer> map;
7275
private List<Integer> nestedInnerInt;
7376
private List<String> nestedInnerString;
7477

75-
public SamplePOJO() {
78+
public QuerySamplePOJO() {
7679
final Random random = new Random();
7780
int8 = random.nextInt(128);
7881
int16 = random.nextInt(32768);
@@ -90,11 +93,16 @@ public SamplePOJO() {
9093

9194
int256 = upper1.or(upper2).or(lower1).or(lower2);
9295

96+
9397
uint8 = random.nextInt(255);
9498
uint16 = random.nextInt(32768);
9599
uint32 = (long) (random.nextDouble() * 4294967295L);
96-
uint64 = (long) (random.nextDouble() * 18446744073709615L);
97100

101+
long rndUInt64 = random.nextLong();
102+
uint64 = BigInteger.valueOf(rndUInt64);
103+
if (rndUInt64 < 0) {
104+
uint64 = uint64.add(BigInteger.ONE.shiftLeft(64));
105+
}
98106

99107
uint128 = upper.or(lower).abs();
100108
uint256 = upper1.or(upper2).or(lower1).or(lower2).abs();
@@ -137,7 +145,8 @@ public SamplePOJO() {
137145
}
138146

139147
array = Arrays.asList("a", "b", "c", "d", "e", "f", "g", "h", "i", "j", "k", "l", "m", "n", "o", "p", "q", "r", "s", "t", "u", "v", "w", "x", "y", "z");
140-
tuple = Arrays.asList(1, 2, 3, 4, 5, 6, 7, 8, 9, 10);
148+
tuple = Arrays.asList(random.nextInt(), random.nextDouble(), "a", "b");
149+
tupleArray = new Object[] {random.nextInt(), random.nextDouble(), "c", "d" };
141150
map = new HashMap<>();
142151
for (int i = 0; i < 10; i++) {
143152
map.put(String.valueOf((char) ('a' + i)), i + 1);
@@ -248,11 +257,11 @@ public void setUint32(long uint32) {
248257
this.uint32 = uint32;
249258
}
250259

251-
public long getUint64() {
260+
public BigInteger getUint64() {
252261
return uint64;
253262
}
254263

255-
public void setUint64(long uint64) {
264+
public void setUint64(BigInteger uint64) {
256265
this.uint64 = uint64;
257266
}
258267

@@ -424,14 +433,22 @@ public void setArray(List<String> array) {
424433
this.array = array;
425434
}
426435

427-
public List<Integer> getTuple() {
436+
public List<?> getTuple() {
428437
return tuple;
429438
}
430439

431-
public void setTuple(List<Integer> tuple) {
440+
public void setTuple(List<?> tuple) {
432441
this.tuple = tuple;
433442
}
434443

444+
public Object[] getTupleArray() {
445+
return tupleArray;
446+
}
447+
448+
public void setTupleArray(Object[] tupleArray) {
449+
this.tupleArray = tupleArray;
450+
}
451+
435452
public Map<String, Integer> getMap() {
436453
return map;
437454
}
@@ -460,18 +477,18 @@ public void setNestedInnerString(List<String> nestedInnerString) {
460477
public boolean equals(Object o) {
461478
if (this == o) return true;
462479
if (o == null || getClass() != o.getClass()) return false;
463-
SamplePOJO that = (SamplePOJO) o;
464-
return int8 == that.int8 && int16 == that.int16 && int32 == that.int32 && int64 == that.int64 && uint8 == that.uint8 && uint16 == that.uint16 && uint32 == that.uint32 && uint64 == that.uint64 && Float.compare(float32, that.float32) == 0 && Double.compare(float64, that.float64) == 0 && bool == that.bool && enum8 == that.enum8 && enum16 == that.enum16 && Objects.equals(int128, that.int128) && Objects.equals(int256, that.int256) && Objects.equals(uint128, that.uint128) && Objects.equals(uint256, that.uint256) && Objects.equals(decimal32, that.decimal32) && Objects.equals(decimal64, that.decimal64) && Objects.equals(decimal128, that.decimal128) && Objects.equals(decimal256, that.decimal256) && Objects.equals(string, that.string) && Objects.equals(fixedString, that.fixedString) && Objects.equals(date, that.date) && Objects.equals(date32, that.date32) && Objects.equals(dateTime, that.dateTime) && Objects.equals(dateTime64, that.dateTime64) && Objects.equals(uuid, that.uuid) && Objects.equals(ipv4, that.ipv4) && Objects.equals(ipv6, that.ipv6) && Objects.equals(array, that.array) && Objects.equals(tuple, that.tuple) && Objects.equals(map, that.map) && Objects.equals(nestedInnerInt, that.nestedInnerInt) && Objects.equals(nestedInnerString, that.nestedInnerString);
480+
QuerySamplePOJO that = (QuerySamplePOJO) o;
481+
return int8 == that.int8 && int8_default == that.int8_default && int16 == that.int16 && int16_default == that.int16_default && int32 == that.int32 && int32_default == that.int32_default && int64 == that.int64 && int64_default == that.int64_default && uint8 == that.uint8 && uint16 == that.uint16 && uint32 == that.uint32 && Float.compare(float32, that.float32) == 0 && Double.compare(float64, that.float64) == 0 && bool == that.bool && enum8 == that.enum8 && enum16 == that.enum16 && Objects.equals(int128, that.int128) && Objects.equals(int128_default, that.int128_default) && Objects.equals(int256, that.int256) && Objects.equals(int256_default, that.int256_default) && Objects.equals(uint64, that.uint64) && Objects.equals(uint128, that.uint128) && Objects.equals(uint256, that.uint256) && Objects.equals(decimal32, that.decimal32) && Objects.equals(decimal64, that.decimal64) && Objects.equals(decimal128, that.decimal128) && Objects.equals(decimal256, that.decimal256) && Objects.equals(string, that.string) && Objects.equals(fixedString, that.fixedString) && Objects.equals(date, that.date) && Objects.equals(date32, that.date32) && Objects.equals(dateTime, that.dateTime) && Objects.equals(dateTime64, that.dateTime64) && Objects.equals(uuid, that.uuid) && Objects.equals(ipv4, that.ipv4) && Objects.equals(ipv6, that.ipv6) && Objects.equals(array, that.array) && Objects.equals(map, that.map) && Objects.equals(nestedInnerInt, that.nestedInnerInt) && Objects.equals(nestedInnerString, that.nestedInnerString);
465482
}
466483

467484
@Override
468485
public int hashCode() {
469-
return Objects.hash(int8, int16, int32, int64, int128, int256, uint8, uint16, uint32, uint64, uint128, uint256, float32, float64, decimal32, decimal64, decimal128, decimal256, bool, string, fixedString, date, date32, dateTime, dateTime64, uuid, enum8, enum16, ipv4, ipv6, array, tuple, map, nestedInnerInt, nestedInnerString);
486+
return Objects.hash(int8, int8_default, int16, int16_default, int32, int32_default, int64, int64_default, int128, int128_default, int256, int256_default, uint8, uint16, uint32, uint64, uint128, uint256, float32, float64, decimal32, decimal64, decimal128, decimal256, bool, string, fixedString, date, date32, dateTime, dateTime64, uuid, enum8, enum16, ipv4, ipv6, array, map, nestedInnerInt, nestedInnerString);
470487
}
471488

472489
@Override
473490
public String toString() {
474-
return "SamplePOJO{" +
491+
return "QuerySamplePOJO{" +
475492
"int8=" + int8 +
476493
", int8_default=" + int8_default +
477494
", int16=" + int16 +
@@ -509,7 +526,6 @@ public String toString() {
509526
", ipv4=" + ipv4 +
510527
", ipv6=" + ipv6 +
511528
", array=" + array +
512-
", tuple=" + tuple +
513529
", map=" + map +
514530
", nestedInnerInt=" + nestedInnerInt +
515531
", nestedInnerString=" + nestedInnerString +
@@ -555,7 +571,8 @@ public static String generateTableCreateSQL(String tableName) {
555571
"ipv4 IPv4, " +
556572
"ipv6 IPv6, " +
557573
"array Array(String), " +
558-
"tuple Tuple(Int32, Int32, Int32, Int32, Int32, Int32, Int32, Int32, Int32, Int32), " +
574+
"tuple Tuple(Int32, Float64, String, String), " +
575+
"tupleArray Tuple(Int32, Float64, String, String), " +
559576
"map Map(String, Int32), " +
560577
"nested Nested (innerInt Int32, innerString String)" +
561578
") ENGINE = Memory";

‎client-v2/src/test/java/com/clickhouse/client/query/QueryTests.java‎

Lines changed: 4 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -31,7 +31,6 @@
3131
import com.fasterxml.jackson.databind.JsonNode;
3232
import com.fasterxml.jackson.databind.MappingIterator;
3333
import com.fasterxml.jackson.databind.ObjectMapper;
34-
import org.testcontainers.shaded.com.google.common.collect.Table;
3534
import org.testng.Assert;
3635
import org.testng.annotations.AfterMethod;
3736
import org.testng.annotations.BeforeMethod;
@@ -47,7 +46,6 @@
4746
import java.io.OutputStreamWriter;
4847
import java.math.BigDecimal;
4948
import java.math.BigInteger;
50-
import java.math.RoundingMode;
5149
import java.net.Inet4Address;
5250
import java.net.Inet6Address;
5351
import java.net.InetAddress;
@@ -1542,13 +1540,13 @@ public void testQueryReadToPOJOWithoutGetters() {
15421540
public void testQueryAllWithPOJO() throws Exception {
15431541

15441542
final String tableName = "test_query_all_with_pojo";
1545-
final String createTableSQL = SamplePOJO.generateTableCreateSQL(tableName);
1543+
final String createTableSQL = QuerySamplePOJO.generateTableCreateSQL(tableName);
15461544
client.execute("DROP TABLE IF EXISTS test_query_all_with_pojo").get();
15471545
client.execute(createTableSQL).get();
15481546

1549-
SamplePOJO pojo = new SamplePOJO();
1547+
QuerySamplePOJO pojo = new QuerySamplePOJO();
15501548
TableSchema schema = client.getTableSchema(tableName);
1551-
client.register(SamplePOJO.class, schema);
1549+
client.register(QuerySamplePOJO.class, schema);
15521550

15531551
client.insert(tableName, Collections.singletonList(pojo)).get();
15541552

@@ -1562,7 +1560,7 @@ public void testQueryAllWithPOJO() throws Exception {
15621560
pojo.setDateTime(pojo.getDateTime().minusNanos(pojo.getDateTime().getNano()));
15631561
pojo.setDateTime64(pojo.getDateTime64().withNano((int) Math.ceil((pojo.getDateTime64().getNano() / 1000_000) * 1000_000)));
15641562

1565-
List<SamplePOJO> pojos = client.queryAll("SELECT * FROM " + tableName + " LIMIT 1", SamplePOJO.class,
1563+
List<QuerySamplePOJO> pojos = client.queryAll("SELECT * FROM " + tableName + " LIMIT 1", QuerySamplePOJO.class,
15661564
schema);
15671565
Assert.assertEquals(pojos.get(0), pojo, "Expected " + pojo + " but got " + pojos.get(0));
15681566
}

0 commit comments

Comments
 (0)