Skip to content

Commit 8e1e2ef

Browse files
chernserchernser
committed
primitive reading into pojo
1 parent 28de7cd commit 8e1e2ef

File tree

7 files changed

+500
-354
lines changed

7 files changed

+500
-354
lines changed

‎client-v2/src/main/java/com/clickhouse/client/api/data_formats/internal/AbstractBinaryFormatReader.java‎

Lines changed: 16 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -73,17 +73,27 @@ protected AbstractBinaryFormatReader(InputStream inputStream, QuerySettings quer
7373

7474
protected AtomicBoolean nextRecordEmpty = new AtomicBoolean(true);
7575

76+
/**
77+
* Reads next record into POJO object using set of serializers.
78+
* There should be a serializer for each column in the record, otherwise it will silently skip a field
79+
* It is done in such a way because it is not the reader concern. Calling code should validate this.
80+
*
81+
* Note: internal API
82+
* @param deserializers
83+
* @param obj
84+
* @return
85+
* @throws IOException
86+
*/
7687
public boolean readToPOJO(Map<String, POJOSetter> deserializers, Object obj ) throws IOException {
7788
boolean firstColumn = true;
7889

7990
for (ClickHouseColumn column : columns) {
8091
try {
81-
Object val = binaryStreamReader.readValue(column);
82-
if (val != null) {
83-
POJOSetter deserializer = deserializers.get(column.getColumnName());
84-
if (deserializer != null) {
85-
deserializer.setValue(obj, val);
86-
}
92+
POJOSetter deserializer = deserializers.get(column.getColumnName());
93+
if (deserializer != null) {
94+
deserializer.setValue(obj, binaryStreamReader, column);
95+
} else {
96+
binaryStreamReader.skipValue(column);
8797
}
8898
firstColumn = false;
8999
} catch (EOFException e) {

‎client-v2/src/main/java/com/clickhouse/client/api/data_formats/internal/BinaryStreamReader.java‎

Lines changed: 286 additions & 88 deletions
Large diffs are not rendered by default.

‎client-v2/src/main/java/com/clickhouse/client/api/internal/SerializerUtils.java‎

Lines changed: 150 additions & 154 deletions
Large diffs are not rendered by default.

‎client-v2/src/main/java/com/clickhouse/client/api/query/POJOSetter.java‎

Lines changed: 4 additions & 33 deletions
Original file line numberDiff line numberDiff line change
@@ -1,6 +1,9 @@
11
package com.clickhouse.client.api.query;
22

33

4+
import com.clickhouse.client.api.data_formats.internal.BinaryStreamReader;
5+
import com.clickhouse.data.ClickHouseColumn;
6+
47
/**
58
* Class used to set value for individual fields in a POJO.
69
* Implementation will have reference to a specific POJO property.
@@ -9,37 +12,5 @@
912
*/
1013
public interface POJOSetter {
1114

12-
default void setValue(Object obj, boolean value) {
13-
throw new UnsupportedOperationException("Unsupported type: boolean");
14-
};
15-
16-
default void setValue(Object obj, byte value) {
17-
throw new UnsupportedOperationException("Unsupported type: byte");
18-
};
19-
20-
default void setValue(Object obj, char value) {
21-
throw new UnsupportedOperationException("Unsupported type: char");
22-
};
23-
24-
default void setValue(Object obj, short value) {
25-
throw new UnsupportedOperationException("Unsupported type: short");
26-
};
27-
28-
default void setValue(Object obj, int value) {
29-
throw new UnsupportedOperationException("Unsupported type: int");
30-
};
31-
32-
default void setValue(Object obj, long value) {
33-
throw new UnsupportedOperationException("Unsupported type: long");
34-
};
35-
36-
default void setValue(Object obj, float value) {
37-
throw new UnsupportedOperationException("Unsupported type: float");
38-
};
39-
40-
default void setValue(Object obj, double value) {
41-
throw new UnsupportedOperationException("Unsupported type: double");
42-
};
43-
44-
void setValue(Object obj, Object value);
15+
void setValue(Object obj, BinaryStreamReader reader, ClickHouseColumn column) throws Exception;
4516
}

‎client-v2/src/test/java/com/clickhouse/client/internal/SerializerUtilsTests.java‎

Lines changed: 12 additions & 46 deletions
Original file line numberDiff line numberDiff line change
@@ -1,62 +1,28 @@
11
package com.clickhouse.client.internal;
22

3-
import com.clickhouse.client.api.internal.SerializerUtils;
3+
import com.clickhouse.client.api.data_formats.internal.BinaryStreamReader;
44
import com.clickhouse.client.api.query.POJOSetter;
5-
import com.clickhouse.client.query.SamplePOJO;
5+
import com.clickhouse.client.query.QuerySamplePOJO;
6+
import com.clickhouse.client.query.SimplePOJO;
67
import com.clickhouse.data.ClickHouseColumn;
7-
import org.testng.Assert;
8-
import org.testng.annotations.Test;
98

10-
import java.lang.reflect.Method;
11-
import java.math.BigInteger;
9+
import java.io.IOException;
1210
import java.time.LocalDateTime;
13-
import java.util.Arrays;
14-
import java.util.HashMap;
15-
import java.util.Map;
16-
import java.util.stream.Collectors;
11+
import java.time.ZonedDateTime;
1712

1813
public class SerializerUtilsTests {
1914

20-
@Test(enabled = false)
21-
public void testDeserialize() throws Exception {
22-
23-
Map<String, POJOSetter> pojoSetterList = new HashMap<>();
24-
for (Method method : SamplePOJOForSerialization.class.getDeclaredMethods()) {
25-
if (method.getName().startsWith("set")) {
26-
pojoSetterList.put(method.getName().substring(3).toLowerCase(),
27-
SerializerUtils.compilePOJOSetter(method, ClickHouseColumn.of(method.getName(),
28-
"String")));
29-
}
30-
}
31-
32-
SamplePOJOForSerialization pojo = new SamplePOJOForSerialization();
33-
pojoSetterList.get("string").setValue(pojo, "John Doe");
34-
pojoSetterList.get("int32").setValue(pojo, Integer.valueOf(30));
35-
pojoSetterList.get("int16").setValue(pojo, 22);
36-
37-
Assert.assertEquals(pojo.getString(), "John Doe");
38-
Assert.assertEquals(pojo.getInt32(), 30);
39-
Assert.assertEquals(pojo.getInt16(), 22);
40-
}
4115

4216
public static class SamplePOJOInt256Setter implements POJOSetter {
4317

44-
45-
46-
47-
/*
48-
public void setValue(java.lang.Object, java.lang.Object);
49-
Code:
50-
0: aload_1
51-
1: checkcast #7 // class com/clickhouse/client/query/SamplePOJO
52-
4: aload_2
53-
5: checkcast #25 // class java/math/BigInteger
54-
8: invokevirtual #27 // Method com/clickhouse/client/query/SamplePOJO.setInt256:(Ljava/math/BigInteger;)V
55-
11: return
56-
*/
5718
@Override
58-
public void setValue(Object obj, Object value) {
59-
Arrays.stream(((Object[]) value)).collect(Collectors.toList());
19+
public void setValue(Object obj, BinaryStreamReader reader, ClickHouseColumn column) throws IOException {
20+
((QuerySamplePOJO)obj).setDateTime(((ZonedDateTime)reader.readValue(column)).toLocalDateTime());
21+
}
22+
23+
public void readValue(Object obj, BinaryStreamReader reader, ClickHouseColumn column) throws IOException {
24+
// ((SamplePOJO)obj).setDateTime(((ZonedDateTime)reader.readValue(column)).toLocalDateTime());
25+
((SimplePOJO)obj).setId(reader.readIntLE());
6026
}
6127
}
6228
}

client-v2/src/test/java/com/clickhouse/client/query/SamplePOJO.java renamed to client-v2/src/test/java/com/clickhouse/client/query/QuerySamplePOJO.java

Lines changed: 24 additions & 20 deletions
Original file line numberDiff line numberDiff line change
@@ -18,7 +18,7 @@
1818
import java.util.Random;
1919
import java.util.UUID;
2020

21-
public class SamplePOJO {
21+
public class QuerySamplePOJO {
2222
private int int8;
2323
private int int8_default;
2424
private int int16;
@@ -35,7 +35,7 @@ public class SamplePOJO {
3535
private int uint8;
3636
private int uint16;
3737
private long uint32;
38-
private long uint64;
38+
private BigInteger uint64;
3939
private BigInteger uint128;
4040
private BigInteger uint256;
4141

@@ -67,12 +67,12 @@ public class SamplePOJO {
6767
private Inet6Address ipv6;
6868

6969
private List<String> array;
70-
private List<Integer> tuple;
70+
// private List<?> tuple;
7171
private Map<String, Integer> map;
7272
private List<Integer> nestedInnerInt;
7373
private List<String> nestedInnerString;
7474

75-
public SamplePOJO() {
75+
public QuerySamplePOJO() {
7676
final Random random = new Random();
7777
int8 = random.nextInt(128);
7878
int16 = random.nextInt(32768);
@@ -90,11 +90,16 @@ public SamplePOJO() {
9090

9191
int256 = upper1.or(upper2).or(lower1).or(lower2);
9292

93+
9394
uint8 = random.nextInt(255);
9495
uint16 = random.nextInt(32768);
9596
uint32 = (long) (random.nextDouble() * 4294967295L);
96-
uint64 = (long) (random.nextDouble() * 18446744073709615L);
9797

98+
long rndUInt64 = random.nextLong();
99+
uint64 = BigInteger.valueOf(rndUInt64);
100+
if (rndUInt64 < 0) {
101+
uint64 = uint64.add(BigInteger.ONE.shiftLeft(64));
102+
}
98103

99104
uint128 = upper.or(lower).abs();
100105
uint256 = upper1.or(upper2).or(lower1).or(lower2).abs();
@@ -137,7 +142,7 @@ public SamplePOJO() {
137142
}
138143

139144
array = Arrays.asList("a", "b", "c", "d", "e", "f", "g", "h", "i", "j", "k", "l", "m", "n", "o", "p", "q", "r", "s", "t", "u", "v", "w", "x", "y", "z");
140-
tuple = Arrays.asList(1, 2, 3, 4, 5, 6, 7, 8, 9, 10);
145+
// tuple = Arrays.asList(new Object[]{random.nextInt(), random.nextDouble(), "a", "b" });
141146
map = new HashMap<>();
142147
for (int i = 0; i < 10; i++) {
143148
map.put(String.valueOf((char) ('a' + i)), i + 1);
@@ -248,11 +253,11 @@ public void setUint32(long uint32) {
248253
this.uint32 = uint32;
249254
}
250255

251-
public long getUint64() {
256+
public BigInteger getUint64() {
252257
return uint64;
253258
}
254259

255-
public void setUint64(long uint64) {
260+
public void setUint64(BigInteger uint64) {
256261
this.uint64 = uint64;
257262
}
258263

@@ -424,13 +429,13 @@ public void setArray(List<String> array) {
424429
this.array = array;
425430
}
426431

427-
public List<Integer> getTuple() {
428-
return tuple;
429-
}
432+
// public List<?> getTuple() {
433+
// return tuple;
434+
// }
430435

431-
public void setTuple(List<Integer> tuple) {
432-
this.tuple = tuple;
433-
}
436+
// public void setTuple(List<?> tuple) {
437+
// this.tuple = tuple;
438+
// }
434439

435440
public Map<String, Integer> getMap() {
436441
return map;
@@ -460,18 +465,18 @@ public void setNestedInnerString(List<String> nestedInnerString) {
460465
public boolean equals(Object o) {
461466
if (this == o) return true;
462467
if (o == null || getClass() != o.getClass()) return false;
463-
SamplePOJO that = (SamplePOJO) o;
464-
return int8 == that.int8 && int16 == that.int16 && int32 == that.int32 && int64 == that.int64 && uint8 == that.uint8 && uint16 == that.uint16 && uint32 == that.uint32 && uint64 == that.uint64 && Float.compare(float32, that.float32) == 0 && Double.compare(float64, that.float64) == 0 && bool == that.bool && enum8 == that.enum8 && enum16 == that.enum16 && Objects.equals(int128, that.int128) && Objects.equals(int256, that.int256) && Objects.equals(uint128, that.uint128) && Objects.equals(uint256, that.uint256) && Objects.equals(decimal32, that.decimal32) && Objects.equals(decimal64, that.decimal64) && Objects.equals(decimal128, that.decimal128) && Objects.equals(decimal256, that.decimal256) && Objects.equals(string, that.string) && Objects.equals(fixedString, that.fixedString) && Objects.equals(date, that.date) && Objects.equals(date32, that.date32) && Objects.equals(dateTime, that.dateTime) && Objects.equals(dateTime64, that.dateTime64) && Objects.equals(uuid, that.uuid) && Objects.equals(ipv4, that.ipv4) && Objects.equals(ipv6, that.ipv6) && Objects.equals(array, that.array) && Objects.equals(tuple, that.tuple) && Objects.equals(map, that.map) && Objects.equals(nestedInnerInt, that.nestedInnerInt) && Objects.equals(nestedInnerString, that.nestedInnerString);
468+
QuerySamplePOJO that = (QuerySamplePOJO) o;
469+
return int8 == that.int8 && int8_default == that.int8_default && int16 == that.int16 && int16_default == that.int16_default && int32 == that.int32 && int32_default == that.int32_default && int64 == that.int64 && int64_default == that.int64_default && uint8 == that.uint8 && uint16 == that.uint16 && uint32 == that.uint32 && Float.compare(float32, that.float32) == 0 && Double.compare(float64, that.float64) == 0 && bool == that.bool && enum8 == that.enum8 && enum16 == that.enum16 && Objects.equals(int128, that.int128) && Objects.equals(int128_default, that.int128_default) && Objects.equals(int256, that.int256) && Objects.equals(int256_default, that.int256_default) && Objects.equals(uint64, that.uint64) && Objects.equals(uint128, that.uint128) && Objects.equals(uint256, that.uint256) && Objects.equals(decimal32, that.decimal32) && Objects.equals(decimal64, that.decimal64) && Objects.equals(decimal128, that.decimal128) && Objects.equals(decimal256, that.decimal256) && Objects.equals(string, that.string) && Objects.equals(fixedString, that.fixedString) && Objects.equals(date, that.date) && Objects.equals(date32, that.date32) && Objects.equals(dateTime, that.dateTime) && Objects.equals(dateTime64, that.dateTime64) && Objects.equals(uuid, that.uuid) && Objects.equals(ipv4, that.ipv4) && Objects.equals(ipv6, that.ipv6) && Objects.equals(array, that.array) && Objects.equals(map, that.map) && Objects.equals(nestedInnerInt, that.nestedInnerInt) && Objects.equals(nestedInnerString, that.nestedInnerString);
465470
}
466471

467472
@Override
468473
public int hashCode() {
469-
return Objects.hash(int8, int16, int32, int64, int128, int256, uint8, uint16, uint32, uint64, uint128, uint256, float32, float64, decimal32, decimal64, decimal128, decimal256, bool, string, fixedString, date, date32, dateTime, dateTime64, uuid, enum8, enum16, ipv4, ipv6, array, tuple, map, nestedInnerInt, nestedInnerString);
474+
return Objects.hash(int8, int8_default, int16, int16_default, int32, int32_default, int64, int64_default, int128, int128_default, int256, int256_default, uint8, uint16, uint32, uint64, uint128, uint256, float32, float64, decimal32, decimal64, decimal128, decimal256, bool, string, fixedString, date, date32, dateTime, dateTime64, uuid, enum8, enum16, ipv4, ipv6, array, map, nestedInnerInt, nestedInnerString);
470475
}
471476

472477
@Override
473478
public String toString() {
474-
return "SamplePOJO{" +
479+
return "QuerySamplePOJO{" +
475480
"int8=" + int8 +
476481
", int8_default=" + int8_default +
477482
", int16=" + int16 +
@@ -509,7 +514,6 @@ public String toString() {
509514
", ipv4=" + ipv4 +
510515
", ipv6=" + ipv6 +
511516
", array=" + array +
512-
", tuple=" + tuple +
513517
", map=" + map +
514518
", nestedInnerInt=" + nestedInnerInt +
515519
", nestedInnerString=" + nestedInnerString +
@@ -555,7 +559,7 @@ public static String generateTableCreateSQL(String tableName) {
555559
"ipv4 IPv4, " +
556560
"ipv6 IPv6, " +
557561
"array Array(String), " +
558-
"tuple Tuple(Int32, Int32, Int32, Int32, Int32, Int32, Int32, Int32, Int32, Int32), " +
562+
// "tuple Tuple(Int32, Int32, Int32, Int32, Int32, Int32, Int32, Int32, Int32, Int32), " +
559563
"map Map(String, Int32), " +
560564
"nested Nested (innerInt Int32, innerString String)" +
561565
") ENGINE = Memory";

‎client-v2/src/test/java/com/clickhouse/client/query/QueryTests.java‎

Lines changed: 8 additions & 7 deletions
Original file line numberDiff line numberDiff line change
@@ -31,7 +31,6 @@
3131
import com.fasterxml.jackson.databind.JsonNode;
3232
import com.fasterxml.jackson.databind.MappingIterator;
3333
import com.fasterxml.jackson.databind.ObjectMapper;
34-
import org.testcontainers.shaded.com.google.common.collect.Table;
3534
import org.testng.Assert;
3635
import org.testng.annotations.AfterMethod;
3736
import org.testng.annotations.BeforeMethod;
@@ -47,7 +46,6 @@
4746
import java.io.OutputStreamWriter;
4847
import java.math.BigDecimal;
4948
import java.math.BigInteger;
50-
import java.math.RoundingMode;
5149
import java.net.Inet4Address;
5250
import java.net.Inet6Address;
5351
import java.net.InetAddress;
@@ -91,6 +89,10 @@ public class QueryTests extends BaseIntegrationTest {
9189

9290
private boolean usePreallocatedBuffers = false;
9391

92+
static {
93+
// System.setProperty("org.slf4j.simpleLogger.defaultLogLevel", "DEBUG");
94+
}
95+
9496
QueryTests(){
9597
}
9698

@@ -1473,7 +1475,6 @@ public void testQueryReadToPOJO() {
14731475
" FROM system.numbers LIMIT " + limit;
14741476
TableSchema schema = client.getTableSchemaFromQuery(sql, "q1");
14751477
client.register(SimplePOJO.class, schema);
1476-
14771478
List<SimplePOJO> pojos = client.queryAll(sql, SimplePOJO.class);
14781479
Assert.assertEquals(pojos.size(), limit);
14791480
}
@@ -1497,12 +1498,12 @@ public void testQueryReadToPOJOWithoutGetters() {
14971498
public void testQueryAllWithPOJO() throws Exception {
14981499

14991500
final String tableName = "test_query_all_with_pojo";
1500-
final String createTableSQL = SamplePOJO.generateTableCreateSQL(tableName);
1501+
final String createTableSQL = QuerySamplePOJO.generateTableCreateSQL(tableName);
15011502
client.execute("DROP TABLE IF EXISTS test_query_all_with_pojo").get();
15021503
client.execute(createTableSQL).get();
15031504

1504-
SamplePOJO pojo = new SamplePOJO();
1505-
client.register(SamplePOJO.class, client.getTableSchema(tableName));
1505+
QuerySamplePOJO pojo = new QuerySamplePOJO();
1506+
client.register(QuerySamplePOJO.class, client.getTableSchema(tableName));
15061507

15071508
client.insert(tableName, Collections.singletonList(pojo)).get();
15081509

@@ -1516,7 +1517,7 @@ public void testQueryAllWithPOJO() throws Exception {
15161517
pojo.setDateTime(pojo.getDateTime().minusNanos(pojo.getDateTime().getNano()));
15171518
pojo.setDateTime64(pojo.getDateTime64().withNano((int) Math.ceil((pojo.getDateTime64().getNano() / 1000_000) * 1000_000)));
15181519

1519-
List<SamplePOJO> pojos = client.queryAll("SELECT * FROM " + tableName + " LIMIT 1", SamplePOJO.class);
1520+
List<QuerySamplePOJO> pojos = client.queryAll("SELECT * FROM " + tableName + " LIMIT 1", QuerySamplePOJO.class);
15201521
Assert.assertEquals(pojos.get(0), pojo, "Expected " + pojo + " but got " + pojos.get(0));
15211522
}
15221523

0 commit comments

Comments
 (0)