Skip to content

Commit 7e4d144

Browse files
chernserchernser
committed
fixed comments and updated example
1 parent 00a18d1 commit 7e4d144

File tree

4 files changed

+55
-44
lines changed

4 files changed

+55
-44
lines changed

‎client-v2/src/main/java/com/clickhouse/client/api/Client.java‎

Lines changed: 5 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -1535,10 +1535,15 @@ public <T> List<T> queryAll(String sqlQuery, Class<T> clazz) {
15351535
}
15361536

15371537
/**
1538+
* WARNING: Experimental API
1539+
*
15381540
* <p>Queries data and returns collection with whole result. Data is read directly to a DTO
15391541
* to save memory on intermediate structures. DTO will be instantiated with default constructor or
15401542
* by using allocator</p>
15411543
* <p>{@code class} should be registered before calling this method using {@link #register(Class, TableSchema)}</p>
1544+
* <p>Internally deserializer is compiled at the register stage. Compilation is done using ASM library by
1545+
* writing a bytecode</p>
1546+
*
15421547
* @param sqlQuery - query to execute
15431548
* @param clazz - class of the DTO
15441549
* @param allocator - optional supplier to create new instances of the DTO.

‎client-v2/src/main/java/com/clickhouse/client/api/internal/SerializerUtils.java‎

Lines changed: 20 additions & 30 deletions
Original file line numberDiff line numberDiff line change
@@ -394,21 +394,7 @@ public static POJOSetter compilePOJOSetter(Method setterMethod, ClickHouseColumn
394394
} else if (targetType.isPrimitive() && !targetType.isArray()) {
395395
// unboxing
396396
mv.visitVarInsn(ALOAD, 2); // load object
397-
String sourceInternalClassName;
398-
if (column.getDataType().isSigned()) {
399-
sourceInternalClassName = Type.getInternalName(sourceType);
400-
} else if (column.getDataType() == ClickHouseDataType.UInt64) {
401-
sourceInternalClassName = Type.getInternalName(BigInteger.class);
402-
} else if (column.getDataType() == ClickHouseDataType.Enum8) {
403-
sourceInternalClassName = Type.getInternalName(Byte.class);
404-
} else if (column.getDataType() == ClickHouseDataType.Enum16) {
405-
sourceInternalClassName = Type.getInternalName(Short.class);
406-
} else {
407-
sourceInternalClassName = Type.getInternalName(
408-
ClickHouseDataType.toObjectType(ClickHouseDataType.toWiderPrimitiveType(
409-
ClickHouseDataType.toPrimitiveType(sourceType))));
410-
System.out.println("sourceInternalClassName: " + sourceInternalClassName + " " + column.getColumnName());
411-
}
397+
String sourceInternalClassName = getSourceInternalClassName(column, sourceType);
412398
mv.visitTypeInsn(CHECKCAST, sourceInternalClassName);
413399
mv.visitMethodInsn(INVOKEVIRTUAL,
414400
sourceInternalClassName,
@@ -417,21 +403,7 @@ public static POJOSetter compilePOJOSetter(Method setterMethod, ClickHouseColumn
417403
false);
418404
} else if (!targetPrimitiveType.isPrimitive()) {
419405
// boxing
420-
String sourceInternalClassName;
421-
if (column.getDataType().isSigned()) {
422-
sourceInternalClassName = Type.getInternalName(sourceType);
423-
} else if (column.getDataType() == ClickHouseDataType.UInt64) {
424-
sourceInternalClassName = Type.getInternalName(BigInteger.class);
425-
} else if (column.getDataType() == ClickHouseDataType.Enum8) {
426-
sourceInternalClassName = Type.getInternalName(Byte.class);
427-
} else if (column.getDataType() == ClickHouseDataType.Enum16) {
428-
sourceInternalClassName = Type.getInternalName(Short.class);
429-
} else {
430-
sourceInternalClassName = Type.getInternalName(
431-
ClickHouseDataType.toObjectType(ClickHouseDataType.toWiderPrimitiveType(
432-
ClickHouseDataType.toPrimitiveType(sourceType))));
433-
System.out.println("sourceInternalClassName: " + sourceInternalClassName + " " + column.getColumnName());
434-
}
406+
String sourceInternalClassName = getSourceInternalClassName(column, sourceType);
435407
mv.visitVarInsn(ALOAD, 2); // load object
436408
mv.visitTypeInsn(CHECKCAST, sourceInternalClassName);
437409
try {
@@ -478,6 +450,24 @@ public static POJOSetter compilePOJOSetter(Method setterMethod, ClickHouseColumn
478450
}
479451
}
480452

453+
private static String getSourceInternalClassName(ClickHouseColumn column, Class<?> sourceType) {
454+
String sourceInternalClassName;
455+
if (column.getDataType().isSigned()) {
456+
sourceInternalClassName = Type.getInternalName(sourceType);
457+
} else if (column.getDataType() == ClickHouseDataType.UInt64) {
458+
sourceInternalClassName = Type.getInternalName(BigInteger.class);
459+
} else if (column.getDataType() == ClickHouseDataType.Enum8) {
460+
sourceInternalClassName = Type.getInternalName(Byte.class);
461+
} else if (column.getDataType() == ClickHouseDataType.Enum16) {
462+
sourceInternalClassName = Type.getInternalName(Short.class);
463+
} else {
464+
sourceInternalClassName = Type.getInternalName(
465+
ClickHouseDataType.toObjectType(ClickHouseDataType.toWiderPrimitiveType(
466+
ClickHouseDataType.toPrimitiveType(sourceType))));
467+
}
468+
return sourceInternalClassName;
469+
}
470+
481471
private static String pojoSetterMethodDescriptor(Class<?> dtoClass, Class<?> argType) {
482472
StringBuilder sb = new StringBuilder();
483473
sb.append('(');

‎examples/demo-service/src/main/java/com/clickhouse/demo_service/DatasetController.java‎

Lines changed: 29 additions & 14 deletions
Original file line numberDiff line numberDiff line change
@@ -27,6 +27,8 @@
2727
import java.util.List;
2828
import java.util.concurrent.ConcurrentLinkedDeque;
2929
import java.util.concurrent.TimeUnit;
30+
import java.util.function.Supplier;
31+
import java.util.stream.Collectors;
3032

3133
/**
3234
* Dataset API:
@@ -84,12 +86,14 @@ VirtualDatasetRecord create() {
8486
" FROM system.numbers";
8587

8688
/**
87-
* Common approach to fetch data from ClickHouse using client v2.
89+
* Fetches data from a DB using row binary reader. VirtualDatasetRecord objects are created on each iteration and
90+
* filled with data from the reader. Gives full control on how data is processed and stored.
91+
* If this method returns a lot of data it may cause application slowdown.
8892
*
8993
* @param limit
9094
* @return
9195
*/
92-
@GetMapping("/direct/dataset/0")
96+
@GetMapping("/dataset/reader")
9397
public List<VirtualDatasetRecord> directDatasetFetch(@RequestParam(name = "limit", required = false) Integer limit) {
9498
limit = limit == null ? 100 : limit;
9599

@@ -118,7 +122,7 @@ public List<VirtualDatasetRecord> directDatasetFetch(@RequestParam(name = "limit
118122
response.getMetrics().getMetric(ClientMetrics.OP_DURATION).getLong(),
119123
TimeUnit.NANOSECONDS.toMillis(response.getServerTime())));
120124

121-
return result;
125+
return result.stream().findFirst().stream().collect(Collectors.toCollection(ArrayList::new));
122126
} catch (Exception e) {
123127
throw new RuntimeException("Failed to fetch dataset", e);
124128
}
@@ -127,12 +131,14 @@ public List<VirtualDatasetRecord> directDatasetFetch(@RequestParam(name = "limit
127131
private JsonMapper jsonMapper = new JsonMapper();
128132

129133
/**
130-
* Current approach is to demonstrate how to 'stream' data from ClickHouse using JSONEachRow format.
131-
* This approach is faster than common one because it bypasses Spring internals and writes directly to http output stream.
134+
* Reads data in JSONEachRow format, parses it into JSON library object (can be used for further processing) and
135+
* writes it back to the response.
136+
* This helps to reduce effort of writing data to the response.
137+
*
132138
* @param httpResp
133139
* @param limit
134140
*/
135-
@GetMapping("/direct/dataset/1")
141+
@GetMapping("/dataset/json_each_row_in_and_out")
136142
@ResponseBody
137143
public void directDataFetchJSONEachRow(HttpServletResponse httpResp, @RequestParam(name = "limit", required = false) Integer limit) {
138144
limit = limit == null ? 100 : limit;
@@ -170,19 +176,25 @@ public void directDataFetchJSONEachRow(HttpServletResponse httpResp, @RequestPar
170176
}
171177

172178
/**
173-
* Using POJO deserialization to fetch data from ClickHouse. Also using a objects cache
174-
* to avoid objects creation on each iteration.
179+
* Using POJO deserialization to fetch data from ClickHouse.
180+
* If cache is enabled, objects are reused from the pool otherwise new objects are created on each iteration.
175181
*
176182
* @param limit
177183
* @return
178184
*/
179-
@GetMapping("/direct/dataset/cached_objects")
180-
public CalculationResult directDatasetFetchCached(@RequestParam(name = "limit", required = false) Integer limit) {
185+
@GetMapping("/dataset/read_to_pojo")
186+
public CalculationResult directDatasetReadToPojo(@RequestParam(name = "limit", required = false) Integer limit,
187+
@RequestParam(name = "cache", required = false) Boolean cache) {
181188
limit = limit == null ? 100 : limit;
189+
cache = cache != null && cache;
190+
return readToPOJO(limit, cache);
191+
}
182192

193+
private CalculationResult readToPOJO(int limit, boolean cache) {
183194
final String query = DATASET_QUERY + " LIMIT " + limit;
184195
List<VirtualDatasetRecord> result = null;
185-
ObjectsPreparedCollection<VirtualDatasetRecord> objectsPool = this.pool.lease(); // take object from the pool
196+
Supplier<VirtualDatasetRecord> objectsPool = cache ? this.pool.lease()
197+
: VirtualDatasetRecord::new;
186198
try {
187199
long start = System.nanoTime();
188200

@@ -193,13 +205,16 @@ public CalculationResult directDatasetFetchCached(@RequestParam(name = "limit",
193205
for (VirtualDatasetRecord record : result) {
194206
p1Sum += record.getP1();
195207
}
196-
log.info(result.toString());
197-
objectsPool.reset(); // reset pool to for next use
208+
if (cache) {
209+
((ObjectsPreparedCollection<VirtualDatasetRecord>) objectsPool).reset();
210+
}
198211
return new CalculationResult(p1Sum);
199212
} catch (Exception e) {
200213
throw new RuntimeException("Failed to fetch dataset", e);
201214
} finally {
202-
this.pool.release(objectsPool);
215+
if (cache) {
216+
this.pool.release((ObjectsPreparedCollection<VirtualDatasetRecord>) objectsPool);
217+
}
203218
}
204219
}
205220
}

‎examples/demo-service/src/main/java/com/clickhouse/demo_service/DbConfiguration.java‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -27,6 +27,7 @@ public Client chDirectClient(@Value("${db.url}") String dbUrl, @Value("${db.user
2727
.setSocketTcpNodelay(true)
2828
.setSocketSndbuf(500_000)
2929
.setClientNetworkBufferSize(500_000)
30+
.allowBinaryReaderToReuseBuffers(true) // using buffer pool for binary reader
3031
.build();
3132
}
3233
}

0 commit comments

Comments
 (0)