Skip to content

Commit 4367546

Browse files
rbygraverbygrave
authored
Merge pull request #3523 from FOCONIS/streaming-tests
set defaultFetchBuffer in findEach/findList in SQL and DTO Queries
2 parents 1c038f4 + 4d08dce commit 4367546

File tree

8 files changed

+232
-6
lines changed

8 files changed

+232
-6
lines changed

‎ebean-core/src/main/java/io/ebeaninternal/api/SpiSqlBinding.java‎

Lines changed: 5 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -53,4 +53,9 @@ public interface SpiSqlBinding extends SpiCancelableQuery {
5353
*/
5454
int getBufferFetchSizeHint();
5555

56+
/**
57+
* Set the JDBC fetchSize buffer hint if not explicitly set.
58+
*/
59+
void setDefaultFetchBuffer(int fetchSize);
60+
5661
}

‎ebean-core/src/main/java/io/ebeaninternal/server/core/AbstractSqlQueryRequest.java‎

Lines changed: 6 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -92,6 +92,12 @@ public String getBindLog() {
9292

9393
protected abstract void requestComplete();
9494

95+
/**
96+
* Set the JDBC buffer fetchSize hint if not set explicitly.
97+
*/
98+
public void setDefaultFetchBuffer(int fetchSize) {
99+
query.setDefaultFetchBuffer(fetchSize);
100+
}
95101
/**
96102
* Close the underlying resources.
97103
*/

‎ebean-core/src/main/java/io/ebeaninternal/server/core/InternalConfiguration.java‎

Lines changed: 3 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -300,11 +300,12 @@ AutoTuneService createAutoTuneService(SpiEbeanServer server) {
300300
}
301301

302302
DtoQueryEngine createDtoQueryEngine() {
303-
return new DtoQueryEngine(binder);
303+
return new DtoQueryEngine(binder, config.getJdbcFetchSizeFindEach(), config.getJdbcFetchSizeFindList());
304304
}
305305

306306
RelationalQueryEngine createRelationalQueryEngine() {
307-
return new DefaultRelationalQueryEngine(binder, config.getDatabaseBooleanTrue(), config.getPlatformConfig().getDbUuid().useBinaryOptimized());
307+
return new DefaultRelationalQueryEngine(binder, config.getDatabaseBooleanTrue(), config.getPlatformConfig().getDbUuid().useBinaryOptimized(),
308+
config.getJdbcFetchSizeFindEach(), config.getJdbcFetchSizeFindList());
308309
}
309310

310311
OrmQueryEngine createOrmQueryEngine() {

‎ebean-core/src/main/java/io/ebeaninternal/server/query/DefaultRelationalQueryEngine.java‎

Lines changed: 24 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -29,12 +29,17 @@ public final class DefaultRelationalQueryEngine implements RelationalQueryEngine
2929
private final String dbTrueValue;
3030
private final boolean binaryOptimizedUUID;
3131
private final TimedMetricMap timedMetricMap;
32+
private final int defaultFetchSizeFindEach;
33+
private final int defaultFetchSizeFindList;
3234

33-
public DefaultRelationalQueryEngine(Binder binder, String dbTrueValue, boolean binaryOptimizedUUID) {
35+
public DefaultRelationalQueryEngine(Binder binder, String dbTrueValue, boolean binaryOptimizedUUID,
36+
int defaultFetchSizeFindEach, int defaultFetchSizeFindList) {
3437
this.binder = binder;
3538
this.dbTrueValue = dbTrueValue == null ? "true" : dbTrueValue;
3639
this.binaryOptimizedUUID = binaryOptimizedUUID;
3740
this.timedMetricMap = MetricFactory.get().createTimedMetricMap("sql.query.");
41+
this.defaultFetchSizeFindEach = defaultFetchSizeFindEach;
42+
this.defaultFetchSizeFindList = defaultFetchSizeFindList;
3843
}
3944

4045
@Override
@@ -59,6 +64,9 @@ private String errMsg(String msg, String sql) {
5964
@Override
6065
public void findEach(RelationalQueryRequest request, RowConsumer consumer) {
6166
try {
67+
if (defaultFetchSizeFindEach > 0) {
68+
request.setDefaultFetchBuffer(defaultFetchSizeFindEach);
69+
}
6270
request.executeSql(binder, SpiQuery.Type.ITERATE);
6371
request.mapEach(consumer);
6472
request.logSummary();
@@ -74,6 +82,9 @@ public void findEach(RelationalQueryRequest request, RowConsumer consumer) {
7482
@Override
7583
public <T> void findEach(RelationalQueryRequest request, RowReader<T> reader, Predicate<T> consumer) {
7684
try {
85+
if (defaultFetchSizeFindEach > 0) {
86+
request.setDefaultFetchBuffer(defaultFetchSizeFindEach);
87+
}
7788
request.executeSql(binder, SpiQuery.Type.ITERATE);
7889
while (request.next()) {
7990
if (!consumer.test(reader.read())) {
@@ -109,6 +120,9 @@ public <T> T findOne(RelationalQueryRequest request, RowMapper<T> mapper) {
109120
@Override
110121
public <T> List<T> findList(RelationalQueryRequest request, RowReader<T> reader) {
111122
try {
123+
if (defaultFetchSizeFindList > 0) {
124+
request.setDefaultFetchBuffer(defaultFetchSizeFindList);
125+
}
112126
request.executeSql(binder, SpiQuery.Type.LIST);
113127
List<T> rows = new ArrayList<>();
114128
while (request.next()) {
@@ -129,6 +143,9 @@ public <T> List<T> findList(RelationalQueryRequest request, RowReader<T> reader)
129143
public <T> T findSingleAttribute(RelationalQueryRequest request, Class<T> cls) {
130144
ScalarType<T> scalarType = (ScalarType<T>) binder.getScalarType(cls);
131145
try {
146+
if (defaultFetchSizeFindList > 0) {
147+
request.setDefaultFetchBuffer(defaultFetchSizeFindList);
148+
}
132149
request.executeSql(binder, SpiQuery.Type.ATTRIBUTE);
133150
final DataReader dataReader = binder.createDataReader(request.resultSet());
134151
T value = null;
@@ -151,6 +168,9 @@ public <T> T findSingleAttribute(RelationalQueryRequest request, Class<T> cls) {
151168
public <T> List<T> findSingleAttributeList(RelationalQueryRequest request, Class<T> cls) {
152169
ScalarType<T> scalarType = (ScalarType<T>) binder.getScalarType(cls);
153170
try {
171+
if (defaultFetchSizeFindList > 0) {
172+
request.setDefaultFetchBuffer(defaultFetchSizeFindList);
173+
}
154174
request.executeSql(binder, SpiQuery.Type.ATTRIBUTE);
155175
final DataReader dataReader = binder.createDataReader(request.resultSet());
156176
List<T> rows = new ArrayList<>();
@@ -173,6 +193,9 @@ public <T> List<T> findSingleAttributeList(RelationalQueryRequest request, Class
173193
public <T> void findSingleAttributeEach(RelationalQueryRequest request, Class<T> cls, Consumer<T> consumer) {
174194
ScalarType<T> scalarType = (ScalarType<T>) binder.getScalarType(cls);
175195
try {
196+
if (defaultFetchSizeFindEach > 0) {
197+
request.setDefaultFetchBuffer(defaultFetchSizeFindEach);
198+
}
176199
request.executeSql(binder, SpiQuery.Type.ATTRIBUTE);
177200
final DataReader dataReader = binder.createDataReader(request.resultSet());
178201
while (dataReader.next()) {

‎ebean-core/src/main/java/io/ebeaninternal/server/query/DtoQueryEngine.java‎

Lines changed: 22 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -4,8 +4,8 @@
44
import io.ebeaninternal.api.SpiQuery;
55
import io.ebeaninternal.server.core.DtoQueryRequest;
66
import io.ebeaninternal.server.persist.Binder;
7-
87
import jakarta.persistence.PersistenceException;
8+
99
import java.sql.SQLException;
1010
import java.util.ArrayList;
1111
import java.util.List;
@@ -15,13 +15,20 @@
1515
public final class DtoQueryEngine {
1616

1717
private final Binder binder;
18+
private final int defaultFetchSizeFindEach;
19+
private final int defaultFetchSizeFindList;
1820

19-
public DtoQueryEngine(Binder binder) {
21+
public DtoQueryEngine(Binder binder, int defaultFetchSizeFindEach, int defaultFetchSizeFindList) {
2022
this.binder = binder;
23+
this.defaultFetchSizeFindEach = defaultFetchSizeFindEach;
24+
this.defaultFetchSizeFindList = defaultFetchSizeFindList;
2125
}
2226

2327
public <T> List<T> findList(DtoQueryRequest<T> request) {
2428
try {
29+
if (defaultFetchSizeFindList > 0) {
30+
request.setDefaultFetchBuffer(defaultFetchSizeFindList);
31+
}
2532
request.executeSql(binder, SpiQuery.Type.LIST);
2633
List<T> rows = new ArrayList<>();
2734
while (request.next()) {
@@ -38,6 +45,9 @@ public <T> List<T> findList(DtoQueryRequest<T> request) {
3845

3946
public <T> QueryIterator<T> findIterate(DtoQueryRequest<T> request) {
4047
try {
48+
if (defaultFetchSizeFindEach > 0) {
49+
request.setDefaultFetchBuffer(defaultFetchSizeFindEach);
50+
}
4151
request.executeSql(binder, SpiQuery.Type.ITERATE);
4252
return new DtoQueryIterator<>(request);
4353
} catch (SQLException e) {
@@ -46,7 +56,10 @@ public <T> QueryIterator<T> findIterate(DtoQueryRequest<T> request) {
4656
}
4757

4858
public <T> void findEach(DtoQueryRequest<T> request, Consumer<T> consumer) {
49-
try {
59+
try {
60+
if (defaultFetchSizeFindEach > 0) {
61+
request.setDefaultFetchBuffer(defaultFetchSizeFindEach);
62+
}
5063
request.executeSql(binder, SpiQuery.Type.ITERATE);
5164
while (request.next()) {
5265
consumer.accept(request.readNextBean());
@@ -61,6 +74,9 @@ public <T> void findEach(DtoQueryRequest<T> request, Consumer<T> consumer) {
6174
public <T> void findEach(DtoQueryRequest<T> request, int batchSize, Consumer<List<T>> consumer) {
6275
try {
6376
List<T> buffer = new ArrayList<>();
77+
if (defaultFetchSizeFindEach > 0) {
78+
request.setDefaultFetchBuffer(defaultFetchSizeFindEach);
79+
}
6480
request.executeSql(binder, SpiQuery.Type.ITERATE);
6581
while (request.next()) {
6682
buffer.add(request.readNextBean());
@@ -82,6 +98,9 @@ public <T> void findEach(DtoQueryRequest<T> request, int batchSize, Consumer<Lis
8298

8399
public <T> void findEachWhile(DtoQueryRequest<T> request, Predicate<T> consumer) {
84100
try {
101+
if (defaultFetchSizeFindEach > 0) {
102+
request.setDefaultFetchBuffer(defaultFetchSizeFindEach);
103+
}
85104
request.executeSql(binder, SpiQuery.Type.ITERATE);
86105
while (request.next()) {
87106
if (!consumer.test(request.readNextBean())) {

‎ebean-core/src/main/java/io/ebeaninternal/server/querydefn/DefaultDtoQuery.java‎

Lines changed: 7 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -335,6 +335,13 @@ public BindParams getBindParams() {
335335
return bindParams;
336336
}
337337

338+
@Override
339+
public void setDefaultFetchBuffer(int fetchSize) {
340+
if (bufferFetchSizeHint == 0) {
341+
bufferFetchSizeHint = fetchSize;
342+
}
343+
}
344+
338345
@Override
339346
public DtoQuery<T> setBufferFetchSizeHint(int bufferFetchSizeHint) {
340347
this.bufferFetchSizeHint = bufferFetchSizeHint;

‎ebean-core/src/main/java/io/ebeaninternal/server/querydefn/DefaultRelationalQuery.java‎

Lines changed: 7 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -206,6 +206,13 @@ public BindParams getBindParams() {
206206
return bindParams;
207207
}
208208

209+
@Override
210+
public final void setDefaultFetchBuffer(int fetchSize) {
211+
if (bufferFetchSizeHint == 0) {
212+
bufferFetchSizeHint = fetchSize;
213+
}
214+
}
215+
209216
@Override
210217
public DefaultRelationalQuery setBufferFetchSizeHint(int bufferFetchSizeHint) {
211218
this.bufferFetchSizeHint = bufferFetchSizeHint;

‎ebean-test/src/test/java/org/tests/query/other/TestFindIterateMariaDb.java‎

Lines changed: 158 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,158 @@
1+
package org.tests.query.other;
2+
3+
import io.ebean.DB;
4+
import io.ebean.Transaction;
5+
import io.ebean.annotation.Platform;
6+
import io.ebean.xtest.BaseTestCase;
7+
import io.ebean.xtest.ForPlatform;
8+
import org.junit.jupiter.api.BeforeEach;
9+
import org.junit.jupiter.api.Test;
10+
import org.tests.model.basic.EBasic;
11+
12+
import java.lang.reflect.Field;
13+
import java.util.concurrent.atomic.AtomicBoolean;
14+
15+
import static org.assertj.core.api.Assertions.assertThat;
16+
17+
/**
18+
* This test ensures that MariaDb uses the correct streaming result in findEach queries.
19+
* See Issue #56.
20+
*/
21+
public class TestFindIterateMariaDb extends BaseTestCase {
22+
23+
@BeforeEach
24+
public void setup() {
25+
// we need at least > fetchSize beans
26+
if (DB.find(EBasic.class).findCount() < 1000) {
27+
for (int i = 0; i < 1000; i++) {
28+
EBasic dumbModel = new EBasic();
29+
dumbModel.setName("Goodbye now");
30+
DB.save(dumbModel);
31+
}
32+
}
33+
}
34+
35+
public static class DtoBasic {
36+
private String name;
37+
38+
public String getName() {
39+
return name;
40+
}
41+
42+
public void setName(String name) {
43+
this.name = name;
44+
}
45+
}
46+
47+
@Test
48+
@ForPlatform(Platform.MARIADB)
49+
public void testStreamingOnSqlQueryFindEach() {
50+
51+
try (Transaction txn = DB.beginTransaction()) {
52+
AtomicBoolean mariadbStreaming = new AtomicBoolean();
53+
DB.sqlQuery("select name from e_basic").findEach(bean -> {
54+
if (!mariadbStreaming.get() && isMariaDbStreaming()) {
55+
mariadbStreaming.set(true);
56+
}
57+
});
58+
assertThat(mariadbStreaming.get()).isTrue();
59+
}
60+
}
61+
62+
@Test
63+
@ForPlatform(Platform.MARIADB)
64+
public void testStreamingOnOrmFindEach() {
65+
66+
try (Transaction txn = DB.beginTransaction()) {
67+
AtomicBoolean mariadbStreaming = new AtomicBoolean();
68+
DB.find(EBasic.class).findEach(bean -> {
69+
if (!mariadbStreaming.get() && isMariaDbStreaming()) {
70+
mariadbStreaming.set(true);
71+
}
72+
});
73+
assertThat(mariadbStreaming.get()).isTrue();
74+
75+
}
76+
}
77+
78+
@Test
79+
@ForPlatform(Platform.MARIADB)
80+
public void testStreamingOnOrmAsDtoFindEach() {
81+
82+
try (Transaction txn = DB.beginTransaction()) {
83+
AtomicBoolean mariadbStreaming = new AtomicBoolean();
84+
DB.find(EBasic.class).select("name").asDto(DtoBasic.class).findEach(bean -> {
85+
if (!mariadbStreaming.get() && isMariaDbStreaming()) {
86+
mariadbStreaming.set(true);
87+
}
88+
});
89+
assertThat(mariadbStreaming.get()).isTrue();
90+
91+
}
92+
}
93+
94+
@Test
95+
@ForPlatform(Platform.MARIADB)
96+
public void testStreamingOnDtoFindEach() {
97+
98+
try (Transaction txn = DB.beginTransaction()) {
99+
AtomicBoolean mariadbStreaming = new AtomicBoolean();
100+
DB.findDto(DtoBasic.class, "select name from e_basic").findEach(bean -> {
101+
if (!mariadbStreaming.get() && isMariaDbStreaming()) {
102+
mariadbStreaming.set(true);
103+
}
104+
});
105+
assertThat(mariadbStreaming.get()).isTrue();
106+
107+
}
108+
}
109+
@Test
110+
@ForPlatform(Platform.MARIADB)
111+
public void testStreamingOnDtoFindEachWhile() {
112+
113+
try (Transaction txn = DB.beginTransaction()) {
114+
AtomicBoolean mariadbStreaming = new AtomicBoolean();
115+
DB.findDto(DtoBasic.class, "select name from e_basic").findEachWhile(bean -> {
116+
if (!mariadbStreaming.get() && isMariaDbStreaming()) {
117+
mariadbStreaming.set(true);
118+
}
119+
return false;
120+
});
121+
assertThat(mariadbStreaming.get()).isTrue();
122+
123+
}
124+
}
125+
126+
@Test
127+
@ForPlatform(Platform.MARIADB)
128+
public void testStreamingOnDtoFindEachBatch() {
129+
130+
try (Transaction txn = DB.beginTransaction()) {
131+
AtomicBoolean mariadbStreaming = new AtomicBoolean();
132+
DB.findDto(DtoBasic.class, "select name from e_basic").findEach(2000, bean -> {
133+
if (!mariadbStreaming.get() && isMariaDbStreaming()) {
134+
mariadbStreaming.set(true);
135+
}
136+
});
137+
assertThat(mariadbStreaming.get()).isTrue();
138+
139+
}
140+
}
141+
142+
/**
143+
* Take a look into the current connection. We are streaming, if there is a result set.
144+
*/
145+
private boolean isMariaDbStreaming() {
146+
try {
147+
org.mariadb.jdbc.Connection conn = Transaction.current().connection().unwrap(org.mariadb.jdbc.Connection.class);
148+
org.mariadb.jdbc.client.impl.StandardClient client = (org.mariadb.jdbc.client.impl.StandardClient) conn.getClient();
149+
Field field = client.getClass().getDeclaredField("streamMsg");
150+
field.setAccessible(true);
151+
return field.get(client) != null;
152+
} catch (Exception e) {
153+
e.printStackTrace();
154+
return false;
155+
}
156+
}
157+
158+
}

0 commit comments

Comments
 (0)