diff --git a/ebean-api/src/main/java/io/ebean/config/dbplatform/DatabasePlatform.java b/ebean-api/src/main/java/io/ebean/config/dbplatform/DatabasePlatform.java index 29b554c481..114f3ba79a 100644 --- a/ebean-api/src/main/java/io/ebean/config/dbplatform/DatabasePlatform.java +++ b/ebean-api/src/main/java/io/ebean/config/dbplatform/DatabasePlatform.java @@ -167,6 +167,7 @@ public class DatabasePlatform { * findIterate() and findVisit(). */ protected boolean forwardOnlyHintOnFindIterate; + protected boolean autoCommitFalseOnFindIterate; /** * If set then use the CONCUR_UPDATABLE hint when creating ResultSets. @@ -528,6 +529,10 @@ public void setForwardOnlyHintOnFindIterate(boolean forwardOnlyHintOnFindIterate this.forwardOnlyHintOnFindIterate = forwardOnlyHintOnFindIterate; } + public boolean autoCommitFalseOnFindIterate() { + return autoCommitFalseOnFindIterate; + } + /** * Return true if the ResultSet CONCUR_UPDATABLE Hint should be used on * createNativeSqlTree() PreparedStatements. @@ -780,4 +785,5 @@ public String inlineSqlComment(String label) { public String inlineSqlHint(String hint) { return "/*+ " + hint + " */ "; } + } diff --git a/ebean-core/src/main/java/io/ebeaninternal/api/SpiTransaction.java b/ebean-core/src/main/java/io/ebeaninternal/api/SpiTransaction.java index ded3644538..67d11c5921 100644 --- a/ebean-core/src/main/java/io/ebeaninternal/api/SpiTransaction.java +++ b/ebean-core/src/main/java/io/ebeaninternal/api/SpiTransaction.java @@ -362,4 +362,14 @@ default int depth() { * Set the transaction to be inactive via external transaction manager. */ void deactivateExternal(); + + /** + * Set autocommit to false for a findIterate query. + *

+ * This is done for specific platforms that need it, in order to make + * use cursors to stream a large or unbounded query result to the client. + */ + default void setAutoCommitOnFindIterate() { + throw new UnsupportedOperationException(); + } } diff --git a/ebean-core/src/main/java/io/ebeaninternal/server/core/AbstractSqlQueryRequest.java b/ebean-core/src/main/java/io/ebeaninternal/server/core/AbstractSqlQueryRequest.java index 1e15939e46..8609fa48c4 100644 --- a/ebean-core/src/main/java/io/ebeaninternal/server/core/AbstractSqlQueryRequest.java +++ b/ebean-core/src/main/java/io/ebeaninternal/server/core/AbstractSqlQueryRequest.java @@ -1,7 +1,6 @@ package io.ebeaninternal.server.core; import io.ebean.CancelableQuery; -import io.ebean.Transaction; import io.ebean.util.JdbcClose; import io.ebeaninternal.api.*; import io.ebeaninternal.server.persist.Binder; @@ -98,6 +97,13 @@ public String getBindLog() { public void setDefaultFetchBuffer(int fetchSize) { query.setDefaultFetchBuffer(fetchSize); } + + public void setAutoCommitOnFindIterate() { + if (createdTransaction) { + transaction.setAutoCommitOnFindIterate(); + } + } + /** * Close the underlying resources. */ diff --git a/ebean-core/src/main/java/io/ebeaninternal/server/core/InternalConfiguration.java b/ebean-core/src/main/java/io/ebeaninternal/server/core/InternalConfiguration.java index a3f5d40524..058ac1559b 100644 --- a/ebean-core/src/main/java/io/ebeaninternal/server/core/InternalConfiguration.java +++ b/ebean-core/src/main/java/io/ebeaninternal/server/core/InternalConfiguration.java @@ -301,12 +301,12 @@ AutoTuneService createAutoTuneService(SpiEbeanServer server) { } DtoQueryEngine createDtoQueryEngine() { - return new DtoQueryEngine(binder, config.getJdbcFetchSizeFindEach(), config.getJdbcFetchSizeFindList()); + return new DtoQueryEngine(binder, config.getJdbcFetchSizeFindEach(), config.getJdbcFetchSizeFindList(), databasePlatform.autoCommitFalseOnFindIterate()); } RelationalQueryEngine createRelationalQueryEngine() { return new DefaultRelationalQueryEngine(binder, config.getDatabaseBooleanTrue(), config.getPlatformConfig().getDbUuid().useBinaryOptimized(), - config.getJdbcFetchSizeFindEach(), config.getJdbcFetchSizeFindList()); + config.getJdbcFetchSizeFindEach(), config.getJdbcFetchSizeFindList(), databasePlatform.autoCommitFalseOnFindIterate()); } OrmQueryEngine createOrmQueryEngine() { diff --git a/ebean-core/src/main/java/io/ebeaninternal/server/core/OrmQueryRequest.java b/ebean-core/src/main/java/io/ebeaninternal/server/core/OrmQueryRequest.java index 941a58af59..ecab0b22bb 100644 --- a/ebean-core/src/main/java/io/ebeaninternal/server/core/OrmQueryRequest.java +++ b/ebean-core/src/main/java/io/ebeaninternal/server/core/OrmQueryRequest.java @@ -786,4 +786,10 @@ public void unmodifiableFreeze(EntityBean bean) { beanDescriptor.freeze(bean); } } + + public void setAutoCommitOnFindIterate() { + if (createdTransaction) { + transaction.setAutoCommitOnFindIterate(); + } + } } diff --git a/ebean-core/src/main/java/io/ebeaninternal/server/query/CQueryEngine.java b/ebean-core/src/main/java/io/ebeaninternal/server/query/CQueryEngine.java index 9ecbddacd5..02b3d34ded 100644 --- a/ebean-core/src/main/java/io/ebeaninternal/server/query/CQueryEngine.java +++ b/ebean-core/src/main/java/io/ebeaninternal/server/query/CQueryEngine.java @@ -37,12 +37,14 @@ public final class CQueryEngine { private final CQueryBuilder queryBuilder; private final CQueryHistorySupport historySupport; private final DatabasePlatform dbPlatform; + private final boolean autoCommitFalseOnFindIterate; public CQueryEngine(DatabaseBuilder.Settings config, DatabasePlatform dbPlatform, Binder binder, Map asOfTableMapping, Map draftTableMap) { this.dbPlatform = dbPlatform; this.defaultFetchSizeFindEach = config.getJdbcFetchSizeFindEach(); this.defaultFetchSizeFindList = config.getJdbcFetchSizeFindList(); this.forwardOnlyHintOnFindIterate = dbPlatform.forwardOnlyHintOnFindIterate(); + this.autoCommitFalseOnFindIterate = dbPlatform.autoCommitFalseOnFindIterate(); this.historySupport = new CQueryHistorySupport(dbPlatform.historySupport(), asOfTableMapping, config.getAsOfSysPeriod()); this.queryBuilder = new CQueryBuilder(config, dbPlatform, binder, historySupport, new CQueryDraftSupport(draftTableMap)); } @@ -181,6 +183,9 @@ public QueryIterator findIterate(OrmQueryRequest request) { if (defaultFetchSizeFindEach > 0) { request.setDefaultFetchBuffer(defaultFetchSizeFindEach); } + if (autoCommitFalseOnFindIterate) { + request.setAutoCommitOnFindIterate(); + } if (!cquery.prepareBindExecuteQueryForwardOnly(forwardOnlyHintOnFindIterate)) { // query has been cancelled already return null; diff --git a/ebean-core/src/main/java/io/ebeaninternal/server/query/DefaultRelationalQueryEngine.java b/ebean-core/src/main/java/io/ebeaninternal/server/query/DefaultRelationalQueryEngine.java index f716d76c07..0fa495d01d 100644 --- a/ebean-core/src/main/java/io/ebeaninternal/server/query/DefaultRelationalQueryEngine.java +++ b/ebean-core/src/main/java/io/ebeaninternal/server/query/DefaultRelationalQueryEngine.java @@ -15,6 +15,8 @@ import io.ebeaninternal.server.persist.Binder; import jakarta.persistence.PersistenceException; + +import java.sql.SQLException; import java.util.ArrayList; import java.util.List; import java.util.function.Consumer; @@ -28,15 +30,18 @@ public final class DefaultRelationalQueryEngine implements RelationalQueryEngine private final Binder binder; private final String dbTrueValue; private final boolean binaryOptimizedUUID; + private final boolean autoCommitFalseOnFindIterate; private final TimedMetricMap timedMetricMap; private final int defaultFetchSizeFindEach; private final int defaultFetchSizeFindList; public DefaultRelationalQueryEngine(Binder binder, String dbTrueValue, boolean binaryOptimizedUUID, - int defaultFetchSizeFindEach, int defaultFetchSizeFindList) { + int defaultFetchSizeFindEach, int defaultFetchSizeFindList, + boolean autoCommitFalseOnFindIterate) { this.binder = binder; this.dbTrueValue = dbTrueValue == null ? "true" : dbTrueValue; this.binaryOptimizedUUID = binaryOptimizedUUID; + this.autoCommitFalseOnFindIterate = autoCommitFalseOnFindIterate; this.timedMetricMap = MetricFactory.get().createTimedMetricMap("sql.query."); this.defaultFetchSizeFindEach = defaultFetchSizeFindEach; this.defaultFetchSizeFindList = defaultFetchSizeFindList; @@ -61,13 +66,20 @@ private String errMsg(String msg, String sql) { return "Query threw SQLException:" + msg + " Query was:" + sql; } + private void prepareForIterate(RelationalQueryRequest request) throws SQLException { + if (defaultFetchSizeFindEach > 0) { + request.setDefaultFetchBuffer(defaultFetchSizeFindEach); + } + if (autoCommitFalseOnFindIterate) { + request.setAutoCommitOnFindIterate(); + } + request.executeSql(binder, SpiQuery.Type.ITERATE); + } + @Override public void findEach(RelationalQueryRequest request, RowConsumer consumer) { try { - if (defaultFetchSizeFindEach > 0) { - request.setDefaultFetchBuffer(defaultFetchSizeFindEach); - } - request.executeSql(binder, SpiQuery.Type.ITERATE); + prepareForIterate(request); request.mapEach(consumer); request.logSummary(); @@ -82,10 +94,7 @@ public void findEach(RelationalQueryRequest request, RowConsumer consumer) { @Override public void findEach(RelationalQueryRequest request, RowReader reader, Predicate consumer) { try { - if (defaultFetchSizeFindEach > 0) { - request.setDefaultFetchBuffer(defaultFetchSizeFindEach); - } - request.executeSql(binder, SpiQuery.Type.ITERATE); + prepareForIterate(request); while (request.next()) { if (!consumer.test(reader.read())) { break; diff --git a/ebean-core/src/main/java/io/ebeaninternal/server/query/DtoQueryEngine.java b/ebean-core/src/main/java/io/ebeaninternal/server/query/DtoQueryEngine.java index 8769428cec..574ae1cd57 100644 --- a/ebean-core/src/main/java/io/ebeaninternal/server/query/DtoQueryEngine.java +++ b/ebean-core/src/main/java/io/ebeaninternal/server/query/DtoQueryEngine.java @@ -17,11 +17,13 @@ public final class DtoQueryEngine { private final Binder binder; private final int defaultFetchSizeFindEach; private final int defaultFetchSizeFindList; + private final boolean autoCommitFalseOnFindIterate; - public DtoQueryEngine(Binder binder, int defaultFetchSizeFindEach, int defaultFetchSizeFindList) { + public DtoQueryEngine(Binder binder, int defaultFetchSizeFindEach, int defaultFetchSizeFindList, boolean autoCommitFalseOnFindIterate) { this.binder = binder; this.defaultFetchSizeFindEach = defaultFetchSizeFindEach; this.defaultFetchSizeFindList = defaultFetchSizeFindList; + this.autoCommitFalseOnFindIterate = autoCommitFalseOnFindIterate; } public List findList(DtoQueryRequest request) { @@ -43,12 +45,19 @@ public List findList(DtoQueryRequest request) { } } + private void prepareForIterate(DtoQueryRequest request) throws SQLException { + if (defaultFetchSizeFindEach > 0) { + request.setDefaultFetchBuffer(defaultFetchSizeFindEach); + } + if (autoCommitFalseOnFindIterate) { + request.setAutoCommitOnFindIterate(); + } + request.executeSql(binder, SpiQuery.Type.ITERATE); + } + public QueryIterator findIterate(DtoQueryRequest request) { try { - if (defaultFetchSizeFindEach > 0) { - request.setDefaultFetchBuffer(defaultFetchSizeFindEach); - } - request.executeSql(binder, SpiQuery.Type.ITERATE); + prepareForIterate(request); return new DtoQueryIterator<>(request); } catch (SQLException e) { throw new PersistenceException(errMsg(e.getMessage(), request.getSql()), e); @@ -57,10 +66,7 @@ public QueryIterator findIterate(DtoQueryRequest request) { public void findEach(DtoQueryRequest request, Consumer consumer) { try { - if (defaultFetchSizeFindEach > 0) { - request.setDefaultFetchBuffer(defaultFetchSizeFindEach); - } - request.executeSql(binder, SpiQuery.Type.ITERATE); + prepareForIterate(request); while (request.next()) { consumer.accept(request.readNextBean()); } @@ -73,11 +79,8 @@ public void findEach(DtoQueryRequest request, Consumer consumer) { public void findEach(DtoQueryRequest request, int batchSize, Consumer> consumer) { try { + prepareForIterate(request); List buffer = new ArrayList<>(); - if (defaultFetchSizeFindEach > 0) { - request.setDefaultFetchBuffer(defaultFetchSizeFindEach); - } - request.executeSql(binder, SpiQuery.Type.ITERATE); while (request.next()) { buffer.add(request.readNextBean()); if (buffer.size() >= batchSize) { @@ -98,10 +101,7 @@ public void findEach(DtoQueryRequest request, int batchSize, Consumer void findEachWhile(DtoQueryRequest request, Predicate consumer) { try { - if (defaultFetchSizeFindEach > 0) { - request.setDefaultFetchBuffer(defaultFetchSizeFindEach); - } - request.executeSql(binder, SpiQuery.Type.ITERATE); + prepareForIterate(request); while (request.next()) { if (!consumer.test(request.readNextBean())) { break; diff --git a/ebean-core/src/main/java/io/ebeaninternal/server/transaction/ImplicitReadOnlyTransaction.java b/ebean-core/src/main/java/io/ebeaninternal/server/transaction/ImplicitReadOnlyTransaction.java index f5d87317bd..84c97f4198 100644 --- a/ebean-core/src/main/java/io/ebeaninternal/server/transaction/ImplicitReadOnlyTransaction.java +++ b/ebean-core/src/main/java/io/ebeaninternal/server/transaction/ImplicitReadOnlyTransaction.java @@ -36,7 +36,7 @@ final class ImplicitReadOnlyTransaction implements SpiTransaction, TxnProfileEve /** * Set false when using autoCommit (as a performance optimisation for the read-only case). */ - private final boolean useCommit; + private boolean useCommit; private final TransactionManager manager; private final SpiTxnLogger logger; private final boolean logSql; @@ -550,6 +550,16 @@ public void setRollbackOnly() { throw new IllegalStateException(notExpectedMessage); } + @Override + public void setAutoCommitOnFindIterate() { + try { + connection.setAutoCommit(false); + useCommit = true; + } catch (SQLException e) { + throw new PersistenceException(e); + } + } + @Override public void rollbackAndContinue() { // do nothing diff --git a/ebean-core/src/main/java/io/ebeaninternal/server/transaction/TransactionFactory.java b/ebean-core/src/main/java/io/ebeaninternal/server/transaction/TransactionFactory.java index 23075fa9f6..2ac8263a24 100644 --- a/ebean-core/src/main/java/io/ebeaninternal/server/transaction/TransactionFactory.java +++ b/ebean-core/src/main/java/io/ebeaninternal/server/transaction/TransactionFactory.java @@ -34,7 +34,7 @@ abstract class TransactionFactory { /** * Set the Transaction Isolation level if required. */ - final SpiTransaction setIsolationLevel(SpiTransaction t, boolean explicit, int isolationLevel) { + final SpiTransaction setIsolationLevel(SpiTransaction t, int isolationLevel) { if (isolationLevel > -1) { Connection connection = t.internalConnection(); try { diff --git a/ebean-core/src/main/java/io/ebeaninternal/server/transaction/TransactionFactoryBasic.java b/ebean-core/src/main/java/io/ebeaninternal/server/transaction/TransactionFactoryBasic.java index 26a13c353a..3e7770beb4 100644 --- a/ebean-core/src/main/java/io/ebeaninternal/server/transaction/TransactionFactoryBasic.java +++ b/ebean-core/src/main/java/io/ebeaninternal/server/transaction/TransactionFactoryBasic.java @@ -39,8 +39,8 @@ public final SpiTransaction createTransaction(boolean explicit, int isolationLev Connection connection = null; try { connection = dataSource.getConnection(); - SpiTransaction t = create(explicit, connection); - return setIsolationLevel(t, explicit, isolationLevel); + SpiTransaction t = manager.createTransaction(explicit, connection); + return setIsolationLevel(t, isolationLevel); } catch (PersistenceException ex) { JdbcClose.close(connection); throw ex; @@ -49,8 +49,4 @@ public final SpiTransaction createTransaction(boolean explicit, int isolationLev } } - private SpiTransaction create(boolean explicit, Connection c) { - return manager.createTransaction(explicit, c); - } - } diff --git a/ebean-core/src/main/java/io/ebeaninternal/server/transaction/TransactionFactoryTenant.java b/ebean-core/src/main/java/io/ebeaninternal/server/transaction/TransactionFactoryTenant.java index 2b842141d9..588222137b 100644 --- a/ebean-core/src/main/java/io/ebeaninternal/server/transaction/TransactionFactoryTenant.java +++ b/ebean-core/src/main/java/io/ebeaninternal/server/transaction/TransactionFactoryTenant.java @@ -25,26 +25,32 @@ class TransactionFactoryTenant extends TransactionFactory { @Override public SpiTransaction createReadOnlyTransaction(Object tenantId, boolean useMaster) { - return create(false, tenantId); + Connection connection = null; + try { + if (tenantId == null) { + tenantId = tenantProvider.currentId(); + } + connection = dataSourceSupplier.readOnlyConnection(tenantId, useMaster); + return new ImplicitReadOnlyTransaction(manager, connection, tenantId); + + } catch (PersistenceException ex) { + JdbcClose.close(connection); + throw ex; + + } catch (SQLException ex) { + throw new PersistenceException(ex); + } } @Override public final SpiTransaction createTransaction(boolean explicit, int isolationLevel) { - SpiTransaction t = create(explicit, null); - return setIsolationLevel(t, explicit, isolationLevel); - } - - private SpiTransaction create(boolean explicit, Object tenantId) { Connection connection = null; try { - if (tenantId == null) { - // tenantId not set (by lazy loading) so get current tenantId - tenantId = tenantProvider.currentId(); - } + Object tenantId = tenantProvider.currentId(); connection = dataSourceSupplier.connection(tenantId); SpiTransaction transaction = manager.createTransaction(explicit, connection); transaction.setTenantId(tenantId); - return transaction; + return setIsolationLevel(transaction, isolationLevel); } catch (PersistenceException ex) { JdbcClose.close(connection); diff --git a/ebean-test/src/test/java/main/StartPostgres.java b/ebean-test/src/test/java/main/StartPostgres.java index 353afc5ac4..4cb54ce73c 100644 --- a/ebean-test/src/test/java/main/StartPostgres.java +++ b/ebean-test/src/test/java/main/StartPostgres.java @@ -5,7 +5,7 @@ public class StartPostgres { public static void main(String[] args) { - PostgresContainer.builder("15") + PostgresContainer.builder("17") .dbName("unit") //.port(6432) //.user("unit") diff --git a/ebean-test/src/test/java/org/tests/query/cancel/EBasicDto.java b/ebean-test/src/test/java/org/tests/query/cancel/EBasicDto.java index 49b6b157c8..25b0bac6be 100644 --- a/ebean-test/src/test/java/org/tests/query/cancel/EBasicDto.java +++ b/ebean-test/src/test/java/org/tests/query/cancel/EBasicDto.java @@ -6,9 +6,11 @@ * DTO for Ebasic Queries. */ public class EBasicDto { - private Integer id; + private Integer id; private Status status; + private String name; + private String description; public Integer getId() { return id; @@ -25,4 +27,22 @@ public Status getStatus() { public void setStatus(Status status) { this.status = status; } -} \ No newline at end of file + + public String getName() { + return name; + } + + public EBasicDto setName(String name) { + this.name = name; + return this; + } + + public String getDescription() { + return description; + } + + public EBasicDto setDescription(String description) { + this.description = description; + return this; + } +} diff --git a/ebean-test/src/test/java/org/tests/query/other/TestFindIterateHeapDump.java b/ebean-test/src/test/java/org/tests/query/other/TestFindIterateHeapDump.java index aa34c2a95c..d77c366df4 100644 --- a/ebean-test/src/test/java/org/tests/query/other/TestFindIterateHeapDump.java +++ b/ebean-test/src/test/java/org/tests/query/other/TestFindIterateHeapDump.java @@ -6,7 +6,8 @@ import io.ebean.Database; import org.junit.jupiter.api.Disabled; import org.junit.jupiter.api.Test; -import org.tests.model.basic.EBasic; +// import org.tests.model.basic.EBasic; +import org.tests.query.cancel.EBasicDto; import javax.management.MBeanServer; import java.io.File; @@ -30,57 +31,50 @@ public class TestFindIterateHeapDump extends BaseTestCase { */ @Disabled @Test - public void test() { - + void test() { Database server = DB.getDefault(); -// Transaction transaction = server.beginTransaction(); -// try { +// String desc = "0123456789".repeat(20); +// try (Transaction transaction = server.beginTransaction()){ // transaction.setBatchMode(true); -// transaction.setBatchSize(20); -// for (int i = 0; i < 20000; i++) { +// transaction.setBatchSize(100); +// for (int i = 0; i < 200_000; i++) { // EBasic dumbModel = new EBasic(); // dumbModel.setName("Goodbye now"); +// dumbModel.setDescription(desc); // server.save(dumbModel); // } // transaction.commit(); -// -// } finally { -// transaction.end(); -// } -// -// if (true) { -// return; // } // Intentionally not iterating through the iterator to - final AtomicInteger counter = new AtomicInteger(); - server.find(EBasic.class) - .findEach(bean -> { - + server.sqlQuery("select id, status, name, description from e_basic") + .findEach(bean -> { +// server.findDto(EBasicDto.class, "select id, status, name, description from e_basic") +// .findStream().forEach(bean -> { +// server.find(EBasic.class) +// .findEach(bean -> { +// server.find(EBasic.class) +// .setBufferFetchSizeHint(10) +// .findStream().forEach(bean -> { +// .findEach(bean -> { int count = counter.incrementAndGet(); - if (count == 1) { - dumpHeap("heap-dump13-initial.snapshot", true); + if (count == 10) { + dumpHeap("s1-dump-initial.snapshot.hprof", true); } - }); - // try { - // while (iterate.hasNext()) { - // EBasic eBasic = iterate.next(); - // eBasic.getDescription(); - // } - // } finally { - // iterate.close(); - // } - - String fileName = "heap-dump13.snapshot"; + if (count == (200_000 - 100)) { + dumpHeap("s1-dump-end.snapshot.hprof", true); + } + }); + String fileName = "s1-dump.snapshot.hprof"; File file = new File(fileName); - if (file.exists()) + if (file.exists()) { file.delete(); - + } dumpHeap(fileName, true); } diff --git a/platforms/postgres/src/main/java/io/ebean/platform/postgres/PostgresPlatform.java b/platforms/postgres/src/main/java/io/ebean/platform/postgres/PostgresPlatform.java index 9b037fbf3f..463ddd9547 100644 --- a/platforms/postgres/src/main/java/io/ebean/platform/postgres/PostgresPlatform.java +++ b/platforms/postgres/src/main/java/io/ebean/platform/postgres/PostgresPlatform.java @@ -35,6 +35,7 @@ public PostgresPlatform() { this.blobDbType = Types.LONGVARBINARY; this.clobDbType = Types.VARCHAR; this.nativeUuidType = true; + this.autoCommitFalseOnFindIterate = true; this.truncateTable = "truncate table %s cascade"; this.dbEncrypt = new PostgresDbEncrypt(); this.historySupport = new PostgresHistorySupport();