diff --git a/fe/fe-common/src/main/java/org/apache/doris/common/Config.java b/fe/fe-common/src/main/java/org/apache/doris/common/Config.java index a0eaf40f77c772..edb01513626042 100644 --- a/fe/fe-common/src/main/java/org/apache/doris/common/Config.java +++ b/fe/fe-common/src/main/java/org/apache/doris/common/Config.java @@ -1386,9 +1386,17 @@ public class Config extends ConfigBase { /** * Set the maximum number of rows that can be cached */ - @ConfField(mutable = true, masterOnly = false) + @ConfField(mutable = true, masterOnly = false, description = {"SQL/Partition Cache可以缓存的最大行数。", + "Maximum number of rows that can be cached in SQL/Partition Cache, is 3000 by default."}) public static int cache_result_max_row_count = 3000; + /** + * Set the maximum data size that can be cached + */ + @ConfField(mutable = true, masterOnly = false, description = {"SQL/Partition Cache可以缓存的最大数据大小。", + "Maximum data size of rows that can be cached in SQL/Partition Cache, is 3000 by default."}) + public static int cache_result_max_data_size = 31457280; // 30M + /** * Used to limit element num of InPredicate in delete statement. */ diff --git a/fe/fe-core/src/main/java/org/apache/doris/qe/cache/Cache.java b/fe/fe-core/src/main/java/org/apache/doris/qe/cache/Cache.java index 2efd805fa8a4db..e68597bec11f73 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/qe/cache/Cache.java +++ b/fe/fe-core/src/main/java/org/apache/doris/qe/cache/Cache.java @@ -41,6 +41,7 @@ public enum HitRange { protected TUniqueId queryId; protected SelectStmt selectStmt; protected RowBatchBuilder rowBatchBuilder; + protected boolean disableCache = false; protected CacheAnalyzer.CacheTable latestTable; protected CacheProxy proxy; protected HitRange hitRange; @@ -75,12 +76,20 @@ public HitRange getHitRange() { public abstract void updateCache(); protected boolean checkRowLimit() { - if (rowBatchBuilder == null) { + if (disableCache || rowBatchBuilder == null) { return false; } if (rowBatchBuilder.getRowSize() > Config.cache_result_max_row_count) { - LOG.info("can not be cached. rowbatch size {} is more than {}", rowBatchBuilder.getRowSize(), + LOG.debug("can not be cached. rowbatch size {} is more than {}", rowBatchBuilder.getRowSize(), Config.cache_result_max_row_count); + rowBatchBuilder.clear(); + disableCache = true; + return false; + } else if (rowBatchBuilder.getDataSize() > Config.cache_result_max_data_size) { + LOG.debug("can not be cached. rowbatch data size {} is more than {}", rowBatchBuilder.getDataSize(), + Config.cache_result_max_data_size); + rowBatchBuilder.clear(); + disableCache = true; return false; } else { return true; diff --git a/fe/fe-core/src/main/java/org/apache/doris/qe/cache/PartitionCache.java b/fe/fe-core/src/main/java/org/apache/doris/qe/cache/PartitionCache.java index 9113bfec82d450..d65d67fdbd2f5b 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/qe/cache/PartitionCache.java +++ b/fe/fe-core/src/main/java/org/apache/doris/qe/cache/PartitionCache.java @@ -123,6 +123,9 @@ public void copyRowBatch(RowBatch rowBatch) { rowBatchBuilder.buildPartitionIndex(selectStmt.getResultExprs(), selectStmt.getColLabels(), partColumn, range.buildUpdatePartitionRange()); } + if (!super.checkRowLimit()) { + return; + } rowBatchBuilder.copyRowData(rowBatch); } diff --git a/fe/fe-core/src/main/java/org/apache/doris/qe/cache/RowBatchBuilder.java b/fe/fe-core/src/main/java/org/apache/doris/qe/cache/RowBatchBuilder.java index 6d5a635a30be85..2cae79597a38b0 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/qe/cache/RowBatchBuilder.java +++ b/fe/fe-core/src/main/java/org/apache/doris/qe/cache/RowBatchBuilder.java @@ -55,6 +55,10 @@ public int getRowSize() { return rowSize; } + public int getDataSize() { + return dataSize; + } + public RowBatchBuilder(CacheAnalyzer.CacheMode model) { cacheMode = model; keyIndex = 0; @@ -99,6 +103,14 @@ public void copyRowData(RowBatch rowBatch) { } } + public void clear() { + rowList = Lists.newArrayList(); + cachePartMap = new HashMap<>(); + batchSize = 0; + rowSize = 0; + dataSize = 0; + } + public InternalService.PUpdateCacheRequest buildSqlUpdateRequest( String sql, long partitionKey, long lastVersion, long lastestTime) { if (updateRequest == null) { diff --git a/fe/fe-core/src/main/java/org/apache/doris/qe/cache/SqlCache.java b/fe/fe-core/src/main/java/org/apache/doris/qe/cache/SqlCache.java index 9cd6c365dcdccc..9d53bb47c4d3cc 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/qe/cache/SqlCache.java +++ b/fe/fe-core/src/main/java/org/apache/doris/qe/cache/SqlCache.java @@ -76,6 +76,9 @@ public void copyRowBatch(RowBatch rowBatch) { if (rowBatchBuilder == null) { rowBatchBuilder = new RowBatchBuilder(CacheAnalyzer.CacheMode.Sql); } + if (!super.checkRowLimit()) { + return; + } rowBatchBuilder.copyRowData(rowBatch); }