Skip to content

Commit c74a21c

Browse files
authored
[fix](mv) Fix sync mv could not be chosen by cbo stable because stats upload from be not in time (#58720)
This PR introduces two key modifications to improve the reliability of the optimizer (CBO) and the testing pipeline concerning materialized views: 1. Fixes unstable selection of synchronous materialized views in the pipeline:​ The root cause was that the statistical information of the base table was not reported promptly. This delay could lead the CBO to estimate selectedPartitionsRowCountas zero for the materialized view, causing it to be incorrectly bypassed during query planning. The fix involves manually injecting the statistical information for the synchronous materialized view​ when available, ensuring the CBO can make a stable and accurate assessment of its cost . 2. Removes the fuzzy value setting for preMaterializedViewRewriteStrategyin SessionVariable:​ The test cases for the TRY_IN_RBO strategy have already been extended to cover scenarios equivalent to both NOT_IN_RBO and FORCE_IN_RBO. Retaining the fuzzy option can introduce unnecessary instability into the pipeline. For instance, a PR might pass tests when the strategy is TRY_IN_RBO but fail if the pipeline randomly selects NOT_IN_RBO, leading to inconsistent results. Removing this fuzzy setting helps eliminate such non-deterministic failures and enhances pipeline stability
1 parent ebec0f2 commit c74a21c

File tree

10 files changed

+22
-16
lines changed

10 files changed

+22
-16
lines changed

fe/fe-core/src/main/java/org/apache/doris/nereids/stats/StatsCalculator.java

Lines changed: 6 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -543,6 +543,12 @@ public Statistics computeOlapScan(OlapScan olapScan) {
543543
optStats.isPresent(), tableRowCount, olapTable.getQualifiedName());
544544
if (optStats.isPresent()) {
545545
double selectedPartitionsRowCount = getSelectedPartitionRowCount(olapScan, tableRowCount);
546+
if (isRegisteredRowCount(olapScan)) {
547+
// If a row count is injected for the materialized view, use it to fix the issue where
548+
// the materialized view cannot be selected by cbo stable due to selectedPartitionsRowCount being 0,
549+
// which is caused by delayed statistics reporting.
550+
selectedPartitionsRowCount = tableRowCount;
551+
}
546552
LOG.info("computeOlapScan optStats is {}, selectedPartitionsRowCount is {}", optStats.get(),
547553
selectedPartitionsRowCount);
548554
// if estimated mv rowCount is more than actual row count, fall back to base table stats

fe/fe-core/src/main/java/org/apache/doris/qe/SessionVariable.java

Lines changed: 0 additions & 15 deletions
Original file line numberDiff line numberDiff line change
@@ -32,7 +32,6 @@
3232
import org.apache.doris.nereids.metrics.EventSwitchParser;
3333
import org.apache.doris.nereids.parser.Dialect;
3434
import org.apache.doris.nereids.rules.RuleType;
35-
import org.apache.doris.nereids.rules.exploration.mv.PreMaterializedViewRewriter.PreRewriteStrategy;
3635
import org.apache.doris.nereids.rules.expression.ExpressionRuleType;
3736
import org.apache.doris.planner.GroupCommitBlockSink;
3837
import org.apache.doris.qe.VariableMgr.VarAttr;
@@ -3301,20 +3300,6 @@ public void initFuzzyModeVariables() {
33013300
this.enableReserveMemory = randomInt % 5 != 0;
33023301
}
33033302

3304-
// random pre materialized view rewrite strategy
3305-
randomInt = random.nextInt(3);
3306-
switch (randomInt % 3) {
3307-
case 0:
3308-
this.preMaterializedViewRewriteStrategy = PreRewriteStrategy.NOT_IN_RBO.name();
3309-
break;
3310-
case 1:
3311-
this.preMaterializedViewRewriteStrategy = PreRewriteStrategy.TRY_IN_RBO.name();
3312-
break;
3313-
case 2:
3314-
default:
3315-
this.preMaterializedViewRewriteStrategy = PreRewriteStrategy.FORCE_IN_RBO.name();
3316-
break;
3317-
}
33183303
setFuzzyForCatalog(random);
33193304
}
33203305

regression-test/suites/mv_p0/ssb/multiple_no_where/multiple_no_where.groovy

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -107,6 +107,8 @@ suite ("multiple_no_where") {
107107
qt_select_star "select * from lineorder_flat order by 1,2, P_MFGR;"
108108

109109
sql """analyze table lineorder_flat with sync;"""
110+
sql """alter table lineorder_flat modify column C_CITY set stats ('row_count'='7');"""
111+
sql """alter table lineorder_flat modify column a3 set stats ('row_count'='1');"""
110112
sql """set enable_stats=false;"""
111113

112114
mv_rewrite_success("""SELECT SUM(LO_EXTENDEDPRICE * LO_DISCOUNT) AS revenue

regression-test/suites/mv_p0/ssb/multiple_ssb/multiple_ssb.groovy

Lines changed: 4 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -154,6 +154,10 @@ suite ("multiple_ssb") {
154154

155155
sql """analyze table lineorder_flat with sync;"""
156156
sql """alter table lineorder_flat modify column LO_ORDERDATE set stats ('row_count'='8');"""
157+
sql """alter table lineorder_flat modify column a1 set stats ('row_count'='1');"""
158+
sql """alter table lineorder_flat modify column a4 set stats ('row_count'='1');"""
159+
sql """alter table lineorder_flat modify column a6 set stats ('row_count'='1');"""
160+
sql """alter table lineorder_flat modify column x2 set stats ('row_count'='1');"""
157161
sql """set enable_stats=false;"""
158162

159163
mv_rewrite_success("""SELECT SUM(LO_EXTENDEDPRICE * LO_DISCOUNT) AS revenue

regression-test/suites/mv_p0/ssb/multiple_ssb_between/multiple_ssb_between.groovy

Lines changed: 5 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -153,7 +153,11 @@ suite ("multiple_ssb_between") {
153153
sql """set enable_stats=true;"""
154154

155155
sql """alter table lineorder_flat modify column LO_ORDERDATE set stats ('row_count'='8');"""
156-
156+
sql """alter table lineorder_flat modify column a1 set stats ('row_count'='1');"""
157+
sql """alter table lineorder_flat modify column a4 set stats ('row_count'='1');"""
158+
sql """alter table lineorder_flat modify column a6 set stats ('row_count'='1');"""
159+
sql """alter table lineorder_flat modify column x2 set stats ('row_count'='1');"""
160+
157161
mv_rewrite_success("""SELECT SUM(LO_EXTENDEDPRICE * LO_DISCOUNT) AS revenue
158162
FROM lineorder_flat
159163
WHERE

regression-test/suites/mv_p0/ssb/q_1_1/q_1_1.groovy

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -102,6 +102,7 @@ suite ("mv_ssb_q_1_1") {
102102

103103
sql "analyze table lineorder_flat with sync;"
104104
sql """alter table lineorder_flat modify column C_CITY set stats ('row_count'='6');"""
105+
sql """alter table lineorder_flat modify column a1 set stats ('row_count'='1');"""
105106
mv_rewrite_success("""SELECT SUM(LO_EXTENDEDPRICE * LO_DISCOUNT) AS revenue
106107
FROM lineorder_flat
107108
WHERE

regression-test/suites/mv_p0/ssb/q_2_1/q_2_1.groovy

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -94,6 +94,7 @@ suite ("mv_ssb_q_2_1") {
9494
qt_select_star "select * from lineorder_flat order by 1,2,P_MFGR;"
9595

9696
sql """alter table lineorder_flat modify column LO_ORDERDATE set stats ('row_count'='2');"""
97+
sql """alter table lineorder_flat modify column a2 set stats ('row_count'='1');"""
9798

9899
mv_rewrite_success("""SELECT
99100
SUM(LO_REVENUE), (LO_ORDERDATE DIV 10000) AS YEAR,

regression-test/suites/mv_p0/ssb/q_3_1/q_3_1.groovy

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -99,6 +99,7 @@ suite ("mv_ssb_q_3_1") {
9999
sql """analyze table lineorder_flat with sync;"""
100100

101101
sql """alter table lineorder_flat modify column LO_ORDERDATE set stats ('row_count'='2');"""
102+
sql """alter table lineorder_flat modify column a1 set stats ('row_count'='1');"""
102103

103104
mv_rewrite_success("""SELECT
104105
C_NATION,

regression-test/suites/mv_p0/ssb/q_4_1/q_4_1.groovy

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -93,6 +93,7 @@ suite ("mv_ssb_q_4_1") {
9393
sql """analyze table lineorder_flat with sync;"""
9494

9595
sql """alter table lineorder_flat modify column LO_ORDERDATE set stats ('row_count'='2');"""
96+
sql """alter table lineorder_flat modify column a2 set stats ('row_count'='1');"""
9697

9798
mv_rewrite_success("""SELECT (LO_ORDERDATE DIV 10000) AS YEAR,
9899
C_NATION,

regression-test/suites/mv_p0/ssb/q_4_1_r1/q_4_1_r1.groovy

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -96,6 +96,7 @@ suite ("q_4_1_r1") {
9696

9797
sql """analyze table lineorder_flat with sync;"""
9898
sql """alter table lineorder_flat modify column LO_ORDERDATE set stats ('row_count'='8');"""
99+
sql """alter table lineorder_flat modify column a1 set stats ('row_count'='1');"""
99100
sql """set enable_stats=false;"""
100101

101102
mv_rewrite_success("""SELECT (LO_ORDERDATE DIV 10000) AS YEAR,

0 commit comments

Comments
 (0)