Skip to content

Commit 69419bd

Browse files
seawindeseawinde
committed
opt mv rewrite performance 3
1 parent 21f4a43 commit 69419bd

17 files changed

+223
-171
lines changed

‎fe/fe-core/src/main/java/org/apache/doris/nereids/memo/Memo.java‎

Lines changed: 12 additions & 8 deletions
Original file line numberDiff line numberDiff line change
@@ -34,6 +34,7 @@
3434
import org.apache.doris.nereids.trees.plans.GroupPlan;
3535
import org.apache.doris.nereids.trees.plans.LeafPlan;
3636
import org.apache.doris.nereids.trees.plans.Plan;
37+
import org.apache.doris.nereids.trees.plans.TableId;
3738
import org.apache.doris.nereids.trees.plans.algebra.CatalogRelation;
3839
import org.apache.doris.nereids.trees.plans.algebra.SetOperation;
3940
import org.apache.doris.nereids.trees.plans.logical.LogicalCatalogRelation;
@@ -75,8 +76,8 @@ public class Memo {
7576
EventChannel.getDefaultChannel().addConsumers(new LogConsumer(GroupMergeEvent.class, EventChannel.LOG)));
7677
private static long stateId = 0;
7778
private final ConnectContext connectContext;
78-
// The key is the query relationId, the value is the refresh version when last refresh, this is needed
79-
// because struct info refresh base on target relationId.
79+
// The key is the query tableId, the value is the refresh version when last refresh, this is needed
80+
// because struct info refresh base on target tableId.
8081
private final Map<Integer, AtomicInteger> refreshVersion = new HashMap<>();
8182
private final Map<Class<? extends AbstractMaterializedViewRule>, Set<Long>> materializationCheckSuccessMap =
8283
new LinkedHashMap<>();
@@ -136,9 +137,9 @@ public Map<Integer, AtomicInteger> getRefreshVersion() {
136137
return refreshVersion;
137138
}
138139

139-
/** return the incremented refresh version for the given relationId*/
140-
public long incrementAndGetRefreshVersion(int relationId) {
141-
return refreshVersion.compute(relationId, (k, v) -> {
140+
/** return the incremented refresh version for the given commonTableId*/
141+
public long incrementAndGetRefreshVersion(int commonTableId) {
142+
return refreshVersion.compute(commonTableId, (k, v) -> {
142143
if (v == null) {
143144
return new AtomicInteger(1);
144145
}
@@ -148,8 +149,9 @@ public long incrementAndGetRefreshVersion(int relationId) {
148149
}
149150

150151
/** return the incremented refresh version for the given relationId set*/
151-
public void incrementAndGetRefreshVersion(BitSet relationIdSet) {
152-
for (int i = relationIdSet.nextSetBit(0); i >= 0; i = relationIdSet.nextSetBit(i + 1)) {
152+
public void incrementAndGetRefreshVersion(BitSet commonTableIdSet) {
153+
for (int i = commonTableIdSet.nextSetBit(0); i >= 0;
154+
i = commonTableIdSet.nextSetBit(i + 1)) {
153155
incrementAndGetRefreshVersion(i);
154156
}
155157
}
@@ -484,7 +486,9 @@ private CopyInResult doCopyIn(Plan plan, @Nullable Group targetGroup, @Nullable
484486
&& plan instanceof LogicalCatalogRelation
485487
&& ((CatalogRelation) plan).getTable() instanceof MTMV
486488
&& !plan.getGroupExpression().isPresent()) {
487-
incrementAndGetRefreshVersion(((CatalogRelation) plan).getRelationId().asInt());
489+
TableId mvCommonTableId
490+
= this.connectContext.getStatementContext().getTableId(((CatalogRelation) plan).getTable());
491+
incrementAndGetRefreshVersion(mvCommonTableId.asInt());
488492
}
489493
Optional<GroupExpression> groupExpr = plan.getGroupExpression();
490494
if (groupExpr.isPresent()) {

‎fe/fe-core/src/main/java/org/apache/doris/nereids/memo/StructInfoMap.java‎

Lines changed: 122 additions & 54 deletions
Large diffs are not rendered by default.

‎fe/fe-core/src/main/java/org/apache/doris/nereids/rules/exploration/mv/AbstractMaterializedViewAggregateRule.java‎

Lines changed: 7 additions & 17 deletions
Original file line numberDiff line numberDiff line change
@@ -59,7 +59,6 @@
5959
import com.google.common.collect.Sets;
6060

6161
import java.util.ArrayList;
62-
import java.util.BitSet;
6362
import java.util.Collections;
6463
import java.util.HashMap;
6564
import java.util.HashSet;
@@ -124,7 +123,6 @@ protected Plan rewriteQueryByView(MatchMode matchMode,
124123
queryTopPlan,
125124
materializationContext.getShuttledExprToScanExprMapping(),
126125
viewToQuerySlotMapping,
127-
queryStructInfo.getTableBitSet(),
128126
ImmutableMap.of(), cascadesContext);
129127
boolean isRewrittenQueryExpressionValid = true;
130128
if (!rewrittenQueryExpressions.isEmpty()) {
@@ -314,11 +312,10 @@ protected Expression tryRewriteExpression(StructInfo queryStructInfo, Expression
314312
MaterializationContext materializationContext, String summaryIfFail, Supplier<String> detailIfFail) {
315313
Expression queryFunctionShuttled = ExpressionUtils.shuttleExpressionWithLineage(
316314
queryExpression,
317-
queryStructInfo.getTopPlan(),
318-
queryStructInfo.getTableBitSet());
315+
queryStructInfo.getTopPlan());
319316
AggregateExpressionRewriteContext expressionRewriteContext = new AggregateExpressionRewriteContext(
320317
rewriteMode, mvShuttledExprToMvScanExprQueryBased, queryStructInfo.getTopPlan(),
321-
queryStructInfo.getTableBitSet(), queryStructInfo.getGroupingId());
318+
queryStructInfo.getGroupingId());
322319
Expression rewrittenExpression = queryFunctionShuttled.accept(AGGREGATE_EXPRESSION_REWRITER,
323320
expressionRewriteContext);
324321
if (!expressionRewriteContext.isValid()) {
@@ -358,7 +355,7 @@ protected boolean canUnionRewrite(Plan queryPlan, AsyncMaterializationContext co
358355
boolean canUnionRewrite = false;
359356
// Check the query plan group by expression contains partition col or not
360357
List<? extends Expression> groupByShuttledExpressions =
361-
ExpressionUtils.shuttleExpressionWithLineage(groupByExpressions, queryPlan, new BitSet());
358+
ExpressionUtils.shuttleExpressionWithLineage(groupByExpressions, queryPlan);
362359
for (Expression expression : groupByShuttledExpressions) {
363360
canUnionRewrite = !expression.collectToSet(expr -> {
364361
if (!(expr instanceof SlotReference) || !((SlotReference) expr).isColumnFromTable()) {
@@ -431,13 +428,13 @@ private boolean isGroupByEquals(Pair<Plan, LogicalAggregate<Plan>> queryTopPlanA
431428
LogicalAggregate<Plan> viewAggregate = viewTopPlanAndAggPair.value();
432429

433430
Set<Expression> queryGroupByShuttledExpression = new HashSet<>(ExpressionUtils.shuttleExpressionWithLineage(
434-
queryAggregate.getGroupByExpressions(), queryTopPlan, queryStructInfo.getTableBitSet()));
431+
queryAggregate.getGroupByExpressions(), queryTopPlan));
435432

436433
// try to eliminate group by dimension by function dependency if group by expression is not in query
437434
Map<Expression, Expression> viewShuttledExpressionQueryBasedToGroupByExpressionMap = new HashMap<>();
438435
List<Expression> viewGroupByExpressions = viewAggregate.getGroupByExpressions();
439436
List<? extends Expression> viewGroupByShuttledExpressions = ExpressionUtils.shuttleExpressionWithLineage(
440-
viewGroupByExpressions, viewTopPlan, viewStructInfo.getTableBitSet());
437+
viewGroupByExpressions, viewTopPlan);
441438

442439
for (int index = 0; index < viewGroupByExpressions.size(); index++) {
443440
Expression viewExpression = viewGroupByExpressions.get(index);
@@ -712,8 +709,7 @@ public Expression visitAggregateFunction(AggregateFunction aggregateFunction,
712709
if (ExpressionRewriteMode.EXPRESSION_ROLL_UP.equals(rewriteContext.getExpressionRewriteMode())) {
713710
Expression queryFunctionShuttled = ExpressionUtils.shuttleExpressionWithLineage(
714711
aggregateFunction,
715-
rewriteContext.getQueryTopPlan(),
716-
rewriteContext.getQueryTableBitSet());
712+
rewriteContext.getQueryTopPlan());
717713
rewrittenFunction = rollup(aggregateFunction, queryFunctionShuttled,
718714
rewriteContext.getMvExprToMvScanExprQueryBasedMapping());
719715
if (rewrittenFunction == null) {
@@ -788,16 +784,14 @@ public static class AggregateExpressionRewriteContext {
788784
private final ExpressionRewriteMode expressionRewriteMode;
789785
private final Map<Expression, Expression> mvExprToMvScanExprQueryBasedMapping;
790786
private final Plan queryTopPlan;
791-
private final BitSet queryTableBitSet;
792787
private final Optional<SlotReference> groupingId;
793788

794789
public AggregateExpressionRewriteContext(ExpressionRewriteMode expressionRewriteMode,
795790
Map<Expression, Expression> mvExprToMvScanExprQueryBasedMapping, Plan queryTopPlan,
796-
BitSet queryTableBitSet, Optional<SlotReference> groupingId) {
791+
Optional<SlotReference> groupingId) {
797792
this.expressionRewriteMode = expressionRewriteMode;
798793
this.mvExprToMvScanExprQueryBasedMapping = mvExprToMvScanExprQueryBasedMapping;
799794
this.queryTopPlan = queryTopPlan;
800-
this.queryTableBitSet = queryTableBitSet;
801795
this.groupingId = groupingId;
802796
}
803797

@@ -821,10 +815,6 @@ public Plan getQueryTopPlan() {
821815
return queryTopPlan;
822816
}
823817

824-
public BitSet getQueryTableBitSet() {
825-
return queryTableBitSet;
826-
}
827-
828818
public Optional<SlotReference> getGroupingId() {
829819
return groupingId;
830820
}

‎fe/fe-core/src/main/java/org/apache/doris/nereids/rules/exploration/mv/AbstractMaterializedViewJoinRule.java‎

Lines changed: 0 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -51,7 +51,6 @@ protected Plan rewriteQueryByView(MatchMode matchMode,
5151
queryStructInfo.getTopPlan(),
5252
materializationContext.getShuttledExprToScanExprMapping(),
5353
targetToSourceMapping,
54-
queryStructInfo.getTableBitSet(),
5554
ImmutableMap.of(), cascadesContext
5655
);
5756
// Can not rewrite, bail out

‎fe/fe-core/src/main/java/org/apache/doris/nereids/rules/exploration/mv/AbstractMaterializedViewRule.java‎

Lines changed: 12 additions & 16 deletions
Original file line numberDiff line numberDiff line change
@@ -79,7 +79,6 @@
7979
import org.apache.logging.log4j.Logger;
8080

8181
import java.util.ArrayList;
82-
import java.util.BitSet;
8382
import java.util.Collection;
8483
import java.util.HashMap;
8584
import java.util.HashSet;
@@ -188,16 +187,15 @@ protected List<StructInfo> getValidQueryStructInfos(Plan queryPlan, CascadesCont
188187
MaterializationContext materializationContext) {
189188
List<StructInfo> validStructInfos = new ArrayList<>();
190189
// For every materialized view we should trigger refreshing struct info map
191-
List<StructInfo> uncheckedQueryStructInfos = MaterializedViewUtils.extractStructInfo(queryPlan, queryPlan,
192-
cascadesContext,
193-
materializationContext.getQueryRelationIdSetMvMatched(cascadesContext.getStatementContext()));
190+
List<StructInfo> uncheckedQueryStructInfos = MaterializedViewUtils.extractStructInfoFuzzy(queryPlan, queryPlan,
191+
cascadesContext, materializationContext.getCommonTableIdSet(cascadesContext.getStatementContext()));
194192
uncheckedQueryStructInfos.forEach(queryStructInfo -> {
195193
boolean valid = checkQueryPattern(queryStructInfo, cascadesContext) && queryStructInfo.isValid();
196194
if (!valid) {
197195
cascadesContext.getMaterializationContexts().forEach(ctx ->
198196
ctx.recordFailReason(queryStructInfo, "Query struct info is invalid",
199197
() -> String.format("query table bitmap is %s, plan is %s",
200-
queryStructInfo.getTableBitSet(), queryPlan.treeString())
198+
queryStructInfo.getRelations(), queryPlan.treeString())
201199
));
202200
} else {
203201
validStructInfos.add(queryStructInfo);
@@ -278,9 +276,7 @@ protected List<Plan> doRewrite(StructInfo queryStructInfo, CascadesContext casca
278276
// Try to rewrite compensate predicates by using mv scan
279277
List<Expression> rewriteCompensatePredicates = rewriteExpression(compensatePredicates.toList(),
280278
queryPlan, materializationContext.getShuttledExprToScanExprMapping(),
281-
viewToQuerySlotMapping, queryStructInfo.getTableBitSet(),
282-
compensatePredicates.getRangePredicateMap(),
283-
cascadesContext);
279+
viewToQuerySlotMapping, compensatePredicates.getRangePredicateMap(), cascadesContext);
284280
if (rewriteCompensatePredicates.isEmpty()) {
285281
materializationContext.recordFailReason(queryStructInfo,
286282
"Rewrite compensate predicate by view fail",
@@ -321,7 +317,7 @@ protected List<Plan> doRewrite(StructInfo queryStructInfo, CascadesContext casca
321317
&& sessionVariable.isEnableMaterializedViewUnionRewrite()) {
322318
MTMV mtmv = ((AsyncMaterializationContext) materializationContext).getMtmv();
323319
Map<List<String>, Set<String>> queryUsedPartitions = PartitionCompensator.getQueryUsedPartitions(
324-
cascadesContext.getStatementContext(), queryStructInfo.getTableBitSet());
320+
cascadesContext.getStatementContext(), queryStructInfo.getRelationBitSet());
325321
Set<MTMVRelatedTableIf> pctTables = mtmv.getMvPartitionInfo().getPctTables();
326322
boolean relateTableUsedPartitionsAnyNull = false;
327323
boolean relateTableUsedPartitionsAllEmpty = true;
@@ -588,14 +584,14 @@ protected Plan rewriteQueryByView(MatchMode matchMode, StructInfo queryStructInf
588584
* then use the corresponding value of mapping to replace it
589585
*/
590586
protected List<Expression> rewriteExpression(List<? extends Expression> sourceExpressionsToWrite, Plan sourcePlan,
591-
ExpressionMapping targetExpressionMapping, SlotMapping targetToSourceMapping, BitSet sourcePlanBitSet,
587+
ExpressionMapping targetExpressionMapping, SlotMapping targetToSourceMapping,
592588
Map<Expression, ExpressionInfo> queryExprToInfoMap, CascadesContext cascadesContext) {
593589
// Firstly, rewrite the target expression using source with inverse mapping
594590
// then try to use the target expression to represent the query. if any of source expressions
595591
// could not be represented by target expressions, return null.
596592
// generate target to target replacement expression mapping, and change target expression to source based
597593
List<? extends Expression> sourceShuttledExpressions = ExpressionUtils.shuttleExpressionWithLineage(
598-
sourceExpressionsToWrite, sourcePlan, sourcePlanBitSet);
594+
sourceExpressionsToWrite, sourcePlan);
599595
ExpressionMapping expressionMappingKeySourceBased = targetExpressionMapping.keyPermute(targetToSourceMapping);
600596
// target to target replacement expression mapping, because mv is 1:1 so get the first element
601597
List<Map<Expression, Expression>> flattenExpressionMap = expressionMappingKeySourceBased.flattenMap();
@@ -917,15 +913,15 @@ private boolean containsNullRejectSlot(Set<Set<Slot>> requireNoNullableViewSlot,
917913
}
918914
// query slot need shuttle to use table slot, avoid alias influence
919915
Set<Expression> queryUsedNeedRejectNullSlotsViewBased = ExpressionUtils.shuttleExpressionWithLineage(
920-
new ArrayList<>(queryNullRejectSlotSet), queryStructInfo.getTopPlan(), new BitSet()).stream()
916+
new ArrayList<>(queryNullRejectSlotSet), queryStructInfo.getTopPlan()).stream()
921917
.map(expr -> ExpressionUtils.replace(expr, queryToViewMapping.toSlotReferenceMap()))
922918
.collect(Collectors.toSet());
923919
// view slot need shuttle to use table slot, avoid alias influence
924920
Set<Set<Slot>> shuttledRequireNoNullableViewSlot = new HashSet<>();
925921
for (Set<Slot> requireNullableSlots : requireNoNullableViewSlot) {
926922
shuttledRequireNoNullableViewSlot.add(
927923
ExpressionUtils.shuttleExpressionWithLineage(new ArrayList<>(requireNullableSlots),
928-
viewStructInfo.getTopPlan(), new BitSet()).stream().map(Slot.class::cast)
924+
viewStructInfo.getTopPlan()).stream().map(Slot.class::cast)
929925
.collect(Collectors.toSet()));
930926
}
931927
// query pulledUp predicates should have null reject predicates and contains any require noNullable slot
@@ -1101,7 +1097,7 @@ protected Plan tryRewriteTopN(LogicalTopN<Plan> queryTopNode, LogicalTopN<Plan>
11011097
List<Expression> queryOrderKeysExpressions = queryOrderKeys.stream()
11021098
.map(OrderKey::getExpr).collect(Collectors.toList());
11031099
List<? extends Expression> queryOrderByExpressionsShuttled = ExpressionUtils.shuttleExpressionWithLineage(
1104-
queryOrderKeysExpressions, queryStructInfo.getTopPlan(), queryStructInfo.getTableBitSet());
1100+
queryOrderKeysExpressions, queryStructInfo.getTopPlan());
11051101

11061102
List<OrderKey> queryShuttledOrderKeys = new ArrayList<>();
11071103
for (int i = 0; i < queryOrderKeys.size(); i++) {
@@ -1112,7 +1108,7 @@ protected Plan tryRewriteTopN(LogicalTopN<Plan> queryTopNode, LogicalTopN<Plan>
11121108
List<OrderKey> viewShuttledOrderKeys = new ArrayList<>();
11131109
List<? extends Expression> viewOrderByExpressionsShuttled = ExpressionUtils.shuttleExpressionWithLineage(
11141110
viewOrderKeys.stream().map(OrderKey::getExpr).collect(Collectors.toList()),
1115-
viewStructInfo.getTopPlan(), new BitSet());
1111+
viewStructInfo.getTopPlan());
11161112
List<Expression> viewOrderByExpressionsQueryBasedSet = ExpressionUtils.replace(
11171113
viewOrderByExpressionsShuttled.stream().map(Expression.class::cast).collect(Collectors.toList()),
11181114
viewToQuerySlotMapping.toSlotReferenceMap());
@@ -1132,7 +1128,7 @@ protected Plan tryRewriteTopN(LogicalTopN<Plan> queryTopNode, LogicalTopN<Plan>
11321128
// try to rewrite the order by expressions using the mv scan slot
11331129
List<Expression> rewrittenExpressions = rewriteExpression(queryOrderKeysExpressions,
11341130
queryStructInfo.getTopPlan(), materializationContext.shuttledExprToScanExprMapping,
1135-
viewToQuerySlotMapping, queryStructInfo.getTableBitSet(), ImmutableMap.of(), cascadesContext);
1131+
viewToQuerySlotMapping, ImmutableMap.of(), cascadesContext);
11361132
if (rewrittenExpressions.isEmpty()) {
11371133
materializationContext.recordFailReason(queryStructInfo,
11381134
"query topN order keys rewrite fail, query topN order keys is not consistent "

‎fe/fe-core/src/main/java/org/apache/doris/nereids/rules/exploration/mv/AbstractMaterializedViewScanRule.java‎

Lines changed: 0 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -51,7 +51,6 @@ protected Plan rewriteQueryByView(MatchMode matchMode,
5151
queryStructInfo.getTopPlan(),
5252
materializationContext.getShuttledExprToScanExprMapping(),
5353
targetToSourceMapping,
54-
queryStructInfo.getTableBitSet(),
5554
ImmutableMap.of(), cascadesContext
5655
);
5756
// Can not rewrite, bail out

‎fe/fe-core/src/main/java/org/apache/doris/nereids/rules/exploration/mv/AbstractMaterializedViewWindowRule.java‎

Lines changed: 0 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -102,7 +102,6 @@ protected Plan rewriteQueryByView(MatchMode matchMode, StructInfo queryStructInf
102102
queryStructInfo.getTopPlan(),
103103
materializationContext.getShuttledExprToScanExprMapping(),
104104
viewToQuerySlotMapping,
105-
queryStructInfo.getTableBitSet(),
106105
ImmutableMap.of(), cascadesContext
107106
);
108107
// Can not rewrite, bail out

0 commit comments

Comments
 (0)