diff --git a/fe/fe-core/src/main/antlr4/org/apache/doris/nereids/DorisLexer.g4 b/fe/fe-core/src/main/antlr4/org/apache/doris/nereids/DorisLexer.g4 index 33282dc4a6b624..132ef2092ebfe0 100644 --- a/fe/fe-core/src/main/antlr4/org/apache/doris/nereids/DorisLexer.g4 +++ b/fe/fe-core/src/main/antlr4/org/apache/doris/nereids/DorisLexer.g4 @@ -83,6 +83,7 @@ APPEND: 'APPEND'; ARRAY: 'ARRAY'; AS: 'AS'; ASC: 'ASC'; +ASOF: 'ASOF'; AT: 'AT'; AUTHORS: 'AUTHORS'; AUTO: 'AUTO'; @@ -342,6 +343,7 @@ MATCH: 'MATCH'; MATCHED: 'MATCHED'; MATCH_ALL: 'MATCH_ALL'; MATCH_ANY: 'MATCH_ANY'; +MATCH_CONDITION: 'MATCH_CONDITION'; MATCH_NAME: 'MATCH_NAME'; MATCH_NAME_GLOB: 'MATCH_NAME_GLOB'; MATCH_PHRASE: 'MATCH_PHRASE'; diff --git a/fe/fe-core/src/main/antlr4/org/apache/doris/nereids/DorisParser.g4 b/fe/fe-core/src/main/antlr4/org/apache/doris/nereids/DorisParser.g4 index d916626420678f..0036b7f0ee734b 100644 --- a/fe/fe-core/src/main/antlr4/org/apache/doris/nereids/DorisParser.g4 +++ b/fe/fe-core/src/main/antlr4/org/apache/doris/nereids/DorisParser.g4 @@ -1277,7 +1277,11 @@ relation ; joinRelation - : (joinType) JOIN distributeType? right=relationPrimary joinCriteria? + : (joinType) JOIN distributeType? right=relationPrimary matchCondition? joinCriteria? + ; + +matchCondition + : MATCH_CONDITION LEFT_PAREN valueExpression RIGHT_PAREN ; // Just like `opt_plan_hints` in legacy CUP parser. @@ -1388,6 +1392,8 @@ joinType | RIGHT SEMI | LEFT ANTI | RIGHT ANTI + | ASOF LEFT? + | ASOF INNER ; joinCriteria diff --git a/fe/fe-core/src/main/java/org/apache/doris/analysis/JoinOperator.java b/fe/fe-core/src/main/java/org/apache/doris/analysis/JoinOperator.java index bd8798bd4d466d..c8bfb9b93e4901 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/analysis/JoinOperator.java +++ b/fe/fe-core/src/main/java/org/apache/doris/analysis/JoinOperator.java @@ -39,7 +39,11 @@ public enum JoinOperator { NULL_AWARE_LEFT_ANTI_JOIN("NULL AWARE LEFT ANTI JOIN", TJoinOp.NULL_AWARE_LEFT_ANTI_JOIN), NULL_AWARE_LEFT_SEMI_JOIN("NULL AWARE LEFT SEMI JOIN", - TJoinOp.NULL_AWARE_LEFT_SEMI_JOIN); + TJoinOp.NULL_AWARE_LEFT_SEMI_JOIN), + ASOF_LEFT_INNER_JOIN("ASOF_LEFT_INNER_JOIN", TJoinOp.ASOF_LEFT_INNER_JOIN), + ASOF_RIGHT_INNER_JOIN("ASOF_RIGHT_INNER_JOIN", TJoinOp.ASOF_RIGHT_INNER_JOIN), + ASOF_LEFT_OUTER_JOIN("ASOF_LEFT_OUTER_JOIN", TJoinOp.ASOF_LEFT_OUTER_JOIN), + ASOF_RIGHT_OUTER_JOIN("ASOF_LEFT_INNER_JOIN", TJoinOp.ASOF_RIGHT_OUTER_JOIN); private final String description; private final TJoinOp thriftJoinOp; @@ -57,66 +61,4 @@ public String toString() { public TJoinOp toThrift() { return thriftJoinOp; } - - public boolean isOuterJoin() { - return this == LEFT_OUTER_JOIN || this == RIGHT_OUTER_JOIN || this == FULL_OUTER_JOIN; - } - - public boolean isSemiAntiJoin() { - return this == LEFT_SEMI_JOIN || this == RIGHT_SEMI_JOIN || this == LEFT_ANTI_JOIN - || this == NULL_AWARE_LEFT_ANTI_JOIN || this == RIGHT_ANTI_JOIN; - } - - public boolean isSemiJoin() { - return this == JoinOperator.LEFT_SEMI_JOIN || this == JoinOperator.LEFT_ANTI_JOIN - || this == JoinOperator.RIGHT_SEMI_JOIN || this == JoinOperator.RIGHT_ANTI_JOIN - || this == JoinOperator.NULL_AWARE_LEFT_ANTI_JOIN; - } - - public boolean isSemiOrAntiJoinNoNullAware() { - return this == JoinOperator.LEFT_SEMI_JOIN || this == JoinOperator.LEFT_ANTI_JOIN - || this == JoinOperator.RIGHT_SEMI_JOIN || this == JoinOperator.RIGHT_ANTI_JOIN; - } - - public boolean isAntiJoinNullAware() { - return this == JoinOperator.NULL_AWARE_LEFT_ANTI_JOIN; - } - - public boolean isAntiJoinNoNullAware() { - return this == JoinOperator.LEFT_ANTI_JOIN || this == JoinOperator.RIGHT_ANTI_JOIN; - } - - public boolean isLeftSemiJoin() { - return this.thriftJoinOp == TJoinOp.LEFT_SEMI_JOIN; - } - - public boolean isInnerJoin() { - return this.thriftJoinOp == TJoinOp.INNER_JOIN; - } - - public boolean isAntiJoin() { - return this == JoinOperator.LEFT_ANTI_JOIN || this == JoinOperator.RIGHT_ANTI_JOIN - || this == JoinOperator.NULL_AWARE_LEFT_ANTI_JOIN; - } - - public boolean supportMarkJoin() { - return this == JoinOperator.LEFT_ANTI_JOIN || this == JoinOperator.LEFT_SEMI_JOIN - || this == JoinOperator.CROSS_JOIN || this == JoinOperator.NULL_AWARE_LEFT_ANTI_JOIN; - } - - public boolean isCrossJoin() { - return this == CROSS_JOIN; - } - - public boolean isFullOuterJoin() { - return this == FULL_OUTER_JOIN; - } - - public boolean isLeftOuterJoin() { - return this == LEFT_OUTER_JOIN; - } - - public boolean isRightOuterJoin() { - return this == RIGHT_OUTER_JOIN; - } } diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/glue/translator/PhysicalPlanTranslator.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/glue/translator/PhysicalPlanTranslator.java index 0181a91979314d..0f84cccd79c70d 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/glue/translator/PhysicalPlanTranslator.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/glue/translator/PhysicalPlanTranslator.java @@ -1705,7 +1705,8 @@ public PlanFragment visitPhysicalHashJoin( } // set slots as nullable for outer join - if (joinType == JoinType.LEFT_OUTER_JOIN || joinType == JoinType.FULL_OUTER_JOIN) { + if (joinType == JoinType.LEFT_OUTER_JOIN || joinType == JoinType.ASOF_LEFT_OUTER_JOIN + || joinType == JoinType.FULL_OUTER_JOIN) { for (SlotDescriptor sd : rightIntermediateSlotDescriptor) { sd.setIsNullable(true); SlotRef slotRef = new SlotRef(sd); @@ -1713,7 +1714,8 @@ public PlanFragment visitPhysicalHashJoin( context.addExprIdSlotRefPair(exprId, slotRef); } } - if (joinType == JoinType.RIGHT_OUTER_JOIN || joinType == JoinType.FULL_OUTER_JOIN) { + if (joinType == JoinType.RIGHT_OUTER_JOIN || joinType == JoinType.ASOF_RIGHT_OUTER_JOIN + || joinType == JoinType.FULL_OUTER_JOIN) { for (SlotDescriptor sd : leftIntermediateSlotDescriptor) { sd.setIsNullable(true); SlotRef slotRef = new SlotRef(sd); @@ -1878,7 +1880,8 @@ public PlanFragment visitPhysicalNestedLoopJoin( } // set slots as nullable for outer join - if (joinType == JoinType.LEFT_OUTER_JOIN || joinType == JoinType.FULL_OUTER_JOIN) { + if (joinType == JoinType.LEFT_OUTER_JOIN || joinType == JoinType.ASOF_LEFT_OUTER_JOIN + || joinType == JoinType.FULL_OUTER_JOIN) { for (SlotDescriptor sd : rightIntermediateSlotDescriptor) { sd.setIsNullable(true); SlotRef slotRef = new SlotRef(sd); @@ -1886,7 +1889,8 @@ public PlanFragment visitPhysicalNestedLoopJoin( context.addExprIdSlotRefPair(exprId, slotRef); } } - if (joinType == JoinType.RIGHT_OUTER_JOIN || joinType == JoinType.FULL_OUTER_JOIN) { + if (joinType == JoinType.RIGHT_OUTER_JOIN || joinType == JoinType.ASOF_RIGHT_OUTER_JOIN + || joinType == JoinType.FULL_OUTER_JOIN) { for (SlotDescriptor sd : leftIntermediateSlotDescriptor) { sd.setIsNullable(true); SlotRef slotRef = new SlotRef(sd); diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/parser/LogicalPlanBuilder.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/parser/LogicalPlanBuilder.java index fe02442ee2aba0..4a3db990a3d362 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/parser/LogicalPlanBuilder.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/parser/LogicalPlanBuilder.java @@ -4304,6 +4304,7 @@ protected LogicalPlan withSelectQuerySpecification( private LogicalPlan withJoinRelations(LogicalPlan input, RelationContext ctx) { LogicalPlan last = input; for (JoinRelationContext join : ctx.joinRelation()) { + boolean isAsofJoin = false; JoinType joinType; if (join.joinType().CROSS() != null) { joinType = JoinType.CROSS_JOIN; @@ -4321,6 +4322,10 @@ private LogicalPlan withJoinRelations(LogicalPlan input, RelationContext ctx) { } else { joinType = JoinType.RIGHT_ANTI_JOIN; } + } else if (join.joinType().ASOF() != null) { + joinType = join.joinType().INNER() != null + ? JoinType.ASOF_LEFT_INNER_JOIN : JoinType.ASOF_LEFT_OUTER_JOIN; + isAsofJoin = true; } else if (join.joinType().LEFT() != null) { joinType = JoinType.LEFT_OUTER_JOIN; } else if (join.joinType().RIGHT() != null) { @@ -4332,6 +4337,23 @@ private LogicalPlan withJoinRelations(LogicalPlan input, RelationContext ctx) { } else { joinType = JoinType.CROSS_JOIN; } + Expression matchCondition = null; + if (join.matchCondition() != null) { + if (!isAsofJoin) { + throw new ParseException("only ASOF JOIN support MATCH_CONDITION", join); + } + matchCondition = typedVisit(join.matchCondition().valueExpression()); + if (!(matchCondition instanceof LessThan + || matchCondition instanceof LessThanEqual + || matchCondition instanceof GreaterThan + || matchCondition instanceof GreaterThanEqual)) { + throw new ParseException("ASOF JOIN's MATCH_CONDITION must be <, <=, >, >=", join); + } + } else { + if (isAsofJoin) { + throw new ParseException("ASOF JOIN must specify MATCH_CONDITION", join); + } + } DistributeHint distributeHint = new DistributeHint(DistributeType.NONE); if (join.distributeType() != null) { distributeHint = visitDistributeType(join.distributeType()); @@ -4352,21 +4374,48 @@ private LogicalPlan withJoinRelations(LogicalPlan input, RelationContext ctx) { .collect(ImmutableList.toImmutableList()); } } else { + if (isAsofJoin) { + throw new ParseException("ASOF JOIN must have on or using clause", join); + } // keep same with original planner, allow cross/inner join if (!joinType.isInnerOrCrossJoin()) { throw new ParseException("on mustn't be empty except for cross/inner join", join); } } + if (ids == null) { - last = new LogicalJoin<>(joinType, ExpressionUtils.EMPTY_CONDITION, - condition.map(ExpressionUtils::extractConjunction) - .orElse(ExpressionUtils.EMPTY_CONDITION), - distributeHint, - Optional.empty(), - last, - plan(join.relationPrimary()), null); + if (isAsofJoin) { + if (!condition.isPresent()) { + throw new ParseException("ASOF JOIN can't be used without ON clause", join); + } + List conjuncts = ExpressionUtils.extractConjunction(condition.get()); + for (Expression expression : conjuncts) { + if (!(expression instanceof EqualTo)) { + throw new ParseException("ASOF JOIN's ON clause must be one or more EQUAL(=) conjuncts", + join); + } + if (expression.child(0).isConstant() || expression.child(1).isConstant()) { + throw new ParseException("ASOF JOIN's EQUAL conjunct's children must not be constant"); + } + } + last = new LogicalJoin<>(joinType, conjuncts, + ImmutableList.of(matchCondition), + distributeHint, + Optional.empty(), + last, + plan(join.relationPrimary()), null); + } else { + last = new LogicalJoin<>(joinType, ExpressionUtils.EMPTY_CONDITION, + condition.map(ExpressionUtils::extractConjunction) + .orElse(ExpressionUtils.EMPTY_CONDITION), + distributeHint, + Optional.empty(), + last, + plan(join.relationPrimary()), null); + } } else { - last = new LogicalUsingJoin<>(joinType, last, plan(join.relationPrimary()), ids, distributeHint); + last = new LogicalUsingJoin<>(joinType, last, plan(join.relationPrimary()), ids, + matchCondition != null ? Optional.of(matchCondition) : Optional.empty(), distributeHint); } if (distributeHint.distributeType != DistributeType.NONE diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/processor/post/RuntimeFilterGenerator.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/processor/post/RuntimeFilterGenerator.java index f3dd019c462908..d032fc6eea1fba 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/processor/post/RuntimeFilterGenerator.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/processor/post/RuntimeFilterGenerator.java @@ -86,6 +86,7 @@ public class RuntimeFilterGenerator extends PlanPostProcessor { JoinType.LEFT_ANTI_JOIN, JoinType.FULL_OUTER_JOIN, JoinType.LEFT_OUTER_JOIN, + JoinType.ASOF_LEFT_OUTER_JOIN, JoinType.NULL_AWARE_LEFT_ANTI_JOIN ); diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/processor/post/RuntimeFilterPushDownVisitor.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/processor/post/RuntimeFilterPushDownVisitor.java index 512f248aac9d9c..fdad35c2b7f1a6 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/processor/post/RuntimeFilterPushDownVisitor.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/processor/post/RuntimeFilterPushDownVisitor.java @@ -29,7 +29,6 @@ import org.apache.doris.nereids.trees.expressions.Slot; import org.apache.doris.nereids.trees.expressions.SlotReference; import org.apache.doris.nereids.trees.expressions.visitor.ExpressionVisitors; -import org.apache.doris.nereids.trees.plans.JoinType; import org.apache.doris.nereids.trees.plans.Plan; import org.apache.doris.nereids.trees.plans.physical.AbstractPhysicalJoin; import org.apache.doris.nereids.trees.plans.physical.AbstractPhysicalPlan; @@ -280,7 +279,7 @@ public Boolean visitPhysicalHashJoin(PhysicalHashJoin bindJoin(MatchingContext otherJoinConjuncts.add(conjunct); } } + List otherConjuncts = otherJoinConjuncts.build(); + if (join.getJoinType().isAsofJoin()) { + // validate match_condition's data type + boolean isValid = false; + if (otherConjuncts.size() == 1) { + Expression conjunct = otherConjuncts.get(0); + if (conjunct instanceof ComparisonPredicate) { + if (conjunct.child(0).getDataType().isDateLikeType() + && conjunct.child(1).getDataType().isDateLikeType()) { + isValid = true; + } else { + throw new AnalysisException("only allow date, datetime and timestamptz in MATCH_CONDITION"); + } + } + } + if (!isValid) { + // other unexpected error + throw new AnalysisException(String.format("MATCH_CONDITION is invalid %s", + Joiner.on(",").join(otherConjuncts))); + } + } return new LogicalJoin<>(join.getJoinType(), - hashJoinConjuncts.build(), otherJoinConjuncts.build(), + hashJoinConjuncts.build(), otherConjuncts, join.getDistributeHint(), join.getMarkJoinSlotReference(), join.getExceptAsteriskOutputs(), join.children(), null); } @@ -762,11 +785,19 @@ private LogicalPlan bindUsingJoin(MatchingContext> hashEqExprs.add(new EqualTo(usingLeftSlot, usingRightSlot)); } - return new LogicalJoin<>( - using.getJoinType() == JoinType.CROSS_JOIN ? JoinType.INNER_JOIN : using.getJoinType(), - hashEqExprs.build(), ImmutableList.of(), - using.getDistributeHint(), Optional.empty(), rightConjunctsSlots, - using.children(), null); + if (using.getJoinType().isAsofJoin()) { + return new LogicalJoin<>( + using.getJoinType(), + hashEqExprs.build(), ImmutableList.of(using.getMatchCondition().get()), + using.getDistributeHint(), Optional.empty(), rightConjunctsSlots, + using.children(), null); + } else { + return new LogicalJoin<>( + using.getJoinType() == JoinType.CROSS_JOIN ? JoinType.INNER_JOIN : using.getJoinType(), + hashEqExprs.build(), ImmutableList.of(), + using.getDistributeHint(), Optional.empty(), rightConjunctsSlots, + using.children(), null); + } } private Plan bindProject(MatchingContext> ctx) { diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/exploration/join/OuterJoinAssocProject.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/exploration/join/OuterJoinAssocProject.java index 0dcb3b8d344435..aeba205dd84d91 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/exploration/join/OuterJoinAssocProject.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/exploration/join/OuterJoinAssocProject.java @@ -58,7 +58,15 @@ public class OuterJoinAssocProject extends OneExplorationRuleFactory { // newBottomJoin Type = topJoin Type, newTopJoin Type = bottomJoin Type public static Set> VALID_TYPE_PAIR_SET = ImmutableSet.of( Pair.of(JoinType.INNER_JOIN, JoinType.LEFT_OUTER_JOIN), - Pair.of(JoinType.LEFT_OUTER_JOIN, JoinType.LEFT_OUTER_JOIN)); + Pair.of(JoinType.INNER_JOIN, JoinType.ASOF_LEFT_OUTER_JOIN), + Pair.of(JoinType.ASOF_LEFT_INNER_JOIN, JoinType.LEFT_OUTER_JOIN), + Pair.of(JoinType.ASOF_LEFT_INNER_JOIN, JoinType.ASOF_LEFT_OUTER_JOIN), + Pair.of(JoinType.ASOF_RIGHT_INNER_JOIN, JoinType.LEFT_OUTER_JOIN), + Pair.of(JoinType.ASOF_RIGHT_INNER_JOIN, JoinType.ASOF_LEFT_OUTER_JOIN), + Pair.of(JoinType.ASOF_LEFT_OUTER_JOIN, JoinType.LEFT_OUTER_JOIN), + Pair.of(JoinType.ASOF_LEFT_OUTER_JOIN, JoinType.ASOF_LEFT_OUTER_JOIN), + Pair.of(JoinType.LEFT_OUTER_JOIN, JoinType.LEFT_OUTER_JOIN), + Pair.of(JoinType.LEFT_OUTER_JOIN, JoinType.ASOF_LEFT_OUTER_JOIN)); @Override public Rule build() { diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/exploration/join/OuterJoinLAsscomProject.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/exploration/join/OuterJoinLAsscomProject.java index ac9787bed48dc1..9ce00e9bb3fcdc 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/exploration/join/OuterJoinLAsscomProject.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/exploration/join/OuterJoinLAsscomProject.java @@ -49,7 +49,18 @@ public class OuterJoinLAsscomProject extends OneExplorationRuleFactory { // newBottomJoin Type = topJoin Type, newTopJoin Type = bottomJoin Type public static Set> VALID_TYPE_PAIR_SET = ImmutableSet.of( Pair.of(JoinType.LEFT_OUTER_JOIN, JoinType.INNER_JOIN), + Pair.of(JoinType.LEFT_OUTER_JOIN, JoinType.ASOF_LEFT_INNER_JOIN), + Pair.of(JoinType.LEFT_OUTER_JOIN, JoinType.ASOF_RIGHT_INNER_JOIN), + Pair.of(JoinType.ASOF_LEFT_OUTER_JOIN, JoinType.INNER_JOIN), + Pair.of(JoinType.ASOF_LEFT_OUTER_JOIN, JoinType.ASOF_LEFT_INNER_JOIN), + Pair.of(JoinType.ASOF_LEFT_OUTER_JOIN, JoinType.ASOF_RIGHT_INNER_JOIN), Pair.of(JoinType.INNER_JOIN, JoinType.LEFT_OUTER_JOIN), + Pair.of(JoinType.INNER_JOIN, JoinType.ASOF_LEFT_OUTER_JOIN), + Pair.of(JoinType.ASOF_LEFT_INNER_JOIN, JoinType.LEFT_OUTER_JOIN), + Pair.of(JoinType.ASOF_LEFT_INNER_JOIN, JoinType.ASOF_LEFT_OUTER_JOIN), + Pair.of(JoinType.ASOF_RIGHT_INNER_JOIN, JoinType.LEFT_OUTER_JOIN), + Pair.of(JoinType.ASOF_RIGHT_INNER_JOIN, JoinType.ASOF_LEFT_OUTER_JOIN), + Pair.of(JoinType.ASOF_LEFT_OUTER_JOIN, JoinType.ASOF_LEFT_OUTER_JOIN), Pair.of(JoinType.LEFT_OUTER_JOIN, JoinType.LEFT_OUTER_JOIN)); /* diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/exploration/mv/AbstractMaterializedViewRule.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/exploration/mv/AbstractMaterializedViewRule.java index 036ffcd225177e..89d7fa9346fc6f 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/exploration/mv/AbstractMaterializedViewRule.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/exploration/mv/AbstractMaterializedViewRule.java @@ -105,7 +105,11 @@ public abstract class AbstractMaterializedViewRule implements ExplorationRuleFac JoinType.RIGHT_SEMI_JOIN, JoinType.LEFT_ANTI_JOIN, JoinType.RIGHT_ANTI_JOIN, - JoinType.NULL_AWARE_LEFT_ANTI_JOIN); + JoinType.NULL_AWARE_LEFT_ANTI_JOIN, + JoinType.ASOF_LEFT_INNER_JOIN, + JoinType.ASOF_RIGHT_INNER_JOIN, + JoinType.ASOF_LEFT_OUTER_JOIN, + JoinType.ASOF_RIGHT_OUTER_JOIN); /** * The abstract template method for query rewrite, it contains the main logic, try to rewrite query by diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/exploration/mv/HyperGraphComparator.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/exploration/mv/HyperGraphComparator.java index 8a6ed611cd1f67..b7c3b28d737d87 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/exploration/mv/HyperGraphComparator.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/exploration/mv/HyperGraphComparator.java @@ -67,12 +67,24 @@ public class HyperGraphComparator { static Map, Pair> canInferredJoinTypeMap = ImmutableMap ., Pair>builder() .put(Pair.of(JoinType.LEFT_SEMI_JOIN, JoinType.INNER_JOIN), Pair.of(false, false)) + .put(Pair.of(JoinType.LEFT_SEMI_JOIN, JoinType.ASOF_LEFT_INNER_JOIN), Pair.of(false, false)) + .put(Pair.of(JoinType.LEFT_SEMI_JOIN, JoinType.ASOF_RIGHT_INNER_JOIN), Pair.of(false, false)) .put(Pair.of(JoinType.RIGHT_SEMI_JOIN, JoinType.INNER_JOIN), Pair.of(false, false)) + .put(Pair.of(JoinType.RIGHT_SEMI_JOIN, JoinType.ASOF_LEFT_INNER_JOIN), Pair.of(false, false)) + .put(Pair.of(JoinType.RIGHT_SEMI_JOIN, JoinType.ASOF_RIGHT_INNER_JOIN), Pair.of(false, false)) .put(Pair.of(JoinType.INNER_JOIN, JoinType.LEFT_OUTER_JOIN), Pair.of(false, true)) .put(Pair.of(JoinType.INNER_JOIN, JoinType.RIGHT_OUTER_JOIN), Pair.of(true, false)) .put(Pair.of(JoinType.INNER_JOIN, JoinType.FULL_OUTER_JOIN), Pair.of(true, true)) + .put(Pair.of(JoinType.ASOF_LEFT_INNER_JOIN, JoinType.LEFT_OUTER_JOIN), Pair.of(false, true)) + .put(Pair.of(JoinType.ASOF_LEFT_INNER_JOIN, JoinType.RIGHT_OUTER_JOIN), Pair.of(true, false)) + .put(Pair.of(JoinType.ASOF_LEFT_INNER_JOIN, JoinType.FULL_OUTER_JOIN), Pair.of(true, true)) + .put(Pair.of(JoinType.ASOF_RIGHT_INNER_JOIN, JoinType.LEFT_OUTER_JOIN), Pair.of(false, true)) + .put(Pair.of(JoinType.ASOF_RIGHT_INNER_JOIN, JoinType.RIGHT_OUTER_JOIN), Pair.of(true, false)) + .put(Pair.of(JoinType.ASOF_RIGHT_INNER_JOIN, JoinType.FULL_OUTER_JOIN), Pair.of(true, true)) .put(Pair.of(JoinType.LEFT_OUTER_JOIN, JoinType.FULL_OUTER_JOIN), Pair.of(true, false)) + .put(Pair.of(JoinType.ASOF_LEFT_OUTER_JOIN, JoinType.FULL_OUTER_JOIN), Pair.of(true, false)) .put(Pair.of(JoinType.RIGHT_OUTER_JOIN, JoinType.FULL_OUTER_JOIN), Pair.of(false, true)) + .put(Pair.of(JoinType.ASOF_RIGHT_OUTER_JOIN, JoinType.FULL_OUTER_JOIN), Pair.of(false, true)) .build(); // record inferred edges when comparing mv diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/rewrite/ConvertOuterJoinToAntiJoin.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/rewrite/ConvertOuterJoinToAntiJoin.java index 46445573055df2..8791bb693f9b06 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/rewrite/ConvertOuterJoinToAntiJoin.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/rewrite/ConvertOuterJoinToAntiJoin.java @@ -79,7 +79,8 @@ public Plan visitLogicalFilter(LogicalFilter filter, Map> filter, Map replaceMap) { LogicalJoin join = filter.child(); - if (!join.getJoinType().isLeftOuterJoin() && !join.getJoinType().isRightOuterJoin()) { + if (join.getJoinType().isAsofJoin() + || (!join.getJoinType().isLeftOuterJoin() && !join.getJoinType().isRightOuterJoin())) { return filter; } diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/rewrite/EliminateConstHashJoinCondition.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/rewrite/EliminateConstHashJoinCondition.java index 5aef6d1e9475a3..56aa0595d229f0 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/rewrite/EliminateConstHashJoinCondition.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/rewrite/EliminateConstHashJoinCondition.java @@ -45,7 +45,7 @@ public class EliminateConstHashJoinCondition extends OneRewriteRuleFactory { public Rule build() { return logicalJoin() .when(join -> join.getJoinType().isInnerJoin() || join.getJoinType().isSemiJoin()) - .whenNot(join -> join.isMarkJoin()) + .whenNot(join -> join.isMarkJoin() || join.getJoinType().isAsofJoin()) .then(EliminateConstHashJoinCondition::eliminateConstHashJoinCondition) .toRule(RuleType.ELIMINATE_CONST_JOIN_CONDITION); } diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/rewrite/EliminateEmptyRelation.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/rewrite/EliminateEmptyRelation.java index 55e79e4742e744..51be745321dc72 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/rewrite/EliminateEmptyRelation.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/rewrite/EliminateEmptyRelation.java @@ -231,12 +231,12 @@ private boolean bothChildrenEmpty(LogicalJoin join) { } private boolean canReplaceJoinByEmptyRelation(LogicalJoin join) { - return !join.isMarkJoin() && ((join.getJoinType() == JoinType.INNER_JOIN + return !join.isMarkJoin() && ((join.getJoinType().isInnerJoin() || join.getJoinType() == JoinType.LEFT_SEMI_JOIN || join.getJoinType() == JoinType.RIGHT_SEMI_JOIN || join.getJoinType() == JoinType.CROSS_JOIN) - || (join.getJoinType() == JoinType.LEFT_OUTER_JOIN && join.left() instanceof EmptyRelation) - || (join.getJoinType() == JoinType.RIGHT_OUTER_JOIN && join.right() instanceof EmptyRelation)); + || (join.getJoinType().isLeftOuterJoin() && join.left() instanceof EmptyRelation) + || (join.getJoinType().isRightOuterJoin() && join.right() instanceof EmptyRelation)); } } diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/rewrite/EliminateJoinByFK.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/rewrite/EliminateJoinByFK.java index b8e2ab6a003dbc..f9a4e015ada771 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/rewrite/EliminateJoinByFK.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/rewrite/EliminateJoinByFK.java @@ -59,7 +59,7 @@ public class EliminateJoinByFK extends OneRewriteRuleFactory { @Override public Rule build() { return logicalProject( - logicalJoin().when(join -> join.getJoinType().isInnerJoin()) + logicalJoin().when(join -> join.getJoinType().isInnerJoin() && !join.getJoinType().isAsofJoin()) ).then(project -> { LogicalJoin join = project.child(); ImmutableEqualSet equalSet = join.getEqualSlots(); diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/rewrite/EliminateJoinByUnique.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/rewrite/EliminateJoinByUnique.java index 204830120728bd..b8144c4846750f 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/rewrite/EliminateJoinByUnique.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/rewrite/EliminateJoinByUnique.java @@ -37,7 +37,8 @@ public Rule build() { return project; } if (!JoinUtils.canEliminateByLeft(join, - join.right().getLogicalProperties().getTrait())) { + join.right().getLogicalProperties().getTrait()) + && !join.getJoinType().isAsofJoin()) { return project; } return project.withChildren(join.left()); diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/rewrite/EliminateOuterJoin.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/rewrite/EliminateOuterJoin.java index b8704f5610b9ea..9a354d3e27cf8a 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/rewrite/EliminateOuterJoin.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/rewrite/EliminateOuterJoin.java @@ -114,10 +114,10 @@ public Rule build() { private JoinType tryEliminateOuterJoin(JoinType joinType, boolean canFilterLeftNull, boolean canFilterRightNull) { if (joinType.isRightOuterJoin() && canFilterLeftNull) { - return JoinType.INNER_JOIN; + return joinType.isAsofOuterJoin() ? JoinType.ASOF_RIGHT_INNER_JOIN : JoinType.INNER_JOIN; } if (joinType.isLeftOuterJoin() && canFilterRightNull) { - return JoinType.INNER_JOIN; + return joinType.isAsofOuterJoin() ? JoinType.ASOF_LEFT_INNER_JOIN : JoinType.INNER_JOIN; } if (joinType.isFullOuterJoin() && canFilterLeftNull && canFilterRightNull) { return JoinType.INNER_JOIN; diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/rewrite/InferPredicates.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/rewrite/InferPredicates.java index f0d4deac1476a9..9c534c337a64e2 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/rewrite/InferPredicates.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/rewrite/InferPredicates.java @@ -104,16 +104,20 @@ public Plan visitLogicalJoin(LogicalJoin join, J right = inferNewPredicate(right, expressions); break; case INNER_JOIN: + case ASOF_LEFT_INNER_JOIN: + case ASOF_RIGHT_INNER_JOIN: case LEFT_SEMI_JOIN: case RIGHT_SEMI_JOIN: left = inferNewPredicateRemoveUselessIsNull(left, expressions, join, context.getCascadesContext()); right = inferNewPredicateRemoveUselessIsNull(right, expressions, join, context.getCascadesContext()); break; case LEFT_OUTER_JOIN: + case ASOF_LEFT_OUTER_JOIN: case LEFT_ANTI_JOIN: right = inferNewPredicateRemoveUselessIsNull(right, expressions, join, context.getCascadesContext()); break; case RIGHT_OUTER_JOIN: + case ASOF_RIGHT_OUTER_JOIN: case RIGHT_ANTI_JOIN: left = inferNewPredicateRemoveUselessIsNull(left, expressions, join, context.getCascadesContext()); break; diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/rewrite/JoinExtractOrFromCaseWhen.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/rewrite/JoinExtractOrFromCaseWhen.java index 929ecbc272391f..87fdf7a48feb2a 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/rewrite/JoinExtractOrFromCaseWhen.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/rewrite/JoinExtractOrFromCaseWhen.java @@ -82,6 +82,9 @@ public List buildRules() { } private boolean needRewrite(LogicalJoin join) { + if (join.getJoinType().isAsofJoin()) { + return false; + } if (PushDownJoinOtherCondition.needRewrite(join) || OrExpansion.INSTANCE.needRewriteJoin(join)) { Set leftSlots = join.left().getOutputSet(); Set rightSlots = join.right().getOutputSet(); diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/rewrite/PullUpPredicates.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/rewrite/PullUpPredicates.java index 8da895d34ab7fb..0a992db6badaea 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/rewrite/PullUpPredicates.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/rewrite/PullUpPredicates.java @@ -252,7 +252,9 @@ public ImmutableSet visitLogicalJoin(LogicalJoin visitLogicalJoin(LogicalJoin visitLogicalJoin(LogicalJoin COULD_PUSH_THROUGH_RIGHT = ImmutableList.of( @@ -55,7 +58,10 @@ public class PushDownFilterThroughJoin extends OneRewriteRuleFactory { JoinType.RIGHT_OUTER_JOIN, JoinType.RIGHT_SEMI_JOIN, JoinType.RIGHT_ANTI_JOIN, - JoinType.CROSS_JOIN + JoinType.CROSS_JOIN, + JoinType.ASOF_LEFT_INNER_JOIN, + JoinType.ASOF_RIGHT_INNER_JOIN, + JoinType.ASOF_RIGHT_OUTER_JOIN ); private static final ImmutableList COULD_PUSH_INSIDE = ImmutableList.of( diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/rewrite/PushDownJoinOnAssertNumRows.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/rewrite/PushDownJoinOnAssertNumRows.java index ef696d1f3bca63..e52def0723cccf 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/rewrite/PushDownJoinOnAssertNumRows.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/rewrite/PushDownJoinOnAssertNumRows.java @@ -24,7 +24,6 @@ import org.apache.doris.nereids.trees.expressions.Expression; import org.apache.doris.nereids.trees.expressions.NamedExpression; import org.apache.doris.nereids.trees.expressions.Slot; -import org.apache.doris.nereids.trees.plans.JoinType; import org.apache.doris.nereids.trees.plans.Plan; import org.apache.doris.nereids.trees.plans.logical.LogicalAssertNumRows; import org.apache.doris.nereids.trees.plans.logical.LogicalJoin; @@ -90,7 +89,7 @@ private boolean pattenCheck(LogicalJoin topJoin) { // 1. right is LogicalAssertNumRows or LogicalProject->LogicalAssertNumRows // 2. left is join or project->join // 3. only one join condition. - if (topJoin.getJoinType() != JoinType.INNER_JOIN && topJoin.getJoinType() != JoinType.CROSS_JOIN) { + if (!topJoin.getJoinType().isInnerOrCrossJoin()) { return false; } LogicalJoin bottomJoin; @@ -107,7 +106,7 @@ private boolean pattenCheck(LogicalJoin topJoin) { return false; } - if (bottomJoin.getJoinType() != JoinType.INNER_JOIN && bottomJoin.getJoinType() != JoinType.CROSS_JOIN) { + if (!bottomJoin.getJoinType().isInnerOrCrossJoin()) { return false; } diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/rewrite/PushDownJoinOtherCondition.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/rewrite/PushDownJoinOtherCondition.java index 75c5e3085d310c..e38d3e9cd419d2 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/rewrite/PushDownJoinOtherCondition.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/rewrite/PushDownJoinOtherCondition.java @@ -42,8 +42,11 @@ public class PushDownJoinOtherCondition extends OneRewriteRuleFactory { */ public static final ImmutableList PUSH_DOWN_LEFT_VALID_TYPE = ImmutableList.of( JoinType.INNER_JOIN, + JoinType.ASOF_LEFT_INNER_JOIN, + JoinType.ASOF_RIGHT_INNER_JOIN, JoinType.LEFT_SEMI_JOIN, JoinType.RIGHT_OUTER_JOIN, + JoinType.ASOF_RIGHT_OUTER_JOIN, JoinType.RIGHT_ANTI_JOIN, JoinType.RIGHT_SEMI_JOIN, JoinType.CROSS_JOIN @@ -54,7 +57,10 @@ public class PushDownJoinOtherCondition extends OneRewriteRuleFactory { */ public static final ImmutableList PUSH_DOWN_RIGHT_VALID_TYPE = ImmutableList.of( JoinType.INNER_JOIN, + JoinType.ASOF_LEFT_INNER_JOIN, + JoinType.ASOF_RIGHT_INNER_JOIN, JoinType.LEFT_OUTER_JOIN, + JoinType.ASOF_LEFT_OUTER_JOIN, JoinType.LEFT_ANTI_JOIN, JoinType.NULL_AWARE_LEFT_ANTI_JOIN, JoinType.LEFT_SEMI_JOIN, diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/rewrite/PushDownLimit.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/rewrite/PushDownLimit.java index 68cf03d7f5b3f3..8aaeba36f32742 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/rewrite/PushDownLimit.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/rewrite/PushDownLimit.java @@ -127,11 +127,13 @@ public List buildRules() { private Plan pushLimitThroughJoin(LogicalLimit limit, LogicalJoin join) { switch (join.getJoinType()) { case LEFT_OUTER_JOIN: + case ASOF_LEFT_OUTER_JOIN: return join.withChildren( limit.withChildren(join.left()), join.right() ); case RIGHT_OUTER_JOIN: + case ASOF_RIGHT_OUTER_JOIN: return join.withChildren( join.left(), limit.withChildren(join.right()) diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/rewrite/PushDownLimitDistinctThroughJoin.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/rewrite/PushDownLimitDistinctThroughJoin.java index db538329339a6c..906007decf4e65 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/rewrite/PushDownLimitDistinctThroughJoin.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/rewrite/PushDownLimitDistinctThroughJoin.java @@ -83,6 +83,7 @@ private Plan pushLimitThroughJoin(LogicalLimit limit, LogicalJoin .flatMap(e -> e.getInputSlots().stream()).collect(Collectors.toList()); switch (join.getJoinType()) { case LEFT_OUTER_JOIN: + case ASOF_LEFT_OUTER_JOIN: if (!(join.left() instanceof Limit) && join.left().getOutputSet().containsAll(groupBySlots) && join.left().getOutputSet().equals(agg.getOutputSet())) { @@ -91,6 +92,7 @@ private Plan pushLimitThroughJoin(LogicalLimit limit, LogicalJoin } return null; case RIGHT_OUTER_JOIN: + case ASOF_RIGHT_OUTER_JOIN: if (!(join.right() instanceof Limit) && join.right().getOutputSet().containsAll(groupBySlots) && join.right().getOutputSet().equals(agg.getOutputSet())) { return join.withChildren(join.left(), limit.withLimitChild(limit.getLimit() + limit.getOffset(), 0, diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/rewrite/PushDownTopNDistinctThroughJoin.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/rewrite/PushDownTopNDistinctThroughJoin.java index 4c6f426f7ac1ba..4b0b8a89dc4991 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/rewrite/PushDownTopNDistinctThroughJoin.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/rewrite/PushDownTopNDistinctThroughJoin.java @@ -98,7 +98,8 @@ private Plan pushTopNThroughJoin(LogicalTopN topN, LogicalJoin

groupBySlots = ((LogicalAggregate) topN.child()).getGroupByExpressions().stream() .flatMap(e -> e.getInputSlots().stream()).collect(Collectors.toSet()); switch (join.getJoinType()) { - case LEFT_OUTER_JOIN: { + case LEFT_OUTER_JOIN: + case ASOF_LEFT_OUTER_JOIN: { if (join.left() instanceof TopN) { return null; } @@ -112,7 +113,8 @@ private Plan pushTopNThroughJoin(LogicalTopN topN, LogicalJoin

topN, LogicalJoin< .flatMap(e -> e.getInputSlots().stream()).collect(Collectors.toList()); switch (join.getJoinType()) { case LEFT_OUTER_JOIN: + case ASOF_LEFT_OUTER_JOIN: if (join.left() instanceof TopN) { return null; } @@ -104,6 +105,7 @@ private Plan pushLimitThroughJoin(LogicalTopN topN, LogicalJoin< } return null; case RIGHT_OUTER_JOIN: + case ASOF_RIGHT_OUTER_JOIN: if (join.right() instanceof TopN) { return null; } diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/rewrite/SaltJoin.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/rewrite/SaltJoin.java index 1227d63ebc8f80..c2ee8171109146 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/rewrite/SaltJoin.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/rewrite/SaltJoin.java @@ -33,7 +33,6 @@ import org.apache.doris.nereids.trees.expressions.InPredicate; import org.apache.doris.nereids.trees.expressions.IsNull; import org.apache.doris.nereids.trees.expressions.NamedExpression; -import org.apache.doris.nereids.trees.expressions.Not; import org.apache.doris.nereids.trees.expressions.NullSafeEqual; import org.apache.doris.nereids.trees.expressions.Or; import org.apache.doris.nereids.trees.expressions.Slot; @@ -50,7 +49,6 @@ import org.apache.doris.nereids.trees.plans.JoinType; import org.apache.doris.nereids.trees.plans.Plan; import org.apache.doris.nereids.trees.plans.algebra.SetOperation.Qualifier; -import org.apache.doris.nereids.trees.plans.logical.LogicalFilter; import org.apache.doris.nereids.trees.plans.logical.LogicalGenerate; import org.apache.doris.nereids.trees.plans.logical.LogicalJoin; import org.apache.doris.nereids.trees.plans.logical.LogicalProject; @@ -63,7 +61,6 @@ import org.apache.doris.qe.ConnectContext; import com.google.common.collect.ImmutableList; -import com.google.common.collect.ImmutableSet; import org.jetbrains.annotations.Nullable; import java.util.HashSet; @@ -205,12 +202,16 @@ public static Plan transform(LogicalJoin join) { switch (join.getJoinType()) { case INNER_JOIN: case LEFT_OUTER_JOIN: + case ASOF_LEFT_INNER_JOIN: + case ASOF_RIGHT_INNER_JOIN: + case ASOF_LEFT_OUTER_JOIN: leftProject = addRandomSlot(leftSkewExpr, skewSideValues, join.left(), factor, type, statementContext); rightProject = expandSkewValueRows(rightSkewExpr, expandSideValues, join.right(), factor, type, statementContext); break; case RIGHT_OUTER_JOIN: + case ASOF_RIGHT_OUTER_JOIN: leftProject = expandSkewValueRows(leftSkewExpr, expandSideValues, join.left(), factor, type, statementContext); rightProject = addRandomSlot(rightSkewExpr, skewSideValues, join.right(), factor, type, @@ -371,37 +372,4 @@ private static List getSaltedSkewValuesForSkewSide(Expression skewCo } return ImmutableList.of(); } - - private static LogicalJoin addNotNull(LogicalJoin join, Expression skewConjunct, - Set skewValuesSet) { - if (skewConjunct instanceof NullSafeEqual) { - return join; - } - boolean containsNull = skewValuesSet.stream().anyMatch(value -> value instanceof NullLiteral); - if (!containsNull) { - return join; - } - - LogicalFilter leftFilter = - new LogicalFilter<>(ImmutableSet.of(new Not(new IsNull(skewConjunct.child(0)))), join.left()); - LogicalFilter rightFilter = - new LogicalFilter<>(ImmutableSet.of(new Not(new IsNull(skewConjunct.child(1)))), join.right()); - DistributeHint hint = join.getDistributeHint(); - switch (join.getJoinType()) { - case INNER_JOIN: - hint.setStatus(HintStatus.SUCCESS); - hint.setSkewInfo(hint.getSkewInfo().withSuccessInSaltJoin(true)); - return join.withDistributeHintChildren(hint, leftFilter, rightFilter); - case LEFT_OUTER_JOIN: - hint.setStatus(HintStatus.SUCCESS); - hint.setSkewInfo(hint.getSkewInfo().withSuccessInSaltJoin(true)); - return join.withDistributeHintChildren(hint, join.left(), rightFilter); - case RIGHT_OUTER_JOIN: - hint.setStatus(HintStatus.SUCCESS); - hint.setSkewInfo(hint.getSkewInfo().withSuccessInSaltJoin(true)); - return join.withDistributeHintChildren(hint, leftFilter, join.right()); - default: - return join; - } - } } diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/rewrite/TransposeSemiJoinLogicalJoin.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/rewrite/TransposeSemiJoinLogicalJoin.java index 7ee1dcf94f98b8..509ffa8bb0c5ad 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/rewrite/TransposeSemiJoinLogicalJoin.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/rewrite/TransposeSemiJoinLogicalJoin.java @@ -21,7 +21,6 @@ import org.apache.doris.nereids.rules.RuleType; import org.apache.doris.nereids.rules.rewrite.TransposeSemiJoinLogicalJoinProject.ContainsType; import org.apache.doris.nereids.trees.expressions.ExprId; -import org.apache.doris.nereids.trees.plans.JoinType; import org.apache.doris.nereids.trees.plans.Plan; import org.apache.doris.nereids.trees.plans.logical.LogicalJoin; import org.apache.doris.qe.ConnectContext; @@ -66,7 +65,7 @@ public Rule build() { * A B A C */ // RIGHT_OUTER_JOIN should be eliminated in rewrite phase - if (bottomJoin.getJoinType() == JoinType.RIGHT_OUTER_JOIN) { + if (bottomJoin.getJoinType().isRightOuterJoin()) { return null; } Plan newBottomSemiJoin = topSemiJoin.withChildren(a, c); @@ -80,7 +79,7 @@ public Rule build() { * A B B C */ // LEFT_OUTER_JOIN should be eliminated in rewrite phase - if (bottomJoin.getJoinType() == JoinType.LEFT_OUTER_JOIN) { + if (bottomJoin.getJoinType().isLeftOuterJoin()) { return null; } Plan newBottomSemiJoin = topSemiJoin.withChildren(b, c); diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/rewrite/TransposeSemiJoinLogicalJoinProject.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/rewrite/TransposeSemiJoinLogicalJoinProject.java index 95065e1b33dab1..40e49e13bcd95a 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/rewrite/TransposeSemiJoinLogicalJoinProject.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/rewrite/TransposeSemiJoinLogicalJoinProject.java @@ -22,7 +22,6 @@ import org.apache.doris.nereids.trees.expressions.ExprId; import org.apache.doris.nereids.trees.expressions.NamedExpression; import org.apache.doris.nereids.trees.expressions.Slot; -import org.apache.doris.nereids.trees.plans.JoinType; import org.apache.doris.nereids.trees.plans.Plan; import org.apache.doris.nereids.trees.plans.logical.LogicalJoin; import org.apache.doris.nereids.trees.plans.logical.LogicalProject; @@ -80,7 +79,7 @@ public Rule build() { // RIGHT_OUTER_JOIN should be eliminated in rewrite phase // TODO: when top join is ANTI JOIN, bottomJoin may be RIGHT_OUTER_JOIN // Can we also do the transformation? - if (bottomJoin.getJoinType() == JoinType.RIGHT_OUTER_JOIN) { + if (bottomJoin.getJoinType().isRightOuterJoin()) { return null; } @@ -100,7 +99,7 @@ public Rule build() { // LEFT_OUTER_JOIN should be eliminated in rewrite phase // TODO: when top join is ANTI JOIN, bottomJoin may be RIGHT_OUTER_JOIN // Can we also do the transformation? - if (bottomJoin.getJoinType() == JoinType.LEFT_OUTER_JOIN) { + if (bottomJoin.getJoinType().isLeftOuterJoin()) { return null; } diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/stats/JoinEstimation.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/stats/JoinEstimation.java index 0e03a6cf7bdb74..e657c586ee29cf 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/stats/JoinEstimation.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/stats/JoinEstimation.java @@ -359,6 +359,83 @@ private static Statistics estimateSemiOrAnti(Statistics leftStats, Statistics ri } } + private static Statistics estimateAsofInnerJoin(Statistics leftStats, Statistics rightStats, + Statistics innerJoinStats, Join join) { + if (joinConditionContainsUnknownColumnStats(leftStats, rightStats, join)) { + double sel = computeSelectivityForBuildSideWhenColStatsUnknown(rightStats, join); + Statistics result; + if (join.getJoinType().isAsofLeftInnerJoin()) { + result = new StatisticsBuilder().setRowCount(leftStats.getRowCount() * sel) + .putColumnStatistics(leftStats.columnStatistics()) + .putColumnStatistics(rightStats.columnStatistics()) + .build(); + } else { + //asof right inner join + result = new StatisticsBuilder().setRowCount(rightStats.getRowCount() * sel) + .putColumnStatistics(leftStats.columnStatistics()) + .putColumnStatistics(rightStats.columnStatistics()) + .build(); + } + result.normalizeColumnStatistics(); + return result; + } + double rowCount = Double.POSITIVE_INFINITY; + for (Expression conjunct : join.getEqualPredicates()) { + double eqRowCount = estimateAsofInnerJoinCountBySlotsEqual(leftStats, rightStats, + join, (EqualPredicate) conjunct); + if (rowCount > eqRowCount) { + rowCount = eqRowCount; + } + } + if (Double.isInfinite(rowCount)) { + //slotsEqual estimation failed, fall back to original algorithm + double baseRowCount = + join.getJoinType().isAsofLeftInnerJoin() ? leftStats.getRowCount() : rightStats.getRowCount(); + rowCount = Math.min(innerJoinStats.getRowCount(), baseRowCount); + return innerJoinStats.withRowCountAndEnforceValid(rowCount); + } else { + StatisticsBuilder builder; + if (join.getJoinType().isAsofLeftInnerJoin()) { + builder = new StatisticsBuilder(leftStats); + } else { + //asof right inner join + builder = new StatisticsBuilder(rightStats); + } + builder.setRowCount(rowCount); + Statistics outputStats = builder.build(); + outputStats.normalizeColumnStatistics(); + return outputStats; + } + } + + private static double estimateAsofInnerJoinCountBySlotsEqual(Statistics leftStats, + Statistics rightStats, Join join, EqualPredicate equalTo) { + Expression eqLeft = equalTo.left(); + Expression eqRight = equalTo.right(); + ColumnStatistic probColStats = leftStats.findColumnStatistics(eqLeft); + ColumnStatistic buildColStats; + if (probColStats == null) { + probColStats = leftStats.findColumnStatistics(eqRight); + buildColStats = rightStats.findColumnStatistics(eqLeft); + } else { + buildColStats = rightStats.findColumnStatistics(eqRight); + } + if (probColStats == null || buildColStats == null) { + return Double.POSITIVE_INFINITY; + } + + double rowCount; + if (join.getJoinType().isAsofLeftInnerJoin()) { + rowCount = StatsMathUtil.divide(leftStats.getRowCount() * buildColStats.ndv, + buildColStats.getOriginalNdv()); + } else { + // asof right inner join + rowCount = StatsMathUtil.divide(rightStats.getRowCount() * probColStats.ndv, + probColStats.getOriginalNdv()); + } + return Math.max(1, rowCount); + } + /** * outer join generates nulls. * for example, T1 left outer join T2, @@ -418,6 +495,20 @@ public static Statistics estimate(Statistics leftStats, Statistics rightStats, J updateNumNullsForOuterJoin(crossJoinStats, innerJoinStats, rightStats, leftStats, rowCount); updateJoinConditionColumnStatistics(crossJoinStats, join); return crossJoinStats.withRowCountAndEnforceValid(rowCount); + } else if (joinType == JoinType.ASOF_LEFT_OUTER_JOIN) { + double rowCount = Math.max(leftStats.getRowCount(), 1); + updateNumNullsForOuterJoin(crossJoinStats, innerJoinStats, leftStats, rightStats, rowCount); + updateJoinConditionColumnStatistics(crossJoinStats, join); + return crossJoinStats.withRowCountAndEnforceValid(rowCount); + } else if (joinType == JoinType.ASOF_RIGHT_OUTER_JOIN) { + double rowCount = Math.max(rightStats.getRowCount(), 1); + updateNumNullsForOuterJoin(crossJoinStats, innerJoinStats, leftStats, rightStats, rowCount); + updateJoinConditionColumnStatistics(crossJoinStats, join); + return crossJoinStats.withRowCountAndEnforceValid(rowCount); + } else if (joinType.isAsofInnerJoin()) { + Statistics outputStats = estimateAsofInnerJoin(leftStats, rightStats, innerJoinStats, join); + updateJoinConditionColumnStatistics(outputStats, join); + return outputStats; } else if (joinType == JoinType.CROSS_JOIN) { updateJoinConditionColumnStatistics(crossJoinStats, join); return crossJoinStats; @@ -443,7 +534,7 @@ private static void updateJoinConditionColumnStatistics(Statistics inputStats, J if (eqRight instanceof Cast) { eqRight = eqRight.child(0); } - if (joinType == JoinType.INNER_JOIN) { + if (joinType.isInnerJoin()) { ColumnStatisticBuilder builder = new ColumnStatisticBuilder(leftColStats); builder.setNdv(Math.min(leftColStats.ndv, rightColStats.ndv)); // update hot values @@ -463,7 +554,7 @@ private static void updateJoinConditionColumnStatistics(Statistics inputStats, J } updatedCols.put(eqLeft, builder.build()); updatedCols.put(eqRight, builder.build()); - } else if (joinType == JoinType.LEFT_OUTER_JOIN) { + } else if (joinType.isLeftOuterJoin()) { ColumnStatisticBuilder rightBuilder = new ColumnStatisticBuilder(rightColStats); rightBuilder.setNdv(Math.min(leftColStats.ndv, rightColStats.ndv)); // update hot values @@ -482,13 +573,11 @@ private static void updateJoinConditionColumnStatistics(Statistics inputStats, J } } updatedCols.put(eqRight, rightBuilder.build()); - } else if (joinType == JoinType.LEFT_SEMI_JOIN - || joinType == JoinType.LEFT_ANTI_JOIN - || joinType == JoinType.NULL_AWARE_LEFT_ANTI_JOIN) { + } else if (joinType.isLeftSemiOrAntiJoin()) { ColumnStatisticBuilder leftBuilder = new ColumnStatisticBuilder(leftColStats); leftBuilder.setNdv(Math.min(leftColStats.ndv, rightColStats.ndv)); updatedCols.put(eqLeft, leftBuilder.build()); - } else if (joinType == JoinType.RIGHT_OUTER_JOIN) { + } else if (joinType.isRightOuterJoin()) { ColumnStatisticBuilder leftBuilder = new ColumnStatisticBuilder(leftColStats); leftBuilder.setNdv(Math.min(leftColStats.ndv, rightColStats.ndv)); // update hot values @@ -507,12 +596,11 @@ private static void updateJoinConditionColumnStatistics(Statistics inputStats, J } } updatedCols.put(eqLeft, leftBuilder.build()); - } else if (joinType == JoinType.RIGHT_SEMI_JOIN - || joinType == JoinType.RIGHT_ANTI_JOIN) { + } else if (joinType.isRightSemiOrAntiJoin()) { ColumnStatisticBuilder rightBuilder = new ColumnStatisticBuilder(rightColStats); rightBuilder.setNdv(Math.min(leftColStats.ndv, rightColStats.ndv)); updatedCols.put(eqRight, rightBuilder.build()); - } else if (joinType == JoinType.FULL_OUTER_JOIN || joinType == JoinType.CROSS_JOIN) { + } else if (joinType.isFullOuterJoin() || joinType.isCrossJoin()) { // ignore } diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/JoinType.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/JoinType.java index 112c1d98a98a1f..2fdf1d9adb327a 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/JoinType.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/JoinType.java @@ -39,6 +39,10 @@ public enum JoinType { RIGHT_ANTI_JOIN, CROSS_JOIN, NULL_AWARE_LEFT_ANTI_JOIN, + ASOF_LEFT_INNER_JOIN, + ASOF_RIGHT_INNER_JOIN, + ASOF_LEFT_OUTER_JOIN, + ASOF_RIGHT_OUTER_JOIN ; private static final Map joinSwapMap = ImmutableMap @@ -52,6 +56,10 @@ public enum JoinType { .put(RIGHT_OUTER_JOIN, LEFT_OUTER_JOIN) .put(LEFT_ANTI_JOIN, RIGHT_ANTI_JOIN) .put(RIGHT_ANTI_JOIN, LEFT_ANTI_JOIN) + .put(ASOF_LEFT_INNER_JOIN, ASOF_RIGHT_INNER_JOIN) + .put(ASOF_RIGHT_INNER_JOIN, ASOF_LEFT_INNER_JOIN) + .put(ASOF_LEFT_OUTER_JOIN, ASOF_RIGHT_OUTER_JOIN) + .put(ASOF_RIGHT_OUTER_JOIN, ASOF_LEFT_OUTER_JOIN) .build(); // TODO: the right-semi/right-anti/right-outer join is not derived in paper. We need to derive them @@ -67,22 +75,34 @@ public enum JoinType { * topJoin - - * bottomJoin + - */ - private static final Map> assocJoinMatrix - = ImmutableMap.>builder() - .put(CROSS_JOIN, ImmutableSet.of(CROSS_JOIN, INNER_JOIN)) - .put(INNER_JOIN, ImmutableSet.of(CROSS_JOIN, INNER_JOIN)) + private static final Map> assocJoinMatrix = ImmutableMap + .>builder() + .put(CROSS_JOIN, ImmutableSet.of(CROSS_JOIN, INNER_JOIN, ASOF_LEFT_INNER_JOIN, ASOF_RIGHT_INNER_JOIN)) + .put(INNER_JOIN, ImmutableSet.of(CROSS_JOIN, INNER_JOIN, ASOF_LEFT_INNER_JOIN, ASOF_RIGHT_INNER_JOIN)) + .put(ASOF_LEFT_INNER_JOIN, + ImmutableSet.of(CROSS_JOIN, INNER_JOIN, ASOF_LEFT_INNER_JOIN, ASOF_RIGHT_INNER_JOIN)) + .put(ASOF_RIGHT_INNER_JOIN, + ImmutableSet.of(CROSS_JOIN, INNER_JOIN, ASOF_LEFT_INNER_JOIN, ASOF_RIGHT_INNER_JOIN)) .build(); - private static final Map> lAssocJoinMatrix - = ImmutableMap.>builder() - .put(CROSS_JOIN, ImmutableSet.of(CROSS_JOIN, INNER_JOIN)) - .put(INNER_JOIN, ImmutableSet.of(CROSS_JOIN, INNER_JOIN)) + private static final Map> lAssocJoinMatrix = ImmutableMap + .>builder() + .put(CROSS_JOIN, ImmutableSet.of(CROSS_JOIN, INNER_JOIN, ASOF_LEFT_INNER_JOIN, ASOF_RIGHT_INNER_JOIN)) + .put(INNER_JOIN, ImmutableSet.of(CROSS_JOIN, INNER_JOIN, ASOF_LEFT_INNER_JOIN, ASOF_RIGHT_INNER_JOIN)) + .put(ASOF_LEFT_INNER_JOIN, + ImmutableSet.of(CROSS_JOIN, INNER_JOIN, ASOF_LEFT_INNER_JOIN, ASOF_RIGHT_INNER_JOIN)) + .put(ASOF_RIGHT_INNER_JOIN, + ImmutableSet.of(CROSS_JOIN, INNER_JOIN, ASOF_LEFT_INNER_JOIN, ASOF_RIGHT_INNER_JOIN)) .build(); - private static final Map> rAssocJoinMatrix - = ImmutableMap.>builder() - .put(CROSS_JOIN, ImmutableSet.of(CROSS_JOIN, INNER_JOIN)) - .put(INNER_JOIN, ImmutableSet.of(CROSS_JOIN, INNER_JOIN)) + private static final Map> rAssocJoinMatrix = ImmutableMap + .>builder() + .put(CROSS_JOIN, ImmutableSet.of(CROSS_JOIN, INNER_JOIN, ASOF_LEFT_INNER_JOIN, ASOF_RIGHT_INNER_JOIN)) + .put(INNER_JOIN, ImmutableSet.of(CROSS_JOIN, INNER_JOIN, ASOF_LEFT_INNER_JOIN, ASOF_RIGHT_INNER_JOIN)) + .put(ASOF_LEFT_INNER_JOIN, + ImmutableSet.of(CROSS_JOIN, INNER_JOIN, ASOF_LEFT_INNER_JOIN, ASOF_RIGHT_INNER_JOIN)) + .put(ASOF_RIGHT_INNER_JOIN, + ImmutableSet.of(CROSS_JOIN, INNER_JOIN, ASOF_LEFT_INNER_JOIN, ASOF_RIGHT_INNER_JOIN)) .build(); /** @@ -112,6 +132,14 @@ public static JoinOperator toJoinOperator(JoinType joinType) { return JoinOperator.LEFT_SEMI_JOIN; case RIGHT_SEMI_JOIN: return JoinOperator.RIGHT_SEMI_JOIN; + case ASOF_LEFT_INNER_JOIN: + return JoinOperator.ASOF_LEFT_INNER_JOIN; + case ASOF_RIGHT_INNER_JOIN: + return JoinOperator.ASOF_RIGHT_INNER_JOIN; + case ASOF_LEFT_OUTER_JOIN: + return JoinOperator.ASOF_LEFT_OUTER_JOIN; + case ASOF_RIGHT_OUTER_JOIN: + return JoinOperator.ASOF_RIGHT_OUTER_JOIN; case CROSS_JOIN: return JoinOperator.CROSS_JOIN; default: @@ -124,20 +152,22 @@ public final boolean isCrossJoin() { } public final boolean isInnerJoin() { - return this == INNER_JOIN; + return this == INNER_JOIN || this == ASOF_LEFT_INNER_JOIN || this == ASOF_RIGHT_INNER_JOIN; } public final boolean isInnerOrCrossJoin() { - return this == INNER_JOIN || this == CROSS_JOIN; + return this == INNER_JOIN || this == CROSS_JOIN + || this == ASOF_LEFT_INNER_JOIN || this == ASOF_RIGHT_INNER_JOIN; } public final boolean isLeftJoin() { return this == LEFT_OUTER_JOIN || this == LEFT_ANTI_JOIN || this == NULL_AWARE_LEFT_ANTI_JOIN - || this == LEFT_SEMI_JOIN; + || this == LEFT_SEMI_JOIN || this == ASOF_LEFT_OUTER_JOIN; } public final boolean isRightJoin() { - return this == RIGHT_OUTER_JOIN || this == RIGHT_ANTI_JOIN || this == RIGHT_SEMI_JOIN; + return this == RIGHT_OUTER_JOIN || this == RIGHT_ANTI_JOIN || this == RIGHT_SEMI_JOIN + || this == ASOF_RIGHT_OUTER_JOIN; } public final boolean isFullOuterJoin() { @@ -145,15 +175,16 @@ public final boolean isFullOuterJoin() { } public final boolean isLeftOuterJoin() { - return this == LEFT_OUTER_JOIN; + return this == LEFT_OUTER_JOIN || this == ASOF_LEFT_OUTER_JOIN; } public final boolean isRightOuterJoin() { - return this == RIGHT_OUTER_JOIN; + return this == RIGHT_OUTER_JOIN || this == ASOF_RIGHT_OUTER_JOIN; } public final boolean isLeftRightOuterOrCrossJoin() { - return this == LEFT_OUTER_JOIN || this == RIGHT_OUTER_JOIN || this == CROSS_JOIN; + return this == LEFT_OUTER_JOIN || this == RIGHT_OUTER_JOIN || this == CROSS_JOIN + || this == ASOF_LEFT_OUTER_JOIN || this == ASOF_RIGHT_OUTER_JOIN; } public final boolean isLeftSemiOrAntiJoin() { @@ -194,11 +225,13 @@ public final boolean isAntiJoin() { } public final boolean isOuterJoin() { - return this == LEFT_OUTER_JOIN || this == RIGHT_OUTER_JOIN || this == FULL_OUTER_JOIN; + return this == LEFT_OUTER_JOIN || this == RIGHT_OUTER_JOIN || this == FULL_OUTER_JOIN + || this == ASOF_LEFT_OUTER_JOIN || this == ASOF_RIGHT_OUTER_JOIN; } public final boolean isOneSideOuterJoin() { - return this == LEFT_OUTER_JOIN || this == RIGHT_OUTER_JOIN; + return this == LEFT_OUTER_JOIN || this == RIGHT_OUTER_JOIN + || this == ASOF_LEFT_OUTER_JOIN || this == ASOF_RIGHT_OUTER_JOIN; } public final boolean isRemainLeftJoin() { @@ -213,6 +246,31 @@ public final boolean isNullAwareLeftAntiJoin() { return this == NULL_AWARE_LEFT_ANTI_JOIN; } + public final boolean isAsofLeftInnerJoin() { + return this == ASOF_LEFT_INNER_JOIN; + } + + public final boolean isAsofInnerJoin() { + return this == ASOF_LEFT_INNER_JOIN || this == ASOF_RIGHT_INNER_JOIN; + } + + public final boolean isAsofOuterJoin() { + return this == ASOF_LEFT_OUTER_JOIN || this == ASOF_RIGHT_OUTER_JOIN; + } + + public final boolean isAsofLeftJoin() { + return this == ASOF_LEFT_INNER_JOIN || this == ASOF_LEFT_OUTER_JOIN; + } + + public final boolean isAsofRightJoin() { + return this == ASOF_RIGHT_INNER_JOIN || this == ASOF_RIGHT_OUTER_JOIN; + } + + public final boolean isAsofJoin() { + return this == ASOF_LEFT_INNER_JOIN || this == ASOF_RIGHT_INNER_JOIN + || this == ASOF_LEFT_OUTER_JOIN || this == ASOF_RIGHT_OUTER_JOIN; + } + public final boolean isSwapJoinType() { return joinSwapMap.containsKey(this); } diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/logical/LogicalJoin.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/logical/LogicalJoin.java index f7bf44b5518cc4..9bb1b103746fad 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/logical/LogicalJoin.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/logical/LogicalJoin.java @@ -582,9 +582,9 @@ public void computeUnique(Builder builder) { // TODO disable function dependence calculation for mark join, but need re-think this in future. return; } - if (joinType.isLeftSemiOrAntiJoin()) { + if (joinType.isLeftSemiOrAntiJoin() || joinType.isAsofLeftJoin()) { builder.addUniqueSlot(left().getLogicalProperties().getTrait()); - } else if (joinType.isRightSemiOrAntiJoin()) { + } else if (joinType.isRightSemiOrAntiJoin() || joinType.isAsofRightJoin()) { builder.addUniqueSlot(right().getLogicalProperties().getTrait()); } // if there is non-equal join conditions, don't propagate unique @@ -607,11 +607,11 @@ public void computeUnique(Builder builder) { // left/right outer join propagate left/right uniforms slots // And if the right/left hash keys is unique, // join can propagate left/right functional dependencies - if (joinType.isLeftOuterJoin() && isRightUnique) { + if (joinType == JoinType.LEFT_OUTER_JOIN && isRightUnique) { builder.addUniqueSlot(left().getLogicalProperties().getTrait()); - } else if (joinType.isRightOuterJoin() && isLeftUnique) { + } else if (joinType == JoinType.RIGHT_OUTER_JOIN && isLeftUnique) { builder.addUniqueSlot(right().getLogicalProperties().getTrait()); - } else if (joinType.isInnerJoin() && isLeftUnique && isRightUnique) { + } else if (joinType == JoinType.INNER_JOIN && isLeftUnique && isRightUnique) { // inner join propagate uniforms slots // And if the hash keys is unique, inner join can propagate all functional dependencies builder.addDataTrait(left().getLogicalProperties().getTrait()); @@ -627,6 +627,8 @@ public void computeUniform(Builder builder) { } switch (joinType) { case INNER_JOIN: + case ASOF_LEFT_INNER_JOIN: + case ASOF_RIGHT_INNER_JOIN: case CROSS_JOIN: builder.addUniformSlot(left().getLogicalProperties().getTrait()); builder.addUniformSlot(right().getLogicalProperties().getTrait()); @@ -641,10 +643,12 @@ public void computeUniform(Builder builder) { builder.addUniformSlot(right().getLogicalProperties().getTrait()); break; case LEFT_OUTER_JOIN: + case ASOF_LEFT_OUTER_JOIN: builder.addUniformSlot(left().getLogicalProperties().getTrait()); builder.addUniformSlotForOuterJoinNullableSide(right().getLogicalProperties().getTrait()); break; case RIGHT_OUTER_JOIN: + case ASOF_RIGHT_OUTER_JOIN: builder.addUniformSlot(right().getLogicalProperties().getTrait()); builder.addUniformSlotForOuterJoinNullableSide(left().getLogicalProperties().getTrait()); break; diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/logical/LogicalUsingJoin.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/logical/LogicalUsingJoin.java index 0b056da686c8de..a6b685a782a91b 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/logical/LogicalUsingJoin.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/logical/LogicalUsingJoin.java @@ -49,21 +49,23 @@ public class LogicalUsingJoin usingSlots; private final DistributeHint hint; + private final Optional matchCondition; public LogicalUsingJoin(JoinType joinType, LEFT_CHILD_TYPE leftChild, RIGHT_CHILD_TYPE rightChild, - List usingSlots, DistributeHint hint) { - this(joinType, leftChild, rightChild, usingSlots, Optional.empty(), Optional.empty(), hint); + List usingSlots, Optional matchCondition, DistributeHint hint) { + this(joinType, leftChild, rightChild, usingSlots, matchCondition, Optional.empty(), Optional.empty(), hint); } /** * Constructor. */ public LogicalUsingJoin(JoinType joinType, LEFT_CHILD_TYPE leftChild, RIGHT_CHILD_TYPE rightChild, - List usingSlots, Optional groupExpression, + List usingSlots, Optional matchCondition, Optional groupExpression, Optional logicalProperties, DistributeHint hint) { super(PlanType.LOGICAL_USING_JOIN, groupExpression, logicalProperties, leftChild, rightChild); this.joinType = joinType; this.usingSlots = ImmutableList.copyOf(usingSlots); + this.matchCondition = matchCondition; this.hint = hint; } @@ -75,20 +77,20 @@ public List computeOutput() { @Override public Plan withGroupExpression(Optional groupExpression) { return new LogicalUsingJoin<>(joinType, child(0), child(1), - usingSlots, groupExpression, Optional.of(getLogicalProperties()), hint); + usingSlots, matchCondition, groupExpression, Optional.of(getLogicalProperties()), hint); } @Override public Plan withGroupExprLogicalPropChildren(Optional groupExpression, Optional logicalProperties, List children) { return new LogicalUsingJoin<>(joinType, children.get(0), children.get(1), - usingSlots, groupExpression, logicalProperties, hint); + usingSlots, matchCondition, groupExpression, logicalProperties, hint); } @Override public Plan withChildren(List children) { return new LogicalUsingJoin<>(joinType, children.get(0), children.get(1), - usingSlots, groupExpression, Optional.of(getLogicalProperties()), hint); + usingSlots, matchCondition, groupExpression, Optional.of(getLogicalProperties()), hint); } @Override @@ -117,6 +119,10 @@ public List getMarkJoinConjuncts() { return ExpressionUtils.EMPTY_CONDITION; } + public Optional getMatchCondition() { + return matchCondition; + } + @Override public boolean equals(Object o) { if (this == o) { @@ -128,12 +134,13 @@ public boolean equals(Object o) { LogicalUsingJoin usingJoin = (LogicalUsingJoin) o; return joinType == usingJoin.joinType && Objects.equals(usingSlots, usingJoin.usingSlots) + && Objects.equals(matchCondition, usingJoin.matchCondition) && Objects.equals(hint, usingJoin.hint); } @Override public int hashCode() { - return Objects.hash(joinType, usingSlots, hint); + return Objects.hash(joinType, usingSlots, matchCondition, hint); } @Override @@ -141,6 +148,7 @@ public String toString() { List args = Lists.newArrayList( "type", joinType, "usingSlots", usingSlots, + "matchCondition", matchCondition, "stats", statistics); if (hint.distributeType != DistributeType.NONE) { args.add("hint"); @@ -155,6 +163,11 @@ public String toDigest() { sb.append(left().toDigest()); sb.append(" ").append(joinType).append(" "); sb.append(right().toDigest()); + if (matchCondition.isPresent()) { + sb.append("MATCH_CONDITION ("); + sb.append(matchCondition.get().toDigest()); + sb.append(")"); + } sb.append(" USING ("); sb.append( usingSlots.stream().map(Expression::toDigest) diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/util/JoinUtils.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/util/JoinUtils.java index a8bc4a0f15b4b2..9ac05adffad2a3 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/util/JoinUtils.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/util/JoinUtils.java @@ -445,11 +445,13 @@ public static List getJoinOutput(JoinType joinType, Plan left, Plan right, case RIGHT_ANTI_JOIN: return ImmutableList.copyOf(rightOutput); case LEFT_OUTER_JOIN: + case ASOF_LEFT_OUTER_JOIN: return ImmutableList.builder() .addAll(leftOutput) .addAll(applyNullable(rightOutput, true)) .build(); case RIGHT_OUTER_JOIN: + case ASOF_RIGHT_OUTER_JOIN: return ImmutableList.builder() .addAll(applyNullable(leftOutput, true)) .addAll(rightOutput) diff --git a/fe/fe-core/src/main/java/org/apache/doris/planner/NestedLoopJoinNode.java b/fe/fe-core/src/main/java/org/apache/doris/planner/NestedLoopJoinNode.java index 520cbf937cd3f3..90ad4ebfc5aaa0 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/planner/NestedLoopJoinNode.java +++ b/fe/fe-core/src/main/java/org/apache/doris/planner/NestedLoopJoinNode.java @@ -54,7 +54,9 @@ public class NestedLoopJoinNode extends JoinNodeBase { public static boolean canParallelize(JoinOperator joinOp) { return joinOp == JoinOperator.CROSS_JOIN || joinOp == JoinOperator.INNER_JOIN || joinOp == JoinOperator.LEFT_OUTER_JOIN || joinOp == JoinOperator.LEFT_SEMI_JOIN - || joinOp == JoinOperator.LEFT_ANTI_JOIN || joinOp == JoinOperator.NULL_AWARE_LEFT_ANTI_JOIN; + || joinOp == JoinOperator.LEFT_ANTI_JOIN || joinOp == JoinOperator.NULL_AWARE_LEFT_ANTI_JOIN + || joinOp == JoinOperator.ASOF_LEFT_INNER_JOIN || joinOp == JoinOperator.ASOF_RIGHT_INNER_JOIN + || joinOp == JoinOperator.ASOF_LEFT_OUTER_JOIN; } diff --git a/fe/fe-core/src/test/java/org/apache/doris/nereids/parser/NereidsParserTest.java b/fe/fe-core/src/test/java/org/apache/doris/nereids/parser/NereidsParserTest.java index 6b134440ae61df..049b2947c7eb36 100644 --- a/fe/fe-core/src/test/java/org/apache/doris/nereids/parser/NereidsParserTest.java +++ b/fe/fe-core/src/test/java/org/apache/doris/nereids/parser/NereidsParserTest.java @@ -254,6 +254,11 @@ public void testParseJoin() { logicalJoin = (LogicalJoin) logicalPlan.child(0); Assertions.assertEquals(JoinType.INNER_JOIN, logicalJoin.getJoinType()); + String asofInnerJoin = "SELECT t1.a FROM t1 ASOF INNER JOIN t2 MATCH_CONDITION(t1.dt < t2.dt) ON t1.id = t2.id;"; + logicalPlan = (LogicalPlan) nereidsParser.parseSingle(asofInnerJoin).child(0); + logicalJoin = (LogicalJoin) logicalPlan.child(0); + Assertions.assertEquals(JoinType.ASOF_LEFT_INNER_JOIN, logicalJoin.getJoinType()); + String leftJoin1 = "SELECT t1.a FROM t1 LEFT JOIN t2 ON t1.id = t2.id;"; logicalPlan = (LogicalPlan) nereidsParser.parseSingle(leftJoin1).child(0); logicalJoin = (LogicalJoin) logicalPlan.child(0); @@ -264,6 +269,11 @@ public void testParseJoin() { logicalJoin = (LogicalJoin) logicalPlan.child(0); Assertions.assertEquals(JoinType.LEFT_OUTER_JOIN, logicalJoin.getJoinType()); + String asofLeftJoin = "SELECT t1.a FROM t1 ASOF JOIN t2 MATCH_CONDITION(t1.dt < t2.dt) ON t1.id = t2.id;"; + logicalPlan = (LogicalPlan) nereidsParser.parseSingle(asofLeftJoin).child(0); + logicalJoin = (LogicalJoin) logicalPlan.child(0); + Assertions.assertEquals(JoinType.ASOF_LEFT_OUTER_JOIN, logicalJoin.getJoinType()); + String rightJoin1 = "SELECT t1.a FROM t1 RIGHT JOIN t2 ON t1.id = t2.id;"; logicalPlan = (LogicalPlan) nereidsParser.parseSingle(rightJoin1).child(0); logicalJoin = (LogicalJoin) logicalPlan.child(0); diff --git a/fe/fe-core/src/test/java/org/apache/doris/nereids/properties/ChildOutputPropertyDeriverTest.java b/fe/fe-core/src/test/java/org/apache/doris/nereids/properties/ChildOutputPropertyDeriverTest.java index a68479421b78e4..a0e2639771a2b0 100644 --- a/fe/fe-core/src/test/java/org/apache/doris/nereids/properties/ChildOutputPropertyDeriverTest.java +++ b/fe/fe-core/src/test/java/org/apache/doris/nereids/properties/ChildOutputPropertyDeriverTest.java @@ -125,7 +125,13 @@ ConnectContext get() { @Test void testInnerJoin() { - PhysicalHashJoin join = new PhysicalHashJoin<>(JoinType.INNER_JOIN, + testInnerJoinHelper(JoinType.INNER_JOIN); + testInnerJoinHelper(JoinType.ASOF_LEFT_INNER_JOIN); + testInnerJoinHelper(JoinType.ASOF_RIGHT_INNER_JOIN); + } + + private void testInnerJoinHelper(JoinType joinType) { + PhysicalHashJoin join = new PhysicalHashJoin<>(joinType, ExpressionUtils.EMPTY_CONDITION, ExpressionUtils.EMPTY_CONDITION, new DistributeHint(DistributeType.NONE), Optional.empty(), logicalProperties, groupPlan, groupPlan); GroupExpression groupExpression = new GroupExpression(join); @@ -203,7 +209,12 @@ ExpressionUtils.EMPTY_CONDITION, ExpressionUtils.EMPTY_CONDITION, new Distribute @Test void testLeftOuterJoin() { - PhysicalHashJoin join = new PhysicalHashJoin<>(JoinType.LEFT_OUTER_JOIN, + testLeftOuterJoinHelper(JoinType.LEFT_OUTER_JOIN); + testLeftOuterJoinHelper(JoinType.ASOF_LEFT_OUTER_JOIN); + } + + private void testLeftOuterJoinHelper(JoinType joinType) { + PhysicalHashJoin join = new PhysicalHashJoin<>(joinType, ExpressionUtils.EMPTY_CONDITION, ExpressionUtils.EMPTY_CONDITION, new DistributeHint(DistributeType.NONE), Optional.empty(), logicalProperties, groupPlan, groupPlan); GroupExpression groupExpression = new GroupExpression(join); @@ -450,7 +461,12 @@ ExpressionUtils.EMPTY_CONDITION, ExpressionUtils.EMPTY_CONDITION, new Distribute @Test void testRightOuterJoin() { - PhysicalHashJoin join = new PhysicalHashJoin<>(JoinType.RIGHT_OUTER_JOIN, + testRightOuterJoinHelper(JoinType.RIGHT_OUTER_JOIN); + testRightOuterJoinHelper(JoinType.ASOF_RIGHT_OUTER_JOIN); + } + + private void testRightOuterJoinHelper(JoinType joinType) { + PhysicalHashJoin join = new PhysicalHashJoin<>(joinType, ExpressionUtils.EMPTY_CONDITION, ExpressionUtils.EMPTY_CONDITION, new DistributeHint(DistributeType.NONE), Optional.empty(), logicalProperties, groupPlan, groupPlan); GroupExpression groupExpression = new GroupExpression(join); diff --git a/fe/fe-core/src/test/java/org/apache/doris/nereids/rules/analysis/BindExpressionTest.java b/fe/fe-core/src/test/java/org/apache/doris/nereids/rules/analysis/BindExpressionTest.java index 50cad2811ac23a..d2f9dffce37a5f 100644 --- a/fe/fe-core/src/test/java/org/apache/doris/nereids/rules/analysis/BindExpressionTest.java +++ b/fe/fe-core/src/test/java/org/apache/doris/nereids/rules/analysis/BindExpressionTest.java @@ -49,7 +49,7 @@ void testJoin() { sql = String.format("select * from t1 %s t2", joinType.toString().replace("_", " ")); } - if (joinType.isNullAwareLeftAntiJoin()) { + if (joinType.isNullAwareLeftAntiJoin() || joinType.isAsofJoin()) { continue; } PlanChecker.from(connectContext) diff --git a/fe/fe-core/src/test/java/org/apache/doris/nereids/rules/analysis/BindSlotReferenceTest.java b/fe/fe-core/src/test/java/org/apache/doris/nereids/rules/analysis/BindSlotReferenceTest.java index 63fa700f23c51a..abb41a6138c7d3 100644 --- a/fe/fe-core/src/test/java/org/apache/doris/nereids/rules/analysis/BindSlotReferenceTest.java +++ b/fe/fe-core/src/test/java/org/apache/doris/nereids/rules/analysis/BindSlotReferenceTest.java @@ -47,6 +47,7 @@ import org.junit.jupiter.api.Test; import java.util.List; +import java.util.Optional; class BindSlotReferenceTest implements MemoPatternMatchSupported { @@ -161,12 +162,12 @@ public void testBindUsingJoinFromLeftToRight() { DistributeHint hint = new DistributeHint(DistributeType.NONE); LogicalUsingJoin, LogicalSubQueryAlias> using1 = new LogicalUsingJoin<>(JoinType.LEFT_OUTER_JOIN, sub1, - sub2, ImmutableList.of(new UnboundSlot("id")), hint); + sub2, ImmutableList.of(new UnboundSlot("id")), Optional.empty(), hint); LogicalUsingJoin, LogicalSubQueryAlias>, LogicalSubQueryAlias> using2 = new LogicalUsingJoin<>( JoinType.LEFT_OUTER_JOIN, using1, sub3, - ImmutableList.of(new UnboundSlot("id")), hint); + ImmutableList.of(new UnboundSlot("id")), Optional.empty(), hint); PlanChecker.from(MemoTestUtils.createConnectContext()) .analyze(using2) diff --git a/fe/fe-core/src/test/java/org/apache/doris/nereids/rules/exploration/join/JoinCommuteTest.java b/fe/fe-core/src/test/java/org/apache/doris/nereids/rules/exploration/join/JoinCommuteTest.java index f0774d9b5e6a0e..84045fc2d33e40 100644 --- a/fe/fe-core/src/test/java/org/apache/doris/nereids/rules/exploration/join/JoinCommuteTest.java +++ b/fe/fe-core/src/test/java/org/apache/doris/nereids/rules/exploration/join/JoinCommuteTest.java @@ -43,11 +43,17 @@ public class JoinCommuteTest implements MemoPatternMatchSupported { @Test void testInnerJoinCommute() { + testInnerJoinCommuteHelper(JoinType.INNER_JOIN); + testInnerJoinCommuteHelper(JoinType.ASOF_LEFT_OUTER_JOIN); + testInnerJoinCommuteHelper(JoinType.ASOF_RIGHT_INNER_JOIN); + } + + private void testInnerJoinCommuteHelper(JoinType joinType) { LogicalOlapScan scan1 = PlanConstructor.newLogicalOlapScan(0, "t1", 0); LogicalOlapScan scan2 = PlanConstructor.newLogicalOlapScan(1, "t2", 0); LogicalPlan join = new LogicalPlanBuilder(scan1) - .join(scan2, JoinType.INNER_JOIN, Pair.of(0, 0)) // t1.id = t2.id + .join(scan2, joinType, Pair.of(0, 0)) // t1.id = t2.id .build(); PlanChecker.from(MemoTestUtils.createConnectContext(), join) @@ -65,11 +71,16 @@ void testInnerJoinCommute() { @Test void testParallelJoinCommute() { + testParallelJoinCommuteHelper(JoinType.LEFT_OUTER_JOIN); + testParallelJoinCommuteHelper(JoinType.ASOF_LEFT_OUTER_JOIN); + } + + private void testParallelJoinCommuteHelper(JoinType joinType) { LogicalOlapScan scan1 = PlanConstructor.newLogicalOlapScan(0, "t1", 0); LogicalOlapScan scan2 = PlanConstructor.newLogicalOlapScan(1, "t2", 0); LogicalJoin join = (LogicalJoin) new LogicalPlanBuilder(scan1) - .join(scan2, JoinType.LEFT_OUTER_JOIN, Pair.of(0, 0)) + .join(scan2, joinType, Pair.of(0, 0)) .build(); join = join.withJoinConjuncts( ImmutableList.of(), @@ -83,11 +94,16 @@ void testParallelJoinCommute() { @Test void testJoinConjunctNullableWhenCommute() { + testJoinConjunctNullableWhenCommuteHelper(JoinType.LEFT_OUTER_JOIN, JoinType.RIGHT_OUTER_JOIN); + testJoinConjunctNullableWhenCommuteHelper(JoinType.ASOF_LEFT_OUTER_JOIN, JoinType.ASOF_RIGHT_OUTER_JOIN); + } + + private void testJoinConjunctNullableWhenCommuteHelper(JoinType joinType1, JoinType joinType2) { LogicalOlapScan scan1 = PlanConstructor.newLogicalOlapScan(0, "t1", 0); LogicalOlapScan scan2 = PlanConstructor.newLogicalOlapScan(1, "t2", 0); LogicalJoin join = (LogicalJoin) new LogicalPlanBuilder(scan1) - .join(scan2, JoinType.LEFT_OUTER_JOIN, Pair.of(0, 0)) + .join(scan2, joinType1, Pair.of(0, 0)) .build(); join = join.withJoinConjuncts( ImmutableList.of(new EqualTo(scan1.getOutput().get(0).withNullable(true), @@ -101,7 +117,7 @@ void testJoinConjunctNullableWhenCommute() { .getAllPlan() .stream() .filter(plan -> plan instanceof LogicalJoin - && ((LogicalJoin) plan).getJoinType() == JoinType.RIGHT_OUTER_JOIN) + && ((LogicalJoin) plan).getJoinType() == joinType2) .collect(Collectors.toList()); Assertions.assertEquals(1, allPlan.size()); // the input slot of join conjuncts should be nullable false after commute diff --git a/fe/fe-core/src/test/java/org/apache/doris/nereids/rules/exploration/join/OuterJoinAsscomProjectTest.java b/fe/fe-core/src/test/java/org/apache/doris/nereids/rules/exploration/join/OuterJoinAsscomProjectTest.java index c5d805953b832c..60d75dadfa2896 100644 --- a/fe/fe-core/src/test/java/org/apache/doris/nereids/rules/exploration/join/OuterJoinAsscomProjectTest.java +++ b/fe/fe-core/src/test/java/org/apache/doris/nereids/rules/exploration/join/OuterJoinAsscomProjectTest.java @@ -49,13 +49,18 @@ public class OuterJoinAsscomProjectTest { @Test public void testJoinConjunctNullableWhenAssociate() { + testJoinConjunctNullableWhenAssociateHelper(JoinType.LEFT_OUTER_JOIN); + testJoinConjunctNullableWhenAssociateHelper(JoinType.ASOF_LEFT_OUTER_JOIN); + } + + public void testJoinConjunctNullableWhenAssociateHelper(JoinType joinType) { // t1 left outer join t2 List bottomHashJoinConjunct = ImmutableList.of( new EqualTo(scan1.getOutput().get(0), scan2.getOutput().get(0))); List bottomOtherJoinConjunct = ImmutableList.of( new GreaterThan(scan1.getOutput().get(1), scan2.getOutput().get(1))); LogicalPlan bottomJoin = new LogicalPlanBuilder(scan1) - .join(scan2, JoinType.LEFT_OUTER_JOIN, bottomHashJoinConjunct, bottomOtherJoinConjunct) + .join(scan2, joinType, bottomHashJoinConjunct, bottomOtherJoinConjunct) .build(); LogicalPlan bottomProject = new LogicalProject<>( bottomJoin.getOutput().stream().map(NamedExpression.class::cast).collect(Collectors.toList()), @@ -67,11 +72,11 @@ public void testJoinConjunctNullableWhenAssociate() { List topOtherJoinConjunct = ImmutableList.of( new GreaterThan(bottomProject.getOutput().get(3).withNullable(true), scan3.getOutput().get(1))); LogicalPlan topJoin = new LogicalPlanBuilder(bottomProject) - .join(scan3, JoinType.LEFT_OUTER_JOIN, topHashJoinConjunct, topOtherJoinConjunct) + .join(scan3, joinType, topHashJoinConjunct, topOtherJoinConjunct) .build(); LogicalPlan plan = new LogicalProject<>( topJoin.getOutput().stream().map(NamedExpression.class::cast).collect( - Collectors.toList()), topJoin); + Collectors.toList()), topJoin); List allPlan = PlanChecker.from(MemoTestUtils.createConnectContext(), plan) .printlnOrigin() diff --git a/fe/fe-core/src/test/java/org/apache/doris/nereids/rules/exploration/join/OuterJoinLAsscomProjectTest.java b/fe/fe-core/src/test/java/org/apache/doris/nereids/rules/exploration/join/OuterJoinLAsscomProjectTest.java index e44d8a19762060..fb701767b3b266 100644 --- a/fe/fe-core/src/test/java/org/apache/doris/nereids/rules/exploration/join/OuterJoinLAsscomProjectTest.java +++ b/fe/fe-core/src/test/java/org/apache/doris/nereids/rules/exploration/join/OuterJoinLAsscomProjectTest.java @@ -52,10 +52,15 @@ class OuterJoinLAsscomProjectTest implements MemoPatternMatchSupported { @Test void testJoinLAsscomProject() { + testJoinLAsscomProjectHelper(JoinType.LEFT_OUTER_JOIN); + testJoinLAsscomProjectHelper(JoinType.ASOF_LEFT_OUTER_JOIN); + } + + private void testJoinLAsscomProjectHelper(JoinType joinType) { LogicalPlan plan = new LogicalPlanBuilder(scan1) - .join(scan2, JoinType.LEFT_OUTER_JOIN, Pair.of(0, 0)) + .join(scan2, joinType, Pair.of(0, 0)) .project(ImmutableList.of(0, 1, 2)) - .join(scan3, JoinType.LEFT_OUTER_JOIN, Pair.of(1, 1)) + .join(scan3, joinType, Pair.of(1, 1)) .projectAll() .build(); @@ -64,17 +69,17 @@ void testJoinLAsscomProject() { .applyExploration(OuterJoinLAsscomProject.INSTANCE.build()) .printlnExploration() .matchesExploration( - logicalProject( - logicalJoin( - logicalProject(logicalJoin( - logicalOlapScan().when(scan -> scan.getTable().getName().equals("t1")), - logicalOlapScan().when(scan -> scan.getTable().getName().equals("t3")) - )), - logicalProject( - logicalOlapScan().when(scan -> scan.getTable().getName().equals("t2")) - ).when(project -> project.getProjects().size() == 1) + logicalProject( + logicalJoin( + logicalProject(logicalJoin( + logicalOlapScan().when(scan -> scan.getTable().getName().equals("t1")), + logicalOlapScan().when(scan -> scan.getTable().getName().equals("t3")) + )), + logicalProject( + logicalOlapScan().when(scan -> scan.getTable().getName().equals("t2")) + ).when(project -> project.getProjects().size() == 1) + ) ) - ) ); } @@ -87,35 +92,68 @@ void testAlias() { .projectAll() .build(); + LogicalPlan plan2 = new LogicalPlanBuilder(scan1) + .join(scan2, JoinType.ASOF_LEFT_OUTER_JOIN, Pair.of(0, 0)) + .alias(ImmutableList.of(0, 2), ImmutableList.of("t1.id", "t2.id")) + .join(scan3, JoinType.ASOF_LEFT_OUTER_JOIN, Pair.of(0, 0)) + .projectAll() + .build(); + PlanChecker.from(MemoTestUtils.createConnectContext(), plan) .applyTopDown(new PushDownAliasThroughJoin()) .printlnTree() .applyExploration(OuterJoinLAsscomProject.INSTANCE.build()) .printlnExploration() .matchesExploration( - logicalProject( - logicalJoin( - logicalProject( + logicalProject( logicalJoin( - logicalProject(logicalOlapScan().when(scan -> scan.getTable().getName().equals("t1"))), - logicalOlapScan().when(scan -> scan.getTable().getName().equals("t3")) + logicalProject( + logicalJoin( + logicalProject(logicalOlapScan().when(scan -> scan.getTable().getName().equals("t1"))), + logicalOlapScan().when(scan -> scan.getTable().getName().equals("t3")) + ) + ).when(project -> project.getProjects().size() == 3), // t1.id Add t3.id, t3.name + logicalProject( + logicalProject(logicalOlapScan().when(scan -> scan.getTable().getName().equals("t2"))) + ).when(project -> project.getProjects().size() == 1) + ) + ) + ); + + PlanChecker.from(MemoTestUtils.createConnectContext(), plan2) + .applyTopDown(new PushDownAliasThroughJoin()) + .printlnTree() + .applyExploration(OuterJoinLAsscomProject.INSTANCE.build()) + .printlnExploration() + .matchesExploration( + logicalProject( + logicalJoin( + logicalProject( + logicalJoin( + logicalProject(logicalOlapScan().when(scan -> scan.getTable().getName().equals("t1"))), + logicalOlapScan().when(scan -> scan.getTable().getName().equals("t3")) + ) + ).when(project -> project.getProjects().size() == 3), // t1.id Add t3.id, t3.name + logicalProject( + logicalProject(logicalOlapScan().when(scan -> scan.getTable().getName().equals("t2"))) + ).when(project -> project.getProjects().size() == 1) ) - ).when(project -> project.getProjects().size() == 3), // t1.id Add t3.id, t3.name - logicalProject( - logicalProject(logicalOlapScan().when(scan -> scan.getTable().getName().equals("t2"))) - ).when(project -> project.getProjects().size() == 1) ) - ) ); } @Test void testAliasTopMultiHashJoin() { + testAliasTopMultiHashJoinHelper(JoinType.LEFT_OUTER_JOIN); + testAliasTopMultiHashJoinHelper(JoinType.ASOF_LEFT_OUTER_JOIN); + } + + private void testAliasTopMultiHashJoinHelper(JoinType joinType) { LogicalPlan plan = new LogicalPlanBuilder(scan1) - .join(scan2, JoinType.LEFT_OUTER_JOIN, Pair.of(0, 0)) // t1.id=t2.id + .join(scan2, joinType, Pair.of(0, 0)) // t1.id=t2.id .alias(ImmutableList.of(0, 2), ImmutableList.of("t1.id", "t2.id")) // t1.id=t3.id t2.id = t3.id - .join(scan3, JoinType.LEFT_OUTER_JOIN, ImmutableList.of(Pair.of(0, 0), Pair.of(1, 0))) + .join(scan3, joinType, ImmutableList.of(Pair.of(0, 0), Pair.of(1, 0))) .build(); PlanChecker.from(MemoTestUtils.createConnectContext(), plan) @@ -126,8 +164,13 @@ void testAliasTopMultiHashJoin() { @Test void testAliasTopMultiHashJoinLeftOuterInner() { + testAliasTopMultiHashJoinLeftOuterInnerHelper(JoinType.LEFT_OUTER_JOIN); + testAliasTopMultiHashJoinLeftOuterInnerHelper(JoinType.ASOF_LEFT_OUTER_JOIN); + } + + private void testAliasTopMultiHashJoinLeftOuterInnerHelper(JoinType joinType) { LogicalPlan plan = new LogicalPlanBuilder(scan1) - .join(scan2, JoinType.LEFT_OUTER_JOIN, Pair.of(0, 0)) // t1.id=t2.id + .join(scan2, joinType, Pair.of(0, 0)) // t1.id=t2.id .alias(ImmutableList.of(0, 2), ImmutableList.of("t1.id", "t2.id")) // t1.id=t3.id t2.id=t3.id .join(scan3, JoinType.INNER_JOIN, ImmutableList.of(Pair.of(0, 0), Pair.of(1, 0))) @@ -141,12 +184,17 @@ void testAliasTopMultiHashJoinLeftOuterInner() { @Test public void testHashAndOther() { + testHashAndOtherHelper(JoinType.LEFT_OUTER_JOIN); + testHashAndOtherHelper(JoinType.ASOF_LEFT_OUTER_JOIN); + } + + private void testHashAndOtherHelper(JoinType joinType) { List bottomHashJoinConjunct = ImmutableList.of( new EqualTo(scan1.getOutput().get(0), scan2.getOutput().get(0))); List bottomOtherJoinConjunct = ImmutableList.of( new GreaterThan(scan1.getOutput().get(1), scan2.getOutput().get(1))); LogicalPlan project = new LogicalPlanBuilder(scan1) - .join(scan2, JoinType.LEFT_OUTER_JOIN, bottomHashJoinConjunct, bottomOtherJoinConjunct) + .join(scan2, joinType, bottomHashJoinConjunct, bottomOtherJoinConjunct) .alias(ImmutableList.of(0, 1, 2, 3), ImmutableList.of("t1.id", "t1.name", "t2.id", "t2.name")) .build(); @@ -157,7 +205,7 @@ public void testHashAndOther() { new GreaterThan(project.getOutput().get(1), scan3.getOutput().get(1)), new GreaterThan(project.getOutput().get(3), scan3.getOutput().get(1))); LogicalPlan plan = new LogicalPlanBuilder(project) - .join(scan3, JoinType.LEFT_OUTER_JOIN, topHashJoinConjunct, topOtherJoinConjunct) + .join(scan3, joinType, topHashJoinConjunct, topOtherJoinConjunct) .build(); PlanChecker.from(MemoTestUtils.createConnectContext(), plan) @@ -170,13 +218,18 @@ public void testHashAndOther() { @Test public void testJoinConjunctNullableWhenLAssociate() { + testJoinConjunctNullableWhenLAssociateHelper(JoinType.LEFT_OUTER_JOIN); + testJoinConjunctNullableWhenLAssociateHelper(JoinType.ASOF_LEFT_OUTER_JOIN); + } + + private void testJoinConjunctNullableWhenLAssociateHelper(JoinType joinType) { // t1 left outer join t2 List bottomHashJoinConjunct = ImmutableList.of( new EqualTo(scan1.getOutput().get(0), scan2.getOutput().get(0))); List bottomOtherJoinConjunct = ImmutableList.of( new GreaterThan(scan1.getOutput().get(1), scan2.getOutput().get(1))); LogicalPlan bottomJoin = new LogicalPlanBuilder(scan1) - .join(scan2, JoinType.LEFT_OUTER_JOIN, bottomHashJoinConjunct, bottomOtherJoinConjunct) + .join(scan2, joinType, bottomHashJoinConjunct, bottomOtherJoinConjunct) .build(); LogicalPlan bottomProject = new LogicalProject<>( bottomJoin.getOutput().stream().map(NamedExpression.class::cast).collect(Collectors.toList()), @@ -188,7 +241,7 @@ public void testJoinConjunctNullableWhenLAssociate() { List topOtherJoinConjunct = ImmutableList.of( new GreaterThan(bottomProject.getOutput().get(1), scan3.getOutput().get(1))); LogicalPlan topJoin = new LogicalPlanBuilder(bottomProject) - .join(scan3, JoinType.LEFT_OUTER_JOIN, topHashJoinConjunct, topOtherJoinConjunct) + .join(scan3, joinType, topHashJoinConjunct, topOtherJoinConjunct) .build(); LogicalPlan plan = new LogicalProject<>( diff --git a/fe/fe-core/src/test/java/org/apache/doris/nereids/rules/exploration/join/PushDownProjectThroughInnerOuterJoinTest.java b/fe/fe-core/src/test/java/org/apache/doris/nereids/rules/exploration/join/PushDownProjectThroughInnerOuterJoinTest.java index e5ba37b3bd283f..4c1370da1c03bd 100644 --- a/fe/fe-core/src/test/java/org/apache/doris/nereids/rules/exploration/join/PushDownProjectThroughInnerOuterJoinTest.java +++ b/fe/fe-core/src/test/java/org/apache/doris/nereids/rules/exploration/join/PushDownProjectThroughInnerOuterJoinTest.java @@ -30,6 +30,7 @@ import org.apache.doris.nereids.util.MemoTestUtils; import org.apache.doris.nereids.util.PlanChecker; import org.apache.doris.nereids.util.PlanConstructor; +import org.apache.doris.qe.ConnectContext; import com.google.common.collect.ImmutableList; import org.junit.jupiter.api.Assertions; @@ -42,8 +43,16 @@ class PushDownProjectThroughInnerOuterJoinTest implements MemoPatternMatchSuppor private final LogicalOlapScan scan2 = PlanConstructor.newLogicalOlapScan(1, "t2", 0); private final LogicalOlapScan scan3 = PlanConstructor.newLogicalOlapScan(2, "t3", 0); + private ConnectContext connectContext = MemoTestUtils.createConnectContext(); + @Test public void pushBothSide() { + pushBothSideHelper(JoinType.INNER_JOIN); + pushBothSideHelper(JoinType.ASOF_LEFT_INNER_JOIN); + pushBothSideHelper(JoinType.ASOF_RIGHT_INNER_JOIN); + } + + private void pushBothSideHelper(JoinType joinType) { // project (t1.id + 1) as alias, t1.name, (t2.id + 1) as alias, t2.name List projectExprs = ImmutableList.of( new Alias(new Add(scan1.getOutput().get(0), Literal.of(1)), "alias"), @@ -53,12 +62,12 @@ public void pushBothSide() { ); // complex projection contain ti.id, which isn't in Join Condition LogicalPlan plan = new LogicalPlanBuilder(scan1) - .join(scan2, JoinType.INNER_JOIN, Pair.of(1, 1)) + .join(scan2, joinType, Pair.of(1, 1)) .projectExprs(projectExprs) - .join(scan3, JoinType.INNER_JOIN, Pair.of(1, 1)) + .join(scan3, joinType, Pair.of(1, 1)) .build(); - PlanChecker.from(MemoTestUtils.createConnectContext(), plan) + PlanChecker.from(connectContext, plan) .applyExploration(PushDownProjectThroughInnerOuterJoin.INSTANCE.buildRules()) .printlnOrigin() .printlnExploration() @@ -75,6 +84,15 @@ public void pushBothSide() { @Test public void pushRightSide() { + pushRightSideHelper(JoinType.LEFT_OUTER_JOIN, JoinType.INNER_JOIN); + pushRightSideHelper(JoinType.LEFT_OUTER_JOIN, JoinType.ASOF_LEFT_INNER_JOIN); + pushRightSideHelper(JoinType.LEFT_OUTER_JOIN, JoinType.ASOF_RIGHT_INNER_JOIN); + pushRightSideHelper(JoinType.ASOF_LEFT_OUTER_JOIN, JoinType.INNER_JOIN); + pushRightSideHelper(JoinType.ASOF_LEFT_OUTER_JOIN, JoinType.ASOF_LEFT_INNER_JOIN); + pushRightSideHelper(JoinType.ASOF_LEFT_OUTER_JOIN, JoinType.ASOF_RIGHT_INNER_JOIN); + } + + private void pushRightSideHelper(JoinType bottom, JoinType top) { // project (t1.id + 1) as alias, t1.name, (t2.id + 1) as alias, t2.name List projectExprs = ImmutableList.of( new Alias(new Add(scan1.getOutput().get(0), Literal.of(1)), "alias"), @@ -83,22 +101,22 @@ public void pushRightSide() { ); // complex projection contain ti.id, which isn't in Join Condition LogicalPlan plan = new LogicalPlanBuilder(scan1) - .join(scan2, JoinType.LEFT_OUTER_JOIN, Pair.of(1, 1)) + .join(scan2, bottom, Pair.of(1, 1)) .projectExprs(projectExprs) - .join(scan3, JoinType.INNER_JOIN, Pair.of(1, 1)) + .join(scan3, top, Pair.of(1, 1)) .build(); - PlanChecker.from(MemoTestUtils.createConnectContext(), plan) + PlanChecker.from(connectContext, plan) .applyExploration(PushDownProjectThroughInnerOuterJoin.INSTANCE.buildRules()) .printlnOrigin() .printlnExploration() .matchesExploration( logicalJoin( logicalProject( - logicalJoin( - logicalProject().when(project -> project.getProjects().size() == 2), - logicalOlapScan() - ) + logicalJoin( + logicalProject().when(project -> project.getProjects().size() == 2), + logicalOlapScan() + ) ), logicalOlapScan() ) @@ -107,6 +125,12 @@ public void pushRightSide() { @Test public void pushNoSide() { + pushNoSideHelper(JoinType.FULL_OUTER_JOIN, JoinType.INNER_JOIN); + pushNoSideHelper(JoinType.FULL_OUTER_JOIN, JoinType.ASOF_LEFT_INNER_JOIN); + pushNoSideHelper(JoinType.FULL_OUTER_JOIN, JoinType.ASOF_RIGHT_INNER_JOIN); + } + + private void pushNoSideHelper(JoinType bottom, JoinType top) { // project (t1.id + 1) as alias, t1.name, (t2.id + 1) as alias, t2.name List projectExprs = ImmutableList.of( new Alias(new Add(scan1.getOutput().get(0), Literal.of(1)), "alias"), @@ -115,12 +139,12 @@ public void pushNoSide() { ); // complex projection contain ti.id, which isn't in Join Condition LogicalPlan plan = new LogicalPlanBuilder(scan1) - .join(scan2, JoinType.FULL_OUTER_JOIN, Pair.of(1, 1)) + .join(scan2, bottom, Pair.of(1, 1)) .projectExprs(projectExprs) - .join(scan3, JoinType.INNER_JOIN, Pair.of(1, 1)) + .join(scan3, top, Pair.of(1, 1)) .build(); - int plansNumber = PlanChecker.from(MemoTestUtils.createConnectContext(), plan) + int plansNumber = PlanChecker.from(connectContext, plan) .applyExploration(PushDownProjectThroughInnerOuterJoin.INSTANCE.buildRules()) .plansNumber(); Assertions.assertEquals(1, plansNumber); @@ -128,6 +152,12 @@ public void pushNoSide() { @Test public void pushdownProjectInCondition() { + pushdownProjectInConditionHelper(JoinType.INNER_JOIN); + pushdownProjectInConditionHelper(JoinType.ASOF_LEFT_INNER_JOIN); + pushdownProjectInConditionHelper(JoinType.ASOF_RIGHT_INNER_JOIN); + } + + private void pushdownProjectInConditionHelper(JoinType joinType) { // project (t1.id + 1) as alias, t1.name, (t2.id + 1) as alias, t2.name List projectExprs = ImmutableList.of( new Alias(new Add(scan1.getOutput().get(0), Literal.of(1)), "alias"), @@ -137,12 +167,12 @@ public void pushdownProjectInCondition() { ); // complex projection contain ti.id, which is in Join Condition LogicalPlan plan = new LogicalPlanBuilder(scan1) - .join(scan2, JoinType.INNER_JOIN, Pair.of(0, 0)) + .join(scan2, joinType, Pair.of(0, 0)) .projectExprs(projectExprs) - .join(scan3, JoinType.INNER_JOIN, Pair.of(1, 1)) + .join(scan3, joinType, Pair.of(1, 1)) .build(); - PlanChecker.from(MemoTestUtils.createConnectContext(), plan) + PlanChecker.from(connectContext, plan) .applyExploration(PushDownProjectThroughInnerOuterJoin.INSTANCE.buildRules()) .printlnOrigin() .printlnExploration() @@ -161,6 +191,12 @@ public void pushdownProjectInCondition() { @Test void pushComplexProject() { + pushComplexProjectHelper(JoinType.INNER_JOIN); + pushComplexProjectHelper(JoinType.ASOF_LEFT_INNER_JOIN); + pushComplexProjectHelper(JoinType.ASOF_RIGHT_INNER_JOIN); + } + + private void pushComplexProjectHelper(JoinType joinType) { // project (t1.id + t1.name) as complex1, (t2.id + t2.name) as complex2 List projectExprs = ImmutableList.of( new Alias(new Add(scan1.getOutput().get(0), scan1.getOutput().get(1)), "complex1"), @@ -168,49 +204,55 @@ void pushComplexProject() { ); // complex projection contain ti.id, which is in Join Condition LogicalPlan plan = new LogicalPlanBuilder(scan1) - .join(scan2, JoinType.INNER_JOIN, Pair.of(0, 0)) + .join(scan2, joinType, Pair.of(0, 0)) .projectExprs(projectExprs) - .join(scan3, JoinType.INNER_JOIN, Pair.of(0, 0)) + .join(scan3, joinType, Pair.of(0, 0)) .build(); - PlanChecker.from(MemoTestUtils.createConnectContext(), plan) + PlanChecker.from(connectContext, plan) .applyExploration(PushDownProjectThroughInnerOuterJoin.INSTANCE.buildRules()) .printlnOrigin() .printlnExploration() .matchesExploration( - logicalJoin( - logicalProject( - logicalJoin( - logicalProject() - .when(project -> - project.getProjects().get(0).toSql().equals("(id + name) AS `complex1`") - && project.getProjects().get(1).toSql().equals("id")), - logicalProject() - .when(project -> - project.getProjects().get(0).toSql().equals("(id + name) AS `complex2`") - && project.getProjects().get(1).toSql().equals("id")) - ) - ).when(project -> project.getProjects().get(0).toSql().equals("complex1") - && project.getProjects().get(1).toSql().equals("complex2")), - logicalOlapScan() - ) + logicalJoin( + logicalProject( + logicalJoin( + logicalProject() + .when(project -> + project.getProjects().get(0).toSql().equals("(id + name) AS `complex1`") + && project.getProjects().get(1).toSql().equals("id")), + logicalProject() + .when(project -> + project.getProjects().get(0).toSql().equals("(id + name) AS `complex2`") + && project.getProjects().get(1).toSql().equals("id")) + ) + ).when(project -> project.getProjects().get(0).toSql().equals("complex1") + && project.getProjects().get(1).toSql().equals("complex2")), + logicalOlapScan() + ) ); } @Test void rejectHyperEdgeProject() { + rejectHyperEdgeProjectHelper(JoinType.INNER_JOIN); + rejectHyperEdgeProjectHelper(JoinType.ASOF_LEFT_INNER_JOIN); + rejectHyperEdgeProjectHelper(JoinType.ASOF_RIGHT_INNER_JOIN); + } + + private void rejectHyperEdgeProjectHelper(JoinType joinType) { // project (t1.id + t2.id) as alias List projectExprs = ImmutableList.of( new Alias(new Add(scan1.getOutput().get(0), scan2.getOutput().get(0)), "alias") ); // complex projection contain ti.id, which is in Join Condition LogicalPlan plan = new LogicalPlanBuilder(scan1) - .join(scan2, JoinType.INNER_JOIN, Pair.of(0, 0)) + .join(scan2, joinType, Pair.of(0, 0)) .projectExprs(projectExprs) - .join(scan3, JoinType.INNER_JOIN, Pair.of(0, 0)) + .join(scan3, joinType, Pair.of(0, 0)) .build(); - PlanChecker.from(MemoTestUtils.createConnectContext(), plan) + PlanChecker.from(connectContext, plan) .applyExploration(PushDownProjectThroughInnerOuterJoin.INSTANCE.buildRules()) .checkMemo(memo -> Assertions.assertEquals(1, memo.getRoot().getLogicalExpressions().size())); } diff --git a/fe/fe-core/src/test/java/org/apache/doris/nereids/rules/rewrite/ConvertOuterJoinToAntiJoinTest.java b/fe/fe-core/src/test/java/org/apache/doris/nereids/rules/rewrite/ConvertOuterJoinToAntiJoinTest.java index 6ecb50e53ff3cf..229459d7824046 100644 --- a/fe/fe-core/src/test/java/org/apache/doris/nereids/rules/rewrite/ConvertOuterJoinToAntiJoinTest.java +++ b/fe/fe-core/src/test/java/org/apache/doris/nereids/rules/rewrite/ConvertOuterJoinToAntiJoinTest.java @@ -81,6 +81,26 @@ void testEliminateRightWithProject() { .matches(logicalJoin().when(join -> join.getJoinType().isRightAntiJoin())); } + @Test + void testNoEliminateAsofWithProject() { + testNoEliminateAsofWithProjectHelper(JoinType.ASOF_LEFT_OUTER_JOIN); + testNoEliminateAsofWithProjectHelper(JoinType.ASOF_RIGHT_OUTER_JOIN); + } + + private void testNoEliminateAsofWithProjectHelper(JoinType joinType) { + LogicalPlan plan = new LogicalPlanBuilder(scan1) + .join(scan2, joinType, Pair.of(0, 0)) // t1.id = t2.id + .filter(new IsNull(scan2.getOutput().get(0))) + .projectExprs(ImmutableList.copyOf(scan1.getOutput())) + .build(); + + PlanChecker.from(MemoTestUtils.createConnectContext(), plan) + .applyTopDown(new InferFilterNotNull()) + .applyCustom(new ConvertOuterJoinToAntiJoin()) + .printlnTree() + .matches(logicalJoin().when(join -> join.getJoinType() == joinType)); + } + @Test void testEliminateLeftWithLeftPredicate() { LogicalPlan plan = new LogicalPlanBuilder(scan1) @@ -99,6 +119,29 @@ void testEliminateLeftWithLeftPredicate() { .matches(logicalJoin().when(join -> join.getJoinType().isLeftAntiJoin())); } + @Test + void testNoEliminateAsofWithLeftPredicate() { + testNoEliminateAsofWithLeftPredicateHelper(JoinType.ASOF_LEFT_OUTER_JOIN); + testNoEliminateAsofWithLeftPredicateHelper(JoinType.ASOF_RIGHT_OUTER_JOIN); + } + + private void testNoEliminateAsofWithLeftPredicateHelper(JoinType joinType) { + LogicalPlan plan = new LogicalPlanBuilder(scan1) + .join(scan2, joinType, Pair.of(0, 0)) // t1.id = t2.id + .filter(Sets.newHashSet( + new IsNull(scan2.getOutput().get(0)), + new EqualTo(scan1.getOutput().get(0), new IntegerLiteral(1))) + ) + .projectExprs(ImmutableList.copyOf(scan1.getOutput())) + .build(); + + PlanChecker.from(MemoTestUtils.createConnectContext(), plan) + .applyTopDown(new InferFilterNotNull()) + .applyCustom(new ConvertOuterJoinToAntiJoin()) + .printlnTree() + .matches(logicalJoin().when(join -> join.getJoinType() == joinType)); + } + @Test void testEliminateLeftWithRightPredicate() { LogicalPlan plan = new LogicalPlanBuilder(scan1) @@ -117,10 +160,39 @@ void testEliminateLeftWithRightPredicate() { .matches(logicalJoin().when(join -> join.getJoinType().isLeftAntiJoin())); } + @Test + void testNoEliminateAsofWithRightPredicate() { + testNoEliminateAsofWithRightPredicateHelper(JoinType.ASOF_LEFT_OUTER_JOIN); + testNoEliminateAsofWithRightPredicateHelper(JoinType.ASOF_RIGHT_OUTER_JOIN); + } + + private void testNoEliminateAsofWithRightPredicateHelper(JoinType joinType) { + LogicalPlan plan = new LogicalPlanBuilder(scan1) + .join(scan2, joinType, Pair.of(0, 0)) // t1.id = t2.id + .filter(Sets.newHashSet( + new IsNull(scan2.getOutput().get(0)), + new EqualTo(scan2.getOutput().get(0), new IntegerLiteral(1))) + ) + .projectExprs(ImmutableList.copyOf(scan1.getOutput())) + .build(); + + PlanChecker.from(MemoTestUtils.createConnectContext(), plan) + .applyTopDown(new InferFilterNotNull()) + .applyCustom(new ConvertOuterJoinToAntiJoin()) + .printlnTree() + .matches(logicalJoin().when(join -> join.getJoinType() == joinType)); + } + @Test void testEliminateLeftWithOrPredicate() { + testEliminateLeftWithOrPredicateHelper(JoinType.LEFT_OUTER_JOIN); + testEliminateLeftWithOrPredicateHelper(JoinType.ASOF_LEFT_OUTER_JOIN); + testEliminateLeftWithOrPredicateHelper(JoinType.ASOF_RIGHT_OUTER_JOIN); + } + + private void testEliminateLeftWithOrPredicateHelper(JoinType joinType) { LogicalPlan plan = new LogicalPlanBuilder(scan1) - .join(scan2, JoinType.LEFT_OUTER_JOIN, Pair.of(0, 0)) // t1.id = t2.id + .join(scan2, joinType, Pair.of(0, 0)) // t1.id = t2.id .filter(Sets.newHashSet( new IsNull(scan1.getOutput().get(0)), new Or(new IsNull(scan1.getOutput().get(0)), new IsNull(scan2.getOutput().get(0)))) @@ -132,13 +204,19 @@ void testEliminateLeftWithOrPredicate() { .applyTopDown(new InferFilterNotNull()) .applyCustom(new ConvertOuterJoinToAntiJoin()) .printlnTree() - .matches(logicalJoin().when(join -> join.getJoinType().isLeftOuterJoin())); + .matches(logicalJoin().when(join -> join.getJoinType() == joinType)); } @Test void testEliminateLeftWithAndPredicate() { + testEliminateLeftWithAndPredicateHelper(JoinType.LEFT_OUTER_JOIN); + testEliminateLeftWithAndPredicateHelper(JoinType.ASOF_LEFT_OUTER_JOIN); + testEliminateLeftWithAndPredicateHelper(JoinType.ASOF_RIGHT_OUTER_JOIN); + } + + private void testEliminateLeftWithAndPredicateHelper(JoinType joinType) { LogicalPlan plan = new LogicalPlanBuilder(scan1) - .join(scan2, JoinType.LEFT_OUTER_JOIN, Pair.of(0, 0)) // t1.id = t2.id + .join(scan2, joinType, Pair.of(0, 0)) // t1.id = t2.id .filter(Sets.newHashSet( new IsNull(scan1.getOutput().get(0)), new EqualTo(scan1.getOutput().get(0), new IntegerLiteral(1)), @@ -151,6 +229,6 @@ void testEliminateLeftWithAndPredicate() { .applyTopDown(new InferFilterNotNull()) .applyCustom(new ConvertOuterJoinToAntiJoin()) .printlnTree() - .matches(logicalJoin().when(join -> join.getJoinType().isLeftOuterJoin())); + .matches(logicalJoin().when(join -> join.getJoinType() == joinType)); } } diff --git a/fe/fe-core/src/test/java/org/apache/doris/nereids/rules/rewrite/EliminateJoinByUniqueTest.java b/fe/fe-core/src/test/java/org/apache/doris/nereids/rules/rewrite/EliminateJoinByUniqueTest.java index 8da050a0e477ea..1fd80462d08502 100644 --- a/fe/fe-core/src/test/java/org/apache/doris/nereids/rules/rewrite/EliminateJoinByUniqueTest.java +++ b/fe/fe-core/src/test/java/org/apache/doris/nereids/rules/rewrite/EliminateJoinByUniqueTest.java @@ -32,13 +32,15 @@ protected void runBeforeAll() throws Exception { createTables( "CREATE TABLE IF NOT EXISTS t1 (\n" + " id1 int not null,\n" - + " id_null int\n" + + " id_null int,\n" + + " dt1 date not null\n" + ")\n" + "DUPLICATE KEY(id1)\n" + "DISTRIBUTED BY HASH(id1) BUCKETS 10\n" + "PROPERTIES (\"replication_num\" = \"1\")\n", "CREATE TABLE IF NOT EXISTS t2 (\n" - + " id2 int not null\n" + + " id2 int not null,\n" + + " dt2 date not null\n" + ")\n" + "DUPLICATE KEY(id2)\n" + "DISTRIBUTED BY HASH(id2) BUCKETS 10\n" @@ -51,6 +53,12 @@ protected void runBeforeAll() throws Exception { @Test void testNotNull() throws Exception { String sql = "select t1.id1 from t1 left outer join t2 on t1.id1 = t2.id2"; + PlanChecker.from(connectContext) + .analyze(sql) + .rewrite() + .nonMatch(logicalJoin()) + .printlnTree(); + sql = "select t1.id1 from t1 asof left join t2 MATCH_CONDITION(t1.dt1 > t2.dt2) on t1.id1 = t2.id2"; PlanChecker.from(connectContext) .analyze(sql) .rewrite() @@ -63,11 +71,23 @@ void testNotNull() throws Exception { .rewrite() .matches(logicalJoin()) .printlnTree(); + sql = "select t2.id2 from t1 asof left join t2 MATCH_CONDITION(t1.dt1 > t2.dt2) on t1.id1 = t2.id2"; + PlanChecker.from(connectContext) + .analyze(sql) + .rewrite() + .matches(logicalJoin()) + .printlnTree(); } @Test void testNull() throws Exception { String sql = "select t1.id1 from t1 left outer join t2 on t1.id_null = t2.id2"; + PlanChecker.from(connectContext) + .analyze(sql) + .rewrite() + .nonMatch(logicalJoin()) + .printlnTree(); + sql = "select t1.id1 from t1 asof left join t2 MATCH_CONDITION(t1.dt1 > t2.dt2) on t1.id_null = t2.id2"; PlanChecker.from(connectContext) .analyze(sql) .rewrite() @@ -80,5 +100,11 @@ void testNull() throws Exception { .rewrite() .matches(logicalJoin()) .printlnTree(); + sql = "select t2.id2 from t1 asof left join t2 MATCH_CONDITION(t1.dt1 > t2.dt2) on t1.id_null = t2.id2"; + PlanChecker.from(connectContext) + .analyze(sql) + .rewrite() + .matches(logicalJoin()) + .printlnTree(); } } diff --git a/fe/fe-core/src/test/java/org/apache/doris/nereids/rules/rewrite/EliminateOuterJoinTest.java b/fe/fe-core/src/test/java/org/apache/doris/nereids/rules/rewrite/EliminateOuterJoinTest.java index f0034410163649..51faf9ccaff863 100644 --- a/fe/fe-core/src/test/java/org/apache/doris/nereids/rules/rewrite/EliminateOuterJoinTest.java +++ b/fe/fe-core/src/test/java/org/apache/doris/nereids/rules/rewrite/EliminateOuterJoinTest.java @@ -49,8 +49,13 @@ public EliminateOuterJoinTest() throws Exception { @Test void testEliminateLeft() { + testEliminateLeftHelper(JoinType.LEFT_OUTER_JOIN); + testEliminateLeftHelper(JoinType.ASOF_LEFT_OUTER_JOIN); + } + + private void testEliminateLeftHelper(JoinType joinType) { LogicalPlan plan = new LogicalPlanBuilder(scan1) - .join(scan2, JoinType.LEFT_OUTER_JOIN, Pair.of(0, 0)) // t1.id = t2.id + .join(scan2, joinType, Pair.of(0, 0)) // t1.id = t2.id .filter(new GreaterThan(scan2.getOutput().get(0), Literal.of(1))) .build(); @@ -68,8 +73,13 @@ void testEliminateLeft() { @Test void testEliminateRight() { + testEliminateRightHelper(JoinType.RIGHT_OUTER_JOIN); + testEliminateRightHelper(JoinType.ASOF_RIGHT_OUTER_JOIN); + } + + private void testEliminateRightHelper(JoinType joinType) { LogicalPlan plan = new LogicalPlanBuilder(scan1) - .join(scan2, JoinType.RIGHT_OUTER_JOIN, Pair.of(0, 0)) // t1.id = t2.id + .join(scan2, joinType, Pair.of(0, 0)) // t1.id = t2.id .filter(new GreaterThan(scan1.getOutput().get(0), new IntegerLiteral(1))) .build(); diff --git a/fe/fe-core/src/test/java/org/apache/doris/nereids/rules/rewrite/InferPredicatesTest.java b/fe/fe-core/src/test/java/org/apache/doris/nereids/rules/rewrite/InferPredicatesTest.java index a131ba7f2ddcb8..0feb99edb9bf4d 100644 --- a/fe/fe-core/src/test/java/org/apache/doris/nereids/rules/rewrite/InferPredicatesTest.java +++ b/fe/fe-core/src/test/java/org/apache/doris/nereids/rules/rewrite/InferPredicatesTest.java @@ -52,6 +52,7 @@ protected void runBeforeAll() throws Exception { createTable("create table test.student (\n" + "id int not null,\n" + "name varchar(128),\n" + + "dt date,\n" + "age int,sex int)\n" + "distributed by hash(id) buckets 10\n" + "properties('replication_num' = '1');"); @@ -59,6 +60,7 @@ protected void runBeforeAll() throws Exception { createTable("create table test.score (\n" + "sid int not null, \n" + "cid int not null, \n" + + "dt date,\n" + "grade double)\n" + "distributed by hash(sid,cid) buckets 10\n" + "properties('replication_num' = '1');"); @@ -66,27 +68,28 @@ protected void runBeforeAll() throws Exception { createTable("create table test.course (\n" + "id int not null, \n" + "name varchar(128), \n" + + "dt date,\n" + "teacher varchar(128))\n" + "distributed by hash(id) buckets 10\n" + "properties('replication_num' = '1');"); createTables("create table test.subquery1\n" - + "(k1 bigint, k2 bigint)\n" + + "(k1 bigint, k2 bigint, dt date)\n" + "duplicate key(k1)\n" + "distributed by hash(k2) buckets 1\n" + "properties('replication_num' = '1');\n", "create table test.subquery2\n" - + "(k1 varchar(10), k2 bigint)\n" + + "(k1 varchar(10), k2 bigint, dt date)\n" + "partition by range(k2)\n" + "(partition p1 values less than(\"10\"))\n" + "distributed by hash(k2) buckets 1\n" + "properties('replication_num' = '1');", "create table test.subquery3\n" - + "(k1 int not null, k2 varchar(128), k3 bigint, v1 bigint, v2 bigint)\n" + + "(k1 int not null, k2 varchar(128), k3 bigint, v1 bigint, v2 bigint, dt date)\n" + "distributed by hash(k2) buckets 1\n" + "properties('replication_num' = '1');", "create table test.subquery4\n" - + "(k1 bigint, k2 bigint)\n" + + "(k1 bigint, k2 bigint, dt date)\n" + "duplicate key(k1)\n" + "distributed by hash(k2) buckets 1\n" + "properties('replication_num' = '1');"); @@ -859,4 +862,48 @@ void pullUpPredicatesShouldIgnoreRightSideForCrossJoinMarkJoin() { Assertions.assertTrue(allPredicates.contains(leftPredicate)); Assertions.assertFalse(allPredicates.contains(rightPredicate)); } + + @Test + void inferPredicatesLeftAsofLeft() { + String sql = "select * from student asof left join score match_condition(student.dt > score.dt) on student.id = score.sid where student.id > 1"; + + PlanChecker.from(connectContext) + .analyze(sql) + .rewrite() + .matches( + logicalJoin( + logicalFilter( + logicalOlapScan() + ).when(filter -> !ExpressionUtils.isInferred(filter.getPredicate()) + & filter.getPredicate().toSql().contains("id > 1")), + logicalFilter( + logicalOlapScan() + ).when(filter -> ExpressionUtils.isInferred(filter.getPredicate()) + & filter.getPredicate().toSql().contains("sid > 1")) + ) + ); + } + + @Test + void inferPredicatesLeftAsofInner() { + // convert left join to inner join + String sql = "select * from student asof left join score match_condition(student.dt > score.dt) on student.id = score.sid where score.sid > 1"; + + PlanChecker.from(connectContext) + .analyze(sql) + .rewrite() + .matches( + logicalJoin( + logicalFilter( + logicalOlapScan() + ).when(filter -> ExpressionUtils.isInferred(filter.getPredicate()) + & filter.getPredicate().toSql().contains("id > 1")), + logicalFilter( + logicalOlapScan() + ).when(filter -> !ExpressionUtils.isInferred(filter.getPredicate()) + & filter.getPredicate().toSql().contains("sid > 1")) + ) + ); + } + } diff --git a/fe/fe-core/src/test/java/org/apache/doris/nereids/rules/rewrite/PushDownFilterThroughJoinTest.java b/fe/fe-core/src/test/java/org/apache/doris/nereids/rules/rewrite/PushDownFilterThroughJoinTest.java index 343619cbd8b08a..5d7b03e9088408 100644 --- a/fe/fe-core/src/test/java/org/apache/doris/nereids/rules/rewrite/PushDownFilterThroughJoinTest.java +++ b/fe/fe-core/src/test/java/org/apache/doris/nereids/rules/rewrite/PushDownFilterThroughJoinTest.java @@ -31,6 +31,7 @@ import org.apache.doris.nereids.util.MemoTestUtils; import org.apache.doris.nereids.util.PlanChecker; import org.apache.doris.nereids.util.PlanConstructor; +import org.apache.doris.qe.ConnectContext; import com.google.common.collect.ImmutableList; import com.google.common.collect.ImmutableSet; @@ -49,6 +50,8 @@ public class PushDownFilterThroughJoinTest implements MemoPatternMatchSupported private LogicalPlan rStudent; private LogicalPlan rScore; + private ConnectContext connectContext; + /** * ut before. */ @@ -57,16 +60,21 @@ public final void beforeEach() { rStudent = new LogicalOlapScan(PlanConstructor.getNextRelationId(), PlanConstructor.student, ImmutableList.of("")); rScore = new LogicalOlapScan(PlanConstructor.getNextRelationId(), PlanConstructor.score, ImmutableList.of("")); + connectContext = MemoTestUtils.createConnectContext(); } @Test public void oneSide() { testLeft(JoinType.CROSS_JOIN); testLeft(JoinType.INNER_JOIN); + testLeft(JoinType.ASOF_LEFT_INNER_JOIN); + testLeft(JoinType.ASOF_RIGHT_INNER_JOIN); testLeft(JoinType.LEFT_OUTER_JOIN); + testLeft(JoinType.ASOF_LEFT_OUTER_JOIN); testLeft(JoinType.LEFT_SEMI_JOIN); testLeft(JoinType.LEFT_ANTI_JOIN); testRight(JoinType.RIGHT_OUTER_JOIN); + testRight(JoinType.ASOF_RIGHT_OUTER_JOIN); testRight(JoinType.RIGHT_SEMI_JOIN); testRight(JoinType.RIGHT_ANTI_JOIN); } @@ -81,7 +89,7 @@ private void testLeft(JoinType joinType) { .filter(whereCondition) .build(); - PlanChecker.from(MemoTestUtils.createConnectContext(), plan) + PlanChecker.from(connectContext, plan) .applyTopDown(PushDownFilterThroughJoin.INSTANCE) .matchesFromRoot( logicalJoin( @@ -102,7 +110,7 @@ private void testRight(JoinType joinType) { .filter(whereCondition) .build(); - PlanChecker.from(MemoTestUtils.createConnectContext(), plan) + PlanChecker.from(connectContext, plan) .applyTopDown(PushDownFilterThroughJoin.INSTANCE) .matchesFromRoot( logicalJoin( @@ -133,7 +141,7 @@ private void bothSideToBothSide(JoinType joinType) { .build(); if (joinType.isInnerJoin()) { - PlanChecker.from(MemoTestUtils.createConnectContext(), plan) + PlanChecker.from(connectContext, plan) .applyTopDown(PushDownFilterThroughJoin.INSTANCE) .printlnTree() .matchesFromRoot( @@ -146,7 +154,7 @@ private void bothSideToBothSide(JoinType joinType) { ); } if (joinType.isCrossJoin()) { - PlanChecker.from(MemoTestUtils.createConnectContext(), plan) + PlanChecker.from(connectContext, plan) .applyTopDown(PushDownFilterThroughJoin.INSTANCE) .printlnTree() .matchesFromRoot( @@ -165,9 +173,11 @@ private void bothSideToBothSide(JoinType joinType) { @Test public void bothSideToOneSide() { bothSideToLeft(JoinType.LEFT_OUTER_JOIN); + bothSideToLeft(JoinType.ASOF_LEFT_OUTER_JOIN); bothSideToLeft(JoinType.LEFT_ANTI_JOIN); bothSideToLeft(JoinType.LEFT_SEMI_JOIN); bothSideToRight(JoinType.RIGHT_OUTER_JOIN); + bothSideToRight(JoinType.ASOF_RIGHT_OUTER_JOIN); bothSideToRight(JoinType.RIGHT_ANTI_JOIN); bothSideToRight(JoinType.RIGHT_SEMI_JOIN); } @@ -182,7 +192,7 @@ private void bothSideToLeft(JoinType joinType) { .filter(whereCondition) .build(); - PlanChecker.from(MemoTestUtils.createConnectContext(), plan) + PlanChecker.from(connectContext, plan) .applyTopDown(PushDownFilterThroughJoin.INSTANCE) .matchesFromRoot( logicalFilter( @@ -205,7 +215,7 @@ private void bothSideToRight(JoinType joinType) { .filter(whereCondition) .build(); - PlanChecker.from(MemoTestUtils.createConnectContext(), plan) + PlanChecker.from(connectContext, plan) .applyTopDown(PushDownFilterThroughJoin.INSTANCE) .matchesFromRoot( logicalFilter( diff --git a/fe/fe-core/src/test/java/org/apache/doris/nereids/rules/rewrite/PushDownJoinOnAssertNumRowsTest.java b/fe/fe-core/src/test/java/org/apache/doris/nereids/rules/rewrite/PushDownJoinOnAssertNumRowsTest.java index aded31bd18fcbf..32b62a0951944d 100644 --- a/fe/fe-core/src/test/java/org/apache/doris/nereids/rules/rewrite/PushDownJoinOnAssertNumRowsTest.java +++ b/fe/fe-core/src/test/java/org/apache/doris/nereids/rules/rewrite/PushDownJoinOnAssertNumRowsTest.java @@ -100,6 +100,12 @@ final void beforeAll() { */ @Test void testPushDownToLeftChild() { + testPushDownToLeftChildHelper(JoinType.INNER_JOIN); + testPushDownToLeftChildHelper(JoinType.ASOF_LEFT_INNER_JOIN); + testPushDownToLeftChildHelper(JoinType.ASOF_RIGHT_INNER_JOIN); + } + + private void testPushDownToLeftChildHelper(JoinType joinType) { // Create a one-row relation wrapped in LogicalAssertNumRows Plan oneRowRelation = new LogicalPlanBuilder(t3) .limit(1) @@ -112,26 +118,26 @@ void testPushDownToLeftChild() { Expression bottomJoinCondition = new EqualTo(t1Slots.get(0), t2Slots.get(1)); LogicalPlan bottomJoin = new LogicalPlanBuilder(t1) - .join(t2, JoinType.INNER_JOIN, ImmutableList.of(bottomJoinCondition), - ImmutableList.of()) + .join(t2, joinType, ImmutableList.of(bottomJoinCondition), + ImmutableList.of()) .build(); // Create top join: (T1 JOIN T2) JOIN assertNumRows on T1.age > course.id Expression topJoinCondition = new GreaterThan(t1Slots.get(1), t3Slots.get(0)); LogicalPlan root = new LogicalPlanBuilder(bottomJoin) - .join(assertNumRows, JoinType.INNER_JOIN, ImmutableList.of(), - ImmutableList.of(topJoinCondition)) + .join(assertNumRows, joinType, ImmutableList.of(), + ImmutableList.of(topJoinCondition)) .build(); // Apply the rule PlanChecker.from(MemoTestUtils.createConnectContext(), root) .applyTopDown(new PushDownJoinOnAssertNumRows()) .matches(logicalJoin( - logicalJoin( - logicalOlapScan(), - logicalAssertNumRows()), - logicalOlapScan())); + logicalJoin( + logicalOlapScan(), + logicalAssertNumRows()), + logicalOlapScan())); } /** @@ -152,6 +158,12 @@ void testPushDownToLeftChild() { */ @Test void testPushDownToRightChild() { + testPushDownToRightChildHelper(JoinType.INNER_JOIN); + testPushDownToRightChildHelper(JoinType.ASOF_LEFT_INNER_JOIN); + testPushDownToRightChildHelper(JoinType.ASOF_RIGHT_INNER_JOIN); + } + + private void testPushDownToRightChildHelper(JoinType joinType) { // Create a one-row relation wrapped in LogicalAssertNumRows Plan oneRowRelation = new LogicalPlanBuilder(t3) .limit(1) @@ -164,8 +176,8 @@ void testPushDownToRightChild() { Expression bottomJoinCondition = new EqualTo(t1Slots.get(0), t2Slots.get(1)); LogicalPlan bottomJoin = new LogicalPlanBuilder(t1) - .join(t2, JoinType.INNER_JOIN, ImmutableList.of(bottomJoinCondition), - ImmutableList.of()) + .join(t2, joinType, ImmutableList.of(bottomJoinCondition), + ImmutableList.of()) .build(); // Create top join: (T1 JOIN T2) JOIN assertNumRows on T2.name > course.name @@ -173,18 +185,18 @@ void testPushDownToRightChild() { Expression topJoinCondition = new GreaterThan(t2Slots.get(2), t3Slots.get(1)); LogicalPlan root = new LogicalPlanBuilder(bottomJoin) - .join(assertNumRows, JoinType.INNER_JOIN, ImmutableList.of(), - ImmutableList.of(topJoinCondition)) + .join(assertNumRows, joinType, ImmutableList.of(), + ImmutableList.of(topJoinCondition)) .build(); // Apply the rule PlanChecker.from(MemoTestUtils.createConnectContext(), root) .applyTopDown(new PushDownJoinOnAssertNumRows()) .matches(logicalJoin( + logicalOlapScan(), + logicalJoin( logicalOlapScan(), - logicalJoin( - logicalOlapScan(), - logicalAssertNumRows()))); + logicalAssertNumRows()))); } /** @@ -207,6 +219,12 @@ void testPushDownToRightChild() { */ @Test void testPushDownWithProjectNode() { + testPushDownWithProjectNodeHelper(JoinType.INNER_JOIN); + testPushDownWithProjectNodeHelper(JoinType.ASOF_LEFT_INNER_JOIN); + testPushDownWithProjectNodeHelper(JoinType.ASOF_RIGHT_INNER_JOIN); + } + + private void testPushDownWithProjectNodeHelper(JoinType joinType) { // Create a one-row relation wrapped in LogicalAssertNumRows Plan oneRowRelation = new LogicalPlanBuilder(t3) .limit(1) @@ -219,8 +237,8 @@ void testPushDownWithProjectNode() { Expression bottomJoinCondition = new EqualTo(t1Slots.get(0), t2Slots.get(1)); LogicalPlan bottomJoin = new LogicalPlanBuilder(t1) - .join(t2, JoinType.INNER_JOIN, ImmutableList.of(bottomJoinCondition), - ImmutableList.of()) + .join(t2, joinType, ImmutableList.of(bottomJoinCondition), + ImmutableList.of()) .build(); // Create project with alias: T1.age + 1 as alias_col @@ -238,19 +256,19 @@ void testPushDownWithProjectNode() { Expression topJoinCondition = new GreaterThan(aliasCol.toSlot(), t3Slots.get(0)); LogicalPlan root = new LogicalPlanBuilder(project) - .join(assertNumRows, JoinType.INNER_JOIN, ImmutableList.of(), - ImmutableList.of(topJoinCondition)) + .join(assertNumRows, joinType, ImmutableList.of(), + ImmutableList.of(topJoinCondition)) .build(); // Apply the rule PlanChecker.from(MemoTestUtils.createConnectContext(), root) .applyTopDown(new PushDownJoinOnAssertNumRows()) .matches(logicalProject( + logicalJoin( logicalJoin( - logicalJoin( - logicalProject(logicalOlapScan()), - logicalAssertNumRows()), - logicalOlapScan()))); + logicalProject(logicalOlapScan()), + logicalAssertNumRows()), + logicalOlapScan()))); } /** @@ -294,6 +312,12 @@ void testWithCrossJoin() { */ @Test void testNoApplyWhenBottomJoinHasAssertNumRows() { + testNoApplyWhenBottomJoinHasAssertNumRowsHelper(JoinType.INNER_JOIN); + testNoApplyWhenBottomJoinHasAssertNumRowsHelper(JoinType.ASOF_LEFT_INNER_JOIN); + testNoApplyWhenBottomJoinHasAssertNumRowsHelper(JoinType.ASOF_RIGHT_INNER_JOIN); + } + + private void testNoApplyWhenBottomJoinHasAssertNumRowsHelper(JoinType joinType) { // Create two one-row relations Plan oneRowRelation1 = new LogicalPlanBuilder(t2) .limit(1) @@ -311,16 +335,16 @@ void testNoApplyWhenBottomJoinHasAssertNumRows() { Expression bottomJoinCondition = new GreaterThan(t1Slots.get(1), t2Slots.get(0)); LogicalPlan bottomJoin = new LogicalPlanBuilder(t1) - .join(assertNumRows1, JoinType.INNER_JOIN, ImmutableList.of(), - ImmutableList.of(bottomJoinCondition)) + .join(assertNumRows1, joinType, ImmutableList.of(), + ImmutableList.of(bottomJoinCondition)) .build(); // Create top join with another assertNumRows Expression topJoinCondition = new GreaterThan(t1Slots.get(1), t3Slots.get(0)); LogicalPlan root = new LogicalPlanBuilder(bottomJoin) - .join(assertNumRows2, JoinType.INNER_JOIN, ImmutableList.of(), - ImmutableList.of(topJoinCondition)) + .join(assertNumRows2, joinType, ImmutableList.of(), + ImmutableList.of(topJoinCondition)) .build(); // Apply the rule - should not transform because bottom join already has @@ -328,10 +352,10 @@ void testNoApplyWhenBottomJoinHasAssertNumRows() { PlanChecker.from(MemoTestUtils.createConnectContext(), root) .applyTopDown(new PushDownJoinOnAssertNumRows()) .matches(logicalJoin( - logicalJoin( - logicalOlapScan(), - logicalAssertNumRows()), - logicalAssertNumRows())); + logicalJoin( + logicalOlapScan(), + logicalAssertNumRows()), + logicalAssertNumRows())); } /** @@ -339,6 +363,12 @@ void testNoApplyWhenBottomJoinHasAssertNumRows() { */ @Test void testNoApplyWithMultipleJoinConditions() { + testNoApplyWithMultipleJoinConditionsHelper(JoinType.INNER_JOIN); + testNoApplyWithMultipleJoinConditionsHelper(JoinType.ASOF_LEFT_INNER_JOIN); + testNoApplyWithMultipleJoinConditionsHelper(JoinType.ASOF_RIGHT_INNER_JOIN); + } + + private void testNoApplyWithMultipleJoinConditionsHelper(JoinType joinType) { // Create a one-row relation wrapped in LogicalAssertNumRows Plan oneRowRelation = new LogicalPlanBuilder(t3) .limit(1) @@ -351,8 +381,8 @@ void testNoApplyWithMultipleJoinConditions() { Expression bottomJoinCondition = new EqualTo(t1Slots.get(0), t2Slots.get(1)); LogicalPlan bottomJoin = new LogicalPlanBuilder(t1) - .join(t2, JoinType.INNER_JOIN, ImmutableList.of(bottomJoinCondition), - ImmutableList.of()) + .join(t2, joinType, ImmutableList.of(bottomJoinCondition), + ImmutableList.of()) .build(); // Create top join with multiple conditions @@ -360,8 +390,8 @@ void testNoApplyWithMultipleJoinConditions() { Expression topJoinCondition2 = new GreaterThan(t2Slots.get(0), t3Slots.get(0)); LogicalPlan root = new LogicalPlanBuilder(bottomJoin) - .join(assertNumRows, JoinType.INNER_JOIN, ImmutableList.of(), - ImmutableList.of(topJoinCondition1, topJoinCondition2)) + .join(assertNumRows, joinType, ImmutableList.of(), + ImmutableList.of(topJoinCondition1, topJoinCondition2)) .build(); // Apply the rule - should not transform because there are multiple join @@ -369,10 +399,10 @@ void testNoApplyWithMultipleJoinConditions() { PlanChecker.from(MemoTestUtils.createConnectContext(), root) .applyTopDown(new PushDownJoinOnAssertNumRows()) .matches(logicalJoin( - logicalJoin( - logicalOlapScan(), - logicalOlapScan()), - logicalAssertNumRows())); + logicalJoin( + logicalOlapScan(), + logicalOlapScan()), + logicalAssertNumRows())); } /** @@ -380,6 +410,15 @@ void testNoApplyWithMultipleJoinConditions() { */ @Test void testNoApplyWithOuterJoin() { + testNoApplyWithOuterJoinHelper(JoinType.LEFT_OUTER_JOIN, JoinType.INNER_JOIN); + testNoApplyWithOuterJoinHelper(JoinType.LEFT_OUTER_JOIN, JoinType.ASOF_LEFT_INNER_JOIN); + testNoApplyWithOuterJoinHelper(JoinType.LEFT_OUTER_JOIN, JoinType.ASOF_RIGHT_INNER_JOIN); + testNoApplyWithOuterJoinHelper(JoinType.ASOF_LEFT_OUTER_JOIN, JoinType.INNER_JOIN); + testNoApplyWithOuterJoinHelper(JoinType.ASOF_LEFT_OUTER_JOIN, JoinType.ASOF_LEFT_INNER_JOIN); + testNoApplyWithOuterJoinHelper(JoinType.ASOF_LEFT_OUTER_JOIN, JoinType.ASOF_RIGHT_INNER_JOIN); + } + + private void testNoApplyWithOuterJoinHelper(JoinType outer, JoinType inner) { // Create a one-row relation wrapped in LogicalAssertNumRows Plan oneRowRelation = new LogicalPlanBuilder(t3) .limit(1) @@ -392,26 +431,26 @@ void testNoApplyWithOuterJoin() { Expression bottomJoinCondition = new EqualTo(t1Slots.get(0), t2Slots.get(1)); LogicalPlan bottomJoin = new LogicalPlanBuilder(t1) - .join(t2, JoinType.LEFT_OUTER_JOIN, ImmutableList.of(bottomJoinCondition), - ImmutableList.of()) + .join(t2, outer, ImmutableList.of(bottomJoinCondition), + ImmutableList.of()) .build(); // Create top join with condition Expression topJoinCondition = new GreaterThan(t1Slots.get(1), t3Slots.get(0)); LogicalPlan root = new LogicalPlanBuilder(bottomJoin) - .join(assertNumRows, JoinType.INNER_JOIN, ImmutableList.of(), - ImmutableList.of(topJoinCondition)) + .join(assertNumRows, inner, ImmutableList.of(), + ImmutableList.of(topJoinCondition)) .build(); // Apply the rule - should not transform because bottom join is outer join PlanChecker.from(MemoTestUtils.createConnectContext(), root) .applyTopDown(new PushDownJoinOnAssertNumRows()) .matches(logicalJoin( - logicalJoin( - logicalOlapScan(), - logicalOlapScan()), - logicalAssertNumRows())); + logicalJoin( + logicalOlapScan(), + logicalOlapScan()), + logicalAssertNumRows())); } /** @@ -419,6 +458,12 @@ void testNoApplyWithOuterJoin() { */ @Test void testWithAssertNumRowsInProject() { + testWithAssertNumRowsInProjectHelper(JoinType.INNER_JOIN); + testWithAssertNumRowsInProjectHelper(JoinType.ASOF_LEFT_INNER_JOIN); + testWithAssertNumRowsInProjectHelper(JoinType.ASOF_RIGHT_INNER_JOIN); + } + + private void testWithAssertNumRowsInProjectHelper(JoinType joinType) { // Create a one-row relation wrapped in LogicalAssertNumRows and then Project Plan oneRowRelation = new LogicalPlanBuilder(t3) .limit(1) @@ -435,27 +480,27 @@ void testWithAssertNumRowsInProject() { Expression bottomJoinCondition = new EqualTo(t1Slots.get(0), t2Slots.get(1)); LogicalPlan bottomJoin = new LogicalPlanBuilder(t1) - .join(t2, JoinType.INNER_JOIN, ImmutableList.of(bottomJoinCondition), - ImmutableList.of()) + .join(t2, joinType, ImmutableList.of(bottomJoinCondition), + ImmutableList.of()) .build(); // Create top join with project-wrapped assertNumRows Expression topJoinCondition = new GreaterThan(t1Slots.get(1), t3Slots.get(0)); LogicalPlan root = new LogicalPlanBuilder(bottomJoin) - .join(assertProject, JoinType.INNER_JOIN, ImmutableList.of(), - ImmutableList.of(topJoinCondition)) + .join(assertProject, joinType, ImmutableList.of(), + ImmutableList.of(topJoinCondition)) .build(); // Apply the rule PlanChecker.from(MemoTestUtils.createConnectContext(), root) .applyTopDown(new PushDownJoinOnAssertNumRows()) .matches(logicalJoin( - logicalJoin( - logicalOlapScan(), - logicalProject( - logicalAssertNumRows())), - logicalOlapScan())); + logicalJoin( + logicalOlapScan(), + logicalProject( + logicalAssertNumRows())), + logicalOlapScan())); } @Test diff --git a/fe/fe-core/src/test/java/org/apache/doris/nereids/rules/rewrite/PushDownJoinOtherConditionTest.java b/fe/fe-core/src/test/java/org/apache/doris/nereids/rules/rewrite/PushDownJoinOtherConditionTest.java index 05642f084d5b58..a9175dc5b3c989 100644 --- a/fe/fe-core/src/test/java/org/apache/doris/nereids/rules/rewrite/PushDownJoinOtherConditionTest.java +++ b/fe/fe-core/src/test/java/org/apache/doris/nereids/rules/rewrite/PushDownJoinOtherConditionTest.java @@ -66,10 +66,14 @@ final void beforeAll() { void oneSide() { oneSide(JoinType.CROSS_JOIN, false); oneSide(JoinType.INNER_JOIN, false); + oneSide(JoinType.ASOF_LEFT_INNER_JOIN, false); + oneSide(JoinType.ASOF_RIGHT_INNER_JOIN, false); oneSide(JoinType.LEFT_OUTER_JOIN, true); + oneSide(JoinType.ASOF_LEFT_OUTER_JOIN, true); oneSide(JoinType.LEFT_SEMI_JOIN, true); oneSide(JoinType.LEFT_ANTI_JOIN, true); oneSide(JoinType.RIGHT_OUTER_JOIN, false); + oneSide(JoinType.ASOF_RIGHT_OUTER_JOIN, false); oneSide(JoinType.RIGHT_SEMI_JOIN, false); oneSide(JoinType.RIGHT_ANTI_JOIN, false); } @@ -114,6 +118,8 @@ private void oneSide(JoinType joinType, boolean testRight) { void bothSideToBothSide() { bothSideToBothSide(JoinType.CROSS_JOIN); bothSideToBothSide(JoinType.INNER_JOIN); + bothSideToBothSide(JoinType.ASOF_LEFT_INNER_JOIN); + bothSideToBothSide(JoinType.ASOF_RIGHT_INNER_JOIN); bothSideToBothSide(JoinType.LEFT_SEMI_JOIN); bothSideToBothSide(JoinType.RIGHT_SEMI_JOIN); } @@ -141,8 +147,10 @@ private void bothSideToBothSide(JoinType joinType) { @Test void bothSideToOneSide() { bothSideToOneSide(JoinType.LEFT_OUTER_JOIN, true); + bothSideToOneSide(JoinType.ASOF_LEFT_OUTER_JOIN, true); bothSideToOneSide(JoinType.LEFT_ANTI_JOIN, true); bothSideToOneSide(JoinType.RIGHT_OUTER_JOIN, false); + bothSideToOneSide(JoinType.ASOF_RIGHT_OUTER_JOIN, false); bothSideToOneSide(JoinType.RIGHT_ANTI_JOIN, false); } diff --git a/fe/fe-core/src/test/java/org/apache/doris/nereids/rules/rewrite/PushDownLimitDistinctThroughJoinTest.java b/fe/fe-core/src/test/java/org/apache/doris/nereids/rules/rewrite/PushDownLimitDistinctThroughJoinTest.java index bce9a2cd207a66..6eb92c791258e7 100644 --- a/fe/fe-core/src/test/java/org/apache/doris/nereids/rules/rewrite/PushDownLimitDistinctThroughJoinTest.java +++ b/fe/fe-core/src/test/java/org/apache/doris/nereids/rules/rewrite/PushDownLimitDistinctThroughJoinTest.java @@ -84,6 +84,20 @@ void testJoin() { ) ); + plan = new LogicalPlanBuilder(scan1) + .join(scan2, JoinType.ASOF_LEFT_OUTER_JOIN, Pair.of(0, 0)) + .distinct(ImmutableList.of(0, 1)) + .limit(10) + .build(); + PlanChecker.from(connectContext, plan) + .applyTopDown(new PushDownLimitDistinctThroughJoin()) + .matches( + logicalJoin( + logicalLimit(logicalAggregate(logicalOlapScan())).when(l -> l.getLimit() == 10), + logicalOlapScan() + ) + ); + plan = new LogicalPlanBuilder(scan1) .join(scan2, JoinType.RIGHT_OUTER_JOIN, Pair.of(0, 0)) .distinct(ImmutableList.of(2, 3)) @@ -98,6 +112,20 @@ void testJoin() { ) ); + plan = new LogicalPlanBuilder(scan1) + .join(scan2, JoinType.ASOF_RIGHT_OUTER_JOIN, Pair.of(0, 0)) + .distinct(ImmutableList.of(2, 3)) + .limit(10) + .build(); + PlanChecker.from(connectContext, plan) + .applyTopDown(new PushDownLimitDistinctThroughJoin()) + .matches( + logicalJoin( + logicalOlapScan(), + logicalLimit(logicalAggregate(logicalOlapScan())).when(l -> l.getLimit() == 10) + ) + ); + plan = new LogicalPlanBuilder(scan1) .join(scan2, JoinType.CROSS_JOIN, Pair.of(0, 0)) .distinct(ImmutableList.of(0, 1)) @@ -143,8 +171,13 @@ void testJoinSql() { @Test void badCaseJoinType() { + badCaseJoinTypeHelper(JoinType.LEFT_OUTER_JOIN); + badCaseJoinTypeHelper(JoinType.ASOF_LEFT_OUTER_JOIN); + } + + private void badCaseJoinTypeHelper(JoinType joinType) { LogicalPlan plan = new LogicalPlanBuilder(scan1) - .join(scan2, JoinType.LEFT_OUTER_JOIN, Pair.of(0, 0)) + .join(scan2, joinType, Pair.of(0, 0)) .distinct(ImmutableList.of(2)) .limit(10) .build(); @@ -155,9 +188,14 @@ void badCaseJoinType() { @Test void badCaseOutput() { + badCaseOutputHelper(JoinType.LEFT_OUTER_JOIN); + badCaseOutputHelper(JoinType.ASOF_LEFT_OUTER_JOIN); + } + + private void badCaseOutputHelper(JoinType joinType) { // distinct agg don't output all group by columns of left child LogicalPlan plan = new LogicalPlanBuilder(scan1) - .join(scan2, JoinType.LEFT_OUTER_JOIN, Pair.of(0, 0)) + .join(scan2, joinType, Pair.of(0, 0)) .distinct(ImmutableList.of(0)) .limit(10) .build(); diff --git a/fe/fe-core/src/test/java/org/apache/doris/nereids/rules/rewrite/PushDownTopNThroughJoinTest.java b/fe/fe-core/src/test/java/org/apache/doris/nereids/rules/rewrite/PushDownTopNThroughJoinTest.java index 1e54444be723f9..6d2ffb3bbbebbe 100644 --- a/fe/fe-core/src/test/java/org/apache/doris/nereids/rules/rewrite/PushDownTopNThroughJoinTest.java +++ b/fe/fe-core/src/test/java/org/apache/doris/nereids/rules/rewrite/PushDownTopNThroughJoinTest.java @@ -76,8 +76,13 @@ protected void runBeforeAll() throws Exception { @Test void testJoin() { + testJoinHelper(JoinType.LEFT_OUTER_JOIN); + testJoinHelper(JoinType.ASOF_LEFT_OUTER_JOIN); + } + + private void testJoinHelper(JoinType joinType) { LogicalPlan plan = new LogicalPlanBuilder(scan1) - .join(scan2, JoinType.LEFT_OUTER_JOIN, Pair.of(0, 0)) + .join(scan2, joinType, Pair.of(0, 0)) .topN(10, 0, ImmutableList.of(0)) .build(); PlanChecker.from(connectContext, plan) @@ -94,10 +99,15 @@ void testJoin() { @Test void testProject1() { + testProject1Helper(JoinType.LEFT_OUTER_JOIN); + testProject1Helper(JoinType.ASOF_LEFT_OUTER_JOIN); + } + + private void testProject1Helper(JoinType joinType) { List projectExpres = ImmutableList.of(scan1.getOutput().get(0), new Cast(scan1.getOutput().get(1), VarcharType.SYSTEM_DEFAULT).alias("cast")); LogicalPlan plan = new LogicalPlanBuilder(scan1) - .join(scan2, JoinType.LEFT_OUTER_JOIN, Pair.of(0, 0)) + .join(scan2, joinType, Pair.of(0, 0)) .projectExprs(projectExpres) .topN(10, 0, ImmutableList.of(0)) .build(); @@ -178,11 +188,16 @@ void testProjectSql() { @Test void rejectTopNUseProjectComplexExpr() { + rejectTopNUseProjectComplexExprHelper(JoinType.LEFT_OUTER_JOIN); + rejectTopNUseProjectComplexExprHelper(JoinType.ASOF_LEFT_OUTER_JOIN); + } + + private void rejectTopNUseProjectComplexExprHelper(JoinType joinType) { List projectExpres = ImmutableList.of( (new Add(scan1.getOutput().get(0), scan1.getOutput().get(1))).alias("add") ); LogicalPlan plan = new LogicalPlanBuilder(scan1) - .join(scan2, JoinType.LEFT_OUTER_JOIN, Pair.of(0, 0)) + .join(scan2, joinType, Pair.of(0, 0)) .projectExprs(projectExpres) .topN(10, 0, ImmutableList.of(0)) .build(); @@ -198,8 +213,13 @@ void rejectTopNUseProjectComplexExpr() { @Test void rejectWrongJoinType() { + rejectWrongJoinTypeHelper(JoinType.RIGHT_OUTER_JOIN); + rejectWrongJoinTypeHelper(JoinType.ASOF_RIGHT_OUTER_JOIN); + } + + private void rejectWrongJoinTypeHelper(JoinType joinType) { LogicalPlan plan = new LogicalPlanBuilder(scan1) - .join(scan2, JoinType.RIGHT_OUTER_JOIN, Pair.of(0, 0)) + .join(scan2, joinType, Pair.of(0, 0)) .topN(10, 0, ImmutableList.of(0)) .build(); PlanChecker.from(connectContext, plan) diff --git a/fe/fe-core/src/test/java/org/apache/doris/nereids/rules/rewrite/PushFilterInsideJoinTest.java b/fe/fe-core/src/test/java/org/apache/doris/nereids/rules/rewrite/PushFilterInsideJoinTest.java index 4fd436c54a09a2..c5f4da1207fee2 100644 --- a/fe/fe-core/src/test/java/org/apache/doris/nereids/rules/rewrite/PushFilterInsideJoinTest.java +++ b/fe/fe-core/src/test/java/org/apache/doris/nereids/rules/rewrite/PushFilterInsideJoinTest.java @@ -108,7 +108,7 @@ public void testPushInsideInnerJoin() { @Test public void testShouldNotPushInsideJoin() { for (JoinType joinType : JoinType.values()) { - if (JoinType.INNER_JOIN == joinType || JoinType.CROSS_JOIN == joinType) { + if (joinType.isInnerOrCrossJoin()) { continue; } shouldNotPushInsideJoin(joinType); diff --git a/fe/fe-core/src/test/java/org/apache/doris/nereids/rules/rewrite/ReorderJoinTest.java b/fe/fe-core/src/test/java/org/apache/doris/nereids/rules/rewrite/ReorderJoinTest.java index 71efbb42ff1da4..63f8d769545cbb 100644 --- a/fe/fe-core/src/test/java/org/apache/doris/nereids/rules/rewrite/ReorderJoinTest.java +++ b/fe/fe-core/src/test/java/org/apache/doris/nereids/rules/rewrite/ReorderJoinTest.java @@ -43,15 +43,20 @@ class ReorderJoinTest implements MemoPatternMatchSupported { @Test public void testLeftOuterJoin() { + testLeftOuterJoinHelper(JoinType.LEFT_OUTER_JOIN); + testLeftOuterJoinHelper(JoinType.ASOF_LEFT_OUTER_JOIN); + } + + private void testLeftOuterJoinHelper(JoinType joinType) { ImmutableList plans = ImmutableList.of( new LogicalPlanBuilder(scan1) - .join(scan2, JoinType.LEFT_OUTER_JOIN, Pair.of(0, 0)) + .join(scan2, joinType, Pair.of(0, 0)) .joinEmptyOn(scan3, JoinType.CROSS_JOIN) .filter(new EqualTo(scan3.getOutput().get(0), scan1.getOutput().get(0))) .build(), new LogicalPlanBuilder(scan1) .joinEmptyOn(scan3, JoinType.CROSS_JOIN) - .join(scan2, JoinType.LEFT_OUTER_JOIN, Pair.of(0, 0)) + .join(scan2, joinType, Pair.of(0, 0)) .filter(new EqualTo(scan3.getOutput().get(0), scan1.getOutput().get(0))) .build() ); @@ -61,15 +66,20 @@ public void testLeftOuterJoin() { @Test public void testRightOuterJoin() { + testRightOuterJoinHelper(JoinType.RIGHT_OUTER_JOIN); + testRightOuterJoinHelper(JoinType.ASOF_RIGHT_OUTER_JOIN); + } + + private void testRightOuterJoinHelper(JoinType joinType) { ImmutableList plans = ImmutableList.of( new LogicalPlanBuilder(scan1) - .join(scan2, JoinType.RIGHT_OUTER_JOIN, Pair.of(0, 0)) + .join(scan2, joinType, Pair.of(0, 0)) .joinEmptyOn(scan3, JoinType.CROSS_JOIN) .filter(new EqualTo(scan3.getOutput().get(0), scan1.getOutput().get(0))) .build(), new LogicalPlanBuilder(scan1) .joinEmptyOn(scan3, JoinType.CROSS_JOIN) - .join(scan2, JoinType.RIGHT_OUTER_JOIN, Pair.of(0, 0)) + .join(scan2, joinType, Pair.of(0, 0)) .filter(new EqualTo(scan3.getOutput().get(0), scan1.getOutput().get(0))) .build() ); @@ -229,11 +239,17 @@ public void check(List plans) { */ @Test public void testInnerOrCrossJoin() { + testInnerOrCrossJoinHelper(JoinType.INNER_JOIN); + testInnerOrCrossJoinHelper(JoinType.ASOF_LEFT_INNER_JOIN); + testInnerOrCrossJoinHelper(JoinType.ASOF_RIGHT_INNER_JOIN); + } + + private void testInnerOrCrossJoinHelper(JoinType joinType) { LogicalPlan leftJoin = new LogicalPlanBuilder(scan1) - .join(scan2, JoinType.INNER_JOIN, Pair.of(0, 0)) + .join(scan2, joinType, Pair.of(0, 0)) .build(); LogicalPlan rightJoin = new LogicalPlanBuilder(scan3) - .join(scan4, JoinType.INNER_JOIN, Pair.of(0, 0)) + .join(scan4, joinType, Pair.of(0, 0)) .build(); LogicalPlan plan = new LogicalPlanBuilder(leftJoin) diff --git a/fe/fe-core/src/test/java/org/apache/doris/nereids/rules/rewrite/SaltJoinTest.java b/fe/fe-core/src/test/java/org/apache/doris/nereids/rules/rewrite/SaltJoinTest.java index 7fcb80862a3dfa..499f0a34d88c5f 100644 --- a/fe/fe-core/src/test/java/org/apache/doris/nereids/rules/rewrite/SaltJoinTest.java +++ b/fe/fe-core/src/test/java/org/apache/doris/nereids/rules/rewrite/SaltJoinTest.java @@ -31,8 +31,8 @@ public class SaltJoinTest extends TestWithFeService implements MemoPatternMatchS @Override protected void runBeforeAll() throws Exception { createDatabase("test"); - createTable("create table test.test_skew9(a int,c varchar(100), b int) distributed by hash(a) buckets 32 properties(\"replication_num\"=\"1\");"); - createTable("create table test.test_skew10(a int,c varchar(100), b int) distributed by hash(a) buckets 32 properties(\"replication_num\"=\"1\");"); + createTable("create table test.test_skew9(a int,c varchar(100), b int, d date) distributed by hash(a) buckets 32 properties(\"replication_num\"=\"1\");"); + createTable("create table test.test_skew10(a int,c varchar(100), b int, d date) distributed by hash(a) buckets 32 properties(\"replication_num\"=\"1\");"); connectContext.setDatabase("test"); connectContext.getSessionVariable().setDisableNereidsRules("PRUNE_EMPTY_PARTITION"); connectContext.getSessionVariable().setParallelResultSink(false); @@ -59,6 +59,27 @@ public void testInnerJoin() { ); } + @Test + public void testAsofInnerJoin() { + PlanChecker.from(connectContext) + .analyze("select * from test_skew9 tl asof inner join [shuffle[skew(tl.b(1,2))]] test_skew10 tr match_condition(tl.d > tr.d) on tl.b = tr.b") + .rewrite() + .printlnTree() + .matches( + logicalJoin( + logicalProject( + logicalOlapScan()), + logicalProject( + logicalJoin( + logicalProject( + logicalGenerate( + logicalUnion())), + logicalOlapScan() + ).when(join -> join.getJoinType() == JoinType.RIGHT_OUTER_JOIN)) + ).when(join -> join.getHashJoinConjuncts().size() == 2 && join.getJoinType() == JoinType.ASOF_LEFT_INNER_JOIN) + ); + } + @Test public void testLeftJoin() { PlanChecker.from(connectContext) @@ -80,6 +101,27 @@ public void testLeftJoin() { ); } + @Test + public void testAsofLeftJoin() { + PlanChecker.from(connectContext) + .analyze("select * from test_skew9 tl asof left join [shuffle[skew(tl.b(1,2))]] test_skew10 tr match_condition(tl.d > tr.d) on tl.b = tr.b;") + .rewrite() + .printlnTree() + .matches( + logicalJoin( + logicalProject( + logicalOlapScan()), + logicalProject( + logicalJoin( + logicalProject( + logicalGenerate( + logicalUnion())), + logicalOlapScan() + ).when(join -> join.getJoinType() == JoinType.RIGHT_OUTER_JOIN)) + ).when(join -> join.getHashJoinConjuncts().size() == 2 && join.getJoinType() == JoinType.ASOF_LEFT_OUTER_JOIN) + ); + } + @Test public void testRightJoin() { PlanChecker.from(connectContext) @@ -146,6 +188,20 @@ public void testRightJoinPhysicalPlan() { ); } + @Test + void testAsofInnerSkewValueIsNull() { + PlanChecker.from(connectContext) + .analyze("select * from test_skew9 tl asof inner join [shuffle[skew(tl.b(null))]] test_skew10 tr match_condition(tl.d > tr.d) on tl.b = tr.b") + .rewrite() + .printlnTree() + .matches( + logicalJoin( + logicalOlapScan(), + logicalOlapScan() + ).when(join -> join.getHashJoinConjuncts().size() == 1 && join.getJoinType() == JoinType.ASOF_LEFT_INNER_JOIN) + ); + } + @Test void testLeftJoinSkewValueIsNull() { PlanChecker.from(connectContext) .analyze("select * from test_skew9 tl left join [shuffle[skew(tl.b(null))]] test_skew10 tr on tl.b = tr.b") @@ -162,6 +218,23 @@ public void testRightJoinPhysicalPlan() { ); } + @Test + void testAsofLeftJoinSkewValueIsNull() { + PlanChecker.from(connectContext) + .analyze("select * from test_skew9 tl asof left join [shuffle[skew(tl.b(null))]] test_skew10 tr match_condition(tl.d > tr.d) on tl.b = tr.b") + .rewrite() + .printlnTree() + .matches( + logicalJoin( + logicalProject( + logicalOlapScan()), + logicalProject( + logicalOlapScan() + ) + ).when(join -> join.getHashJoinConjuncts().size() == 2 && join.getJoinType() == JoinType.ASOF_LEFT_OUTER_JOIN) + ); + } + @Test void testRightJoinSkewValueIsNull() { PlanChecker.from(connectContext) .analyze("select * from test_skew9 tl right join [shuffle[skew(tr.b(null))]] test_skew10 tr on tl.b = tr.b") diff --git a/gensrc/thrift/PlanNodes.thrift b/gensrc/thrift/PlanNodes.thrift index 8eea8f078db367..f9dfdb4dde5ba0 100644 --- a/gensrc/thrift/PlanNodes.thrift +++ b/gensrc/thrift/PlanNodes.thrift @@ -886,7 +886,11 @@ enum TJoinOp { // be rejected (ANTI-join), based on the other join conjuncts. This is in contrast // to LEFT_ANTI_JOIN where NULLs are not matches and therefore always returned. NULL_AWARE_LEFT_ANTI_JOIN = 10, - NULL_AWARE_LEFT_SEMI_JOIN = 11 + NULL_AWARE_LEFT_SEMI_JOIN = 11, + ASOF_LEFT_INNER_JOIN = 12, + ASOF_RIGHT_INNER_JOIN = 13, + ASOF_LEFT_OUTER_JOIN = 14, + ASOF_RIGHT_OUTER_JOIN = 15 } enum TJoinDistributionType {