Skip to content

Commit e154ec3

Browse files
924060929924060929
committed
support
1 parent a0ecbae commit e154ec3

File tree

6 files changed

+231
-27
lines changed

6 files changed

+231
-27
lines changed

‎fe/fe-core/src/main/antlr4/org/apache/doris/nereids/DorisParser.g4‎

Lines changed: 3 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -2052,6 +2052,7 @@ nonReserved
20522052
| JOBS
20532053
| JSON
20542054
| JSONB
2055+
| KEYS
20552056
| LABEL
20562057
| LAST
20572058
| LDAP
@@ -2180,6 +2181,7 @@ nonReserved
21802181
| SESSION
21812182
| SESSION_USER
21822183
| SHAPE
2184+
| SIZE
21832185
| SKEW
21842186
| SNAPSHOT
21852187
| SNAPSHOTS
@@ -2223,6 +2225,7 @@ nonReserved
22232225
| UP
22242226
| USER
22252227
| VALUE
2228+
| VALUES
22262229
| VARBINARY
22272230
| VARCHAR
22282231
| VARIABLE

‎fe/fe-core/src/main/java/org/apache/doris/nereids/parser/LogicalPlanBuilder.java‎

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -515,6 +515,7 @@
515515
import org.apache.doris.nereids.trees.expressions.CaseWhen;
516516
import org.apache.doris.nereids.trees.expressions.Cast;
517517
import org.apache.doris.nereids.trees.expressions.DefaultValueSlot;
518+
import org.apache.doris.nereids.trees.expressions.DereferenceExpression;
518519
import org.apache.doris.nereids.trees.expressions.Divide;
519520
import org.apache.doris.nereids.trees.expressions.EqualTo;
520521
import org.apache.doris.nereids.trees.expressions.Exists;
@@ -3404,8 +3405,7 @@ public Expression visitDereference(DereferenceContext ctx) {
34043405
UnboundSlot slot = new UnboundSlot(nameParts, Optional.empty());
34053406
return slot;
34063407
} else {
3407-
// todo: base is an expression, may be not a table name.
3408-
throw new ParseException("Unsupported dereference expression: " + ctx.getText(), ctx);
3408+
return new DereferenceExpression(e, new StringLiteral(ctx.identifier().getText()));
34093409
}
34103410
});
34113411
}

‎fe/fe-core/src/main/java/org/apache/doris/nereids/rules/analysis/BindExpression.java‎

Lines changed: 11 additions & 11 deletions
Original file line numberDiff line numberDiff line change
@@ -469,12 +469,12 @@ private LogicalHaving<Plan> bindHavingAggregate(
469469
Scope groupBySlotsScope = toScope(cascadesContext, groupBySlots.build());
470470

471471
return (analyzer, unboundSlot) -> {
472-
List<Slot> boundInGroupBy = analyzer.bindSlotByScope(unboundSlot, groupBySlotsScope);
472+
List<Expression> boundInGroupBy = analyzer.bindSlotByScope(unboundSlot, groupBySlotsScope);
473473
if (!boundInGroupBy.isEmpty()) {
474474
return ImmutableList.of(boundInGroupBy.get(0));
475475
}
476476

477-
List<Slot> boundInAggOutput = analyzer.bindSlotByScope(unboundSlot, aggOutputScope);
477+
List<Expression> boundInAggOutput = analyzer.bindSlotByScope(unboundSlot, aggOutputScope);
478478
if (!boundInAggOutput.isEmpty()) {
479479
return ImmutableList.of(boundInAggOutput.get(0));
480480
}
@@ -553,7 +553,7 @@ private LogicalHaving<Plan> bindHavingByScopes(
553553
SimpleExprAnalyzer analyzer = buildCustomSlotBinderAnalyzer(
554554
having, cascadesContext, defaultScope, false, true,
555555
(self, unboundSlot) -> {
556-
List<Slot> slots = self.bindSlotByScope(unboundSlot, defaultScope);
556+
List<Expression> slots = self.bindSlotByScope(unboundSlot, defaultScope);
557557
if (!slots.isEmpty()) {
558558
return slots;
559559
}
@@ -1006,7 +1006,7 @@ private void bindQualifyByProject(LogicalProject<? extends Plan> project, Cascad
10061006
SimpleExprAnalyzer analyzer = buildCustomSlotBinderAnalyzer(
10071007
qualify, cascadesContext, defaultScope.get(), true, true,
10081008
(self, unboundSlot) -> {
1009-
List<Slot> slots = self.bindSlotByScope(unboundSlot, defaultScope.get());
1009+
List<Expression> slots = self.bindSlotByScope(unboundSlot, defaultScope.get());
10101010
if (!slots.isEmpty()) {
10111011
return slots;
10121012
}
@@ -1044,11 +1044,11 @@ private void bindQualifyByAggregate(Aggregate<? extends Plan> aggregate, Cascade
10441044
Scope groupBySlotsScope = toScope(cascadesContext, groupBySlots.build());
10451045

10461046
return (analyzer, unboundSlot) -> {
1047-
List<Slot> boundInGroupBy = analyzer.bindSlotByScope(unboundSlot, groupBySlotsScope);
1047+
List<Expression> boundInGroupBy = analyzer.bindSlotByScope(unboundSlot, groupBySlotsScope);
10481048
if (!boundInGroupBy.isEmpty()) {
10491049
return ImmutableList.of(boundInGroupBy.get(0));
10501050
}
1051-
List<Slot> boundInAggOutput = analyzer.bindSlotByScope(unboundSlot, aggOutputScope);
1051+
List<Expression> boundInAggOutput = analyzer.bindSlotByScope(unboundSlot, aggOutputScope);
10521052
if (!boundInAggOutput.isEmpty()) {
10531053
return ImmutableList.of(boundInAggOutput.get(0));
10541054
}
@@ -1368,15 +1368,15 @@ private List<Expression> bindGroupBy(
13681368
// see: https://github.com/apache/doris/pull/15240
13691369
//
13701370
// first, try to bind by agg.child.output
1371-
List<Slot> slotsInChildren = self.bindExactSlotsByThisScope(unboundSlot, childOutputScope);
1371+
List<Expression> slotsInChildren = self.bindExactSlotsByThisScope(unboundSlot, childOutputScope);
13721372
if (slotsInChildren.size() == 1) {
13731373
// bind succeed
13741374
return slotsInChildren;
13751375
}
13761376
// second, bind failed:
13771377
// if the slot not found, or more than one candidate slots found in agg.child.output,
13781378
// then try to bind by agg.output
1379-
List<Slot> slotsInOutput = self.bindExactSlotsByThisScope(
1379+
List<Expression> slotsInOutput = self.bindExactSlotsByThisScope(
13801380
unboundSlot, aggOutputScopeWithoutAggFun.get());
13811381
if (slotsInOutput.isEmpty()) {
13821382
// if slotsInChildren.size() > 1 && slotsInOutput.isEmpty(),
@@ -1385,7 +1385,7 @@ private List<Expression> bindGroupBy(
13851385
}
13861386

13871387
Builder<Expression> useOutputExpr = ImmutableList.builderWithExpectedSize(slotsInOutput.size());
1388-
for (Slot slotInOutput : slotsInOutput) {
1388+
for (Expression slotInOutput : slotsInOutput) {
13891389
// mappingSlot is provided by aggOutputScopeWithoutAggFun
13901390
// and no non-MappingSlot slot exist in the Scope, so we
13911391
// can direct cast it safely
@@ -1476,7 +1476,7 @@ private Plan bindSortWithoutSetOperation(MatchingContext<LogicalSort<Plan>> ctx)
14761476
sort, cascadesContext, inputScope, true, false,
14771477
(self, unboundSlot) -> {
14781478
// first, try to bind slot in Scope(input.output)
1479-
List<Slot> slotsInInput = self.bindExactSlotsByThisScope(unboundSlot, inputScope);
1479+
List<Expression> slotsInInput = self.bindExactSlotsByThisScope(unboundSlot, inputScope);
14801480
if (!slotsInInput.isEmpty()) {
14811481
// bind succeed
14821482
return ImmutableList.of(slotsInInput.get(0));
@@ -1678,7 +1678,7 @@ private SimpleExprAnalyzer getAnalyzerForOrderByAggFunc(Plan finalInput, Cascade
16781678
sort, cascadesContext, inputScope, true, false,
16791679
(analyzer, unboundSlot) -> {
16801680
if (finalInput instanceof LogicalAggregate) {
1681-
List<Slot> boundInOutputWithoutAggFunc = analyzer.bindSlotByScope(unboundSlot,
1681+
List<Expression> boundInOutputWithoutAggFunc = analyzer.bindSlotByScope(unboundSlot,
16821682
outputWithoutAggFunc);
16831683
if (!boundInOutputWithoutAggFunc.isEmpty()) {
16841684
return ImmutableList.of(boundInOutputWithoutAggFunc.get(0));

‎fe/fe-core/src/main/java/org/apache/doris/nereids/rules/analysis/ExpressionAnalyzer.java‎

Lines changed: 169 additions & 14 deletions
Original file line numberDiff line numberDiff line change
@@ -50,6 +50,7 @@
5050
import org.apache.doris.nereids.trees.expressions.Cast;
5151
import org.apache.doris.nereids.trees.expressions.ComparisonPredicate;
5252
import org.apache.doris.nereids.trees.expressions.CompoundPredicate;
53+
import org.apache.doris.nereids.trees.expressions.DereferenceExpression;
5354
import org.apache.doris.nereids.trees.expressions.Divide;
5455
import org.apache.doris.nereids.trees.expressions.EqualTo;
5556
import org.apache.doris.nereids.trees.expressions.ExprId;
@@ -75,7 +76,13 @@
7576
import org.apache.doris.nereids.trees.expressions.functions.agg.AggregateFunction;
7677
import org.apache.doris.nereids.trees.expressions.functions.agg.NullableAggregateFunction;
7778
import org.apache.doris.nereids.trees.expressions.functions.agg.SupportMultiDistinct;
79+
import org.apache.doris.nereids.trees.expressions.functions.scalar.Cardinality;
80+
import org.apache.doris.nereids.trees.expressions.functions.scalar.ElementAt;
7881
import org.apache.doris.nereids.trees.expressions.functions.scalar.Lambda;
82+
import org.apache.doris.nereids.trees.expressions.functions.scalar.MapKeys;
83+
import org.apache.doris.nereids.trees.expressions.functions.scalar.MapSize;
84+
import org.apache.doris.nereids.trees.expressions.functions.scalar.MapValues;
85+
import org.apache.doris.nereids.trees.expressions.functions.scalar.StructElement;
7986
import org.apache.doris.nereids.trees.expressions.functions.udf.AliasUdfBuilder;
8087
import org.apache.doris.nereids.trees.expressions.functions.udf.JavaUdaf;
8188
import org.apache.doris.nereids.trees.expressions.functions.udf.JavaUdf;
@@ -94,6 +101,8 @@
94101
import org.apache.doris.nereids.types.BooleanType;
95102
import org.apache.doris.nereids.types.DataType;
96103
import org.apache.doris.nereids.types.StringType;
104+
import org.apache.doris.nereids.types.StructField;
105+
import org.apache.doris.nereids.types.StructType;
97106
import org.apache.doris.nereids.types.TinyIntType;
98107
import org.apache.doris.nereids.util.ExpressionUtils;
99108
import org.apache.doris.nereids.util.TypeCoercionUtils;
@@ -256,6 +265,35 @@ public Expression visitUnboundAlias(UnboundAlias unboundAlias, ExpressionRewrite
256265
}
257266
}
258267

268+
@Override
269+
public Expression visitDereferenceExpression(DereferenceExpression dereferenceExpression,
270+
ExpressionRewriteContext context) {
271+
Expression expression = dereferenceExpression.child(0).accept(this, context);
272+
DataType dataType = expression.getDataType();
273+
if (dataType.isStructType()) {
274+
StructType structType = (StructType) dataType;
275+
StructField field = structType.getField(dereferenceExpression.fieldName);
276+
if (field != null) {
277+
return new StructElement(expression, dereferenceExpression.child(1));
278+
}
279+
} else if (dataType.isArrayType()) {
280+
if (dereferenceExpression.fieldName.equalsIgnoreCase("size")) {
281+
return new Cardinality(expression);
282+
}
283+
} else if (dataType.isMapType()) {
284+
if (dereferenceExpression.fieldName.equalsIgnoreCase("size")) {
285+
return new MapSize(expression);
286+
} else if (dereferenceExpression.fieldName.equalsIgnoreCase("keys")) {
287+
return new MapKeys(expression);
288+
} else if (dereferenceExpression.fieldName.equalsIgnoreCase("values")) {
289+
return new MapValues(expression);
290+
} else {
291+
return new ElementAt(expression, dereferenceExpression.child(1));
292+
}
293+
}
294+
throw new AnalysisException("Can not dereference field: " + dereferenceExpression.fieldName);
295+
}
296+
259297
@Override
260298
public Expression visitUnboundSlot(UnboundSlot unboundSlot, ExpressionRewriteContext context) {
261299
Optional<Scope> outerScope = getScope().getOuterScope();
@@ -937,13 +975,13 @@ protected List<? extends Expression> bindSlotByThisScope(UnboundSlot unboundSlot
937975
return bindSlotByScope(unboundSlot, getScope());
938976
}
939977

940-
protected List<Slot> bindExactSlotsByThisScope(UnboundSlot unboundSlot, Scope scope) {
941-
List<Slot> candidates = bindSlotByScope(unboundSlot, scope);
978+
protected List<Expression> bindExactSlotsByThisScope(UnboundSlot unboundSlot, Scope scope) {
979+
List<Expression> candidates = bindSlotByScope(unboundSlot, scope);
942980
if (candidates.size() == 1) {
943981
return candidates;
944982
}
945-
List<Slot> extractSlots = Utils.filterImmutableList(candidates, bound ->
946-
unboundSlot.getNameParts().size() == bound.getQualifier().size() + 1
983+
List<Expression> extractSlots = Utils.filterImmutableList(candidates, bound ->
984+
bound instanceof Slot && unboundSlot.getNameParts().size() == ((Slot) bound).getQualifier().size() + 1
947985
);
948986
// we should return origin candidates slots if extract slots is empty,
949987
// and then throw an ambiguous exception
@@ -962,33 +1000,150 @@ private List<Slot> addSqlIndexInfo(List<Slot> slots, Optional<Pair<Integer, Inte
9621000
}
9631001

9641002
/** bindSlotByScope */
965-
public List<Slot> bindSlotByScope(UnboundSlot unboundSlot, Scope scope) {
1003+
public List<Expression> bindSlotByScope(UnboundSlot unboundSlot, Scope scope) {
9661004
List<String> nameParts = unboundSlot.getNameParts();
9671005
Optional<Pair<Integer, Integer>> idxInSql = unboundSlot.getIndexInSqlString();
9681006
int namePartSize = nameParts.size();
9691007
switch (namePartSize) {
9701008
// column
9711009
case 1: {
972-
return addSqlIndexInfo(bindSingleSlotByName(nameParts.get(0), scope), idxInSql);
1010+
return (List<Expression>) bindExpressionByColumn(unboundSlot, nameParts, idxInSql, scope);
9731011
}
9741012
// table.column
9751013
case 2: {
976-
return addSqlIndexInfo(bindSingleSlotByTable(nameParts.get(0), nameParts.get(1), scope), idxInSql);
1014+
return (List<Expression>) bindExpressionByTableColumn(unboundSlot, nameParts, idxInSql, scope);
9771015
}
9781016
// db.table.column
9791017
case 3: {
980-
return addSqlIndexInfo(bindSingleSlotByDb(nameParts.get(0), nameParts.get(1), nameParts.get(2), scope),
981-
idxInSql);
1018+
return (List<Expression>) bindExpressionByDbTableColumn(unboundSlot, nameParts, idxInSql, scope);
9821019
}
9831020
// catalog.db.table.column
984-
case 4: {
985-
return addSqlIndexInfo(bindSingleSlotByCatalog(
986-
nameParts.get(0), nameParts.get(1), nameParts.get(2), nameParts.get(3), scope), idxInSql);
987-
}
9881021
default: {
989-
throw new AnalysisException("Not supported name: " + StringUtils.join(nameParts, "."));
1022+
return (List<Expression>) bindExpressionByCatalogDbTableColumn(unboundSlot, nameParts, idxInSql, scope);
1023+
}
1024+
}
1025+
}
1026+
1027+
private List<? extends Expression> bindExpressionByCatalogDbTableColumn(
1028+
UnboundSlot unboundSlot, List<String> nameParts, Optional<Pair<Integer, Integer>> idxInSql, Scope scope) {
1029+
List<Slot> slots = addSqlIndexInfo(bindSingleSlotByCatalog(
1030+
nameParts.get(0), nameParts.get(1), nameParts.get(2), nameParts.get(3), scope), idxInSql);
1031+
if (slots.isEmpty()) {
1032+
return bindExpressionByDbTableColumn(unboundSlot, nameParts, idxInSql, scope);
1033+
} else if (slots.size() > 1) {
1034+
return slots;
1035+
}
1036+
if (nameParts.size() == 4) {
1037+
return slots;
1038+
}
1039+
1040+
Optional<Expression> expression = bindNestedFields(
1041+
unboundSlot, slots.get(0), nameParts.subList(4, nameParts.size())
1042+
);
1043+
if (!expression.isPresent()) {
1044+
return slots;
1045+
}
1046+
return ImmutableList.of(expression.get());
1047+
}
1048+
1049+
private List<? extends Expression> bindExpressionByDbTableColumn(
1050+
UnboundSlot unboundSlot, List<String> nameParts, Optional<Pair<Integer, Integer>> idxInSql, Scope scope) {
1051+
List<Slot> slots = addSqlIndexInfo(
1052+
bindSingleSlotByDb(nameParts.get(0), nameParts.get(1), nameParts.get(2), scope), idxInSql);
1053+
if (slots.isEmpty()) {
1054+
return bindExpressionByTableColumn(unboundSlot, nameParts, idxInSql, scope);
1055+
} else if (slots.size() > 1) {
1056+
return slots;
1057+
}
1058+
if (nameParts.size() == 3) {
1059+
return slots;
1060+
}
1061+
1062+
Optional<Expression> expression = bindNestedFields(
1063+
unboundSlot, slots.get(0), nameParts.subList(3, nameParts.size())
1064+
);
1065+
if (!expression.isPresent()) {
1066+
return slots;
1067+
}
1068+
return ImmutableList.of(expression.get());
1069+
}
1070+
1071+
private List<? extends Expression> bindExpressionByTableColumn(
1072+
UnboundSlot unboundSlot, List<String> nameParts, Optional<Pair<Integer, Integer>> idxInSql, Scope scope) {
1073+
List<Slot> slots = addSqlIndexInfo(bindSingleSlotByTable(nameParts.get(0), nameParts.get(1), scope), idxInSql);
1074+
if (slots.isEmpty()) {
1075+
return bindExpressionByColumn(unboundSlot, nameParts, idxInSql, scope);
1076+
} else if (slots.size() > 1) {
1077+
return slots;
1078+
}
1079+
if (nameParts.size() == 2) {
1080+
return slots;
1081+
}
1082+
1083+
Optional<Expression> expression = bindNestedFields(
1084+
unboundSlot, slots.get(0), nameParts.subList(2, nameParts.size())
1085+
);
1086+
if (!expression.isPresent()) {
1087+
return slots;
1088+
}
1089+
return ImmutableList.of(expression.get());
1090+
}
1091+
1092+
private List<? extends Expression> bindExpressionByColumn(
1093+
UnboundSlot unboundSlot, List<String> nameParts, Optional<Pair<Integer, Integer>> idxInSql, Scope scope) {
1094+
List<Slot> slots = addSqlIndexInfo(bindSingleSlotByName(nameParts.get(0), scope), idxInSql);
1095+
if (slots.size() != 1) {
1096+
return slots;
1097+
}
1098+
if (nameParts.size() == 1) {
1099+
return slots;
1100+
}
1101+
Optional<Expression> expression = bindNestedFields(
1102+
unboundSlot, slots.get(0), nameParts.subList(1, nameParts.size())
1103+
);
1104+
if (!expression.isPresent()) {
1105+
return slots;
1106+
}
1107+
return ImmutableList.of(expression.get());
1108+
}
1109+
1110+
private Optional<Expression> bindNestedFields(UnboundSlot unboundSlot, Slot slot, List<String> fieldNames) {
1111+
Expression expression = slot;
1112+
String lastFieldName = slot.getName();
1113+
for (String fieldName : fieldNames) {
1114+
DataType dataType = expression.getDataType();
1115+
if (dataType.isStructType()) {
1116+
StructType structType = (StructType) dataType;
1117+
StructField field = structType.getField(fieldName);
1118+
if (field == null) {
1119+
throw new AnalysisException("No such struct field " + fieldName + " in " + lastFieldName);
1120+
}
1121+
lastFieldName = fieldName;
1122+
expression = new StructElement(expression, new StringLiteral(fieldName));
1123+
continue;
1124+
} else if (dataType.isMapType()) {
1125+
if (fieldName.equalsIgnoreCase("keys")) {
1126+
expression = new MapKeys(expression);
1127+
continue;
1128+
} else if (fieldName.equalsIgnoreCase("values")) {
1129+
expression = new MapValues(expression);
1130+
continue;
1131+
} else if (fieldName.equalsIgnoreCase("size")) {
1132+
expression = new MapSize(expression);
1133+
continue;
1134+
} else {
1135+
expression = new ElementAt(expression, new StringLiteral(fieldName));
1136+
continue;
1137+
}
1138+
} else if (dataType.isArrayType()) {
1139+
if (fieldName.equalsIgnoreCase("size")) {
1140+
expression = new Cardinality(expression);
1141+
continue;
1142+
}
9901143
}
1144+
throw new AnalysisException("No such field " + fieldName + " in " + lastFieldName);
9911145
}
1146+
return Optional.of(new Alias(expression));
9921147
}
9931148

9941149
public static boolean sameTableName(String boundSlot, String unboundSlot) {

0 commit comments

Comments
 (0)