Skip to content

Commit d0000c5

Browse files
morrySnowmorrySnow
authored
[refactor](load) remove castToSlot in load planner (#57489)
castToSlot use lagecy planner's Expr#castTo. we should not rely on it anymore
1 parent 99bd0cb commit d0000c5

File tree

1 file changed

+39
-44
lines changed

1 file changed

+39
-44
lines changed

‎fe/fe-core/src/main/java/org/apache/doris/nereids/load/NereidsLoadPlanInfoCollector.java‎

Lines changed: 39 additions & 44 deletions
Original file line numberDiff line numberDiff line change
@@ -19,10 +19,8 @@
1919

2020
import org.apache.doris.analysis.DescriptorTable;
2121
import org.apache.doris.analysis.Expr;
22-
import org.apache.doris.analysis.NullLiteral;
2322
import org.apache.doris.analysis.SlotDescriptor;
2423
import org.apache.doris.analysis.SlotId;
25-
import org.apache.doris.analysis.StringLiteral;
2624
import org.apache.doris.analysis.TupleDescriptor;
2725
import org.apache.doris.analysis.TupleId;
2826
import org.apache.doris.catalog.AggregateFunction;
@@ -33,7 +31,6 @@
3331
import org.apache.doris.catalog.PartitionInfo;
3432
import org.apache.doris.catalog.PartitionItem;
3533
import org.apache.doris.catalog.PartitionType;
36-
import org.apache.doris.catalog.PrimitiveType;
3734
import org.apache.doris.catalog.TableIf;
3835
import org.apache.doris.common.Config;
3936
import org.apache.doris.common.DdlException;
@@ -44,16 +41,24 @@
4441
import org.apache.doris.info.PartitionNamesInfo;
4542
import org.apache.doris.nereids.CascadesContext;
4643
import org.apache.doris.nereids.StatementContext;
44+
import org.apache.doris.nereids.analyzer.Scope;
4745
import org.apache.doris.nereids.exceptions.AnalysisException;
4846
import org.apache.doris.nereids.glue.translator.ExpressionTranslator;
4947
import org.apache.doris.nereids.glue.translator.PlanTranslatorContext;
48+
import org.apache.doris.nereids.parser.NereidsParser;
5049
import org.apache.doris.nereids.properties.PhysicalProperties;
50+
import org.apache.doris.nereids.rules.analysis.ExpressionAnalyzer;
51+
import org.apache.doris.nereids.rules.expression.rules.ConvertAggStateCast;
5152
import org.apache.doris.nereids.rules.expression.rules.PartitionPruner;
5253
import org.apache.doris.nereids.rules.expression.rules.SortedPartitionRanges;
54+
import org.apache.doris.nereids.trees.expressions.Alias;
55+
import org.apache.doris.nereids.trees.expressions.Cast;
5356
import org.apache.doris.nereids.trees.expressions.Expression;
5457
import org.apache.doris.nereids.trees.expressions.NamedExpression;
5558
import org.apache.doris.nereids.trees.expressions.Slot;
5659
import org.apache.doris.nereids.trees.expressions.SlotReference;
60+
import org.apache.doris.nereids.trees.expressions.literal.NullLiteral;
61+
import org.apache.doris.nereids.trees.expressions.literal.StringLiteral;
5762
import org.apache.doris.nereids.trees.plans.Plan;
5863
import org.apache.doris.nereids.trees.plans.logical.LogicalFilter;
5964
import org.apache.doris.nereids.trees.plans.logical.LogicalOlapTableSink;
@@ -62,6 +67,9 @@
6267
import org.apache.doris.nereids.trees.plans.logical.LogicalPreFilter;
6368
import org.apache.doris.nereids.trees.plans.logical.LogicalProject;
6469
import org.apache.doris.nereids.trees.plans.visitor.DefaultPlanVisitor;
70+
import org.apache.doris.nereids.types.DataType;
71+
import org.apache.doris.nereids.types.VarcharType;
72+
import org.apache.doris.nereids.util.TypeCoercionUtils;
6573
import org.apache.doris.planner.GroupCommitBlockSink;
6674
import org.apache.doris.planner.OlapTableSink;
6775
import org.apache.doris.thrift.TExpr;
@@ -75,6 +83,7 @@
7583
import org.apache.doris.thrift.TUniqueKeyUpdateMode;
7684

7785
import com.google.common.base.Preconditions;
86+
import com.google.common.collect.ImmutableList;
7887
import com.google.common.collect.Lists;
7988
import com.google.common.collect.Maps;
8089

@@ -339,9 +348,6 @@ public Void visitLogicalProject(LogicalProject<? extends Plan> logicalProject, P
339348
}
340349
}
341350

342-
List<Expr> projectList = outputs.stream().map(e -> ExpressionTranslator.translate(e, context))
343-
.collect(Collectors.toList());
344-
345351
// For Broker load with multiple file groups, all file groups share the same destTuple.
346352
// Create slots for destTuple only when processing the first file group (when slots are empty).
347353
// Subsequent file groups will reuse the slots created by the first file group.
@@ -374,23 +380,23 @@ public Void visitLogicalProject(LogicalProject<? extends Plan> logicalProject, P
374380
context.createSlotDesc(loadPlanInfo.destTuple, (SlotReference) slot, destTable);
375381
}
376382
}
377-
378-
loadPlanInfo.destSlotIdToExprMap = Maps.newHashMap();
379383
List<SlotDescriptor> slotDescriptorList = loadPlanInfo.destTuple.getSlots();
384+
loadPlanInfo.destSlotIdToExprMap = Maps.newHashMap();
380385
for (int i = 0; i < slotDescriptorList.size(); ++i) {
381-
SlotDescriptor slotDescriptor = slotDescriptorList.get(i);
382-
Expr expr = projectList.get(i);
383-
PrimitiveType dstType = slotDescriptor.getType().getPrimitiveType();
384-
PrimitiveType srcType = expr.getType().getPrimitiveType();
385-
if (!(dstType == PrimitiveType.JSONB
386-
&& (srcType == PrimitiveType.VARCHAR || srcType == PrimitiveType.STRING))) {
387-
try {
388-
expr = castToSlot(slotDescriptor, expr);
389-
} catch (org.apache.doris.common.AnalysisException e) {
390-
throw new AnalysisException(e.getMessage(), e.getCause());
386+
DataType targetType = DataType.fromCatalogType(slotDescriptorList.get(i).getType());
387+
Expression output = outputs.get(i);
388+
if (!(targetType.isJsonType() && output.getDataType().isStringLikeType())) {
389+
if (output instanceof Alias) {
390+
output = TypeCoercionUtils.castIfNotSameType(((Alias) output).child(), targetType);
391+
} else {
392+
output = TypeCoercionUtils.castIfNotSameType(output, targetType);
393+
}
394+
if (output instanceof Cast && output.getDataType().isAggStateType()) {
395+
output = ConvertAggStateCast.convert((Cast) output);
391396
}
392397
}
393-
loadPlanInfo.destSlotIdToExprMap.put(slotDescriptor.getId(), expr);
398+
Expr expr = ExpressionTranslator.translate(output, context);
399+
loadPlanInfo.destSlotIdToExprMap.put(slotDescriptorList.get(i).getId(), expr);
394400
}
395401
return null;
396402
}
@@ -470,33 +476,36 @@ public Void visitLogicalOneRowRelation(LogicalOneRowRelation oneRowRelation,
470476
for (SlotDescriptor slotDescriptor : oneRowTuple.getSlots()) {
471477
Column column = destTable.getColumn(slotDescriptor.getColumn().getName());
472478
if (column != null) {
473-
Expr expr;
479+
Expression expression;
474480
if (column.getDefaultValue() != null) {
475481
if (column.getDefaultValueExprDef() != null) {
476482
try {
477-
expr = column.getDefaultValueExpr();
483+
expression = new NereidsParser().parseExpression(
484+
column.getDefaultValueExpr().toSqlWithoutTbl());
485+
ExpressionAnalyzer analyzer = new ExpressionAnalyzer(
486+
null, new Scope(ImmutableList.of()), null, true, true);
487+
expression = analyzer.analyze(expression);
478488
} catch (org.apache.doris.common.AnalysisException e) {
479489
throw new AnalysisException(e.getMessage(), e.getCause());
480490
}
481491
} else {
482-
expr = new StringLiteral(column.getDefaultValue());
492+
expression = new StringLiteral(column.getDefaultValue());
483493
}
484494
} else {
485495
if (column.isAllowNull()) {
486-
expr = NullLiteral.create(org.apache.doris.catalog.Type.VARCHAR);
496+
expression = new NullLiteral(VarcharType.SYSTEM_DEFAULT);
487497
} else {
488-
expr = null;
498+
expression = null;
489499
}
490500
}
491501
if (exprMap.containsKey(column.getName())) {
492502
continue;
493503
}
494-
if (expr != null) {
495-
try {
496-
expr = castToSlot(slotDescriptor, expr);
497-
} catch (org.apache.doris.common.AnalysisException e) {
498-
throw new AnalysisException(e.getMessage(), e.getCause());
499-
}
504+
Expr expr = null;
505+
if (expression != null) {
506+
expression = TypeCoercionUtils.castIfNotSameType(expression,
507+
DataType.fromCatalogType(slotDescriptor.getType()));
508+
expr = ExpressionTranslator.translate(expression, context);
500509
}
501510
loadPlanInfo.srcSlotIdToDefaultValueMap.put(slotDescriptor.getId(), expr);
502511
srcSlots.put(column.getName(), slotDescriptor);
@@ -514,20 +523,6 @@ private TupleDescriptor generateTupleDesc(List<Slot> slotList, TableIf table, Pl
514523
return tupleDescriptor;
515524
}
516525

517-
private Expr castToSlot(SlotDescriptor slotDesc, Expr expr) throws org.apache.doris.common.AnalysisException {
518-
PrimitiveType dstType = slotDesc.getType().getPrimitiveType();
519-
PrimitiveType srcType = expr.getType().getPrimitiveType();
520-
if (PrimitiveType.typeWithPrecision.contains(dstType) && PrimitiveType.typeWithPrecision.contains(srcType)
521-
&& !slotDesc.getType().equals(expr.getType())) {
522-
return expr.castTo(slotDesc.getType());
523-
} else if (dstType != srcType || slotDesc.getType().isAggStateType() && expr.getType().isAggStateType()
524-
&& !slotDesc.getType().equals(expr.getType())) {
525-
return expr.castTo(slotDesc.getType());
526-
} else {
527-
return expr;
528-
}
529-
}
530-
531526
// get all specified partition ids.
532527
// if no partition specified, return null
533528
private List<Long> getAllPartitionIds() throws DdlException, AnalysisException {

0 commit comments

Comments
 (0)