Skip to content

Commit e723190

Browse files
starocean999starocean999
authored
[refactor](rec_cte)recursive-CTE refactor (apache#59872)
Refactor recursive-CTE fe part. Related PR: apache#58916 **Summary** - **Description:** Replace the legacy RecursiveCTE model with a clearer RecursiveUnion model (anchor / producer / work-table reference), update planner/physical nodes, runtime Thrift wiring, stats, visitors and tests to match. - **Motivation:** Separate anchor (initial rows) and producer (recursive body) responsibilities; make work-table explicit; deliver BE control/reset info via Thrift.
1 parent 6771470 commit e723190

File tree

56 files changed

+1548
-901
lines changed

Some content is hidden

Large Commits have some content hidden by default. Use the searchbox below for content that may be hidden.

56 files changed

+1548
-901
lines changed

‎fe/fe-core/src/main/java/org/apache/doris/analysis/DescriptorTable.java‎

Lines changed: 0 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -20,7 +20,6 @@
2020

2121
package org.apache.doris.analysis;
2222

23-
import org.apache.doris.catalog.RecursiveCteTempTable;
2423
import org.apache.doris.catalog.TableIf;
2524
import org.apache.doris.common.IdGenerator;
2625
import org.apache.doris.thrift.TDescriptorTable;
@@ -101,10 +100,6 @@ public TDescriptorTable toThrift() {
101100
}
102101

103102
for (TableIf tbl : referencedTbls.values()) {
104-
if (tbl instanceof RecursiveCteTempTable) {
105-
// skip recursive cte temp table
106-
continue;
107-
}
108103
result.addToTableDescriptors(tbl.toThrift());
109104
}
110105
thriftDescTable = result;

‎fe/fe-core/src/main/java/org/apache/doris/catalog/RecursiveCteTempTable.java‎

Lines changed: 0 additions & 35 deletions
This file was deleted.

‎fe/fe-core/src/main/java/org/apache/doris/catalog/TableIf.java‎

Lines changed: 1 addition & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -450,8 +450,7 @@ enum TableType {
450450
HUDI, JDBC,
451451
TABLE_VALUED_FUNCTION, HMS_EXTERNAL_TABLE, ES_EXTERNAL_TABLE, MATERIALIZED_VIEW, JDBC_EXTERNAL_TABLE,
452452
ICEBERG_EXTERNAL_TABLE, TEST_EXTERNAL_TABLE, PAIMON_EXTERNAL_TABLE, MAX_COMPUTE_EXTERNAL_TABLE,
453-
HUDI_EXTERNAL_TABLE, TRINO_CONNECTOR_EXTERNAL_TABLE, LAKESOUl_EXTERNAL_TABLE, DICTIONARY, DORIS_EXTERNAL_TABLE,
454-
RECURSIVE_CTE_TEMP_TABLE;
453+
HUDI_EXTERNAL_TABLE, TRINO_CONNECTOR_EXTERNAL_TABLE, LAKESOUl_EXTERNAL_TABLE, DICTIONARY, DORIS_EXTERNAL_TABLE;
455454

456455
public String toEngineName() {
457456
switch (this) {
@@ -494,8 +493,6 @@ public String toEngineName() {
494493
return "dictionary";
495494
case DORIS_EXTERNAL_TABLE:
496495
return "External_Doris";
497-
case RECURSIVE_CTE_TEMP_TABLE:
498-
return "RecursiveCteTempTable";
499496
default:
500497
return null;
501498
}
@@ -535,7 +532,6 @@ public String toMysqlType() {
535532
case MATERIALIZED_VIEW:
536533
case TRINO_CONNECTOR_EXTERNAL_TABLE:
537534
case DORIS_EXTERNAL_TABLE:
538-
case RECURSIVE_CTE_TEMP_TABLE:
539535
return "BASE TABLE";
540536
default:
541537
return null;

‎fe/fe-core/src/main/java/org/apache/doris/nereids/CTEContext.java‎

Lines changed: 25 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -19,13 +19,16 @@
1919

2020
import org.apache.doris.nereids.exceptions.AnalysisException;
2121
import org.apache.doris.nereids.trees.expressions.CTEId;
22+
import org.apache.doris.nereids.trees.expressions.Slot;
2223
import org.apache.doris.nereids.trees.plans.Plan;
2324
import org.apache.doris.nereids.trees.plans.logical.LogicalPlan;
2425
import org.apache.doris.nereids.trees.plans.logical.LogicalSubQueryAlias;
2526
import org.apache.doris.qe.GlobalVariable;
2627

28+
import com.google.common.collect.ImmutableList;
2729
import com.google.common.collect.ImmutableMap;
2830

31+
import java.util.List;
2932
import java.util.Locale;
3033
import java.util.Map;
3134
import java.util.Optional;
@@ -38,14 +41,15 @@ public class CTEContext {
3841

3942
private final CTEId cteId;
4043
private final String name;
44+
private List<Slot> recursiveCteOutputs;
4145
// this cache only use once
4246
private LogicalPlan analyzedPlan;
4347

4448
private final Map<String, CTEContext> cteContextMap;
4549

4650
/* build head CTEContext */
4751
public CTEContext() {
48-
this(CTEId.DEFAULT, null, null);
52+
this(CTEId.DEFAULT, null, (CTEContext) null);
4953
}
5054

5155
/**
@@ -66,6 +70,26 @@ public CTEContext(CTEId cteId, @Nullable LogicalSubQueryAlias<Plan> parsedPlan,
6670
// if inner name same with outer name, use inner name in this scope.
6771
.buildKeepingLast();
6872
this.cteId = cteId;
73+
this.recursiveCteOutputs = ImmutableList.of();
74+
}
75+
76+
/**
77+
* CTEContext for recursive cte
78+
*/
79+
public CTEContext(CTEId cteId, String cteName, List<Slot> recursiveCteOutputs) {
80+
this.cteId = cteId;
81+
this.name = GlobalVariable.lowerCaseTableNames != 0 ? cteName.toLowerCase(Locale.ROOT) : cteName;
82+
this.recursiveCteOutputs = recursiveCteOutputs != null ? ImmutableList.copyOf(recursiveCteOutputs)
83+
: ImmutableList.of();
84+
this.cteContextMap = ImmutableMap.of(name, this);
85+
}
86+
87+
public void setRecursiveCteOutputs(List<Slot> recursiveCteOutputs) {
88+
this.recursiveCteOutputs = recursiveCteOutputs;
89+
}
90+
91+
public List<Slot> getRecursiveCteOutputs() {
92+
return recursiveCteOutputs;
6993
}
7094

7195
public void setAnalyzedPlan(LogicalPlan analyzedPlan) {

‎fe/fe-core/src/main/java/org/apache/doris/nereids/CascadesContext.java‎

Lines changed: 16 additions & 21 deletions
Original file line numberDiff line numberDiff line change
@@ -133,8 +133,7 @@ public class CascadesContext implements ScheduleContext {
133133
private final boolean isEnableExprTrace;
134134

135135
private int groupExpressionCount = 0;
136-
private Optional<String> currentRecursiveCteName;
137-
private List<Slot> recursiveCteOutputs;
136+
private Optional<CTEContext> recursiveCteContext;
138137

139138
/**
140139
* Constructor of OptimizerContext.
@@ -145,7 +144,7 @@ public class CascadesContext implements ScheduleContext {
145144
private CascadesContext(Optional<CascadesContext> parent, Optional<CTEId> currentTree,
146145
StatementContext statementContext, Plan plan, Memo memo,
147146
CTEContext cteContext, PhysicalProperties requireProperties, boolean isLeadingDisableJoinReorder,
148-
Optional<String> currentRecursiveCteName, List<Slot> recursiveCteOutputs) {
147+
CTEContext recursiveCteContext) {
149148
this.parent = Objects.requireNonNull(parent, "parent should not null");
150149
this.currentTree = Objects.requireNonNull(currentTree, "currentTree should not null");
151150
this.statementContext = Objects.requireNonNull(statementContext, "statementContext should not null");
@@ -170,8 +169,7 @@ private CascadesContext(Optional<CascadesContext> parent, Optional<CTEId> curren
170169
this.isEnableExprTrace = false;
171170
}
172171
this.isLeadingDisableJoinReorder = isLeadingDisableJoinReorder;
173-
this.currentRecursiveCteName = currentRecursiveCteName;
174-
this.recursiveCteOutputs = recursiveCteOutputs;
172+
this.recursiveCteContext = Optional.ofNullable(recursiveCteContext);
175173
}
176174

177175
/** init a temporary context to rewrite expression */
@@ -186,7 +184,7 @@ public static CascadesContext initTempContext() {
186184
}
187185
return newContext(Optional.empty(), Optional.empty(),
188186
statementContext, DUMMY_PLAN,
189-
new CTEContext(), PhysicalProperties.ANY, false, Optional.empty(), ImmutableList.of());
187+
new CTEContext(), PhysicalProperties.ANY, false, null);
190188
}
191189

192190
/**
@@ -195,25 +193,23 @@ public static CascadesContext initTempContext() {
195193
public static CascadesContext initContext(StatementContext statementContext,
196194
Plan initPlan, PhysicalProperties requireProperties) {
197195
return newContext(Optional.empty(), Optional.empty(), statementContext,
198-
initPlan, new CTEContext(), requireProperties, false, Optional.empty(), ImmutableList.of());
196+
initPlan, new CTEContext(), requireProperties, false, null);
199197
}
200198

201199
/**
202200
* use for analyze cte. we must pass CteContext from outer since we need to get right scope of cte
203201
*/
204202
public static CascadesContext newContextWithCteContext(CascadesContext cascadesContext,
205-
Plan initPlan, CTEContext cteContext, Optional<String> currentRecursiveCteName,
206-
List<Slot> recursiveCteOutputs) {
203+
Plan initPlan, CTEContext cteContext, CTEContext recursiveCteContext) {
207204
return newContext(Optional.of(cascadesContext), Optional.empty(),
208205
cascadesContext.getStatementContext(), initPlan, cteContext, PhysicalProperties.ANY,
209-
cascadesContext.isLeadingDisableJoinReorder, currentRecursiveCteName, recursiveCteOutputs);
206+
cascadesContext.isLeadingDisableJoinReorder, recursiveCteContext);
210207
}
211208

212209
public static CascadesContext newCurrentTreeContext(CascadesContext context) {
213210
return CascadesContext.newContext(context.getParent(), context.getCurrentTree(), context.getStatementContext(),
214211
context.getRewritePlan(), context.getCteContext(),
215-
context.getCurrentJobContext().getRequiredProperties(), context.isLeadingDisableJoinReorder,
216-
Optional.empty(), ImmutableList.of());
212+
context.getCurrentJobContext().getRequiredProperties(), context.isLeadingDisableJoinReorder, null);
217213
}
218214

219215
/**
@@ -222,17 +218,15 @@ public static CascadesContext newCurrentTreeContext(CascadesContext context) {
222218
public static CascadesContext newSubtreeContext(Optional<CTEId> subtree, CascadesContext context,
223219
Plan plan, PhysicalProperties requireProperties) {
224220
return CascadesContext.newContext(Optional.of(context), subtree, context.getStatementContext(),
225-
plan, context.getCteContext(), requireProperties, context.isLeadingDisableJoinReorder, Optional.empty(),
226-
ImmutableList.of());
221+
plan, context.getCteContext(), requireProperties, context.isLeadingDisableJoinReorder, null);
227222
}
228223

229224
private static CascadesContext newContext(Optional<CascadesContext> parent, Optional<CTEId> subtree,
230225
StatementContext statementContext, Plan initPlan, CTEContext cteContext,
231226
PhysicalProperties requireProperties, boolean isLeadingDisableJoinReorder,
232-
Optional<String> currentRecursiveCteName, List<Slot> recursiveCteOutputs) {
227+
CTEContext recursiveCteContext) {
233228
return new CascadesContext(parent, subtree, statementContext, initPlan, null,
234-
cteContext, requireProperties, isLeadingDisableJoinReorder, currentRecursiveCteName,
235-
recursiveCteOutputs);
229+
cteContext, requireProperties, isLeadingDisableJoinReorder, recursiveCteContext);
236230
}
237231

238232
public CascadesContext getRoot() {
@@ -259,16 +253,17 @@ public synchronized boolean isTimeout() {
259253
return isTimeout;
260254
}
261255

262-
public Optional<String> getCurrentRecursiveCteName() {
263-
return currentRecursiveCteName;
256+
public Optional<CTEContext> getRecursiveCteContext() {
257+
return recursiveCteContext;
264258
}
265259

266260
public List<Slot> getRecursiveCteOutputs() {
267-
return recursiveCteOutputs;
261+
return recursiveCteContext.isPresent() ? recursiveCteContext.get().getRecursiveCteOutputs()
262+
: ImmutableList.of();
268263
}
269264

270265
public boolean isAnalyzingRecursiveCteAnchorChild() {
271-
return currentRecursiveCteName.isPresent() && recursiveCteOutputs.isEmpty();
266+
return recursiveCteContext.isPresent() && recursiveCteContext.get().getRecursiveCteOutputs().isEmpty();
272267
}
273268

274269
/**

‎fe/fe-core/src/main/java/org/apache/doris/nereids/StatementContext.java‎

Lines changed: 10 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -291,6 +291,8 @@ public enum TableFrom {
291291
private List<org.apache.iceberg.FileScanTask> icebergRewriteFileScanTasks = null;
292292
private boolean hasNestedColumns;
293293

294+
private final Set<CTEId> mustInlineCTE = new HashSet<>();
295+
294296
public StatementContext() {
295297
this(ConnectContext.get(), null, 0);
296298
}
@@ -1060,4 +1062,12 @@ public boolean hasNestedColumns() {
10601062
public void setHasNestedColumns(boolean hasNestedColumns) {
10611063
this.hasNestedColumns = hasNestedColumns;
10621064
}
1065+
1066+
public void addToMustLineCTEs(CTEId cteId) {
1067+
mustInlineCTE.add(cteId);
1068+
}
1069+
1070+
public Set<CTEId> getMustInlineCTEs() {
1071+
return mustInlineCTE;
1072+
}
10631073
}

0 commit comments

Comments
 (0)