Skip to content

Commit 69cda2d

Browse files
Wei-hao-LiWei-hao-Li
authored
Add more optimizers for union (apache#16689)
1 parent 685141a commit 69cda2d

File tree

6 files changed

+511
-8
lines changed

6 files changed

+511
-8
lines changed

‎iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/planner/iterative/rule/PushLimitThroughUnion.java‎

Lines changed: 107 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,107 @@
1+
/*
2+
* Licensed to the Apache Software Foundation (ASF) under one
3+
* or more contributor license agreements. See the NOTICE file
4+
* distributed with this work for additional information
5+
* regarding copyright ownership. The ASF licenses this file
6+
* to you under the Apache License, Version 2.0 (the
7+
* "License"); you may not use this file except in compliance
8+
* with the License. You may obtain a copy of the License at
9+
*
10+
* http://www.apache.org/licenses/LICENSE-2.0
11+
*
12+
* Unless required by applicable law or agreed to in writing,
13+
* software distributed under the License is distributed on an
14+
* "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
15+
* KIND, either express or implied. See the License for the
16+
* specific language governing permissions and limitations
17+
* under the License.
18+
*/
19+
20+
package org.apache.iotdb.db.queryengine.plan.relational.planner.iterative.rule;
21+
22+
import org.apache.iotdb.db.queryengine.plan.planner.plan.node.PlanNode;
23+
import org.apache.iotdb.db.queryengine.plan.relational.planner.iterative.Rule;
24+
import org.apache.iotdb.db.queryengine.plan.relational.planner.node.LimitNode;
25+
import org.apache.iotdb.db.queryengine.plan.relational.planner.node.UnionNode;
26+
import org.apache.iotdb.db.queryengine.plan.relational.utils.matching.Capture;
27+
import org.apache.iotdb.db.queryengine.plan.relational.utils.matching.Captures;
28+
import org.apache.iotdb.db.queryengine.plan.relational.utils.matching.Pattern;
29+
30+
import com.google.common.collect.ImmutableList;
31+
32+
import java.util.Optional;
33+
34+
import static org.apache.iotdb.db.queryengine.plan.relational.planner.node.Patterns.Limit.requiresPreSortedInputs;
35+
import static org.apache.iotdb.db.queryengine.plan.relational.planner.node.Patterns.limit;
36+
import static org.apache.iotdb.db.queryengine.plan.relational.planner.node.Patterns.source;
37+
import static org.apache.iotdb.db.queryengine.plan.relational.planner.node.Patterns.union;
38+
import static org.apache.iotdb.db.queryengine.plan.relational.planner.optimizations.QueryCardinalityUtil.isAtMost;
39+
import static org.apache.iotdb.db.queryengine.plan.relational.utils.matching.Capture.newCapture;
40+
41+
/**
42+
* Transforms:
43+
*
44+
* <pre>
45+
* - Limit
46+
* - Union
47+
* - relation1
48+
* - relation2
49+
* ..
50+
* </pre>
51+
*
52+
* Into:
53+
*
54+
* <pre>
55+
* - Limit
56+
* - Union
57+
* - Limit
58+
* - relation1
59+
* - Limit
60+
* - relation2
61+
* ..
62+
* </pre>
63+
*
64+
* Applies to LimitNode without ties only to avoid optimizer loop.
65+
*/
66+
public class PushLimitThroughUnion implements Rule<LimitNode> {
67+
private static final Capture<UnionNode> CHILD = newCapture();
68+
69+
private static final Pattern<LimitNode> PATTERN =
70+
limit()
71+
.matching(limit -> !limit.isWithTies())
72+
.with(requiresPreSortedInputs().equalTo(false))
73+
.with(source().matching(union().capturedAs(CHILD)));
74+
75+
@Override
76+
public Pattern<LimitNode> getPattern() {
77+
return PATTERN;
78+
}
79+
80+
@Override
81+
public Result apply(LimitNode parent, Captures captures, Context context) {
82+
UnionNode unionNode = captures.get(CHILD);
83+
ImmutableList.Builder<PlanNode> builder = ImmutableList.builder();
84+
boolean shouldApply = false;
85+
for (PlanNode child : unionNode.getChildren()) {
86+
// This check is to ensure that we don't fire the optimizer if it was previously applied.
87+
if (isAtMost(child, context.getLookup(), parent.getCount())) {
88+
builder.add(child);
89+
} else {
90+
shouldApply = true;
91+
builder.add(
92+
new LimitNode(
93+
context.getIdAllocator().genPlanNodeId(),
94+
child,
95+
parent.getCount(),
96+
Optional.empty()));
97+
}
98+
}
99+
100+
if (!shouldApply) {
101+
return Result.empty();
102+
}
103+
104+
return Result.ofPlanNode(
105+
parent.replaceChildren(ImmutableList.of(unionNode.replaceChildren(builder.build()))));
106+
}
107+
}

‎iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/planner/iterative/rule/PushProjectionThroughUnion.java‎

Lines changed: 111 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,111 @@
1+
/*
2+
* Licensed to the Apache Software Foundation (ASF) under one
3+
* or more contributor license agreements. See the NOTICE file
4+
* distributed with this work for additional information
5+
* regarding copyright ownership. The ASF licenses this file
6+
* to you under the Apache License, Version 2.0 (the
7+
* "License"); you may not use this file except in compliance
8+
* with the License. You may obtain a copy of the License at
9+
*
10+
* http://www.apache.org/licenses/LICENSE-2.0
11+
*
12+
* Unless required by applicable law or agreed to in writing,
13+
* software distributed under the License is distributed on an
14+
* "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
15+
* KIND, either express or implied. See the License for the
16+
* specific language governing permissions and limitations
17+
* under the License.
18+
*/
19+
20+
package org.apache.iotdb.db.queryengine.plan.relational.planner.iterative.rule;
21+
22+
import org.apache.iotdb.db.queryengine.plan.planner.plan.node.PlanNode;
23+
import org.apache.iotdb.db.queryengine.plan.relational.planner.Assignments;
24+
import org.apache.iotdb.db.queryengine.plan.relational.planner.Symbol;
25+
import org.apache.iotdb.db.queryengine.plan.relational.planner.iterative.Rule;
26+
import org.apache.iotdb.db.queryengine.plan.relational.planner.node.ProjectNode;
27+
import org.apache.iotdb.db.queryengine.plan.relational.planner.node.UnionNode;
28+
import org.apache.iotdb.db.queryengine.plan.relational.sql.ast.Expression;
29+
import org.apache.iotdb.db.queryengine.plan.relational.sql.ast.SymbolReference;
30+
import org.apache.iotdb.db.queryengine.plan.relational.utils.matching.Capture;
31+
import org.apache.iotdb.db.queryengine.plan.relational.utils.matching.Captures;
32+
import org.apache.iotdb.db.queryengine.plan.relational.utils.matching.Pattern;
33+
34+
import com.google.common.collect.ImmutableList;
35+
import com.google.common.collect.ImmutableListMultimap;
36+
import org.apache.tsfile.read.common.type.Type;
37+
38+
import java.util.HashMap;
39+
import java.util.List;
40+
import java.util.Map;
41+
42+
import static org.apache.iotdb.db.queryengine.plan.relational.planner.ExpressionSymbolInliner.inlineSymbols;
43+
import static org.apache.iotdb.db.queryengine.plan.relational.planner.node.Patterns.project;
44+
import static org.apache.iotdb.db.queryengine.plan.relational.planner.node.Patterns.source;
45+
import static org.apache.iotdb.db.queryengine.plan.relational.planner.node.Patterns.union;
46+
import static org.apache.iotdb.db.queryengine.plan.relational.utils.matching.Capture.newCapture;
47+
48+
public class PushProjectionThroughUnion implements Rule<ProjectNode> {
49+
private static final Capture<UnionNode> CHILD = newCapture();
50+
51+
private static final Pattern<ProjectNode> PATTERN =
52+
project()
53+
.matching(PushProjectionThroughUnion::nonTrivialProjection)
54+
.with(source().matching(union().capturedAs(CHILD)));
55+
56+
@Override
57+
public Pattern<ProjectNode> getPattern() {
58+
return PATTERN;
59+
}
60+
61+
@Override
62+
public Result apply(ProjectNode parent, Captures captures, Context context) {
63+
UnionNode child = captures.get(CHILD);
64+
65+
// OutputLayout of the resultant Union, will be same as the layout of the Project
66+
List<Symbol> outputLayout = parent.getOutputSymbols();
67+
68+
// Mapping from the output symbol to ordered list of symbols from each of the children
69+
ImmutableListMultimap.Builder<Symbol, Symbol> mappings = ImmutableListMultimap.builder();
70+
71+
// sources for the resultant UnionNode
72+
ImmutableList.Builder<PlanNode> outputSources = ImmutableList.builder();
73+
74+
for (int i = 0; i < child.getChildren().size(); i++) {
75+
Map<Symbol, SymbolReference> outputToInput =
76+
child.sourceSymbolMap(i); // Map: output of union -> input of this child to the union
77+
Assignments.Builder assignments =
78+
Assignments.builder(); // assignments for the new ProjectNode
79+
80+
// mapping from current ProjectNode to new ProjectNode, used to identify the output layout
81+
Map<Symbol, Symbol> projectSymbolMapping = new HashMap<>();
82+
83+
// Translate the assignments in the ProjectNode using symbols of the source of the UnionNode
84+
for (Map.Entry<Symbol, Expression> entry : parent.getAssignments().entrySet()) {
85+
Expression translatedExpression = inlineSymbols(outputToInput, entry.getValue());
86+
Type type = context.getSymbolAllocator().getTypes().getTableModelType(entry.getKey());
87+
Symbol symbol = context.getSymbolAllocator().newSymbol(translatedExpression, type);
88+
assignments.put(symbol, translatedExpression);
89+
projectSymbolMapping.put(entry.getKey(), symbol);
90+
}
91+
outputSources.add(
92+
new ProjectNode(
93+
context.getIdAllocator().genPlanNodeId(),
94+
child.getChildren().get(i),
95+
assignments.build()));
96+
outputLayout.forEach(symbol -> mappings.put(symbol, projectSymbolMapping.get(symbol)));
97+
}
98+
99+
return Result.ofPlanNode(
100+
new UnionNode(
101+
parent.getPlanNodeId(),
102+
outputSources.build(),
103+
mappings.build(),
104+
ImmutableList.copyOf(mappings.build().keySet())));
105+
}
106+
107+
private static boolean nonTrivialProjection(ProjectNode project) {
108+
return !project.getAssignments().getExpressions().stream()
109+
.allMatch(SymbolReference.class::isInstance);
110+
}
111+
}

‎iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/planner/iterative/rule/PushTopKThroughUnion.java‎

Lines changed: 102 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,102 @@
1+
/*
2+
* Licensed to the Apache Software Foundation (ASF) under one
3+
* or more contributor license agreements. See the NOTICE file
4+
* distributed with this work for additional information
5+
* regarding copyright ownership. The ASF licenses this file
6+
* to you under the Apache License, Version 2.0 (the
7+
* "License"); you may not use this file except in compliance
8+
* with the License. You may obtain a copy of the License at
9+
*
10+
* http://www.apache.org/licenses/LICENSE-2.0
11+
*
12+
* Unless required by applicable law or agreed to in writing,
13+
* software distributed under the License is distributed on an
14+
* "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
15+
* KIND, either express or implied. See the License for the
16+
* specific language governing permissions and limitations
17+
* under the License.
18+
*/
19+
20+
package org.apache.iotdb.db.queryengine.plan.relational.planner.iterative.rule;
21+
22+
import org.apache.iotdb.db.queryengine.plan.planner.plan.node.PlanNode;
23+
import org.apache.iotdb.db.queryengine.plan.relational.planner.Symbol;
24+
import org.apache.iotdb.db.queryengine.plan.relational.planner.iterative.Rule;
25+
import org.apache.iotdb.db.queryengine.plan.relational.planner.node.TopKNode;
26+
import org.apache.iotdb.db.queryengine.plan.relational.planner.node.UnionNode;
27+
import org.apache.iotdb.db.queryengine.plan.relational.planner.optimizations.SymbolMapper;
28+
import org.apache.iotdb.db.queryengine.plan.relational.utils.matching.Capture;
29+
import org.apache.iotdb.db.queryengine.plan.relational.utils.matching.Captures;
30+
import org.apache.iotdb.db.queryengine.plan.relational.utils.matching.Pattern;
31+
32+
import com.google.common.collect.ImmutableList;
33+
import com.google.common.collect.ImmutableSet;
34+
35+
import java.util.Collections;
36+
import java.util.Set;
37+
38+
import static com.google.common.collect.Iterables.getLast;
39+
import static com.google.common.collect.Sets.intersection;
40+
import static org.apache.iotdb.db.queryengine.plan.relational.planner.node.Patterns.source;
41+
import static org.apache.iotdb.db.queryengine.plan.relational.planner.node.Patterns.topK;
42+
import static org.apache.iotdb.db.queryengine.plan.relational.planner.node.Patterns.union;
43+
import static org.apache.iotdb.db.queryengine.plan.relational.planner.optimizations.QueryCardinalityUtil.isAtMost;
44+
import static org.apache.iotdb.db.queryengine.plan.relational.utils.matching.Capture.newCapture;
45+
46+
public class PushTopKThroughUnion implements Rule<TopKNode> {
47+
private static final Capture<UnionNode> CHILD = newCapture();
48+
49+
private static final Pattern<TopKNode> PATTERN =
50+
topK().with(source().matching(union().capturedAs(CHILD)));
51+
52+
@Override
53+
public Pattern<TopKNode> getPattern() {
54+
return PATTERN;
55+
}
56+
57+
@Override
58+
public Result apply(TopKNode topKNode, Captures captures, Context context) {
59+
UnionNode unionNode = captures.get(CHILD);
60+
61+
ImmutableList.Builder<PlanNode> children = ImmutableList.builder();
62+
63+
boolean shouldApply = false;
64+
for (PlanNode child : unionNode.getChildren()) {
65+
SymbolMapper.Builder symbolMapper = SymbolMapper.builder();
66+
Set<Symbol> sourceOutputSymbols = ImmutableSet.copyOf(child.getOutputSymbols());
67+
// This check is to ensure that we don't fire the optimizer if it was previously applied,
68+
// which is the same as PushLimitThroughUnion.
69+
if (isAtMost(child, context.getLookup(), topKNode.getCount())) {
70+
children.add(child);
71+
} else {
72+
shouldApply = true;
73+
for (Symbol unionOutput : unionNode.getOutputSymbols()) {
74+
Set<Symbol> inputSymbols =
75+
ImmutableSet.copyOf(unionNode.getSymbolMapping().get(unionOutput));
76+
Symbol unionInput = getLast(intersection(inputSymbols, sourceOutputSymbols));
77+
symbolMapper.put(unionOutput, unionInput);
78+
}
79+
children.add(
80+
symbolMapper
81+
.build()
82+
.map(
83+
topKNode,
84+
Collections.singletonList(child),
85+
context.getIdAllocator().genPlanNodeId()));
86+
}
87+
}
88+
89+
if (!shouldApply) {
90+
return Result.empty();
91+
}
92+
93+
return Result.ofPlanNode(
94+
topKNode.replaceChildren(
95+
Collections.singletonList(
96+
new UnionNode(
97+
unionNode.getPlanNodeId(),
98+
children.build(),
99+
unionNode.getSymbolMapping(),
100+
unionNode.getOutputSymbols()))));
101+
}
102+
}

0 commit comments

Comments
 (0)