From 010aec8df6a60aec3ec37e6af40a77f766ec3149 Mon Sep 17 00:00:00 2001 From: lancelly <1435078631@qq.com> Date: Fri, 6 Dec 2024 16:29:32 +0800 Subject: [PATCH 01/26] Add support for row type && Add SemiJoinNode && Add uncorrelated InPredicate related rules --- .../plan/planner/plan/node/PlanNodeType.java | 4 + .../plan/planner/plan/node/PlanVisitor.java | 5 + .../relational/planner/IrTypeAnalyzer.java | 11 ++ .../relational/planner/SubqueryPlanner.java | 7 +- .../relational/planner/TranslationMap.java | 8 + .../planner/ir/ExpressionRewriter.java | 6 + .../planner/ir/ExpressionTreeRewriter.java | 44 ++++++ .../iterative/rule/PruneApplyColumns.java | 138 ++++++++++++++++ .../iterative/rule/PruneApplyCorrelation.java | 70 ++++++++ .../rule/PruneApplySourceColumns.java | 95 +++++++++++ .../RemoveUnreferencedScalarApplyNodes.java | 42 +++++ .../RemoveUnreferencedScalarSubqueries.java | 70 ++++++++ ...rrelatedInPredicateSubqueryToSemiJoin.java | 95 +++++++++++ .../relational/planner/node/SemiJoinNode.java | 148 +++++++++++++++++ .../optimizations/LogicalOptimizeFactory.java | 13 +- .../UnaliasSymbolReferences.java | 29 ++++ .../plan/relational/sql/ast/AstVisitor.java | 8 + .../plan/relational/sql/ast/RowDataType.java | 149 ++++++++++++++++++ .../sql/util/ExpressionFormatter.java | 20 +++ .../relational/type/InternalTypeManager.java | 44 +++++- .../plan/relational/type/NamedType.java | 62 ++++++++ .../plan/relational/type/ParametricType.java | 30 ++++ .../relational/type/RowParametricType.java | 59 +++++++ .../plan/relational/type/TypeParameter.java | 128 +++++++++++++++ .../type/TypeSignatureTranslator.java | 48 +++++- 25 files changed, 1322 insertions(+), 11 deletions(-) create mode 100644 iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/planner/iterative/rule/PruneApplyColumns.java create mode 100644 iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/planner/iterative/rule/PruneApplyCorrelation.java create mode 100644 iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/planner/iterative/rule/PruneApplySourceColumns.java create mode 100644 iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/planner/iterative/rule/RemoveUnreferencedScalarApplyNodes.java create mode 100644 iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/planner/iterative/rule/RemoveUnreferencedScalarSubqueries.java create mode 100644 iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/planner/iterative/rule/TransformUncorrelatedInPredicateSubqueryToSemiJoin.java create mode 100644 iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/planner/node/SemiJoinNode.java create mode 100644 iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/sql/ast/RowDataType.java create mode 100644 iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/type/NamedType.java create mode 100644 iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/type/ParametricType.java create mode 100644 iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/type/RowParametricType.java create mode 100644 iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/type/TypeParameter.java diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/planner/plan/node/PlanNodeType.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/planner/plan/node/PlanNodeType.java index 3e53bc696909..62fa6362edc9 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/planner/plan/node/PlanNodeType.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/planner/plan/node/PlanNodeType.java @@ -120,6 +120,7 @@ import org.apache.iotdb.db.queryengine.plan.relational.planner.node.GapFillNode; import org.apache.iotdb.db.queryengine.plan.relational.planner.node.LinearFillNode; import org.apache.iotdb.db.queryengine.plan.relational.planner.node.PreviousFillNode; +import org.apache.iotdb.db.queryengine.plan.relational.planner.node.SemiJoinNode; import org.apache.iotdb.db.queryengine.plan.relational.planner.node.ValueFillNode; import org.apache.iotdb.db.queryengine.plan.relational.planner.node.schema.ConstructTableDevicesBlackListNode; import org.apache.iotdb.db.queryengine.plan.relational.planner.node.schema.CreateOrUpdateTableDeviceNode; @@ -280,6 +281,7 @@ public enum PlanNodeType { TABLE_EXCHANGE_NODE((short) 1018), TABLE_EXPLAIN_ANALYZE_NODE((short) 1019), TABLE_ENFORCE_SINGLE_ROW_NODE((short) 1020), + TABLE_SEMI_JOIN_NODE((short) 1021), RELATIONAL_INSERT_TABLET((short) 2000), RELATIONAL_INSERT_ROW((short) 2001), @@ -637,6 +639,8 @@ public static PlanNode deserialize(ByteBuffer buffer, short nodeType) { throw new UnsupportedOperationException("ExplainAnalyzeNode should not be deserialized"); case 1020: return EnforceSingleRowNode.deserialize(buffer); + case 1021: + return SemiJoinNode.deserialize(buffer); case 2000: return RelationalInsertTabletNode.deserialize(buffer); case 2001: diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/planner/plan/node/PlanVisitor.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/planner/plan/node/PlanVisitor.java index 682f2a65202a..0c09911b053e 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/planner/plan/node/PlanVisitor.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/planner/plan/node/PlanVisitor.java @@ -124,6 +124,7 @@ import org.apache.iotdb.db.queryengine.plan.relational.planner.node.GapFillNode; import org.apache.iotdb.db.queryengine.plan.relational.planner.node.LinearFillNode; import org.apache.iotdb.db.queryengine.plan.relational.planner.node.PreviousFillNode; +import org.apache.iotdb.db.queryengine.plan.relational.planner.node.SemiJoinNode; import org.apache.iotdb.db.queryengine.plan.relational.planner.node.TableScanNode; import org.apache.iotdb.db.queryengine.plan.relational.planner.node.ValueFillNode; import org.apache.iotdb.db.queryengine.plan.relational.planner.node.schema.ConstructTableDevicesBlackListNode; @@ -740,6 +741,10 @@ public R visitJoin( return visitTwoChildProcess(node, context); } + public R visitSemiJoin(SemiJoinNode node, C context) { + return visitPlan(node, context); + } + public R visitGroupReference(GroupReference node, C context) { return visitPlan(node, context); } diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/planner/IrTypeAnalyzer.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/planner/IrTypeAnalyzer.java index cdf01015f40e..1a19f40132f1 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/planner/IrTypeAnalyzer.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/planner/IrTypeAnalyzer.java @@ -54,6 +54,7 @@ import org.apache.iotdb.db.queryengine.plan.relational.sql.ast.NotExpression; import org.apache.iotdb.db.queryengine.plan.relational.sql.ast.NullIfExpression; import org.apache.iotdb.db.queryengine.plan.relational.sql.ast.NullLiteral; +import org.apache.iotdb.db.queryengine.plan.relational.sql.ast.Row; import org.apache.iotdb.db.queryengine.plan.relational.sql.ast.SearchedCaseExpression; import org.apache.iotdb.db.queryengine.plan.relational.sql.ast.SimpleCaseExpression; import org.apache.iotdb.db.queryengine.plan.relational.sql.ast.StringLiteral; @@ -63,6 +64,7 @@ import com.google.common.collect.ImmutableMap; import org.apache.tsfile.read.common.type.BlobType; import org.apache.tsfile.read.common.type.DateType; +import org.apache.tsfile.read.common.type.RowType; import org.apache.tsfile.read.common.type.StringType; import org.apache.tsfile.read.common.type.TimestampType; import org.apache.tsfile.read.common.type.Type; @@ -75,6 +77,7 @@ import java.util.stream.Collectors; import static com.google.common.base.Preconditions.checkArgument; +import static com.google.common.collect.ImmutableList.toImmutableList; import static java.util.Objects.requireNonNull; import static org.apache.iotdb.db.queryengine.plan.relational.type.TypeSignatureTranslator.toTypeSignature; import static org.apache.tsfile.read.common.type.BooleanType.BOOLEAN; @@ -416,6 +419,14 @@ protected Type visitInPredicate(InPredicate node, Context context) { return setExpressionType(node, BOOLEAN); } + @Override + protected Type visitRow(Row node, Context context) { + List types = + node.getItems().stream().map(child -> process(child, context)).collect(toImmutableList()); + + return setExpressionType(node, RowType.anonymous(types)); + } + @Override protected Type visitLikePredicate(LikePredicate node, Context context) { process(node.getValue(), context); diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/planner/SubqueryPlanner.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/planner/SubqueryPlanner.java index 82f90ac36da9..59b60abba088 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/planner/SubqueryPlanner.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/planner/SubqueryPlanner.java @@ -571,12 +571,7 @@ private PlanAndMappings planSubquery( new ProjectNode( idAllocator.genPlanNodeId(), relationPlan.getRoot(), - Assignments.of( - column, - new Cast( - new Row(fields.build()), - new GenericDataType( - new Identifier(type.toString()), ImmutableList.of()))))); + Assignments.of(column, new Cast(new Row(fields.build()), toSqlType(type))))); return coerceIfNecessary(subqueryPlan, column, subquery, coercion); } diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/planner/TranslationMap.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/planner/TranslationMap.java index 6a01f59e2b7c..80300c47760f 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/planner/TranslationMap.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/planner/TranslationMap.java @@ -28,6 +28,7 @@ import org.apache.iotdb.db.queryengine.plan.relational.sql.ast.LikePredicate; import org.apache.iotdb.db.queryengine.plan.relational.sql.ast.Parameter; import org.apache.iotdb.db.queryengine.plan.relational.sql.ast.QualifiedName; +import org.apache.iotdb.db.queryengine.plan.relational.sql.ast.RowDataType; import org.apache.iotdb.db.queryengine.plan.relational.sql.ast.SymbolReference; import org.apache.iotdb.db.queryengine.plan.relational.sql.ast.Trim; import org.apache.iotdb.db.queryengine.plan.relational.sql.util.AstUtil; @@ -357,6 +358,13 @@ public Expression rewriteGenericDataType( // do not rewrite identifiers within type parameters return node; } + + @Override + public Expression rewriteRowDataType( + RowDataType node, Void context, ExpressionTreeRewriter treeRewriter) { + // do not rewrite identifiers in field names + return node; + } }, expression); } diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/planner/ir/ExpressionRewriter.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/planner/ir/ExpressionRewriter.java index d232101d6037..735b74250410 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/planner/ir/ExpressionRewriter.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/planner/ir/ExpressionRewriter.java @@ -47,6 +47,7 @@ import org.apache.iotdb.db.queryengine.plan.relational.sql.ast.Parameter; import org.apache.iotdb.db.queryengine.plan.relational.sql.ast.QuantifiedComparisonExpression; import org.apache.iotdb.db.queryengine.plan.relational.sql.ast.Row; +import org.apache.iotdb.db.queryengine.plan.relational.sql.ast.RowDataType; import org.apache.iotdb.db.queryengine.plan.relational.sql.ast.SearchedCaseExpression; import org.apache.iotdb.db.queryengine.plan.relational.sql.ast.SimpleCaseExpression; import org.apache.iotdb.db.queryengine.plan.relational.sql.ast.SubqueryExpression; @@ -231,4 +232,9 @@ public Expression rewriteGenericDataType( GenericDataType node, C context, ExpressionTreeRewriter treeRewriter) { return rewriteExpression(node, context, treeRewriter); } + + public Expression rewriteRowDataType( + RowDataType node, C context, ExpressionTreeRewriter treeRewriter) { + return rewriteExpression(node, context, treeRewriter); + } } diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/planner/ir/ExpressionTreeRewriter.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/planner/ir/ExpressionTreeRewriter.java index cab412becfcb..1632fb513396 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/planner/ir/ExpressionTreeRewriter.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/planner/ir/ExpressionTreeRewriter.java @@ -50,6 +50,7 @@ import org.apache.iotdb.db.queryengine.plan.relational.sql.ast.Parameter; import org.apache.iotdb.db.queryengine.plan.relational.sql.ast.QuantifiedComparisonExpression; import org.apache.iotdb.db.queryengine.plan.relational.sql.ast.Row; +import org.apache.iotdb.db.queryengine.plan.relational.sql.ast.RowDataType; import org.apache.iotdb.db.queryengine.plan.relational.sql.ast.SearchedCaseExpression; import org.apache.iotdb.db.queryengine.plan.relational.sql.ast.SimpleCaseExpression; import org.apache.iotdb.db.queryengine.plan.relational.sql.ast.SubqueryExpression; @@ -642,6 +643,49 @@ public Expression visitCast(Cast node, Context context) { return node; } + @Override + protected Expression visitRowDataType(RowDataType node, Context context) { + if (!context.isDefaultRewrite()) { + Expression result = + rewriter.rewriteRowDataType(node, context.get(), ExpressionTreeRewriter.this); + if (result != null) { + return result; + } + } + + ImmutableList.Builder rewritten = ImmutableList.builder(); + for (RowDataType.Field field : node.getFields()) { + DataType dataType = rewrite(field.getType(), context.get()); + + Optional name = field.getName(); + + if (field.getName().isPresent()) { + Identifier identifier = field.getName().get(); + Identifier rewrittenIdentifier = rewrite(identifier, context.get()); + + if (identifier != rewrittenIdentifier) { + name = Optional.of(rewrittenIdentifier); + } + } + + @SuppressWarnings("OptionalEquality") + boolean nameRewritten = name != field.getName(); + if (dataType != field.getType() || nameRewritten) { + rewritten.add(new RowDataType.Field(field.getLocation(), name, dataType)); + } else { + rewritten.add(field); + } + } + + List fields = rewritten.build(); + + if (!sameElements(fields, node.getFields())) { + return new RowDataType(node.getLocation(), fields); + } + + return node; + } + @Override public Expression visitCurrentDatabase(final CurrentDatabase node, final Context context) { if (!context.isDefaultRewrite()) { diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/planner/iterative/rule/PruneApplyColumns.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/planner/iterative/rule/PruneApplyColumns.java new file mode 100644 index 000000000000..11dcaefba17a --- /dev/null +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/planner/iterative/rule/PruneApplyColumns.java @@ -0,0 +1,138 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.iotdb.db.queryengine.plan.relational.planner.iterative.rule; + +import org.apache.iotdb.db.queryengine.plan.planner.plan.node.PlanNode; +import org.apache.iotdb.db.queryengine.plan.relational.planner.Symbol; +import org.apache.iotdb.db.queryengine.plan.relational.planner.node.ApplyNode; + +import com.google.common.collect.ImmutableMap; +import com.google.common.collect.ImmutableSet; + +import java.util.Map; +import java.util.Optional; +import java.util.Set; + +import static com.google.common.collect.Sets.intersection; +import static org.apache.iotdb.db.queryengine.plan.relational.planner.iterative.rule.Util.restrictOutputs; +import static org.apache.iotdb.db.queryengine.plan.relational.planner.node.Patterns.applyNode; + +/** + * This rule restricts the outputs of ApplyNode's input and subquery based on which ApplyNode's + * output symbols are referenced. + * + *

A symbol from input source can be pruned, when - it is not a referenced output symbol - it is + * not a correlation symbol - it is not referenced in subqueryAssignments + * + *

A symbol from subquery source can be pruned, when it is not referenced in subqueryAssignments. + * + *

A subquery assignment can be removed, when its key is not a referenced output symbol. + * + *

Note: this rule does not remove any symbols from the correlation list. This is responsibility + * of PruneApplyCorrelation rule. + * + *

Transforms: + * + * 

+ * - Project (i1, r1)
+ *      - Apply
+ *          correlation: [corr]
+ *          assignments:
+ *              r1 -> a in s1,
+ *              r2 -> b in s2,
+ *          - Input (a, b, corr)
+ *          - Subquery (s1, s2)
+ *
+ * + * Into: + * + * 
+ * - Project (i1, r1)
+ *      - Apply
+ *          correlation: [corr]
+ *          assignments:
+ *              r1 -> a in s1,
+ *          - Project (a, corr)
+ *              - Input (a, b, corr)
+ *          - Project (s1)
+ *              - Subquery (s1, s2)
+ *
+ */ +public class PruneApplyColumns extends ProjectOffPushDownRule { + public PruneApplyColumns() { + super(applyNode()); + } + + @Override + protected Optional pushDownProjectOff( + Context context, ApplyNode applyNode, Set referencedOutputs) { + // remove unused apply node + if (intersection(applyNode.getSubqueryAssignments().keySet(), referencedOutputs).isEmpty()) { + return Optional.of(applyNode.getInput()); + } + + // extract referenced assignments + ImmutableSet.Builder requiredAssignmentsSymbols = ImmutableSet.builder(); + ImmutableMap.Builder newSubqueryAssignments = + ImmutableMap.builder(); + for (Map.Entry entry : + applyNode.getSubqueryAssignments().entrySet()) { + if (referencedOutputs.contains(entry.getKey())) { + requiredAssignmentsSymbols.addAll(entry.getValue().inputs()); + newSubqueryAssignments.put(entry); + } + } + + // prune subquery symbols + Optional newSubquery = + restrictOutputs( + context.getIdAllocator(), applyNode.getSubquery(), requiredAssignmentsSymbols.build()); + + // prune input symbols + Set requiredInputSymbols = + ImmutableSet.builder() + .addAll(referencedOutputs) + .addAll(applyNode.getCorrelation()) + .addAll(requiredAssignmentsSymbols.build()) + .build(); + + Optional newInput = + restrictOutputs(context.getIdAllocator(), applyNode.getInput(), requiredInputSymbols); + + boolean pruned = + newSubquery.isPresent() + || newInput.isPresent() + || newSubqueryAssignments.buildOrThrow().size() + < applyNode.getSubqueryAssignments().size(); + + if (pruned) { + return Optional.of( + new ApplyNode( + applyNode.getPlanNodeId(), + newInput.orElse(applyNode.getInput()), + newSubquery.orElse(applyNode.getSubquery()), + newSubqueryAssignments.buildOrThrow(), + applyNode.getCorrelation(), + applyNode.getOriginSubquery())); + } + + return Optional.empty(); + } +} diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/planner/iterative/rule/PruneApplyCorrelation.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/planner/iterative/rule/PruneApplyCorrelation.java new file mode 100644 index 000000000000..942c5ba6e639 --- /dev/null +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/planner/iterative/rule/PruneApplyCorrelation.java @@ -0,0 +1,70 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.iotdb.db.queryengine.plan.relational.planner.iterative.rule; + +import org.apache.iotdb.db.queryengine.plan.relational.planner.Symbol; +import org.apache.iotdb.db.queryengine.plan.relational.planner.iterative.Rule; +import org.apache.iotdb.db.queryengine.plan.relational.planner.node.ApplyNode; +import org.apache.iotdb.db.queryengine.plan.relational.utils.matching.Captures; +import org.apache.iotdb.db.queryengine.plan.relational.utils.matching.Pattern; + +import java.util.List; +import java.util.Set; + +import static com.google.common.collect.ImmutableList.toImmutableList; +import static org.apache.iotdb.db.queryengine.plan.relational.planner.SymbolsExtractor.extractUnique; +import static org.apache.iotdb.db.queryengine.plan.relational.planner.node.Patterns.applyNode; + +/** + * This rule updates ApplyNode's correlation list. A symbol can be removed from the correlation list + * if it is not referenced by the subquery node. Note: This rule does not restrict ApplyNode's + * children outputs. It requires additional information about context (symbols required by the outer + * plan) and is done in PruneApplyColumns rule. + */ +public class PruneApplyCorrelation implements Rule { + private static final Pattern PATTERN = applyNode(); + + @Override + public Pattern getPattern() { + return PATTERN; + } + + @Override + public Result apply(ApplyNode applyNode, Captures captures, Context context) { + Set subquerySymbols = extractUnique(applyNode.getSubquery(), context.getLookup()); + List newCorrelation = + applyNode.getCorrelation().stream() + .filter(subquerySymbols::contains) + .collect(toImmutableList()); + + if (newCorrelation.size() < applyNode.getCorrelation().size()) { + return Result.ofPlanNode( + new ApplyNode( + applyNode.getPlanNodeId(), + applyNode.getInput(), + applyNode.getSubquery(), + applyNode.getSubqueryAssignments(), + newCorrelation, + applyNode.getOriginSubquery())); + } + + return Result.empty(); + } +} diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/planner/iterative/rule/PruneApplySourceColumns.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/planner/iterative/rule/PruneApplySourceColumns.java new file mode 100644 index 000000000000..746866102d2c --- /dev/null +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/planner/iterative/rule/PruneApplySourceColumns.java @@ -0,0 +1,95 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.iotdb.db.queryengine.plan.relational.planner.iterative.rule; + +import org.apache.iotdb.db.queryengine.plan.planner.plan.node.PlanNode; +import org.apache.iotdb.db.queryengine.plan.relational.planner.Symbol; +import org.apache.iotdb.db.queryengine.plan.relational.planner.iterative.Rule; +import org.apache.iotdb.db.queryengine.plan.relational.planner.node.ApplyNode; +import org.apache.iotdb.db.queryengine.plan.relational.utils.matching.Captures; +import org.apache.iotdb.db.queryengine.plan.relational.utils.matching.Pattern; + +import com.google.common.collect.ImmutableList; + +import java.util.Optional; +import java.util.Set; + +import static com.google.common.collect.ImmutableSet.toImmutableSet; +import static org.apache.iotdb.db.queryengine.plan.relational.planner.iterative.rule.Util.restrictOutputs; +import static org.apache.iotdb.db.queryengine.plan.relational.planner.node.Patterns.applyNode; + +/** + * This rule restricts outputs of ApplyNode's subquery to include only the symbols needed for + * subqueryAssignments. Symbols from the subquery are not produced at ApplyNode's output. They are + * only used for the assignments. Transforms: + * + * 
+ * - Apply
+ *      correlation: [corr_symbol]
+ *      assignments:
+ *          result_1 -> a in subquery_symbol_1,
+ *          result_2 -> b > ALL subquery_symbol_2
+ *    - Input (a, b, corr_symbol)
+ *    - Subquery (subquery_symbol_1, subquery_symbol_2, subquery_symbol_3)
+ *
+ * + * Into: + * + * 
+ * - Apply
+ *      correlation: [corr_symbol]
+ *      assignments:
+ *          result_1 -> a in subquery_symbol_1,
+ *          result_2 -> b > ALL subquery_symbol_2
+ *    - Input (a, b, corr_symbol)
+ *    - Project
+ *          subquery_symbol_1 -> subquery_symbol_1
+ *          subquery_symbol_2 -> subquery_symbol_2
+ *        - Subquery (subquery_symbol_1, subquery_symbol_2, subquery_symbol_3)
+ *
+ * + * Note: ApplyNode's input symbols are produced on ApplyNode's output. They cannot be pruned without + * outer context. + */ +public class PruneApplySourceColumns implements Rule { + private static final Pattern PATTERN = applyNode(); + + @Override + public Pattern getPattern() { + return PATTERN; + } + + @Override + public Result apply(ApplyNode applyNode, Captures captures, Context context) { + Set subqueryAssignmentsSymbols = + applyNode.getSubqueryAssignments().values().stream() + .flatMap(expression -> expression.inputs().stream()) + .collect(toImmutableSet()); + + Optional prunedSubquery = + restrictOutputs( + context.getIdAllocator(), applyNode.getSubquery(), subqueryAssignmentsSymbols); + return prunedSubquery + .map( + subquery -> applyNode.replaceChildren(ImmutableList.of(applyNode.getInput(), subquery))) + .map(Result::ofPlanNode) + .orElse(Result.empty()); + } +} diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/planner/iterative/rule/RemoveUnreferencedScalarApplyNodes.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/planner/iterative/rule/RemoveUnreferencedScalarApplyNodes.java new file mode 100644 index 000000000000..fcecc2c3b821 --- /dev/null +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/planner/iterative/rule/RemoveUnreferencedScalarApplyNodes.java @@ -0,0 +1,42 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.iotdb.db.queryengine.plan.relational.planner.iterative.rule; + +import org.apache.iotdb.db.queryengine.plan.relational.planner.iterative.Rule; +import org.apache.iotdb.db.queryengine.plan.relational.planner.node.ApplyNode; +import org.apache.iotdb.db.queryengine.plan.relational.utils.matching.Captures; +import org.apache.iotdb.db.queryengine.plan.relational.utils.matching.Pattern; + +import static org.apache.iotdb.db.queryengine.plan.relational.planner.node.Patterns.applyNode; + +public class RemoveUnreferencedScalarApplyNodes implements Rule { + private static final Pattern PATTERN = + applyNode().matching(applyNode -> applyNode.getSubqueryAssignments().isEmpty()); + + @Override + public Pattern getPattern() { + return PATTERN; + } + + @Override + public Result apply(ApplyNode applyNode, Captures captures, Context context) { + return Result.ofPlanNode(applyNode.getInput()); + } +} diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/planner/iterative/rule/RemoveUnreferencedScalarSubqueries.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/planner/iterative/rule/RemoveUnreferencedScalarSubqueries.java new file mode 100644 index 000000000000..badc7dff3c49 --- /dev/null +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/planner/iterative/rule/RemoveUnreferencedScalarSubqueries.java @@ -0,0 +1,70 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.iotdb.db.queryengine.plan.relational.planner.iterative.rule; + +import org.apache.iotdb.db.queryengine.plan.planner.plan.node.PlanNode; +import org.apache.iotdb.db.queryengine.plan.relational.planner.iterative.Lookup; +import org.apache.iotdb.db.queryengine.plan.relational.planner.iterative.Rule; +import org.apache.iotdb.db.queryengine.plan.relational.planner.node.CorrelatedJoinNode; +import org.apache.iotdb.db.queryengine.plan.relational.utils.matching.Captures; +import org.apache.iotdb.db.queryengine.plan.relational.utils.matching.Pattern; + +import static org.apache.iotdb.db.queryengine.plan.relational.planner.node.JoinNode.JoinType.INNER; +import static org.apache.iotdb.db.queryengine.plan.relational.planner.node.JoinNode.JoinType.RIGHT; +import static org.apache.iotdb.db.queryengine.plan.relational.planner.node.Patterns.CorrelatedJoin.filter; +import static org.apache.iotdb.db.queryengine.plan.relational.planner.node.Patterns.correlatedJoin; +import static org.apache.iotdb.db.queryengine.plan.relational.planner.optimizations.QueryCardinalityUtil.isAtLeastScalar; +import static org.apache.iotdb.db.queryengine.plan.relational.planner.optimizations.QueryCardinalityUtil.isScalar; +import static org.apache.iotdb.db.queryengine.plan.relational.sql.ast.BooleanLiteral.TRUE_LITERAL; + +public class RemoveUnreferencedScalarSubqueries implements Rule { + private static final Pattern PATTERN = + correlatedJoin().with(filter().equalTo(TRUE_LITERAL)); + + @Override + public Pattern getPattern() { + return PATTERN; + } + + @Override + public Result apply(CorrelatedJoinNode correlatedJoinNode, Captures captures, Context context) { + PlanNode input = correlatedJoinNode.getInput(); + PlanNode subquery = correlatedJoinNode.getSubquery(); + + if (isUnreferencedScalar(input, context.getLookup()) + && correlatedJoinNode.getCorrelation().isEmpty()) { + if (correlatedJoinNode.getJoinType() == INNER + || correlatedJoinNode.getJoinType() == RIGHT + || isAtLeastScalar(subquery, context.getLookup())) { + return Result.ofPlanNode(subquery); + } + } + + if (isUnreferencedScalar(subquery, context.getLookup())) { + return Result.ofPlanNode(input); + } + + return Result.empty(); + } + + private boolean isUnreferencedScalar(PlanNode planNode, Lookup lookup) { + return planNode.getOutputSymbols().isEmpty() && isScalar(planNode, lookup); + } +} diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/planner/iterative/rule/TransformUncorrelatedInPredicateSubqueryToSemiJoin.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/planner/iterative/rule/TransformUncorrelatedInPredicateSubqueryToSemiJoin.java new file mode 100644 index 000000000000..80fe79c7e5f4 --- /dev/null +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/planner/iterative/rule/TransformUncorrelatedInPredicateSubqueryToSemiJoin.java @@ -0,0 +1,95 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.iotdb.db.queryengine.plan.relational.planner.iterative.rule; + +import org.apache.iotdb.db.queryengine.plan.relational.planner.Symbol; +import org.apache.iotdb.db.queryengine.plan.relational.planner.iterative.Rule; +import org.apache.iotdb.db.queryengine.plan.relational.planner.node.ApplyNode; +import org.apache.iotdb.db.queryengine.plan.relational.planner.node.SemiJoinNode; +import org.apache.iotdb.db.queryengine.plan.relational.utils.matching.Captures; +import org.apache.iotdb.db.queryengine.plan.relational.utils.matching.Pattern; + +import static com.google.common.collect.Iterables.getOnlyElement; +import static org.apache.iotdb.db.queryengine.plan.relational.planner.node.Patterns.Apply.correlation; +import static org.apache.iotdb.db.queryengine.plan.relational.planner.node.Patterns.applyNode; +import static org.apache.iotdb.db.queryengine.plan.relational.utils.matching.Pattern.empty; + +/** + * This optimizers looks for InPredicate expressions in ApplyNodes and replaces the nodes with + * SemiJoin nodes. + * + *

Plan before optimizer: + * + * 

+ * Filter(a IN b):
+ *   Apply
+ *     - correlation: []  // empty
+ *     - input: some plan A producing symbol a
+ *     - subquery: some plan B producing symbol b
+ *
+ * + *

Plan after optimizer: + * + * 

+ * Filter(semijoinresult):
+ *   SemiJoin
+ *     - source: plan A
+ *     - filteringSource: B
+ *     - sourceJoinSymbol: symbol a
+ *     - filteringSourceJoinSymbol: symbol b
+ *     - semiJoinOutput: semijoinresult
+ *
+ */ +public class TransformUncorrelatedInPredicateSubqueryToSemiJoin implements Rule { + private static final Pattern PATTERN = applyNode().with(empty(correlation())); + + @Override + public Pattern getPattern() { + return PATTERN; + } + + @Override + public Result apply(ApplyNode applyNode, Captures captures, Context context) { + if (applyNode.getSubqueryAssignments().size() != 1) { + return Result.empty(); + } + + ApplyNode.SetExpression expression = + getOnlyElement(applyNode.getSubqueryAssignments().values()); + if (!(expression instanceof ApplyNode.In)) { + return Result.empty(); + } + + ApplyNode.In inPredicate = (ApplyNode.In) expression; + + Symbol semiJoinSymbol = getOnlyElement(applyNode.getSubqueryAssignments().keySet()); + + SemiJoinNode replacement = + new SemiJoinNode( + context.getIdAllocator().genPlanNodeId(), + applyNode.getInput(), + applyNode.getSubquery(), + inPredicate.getValue(), + inPredicate.getReference(), + semiJoinSymbol); + + return Result.ofPlanNode(replacement); + } +} diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/planner/node/SemiJoinNode.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/planner/node/SemiJoinNode.java new file mode 100644 index 000000000000..94cdb47cddc6 --- /dev/null +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/planner/node/SemiJoinNode.java @@ -0,0 +1,148 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.iotdb.db.queryengine.plan.relational.planner.node; + +import org.apache.iotdb.db.queryengine.plan.planner.plan.node.PlanNode; +import org.apache.iotdb.db.queryengine.plan.planner.plan.node.PlanNodeId; +import org.apache.iotdb.db.queryengine.plan.planner.plan.node.PlanNodeType; +import org.apache.iotdb.db.queryengine.plan.planner.plan.node.PlanVisitor; +import org.apache.iotdb.db.queryengine.plan.planner.plan.node.process.TwoChildProcessNode; +import org.apache.iotdb.db.queryengine.plan.relational.planner.Symbol; + +import com.google.common.collect.ImmutableList; + +import java.io.DataOutputStream; +import java.io.IOException; +import java.nio.ByteBuffer; +import java.util.List; + +import static com.google.common.base.Preconditions.checkArgument; +import static java.util.Objects.requireNonNull; + +public class SemiJoinNode extends TwoChildProcessNode { + private final Symbol sourceJoinSymbol; + private final Symbol filteringSourceJoinSymbol; + private final Symbol semiJoinOutput; + + public SemiJoinNode( + PlanNodeId id, + PlanNode source, + PlanNode filteringSource, + Symbol sourceJoinSymbol, + Symbol filteringSourceJoinSymbol, + Symbol semiJoinOutput) { + super(id, source, filteringSource); + this.sourceJoinSymbol = requireNonNull(sourceJoinSymbol, "sourceJoinSymbol is null"); + this.filteringSourceJoinSymbol = + requireNonNull(filteringSourceJoinSymbol, "filteringSourceJoinSymbol is null"); + this.semiJoinOutput = requireNonNull(semiJoinOutput, "semiJoinOutput is null"); + + checkArgument( + source.getOutputSymbols().contains(sourceJoinSymbol), + "Source does not contain join symbol"); + checkArgument( + filteringSource.getOutputSymbols().contains(filteringSourceJoinSymbol), + "Filtering source does not contain filtering join symbol"); + } + + public PlanNode getSource() { + return leftChild; + } + + public PlanNode getFilteringSource() { + return rightChild; + } + + public Symbol getSourceJoinSymbol() { + return sourceJoinSymbol; + } + + public Symbol getFilteringSourceJoinSymbol() { + return filteringSourceJoinSymbol; + } + + public Symbol getSemiJoinOutput() { + return semiJoinOutput; + } + + @Override + public List getOutputSymbols() { + return ImmutableList.builder() + .addAll(leftChild.getOutputSymbols()) + .add(semiJoinOutput) + .build(); + } + + @Override + public R accept(PlanVisitor visitor, C context) { + return visitor.visitSemiJoin(this, context); + } + + @Override + public PlanNode replaceChildren(List newChildren) { + checkArgument(newChildren.size() == 2, "expected newChildren to contain 2 nodes"); + return new SemiJoinNode( + getPlanNodeId(), + newChildren.get(0), + newChildren.get(1), + sourceJoinSymbol, + filteringSourceJoinSymbol, + semiJoinOutput); + } + + @Override + public PlanNode clone() { + // clone without children + return new SemiJoinNode( + getPlanNodeId(), null, null, sourceJoinSymbol, filteringSourceJoinSymbol, semiJoinOutput); + } + + @Override + public List getOutputColumnNames() { + throw new UnsupportedOperationException(); + } + + @Override + protected void serializeAttributes(ByteBuffer byteBuffer) { + PlanNodeType.TABLE_SEMI_JOIN_NODE.serialize(byteBuffer); + + Symbol.serialize(sourceJoinSymbol, byteBuffer); + Symbol.serialize(filteringSourceJoinSymbol, byteBuffer); + Symbol.serialize(semiJoinOutput, byteBuffer); + } + + @Override + protected void serializeAttributes(DataOutputStream stream) throws IOException { + PlanNodeType.TABLE_SEMI_JOIN_NODE.serialize(stream); + + Symbol.serialize(sourceJoinSymbol, stream); + Symbol.serialize(filteringSourceJoinSymbol, stream); + Symbol.serialize(semiJoinOutput, stream); + } + + public static SemiJoinNode deserialize(ByteBuffer byteBuffer) { + Symbol sourceJoinSymbol = Symbol.deserialize(byteBuffer); + Symbol filteringSourceJoinSymbol = Symbol.deserialize(byteBuffer); + Symbol semiJoinOutput = Symbol.deserialize(byteBuffer); + PlanNodeId planNodeId = PlanNodeId.deserialize(byteBuffer); + return new SemiJoinNode( + planNodeId, null, null, sourceJoinSymbol, filteringSourceJoinSymbol, semiJoinOutput); + } +} diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/planner/optimizations/LogicalOptimizeFactory.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/planner/optimizations/LogicalOptimizeFactory.java index 304917842877..42535ef68efe 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/planner/optimizations/LogicalOptimizeFactory.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/planner/optimizations/LogicalOptimizeFactory.java @@ -32,6 +32,9 @@ import org.apache.iotdb.db.queryengine.plan.relational.planner.iterative.rule.MergeLimits; import org.apache.iotdb.db.queryengine.plan.relational.planner.iterative.rule.PruneAggregationColumns; import org.apache.iotdb.db.queryengine.plan.relational.planner.iterative.rule.PruneAggregationSourceColumns; +import org.apache.iotdb.db.queryengine.plan.relational.planner.iterative.rule.PruneApplyColumns; +import org.apache.iotdb.db.queryengine.plan.relational.planner.iterative.rule.PruneApplyCorrelation; +import org.apache.iotdb.db.queryengine.plan.relational.planner.iterative.rule.PruneApplySourceColumns; import org.apache.iotdb.db.queryengine.plan.relational.planner.iterative.rule.PruneCorrelatedJoinColumns; import org.apache.iotdb.db.queryengine.plan.relational.planner.iterative.rule.PruneCorrelatedJoinCorrelation; import org.apache.iotdb.db.queryengine.plan.relational.planner.iterative.rule.PruneEnforceSingleRowColumns; @@ -53,7 +56,9 @@ import org.apache.iotdb.db.queryengine.plan.relational.planner.iterative.rule.RemoveRedundantEnforceSingleRowNode; import org.apache.iotdb.db.queryengine.plan.relational.planner.iterative.rule.RemoveRedundantIdentityProjections; import org.apache.iotdb.db.queryengine.plan.relational.planner.iterative.rule.RemoveTrivialFilters; +import org.apache.iotdb.db.queryengine.plan.relational.planner.iterative.rule.RemoveUnreferencedScalarSubqueries; import org.apache.iotdb.db.queryengine.plan.relational.planner.iterative.rule.SimplifyExpressions; +import org.apache.iotdb.db.queryengine.plan.relational.planner.iterative.rule.TransformUncorrelatedInPredicateSubqueryToSemiJoin; import org.apache.iotdb.db.queryengine.plan.relational.planner.iterative.rule.TransformUncorrelatedSubqueryToJoin; import com.google.common.collect.ImmutableList; @@ -77,6 +82,9 @@ public LogicalOptimizeFactory(PlannerContext plannerContext) { // TODO After ValuesNode introduced // new RemoveEmptyGlobalAggregation(), new PruneAggregationSourceColumns(), + new PruneApplyColumns(), + new PruneApplyCorrelation(), + new PruneApplySourceColumns(), new PruneCorrelatedJoinColumns(), new PruneCorrelatedJoinCorrelation(), new PruneEnforceSingleRowColumns(), @@ -203,8 +211,9 @@ public LogicalOptimizeFactory(PlannerContext plannerContext) { plannerContext, ruleStats, ImmutableSet.of( - new RemoveRedundantEnforceSingleRowNode(), - new TransformUncorrelatedSubqueryToJoin())), + new RemoveRedundantEnforceSingleRowNode(), new RemoveUnreferencedScalarSubqueries(), + new TransformUncorrelatedSubqueryToJoin(), + new TransformUncorrelatedInPredicateSubqueryToSemiJoin())), new CheckSubqueryNodesAreRewritten(), simplifyOptimizer, new PushPredicateIntoTableScan(), diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/planner/optimizations/UnaliasSymbolReferences.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/planner/optimizations/UnaliasSymbolReferences.java index aedd97bff6dc..736272117a1b 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/planner/optimizations/UnaliasSymbolReferences.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/planner/optimizations/UnaliasSymbolReferences.java @@ -37,6 +37,7 @@ import org.apache.iotdb.db.queryengine.plan.relational.planner.node.OutputNode; import org.apache.iotdb.db.queryengine.plan.relational.planner.node.PreviousFillNode; import org.apache.iotdb.db.queryengine.plan.relational.planner.node.ProjectNode; +import org.apache.iotdb.db.queryengine.plan.relational.planner.node.SemiJoinNode; import org.apache.iotdb.db.queryengine.plan.relational.planner.node.SortNode; import org.apache.iotdb.db.queryengine.plan.relational.planner.node.TableScanNode; import org.apache.iotdb.db.queryengine.plan.relational.planner.node.TopKNode; @@ -658,6 +659,34 @@ public PlanAndMappings visitJoin(JoinNode node, UnaliasContext context) { node.isSpillable()), outputMapping); } + + @Override + public PlanAndMappings visitSemiJoin(SemiJoinNode node, UnaliasContext context) { + // it is assumed that symbols are distinct between SemiJoin source and filtering source. Only + // symbols from outer correlation might be the exception + PlanAndMappings rewrittenSource = node.getSource().accept(this, context); + PlanAndMappings rewrittenFilteringSource = node.getFilteringSource().accept(this, context); + + Map outputMapping = new HashMap<>(); + outputMapping.putAll(rewrittenSource.getMappings()); + outputMapping.putAll(rewrittenFilteringSource.getMappings()); + + SymbolMapper mapper = symbolMapper(outputMapping); + + Symbol newSourceJoinSymbol = mapper.map(node.getSourceJoinSymbol()); + Symbol newFilteringSourceJoinSymbol = mapper.map(node.getFilteringSourceJoinSymbol()); + Symbol newSemiJoinOutput = mapper.map(node.getSemiJoinOutput()); + + return new PlanAndMappings( + new SemiJoinNode( + node.getPlanNodeId(), + rewrittenSource.getRoot(), + rewrittenFilteringSource.getRoot(), + newSourceJoinSymbol, + newFilteringSourceJoinSymbol, + newSemiJoinOutput), + outputMapping); + } } private static class UnaliasContext { diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/sql/ast/AstVisitor.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/sql/ast/AstVisitor.java index 1a2e3b9f3a2c..eb3b6b98fd50 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/sql/ast/AstVisitor.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/sql/ast/AstVisitor.java @@ -477,6 +477,14 @@ protected R visitGenericDataType(GenericDataType node, C context) { return visitDataType(node, context); } + protected R visitRowDataType(RowDataType node, C context) { + return visitDataType(node, context); + } + + protected R visitRowField(RowDataType.Field node, C context) { + return visitNode(node, context); + } + protected R visitDataTypeParameter(DataTypeParameter node, C context) { return visitNode(node, context); } diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/sql/ast/RowDataType.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/sql/ast/RowDataType.java new file mode 100644 index 000000000000..77452dab7f9c --- /dev/null +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/sql/ast/RowDataType.java @@ -0,0 +1,149 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.iotdb.db.queryengine.plan.relational.sql.ast; + +import com.google.common.collect.ImmutableList; + +import java.util.List; +import java.util.Objects; +import java.util.Optional; + +import static java.util.Objects.requireNonNull; + +public class RowDataType extends DataType { + private final List fields; + + public RowDataType(Optional location, List fields) { + this(location.orElse(null), fields); + } + + public RowDataType(NodeLocation location, List fields) { + super(location); + this.fields = ImmutableList.copyOf(fields); + } + + public List getFields() { + return fields; + } + + @Override + public List getChildren() { + return fields; + } + + @Override + public R accept(AstVisitor visitor, C context) { + return visitor.visitRowDataType(this, context); + } + + @Override + public boolean equals(Object o) { + if (this == o) { + return true; + } + if (o == null || getClass() != o.getClass()) { + return false; + } + RowDataType that = (RowDataType) o; + return fields.equals(that.fields); + } + + @Override + public int hashCode() { + return Objects.hash(fields); + } + + public static class Field extends Node { + private final Optional name; + private final DataType type; + + public Field(Optional location, Optional name, DataType type) { + this(location.orElse(null), name, type); + } + + public Field(NodeLocation location, Optional name, DataType type) { + super(location); + + this.name = requireNonNull(name, "name is null"); + this.type = requireNonNull(type, "type is null"); + } + + public Optional getName() { + return name; + } + + public DataType getType() { + return type; + } + + @Override + public List getChildren() { + ImmutableList.Builder children = ImmutableList.builder(); + name.ifPresent(children::add); + children.add(type); + + return children.build(); + } + + @Override + protected R accept(AstVisitor visitor, C context) { + return visitor.visitRowField(this, context); + } + + @Override + public String toString() { + StringBuilder builder = new StringBuilder(); + if (name.isPresent()) { + builder.append(name.get()); + builder.append(" "); + } + builder.append(type); + + return builder.toString(); + } + + @Override + public boolean equals(Object o) { + if (this == o) { + return true; + } + if (o == null || getClass() != o.getClass()) { + return false; + } + Field field = (Field) o; + return name.equals(field.name) && type.equals(field.type); + } + + @Override + public int hashCode() { + return Objects.hash(name, type); + } + + @Override + public boolean shallowEquals(Node other) { + return sameClass(this, other); + } + } + + @Override + public boolean shallowEquals(Node other) { + return sameClass(this, other); + } +} diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/sql/util/ExpressionFormatter.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/sql/util/ExpressionFormatter.java index e36a272b363c..c584947c9678 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/sql/util/ExpressionFormatter.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/sql/util/ExpressionFormatter.java @@ -64,6 +64,7 @@ import org.apache.iotdb.db.queryengine.plan.relational.sql.ast.QualifiedName; import org.apache.iotdb.db.queryengine.plan.relational.sql.ast.QuantifiedComparisonExpression; import org.apache.iotdb.db.queryengine.plan.relational.sql.ast.Row; +import org.apache.iotdb.db.queryengine.plan.relational.sql.ast.RowDataType; import org.apache.iotdb.db.queryengine.plan.relational.sql.ast.SearchedCaseExpression; import org.apache.iotdb.db.queryengine.plan.relational.sql.ast.SimpleCaseExpression; import org.apache.iotdb.db.queryengine.plan.relational.sql.ast.SimpleGroupBy; @@ -532,6 +533,25 @@ protected String visitGenericDataType(GenericDataType node, Void context) { return result.toString(); } + @Override + protected String visitRowDataType(RowDataType node, Void context) { + return node.getFields().stream().map(this::process).collect(joining(", ", "ROW(", ")")); + } + + @Override + protected String visitRowField(RowDataType.Field node, Void context) { + StringBuilder result = new StringBuilder(); + + if (node.getName().isPresent()) { + result.append(process(node.getName().get(), context)); + result.append(" "); + } + + result.append(process(node.getType(), context)); + + return result.toString(); + } + @Override protected String visitTypeParameter(TypeParameter node, Void context) { return process(node.getValue(), context); diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/type/InternalTypeManager.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/type/InternalTypeManager.java index 6f2aaef76ab1..b3b7fb77154c 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/type/InternalTypeManager.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/type/InternalTypeManager.java @@ -23,10 +23,14 @@ import org.apache.tsfile.read.common.type.Type; import org.apache.tsfile.read.common.type.TypeEnum; +import java.util.ArrayList; +import java.util.List; import java.util.Locale; import java.util.concurrent.ConcurrentHashMap; import java.util.concurrent.ConcurrentMap; +import static com.google.common.base.Preconditions.checkArgument; +import static org.apache.iotdb.db.queryengine.plan.relational.type.RowParametricType.ROW; import static org.apache.tsfile.read.common.type.BinaryType.TEXT; import static org.apache.tsfile.read.common.type.BlobType.BLOB; import static org.apache.tsfile.read.common.type.BooleanType.BOOLEAN; @@ -43,6 +47,8 @@ public class InternalTypeManager implements TypeManager { private final ConcurrentMap types = new ConcurrentHashMap<>(); + private final ConcurrentMap parametricTypes = new ConcurrentHashMap<>(); + public InternalTypeManager() { types.put(new TypeSignature(TypeEnum.DOUBLE.name().toLowerCase(Locale.ENGLISH)), DOUBLE); types.put(new TypeSignature(TypeEnum.FLOAT.name().toLowerCase(Locale.ENGLISH)), FLOAT); @@ -55,17 +61,53 @@ public InternalTypeManager() { types.put(new TypeSignature(TypeEnum.DATE.name().toLowerCase(Locale.ENGLISH)), DATE); types.put(new TypeSignature(TypeEnum.TIMESTAMP.name().toLowerCase(Locale.ENGLISH)), TIMESTAMP); types.put(new TypeSignature(TypeEnum.UNKNOWN.name().toLowerCase(Locale.ENGLISH)), UNKNOWN); + + addParametricType(ROW); } @Override public Type getType(TypeSignature signature) throws TypeNotFoundException { Type type = types.get(signature); if (type == null) { - throw new TypeNotFoundException(signature); + return instantiateParametricType(signature); } return type; } + private void addParametricType(ParametricType parametricType) { + String name = parametricType.getName().toLowerCase(Locale.ENGLISH); + if ("ROW".equals(name)) { + name = "row"; + } + checkArgument( + !parametricTypes.containsKey(name), "Parametric type already registered: %s", name); + parametricTypes.putIfAbsent(name, parametricType); + } + + private Type instantiateParametricType(TypeSignature signature) { + List parameters = new ArrayList<>(); + + for (TypeSignatureParameter parameter : signature.getParameters()) { + TypeParameter typeParameter = TypeParameter.of(parameter, this); + parameters.add(typeParameter); + } + + ParametricType parametricType = + parametricTypes.get(signature.getBase().toLowerCase(Locale.ENGLISH)); + if (parametricType == null) { + throw new TypeNotFoundException(signature); + } + + Type instantiatedType; + try { + instantiatedType = parametricType.createType(this, parameters); + } catch (IllegalArgumentException e) { + throw new TypeNotFoundException(signature, e); + } + + return instantiatedType; + } + @Override public Type fromSqlType(String type) { throw new UnsupportedOperationException(); diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/type/NamedType.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/type/NamedType.java new file mode 100644 index 000000000000..93e65cfab7b5 --- /dev/null +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/type/NamedType.java @@ -0,0 +1,62 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.iotdb.db.queryengine.plan.relational.type; + +import org.apache.tsfile.read.common.type.Type; + +import java.util.Objects; +import java.util.Optional; + +public class NamedType { + private final Optional name; + private final Type type; + + public NamedType(Optional name, Type type) { + this.name = name; + this.type = type; + } + + public Optional getName() { + return name; + } + + public Type getType() { + return type; + } + + @Override + public boolean equals(Object o) { + if (this == o) { + return true; + } + if (o == null || getClass() != o.getClass()) { + return false; + } + + NamedType other = (NamedType) o; + + return Objects.equals(this.name, other.name) && Objects.equals(this.type, other.type); + } + + @Override + public int hashCode() { + return Objects.hash(name, type); + } +} diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/type/ParametricType.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/type/ParametricType.java new file mode 100644 index 000000000000..07c5d8a2899d --- /dev/null +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/type/ParametricType.java @@ -0,0 +1,30 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.iotdb.db.queryengine.plan.relational.type; + +import org.apache.tsfile.read.common.type.Type; + +import java.util.List; + +public interface ParametricType { + String getName(); + + Type createType(TypeManager typeManager, List parameters); +} diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/type/RowParametricType.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/type/RowParametricType.java new file mode 100644 index 000000000000..005f6b2c4193 --- /dev/null +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/type/RowParametricType.java @@ -0,0 +1,59 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.iotdb.db.queryengine.plan.relational.type; + +import org.apache.tsfile.read.common.type.RowType; +import org.apache.tsfile.read.common.type.Type; + +import java.util.List; + +import static com.google.common.base.Preconditions.checkArgument; +import static java.util.stream.Collectors.toList; + +public final class RowParametricType implements ParametricType { + public static final RowParametricType ROW = new RowParametricType(); + + private RowParametricType() {} + + @Override + public String getName() { + return StandardTypes.ROW; + } + + @Override + public Type createType(TypeManager typeManager, List parameters) { + checkArgument(!parameters.isEmpty(), "Row type must have at least one parameter"); + checkArgument( + parameters.stream().allMatch(parameter -> parameter.getKind() == ParameterKind.NAMED_TYPE), + "Expected only named types as a parameters, got %s", + parameters); + + List fields = + parameters.stream() + .map(TypeParameter::getNamedType) + .map( + parameter -> + new RowType.Field( + parameter.getName().map(RowFieldName::getName), parameter.getType())) + .collect(toList()); + + return RowType.createWithTypeSignature(fields); + } +} diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/type/TypeParameter.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/type/TypeParameter.java new file mode 100644 index 000000000000..9c4aa1333534 --- /dev/null +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/type/TypeParameter.java @@ -0,0 +1,128 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.iotdb.db.queryengine.plan.relational.type; + +import org.apache.tsfile.read.common.type.Type; + +import java.util.Objects; + +import static java.lang.String.format; + +public class TypeParameter { + private final ParameterKind kind; + private final Object value; + + private TypeParameter(ParameterKind kind, Object value) { + this.kind = kind; + this.value = value; + } + + public static TypeParameter of(Type type) { + return new TypeParameter(ParameterKind.TYPE, type); + } + + public static TypeParameter of(long longLiteral) { + return new TypeParameter(ParameterKind.LONG, longLiteral); + } + + public static TypeParameter of(NamedType namedType) { + return new TypeParameter(ParameterKind.NAMED_TYPE, namedType); + } + + public static TypeParameter of(String variable) { + return new TypeParameter(ParameterKind.VARIABLE, variable); + } + + public static TypeParameter of(TypeSignatureParameter parameter, TypeManager typeManager) { + switch (parameter.getKind()) { + case TYPE: + { + Type type = typeManager.getType(parameter.getTypeSignature()); + return of(type); + } + case LONG: + return of(parameter.getLongLiteral()); + case NAMED_TYPE: + { + Type type = typeManager.getType(parameter.getNamedTypeSignature().getTypeSignature()); + return of(new NamedType(parameter.getNamedTypeSignature().getFieldName(), type)); + } + case VARIABLE: + return of(parameter.getVariable()); + } + throw new UnsupportedOperationException(format("Unsupported parameter [%s]", parameter)); + } + + public ParameterKind getKind() { + return kind; + } + + public A getValue(ParameterKind expectedParameterKind, Class target) { + if (kind != expectedParameterKind) { + throw new AssertionError( + format("ParameterKind is [%s] but expected [%s]", kind, expectedParameterKind)); + } + return target.cast(value); + } + + public boolean isLongLiteral() { + return kind == ParameterKind.LONG; + } + + public Type getType() { + return getValue(ParameterKind.TYPE, Type.class); + } + + public Long getLongLiteral() { + return getValue(ParameterKind.LONG, Long.class); + } + + public NamedType getNamedType() { + return getValue(ParameterKind.NAMED_TYPE, NamedType.class); + } + + public String getVariable() { + return getValue(ParameterKind.VARIABLE, String.class); + } + + @Override + public String toString() { + return value.toString(); + } + + @Override + public boolean equals(Object o) { + if (this == o) { + return true; + } + if (o == null || getClass() != o.getClass()) { + return false; + } + + TypeParameter other = (TypeParameter) o; + + return this.kind == other.kind && Objects.equals(this.value, other.value); + } + + @Override + public int hashCode() { + return Objects.hash(kind, value); + } +} diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/type/TypeSignatureTranslator.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/type/TypeSignatureTranslator.java index 9390cd4826a2..383d81a0e669 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/type/TypeSignatureTranslator.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/type/TypeSignatureTranslator.java @@ -25,16 +25,23 @@ import org.apache.iotdb.db.queryengine.plan.relational.sql.ast.GenericDataType; import org.apache.iotdb.db.queryengine.plan.relational.sql.ast.Identifier; import org.apache.iotdb.db.queryengine.plan.relational.sql.ast.NumericParameter; +import org.apache.iotdb.db.queryengine.plan.relational.sql.ast.RowDataType; import org.apache.iotdb.db.queryengine.plan.relational.sql.ast.TypeParameter; import com.google.common.collect.ImmutableList; +import org.apache.tsfile.read.common.type.RowType; import org.apache.tsfile.read.common.type.Type; import java.util.Collections; +import java.util.List; import java.util.Locale; +import java.util.Optional; import java.util.Set; import static com.google.common.base.Preconditions.checkArgument; +import static com.google.common.collect.ImmutableList.toImmutableList; +import static org.apache.iotdb.db.queryengine.plan.relational.type.StandardTypes.ROW; +import static org.apache.iotdb.db.queryengine.plan.relational.type.TypeSignatureParameter.namedTypeParameter; import static org.apache.iotdb.db.queryengine.plan.relational.type.TypeSignatureParameter.numericParameter; import static org.apache.iotdb.db.queryengine.plan.relational.type.TypeSignatureParameter.typeParameter; import static org.apache.iotdb.db.queryengine.plan.relational.type.TypeSignatureParameter.typeVariable; @@ -44,8 +51,25 @@ public class TypeSignatureTranslator { private TypeSignatureTranslator() {} public static DataType toSqlType(Type type) { - return new GenericDataType( - new Identifier(type.getTypeEnum().name(), false), Collections.emptyList()); + if (type instanceof RowType) { + RowType rowType = (RowType) type; + return new RowDataType( + Optional.empty(), + rowType.getFields().stream() + .map( + field -> + new RowDataType.Field( + Optional.empty(), + field + .getName() + .map(RowFieldName::new) + .map(fieldName -> new Identifier(fieldName.getName(), false)), + toSqlType(field.getType()))) + .collect(toImmutableList())); + } else { + return new GenericDataType( + new Identifier(type.getTypeEnum().name(), false), Collections.emptyList()); + } } public static TypeSignature toTypeSignature(DataType type) { @@ -56,10 +80,30 @@ private static TypeSignature toTypeSignature(DataType type, Set typeVari if (type instanceof GenericDataType) { return toTypeSignature((GenericDataType) type, typeVariables); } + if (type instanceof RowDataType) { + return toTypeSignature((RowDataType) type, typeVariables); + } throw new UnsupportedOperationException("Unsupported DataType: " + type.getClass().getName()); } + private static TypeSignature toTypeSignature(RowDataType type, Set typeVariables) { + List parameters = + type.getFields().stream() + .map( + field -> + namedTypeParameter( + new NamedTypeSignature( + field + .getName() + .map(TypeSignatureTranslator::canonicalize) + .map(RowFieldName::new), + toTypeSignature(field.getType(), typeVariables)))) + .collect(toImmutableList()); + + return new TypeSignature(ROW, parameters); + } + private static TypeSignature toTypeSignature(GenericDataType type, Set typeVariables) { ImmutableList.Builder parameters = ImmutableList.builder(); From 95d162ccde8db1a85a7efb97ff73e89af4b94392 Mon Sep 17 00:00:00 2001 From: lancelly <1435078631@qq.com> Date: Sun, 8 Dec 2024 15:18:58 +0800 Subject: [PATCH 02/26] add operator and workaround with RowType --- .../relational/MergeSortSemiJoinOperator.java | 171 ++++++++++++++++ .../relational/ColumnTransformerBuilder.java | 15 ++ .../plan/planner/TableOperatorGenerator.java | 75 ++++++- .../planner/plan/node/PlanGraphPrinter.java | 12 ++ .../relational/planner/SubqueryPlanner.java | 25 ++- .../TableDistributedPlanGenerator.java | 16 ++ ...TransformFilteringSemiJoinToInnerJoin.java | 153 +++++++++++++++ .../relational/planner/node/Patterns.java | 7 +- .../relational/planner/node/SemiJoinNode.java | 17 +- .../optimizations/LogicalOptimizeFactory.java | 5 + .../PushPredicateIntoTableScan.java | 185 ++++++++++++++++++ .../plan/relational/sql/ast/RowDataType.java | 16 +- .../iotdb/commons/conf/CommonConfig.java | 2 +- 13 files changed, 663 insertions(+), 36 deletions(-) create mode 100644 iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/execution/operator/source/relational/MergeSortSemiJoinOperator.java create mode 100644 iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/planner/iterative/rule/TransformFilteringSemiJoinToInnerJoin.java diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/execution/operator/source/relational/MergeSortSemiJoinOperator.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/execution/operator/source/relational/MergeSortSemiJoinOperator.java new file mode 100644 index 000000000000..34ef6d6cd3f0 --- /dev/null +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/execution/operator/source/relational/MergeSortSemiJoinOperator.java @@ -0,0 +1,171 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.iotdb.db.queryengine.execution.operator.source.relational; + +import org.apache.iotdb.db.queryengine.execution.MemoryEstimationHelper; +import org.apache.iotdb.db.queryengine.execution.operator.Operator; +import org.apache.iotdb.db.queryengine.execution.operator.OperatorContext; +import org.apache.iotdb.db.queryengine.execution.operator.process.join.merge.comparator.JoinKeyComparator; + +import org.apache.tsfile.enums.TSDataType; +import org.apache.tsfile.read.common.type.Type; +import org.apache.tsfile.utils.RamUsageEstimator; + +import java.util.List; + +public class MergeSortSemiJoinOperator extends AbstractMergeSortJoinOperator { + private static final long INSTANCE_SIZE = + RamUsageEstimator.shallowSizeOfInstance(MergeSortSemiJoinOperator.class); + + public MergeSortSemiJoinOperator( + OperatorContext operatorContext, + Operator leftChild, + int leftJoinKeyPosition, + int[] leftOutputSymbolIdx, + Operator rightChild, + int rightJoinKeyPosition, + JoinKeyComparator joinKeyComparator, + List dataTypes, + Type joinKeyType) { + super( + operatorContext, + leftChild, + leftJoinKeyPosition, + leftOutputSymbolIdx, + rightChild, + rightJoinKeyPosition, + null, + joinKeyComparator, + dataTypes, + joinKeyType); + } + + @Override + public boolean hasNext() throws Exception { + if (retainedTsBlock != null) { + return true; + } + + return !leftFinished && !rightFinished; + } + + @Override + protected boolean prepareInput() throws Exception { + gotCandidateBlocks(); + return leftBlockNotEmpty() && rightBlockNotEmpty() && gotNextRightBlock(); + } + + @Override + protected boolean processFinished() { + // all the join keys in rightTsBlock are less than leftTsBlock, just skip right + if (allRightLessThanLeft()) { + resetRightBlockList(); + return true; + } + + // all the join Keys in leftTsBlock are less than rightTsBlock, just skip left + if (allLeftLessThanRight()) { + resetLeftBlock(); + return true; + } + + // continue right < left, until right >= left + while (comparator.lessThan( + rightBlockList.get(rightBlockListIdx), + rightJoinKeyPosition, + rightIndex, + leftBlock, + leftJoinKeyPosition, + leftIndex)) { + if (rightFinishedWithIncIndex()) { + return true; + } + } + if (currentRoundNeedStop()) { + return true; + } + + // continue left < right, until left >= right + while (comparator.lessThan( + leftBlock, + leftJoinKeyPosition, + leftIndex, + rightBlockList.get(rightBlockListIdx), + rightJoinKeyPosition, + rightIndex)) { + leftIndex++; + if (leftIndex >= leftBlock.getPositionCount()) { + resetLeftBlock(); + return true; + } + } + if (currentRoundNeedStop()) { + return true; + } + + // has right values equal to current left, append to join result, inc leftIndex + if (hasMatchedRightValueToProbeLeft()) { + leftIndex++; + if (leftIndex >= leftBlock.getPositionCount()) { + resetLeftBlock(); + return true; + } + } + + return false; + } + + @Override + protected boolean hasMatchedRightValueToProbeLeft() { + if (comparator.equalsTo( + leftBlock, + leftJoinKeyPosition, + leftIndex, + rightBlockList.get(rightBlockListIdx), + rightJoinKeyPosition, + rightIndex)) { + recordsWhenDataMatches(); + appendValueToResultWhenMatches(); + return true; + } + return false; + } + + protected void appendValueToResultWhenMatches() { + appendLeftBlockData(leftOutputSymbolIdx, resultBuilder, leftBlock, leftIndex); + resultBuilder.declarePosition(); + } + + @Override + protected void recordsWhenDataMatches() { + // do nothing + } + + @Override + public long ramBytesUsed() { + return INSTANCE_SIZE + + MemoryEstimationHelper.getEstimatedSizeOfAccountableObject(leftChild) + + MemoryEstimationHelper.getEstimatedSizeOfAccountableObject(rightChild) + + RamUsageEstimator.sizeOf(leftOutputSymbolIdx) + + RamUsageEstimator.sizeOf(rightOutputSymbolIdx) + + MemoryEstimationHelper.getEstimatedSizeOfAccountableObject(operatorContext) + + resultBuilder.getRetainedSizeInBytes(); + } +} diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/execution/relational/ColumnTransformerBuilder.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/execution/relational/ColumnTransformerBuilder.java index 9f9658c9b05b..3c2faa93fdf9 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/execution/relational/ColumnTransformerBuilder.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/execution/relational/ColumnTransformerBuilder.java @@ -59,6 +59,8 @@ import org.apache.iotdb.db.queryengine.plan.relational.sql.ast.NotExpression; import org.apache.iotdb.db.queryengine.plan.relational.sql.ast.NullIfExpression; import org.apache.iotdb.db.queryengine.plan.relational.sql.ast.NullLiteral; +import org.apache.iotdb.db.queryengine.plan.relational.sql.ast.Row; +import org.apache.iotdb.db.queryengine.plan.relational.sql.ast.RowDataType; import org.apache.iotdb.db.queryengine.plan.relational.sql.ast.SearchedCaseExpression; import org.apache.iotdb.db.queryengine.plan.relational.sql.ast.SimpleCaseExpression; import org.apache.iotdb.db.queryengine.plan.relational.sql.ast.StringLiteral; @@ -177,6 +179,7 @@ import java.util.Set; import java.util.stream.Collectors; +import static com.google.common.base.Preconditions.checkArgument; import static org.apache.iotdb.db.queryengine.plan.expression.unary.LikeExpression.getEscapeCharacter; import static org.apache.iotdb.db.queryengine.plan.relational.analyzer.predicate.PredicatePushIntoMetadataChecker.isStringLiteral; import static org.apache.iotdb.db.queryengine.plan.relational.type.InternalTypeManager.getTSDataType; @@ -350,6 +353,12 @@ protected ColumnTransformer visitCast(Cast node, Context context) { Type type; try { type = context.metadata.getType(toTypeSignature(node.getType())); + // For now, we only support casting to scalar types and Row can only have one item. + if (type instanceof RowDataType) { + type = + context.metadata.getType( + toTypeSignature(((RowDataType) type).getFields().get(0).getType())); + } } catch (TypeNotFoundException e) { throw new SemanticException(String.format("Unknown type: %s", node.getType())); } @@ -1422,6 +1431,12 @@ protected ColumnTransformer visitSearchedCaseExpression( return res; } + @Override + protected ColumnTransformer visitRow(Row node, Context context) { + checkArgument(node.getItems().size() == 1, "Row should only have one item for now."); + return process(node.getItems().get(0), context); + } + @Override protected ColumnTransformer visitTrim(Trim node, Context context) { throw new UnsupportedOperationException(String.format(UNSUPPORTED_EXPRESSION, node)); diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/planner/TableOperatorGenerator.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/planner/TableOperatorGenerator.java index 1bc778360955..3c2a8f05dfdd 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/planner/TableOperatorGenerator.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/planner/TableOperatorGenerator.java @@ -73,6 +73,7 @@ import org.apache.iotdb.db.queryengine.execution.operator.source.ExchangeOperator; import org.apache.iotdb.db.queryengine.execution.operator.source.relational.MergeSortFullOuterJoinOperator; import org.apache.iotdb.db.queryengine.execution.operator.source.relational.MergeSortInnerJoinOperator; +import org.apache.iotdb.db.queryengine.execution.operator.source.relational.MergeSortSemiJoinOperator; import org.apache.iotdb.db.queryengine.execution.operator.source.relational.TableAggregationTableScanOperator; import org.apache.iotdb.db.queryengine.execution.operator.source.relational.TableScanOperator; import org.apache.iotdb.db.queryengine.execution.operator.source.relational.aggregation.AggregationOperator; @@ -124,6 +125,7 @@ import org.apache.iotdb.db.queryengine.plan.relational.planner.node.OutputNode; import org.apache.iotdb.db.queryengine.plan.relational.planner.node.PreviousFillNode; import org.apache.iotdb.db.queryengine.plan.relational.planner.node.ProjectNode; +import org.apache.iotdb.db.queryengine.plan.relational.planner.node.SemiJoinNode; import org.apache.iotdb.db.queryengine.plan.relational.planner.node.SortNode; import org.apache.iotdb.db.queryengine.plan.relational.planner.node.StreamSortNode; import org.apache.iotdb.db.queryengine.plan.relational.planner.node.TableScanNode; @@ -162,6 +164,7 @@ import org.apache.tsfile.read.common.type.BinaryType; import org.apache.tsfile.read.common.type.BlobType; import org.apache.tsfile.read.common.type.BooleanType; +import org.apache.tsfile.read.common.type.RowType; import org.apache.tsfile.read.common.type.Type; import org.apache.tsfile.read.common.type.TypeEnum; import org.apache.tsfile.read.filter.basic.Filter; @@ -188,6 +191,7 @@ import java.util.stream.Collectors; import static com.google.common.base.Preconditions.checkArgument; +import static com.google.common.base.Preconditions.checkNotNull; import static com.google.common.collect.ImmutableList.toImmutableList; import static java.util.Objects.requireNonNull; import static org.apache.iotdb.commons.schema.table.column.TsTableColumnCategory.MEASUREMENT; @@ -1251,12 +1255,10 @@ public Operator visitJoin(JoinNode node, LocalExecutionPlanContext context) { throw new IllegalStateException("Right child of JoinNode doesn't contain right join key."); } - Type leftJoinKeyType = - context.getTypeProvider().getTableModelType(node.getCriteria().get(0).getLeft()); + Type leftJoinKeyType = getJoinKeyType(context, node.getCriteria().get(0).getLeft()); checkArgument( - leftJoinKeyType - == context.getTypeProvider().getTableModelType(node.getCriteria().get(0).getRight()), + leftJoinKeyType == getJoinKeyType(context, node.getCriteria().get(0).getRight()), "Join key type mismatch."); if (requireNonNull(node.getJoinType()) == JoinNode.JoinType.INNER) { @@ -1351,6 +1353,71 @@ private BiFunction buildUpdateLastRowFunction(TypeEnum } } + @Override + public Operator visitSemiJoin(SemiJoinNode node, LocalExecutionPlanContext context) { + List dataTypes = getOutputColumnTypes(node, context.getTypeProvider()); + + Operator leftChild = node.getLeftChild().accept(this, context); + Operator rightChild = node.getRightChild().accept(this, context); + + ImmutableMap sourceColumnNamesMap = + makeLayoutFromOutputSymbols(node.getSource().getOutputSymbols()); + List sourceOutputSymbols = node.getSource().getOutputSymbols(); + int[] sourceOutputSymbolIdx = new int[node.getSource().getOutputSymbols().size()]; + for (int i = 0; i < sourceOutputSymbolIdx.length; i++) { + Integer index = sourceColumnNamesMap.get(sourceOutputSymbols.get(i)); + checkNotNull(index, "Source of SemiJoinNode doesn't contain sourceOutputSymbol."); + sourceOutputSymbolIdx[i] = index; + } + + ImmutableMap filteringSourceColumnNamesMap = + makeLayoutFromOutputSymbols(node.getRightChild().getOutputSymbols()); + + Integer sourceJoinKeyPosition = sourceColumnNamesMap.get(node.getSourceJoinSymbol()); + checkNotNull(sourceJoinKeyPosition, "Source of SemiJoinNode doesn't contain sourceJoinSymbol."); + + Integer filteringSourceJoinKeyPosition = + filteringSourceColumnNamesMap.get(node.getFilteringSourceJoinSymbol()); + checkNotNull( + filteringSourceJoinKeyPosition, + "FilteringSource of SemiJoinNode doesn't contain filteringSourceJoinSymbol."); + + Type sourceJoinKeyType = getJoinKeyType(context, node.getSourceJoinSymbol()); + + checkArgument( + sourceJoinKeyType == getJoinKeyType(context, node.getFilteringSourceJoinSymbol()), + "Join key type mismatch."); + OperatorContext operatorContext = + context + .getDriverContext() + .addOperatorContext( + context.getNextOperatorId(), + node.getPlanNodeId(), + MergeSortSemiJoinOperator.class.getSimpleName()); + return new MergeSortSemiJoinOperator( + operatorContext, + leftChild, + sourceJoinKeyPosition, + sourceOutputSymbolIdx, + rightChild, + filteringSourceJoinKeyPosition, + JoinKeyComparatorFactory.getComparator(sourceJoinKeyType, true), + dataTypes, + sourceJoinKeyType); + } + + private Type getJoinKeyType(LocalExecutionPlanContext context, Symbol symbol) { + Type type = context.getTypeProvider().getTableModelType(symbol); + if (type instanceof RowType) { + RowType rowType = (RowType) type; + // For now, we only RowType with single column. + checkArgument( + rowType.getFields().size() == 1, "RowType with multiple columns is not supported."); + return rowType.getFields().get(0).getType(); + } + return type; + } + @Override public Operator visitEnforceSingleRow( EnforceSingleRowNode node, LocalExecutionPlanContext context) { diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/planner/plan/node/PlanGraphPrinter.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/planner/plan/node/PlanGraphPrinter.java index 1939f6c6d75d..fff799dbfd05 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/planner/plan/node/PlanGraphPrinter.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/planner/plan/node/PlanGraphPrinter.java @@ -70,6 +70,7 @@ import org.apache.iotdb.db.queryengine.plan.relational.planner.node.GapFillNode; import org.apache.iotdb.db.queryengine.plan.relational.planner.node.LinearFillNode; import org.apache.iotdb.db.queryengine.plan.relational.planner.node.PreviousFillNode; +import org.apache.iotdb.db.queryengine.plan.relational.planner.node.SemiJoinNode; import org.apache.iotdb.db.queryengine.plan.relational.planner.node.TableScanNode; import org.apache.iotdb.db.queryengine.plan.relational.planner.node.ValueFillNode; @@ -897,6 +898,17 @@ public List visitJoin( return render(node, boxValue, context); } + @Override + public List visitSemiJoin(SemiJoinNode node, GraphContext context) { + List boxValue = new ArrayList<>(); + boxValue.add(String.format("SemiJoin-%s", node.getPlanNodeId().getId())); + boxValue.add(String.format("OutputSymbols: %s", node.getOutputSymbols())); + boxValue.add(String.format("SourceJoinSymbol: %s", node.getSourceJoinSymbol())); + boxValue.add( + String.format("FilteringSourceJoinSymbol: %s", node.getFilteringSourceJoinSymbol())); + return render(node, boxValue, context); + } + private String printRegion(TRegionReplicaSet regionReplicaSet) { return String.format( "Partition: %s", diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/planner/SubqueryPlanner.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/planner/SubqueryPlanner.java index 59b60abba088..7e58dfdd1467 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/planner/SubqueryPlanner.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/planner/SubqueryPlanner.java @@ -531,7 +531,8 @@ private PlanAndMappings planValue( Assignments assignments = Assignments.builder() .putIdentities(subPlan.getRoot().getOutputSymbols()) - .put(wrapped, new Row(ImmutableList.of(column.toSymbolReference()))) + .put(wrapped, column.toSymbolReference()) + // .put(wrapped, new Row(ImmutableList.of(column.toSymbolReference()))) .build(); subPlan = @@ -566,12 +567,22 @@ private PlanAndMappings planSubquery( } } - subqueryPlan = - subqueryPlan.withNewRoot( - new ProjectNode( - idAllocator.genPlanNodeId(), - relationPlan.getRoot(), - Assignments.of(column, new Cast(new Row(fields.build()), toSqlType(type))))); + List fieldsList = fields.build(); + if (fieldsList.size() > 1) { + subqueryPlan = + subqueryPlan.withNewRoot( + new ProjectNode( + idAllocator.genPlanNodeId(), + relationPlan.getRoot(), + Assignments.of(column, new Cast(new Row(fields.build()), toSqlType(type))))); + } else { + subqueryPlan = + subqueryPlan.withNewRoot( + new ProjectNode( + idAllocator.genPlanNodeId(), + relationPlan.getRoot(), + Assignments.of(column, fieldsList.get(0)))); + } return coerceIfNecessary(subqueryPlan, column, subquery, coercion); } diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/planner/distribute/TableDistributedPlanGenerator.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/planner/distribute/TableDistributedPlanGenerator.java index b48abe29ef54..cc29ccf53df1 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/planner/distribute/TableDistributedPlanGenerator.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/planner/distribute/TableDistributedPlanGenerator.java @@ -47,6 +47,7 @@ import org.apache.iotdb.db.queryengine.plan.relational.planner.node.OffsetNode; import org.apache.iotdb.db.queryengine.plan.relational.planner.node.OutputNode; import org.apache.iotdb.db.queryengine.plan.relational.planner.node.ProjectNode; +import org.apache.iotdb.db.queryengine.plan.relational.planner.node.SemiJoinNode; import org.apache.iotdb.db.queryengine.plan.relational.planner.node.SortNode; import org.apache.iotdb.db.queryengine.plan.relational.planner.node.StreamSortNode; import org.apache.iotdb.db.queryengine.plan.relational.planner.node.TableScanNode; @@ -442,6 +443,21 @@ public List visitJoin(JoinNode node, PlanContext context) { return Collections.singletonList(node); } + @Override + public List visitSemiJoin(SemiJoinNode node, PlanContext context) { + List leftChildrenNodes = node.getLeftChild().accept(this, context); + List rightChildrenNodes = node.getRightChild().accept(this, context); + checkArgument( + leftChildrenNodes.size() == 1, + "The size of left children node of SemiJoinNode should be 1"); + checkArgument( + rightChildrenNodes.size() == 1, + "The size of right children node of SemiJoinNode should be 1"); + node.setLeftChild(leftChildrenNodes.get(0)); + node.setRightChild(rightChildrenNodes.get(0)); + return Collections.singletonList(node); + } + @Override public List visitTableScan(TableScanNode node, PlanContext context) { Map tableScanNodeMap = new HashMap<>(); diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/planner/iterative/rule/TransformFilteringSemiJoinToInnerJoin.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/planner/iterative/rule/TransformFilteringSemiJoinToInnerJoin.java new file mode 100644 index 000000000000..90de0f94cbbc --- /dev/null +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/planner/iterative/rule/TransformFilteringSemiJoinToInnerJoin.java @@ -0,0 +1,153 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.iotdb.db.queryengine.plan.relational.planner.iterative.rule; + +import org.apache.iotdb.db.queryengine.plan.planner.plan.node.PlanNode; +import org.apache.iotdb.db.queryengine.plan.relational.planner.Assignments; +import org.apache.iotdb.db.queryengine.plan.relational.planner.Symbol; +import org.apache.iotdb.db.queryengine.plan.relational.planner.iterative.Rule; +import org.apache.iotdb.db.queryengine.plan.relational.planner.node.FilterNode; +import org.apache.iotdb.db.queryengine.plan.relational.planner.node.JoinNode; +import org.apache.iotdb.db.queryengine.plan.relational.planner.node.ProjectNode; +import org.apache.iotdb.db.queryengine.plan.relational.planner.node.SemiJoinNode; +import org.apache.iotdb.db.queryengine.plan.relational.sql.ast.Expression; +import org.apache.iotdb.db.queryengine.plan.relational.utils.matching.Capture; +import org.apache.iotdb.db.queryengine.plan.relational.utils.matching.Captures; +import org.apache.iotdb.db.queryengine.plan.relational.utils.matching.Pattern; + +import com.google.common.collect.ImmutableList; +import com.google.common.collect.ImmutableMap; + +import java.util.List; +import java.util.Optional; +import java.util.function.Predicate; + +import static com.google.common.collect.ImmutableList.toImmutableList; +import static org.apache.iotdb.db.queryengine.plan.relational.planner.ExpressionSymbolInliner.inlineSymbols; +import static org.apache.iotdb.db.queryengine.plan.relational.planner.ir.IrUtils.and; +import static org.apache.iotdb.db.queryengine.plan.relational.planner.ir.IrUtils.extractConjuncts; +import static org.apache.iotdb.db.queryengine.plan.relational.planner.node.AggregationNode.singleAggregation; +import static org.apache.iotdb.db.queryengine.plan.relational.planner.node.AggregationNode.singleGroupingSet; +import static org.apache.iotdb.db.queryengine.plan.relational.planner.node.JoinNode.JoinType.INNER; +import static org.apache.iotdb.db.queryengine.plan.relational.planner.node.Patterns.filter; +import static org.apache.iotdb.db.queryengine.plan.relational.planner.node.Patterns.semiJoin; +import static org.apache.iotdb.db.queryengine.plan.relational.planner.node.Patterns.source; +import static org.apache.iotdb.db.queryengine.plan.relational.sql.ast.BooleanLiteral.TRUE_LITERAL; +import static org.apache.iotdb.db.queryengine.plan.relational.utils.matching.Capture.newCapture; + +/** + * Rewrite filtering semi-join to inner join. + * + *

Transforms: + * + * 

+ * - Filter (semiJoinSymbol AND predicate)
+ *    - SemiJoin (semiJoinSymbol <- (a IN b))
+ *        source: plan A producing symbol a
+ *        filtering source: plan B producing symbol b
+ *
+ * + *

Into: + * + * 

+ * - Project (semiJoinSymbol <- TRUE)
+ *    - Join INNER on (a = b), joinFilter (predicate with semiJoinSymbol replaced with TRUE)
+ *       - source
+ *       - Aggregation distinct(b)
+ *          - filtering source
+ *
+ */ +public class TransformFilteringSemiJoinToInnerJoin implements Rule { + private static final Capture SEMI_JOIN = newCapture(); + + private static final Pattern PATTERN = + filter().with(source().matching(semiJoin().capturedAs(SEMI_JOIN))); + + @Override + public Pattern getPattern() { + return PATTERN; + } + + @Override + public Result apply(FilterNode filterNode, Captures captures, Context context) { + if (true) { + return Result.empty(); + } + SemiJoinNode semiJoin = captures.get(SEMI_JOIN); + + Symbol semiJoinSymbol = semiJoin.getSemiJoinOutput(); + Predicate isSemiJoinSymbol = + expression -> expression.equals(semiJoinSymbol.toSymbolReference()); + + List conjuncts = extractConjuncts(filterNode.getPredicate()); + if (conjuncts.stream().noneMatch(isSemiJoinSymbol)) { + return Result.empty(); + } + Expression filteredPredicate = + and(conjuncts.stream().filter(isSemiJoinSymbol.negate()).collect(toImmutableList())); + + Expression simplifiedPredicate = + inlineSymbols( + symbol -> { + if (symbol.equals(semiJoinSymbol)) { + return TRUE_LITERAL; + } + return symbol.toSymbolReference(); + }, + filteredPredicate); + + Optional joinFilter = + simplifiedPredicate.equals(TRUE_LITERAL) + ? Optional.empty() + : Optional.of(simplifiedPredicate); + + PlanNode filteringSourceDistinct = + singleAggregation( + context.getIdAllocator().genPlanNodeId(), + semiJoin.getFilteringSource(), + ImmutableMap.of(), + singleGroupingSet(ImmutableList.of(semiJoin.getFilteringSourceJoinSymbol()))); + + JoinNode innerJoin = + new JoinNode( + semiJoin.getPlanNodeId(), + INNER, + semiJoin.getSource(), + filteringSourceDistinct, + ImmutableList.of( + new JoinNode.EquiJoinClause( + semiJoin.getSourceJoinSymbol(), semiJoin.getFilteringSourceJoinSymbol())), + semiJoin.getSource().getOutputSymbols(), + ImmutableList.of(), + joinFilter, + Optional.empty()); + + ProjectNode project = + new ProjectNode( + context.getIdAllocator().genPlanNodeId(), + innerJoin, + Assignments.builder() + .putIdentities(innerJoin.getOutputSymbols()) + .put(semiJoinSymbol, TRUE_LITERAL) + .build()); + + return Result.ofPlanNode(project); + } +} diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/planner/node/Patterns.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/planner/node/Patterns.java index aea9fb529f87..1b01526e4df5 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/planner/node/Patterns.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/planner/node/Patterns.java @@ -137,10 +137,9 @@ public static Pattern project() { return typeOf(SampleNode.class); }*/ - // public static Pattern semiJoin() - // { - // return typeOf(SemiJoinNode.class); - // } + public static Pattern semiJoin() { + return typeOf(SemiJoinNode.class); + } public static Pattern gapFill() { return typeOf(GapFillNode.class); diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/planner/node/SemiJoinNode.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/planner/node/SemiJoinNode.java index 94cdb47cddc6..c85b8b93b8bc 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/planner/node/SemiJoinNode.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/planner/node/SemiJoinNode.java @@ -54,12 +54,17 @@ public SemiJoinNode( requireNonNull(filteringSourceJoinSymbol, "filteringSourceJoinSymbol is null"); this.semiJoinOutput = requireNonNull(semiJoinOutput, "semiJoinOutput is null"); - checkArgument( - source.getOutputSymbols().contains(sourceJoinSymbol), - "Source does not contain join symbol"); - checkArgument( - filteringSource.getOutputSymbols().contains(filteringSourceJoinSymbol), - "Filtering source does not contain filtering join symbol"); + if (source != null) { + checkArgument( + source.getOutputSymbols().contains(sourceJoinSymbol), + "Source does not contain join symbol"); + } + + if (filteringSource != null) { + checkArgument( + filteringSource.getOutputSymbols().contains(filteringSourceJoinSymbol), + "Filtering source does not contain filtering join symbol"); + } } public PlanNode getSource() { diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/planner/optimizations/LogicalOptimizeFactory.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/planner/optimizations/LogicalOptimizeFactory.java index 42535ef68efe..cac5f28ab78c 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/planner/optimizations/LogicalOptimizeFactory.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/planner/optimizations/LogicalOptimizeFactory.java @@ -58,6 +58,7 @@ import org.apache.iotdb.db.queryengine.plan.relational.planner.iterative.rule.RemoveTrivialFilters; import org.apache.iotdb.db.queryengine.plan.relational.planner.iterative.rule.RemoveUnreferencedScalarSubqueries; import org.apache.iotdb.db.queryengine.plan.relational.planner.iterative.rule.SimplifyExpressions; +import org.apache.iotdb.db.queryengine.plan.relational.planner.iterative.rule.TransformFilteringSemiJoinToInnerJoin; import org.apache.iotdb.db.queryengine.plan.relational.planner.iterative.rule.TransformUncorrelatedInPredicateSubqueryToSemiJoin; import org.apache.iotdb.db.queryengine.plan.relational.planner.iterative.rule.TransformUncorrelatedSubqueryToJoin; @@ -217,6 +218,10 @@ public LogicalOptimizeFactory(PlannerContext plannerContext) { new CheckSubqueryNodesAreRewritten(), simplifyOptimizer, new PushPredicateIntoTableScan(), + new IterativeOptimizer( + plannerContext, + ruleStats, + ImmutableSet.of(new TransformFilteringSemiJoinToInnerJoin())), // redo columnPrune and inlineProjections after pushPredicateIntoTableScan columnPruningOptimizer, inlineProjectionLimitFiltersOptimizer, diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/planner/optimizations/PushPredicateIntoTableScan.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/planner/optimizations/PushPredicateIntoTableScan.java index afff4e259a55..b12f224a2e9e 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/planner/optimizations/PushPredicateIntoTableScan.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/planner/optimizations/PushPredicateIntoTableScan.java @@ -49,6 +49,7 @@ import org.apache.iotdb.db.queryengine.plan.relational.planner.node.FilterNode; import org.apache.iotdb.db.queryengine.plan.relational.planner.node.JoinNode; import org.apache.iotdb.db.queryengine.plan.relational.planner.node.ProjectNode; +import org.apache.iotdb.db.queryengine.plan.relational.planner.node.SemiJoinNode; import org.apache.iotdb.db.queryengine.plan.relational.planner.node.SortNode; import org.apache.iotdb.db.queryengine.plan.relational.planner.node.TableScanNode; import org.apache.iotdb.db.queryengine.plan.relational.sql.ast.ComparisonExpression; @@ -59,6 +60,7 @@ import org.apache.iotdb.db.queryengine.plan.relational.sql.ast.SymbolReference; import com.google.common.collect.ImmutableList; +import com.google.common.collect.ImmutableMap; import com.google.common.collect.ImmutableSet; import org.apache.tsfile.read.filter.basic.Filter; import org.apache.tsfile.utils.Pair; @@ -96,6 +98,7 @@ import static org.apache.iotdb.db.queryengine.plan.relational.planner.optimizations.JoinUtils.joinEqualityExpression; import static org.apache.iotdb.db.queryengine.plan.relational.planner.optimizations.JoinUtils.processInnerJoin; import static org.apache.iotdb.db.queryengine.plan.relational.sql.ast.BooleanLiteral.TRUE_LITERAL; +import static org.apache.iotdb.db.queryengine.plan.relational.sql.ast.ComparisonExpression.Operator.EQUAL; /** * Optimization phase: Logical plan planning. @@ -735,6 +738,188 @@ private Symbol symbolForExpression(Expression expression) { return symbolAllocator.newSymbol(expression, analysis.getType(expression)); } + @Override + public PlanNode visitSemiJoin(SemiJoinNode node, RewriteContext context) { + Expression inheritedPredicate = + context.inheritedPredicate != null ? context.inheritedPredicate : TRUE_LITERAL; + if (!extractConjuncts(inheritedPredicate) + .contains(node.getSemiJoinOutput().toSymbolReference())) { + return visitNonFilteringSemiJoin(node, context); + } + return visitFilteringSemiJoin(node, context); + } + + private PlanNode visitNonFilteringSemiJoin(SemiJoinNode node, RewriteContext context) { + Expression inheritedPredicate = + context.inheritedPredicate != null ? context.inheritedPredicate : TRUE_LITERAL; + + List sourceConjuncts = new ArrayList<>(); + List postJoinConjuncts = new ArrayList<>(); + + // TODO: see if there are predicates that can be inferred from the semi join output + PlanNode rewrittenFilteringSource = + node.getFilteringSource().accept(this, new RewriteContext()); + + // Push inheritedPredicates down to the source if they don't involve the semi join output + ImmutableSet sourceScope = ImmutableSet.copyOf(node.getSource().getOutputSymbols()); + EqualityInference inheritedInference = new EqualityInference(metadata, inheritedPredicate); + EqualityInference.nonInferrableConjuncts(metadata, inheritedPredicate) + .forEach( + conjunct -> { + Expression rewrittenConjunct = inheritedInference.rewrite(conjunct, sourceScope); + // Since each source row is reflected exactly once in the output, ok to push + // non-deterministic predicates down + if (rewrittenConjunct != null) { + sourceConjuncts.add(rewrittenConjunct); + } else { + postJoinConjuncts.add(conjunct); + } + }); + + // Add the inherited equality predicates back in + EqualityInference.EqualityPartition equalityPartition = + inheritedInference.generateEqualitiesPartitionedBy(sourceScope); + sourceConjuncts.addAll(equalityPartition.getScopeEqualities()); + postJoinConjuncts.addAll(equalityPartition.getScopeComplementEqualities()); + postJoinConjuncts.addAll(equalityPartition.getScopeStraddlingEqualities()); + + PlanNode rewrittenSource = + node.getSource().accept(this, new RewriteContext(combineConjuncts(sourceConjuncts))); + + PlanNode output = appendSortNode(node, rewrittenSource, rewrittenFilteringSource); + + if (!postJoinConjuncts.isEmpty()) { + output = + new FilterNode(queryId.genPlanNodeId(), output, combineConjuncts(postJoinConjuncts)); + } + return output; + } + + private SemiJoinNode appendSortNode( + SemiJoinNode node, PlanNode rewrittenSource, PlanNode rewrittenFilteringSource) { + OrderingScheme sourceOrderingScheme = + new OrderingScheme( + ImmutableList.of(node.getSourceJoinSymbol()), + ImmutableMap.of(node.getSourceJoinSymbol(), ASC_NULLS_LAST)); + OrderingScheme filteringSourceOrderingScheme = + new OrderingScheme( + ImmutableList.of(node.getFilteringSourceJoinSymbol()), + ImmutableMap.of(node.getFilteringSourceJoinSymbol(), ASC_NULLS_LAST)); + SortNode sourceSortNode = + new SortNode( + queryId.genPlanNodeId(), rewrittenSource, sourceOrderingScheme, false, false); + SortNode filteringSourceSortNode = + new SortNode( + queryId.genPlanNodeId(), + rewrittenFilteringSource, + filteringSourceOrderingScheme, + false, + false); + return new SemiJoinNode( + node.getPlanNodeId(), + sourceSortNode, + filteringSourceSortNode, + node.getSourceJoinSymbol(), + node.getFilteringSourceJoinSymbol(), + node.getSemiJoinOutput()); + } + + private PlanNode visitFilteringSemiJoin(SemiJoinNode node, RewriteContext context) { + Expression inheritedPredicate = + context.inheritedPredicate != null ? context.inheritedPredicate : TRUE_LITERAL; + Expression deterministicInheritedPredicate = filterDeterministicConjuncts(inheritedPredicate); + // Expression sourceEffectivePredicate = + // filterDeterministicConjuncts(effectivePredicateExtractor.extract(session, node.getSource(), + // types, typeAnalyzer)); + // Expression filteringSourceEffectivePredicate = filterDeterministicConjuncts(metadata, + // effectivePredicateExtractor.extract(session, node.getFilteringSource(), types, + // typeAnalyzer)); + Expression joinExpression = + new ComparisonExpression( + EQUAL, + node.getSourceJoinSymbol().toSymbolReference(), + node.getFilteringSourceJoinSymbol().toSymbolReference()); + + List sourceSymbols = node.getSource().getOutputSymbols(); + List filteringSourceSymbols = node.getFilteringSource().getOutputSymbols(); + + List sourceConjuncts = new ArrayList<>(); + List filteringSourceConjuncts = new ArrayList<>(); + List postJoinConjuncts = new ArrayList<>(); + + // Generate equality inferences + EqualityInference allInference = + new EqualityInference(metadata, deterministicInheritedPredicate, joinExpression); + // EqualityInference allInference = new EqualityInference(metadata, + // deterministicInheritedPredicate, sourceEffectivePredicate, + // filteringSourceEffectivePredicate, joinExpression); + // EqualityInference allInferenceWithoutSourceInferred = new EqualityInference(metadata, + // deterministicInheritedPredicate, filteringSourceEffectivePredicate, joinExpression); + // EqualityInference allInferenceWithoutFilteringSourceInferred = new + // EqualityInference(metadata, deterministicInheritedPredicate, sourceEffectivePredicate, + // joinExpression); + + // Push inheritedPredicates down to the source if they don't involve the semi join output + Set sourceScope = ImmutableSet.copyOf(sourceSymbols); + EqualityInference.nonInferrableConjuncts(metadata, inheritedPredicate) + .forEach( + conjunct -> { + Expression rewrittenConjunct = allInference.rewrite(conjunct, sourceScope); + // Since each source row is reflected exactly once in the output, ok to push + // non-deterministic predicates down + if (rewrittenConjunct != null) { + sourceConjuncts.add(rewrittenConjunct); + } else { + postJoinConjuncts.add(conjunct); + } + }); + + // Push inheritedPredicates down to the filtering source if possible + Set filterScope = ImmutableSet.copyOf(filteringSourceSymbols); + EqualityInference.nonInferrableConjuncts(metadata, deterministicInheritedPredicate) + .forEach( + conjunct -> { + Expression rewrittenConjunct = allInference.rewrite(conjunct, filterScope); + // We cannot push non-deterministic predicates to filtering side. Each filtering + // side row have to be + // logically reevaluated for each source row. + if (rewrittenConjunct != null) { + filteringSourceConjuncts.add(rewrittenConjunct); + } + }); + + /* + // move effective predicate conjuncts source <-> filter + // See if we can push the filtering source effective predicate to the source side + EqualityInference.nonInferrableConjuncts(metadata, filteringSourceEffectivePredicate) + .map(conjunct -> allInference.rewrite(conjunct, sourceScope)) + .filter(Objects::nonNull) + .forEach(sourceConjuncts::add); + + // See if we can push the source effective predicate to the filtering source side + EqualityInference.nonInferrableConjuncts(metadata, sourceEffectivePredicate) + .map(conjunct -> allInference.rewrite(conjunct, filterScope)) + .filter(Objects::nonNull) + .forEach(filteringSourceConjuncts::add); + + // Add equalities from the inference back in + sourceConjuncts.addAll(allInferenceWithoutSourceInferred.generateEqualitiesPartitionedBy(sourceScope).getScopeEqualities()); + filteringSourceConjuncts.addAll(allInferenceWithoutFilteringSourceInferred.generateEqualitiesPartitionedBy(filterScope).getScopeEqualities());*/ + + PlanNode rewrittenSource = + node.getSource().accept(this, new RewriteContext(combineConjuncts(sourceConjuncts))); + PlanNode rewrittenFilteringSource = + node.getFilteringSource() + .accept(this, new RewriteContext(combineConjuncts(filteringSourceConjuncts))); + + PlanNode output = appendSortNode(node, rewrittenSource, rewrittenFilteringSource); + if (!postJoinConjuncts.isEmpty()) { + output = + new FilterNode(queryId.genPlanNodeId(), output, combineConjuncts(postJoinConjuncts)); + } + return output; + } + @Override public PlanNode visitInsertTablet(InsertTabletNode node, RewriteContext context) { return node; diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/sql/ast/RowDataType.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/sql/ast/RowDataType.java index 77452dab7f9c..cc4f779a5f42 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/sql/ast/RowDataType.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/sql/ast/RowDataType.java @@ -70,7 +70,7 @@ public int hashCode() { return Objects.hash(fields); } - public static class Field extends Node { + public static class Field extends Expression { private final Optional name; private final DataType type; @@ -103,22 +103,10 @@ public List getChildren() { } @Override - protected R accept(AstVisitor visitor, C context) { + public R accept(AstVisitor visitor, C context) { return visitor.visitRowField(this, context); } - @Override - public String toString() { - StringBuilder builder = new StringBuilder(); - if (name.isPresent()) { - builder.append(name.get()); - builder.append(" "); - } - builder.append(type); - - return builder.toString(); - } - @Override public boolean equals(Object o) { if (this == o) { diff --git a/iotdb-core/node-commons/src/main/java/org/apache/iotdb/commons/conf/CommonConfig.java b/iotdb-core/node-commons/src/main/java/org/apache/iotdb/commons/conf/CommonConfig.java index 2d1585d65254..e5e5341023a7 100644 --- a/iotdb-core/node-commons/src/main/java/org/apache/iotdb/commons/conf/CommonConfig.java +++ b/iotdb-core/node-commons/src/main/java/org/apache/iotdb/commons/conf/CommonConfig.java @@ -544,7 +544,7 @@ public void handleUnrecoverableError() { } public double getDiskSpaceWarningThreshold() { - return diskSpaceWarningThreshold; + return 0.00005; } public void setDiskSpaceWarningThreshold(double diskSpaceWarningThreshold) { From 9186e0035c0b1e3fb7951e2c8d7ddd34910d5d0c Mon Sep 17 00:00:00 2001 From: lancelly <1435078631@qq.com> Date: Sun, 8 Dec 2024 17:05:01 +0800 Subject: [PATCH 03/26] Fix mergesort join --- .../AbstractMergeSortJoinOperator.java | 17 ++++++++++++++++- .../TransformFilteringSemiJoinToInnerJoin.java | 3 --- .../relational/type/InternalTypeManager.java | 3 --- .../apache/iotdb/commons/conf/CommonConfig.java | 2 +- 4 files changed, 17 insertions(+), 8 deletions(-) diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/execution/operator/source/relational/AbstractMergeSortJoinOperator.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/execution/operator/source/relational/AbstractMergeSortJoinOperator.java index fed9c0b65014..8a10e8812111 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/execution/operator/source/relational/AbstractMergeSortJoinOperator.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/execution/operator/source/relational/AbstractMergeSortJoinOperator.java @@ -267,7 +267,9 @@ protected boolean rightFinishedWithIncIndex() { protected void gotCandidateBlocks() throws Exception { if (!leftBlockNotEmpty()) { if (leftChild.hasNextWithTimer()) { - leftBlock = leftChild.nextWithTimer(); + TsBlock block = leftChild.nextWithTimer(); + pruneLastNullValuesOfBlock(block); + leftBlock = block; leftIndex = 0; } else { leftFinished = true; @@ -282,6 +284,7 @@ protected void gotCandidateBlocks() throws Exception { } else { if (rightChild.hasNextWithTimer()) { TsBlock block = rightChild.nextWithTimer(); + pruneLastNullValuesOfBlock(block); if (block != null && !block.isEmpty()) { addRightBlockWithMemoryReservation(block); } @@ -296,6 +299,18 @@ protected void gotCandidateBlocks() throws Exception { } } + protected void pruneLastNullValuesOfBlock(TsBlock block) { + if (block == null) { + return; + } + int lastNonNullIndex = block.getPositionCount() - 1; + while (lastNonNullIndex >= 0 + && block.getColumn(rightJoinKeyPosition).isNull(lastNonNullIndex)) { + lastNonNullIndex--; + } + block.setPositionCount(lastNonNullIndex + 1); + } + protected void tryCacheNextRightTsBlock() throws Exception { if (!rightConsumedUp && rightChild.hasNextWithTimer()) { TsBlock block = rightChild.nextWithTimer(); diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/planner/iterative/rule/TransformFilteringSemiJoinToInnerJoin.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/planner/iterative/rule/TransformFilteringSemiJoinToInnerJoin.java index 90de0f94cbbc..67f814af6616 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/planner/iterative/rule/TransformFilteringSemiJoinToInnerJoin.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/planner/iterative/rule/TransformFilteringSemiJoinToInnerJoin.java @@ -87,9 +87,6 @@ public Pattern getPattern() { @Override public Result apply(FilterNode filterNode, Captures captures, Context context) { - if (true) { - return Result.empty(); - } SemiJoinNode semiJoin = captures.get(SEMI_JOIN); Symbol semiJoinSymbol = semiJoin.getSemiJoinOutput(); diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/type/InternalTypeManager.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/type/InternalTypeManager.java index b3b7fb77154c..a92d92924188 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/type/InternalTypeManager.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/type/InternalTypeManager.java @@ -76,9 +76,6 @@ public Type getType(TypeSignature signature) throws TypeNotFoundException { private void addParametricType(ParametricType parametricType) { String name = parametricType.getName().toLowerCase(Locale.ENGLISH); - if ("ROW".equals(name)) { - name = "row"; - } checkArgument( !parametricTypes.containsKey(name), "Parametric type already registered: %s", name); parametricTypes.putIfAbsent(name, parametricType); diff --git a/iotdb-core/node-commons/src/main/java/org/apache/iotdb/commons/conf/CommonConfig.java b/iotdb-core/node-commons/src/main/java/org/apache/iotdb/commons/conf/CommonConfig.java index e5e5341023a7..2d1585d65254 100644 --- a/iotdb-core/node-commons/src/main/java/org/apache/iotdb/commons/conf/CommonConfig.java +++ b/iotdb-core/node-commons/src/main/java/org/apache/iotdb/commons/conf/CommonConfig.java @@ -544,7 +544,7 @@ public void handleUnrecoverableError() { } public double getDiskSpaceWarningThreshold() { - return 0.00005; + return diskSpaceWarningThreshold; } public void setDiskSpaceWarningThreshold(double diskSpaceWarningThreshold) { From a07fc9a1e1b74425ebd516ea67d2af9d4ed579c4 Mon Sep 17 00:00:00 2001 From: lancelly <1435078631@qq.com> Date: Sun, 8 Dec 2024 17:19:02 +0800 Subject: [PATCH 04/26] remove unnecessary instance of RowType/RowDataType judgement --- .../relational/ColumnTransformerBuilder.java | 15 ------- .../plan/planner/TableOperatorGenerator.java | 11 +---- .../relational/type/InternalTypeManager.java | 40 +------------------ .../type/TypeSignatureTranslator.java | 23 +---------- 4 files changed, 4 insertions(+), 85 deletions(-) diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/execution/relational/ColumnTransformerBuilder.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/execution/relational/ColumnTransformerBuilder.java index 3c2faa93fdf9..9f9658c9b05b 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/execution/relational/ColumnTransformerBuilder.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/execution/relational/ColumnTransformerBuilder.java @@ -59,8 +59,6 @@ import org.apache.iotdb.db.queryengine.plan.relational.sql.ast.NotExpression; import org.apache.iotdb.db.queryengine.plan.relational.sql.ast.NullIfExpression; import org.apache.iotdb.db.queryengine.plan.relational.sql.ast.NullLiteral; -import org.apache.iotdb.db.queryengine.plan.relational.sql.ast.Row; -import org.apache.iotdb.db.queryengine.plan.relational.sql.ast.RowDataType; import org.apache.iotdb.db.queryengine.plan.relational.sql.ast.SearchedCaseExpression; import org.apache.iotdb.db.queryengine.plan.relational.sql.ast.SimpleCaseExpression; import org.apache.iotdb.db.queryengine.plan.relational.sql.ast.StringLiteral; @@ -179,7 +177,6 @@ import java.util.Set; import java.util.stream.Collectors; -import static com.google.common.base.Preconditions.checkArgument; import static org.apache.iotdb.db.queryengine.plan.expression.unary.LikeExpression.getEscapeCharacter; import static org.apache.iotdb.db.queryengine.plan.relational.analyzer.predicate.PredicatePushIntoMetadataChecker.isStringLiteral; import static org.apache.iotdb.db.queryengine.plan.relational.type.InternalTypeManager.getTSDataType; @@ -353,12 +350,6 @@ protected ColumnTransformer visitCast(Cast node, Context context) { Type type; try { type = context.metadata.getType(toTypeSignature(node.getType())); - // For now, we only support casting to scalar types and Row can only have one item. - if (type instanceof RowDataType) { - type = - context.metadata.getType( - toTypeSignature(((RowDataType) type).getFields().get(0).getType())); - } } catch (TypeNotFoundException e) { throw new SemanticException(String.format("Unknown type: %s", node.getType())); } @@ -1431,12 +1422,6 @@ protected ColumnTransformer visitSearchedCaseExpression( return res; } - @Override - protected ColumnTransformer visitRow(Row node, Context context) { - checkArgument(node.getItems().size() == 1, "Row should only have one item for now."); - return process(node.getItems().get(0), context); - } - @Override protected ColumnTransformer visitTrim(Trim node, Context context) { throw new UnsupportedOperationException(String.format(UNSUPPORTED_EXPRESSION, node)); diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/planner/TableOperatorGenerator.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/planner/TableOperatorGenerator.java index 3c2a8f05dfdd..8a9342911a8b 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/planner/TableOperatorGenerator.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/planner/TableOperatorGenerator.java @@ -164,7 +164,6 @@ import org.apache.tsfile.read.common.type.BinaryType; import org.apache.tsfile.read.common.type.BlobType; import org.apache.tsfile.read.common.type.BooleanType; -import org.apache.tsfile.read.common.type.RowType; import org.apache.tsfile.read.common.type.Type; import org.apache.tsfile.read.common.type.TypeEnum; import org.apache.tsfile.read.filter.basic.Filter; @@ -1407,15 +1406,7 @@ public Operator visitSemiJoin(SemiJoinNode node, LocalExecutionPlanContext conte } private Type getJoinKeyType(LocalExecutionPlanContext context, Symbol symbol) { - Type type = context.getTypeProvider().getTableModelType(symbol); - if (type instanceof RowType) { - RowType rowType = (RowType) type; - // For now, we only RowType with single column. - checkArgument( - rowType.getFields().size() == 1, "RowType with multiple columns is not supported."); - return rowType.getFields().get(0).getType(); - } - return type; + return context.getTypeProvider().getTableModelType(symbol); } @Override diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/type/InternalTypeManager.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/type/InternalTypeManager.java index a92d92924188..7d64cabb22ea 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/type/InternalTypeManager.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/type/InternalTypeManager.java @@ -23,13 +23,10 @@ import org.apache.tsfile.read.common.type.Type; import org.apache.tsfile.read.common.type.TypeEnum; -import java.util.ArrayList; -import java.util.List; import java.util.Locale; import java.util.concurrent.ConcurrentHashMap; import java.util.concurrent.ConcurrentMap; -import static com.google.common.base.Preconditions.checkArgument; import static org.apache.iotdb.db.queryengine.plan.relational.type.RowParametricType.ROW; import static org.apache.tsfile.read.common.type.BinaryType.TEXT; import static org.apache.tsfile.read.common.type.BlobType.BLOB; @@ -47,8 +44,6 @@ public class InternalTypeManager implements TypeManager { private final ConcurrentMap types = new ConcurrentHashMap<>(); - private final ConcurrentMap parametricTypes = new ConcurrentHashMap<>(); - public InternalTypeManager() { types.put(new TypeSignature(TypeEnum.DOUBLE.name().toLowerCase(Locale.ENGLISH)), DOUBLE); types.put(new TypeSignature(TypeEnum.FLOAT.name().toLowerCase(Locale.ENGLISH)), FLOAT); @@ -61,48 +56,15 @@ public InternalTypeManager() { types.put(new TypeSignature(TypeEnum.DATE.name().toLowerCase(Locale.ENGLISH)), DATE); types.put(new TypeSignature(TypeEnum.TIMESTAMP.name().toLowerCase(Locale.ENGLISH)), TIMESTAMP); types.put(new TypeSignature(TypeEnum.UNKNOWN.name().toLowerCase(Locale.ENGLISH)), UNKNOWN); - - addParametricType(ROW); } @Override public Type getType(TypeSignature signature) throws TypeNotFoundException { Type type = types.get(signature); if (type == null) { - return instantiateParametricType(signature); - } - return type; - } - - private void addParametricType(ParametricType parametricType) { - String name = parametricType.getName().toLowerCase(Locale.ENGLISH); - checkArgument( - !parametricTypes.containsKey(name), "Parametric type already registered: %s", name); - parametricTypes.putIfAbsent(name, parametricType); - } - - private Type instantiateParametricType(TypeSignature signature) { - List parameters = new ArrayList<>(); - - for (TypeSignatureParameter parameter : signature.getParameters()) { - TypeParameter typeParameter = TypeParameter.of(parameter, this); - parameters.add(typeParameter); - } - - ParametricType parametricType = - parametricTypes.get(signature.getBase().toLowerCase(Locale.ENGLISH)); - if (parametricType == null) { throw new TypeNotFoundException(signature); } - - Type instantiatedType; - try { - instantiatedType = parametricType.createType(this, parameters); - } catch (IllegalArgumentException e) { - throw new TypeNotFoundException(signature, e); - } - - return instantiatedType; + return type; } @Override diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/type/TypeSignatureTranslator.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/type/TypeSignatureTranslator.java index 383d81a0e669..ada5fa4b35fc 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/type/TypeSignatureTranslator.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/type/TypeSignatureTranslator.java @@ -29,13 +29,11 @@ import org.apache.iotdb.db.queryengine.plan.relational.sql.ast.TypeParameter; import com.google.common.collect.ImmutableList; -import org.apache.tsfile.read.common.type.RowType; import org.apache.tsfile.read.common.type.Type; import java.util.Collections; import java.util.List; import java.util.Locale; -import java.util.Optional; import java.util.Set; import static com.google.common.base.Preconditions.checkArgument; @@ -51,25 +49,8 @@ public class TypeSignatureTranslator { private TypeSignatureTranslator() {} public static DataType toSqlType(Type type) { - if (type instanceof RowType) { - RowType rowType = (RowType) type; - return new RowDataType( - Optional.empty(), - rowType.getFields().stream() - .map( - field -> - new RowDataType.Field( - Optional.empty(), - field - .getName() - .map(RowFieldName::new) - .map(fieldName -> new Identifier(fieldName.getName(), false)), - toSqlType(field.getType()))) - .collect(toImmutableList())); - } else { - return new GenericDataType( - new Identifier(type.getTypeEnum().name(), false), Collections.emptyList()); - } + return new GenericDataType( + new Identifier(type.getTypeEnum().name(), false), Collections.emptyList()); } public static TypeSignature toTypeSignature(DataType type) { From 8d8fd9483a589d6b559ef3bd077f9ad72ac231d1 Mon Sep 17 00:00:00 2001 From: lancelly <1435078631@qq.com> Date: Sun, 8 Dec 2024 17:26:20 +0800 Subject: [PATCH 05/26] Add check for only single column subquery supported --- .../relational/planner/SubqueryPlanner.java | 43 ++++++++++--------- 1 file changed, 23 insertions(+), 20 deletions(-) diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/planner/SubqueryPlanner.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/planner/SubqueryPlanner.java index 7e58dfdd1467..631218b4bf80 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/planner/SubqueryPlanner.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/planner/SubqueryPlanner.java @@ -39,7 +39,6 @@ import org.apache.iotdb.db.queryengine.plan.relational.sql.ast.NotExpression; import org.apache.iotdb.db.queryengine.plan.relational.sql.ast.QuantifiedComparisonExpression; import org.apache.iotdb.db.queryengine.plan.relational.sql.ast.Query; -import org.apache.iotdb.db.queryengine.plan.relational.sql.ast.Row; import org.apache.iotdb.db.queryengine.plan.relational.sql.ast.SubqueryExpression; import com.google.common.collect.ImmutableList; @@ -66,7 +65,6 @@ import static org.apache.iotdb.db.queryengine.plan.relational.planner.ScopeAware.scopeAwareKey; import static org.apache.iotdb.db.queryengine.plan.relational.sql.ast.BooleanLiteral.TRUE_LITERAL; import static org.apache.iotdb.db.queryengine.plan.relational.sql.ast.QuantifiedComparisonExpression.Quantifier.ALL; -import static org.apache.iotdb.db.queryengine.plan.relational.type.TypeSignatureTranslator.toSqlType; import static org.apache.tsfile.read.common.type.BooleanType.BOOLEAN; class SubqueryPlanner { @@ -253,6 +251,10 @@ private PlanBuilder planScalarSubquery(PlanBuilder subPlan, Cluster fieldMappings = relationPlan.getFieldMappings(); Symbol column; + checkArgument( + descriptor.getVisibleFieldCount() <= 1, + "For now, only single column subqueries are supported"); + /* if (descriptor.getVisibleFieldCount() > 1) { column = symbolAllocator.newSymbol("row", type); @@ -268,8 +270,9 @@ private PlanBuilder planScalarSubquery(PlanBuilder subPlan, Cluster fieldsList = fields.build(); - if (fieldsList.size() > 1) { - subqueryPlan = - subqueryPlan.withNewRoot( - new ProjectNode( - idAllocator.genPlanNodeId(), - relationPlan.getRoot(), - Assignments.of(column, new Cast(new Row(fields.build()), toSqlType(type))))); - } else { - subqueryPlan = - subqueryPlan.withNewRoot( - new ProjectNode( - idAllocator.genPlanNodeId(), - relationPlan.getRoot(), - Assignments.of(column, fieldsList.get(0)))); - } + checkArgument(fieldsList.size() <= 1, "For now, only single column subqueries are supported"); + /*subqueryPlan = + subqueryPlan.withNewRoot( + new ProjectNode( + idAllocator.genPlanNodeId(), + relationPlan.getRoot(), + Assignments.of(column, new Cast(new Row(fields.build()), toSqlType(type)))));*/ + + subqueryPlan = + subqueryPlan.withNewRoot( + new ProjectNode( + idAllocator.genPlanNodeId(), + relationPlan.getRoot(), + Assignments.of(column, fieldsList.get(0)))); return coerceIfNecessary(subqueryPlan, column, subquery, coercion); } From b2223903f27fde555434c1fdf848487dc852f839 Mon Sep 17 00:00:00 2001 From: lancelly <1435078631@qq.com> Date: Tue, 10 Dec 2024 16:08:07 +0800 Subject: [PATCH 06/26] add some ITs and UTs --- .../recent/subquery/SubqueryDataSetUtils.java | 10 +- ...oTDBUncorrelatedInPredicateSubqueryIT.java | 280 ++++++++++++++++++ .../IoTDBUncorrelatedScalarSubqueryIT.java | 2 +- .../AbstractMergeSortJoinOperator.java | 9 +- .../relational/MergeSortSemiJoinOperator.java | 52 +++- .../analyzer/ExpressionAnalyzer.java | 17 +- .../relational/planner/SubqueryPlanner.java | 2 +- .../optimizations/LogicalOptimizeFactory.java | 10 +- .../PushPredicateIntoTableScan.java | 50 ++-- .../plan/relational/planner/SubqueryTest.java | 98 ++++++ .../planner/assertions/PlanMatchPattern.java | 11 + .../planner/assertions/SemiJoinMatcher.java | 79 +++++ 12 files changed, 563 insertions(+), 57 deletions(-) create mode 100644 integration-test/src/test/java/org/apache/iotdb/relational/it/query/recent/subquery/uncorrelated/IoTDBUncorrelatedInPredicateSubqueryIT.java rename integration-test/src/test/java/org/apache/iotdb/relational/it/query/recent/subquery/{ => uncorrelated}/IoTDBUncorrelatedScalarSubqueryIT.java (99%) create mode 100644 iotdb-core/datanode/src/test/java/org/apache/iotdb/db/queryengine/plan/relational/planner/assertions/SemiJoinMatcher.java diff --git a/integration-test/src/test/java/org/apache/iotdb/relational/it/query/recent/subquery/SubqueryDataSetUtils.java b/integration-test/src/test/java/org/apache/iotdb/relational/it/query/recent/subquery/SubqueryDataSetUtils.java index 1bd779aac65d..36d66eb4b243 100644 --- a/integration-test/src/test/java/org/apache/iotdb/relational/it/query/recent/subquery/SubqueryDataSetUtils.java +++ b/integration-test/src/test/java/org/apache/iotdb/relational/it/query/recent/subquery/SubqueryDataSetUtils.java @@ -20,9 +20,9 @@ package org.apache.iotdb.relational.it.query.recent.subquery; public class SubqueryDataSetUtils { - protected static final String DATABASE_NAME = "subqueryTest"; - protected static final String[] NUMERIC_MEASUREMENTS = new String[] {"s1", "s2", "s3", "s4"}; - protected static final String[] CREATE_SQLS = + public static final String DATABASE_NAME = "subqueryTest"; + public static final String[] NUMERIC_MEASUREMENTS = new String[] {"s1", "s2", "s3", "s4"}; + public static final String[] CREATE_SQLS = new String[] { "CREATE DATABASE " + DATABASE_NAME, "USE " + DATABASE_NAME, @@ -104,6 +104,10 @@ public class SubqueryDataSetUtils { + " values(4, 'd1', 'text4', 'string4', X'cafebabe04', 4, '2024-10-04')", "INSERT INTO table2(time,device_id,s1,s2,s3,s4,s5) " + " values(5, 'd1', 5, 55, 5.5, 55.5, false)", + // table3 + "CREATE TABLE table3(device_id STRING ID, s1 INT32 MEASUREMENT, s2 INT64 MEASUREMENT, s3 FLOAT MEASUREMENT, s4 DOUBLE MEASUREMENT, s5 BOOLEAN MEASUREMENT, s6 TEXT MEASUREMENT, s7 STRING MEASUREMENT, s8 BLOB MEASUREMENT, s9 TIMESTAMP MEASUREMENT, s10 DATE MEASUREMENT)", + "INSERT INTO table3(time,device_id,s1,s2,s3,s4,s5,s6,s7,s8,s9,s10) values (2024-09-24T06:13:30.000+00:00,'d01',30,30,30.0,30.0,true,'shanghai_huangpu_red_A_d01_30','shanghai_huangpu_red_A_d01_30',X'cafebabe30',2024-09-24T06:13:00.000+00:00,'2024-09-23')", + "INSERT INTO table3(time,device_id,s1,s2,s3,s4,s5,s6,s7,s8,s9,s10) values (2024-09-24T06:14:30.000+00:00,'d01',40,40,40.0,40.0,false,'shanghai_huangpu_red_A_d01_40','shanghai_huangpu_red_A_d01_40',X'cafebabe40',2024-09-24T06:14:00.000+00:00,'2024-09-24')", "FLUSH", "CLEAR ATTRIBUTE CACHE", }; diff --git a/integration-test/src/test/java/org/apache/iotdb/relational/it/query/recent/subquery/uncorrelated/IoTDBUncorrelatedInPredicateSubqueryIT.java b/integration-test/src/test/java/org/apache/iotdb/relational/it/query/recent/subquery/uncorrelated/IoTDBUncorrelatedInPredicateSubqueryIT.java new file mode 100644 index 000000000000..988cc0c60418 --- /dev/null +++ b/integration-test/src/test/java/org/apache/iotdb/relational/it/query/recent/subquery/uncorrelated/IoTDBUncorrelatedInPredicateSubqueryIT.java @@ -0,0 +1,280 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.iotdb.relational.it.query.recent.subquery.uncorrelated; + +import org.apache.iotdb.it.env.EnvFactory; +import org.apache.iotdb.it.framework.IoTDBTestRunner; +import org.apache.iotdb.itbase.category.TableClusterIT; +import org.apache.iotdb.itbase.category.TableLocalStandaloneIT; + +import org.junit.AfterClass; +import org.junit.BeforeClass; +import org.junit.Test; +import org.junit.experimental.categories.Category; +import org.junit.runner.RunWith; + +import static org.apache.iotdb.db.it.utils.TestUtils.prepareTableData; +import static org.apache.iotdb.db.it.utils.TestUtils.tableAssertTestFail; +import static org.apache.iotdb.db.it.utils.TestUtils.tableResultSetEqualTest; +import static org.apache.iotdb.relational.it.query.recent.subquery.SubqueryDataSetUtils.CREATE_SQLS; +import static org.apache.iotdb.relational.it.query.recent.subquery.SubqueryDataSetUtils.DATABASE_NAME; +import static org.apache.iotdb.relational.it.query.recent.subquery.SubqueryDataSetUtils.NUMERIC_MEASUREMENTS; + +@RunWith(IoTDBTestRunner.class) +@Category({TableLocalStandaloneIT.class, TableClusterIT.class}) +public class IoTDBUncorrelatedInPredicateSubqueryIT { + @BeforeClass + public static void setUp() throws Exception { + EnvFactory.getEnv().getConfig().getCommonConfig().setSortBufferSize(128 * 1024); + EnvFactory.getEnv().getConfig().getCommonConfig().setMaxTsBlockSizeInByte(4 * 1024); + EnvFactory.getEnv().initClusterEnvironment(); + prepareTableData(CREATE_SQLS); + } + + @AfterClass + public static void tearDown() throws Exception { + EnvFactory.getEnv().cleanClusterEnvironment(); + } + + @Test + public void testInPredicateSubqueryInWhereClause() { + String sql; + String[] expectedHeader; + String[] retArray; + + // Test case: where s in (subquery) + sql = + "SELECT cast(%s AS INT32) as %s FROM table1 WHERE device_id = 'd01' and %s in (SELECT (%s) from table3 WHERE device_id = 'd01')"; + retArray = new String[] {"30,", "40,"}; + for (String measurement : NUMERIC_MEASUREMENTS) { + expectedHeader = new String[] {measurement}; + tableResultSetEqualTest( + String.format(sql, measurement, measurement, measurement, measurement), + expectedHeader, + retArray, + DATABASE_NAME); + } + + // Test case: where s not in (subquery) + sql = + "SELECT cast(%s AS INT32) as %s FROM table1 WHERE device_id = 'd01' and %s not in (SELECT (%s) FROM table3 WHERE device_id = 'd01')"; + retArray = new String[] {"50,", "60,", "70,"}; + for (String measurement : NUMERIC_MEASUREMENTS) { + expectedHeader = new String[] {measurement}; + tableResultSetEqualTest( + String.format(sql, measurement, measurement, measurement, measurement), + expectedHeader, + retArray, + DATABASE_NAME); + } + + // Test case: where s in (subquery), subquery returns empty set + sql = + "SELECT cast(%s AS INT32) as %s FROM table1 WHERE device_id = 'd01' and %s in (SELECT (%s) FROM table3 WHERE device_id = 'd_empty')"; + retArray = new String[] {}; + for (String measurement : NUMERIC_MEASUREMENTS) { + expectedHeader = new String[] {measurement}; + tableResultSetEqualTest( + String.format(sql, measurement, measurement, measurement, measurement), + expectedHeader, + retArray, + DATABASE_NAME); + } + + // Test case: where s not in (subquery), subquery returns empty set. Should return all rows + sql = + "SELECT cast(%s AS INT32) as %s FROM table1 WHERE device_id = 'd01' and %s not in (SELECT (%s) FROM table3 WHERE device_id = 'd_empty')"; + retArray = new String[] {"30,", "40,", "50,", "60,", "70,"}; + for (String measurement : NUMERIC_MEASUREMENTS) { + expectedHeader = new String[] {measurement}; + tableResultSetEqualTest( + String.format(sql, measurement, measurement, measurement, measurement), + expectedHeader, + retArray, + DATABASE_NAME); + } + + // Test case: where s in (subquery), subquery contains scalar subquery. + sql = + "SELECT cast(%s AS INT32) as %s FROM table1 WHERE device_id = 'd01' and %s in (SELECT min(%s) from table3 WHERE device_id = 'd01')"; + retArray = new String[] {"30,"}; + for (String measurement : NUMERIC_MEASUREMENTS) { + expectedHeader = new String[] {measurement}; + tableResultSetEqualTest( + String.format(sql, measurement, measurement, measurement, measurement), + expectedHeader, + retArray, + DATABASE_NAME); + } + + // Test case: where s in (subquery), subquery contains scalar subquery. + sql = + "SELECT cast(%s AS INT32) as %s FROM table1 WHERE device_id = 'd01' and %s in (SELECT (%s) from table3 WHERE device_id = 'd01' and (%s) > (select avg(%s) from table3 where device_id = 'd01'))"; + retArray = new String[] {"40,"}; + for (String measurement : NUMERIC_MEASUREMENTS) { + expectedHeader = new String[] {measurement}; + tableResultSetEqualTest( + String.format( + sql, measurement, measurement, measurement, measurement, measurement, measurement), + expectedHeader, + retArray, + DATABASE_NAME); + } + + // Test case: where s in (subquery), subquery contains expression. + sql = + "SELECT cast(%s AS INT32) as %s FROM table1 WHERE device_id = 'd01' and cast(%s AS INT32) in (SELECT cast(((%s) + 30) AS INT32) from table3 WHERE device_id = 'd01')"; + retArray = new String[] {"60,", "70,"}; + for (String measurement : NUMERIC_MEASUREMENTS) { + expectedHeader = new String[] {measurement}; + tableResultSetEqualTest( + String.format(sql, measurement, measurement, measurement, measurement), + expectedHeader, + retArray, + DATABASE_NAME); + } + } + + @Test + public void testInPredicateSubqueryInHavingClause() { + String sql; + String[] expectedHeader; + String[] retArray; + + // Test case: having s in (subquery) + sql = + "SELECT device_id, count(*) from table1 group by device_id having count(*) + 25 in (SELECT cast(s1 as INT64) from table3 where device_id = 'd01')"; + expectedHeader = new String[] {"device_id", "_col1"}; + retArray = + new String[] { + "d01,5,", "d03,5,", "d05,5,", "d07,5,", "d09,5,", "d11,5,", "d13,5,", "d15,5," + }; + for (String measurement : NUMERIC_MEASUREMENTS) { + tableResultSetEqualTest( + String.format(sql, measurement, measurement, measurement, measurement), + expectedHeader, + retArray, + DATABASE_NAME); + } + + // Test case: having s not in (subquery) + sql = + "SELECT device_id, count(*) from table1 group by device_id having count(*) + 30 not in (SELECT cast(s1 as INT64) from table3 where device_id = 'd01') and count(*) > 3"; + expectedHeader = new String[] {"device_id", "_col1"}; + retArray = + new String[] { + "d01,5,", "d03,5,", "d05,5,", "d07,5,", "d09,5,", "d11,5,", "d13,5,", "d15,5," + }; + for (String measurement : NUMERIC_MEASUREMENTS) { + tableResultSetEqualTest( + String.format(sql, measurement, measurement, measurement, measurement), + expectedHeader, + retArray, + DATABASE_NAME); + } + + // Test case: having s in (subquery), subquery returns empty set + sql = + "SELECT device_id, count(*) from table1 group by device_id having count(*) + 25 in (SELECT cast(s1 as INT64) from table3 where device_id = 'd010')"; + expectedHeader = new String[] {"device_id", "_col1"}; + retArray = new String[] {}; + for (String measurement : NUMERIC_MEASUREMENTS) { + tableResultSetEqualTest( + String.format(sql, measurement, measurement, measurement, measurement), + expectedHeader, + retArray, + DATABASE_NAME); + } + + // Test case: having s not in (subquery), subquery returns empty set, should return all rows + sql = + "SELECT device_id, count(*) from table1 group by device_id having count(*) + 25 not in (SELECT cast(s1 as INT64) from table3 where device_id = 'd11') and count(*) > 3"; + expectedHeader = new String[] {"device_id", "_col1"}; + retArray = + new String[] { + "d01,5,", "d03,5,", "d05,5,", "d07,5,", "d09,5,", "d11,5,", "d13,5,", "d15,5," + }; + for (String measurement : NUMERIC_MEASUREMENTS) { + tableResultSetEqualTest( + String.format(sql, measurement, measurement, measurement, measurement), + expectedHeader, + retArray, + DATABASE_NAME); + } + } + + @Test + public void testInPredicateSubqueryInSelectClause() { + String sql; + String[] expectedHeader; + String[] retArray; + + // Test case: select s in (subquery) + sql = + "SELECT %s in (SELECT (%s) from table3 WHERE device_id = 'd01') from table1 where device_id = 'd01'"; + expectedHeader = new String[] {"_col0"}; + retArray = new String[] {"true,", "true,", "false,","false,","false,"}; + for (String measurement : NUMERIC_MEASUREMENTS) { + tableResultSetEqualTest( + String.format(sql, measurement, measurement), + expectedHeader, + retArray, + DATABASE_NAME); + } + + // Test case: select s not in (subquery) + sql = + "SELECT %s not in (SELECT (%s) from table3 WHERE device_id = 'd01') from table1 where device_id = 'd01'"; + expectedHeader = new String[] {"_col0"}; + retArray = new String[] {"false,", "false,", "true,","true,","true,"}; + for (String measurement : NUMERIC_MEASUREMENTS) { + tableResultSetEqualTest( + String.format(sql, measurement, measurement), + expectedHeader, + retArray, + DATABASE_NAME); + } + } + + @Test + public void testInPredicateSubqueryLegalityCheck() { + // Legality check: Multiple parentheses around subquery, this behaves the same as Trino + tableAssertTestFail( + "select s1 from table1 where device_id = 'd01' and s1 not in ((select s1 from table3 where device_id = 'd01'))", + "301: Scalar sub-query has returned multiple rows.", + DATABASE_NAME); + + // Legality check: Join key type mismatch.(left key is int and right key is double) + tableAssertTestFail( + "select s1 from table1 where device_id = 'd01' and s1 in (select s1 + 30.0 from table3 where device_id = 'd01')", + "301: Join key type mismatch.", + DATABASE_NAME); + + // Legality check: subquery can not be parsed(without parentheses) + tableAssertTestFail( + "select s1 from table1 where s1 in select s1 from table1", + "mismatched input", + DATABASE_NAME); + + // Legality check: subquery can not be parsed + tableAssertTestFail( + "select s1 from table1 where s1 in (select s1 from)", "mismatched input", DATABASE_NAME); + } +} diff --git a/integration-test/src/test/java/org/apache/iotdb/relational/it/query/recent/subquery/IoTDBUncorrelatedScalarSubqueryIT.java b/integration-test/src/test/java/org/apache/iotdb/relational/it/query/recent/subquery/uncorrelated/IoTDBUncorrelatedScalarSubqueryIT.java similarity index 99% rename from integration-test/src/test/java/org/apache/iotdb/relational/it/query/recent/subquery/IoTDBUncorrelatedScalarSubqueryIT.java rename to integration-test/src/test/java/org/apache/iotdb/relational/it/query/recent/subquery/uncorrelated/IoTDBUncorrelatedScalarSubqueryIT.java index 1d93e36197e8..b715b5cfeb97 100644 --- a/integration-test/src/test/java/org/apache/iotdb/relational/it/query/recent/subquery/IoTDBUncorrelatedScalarSubqueryIT.java +++ b/integration-test/src/test/java/org/apache/iotdb/relational/it/query/recent/subquery/uncorrelated/IoTDBUncorrelatedScalarSubqueryIT.java @@ -17,7 +17,7 @@ * under the License. */ -package org.apache.iotdb.relational.it.query.recent.subquery; +package org.apache.iotdb.relational.it.query.recent.subquery.uncorrelated; import org.apache.iotdb.it.env.EnvFactory; import org.apache.iotdb.it.framework.IoTDBTestRunner; diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/execution/operator/source/relational/AbstractMergeSortJoinOperator.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/execution/operator/source/relational/AbstractMergeSortJoinOperator.java index 8a10e8812111..0529f6738174 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/execution/operator/source/relational/AbstractMergeSortJoinOperator.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/execution/operator/source/relational/AbstractMergeSortJoinOperator.java @@ -268,7 +268,7 @@ protected void gotCandidateBlocks() throws Exception { if (!leftBlockNotEmpty()) { if (leftChild.hasNextWithTimer()) { TsBlock block = leftChild.nextWithTimer(); - pruneLastNullValuesOfBlock(block); + pruneLastNullValuesOfBlock(block, leftJoinKeyPosition); leftBlock = block; leftIndex = 0; } else { @@ -284,7 +284,7 @@ protected void gotCandidateBlocks() throws Exception { } else { if (rightChild.hasNextWithTimer()) { TsBlock block = rightChild.nextWithTimer(); - pruneLastNullValuesOfBlock(block); + pruneLastNullValuesOfBlock(block, rightJoinKeyPosition); if (block != null && !block.isEmpty()) { addRightBlockWithMemoryReservation(block); } @@ -299,13 +299,12 @@ protected void gotCandidateBlocks() throws Exception { } } - protected void pruneLastNullValuesOfBlock(TsBlock block) { + protected void pruneLastNullValuesOfBlock(TsBlock block, int columnIndex) { if (block == null) { return; } int lastNonNullIndex = block.getPositionCount() - 1; - while (lastNonNullIndex >= 0 - && block.getColumn(rightJoinKeyPosition).isNull(lastNonNullIndex)) { + while (lastNonNullIndex >= 0 && block.getColumn(columnIndex).isNull(lastNonNullIndex)) { lastNonNullIndex--; } block.setPositionCount(lastNonNullIndex + 1); diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/execution/operator/source/relational/MergeSortSemiJoinOperator.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/execution/operator/source/relational/MergeSortSemiJoinOperator.java index 34ef6d6cd3f0..824355c8c574 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/execution/operator/source/relational/MergeSortSemiJoinOperator.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/execution/operator/source/relational/MergeSortSemiJoinOperator.java @@ -24,6 +24,7 @@ import org.apache.iotdb.db.queryengine.execution.operator.OperatorContext; import org.apache.iotdb.db.queryengine.execution.operator.process.join.merge.comparator.JoinKeyComparator; +import org.apache.tsfile.block.column.ColumnBuilder; import org.apache.tsfile.enums.TSDataType; import org.apache.tsfile.read.common.type.Type; import org.apache.tsfile.utils.RamUsageEstimator; @@ -34,6 +35,8 @@ public class MergeSortSemiJoinOperator extends AbstractMergeSortJoinOperator { private static final long INSTANCE_SIZE = RamUsageEstimator.shallowSizeOfInstance(MergeSortSemiJoinOperator.class); + private final int outputColumnNum; + public MergeSortSemiJoinOperator( OperatorContext operatorContext, Operator leftChild, @@ -55,6 +58,7 @@ public MergeSortSemiJoinOperator( joinKeyComparator, dataTypes, joinKeyType); + outputColumnNum = dataTypes.size(); } @Override @@ -63,17 +67,24 @@ public boolean hasNext() throws Exception { return true; } - return !leftFinished && !rightFinished; + return !leftFinished; } @Override protected boolean prepareInput() throws Exception { gotCandidateBlocks(); + if (rightFinished) { + return leftBlockNotEmpty(); + } return leftBlockNotEmpty() && rightBlockNotEmpty() && gotNextRightBlock(); } @Override protected boolean processFinished() { + if (rightFinished) { + buildUseRemainingBlocks(); + return true; + } // all the join keys in rightTsBlock are less than leftTsBlock, just skip right if (allRightLessThanLeft()) { resetRightBlockList(); @@ -110,6 +121,8 @@ protected boolean processFinished() { rightBlockList.get(rightBlockListIdx), rightJoinKeyPosition, rightIndex)) { + // current left won't match any right, append left with false SemiJoin result + appendValueToResult(false); leftIndex++; if (leftIndex >= leftBlock.getPositionCount()) { resetLeftBlock(); @@ -134,25 +147,36 @@ protected boolean processFinished() { @Override protected boolean hasMatchedRightValueToProbeLeft() { - if (comparator.equalsTo( - leftBlock, - leftJoinKeyPosition, - leftIndex, - rightBlockList.get(rightBlockListIdx), - rightJoinKeyPosition, - rightIndex)) { - recordsWhenDataMatches(); - appendValueToResultWhenMatches(); - return true; - } - return false; + boolean matches = + comparator.equalsTo( + leftBlock, + leftJoinKeyPosition, + leftIndex, + rightBlockList.get(rightBlockListIdx), + rightJoinKeyPosition, + rightIndex); + appendValueToResult(matches); + return matches; } - protected void appendValueToResultWhenMatches() { + private void appendValueToResult(boolean matches) { appendLeftBlockData(leftOutputSymbolIdx, resultBuilder, leftBlock, leftIndex); + appendSemiJoinOutput(matches); resultBuilder.declarePosition(); } + private void appendSemiJoinOutput(boolean value) { + ColumnBuilder columnBuilder = resultBuilder.getColumnBuilder(outputColumnNum - 1); + columnBuilder.writeBoolean(value); + } + + private void buildUseRemainingBlocks() { + while (leftBlockNotEmpty()) { + appendValueToResult(false); + leftIndex++; + } + } + @Override protected void recordsWhenDataMatches() { // do nothing diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/analyzer/ExpressionAnalyzer.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/analyzer/ExpressionAnalyzer.java index 404c88b2e1e8..dd54a54bc739 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/analyzer/ExpressionAnalyzer.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/analyzer/ExpressionAnalyzer.java @@ -106,6 +106,7 @@ import static com.google.common.base.Preconditions.checkState; import static com.google.common.collect.ImmutableList.toImmutableList; +import static com.google.common.collect.Iterators.getOnlyElement; import static java.lang.String.format; import static java.util.Collections.unmodifiableMap; import static java.util.Collections.unmodifiableSet; @@ -996,21 +997,15 @@ private Type analyzePredicateWithSubquery( Type declaredValueType, SubqueryExpression subquery, StackableAstVisitorContext context) { + // For now, we only support one column in subqueries, we have checked this before. Type valueRowType = declaredValueType; - if (!(declaredValueType instanceof RowType) && !(declaredValueType instanceof UnknownType)) { + /*if (!(declaredValueType instanceof RowType) && !(declaredValueType instanceof UnknownType)) { valueRowType = RowType.anonymous(ImmutableList.of(declaredValueType)); - } + }*/ Type subqueryType = analyzeSubquery(subquery, context); setExpressionType(subquery, subqueryType); - if (subqueryType.equals(valueRowType)) { - throw new SemanticException( - String.format( - "Value expression and result of subquery must be of the same type: %s vs %s", - valueRowType, subqueryType)); - } - Optional valueCoercion = Optional.empty(); // if (!valueRowType.equals(commonType.get())) { // valueCoercion = commonType; @@ -1048,7 +1043,9 @@ private Type analyzeSubquery( } sourceFields.addAll(queryScope.getRelationType().getVisibleFields()); - return RowType.from(fields.build()); + // return RowType.from(fields.build()); + // For now, we only support one column in subqueries, we have checked this before. + return getOnlyElement(fields.build().stream().iterator()).getType(); } @Override diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/planner/SubqueryPlanner.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/planner/SubqueryPlanner.java index 631218b4bf80..222590f5d7d2 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/planner/SubqueryPlanner.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/planner/SubqueryPlanner.java @@ -572,7 +572,7 @@ private PlanAndMappings planSubquery( } List fieldsList = fields.build(); - checkArgument(fieldsList.size() <= 1, "For now, only single column subqueries are supported"); + checkArgument(fieldsList.size() == 1, "For now, only single column subqueries are supported"); /*subqueryPlan = subqueryPlan.withNewRoot( new ProjectNode( diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/planner/optimizations/LogicalOptimizeFactory.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/planner/optimizations/LogicalOptimizeFactory.java index cac5f28ab78c..ce18b64c2914 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/planner/optimizations/LogicalOptimizeFactory.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/planner/optimizations/LogicalOptimizeFactory.java @@ -58,7 +58,6 @@ import org.apache.iotdb.db.queryengine.plan.relational.planner.iterative.rule.RemoveTrivialFilters; import org.apache.iotdb.db.queryengine.plan.relational.planner.iterative.rule.RemoveUnreferencedScalarSubqueries; import org.apache.iotdb.db.queryengine.plan.relational.planner.iterative.rule.SimplifyExpressions; -import org.apache.iotdb.db.queryengine.plan.relational.planner.iterative.rule.TransformFilteringSemiJoinToInnerJoin; import org.apache.iotdb.db.queryengine.plan.relational.planner.iterative.rule.TransformUncorrelatedInPredicateSubqueryToSemiJoin; import org.apache.iotdb.db.queryengine.plan.relational.planner.iterative.rule.TransformUncorrelatedSubqueryToJoin; @@ -218,10 +217,11 @@ public LogicalOptimizeFactory(PlannerContext plannerContext) { new CheckSubqueryNodesAreRewritten(), simplifyOptimizer, new PushPredicateIntoTableScan(), - new IterativeOptimizer( - plannerContext, - ruleStats, - ImmutableSet.of(new TransformFilteringSemiJoinToInnerJoin())), + // Currently, Distinct is not supported, so we cant use this rule for now. + // new IterativeOptimizer( + // plannerContext, + // ruleStats, + // ImmutableSet.of(new TransformFilteringSemiJoinToInnerJoin())), // redo columnPrune and inlineProjections after pushPredicateIntoTableScan columnPruningOptimizer, inlineProjectionLimitFiltersOptimizer, diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/planner/optimizations/PushPredicateIntoTableScan.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/planner/optimizations/PushPredicateIntoTableScan.java index fff7f4df740d..7ba4d2514fd3 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/planner/optimizations/PushPredicateIntoTableScan.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/planner/optimizations/PushPredicateIntoTableScan.java @@ -72,6 +72,7 @@ import java.util.HashSet; import java.util.List; import java.util.Map; +import java.util.Objects; import java.util.Optional; import java.util.Set; import java.util.stream.Collectors; @@ -832,6 +833,8 @@ private PlanNode visitFilteringSemiJoin(SemiJoinNode node, RewriteContext contex Expression inheritedPredicate = context.inheritedPredicate != null ? context.inheritedPredicate : TRUE_LITERAL; Expression deterministicInheritedPredicate = filterDeterministicConjuncts(inheritedPredicate); + Expression sourceEffectivePredicate = TRUE_LITERAL; + Expression filteringSourceEffectivePredicate = TRUE_LITERAL; // Expression sourceEffectivePredicate = // filterDeterministicConjuncts(effectivePredicateExtractor.extract(session, node.getSource(), // types, typeAnalyzer)); @@ -853,15 +856,21 @@ private PlanNode visitFilteringSemiJoin(SemiJoinNode node, RewriteContext contex // Generate equality inferences EqualityInference allInference = - new EqualityInference(metadata, deterministicInheritedPredicate, joinExpression); - // EqualityInference allInference = new EqualityInference(metadata, - // deterministicInheritedPredicate, sourceEffectivePredicate, - // filteringSourceEffectivePredicate, joinExpression); - // EqualityInference allInferenceWithoutSourceInferred = new EqualityInference(metadata, - // deterministicInheritedPredicate, filteringSourceEffectivePredicate, joinExpression); - // EqualityInference allInferenceWithoutFilteringSourceInferred = new - // EqualityInference(metadata, deterministicInheritedPredicate, sourceEffectivePredicate, - // joinExpression); + new EqualityInference( + metadata, + deterministicInheritedPredicate, + sourceEffectivePredicate, + filteringSourceEffectivePredicate, + joinExpression); + EqualityInference allInferenceWithoutSourceInferred = + new EqualityInference( + metadata, + deterministicInheritedPredicate, + filteringSourceEffectivePredicate, + joinExpression); + EqualityInference allInferenceWithoutFilteringSourceInferred = + new EqualityInference( + metadata, deterministicInheritedPredicate, sourceEffectivePredicate, joinExpression); // Push inheritedPredicates down to the source if they don't involve the semi join output Set sourceScope = ImmutableSet.copyOf(sourceSymbols); @@ -892,23 +901,28 @@ private PlanNode visitFilteringSemiJoin(SemiJoinNode node, RewriteContext contex } }); - /* // move effective predicate conjuncts source <-> filter // See if we can push the filtering source effective predicate to the source side EqualityInference.nonInferrableConjuncts(metadata, filteringSourceEffectivePredicate) - .map(conjunct -> allInference.rewrite(conjunct, sourceScope)) - .filter(Objects::nonNull) - .forEach(sourceConjuncts::add); + .map(conjunct -> allInference.rewrite(conjunct, sourceScope)) + .filter(Objects::nonNull) + .forEach(sourceConjuncts::add); // See if we can push the source effective predicate to the filtering source side EqualityInference.nonInferrableConjuncts(metadata, sourceEffectivePredicate) - .map(conjunct -> allInference.rewrite(conjunct, filterScope)) - .filter(Objects::nonNull) - .forEach(filteringSourceConjuncts::add); + .map(conjunct -> allInference.rewrite(conjunct, filterScope)) + .filter(Objects::nonNull) + .forEach(filteringSourceConjuncts::add); // Add equalities from the inference back in - sourceConjuncts.addAll(allInferenceWithoutSourceInferred.generateEqualitiesPartitionedBy(sourceScope).getScopeEqualities()); - filteringSourceConjuncts.addAll(allInferenceWithoutFilteringSourceInferred.generateEqualitiesPartitionedBy(filterScope).getScopeEqualities());*/ + sourceConjuncts.addAll( + allInferenceWithoutSourceInferred + .generateEqualitiesPartitionedBy(sourceScope) + .getScopeEqualities()); + filteringSourceConjuncts.addAll( + allInferenceWithoutFilteringSourceInferred + .generateEqualitiesPartitionedBy(filterScope) + .getScopeEqualities()); PlanNode rewrittenSource = node.getSource().accept(this, new RewriteContext(combineConjuncts(sourceConjuncts))); diff --git a/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/queryengine/plan/relational/planner/SubqueryTest.java b/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/queryengine/plan/relational/planner/SubqueryTest.java index 652472b05918..a45dfb686f9e 100644 --- a/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/queryengine/plan/relational/planner/SubqueryTest.java +++ b/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/queryengine/plan/relational/planner/SubqueryTest.java @@ -24,6 +24,7 @@ import org.apache.iotdb.db.queryengine.plan.relational.planner.node.JoinNode; import org.apache.iotdb.db.queryengine.plan.relational.sql.ast.ComparisonExpression; import org.apache.iotdb.db.queryengine.plan.relational.sql.ast.Expression; +import org.apache.iotdb.db.queryengine.plan.relational.sql.ast.NotExpression; import org.apache.iotdb.db.queryengine.plan.relational.sql.ast.SymbolReference; import com.google.common.collect.ImmutableList; @@ -45,9 +46,12 @@ import static org.apache.iotdb.db.queryengine.plan.relational.planner.assertions.PlanMatchPattern.exchange; import static org.apache.iotdb.db.queryengine.plan.relational.planner.assertions.PlanMatchPattern.filter; import static org.apache.iotdb.db.queryengine.plan.relational.planner.assertions.PlanMatchPattern.join; +import static org.apache.iotdb.db.queryengine.plan.relational.planner.assertions.PlanMatchPattern.mergeSort; import static org.apache.iotdb.db.queryengine.plan.relational.planner.assertions.PlanMatchPattern.output; import static org.apache.iotdb.db.queryengine.plan.relational.planner.assertions.PlanMatchPattern.project; +import static org.apache.iotdb.db.queryengine.plan.relational.planner.assertions.PlanMatchPattern.semiJoin; import static org.apache.iotdb.db.queryengine.plan.relational.planner.assertions.PlanMatchPattern.singleGroupingSet; +import static org.apache.iotdb.db.queryengine.plan.relational.planner.assertions.PlanMatchPattern.sort; import static org.apache.iotdb.db.queryengine.plan.relational.planner.assertions.PlanMatchPattern.tableScan; import static org.apache.iotdb.db.queryengine.plan.relational.planner.node.AggregationNode.Step.FINAL; import static org.apache.iotdb.db.queryengine.plan.relational.planner.node.AggregationNode.Step.INTERMEDIATE; @@ -223,4 +227,98 @@ public void testUncorrelatedScalarSubqueryInWhereClauseWithEnforceSingleRowNode( JoinNode.JoinType.INNER, builder -> builder.left(tableScan1).right(enforceSingleRow(any()))))))); } + + @Test + public void testUncorrelatedInPredicateSubquery() { + PlanTester planTester = new PlanTester(); + + String sql = "SELECT s1 FROM table1 where s1 in (select s1 from table1)"; + + LogicalQueryPlan logicalQueryPlan = planTester.createPlan(sql); + + Expression filterPredicate = new SymbolReference("expr"); + + PlanMatchPattern tableScan1 = + tableScan("testdb.table1", ImmutableList.of("s1"), ImmutableSet.of("s1")); + + PlanMatchPattern tableScan2 = tableScan("testdb.table1", ImmutableMap.of("s1_6", "s1")); + + // Verify full LogicalPlan + /* + * └──OutputNode + * └──ProjectNode + * └──FilterNode + * └──SemiJoinNode + * |──SortNode + * | └──TableScanNode + * ├──SortNode + * │ └──TableScanNode + + */ + assertPlan( + logicalQueryPlan, + output( + project( + filter( + filterPredicate, + semiJoin("s1", "s1_6", "expr", sort(tableScan1), sort(tableScan2)))))); + + // Verify DistributionPlan + assertPlan( + planTester.getFragmentPlan(0), + output( + project( + filter( + filterPredicate, + semiJoin( + "s1", + "s1_6", + "expr", + mergeSort(exchange(), sort(tableScan1), exchange()), + mergeSort(exchange(), sort(tableScan2), exchange())))))); + + assertPlan(planTester.getFragmentPlan(1), sort(tableScan1)); + + assertPlan(planTester.getFragmentPlan(2), sort(tableScan1)); + + assertPlan(planTester.getFragmentPlan(3), sort(tableScan2)); + + assertPlan(planTester.getFragmentPlan(4), sort(tableScan2)); + } + + @Test + public void testUncorrelatedNotInPredicateSubquery() { + PlanTester planTester = new PlanTester(); + + String sql = "SELECT s1 FROM table1 where s1 not in (select s1 from table1)"; + + LogicalQueryPlan logicalQueryPlan = planTester.createPlan(sql); + + Expression filterPredicate = new NotExpression(new SymbolReference("expr")); + + PlanMatchPattern tableScan1 = + tableScan("testdb.table1", ImmutableList.of("s1"), ImmutableSet.of("s1")); + + PlanMatchPattern tableScan2 = tableScan("testdb.table1", ImmutableMap.of("s1_6", "s1")); + + // Verify full LogicalPlan + /* + * └──OutputNode + * └──ProjectNode + * └──FilterNode + * └──SemiJoinNode + * |──SortNode + * | └──TableScanNode + * ├──SortNode + * │ └──TableScanNode + + */ + assertPlan( + logicalQueryPlan, + output( + project( + filter( + filterPredicate, + semiJoin("s1", "s1_6", "expr", sort(tableScan1), sort(tableScan2)))))); + } } diff --git a/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/queryengine/plan/relational/planner/assertions/PlanMatchPattern.java b/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/queryengine/plan/relational/planner/assertions/PlanMatchPattern.java index 09740bdbed39..30580bf0f4bf 100644 --- a/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/queryengine/plan/relational/planner/assertions/PlanMatchPattern.java +++ b/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/queryengine/plan/relational/planner/assertions/PlanMatchPattern.java @@ -33,6 +33,7 @@ import org.apache.iotdb.db.queryengine.plan.relational.planner.node.OffsetNode; import org.apache.iotdb.db.queryengine.plan.relational.planner.node.OutputNode; import org.apache.iotdb.db.queryengine.plan.relational.planner.node.ProjectNode; +import org.apache.iotdb.db.queryengine.plan.relational.planner.node.SemiJoinNode; import org.apache.iotdb.db.queryengine.plan.relational.planner.node.SortNode; import org.apache.iotdb.db.queryengine.plan.relational.planner.node.StreamSortNode; import org.apache.iotdb.db.queryengine.plan.relational.sql.ast.DataType; @@ -375,6 +376,16 @@ public static PlanMatchPattern join( return builder.build(); } + public static PlanMatchPattern semiJoin( + String sourceSymbolAlias, + String filteringSymbolAlias, + String outputAlias, + PlanMatchPattern source, + PlanMatchPattern filtering) { + return node(SemiJoinNode.class, source, filtering) + .with(new SemiJoinMatcher(sourceSymbolAlias, filteringSymbolAlias, outputAlias)); + } + public static PlanMatchPattern sort(PlanMatchPattern source) { return node(SortNode.class, source); } diff --git a/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/queryengine/plan/relational/planner/assertions/SemiJoinMatcher.java b/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/queryengine/plan/relational/planner/assertions/SemiJoinMatcher.java new file mode 100644 index 000000000000..7ee9192254e7 --- /dev/null +++ b/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/queryengine/plan/relational/planner/assertions/SemiJoinMatcher.java @@ -0,0 +1,79 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.iotdb.db.queryengine.plan.relational.planner.assertions; + +import org.apache.iotdb.db.queryengine.common.SessionInfo; +import org.apache.iotdb.db.queryengine.plan.planner.plan.node.PlanNode; +import org.apache.iotdb.db.queryengine.plan.relational.metadata.Metadata; +import org.apache.iotdb.db.queryengine.plan.relational.planner.node.SemiJoinNode; + +import static com.google.common.base.MoreObjects.toStringHelper; +import static com.google.common.base.Preconditions.checkState; +import static java.util.Objects.requireNonNull; +import static org.apache.iotdb.db.queryengine.plan.relational.planner.assertions.MatchResult.NO_MATCH; +import static org.apache.iotdb.db.queryengine.plan.relational.planner.assertions.MatchResult.match; + +final class SemiJoinMatcher implements Matcher { + private final String sourceSymbolAlias; + private final String filteringSymbolAlias; + private final String outputAlias; + + SemiJoinMatcher(String sourceSymbolAlias, String filteringSymbolAlias, String outputAlias) { + this.sourceSymbolAlias = requireNonNull(sourceSymbolAlias, "sourceSymbolAlias is null"); + this.filteringSymbolAlias = + requireNonNull(filteringSymbolAlias, "filteringSymbolAlias is null"); + this.outputAlias = requireNonNull(outputAlias, "outputAlias is null"); + } + + @Override + public boolean shapeMatches(PlanNode node) { + return node instanceof SemiJoinNode; + } + + @Override + public MatchResult detailMatches( + PlanNode node, SessionInfo sessionInfo, Metadata metadata, SymbolAliases symbolAliases) { + checkState( + shapeMatches(node), + "Plan testing framework error: shapeMatches returned false in detailMatches in %s", + this.getClass().getName()); + + SemiJoinNode semiJoinNode = (SemiJoinNode) node; + if (!(symbolAliases + .get(sourceSymbolAlias) + .equals(semiJoinNode.getSourceJoinSymbol().toSymbolReference()) + && symbolAliases + .get(filteringSymbolAlias) + .equals(semiJoinNode.getFilteringSourceJoinSymbol().toSymbolReference()))) { + return NO_MATCH; + } + + return match(outputAlias, semiJoinNode.getSemiJoinOutput().toSymbolReference()); + } + + @Override + public String toString() { + return toStringHelper(this) + .add("filteringSymbolAlias", filteringSymbolAlias) + .add("sourceSymbolAlias", sourceSymbolAlias) + .add("outputAlias", outputAlias) + .toString(); + } +} From 66219cdd92c897b8af1a433ee408d14fbb3b097b Mon Sep 17 00:00:00 2001 From: lancelly <1435078631@qq.com> Date: Tue, 10 Dec 2024 16:19:31 +0800 Subject: [PATCH 07/26] fix semi join operator --- .../operator/source/relational/MergeSortSemiJoinOperator.java | 1 + 1 file changed, 1 insertion(+) diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/execution/operator/source/relational/MergeSortSemiJoinOperator.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/execution/operator/source/relational/MergeSortSemiJoinOperator.java index 824355c8c574..d5c8e3dfd264 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/execution/operator/source/relational/MergeSortSemiJoinOperator.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/execution/operator/source/relational/MergeSortSemiJoinOperator.java @@ -93,6 +93,7 @@ protected boolean processFinished() { // all the join Keys in leftTsBlock are less than rightTsBlock, just skip left if (allLeftLessThanRight()) { + appendValueToResult(false); resetLeftBlock(); return true; } From 0a5e0986c5031354bfeeee74f7264ee4a887bd97 Mon Sep 17 00:00:00 2001 From: lancelly <1435078631@qq.com> Date: Tue, 10 Dec 2024 16:22:48 +0800 Subject: [PATCH 08/26] fix semi join operator --- .../source/relational/MergeSortSemiJoinOperator.java | 9 +++++---- 1 file changed, 5 insertions(+), 4 deletions(-) diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/execution/operator/source/relational/MergeSortSemiJoinOperator.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/execution/operator/source/relational/MergeSortSemiJoinOperator.java index d5c8e3dfd264..95b0e6d0848d 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/execution/operator/source/relational/MergeSortSemiJoinOperator.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/execution/operator/source/relational/MergeSortSemiJoinOperator.java @@ -82,7 +82,7 @@ protected boolean prepareInput() throws Exception { @Override protected boolean processFinished() { if (rightFinished) { - buildUseRemainingBlocks(); + appendAllLeftBlock(); return true; } // all the join keys in rightTsBlock are less than leftTsBlock, just skip right @@ -91,9 +91,10 @@ protected boolean processFinished() { return true; } - // all the join Keys in leftTsBlock are less than rightTsBlock, just skip left + // all the join Keys in leftTsBlock are less than rightTsBlock, just append the left value with + // match flag as false if (allLeftLessThanRight()) { - appendValueToResult(false); + appendAllLeftBlock(); resetLeftBlock(); return true; } @@ -171,7 +172,7 @@ private void appendSemiJoinOutput(boolean value) { columnBuilder.writeBoolean(value); } - private void buildUseRemainingBlocks() { + private void appendAllLeftBlock() { while (leftBlockNotEmpty()) { appendValueToResult(false); leftIndex++; From 3ba0391d21c2f4f748a5fce23ac3b773ab81d963 Mon Sep 17 00:00:00 2001 From: lancelly <1435078631@qq.com> Date: Tue, 10 Dec 2024 20:10:48 +0800 Subject: [PATCH 09/26] add check for column num of subquery --- ...oTDBUncorrelatedInPredicateSubqueryIT.java | 38 +++++++++++-------- .../analyzer/ExpressionAnalyzer.java | 14 +++++++ .../relational/planner/SubqueryPlanner.java | 4 +- 3 files changed, 39 insertions(+), 17 deletions(-) diff --git a/integration-test/src/test/java/org/apache/iotdb/relational/it/query/recent/subquery/uncorrelated/IoTDBUncorrelatedInPredicateSubqueryIT.java b/integration-test/src/test/java/org/apache/iotdb/relational/it/query/recent/subquery/uncorrelated/IoTDBUncorrelatedInPredicateSubqueryIT.java index 988cc0c60418..a743a0125002 100644 --- a/integration-test/src/test/java/org/apache/iotdb/relational/it/query/recent/subquery/uncorrelated/IoTDBUncorrelatedInPredicateSubqueryIT.java +++ b/integration-test/src/test/java/org/apache/iotdb/relational/it/query/recent/subquery/uncorrelated/IoTDBUncorrelatedInPredicateSubqueryIT.java @@ -228,28 +228,22 @@ public void testInPredicateSubqueryInSelectClause() { // Test case: select s in (subquery) sql = - "SELECT %s in (SELECT (%s) from table3 WHERE device_id = 'd01') from table1 where device_id = 'd01'"; + "SELECT %s in (SELECT (%s) from table3 WHERE device_id = 'd01') from table1 where device_id = 'd01'"; expectedHeader = new String[] {"_col0"}; - retArray = new String[] {"true,", "true,", "false,","false,","false,"}; + retArray = new String[] {"true,", "true,", "false,", "false,", "false,"}; for (String measurement : NUMERIC_MEASUREMENTS) { tableResultSetEqualTest( - String.format(sql, measurement, measurement), - expectedHeader, - retArray, - DATABASE_NAME); + String.format(sql, measurement, measurement), expectedHeader, retArray, DATABASE_NAME); } // Test case: select s not in (subquery) sql = - "SELECT %s not in (SELECT (%s) from table3 WHERE device_id = 'd01') from table1 where device_id = 'd01'"; + "SELECT %s not in (SELECT (%s) from table3 WHERE device_id = 'd01') from table1 where device_id = 'd01'"; expectedHeader = new String[] {"_col0"}; - retArray = new String[] {"false,", "false,", "true,","true,","true,"}; + retArray = new String[] {"false,", "false,", "true,", "true,", "true,"}; for (String measurement : NUMERIC_MEASUREMENTS) { tableResultSetEqualTest( - String.format(sql, measurement, measurement), - expectedHeader, - retArray, - DATABASE_NAME); + String.format(sql, measurement, measurement), expectedHeader, retArray, DATABASE_NAME); } } @@ -267,14 +261,26 @@ public void testInPredicateSubqueryLegalityCheck() { "301: Join key type mismatch.", DATABASE_NAME); + // Legality check: Row Type is not supported for now. + tableAssertTestFail( + "select s1, s2 in (select (s1, s2) from table1) from table1", + "701: Subquery must return only one column for now. Row Type is not supported for now.", + DATABASE_NAME); + + // Legality check: Row Type is not supported for now. + tableAssertTestFail( + "select (s1, s2) in (select (s1, s2) from table1) from table1", + "701: Subquery must return only one column for now. Row Type is not supported for now.", + DATABASE_NAME); + // Legality check: subquery can not be parsed(without parentheses) tableAssertTestFail( - "select s1 from table1 where s1 in select s1 from table1", - "mismatched input", - DATABASE_NAME); + "select s1 from table1 where s1 in select s1 from table1", + "mismatched input", + DATABASE_NAME); // Legality check: subquery can not be parsed tableAssertTestFail( - "select s1 from table1 where s1 in (select s1 from)", "mismatched input", DATABASE_NAME); + "select s1 from table1 where s1 in (select s1 from)", "mismatched input", DATABASE_NAME); } } diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/analyzer/ExpressionAnalyzer.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/analyzer/ExpressionAnalyzer.java index dd54a54bc739..a196a7ac9b15 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/analyzer/ExpressionAnalyzer.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/analyzer/ExpressionAnalyzer.java @@ -165,6 +165,9 @@ public class ExpressionAnalyzer { private final Map, LabelPrefixedReference> labelDereferences = new LinkedHashMap<>(); + private static final String SUBQUERY_COLUMN_NUM_CHECK = + "Subquery must return only one column for now. Row Type is not supported for now."; + private ExpressionAnalyzer( Metadata metadata, MPPQueryContext context, @@ -929,6 +932,10 @@ public Type visitCast(Cast node, StackableAstVisitorContext context) { @Override protected Type visitInPredicate(InPredicate node, StackableAstVisitorContext context) { Expression value = node.getValue(); + // todo: remove this check after supporting RowType + if (value instanceof Row) { + throw new SemanticException(SUBQUERY_COLUMN_NUM_CHECK); + } Expression valueList = node.getValueList(); // When an IN-predicate containing a subquery: `x IN (SELECT ...)` is planned, both `value` @@ -1042,6 +1049,13 @@ private Type analyzeSubquery( } } + List fieldList = fields.build(); + + // todo: remove this check after supporting RowType + if (fieldList.size() != 1 || fieldList.get(0).getType() instanceof RowType) { + throw new SemanticException(SUBQUERY_COLUMN_NUM_CHECK); + } + sourceFields.addAll(queryScope.getRelationType().getVisibleFields()); // return RowType.from(fields.build()); // For now, we only support one column in subqueries, we have checked this before. diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/planner/SubqueryPlanner.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/planner/SubqueryPlanner.java index 222590f5d7d2..edae8267c93a 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/planner/SubqueryPlanner.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/planner/SubqueryPlanner.java @@ -251,6 +251,7 @@ private PlanBuilder planScalarSubquery(PlanBuilder subPlan, Cluster fieldMappings = relationPlan.getFieldMappings(); Symbol column; + // todo: remove this check after supporting RowType checkArgument( descriptor.getVisibleFieldCount() <= 1, "For now, only single column subqueries are supported"); @@ -572,7 +573,8 @@ private PlanAndMappings planSubquery( } List fieldsList = fields.build(); - checkArgument(fieldsList.size() == 1, "For now, only single column subqueries are supported"); + // todo: remove this check after supporting RowType + checkArgument(fieldsList.size() == 1, "For now, only single column subqueries are supported."); /*subqueryPlan = subqueryPlan.withNewRoot( new ProjectNode( From 6fbbf92e5b3627e36fc001a9db7e8c2ba65a003f Mon Sep 17 00:00:00 2001 From: lancelly <1435078631@qq.com> Date: Tue, 10 Dec 2024 22:09:09 +0800 Subject: [PATCH 10/26] remove outdated IT --- .../relational/it/query/recent/IoTDBTableAggregationIT.java | 5 ----- 1 file changed, 5 deletions(-) diff --git a/integration-test/src/test/java/org/apache/iotdb/relational/it/query/recent/IoTDBTableAggregationIT.java b/integration-test/src/test/java/org/apache/iotdb/relational/it/query/recent/IoTDBTableAggregationIT.java index ae32bb241f64..204bdefa292a 100644 --- a/integration-test/src/test/java/org/apache/iotdb/relational/it/query/recent/IoTDBTableAggregationIT.java +++ b/integration-test/src/test/java/org/apache/iotdb/relational/it/query/recent/IoTDBTableAggregationIT.java @@ -3711,11 +3711,6 @@ public void modeTest() { @Test public void exceptionTest() { - tableAssertTestFail( - "select s1 from table1 where s2 in (select s2 from table1)", - "Not a valid IR expression", - DATABASE_NAME); - tableAssertTestFail( "select avg() from table1", "701: Aggregate functions [avg] should only have one argument", From 95b85207b72cc90b824bbecaca04f0d4b9daf931 Mon Sep 17 00:00:00 2001 From: lancelly <1435078631@qq.com> Date: Mon, 16 Dec 2024 12:09:06 +0800 Subject: [PATCH 11/26] remove row type related code --- .../planner/ir/ExpressionTreeRewriter.java | 44 ------ .../plan/relational/sql/ast/AstVisitor.java | 8 - .../plan/relational/sql/ast/RowDataType.java | 137 ------------------ .../sql/util/ExpressionFormatter.java | 20 --- .../relational/type/InternalTypeManager.java | 1 - .../plan/relational/type/NamedType.java | 62 -------- .../plan/relational/type/ParametricType.java | 30 ---- .../relational/type/RowParametricType.java | 59 -------- .../plan/relational/type/TypeParameter.java | 128 ---------------- .../type/TypeSignatureTranslator.java | 25 ---- 10 files changed, 514 deletions(-) delete mode 100644 iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/sql/ast/RowDataType.java delete mode 100644 iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/type/NamedType.java delete mode 100644 iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/type/ParametricType.java delete mode 100644 iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/type/RowParametricType.java delete mode 100644 iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/type/TypeParameter.java diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/planner/ir/ExpressionTreeRewriter.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/planner/ir/ExpressionTreeRewriter.java index e45215cc8817..a917d3d040f8 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/planner/ir/ExpressionTreeRewriter.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/planner/ir/ExpressionTreeRewriter.java @@ -51,7 +51,6 @@ import org.apache.iotdb.db.queryengine.plan.relational.sql.ast.Parameter; import org.apache.iotdb.db.queryengine.plan.relational.sql.ast.QuantifiedComparisonExpression; import org.apache.iotdb.db.queryengine.plan.relational.sql.ast.Row; -import org.apache.iotdb.db.queryengine.plan.relational.sql.ast.RowDataType; import org.apache.iotdb.db.queryengine.plan.relational.sql.ast.SearchedCaseExpression; import org.apache.iotdb.db.queryengine.plan.relational.sql.ast.SimpleCaseExpression; import org.apache.iotdb.db.queryengine.plan.relational.sql.ast.SubqueryExpression; @@ -644,49 +643,6 @@ public Expression visitCast(Cast node, Context context) { return node; } - @Override - protected Expression visitRowDataType(RowDataType node, Context context) { - if (!context.isDefaultRewrite()) { - Expression result = - rewriter.rewriteRowDataType(node, context.get(), ExpressionTreeRewriter.this); - if (result != null) { - return result; - } - } - - ImmutableList.Builder rewritten = ImmutableList.builder(); - for (RowDataType.Field field : node.getFields()) { - DataType dataType = rewrite(field.getType(), context.get()); - - Optional name = field.getName(); - - if (field.getName().isPresent()) { - Identifier identifier = field.getName().get(); - Identifier rewrittenIdentifier = rewrite(identifier, context.get()); - - if (identifier != rewrittenIdentifier) { - name = Optional.of(rewrittenIdentifier); - } - } - - @SuppressWarnings("OptionalEquality") - boolean nameRewritten = name != field.getName(); - if (dataType != field.getType() || nameRewritten) { - rewritten.add(new RowDataType.Field(field.getLocation(), name, dataType)); - } else { - rewritten.add(field); - } - } - - List fields = rewritten.build(); - - if (!sameElements(fields, node.getFields())) { - return new RowDataType(node.getLocation(), fields); - } - - return node; - } - @Override public Expression visitCurrentDatabase(final CurrentDatabase node, final Context context) { if (!context.isDefaultRewrite()) { diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/sql/ast/AstVisitor.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/sql/ast/AstVisitor.java index 85db1a02f79f..43bc9223b114 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/sql/ast/AstVisitor.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/sql/ast/AstVisitor.java @@ -481,14 +481,6 @@ protected R visitGenericDataType(GenericDataType node, C context) { return visitDataType(node, context); } - protected R visitRowDataType(RowDataType node, C context) { - return visitDataType(node, context); - } - - protected R visitRowField(RowDataType.Field node, C context) { - return visitNode(node, context); - } - protected R visitDataTypeParameter(DataTypeParameter node, C context) { return visitNode(node, context); } diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/sql/ast/RowDataType.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/sql/ast/RowDataType.java deleted file mode 100644 index cc4f779a5f42..000000000000 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/sql/ast/RowDataType.java +++ /dev/null @@ -1,137 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, - * software distributed under the License is distributed on an - * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - * KIND, either express or implied. See the License for the - * specific language governing permissions and limitations - * under the License. - */ - -package org.apache.iotdb.db.queryengine.plan.relational.sql.ast; - -import com.google.common.collect.ImmutableList; - -import java.util.List; -import java.util.Objects; -import java.util.Optional; - -import static java.util.Objects.requireNonNull; - -public class RowDataType extends DataType { - private final List fields; - - public RowDataType(Optional location, List fields) { - this(location.orElse(null), fields); - } - - public RowDataType(NodeLocation location, List fields) { - super(location); - this.fields = ImmutableList.copyOf(fields); - } - - public List getFields() { - return fields; - } - - @Override - public List getChildren() { - return fields; - } - - @Override - public R accept(AstVisitor visitor, C context) { - return visitor.visitRowDataType(this, context); - } - - @Override - public boolean equals(Object o) { - if (this == o) { - return true; - } - if (o == null || getClass() != o.getClass()) { - return false; - } - RowDataType that = (RowDataType) o; - return fields.equals(that.fields); - } - - @Override - public int hashCode() { - return Objects.hash(fields); - } - - public static class Field extends Expression { - private final Optional name; - private final DataType type; - - public Field(Optional location, Optional name, DataType type) { - this(location.orElse(null), name, type); - } - - public Field(NodeLocation location, Optional name, DataType type) { - super(location); - - this.name = requireNonNull(name, "name is null"); - this.type = requireNonNull(type, "type is null"); - } - - public Optional getName() { - return name; - } - - public DataType getType() { - return type; - } - - @Override - public List getChildren() { - ImmutableList.Builder children = ImmutableList.builder(); - name.ifPresent(children::add); - children.add(type); - - return children.build(); - } - - @Override - public R accept(AstVisitor visitor, C context) { - return visitor.visitRowField(this, context); - } - - @Override - public boolean equals(Object o) { - if (this == o) { - return true; - } - if (o == null || getClass() != o.getClass()) { - return false; - } - Field field = (Field) o; - return name.equals(field.name) && type.equals(field.type); - } - - @Override - public int hashCode() { - return Objects.hash(name, type); - } - - @Override - public boolean shallowEquals(Node other) { - return sameClass(this, other); - } - } - - @Override - public boolean shallowEquals(Node other) { - return sameClass(this, other); - } -} diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/sql/util/ExpressionFormatter.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/sql/util/ExpressionFormatter.java index d8d83350d4aa..d6222564ec7e 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/sql/util/ExpressionFormatter.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/sql/util/ExpressionFormatter.java @@ -64,7 +64,6 @@ import org.apache.iotdb.db.queryengine.plan.relational.sql.ast.QualifiedName; import org.apache.iotdb.db.queryengine.plan.relational.sql.ast.QuantifiedComparisonExpression; import org.apache.iotdb.db.queryengine.plan.relational.sql.ast.Row; -import org.apache.iotdb.db.queryengine.plan.relational.sql.ast.RowDataType; import org.apache.iotdb.db.queryengine.plan.relational.sql.ast.SearchedCaseExpression; import org.apache.iotdb.db.queryengine.plan.relational.sql.ast.SimpleCaseExpression; import org.apache.iotdb.db.queryengine.plan.relational.sql.ast.SimpleGroupBy; @@ -533,25 +532,6 @@ protected String visitGenericDataType(GenericDataType node, Void context) { return result.toString(); } - @Override - protected String visitRowDataType(RowDataType node, Void context) { - return node.getFields().stream().map(this::process).collect(joining(", ", "ROW(", ")")); - } - - @Override - protected String visitRowField(RowDataType.Field node, Void context) { - StringBuilder result = new StringBuilder(); - - if (node.getName().isPresent()) { - result.append(process(node.getName().get(), context)); - result.append(" "); - } - - result.append(process(node.getType(), context)); - - return result.toString(); - } - @Override protected String visitTypeParameter(TypeParameter node, Void context) { return process(node.getValue(), context); diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/type/InternalTypeManager.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/type/InternalTypeManager.java index 7d64cabb22ea..6f2aaef76ab1 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/type/InternalTypeManager.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/type/InternalTypeManager.java @@ -27,7 +27,6 @@ import java.util.concurrent.ConcurrentHashMap; import java.util.concurrent.ConcurrentMap; -import static org.apache.iotdb.db.queryengine.plan.relational.type.RowParametricType.ROW; import static org.apache.tsfile.read.common.type.BinaryType.TEXT; import static org.apache.tsfile.read.common.type.BlobType.BLOB; import static org.apache.tsfile.read.common.type.BooleanType.BOOLEAN; diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/type/NamedType.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/type/NamedType.java deleted file mode 100644 index 93e65cfab7b5..000000000000 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/type/NamedType.java +++ /dev/null @@ -1,62 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, - * software distributed under the License is distributed on an - * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - * KIND, either express or implied. See the License for the - * specific language governing permissions and limitations - * under the License. - */ - -package org.apache.iotdb.db.queryengine.plan.relational.type; - -import org.apache.tsfile.read.common.type.Type; - -import java.util.Objects; -import java.util.Optional; - -public class NamedType { - private final Optional name; - private final Type type; - - public NamedType(Optional name, Type type) { - this.name = name; - this.type = type; - } - - public Optional getName() { - return name; - } - - public Type getType() { - return type; - } - - @Override - public boolean equals(Object o) { - if (this == o) { - return true; - } - if (o == null || getClass() != o.getClass()) { - return false; - } - - NamedType other = (NamedType) o; - - return Objects.equals(this.name, other.name) && Objects.equals(this.type, other.type); - } - - @Override - public int hashCode() { - return Objects.hash(name, type); - } -} diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/type/ParametricType.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/type/ParametricType.java deleted file mode 100644 index 07c5d8a2899d..000000000000 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/type/ParametricType.java +++ /dev/null @@ -1,30 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, - * software distributed under the License is distributed on an - * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - * KIND, either express or implied. See the License for the - * specific language governing permissions and limitations - * under the License. - */ - -package org.apache.iotdb.db.queryengine.plan.relational.type; - -import org.apache.tsfile.read.common.type.Type; - -import java.util.List; - -public interface ParametricType { - String getName(); - - Type createType(TypeManager typeManager, List parameters); -} diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/type/RowParametricType.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/type/RowParametricType.java deleted file mode 100644 index 005f6b2c4193..000000000000 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/type/RowParametricType.java +++ /dev/null @@ -1,59 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, - * software distributed under the License is distributed on an - * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - * KIND, either express or implied. See the License for the - * specific language governing permissions and limitations - * under the License. - */ - -package org.apache.iotdb.db.queryengine.plan.relational.type; - -import org.apache.tsfile.read.common.type.RowType; -import org.apache.tsfile.read.common.type.Type; - -import java.util.List; - -import static com.google.common.base.Preconditions.checkArgument; -import static java.util.stream.Collectors.toList; - -public final class RowParametricType implements ParametricType { - public static final RowParametricType ROW = new RowParametricType(); - - private RowParametricType() {} - - @Override - public String getName() { - return StandardTypes.ROW; - } - - @Override - public Type createType(TypeManager typeManager, List parameters) { - checkArgument(!parameters.isEmpty(), "Row type must have at least one parameter"); - checkArgument( - parameters.stream().allMatch(parameter -> parameter.getKind() == ParameterKind.NAMED_TYPE), - "Expected only named types as a parameters, got %s", - parameters); - - List fields = - parameters.stream() - .map(TypeParameter::getNamedType) - .map( - parameter -> - new RowType.Field( - parameter.getName().map(RowFieldName::getName), parameter.getType())) - .collect(toList()); - - return RowType.createWithTypeSignature(fields); - } -} diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/type/TypeParameter.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/type/TypeParameter.java deleted file mode 100644 index 9c4aa1333534..000000000000 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/type/TypeParameter.java +++ /dev/null @@ -1,128 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, - * software distributed under the License is distributed on an - * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - * KIND, either express or implied. See the License for the - * specific language governing permissions and limitations - * under the License. - */ - -package org.apache.iotdb.db.queryengine.plan.relational.type; - -import org.apache.tsfile.read.common.type.Type; - -import java.util.Objects; - -import static java.lang.String.format; - -public class TypeParameter { - private final ParameterKind kind; - private final Object value; - - private TypeParameter(ParameterKind kind, Object value) { - this.kind = kind; - this.value = value; - } - - public static TypeParameter of(Type type) { - return new TypeParameter(ParameterKind.TYPE, type); - } - - public static TypeParameter of(long longLiteral) { - return new TypeParameter(ParameterKind.LONG, longLiteral); - } - - public static TypeParameter of(NamedType namedType) { - return new TypeParameter(ParameterKind.NAMED_TYPE, namedType); - } - - public static TypeParameter of(String variable) { - return new TypeParameter(ParameterKind.VARIABLE, variable); - } - - public static TypeParameter of(TypeSignatureParameter parameter, TypeManager typeManager) { - switch (parameter.getKind()) { - case TYPE: - { - Type type = typeManager.getType(parameter.getTypeSignature()); - return of(type); - } - case LONG: - return of(parameter.getLongLiteral()); - case NAMED_TYPE: - { - Type type = typeManager.getType(parameter.getNamedTypeSignature().getTypeSignature()); - return of(new NamedType(parameter.getNamedTypeSignature().getFieldName(), type)); - } - case VARIABLE: - return of(parameter.getVariable()); - } - throw new UnsupportedOperationException(format("Unsupported parameter [%s]", parameter)); - } - - public ParameterKind getKind() { - return kind; - } - - public A getValue(ParameterKind expectedParameterKind, Class target) { - if (kind != expectedParameterKind) { - throw new AssertionError( - format("ParameterKind is [%s] but expected [%s]", kind, expectedParameterKind)); - } - return target.cast(value); - } - - public boolean isLongLiteral() { - return kind == ParameterKind.LONG; - } - - public Type getType() { - return getValue(ParameterKind.TYPE, Type.class); - } - - public Long getLongLiteral() { - return getValue(ParameterKind.LONG, Long.class); - } - - public NamedType getNamedType() { - return getValue(ParameterKind.NAMED_TYPE, NamedType.class); - } - - public String getVariable() { - return getValue(ParameterKind.VARIABLE, String.class); - } - - @Override - public String toString() { - return value.toString(); - } - - @Override - public boolean equals(Object o) { - if (this == o) { - return true; - } - if (o == null || getClass() != o.getClass()) { - return false; - } - - TypeParameter other = (TypeParameter) o; - - return this.kind == other.kind && Objects.equals(this.value, other.value); - } - - @Override - public int hashCode() { - return Objects.hash(kind, value); - } -} diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/type/TypeSignatureTranslator.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/type/TypeSignatureTranslator.java index ada5fa4b35fc..9390cd4826a2 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/type/TypeSignatureTranslator.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/type/TypeSignatureTranslator.java @@ -25,21 +25,16 @@ import org.apache.iotdb.db.queryengine.plan.relational.sql.ast.GenericDataType; import org.apache.iotdb.db.queryengine.plan.relational.sql.ast.Identifier; import org.apache.iotdb.db.queryengine.plan.relational.sql.ast.NumericParameter; -import org.apache.iotdb.db.queryengine.plan.relational.sql.ast.RowDataType; import org.apache.iotdb.db.queryengine.plan.relational.sql.ast.TypeParameter; import com.google.common.collect.ImmutableList; import org.apache.tsfile.read.common.type.Type; import java.util.Collections; -import java.util.List; import java.util.Locale; import java.util.Set; import static com.google.common.base.Preconditions.checkArgument; -import static com.google.common.collect.ImmutableList.toImmutableList; -import static org.apache.iotdb.db.queryengine.plan.relational.type.StandardTypes.ROW; -import static org.apache.iotdb.db.queryengine.plan.relational.type.TypeSignatureParameter.namedTypeParameter; import static org.apache.iotdb.db.queryengine.plan.relational.type.TypeSignatureParameter.numericParameter; import static org.apache.iotdb.db.queryengine.plan.relational.type.TypeSignatureParameter.typeParameter; import static org.apache.iotdb.db.queryengine.plan.relational.type.TypeSignatureParameter.typeVariable; @@ -61,30 +56,10 @@ private static TypeSignature toTypeSignature(DataType type, Set typeVari if (type instanceof GenericDataType) { return toTypeSignature((GenericDataType) type, typeVariables); } - if (type instanceof RowDataType) { - return toTypeSignature((RowDataType) type, typeVariables); - } throw new UnsupportedOperationException("Unsupported DataType: " + type.getClass().getName()); } - private static TypeSignature toTypeSignature(RowDataType type, Set typeVariables) { - List parameters = - type.getFields().stream() - .map( - field -> - namedTypeParameter( - new NamedTypeSignature( - field - .getName() - .map(TypeSignatureTranslator::canonicalize) - .map(RowFieldName::new), - toTypeSignature(field.getType(), typeVariables)))) - .collect(toImmutableList()); - - return new TypeSignature(ROW, parameters); - } - private static TypeSignature toTypeSignature(GenericDataType type, Set typeVariables) { ImmutableList.Builder parameters = ImmutableList.builder(); From 678bb5e04e4312ae6fe61c09c3d337bb4b2bf30d Mon Sep 17 00:00:00 2001 From: lancelly <1435078631@qq.com> Date: Mon, 16 Dec 2024 14:13:04 +0800 Subject: [PATCH 12/26] add IT for null value in source --- .../recent/subquery/SubqueryDataSetUtils.java | 2 ++ .../IoTDBUncorrelatedInPredicateSubqueryIT.java | 13 +++++++++++++ .../relational/MergeSortSemiJoinOperator.java | 14 +++++++++++++- .../plan/relational/planner/TranslationMap.java | 8 -------- .../relational/planner/ir/ExpressionRewriter.java | 6 ------ 5 files changed, 28 insertions(+), 15 deletions(-) diff --git a/integration-test/src/test/java/org/apache/iotdb/relational/it/query/recent/subquery/SubqueryDataSetUtils.java b/integration-test/src/test/java/org/apache/iotdb/relational/it/query/recent/subquery/SubqueryDataSetUtils.java index 36d66eb4b243..f3f79eb61bac 100644 --- a/integration-test/src/test/java/org/apache/iotdb/relational/it/query/recent/subquery/SubqueryDataSetUtils.java +++ b/integration-test/src/test/java/org/apache/iotdb/relational/it/query/recent/subquery/SubqueryDataSetUtils.java @@ -108,6 +108,8 @@ public class SubqueryDataSetUtils { "CREATE TABLE table3(device_id STRING ID, s1 INT32 MEASUREMENT, s2 INT64 MEASUREMENT, s3 FLOAT MEASUREMENT, s4 DOUBLE MEASUREMENT, s5 BOOLEAN MEASUREMENT, s6 TEXT MEASUREMENT, s7 STRING MEASUREMENT, s8 BLOB MEASUREMENT, s9 TIMESTAMP MEASUREMENT, s10 DATE MEASUREMENT)", "INSERT INTO table3(time,device_id,s1,s2,s3,s4,s5,s6,s7,s8,s9,s10) values (2024-09-24T06:13:30.000+00:00,'d01',30,30,30.0,30.0,true,'shanghai_huangpu_red_A_d01_30','shanghai_huangpu_red_A_d01_30',X'cafebabe30',2024-09-24T06:13:00.000+00:00,'2024-09-23')", "INSERT INTO table3(time,device_id,s1,s2,s3,s4,s5,s6,s7,s8,s9,s10) values (2024-09-24T06:14:30.000+00:00,'d01',40,40,40.0,40.0,false,'shanghai_huangpu_red_A_d01_40','shanghai_huangpu_red_A_d01_40',X'cafebabe40',2024-09-24T06:14:00.000+00:00,'2024-09-24')", + "INSERT INTO table3(time,device_id,s1,s2,s3,s4,s5,s6,s7,s8,s9,s10) values (2024-09-24T06:13:30.000+00:00,'d_null',30,30,30.0,30.0,true,'shanghai_huangpu_red_A_d01_30','shanghai_huangpu_red_A_d01_30',X'cafebabe30',2024-09-24T06:13:00.000+00:00,'2024-09-23')", + "INSERT INTO table3(time,device_id,s2,s3,s4,s5,s6,s7,s8,s9,s10) values (2024-09-24T06:14:30.000+00:00,'d_null',40,40.0,40.0,false,'shanghai_huangpu_red_A_d01_40','shanghai_huangpu_red_A_d01_40',X'cafebabe40',2024-09-24T06:14:00.000+00:00,'2024-09-24')", "FLUSH", "CLEAR ATTRIBUTE CACHE", }; diff --git a/integration-test/src/test/java/org/apache/iotdb/relational/it/query/recent/subquery/uncorrelated/IoTDBUncorrelatedInPredicateSubqueryIT.java b/integration-test/src/test/java/org/apache/iotdb/relational/it/query/recent/subquery/uncorrelated/IoTDBUncorrelatedInPredicateSubqueryIT.java index a743a0125002..a4accf01fb12 100644 --- a/integration-test/src/test/java/org/apache/iotdb/relational/it/query/recent/subquery/uncorrelated/IoTDBUncorrelatedInPredicateSubqueryIT.java +++ b/integration-test/src/test/java/org/apache/iotdb/relational/it/query/recent/subquery/uncorrelated/IoTDBUncorrelatedInPredicateSubqueryIT.java @@ -150,6 +150,12 @@ public void testInPredicateSubqueryInWhereClause() { retArray, DATABASE_NAME); } + + // Test case: where s in (subquery), s contains null value + sql = "SELECT s1 FROM table3 WHERE device_id = 'd_null' and s1 in (SELECT (s1) from table3)"; + expectedHeader = new String[] {"s1"}; + retArray = new String[] {"30,"}; + tableResultSetEqualTest(sql, expectedHeader, retArray, DATABASE_NAME); } @Test @@ -245,6 +251,13 @@ public void testInPredicateSubqueryInSelectClause() { tableResultSetEqualTest( String.format(sql, measurement, measurement), expectedHeader, retArray, DATABASE_NAME); } + + // Test case: select s in (subquery), s contains null value. The result should also be null when + // s is null. + sql = "SELECT s1 in (SELECT s1 from table3) from table3 where device_id = 'd_null'"; + expectedHeader = new String[] {"_col0"}; + retArray = new String[] {"true,", "null,"}; + tableResultSetEqualTest(sql, expectedHeader, retArray, DATABASE_NAME); } @Test diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/execution/operator/source/relational/MergeSortSemiJoinOperator.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/execution/operator/source/relational/MergeSortSemiJoinOperator.java index fee63beb627a..3dbc4d0e328e 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/execution/operator/source/relational/MergeSortSemiJoinOperator.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/execution/operator/source/relational/MergeSortSemiJoinOperator.java @@ -120,8 +120,9 @@ protected boolean processFinished() { return true; } - // skip all NULL values in left, because NULL value will not match the left value + // if current left is null, append null to result while (currentLeftHasNullValue()) { + appendNullValueToResult(); if (leftFinishedWithIncIndex()) { return true; } @@ -184,6 +185,17 @@ private void appendSemiJoinOutput(boolean value) { columnBuilder.writeBoolean(value); } + private void appendNullValueToResult() { + appendLeftBlockData(leftOutputSymbolIdx, resultBuilder, leftBlock, leftIndex); + appendNullToSemiJoinOutput(); + resultBuilder.declarePosition(); + } + + private void appendNullToSemiJoinOutput() { + ColumnBuilder columnBuilder = resultBuilder.getColumnBuilder(outputColumnNum - 1); + columnBuilder.appendNull(); + } + private void appendAllLeftBlock() { while (leftBlockNotEmpty()) { appendValueToResult(false); diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/planner/TranslationMap.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/planner/TranslationMap.java index 9d7b9ed3961a..5cbb161654d5 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/planner/TranslationMap.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/planner/TranslationMap.java @@ -34,7 +34,6 @@ import org.apache.iotdb.db.queryengine.plan.relational.sql.ast.LikePredicate; import org.apache.iotdb.db.queryengine.plan.relational.sql.ast.Parameter; import org.apache.iotdb.db.queryengine.plan.relational.sql.ast.QualifiedName; -import org.apache.iotdb.db.queryengine.plan.relational.sql.ast.RowDataType; import org.apache.iotdb.db.queryengine.plan.relational.sql.ast.SymbolReference; import org.apache.iotdb.db.queryengine.plan.relational.sql.ast.Trim; import org.apache.iotdb.db.queryengine.plan.relational.sql.util.AstUtil; @@ -364,13 +363,6 @@ public Expression rewriteGenericDataType( // do not rewrite identifiers within type parameters return node; } - - @Override - public Expression rewriteRowDataType( - RowDataType node, Void context, ExpressionTreeRewriter treeRewriter) { - // do not rewrite identifiers in field names - return node; - } }, expression); } diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/planner/ir/ExpressionRewriter.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/planner/ir/ExpressionRewriter.java index 735b74250410..d232101d6037 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/planner/ir/ExpressionRewriter.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/planner/ir/ExpressionRewriter.java @@ -47,7 +47,6 @@ import org.apache.iotdb.db.queryengine.plan.relational.sql.ast.Parameter; import org.apache.iotdb.db.queryengine.plan.relational.sql.ast.QuantifiedComparisonExpression; import org.apache.iotdb.db.queryengine.plan.relational.sql.ast.Row; -import org.apache.iotdb.db.queryengine.plan.relational.sql.ast.RowDataType; import org.apache.iotdb.db.queryengine.plan.relational.sql.ast.SearchedCaseExpression; import org.apache.iotdb.db.queryengine.plan.relational.sql.ast.SimpleCaseExpression; import org.apache.iotdb.db.queryengine.plan.relational.sql.ast.SubqueryExpression; @@ -232,9 +231,4 @@ public Expression rewriteGenericDataType( GenericDataType node, C context, ExpressionTreeRewriter treeRewriter) { return rewriteExpression(node, context, treeRewriter); } - - public Expression rewriteRowDataType( - RowDataType node, C context, ExpressionTreeRewriter treeRewriter) { - return rewriteExpression(node, context, treeRewriter); - } } From f5d0be8a0ad62883be1937123e1907fe310d6699 Mon Sep 17 00:00:00 2001 From: lancelly <1435078631@qq.com> Date: Mon, 16 Dec 2024 14:20:09 +0800 Subject: [PATCH 13/26] modify semijoin according to innerjoin && fullouter join --- .../source/relational/MergeSortSemiJoinOperator.java | 12 ++---------- 1 file changed, 2 insertions(+), 10 deletions(-) diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/execution/operator/source/relational/MergeSortSemiJoinOperator.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/execution/operator/source/relational/MergeSortSemiJoinOperator.java index 3dbc4d0e328e..48daa48525e5 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/execution/operator/source/relational/MergeSortSemiJoinOperator.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/execution/operator/source/relational/MergeSortSemiJoinOperator.java @@ -148,16 +148,8 @@ protected boolean processFinished() { return true; } - // has right values equal to current left, append to join result, inc leftIndex - if (hasMatchedRightValueToProbeLeft()) { - leftIndex++; - if (leftIndex >= leftBlock.getPositionCount()) { - resetLeftBlock(); - return true; - } - } - - return false; + // has right value equals to current left, append to join result, inc leftIndex + return hasMatchedRightValueToProbeLeft() && leftFinishedWithIncIndex(); } @Override From 879b0ce6a8d61c5eca596a5eaebd0027db656182 Mon Sep 17 00:00:00 2001 From: lancelly <1435078631@qq.com> Date: Mon, 16 Dec 2024 14:23:04 +0800 Subject: [PATCH 14/26] remove useless code --- .../plan/planner/TableOperatorGenerator.java | 10 ++++------ 1 file changed, 4 insertions(+), 6 deletions(-) diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/planner/TableOperatorGenerator.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/planner/TableOperatorGenerator.java index 740c7584454d..836801a652fa 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/planner/TableOperatorGenerator.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/planner/TableOperatorGenerator.java @@ -1396,10 +1396,12 @@ public Operator visitSemiJoin(SemiJoinNode node, LocalExecutionPlanContext conte filteringSourceJoinKeyPosition, "FilteringSource of SemiJoinNode doesn't contain filteringSourceJoinSymbol."); - Type sourceJoinKeyType = getJoinKeyType(context, node.getSourceJoinSymbol()); + Type sourceJoinKeyType = + context.getTypeProvider().getTableModelType(node.getSourceJoinSymbol()); checkArgument( - sourceJoinKeyType == getJoinKeyType(context, node.getFilteringSourceJoinSymbol()), + sourceJoinKeyType + == context.getTypeProvider().getTableModelType(node.getFilteringSourceJoinSymbol()), "Join key type mismatch."); OperatorContext operatorContext = context @@ -1419,10 +1421,6 @@ public Operator visitSemiJoin(SemiJoinNode node, LocalExecutionPlanContext conte dataTypes); } - private Type getJoinKeyType(LocalExecutionPlanContext context, Symbol symbol) { - return context.getTypeProvider().getTableModelType(symbol); - } - @Override public Operator visitEnforceSingleRow( EnforceSingleRowNode node, LocalExecutionPlanContext context) { From 034e4f305919851859096c2f99ebd51258274dc0 Mon Sep 17 00:00:00 2001 From: lancelly <1435078631@qq.com> Date: Wed, 18 Dec 2024 09:29:27 +0800 Subject: [PATCH 15/26] add optimize rule --- .../aggregation/AccumulatorFactory.java | 3 + .../aggregation/CountAllAccumulator.java | 97 +++++ .../metadata/TableMetadataImpl.java | 1 + .../relational/planner/IrTypeAnalyzer.java | 7 +- .../planner/SimplePlanRewriter.java | 88 +++++ .../optimizations/LogicalOptimizeFactory.java | 1 + .../PushPredicateIntoTableScan.java | 31 +- ...tifiedComparisonApplyToCorrelatedJoin.java | 341 ++++++++++++++++++ .../iotdb/db/utils/constant/SqlConstant.java | 1 + .../TableBuiltinAggregationFunction.java | 2 + .../src/main/thrift/common.thrift | 3 +- 11 files changed, 569 insertions(+), 6 deletions(-) create mode 100644 iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/execution/operator/source/relational/aggregation/CountAllAccumulator.java create mode 100644 iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/planner/SimplePlanRewriter.java create mode 100644 iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/planner/optimizations/TransformQuantifiedComparisonApplyToCorrelatedJoin.java diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/execution/operator/source/relational/aggregation/AccumulatorFactory.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/execution/operator/source/relational/aggregation/AccumulatorFactory.java index 125fa7563029..2553050ee87e 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/execution/operator/source/relational/aggregation/AccumulatorFactory.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/execution/operator/source/relational/aggregation/AccumulatorFactory.java @@ -44,6 +44,7 @@ import java.util.Map; import static com.google.common.base.Preconditions.checkState; +import static org.apache.iotdb.commons.udf.builtin.relational.TableBuiltinAggregationFunction.COUNT_ALL; import static org.apache.iotdb.commons.udf.builtin.relational.TableBuiltinAggregationFunction.FIRST_BY; import static org.apache.iotdb.commons.udf.builtin.relational.TableBuiltinAggregationFunction.LAST_BY; import static org.apache.iotdb.db.queryengine.plan.relational.planner.ir.GlobalTimePredicateExtractVisitor.isTimeColumn; @@ -167,6 +168,8 @@ public static TableAccumulator createBuiltinAccumulator( switch (aggregationType) { case COUNT: return new CountAccumulator(); + case COUNT_ALL: + return new CountAllAccumulator(); case AVG: return new AvgAccumulator(inputDataTypes.get(0)); case SUM: diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/execution/operator/source/relational/aggregation/CountAllAccumulator.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/execution/operator/source/relational/aggregation/CountAllAccumulator.java new file mode 100644 index 000000000000..78c94f3e6e69 --- /dev/null +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/execution/operator/source/relational/aggregation/CountAllAccumulator.java @@ -0,0 +1,97 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.iotdb.db.queryengine.execution.operator.source.relational.aggregation; + +import org.apache.tsfile.block.column.Column; +import org.apache.tsfile.block.column.ColumnBuilder; +import org.apache.tsfile.file.metadata.statistics.Statistics; +import org.apache.tsfile.utils.RamUsageEstimator; + +import static com.google.common.base.Preconditions.checkArgument; + +public class CountAllAccumulator implements TableAccumulator { + private static final long INSTANCE_SIZE = + RamUsageEstimator.shallowSizeOfInstance(CountAllAccumulator.class); + private long countState = 0; + + @Override + public long getEstimatedSize() { + return INSTANCE_SIZE; + } + + @Override + public TableAccumulator copy() { + return new CountAllAccumulator(); + } + + @Override + public void addInput(Column[] arguments) { + checkArgument(arguments.length == 1, "argument of Count should be one column"); + int count = arguments[0].getPositionCount(); + countState += count; + } + + @Override + public void removeInput(Column[] arguments) { + checkArgument(arguments.length == 1, "argument of Count should be one column"); + int count = arguments[0].getPositionCount(); + countState -= count; + } + + @Override + public void addIntermediate(Column argument) { + for (int i = 0; i < argument.getPositionCount(); i++) { + if (argument.isNull(i)) { + continue; + } + countState += argument.getLong(i); + } + } + + @Override + public void evaluateIntermediate(ColumnBuilder columnBuilder) { + columnBuilder.writeLong(countState); + } + + @Override + public void evaluateFinal(ColumnBuilder columnBuilder) { + columnBuilder.writeLong(countState); + } + + @Override + public boolean hasFinalResult() { + return false; + } + + @Override + public void addStatistics(Statistics[] statistics) { + throw new UnsupportedOperationException("CountAllAccumulator does not support statistics."); + } + + @Override + public void reset() { + countState = 0; + } + + @Override + public boolean removable() { + return true; + } +} diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/metadata/TableMetadataImpl.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/metadata/TableMetadataImpl.java index c88a73d87c9e..4c2798e448e4 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/metadata/TableMetadataImpl.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/metadata/TableMetadataImpl.java @@ -614,6 +614,7 @@ && isIntegerNumber(argumentTypes.get(2)))) { // get return type switch (functionName.toLowerCase(Locale.ENGLISH)) { case SqlConstant.COUNT: + case SqlConstant.COUNT_ALL: return INT64; case SqlConstant.FIRST_AGGREGATION: case SqlConstant.LAST_AGGREGATION: diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/planner/IrTypeAnalyzer.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/planner/IrTypeAnalyzer.java index 1a19f40132f1..ab84934d166e 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/planner/IrTypeAnalyzer.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/planner/IrTypeAnalyzer.java @@ -82,7 +82,6 @@ import static org.apache.iotdb.db.queryengine.plan.relational.type.TypeSignatureTranslator.toTypeSignature; import static org.apache.tsfile.read.common.type.BooleanType.BOOLEAN; import static org.apache.tsfile.read.common.type.DoubleType.DOUBLE; -import static org.apache.tsfile.read.common.type.IntType.INT32; import static org.apache.tsfile.read.common.type.LongType.INT64; import static org.apache.tsfile.read.common.type.UnknownType.UNKNOWN; @@ -336,11 +335,11 @@ protected Type visitBinaryLiteral(BinaryLiteral node, Context context) { @Override protected Type visitLongLiteral(LongLiteral node, Context context) { - if (node.getParsedValue() >= Integer.MIN_VALUE + /*if (node.getParsedValue() >= Integer.MIN_VALUE && node.getParsedValue() <= Integer.MAX_VALUE) { return setExpressionType(node, INT32); - } - + }*/ + // keep the original type return setExpressionType(node, INT64); } diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/planner/SimplePlanRewriter.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/planner/SimplePlanRewriter.java new file mode 100644 index 000000000000..d2a8ec781a19 --- /dev/null +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/planner/SimplePlanRewriter.java @@ -0,0 +1,88 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.iotdb.db.queryengine.plan.relational.planner; + +import org.apache.iotdb.db.queryengine.plan.planner.plan.node.PlanNode; +import org.apache.iotdb.db.queryengine.plan.planner.plan.node.PlanVisitor; + +import com.google.common.collect.ImmutableList; + +import static com.google.common.base.Verify.verifyNotNull; +import static org.apache.iotdb.db.queryengine.plan.relational.planner.node.ChildReplacer.replaceChildren; + +public abstract class SimplePlanRewriter + extends PlanVisitor> { + public static PlanNode rewriteWith(SimplePlanRewriter rewriter, PlanNode node) { + return node.accept(rewriter, new RewriteContext<>(rewriter, null)); + } + + public static PlanNode rewriteWith(SimplePlanRewriter rewriter, PlanNode node, C context) { + return node.accept(rewriter, new RewriteContext<>(rewriter, context)); + } + + @Override + public PlanNode visitPlan(PlanNode node, RewriteContext context) { + return context.defaultRewrite(node, context.get()); + } + + public static class RewriteContext { + private final C userContext; + private final SimplePlanRewriter nodeRewriter; + + private RewriteContext(SimplePlanRewriter nodeRewriter, C userContext) { + this.nodeRewriter = nodeRewriter; + this.userContext = userContext; + } + + public C get() { + return userContext; + } + + /** + * Invoke the rewrite logic recursively on children of the given node and swap it out with an + * identical copy with the rewritten children + */ + public PlanNode defaultRewrite(PlanNode node) { + return defaultRewrite(node, null); + } + + /** + * Invoke the rewrite logic recursively on children of the given node and swap it out with an + * identical copy with the rewritten children + */ + public PlanNode defaultRewrite(PlanNode node, C context) { + ImmutableList.Builder children = + ImmutableList.builderWithExpectedSize(node.getChildren().size()); + node.getChildren().forEach(source -> children.add(rewrite(source, context))); + return replaceChildren(node, children.build()); + } + + /** This method is meant for invoking the rewrite logic on children while processing a node */ + public PlanNode rewrite(PlanNode node, C userContext) { + PlanNode result = node.accept(nodeRewriter, new RewriteContext<>(nodeRewriter, userContext)); + return verifyNotNull(result, "nodeRewriter returned null for %s", node.getClass().getName()); + } + + /** This method is meant for invoking the rewrite logic on children while processing a node */ + public PlanNode rewrite(PlanNode node) { + return rewrite(node, null); + } + } +} diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/planner/optimizations/LogicalOptimizeFactory.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/planner/optimizations/LogicalOptimizeFactory.java index 62bdd1bf1b9d..d95d246a20a9 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/planner/optimizations/LogicalOptimizeFactory.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/planner/optimizations/LogicalOptimizeFactory.java @@ -208,6 +208,7 @@ public LogicalOptimizeFactory(PlannerContext plannerContext) { new UnaliasSymbolReferences(plannerContext.getMetadata()), columnPruningOptimizer, inlineProjectionLimitFiltersOptimizer, + new TransformQuantifiedComparisonApplyToCorrelatedJoin(metadata), new IterativeOptimizer( plannerContext, ruleStats, diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/planner/optimizations/PushPredicateIntoTableScan.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/planner/optimizations/PushPredicateIntoTableScan.java index f5313b1a5374..8998c9517c27 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/planner/optimizations/PushPredicateIntoTableScan.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/planner/optimizations/PushPredicateIntoTableScan.java @@ -209,7 +209,7 @@ public PlanNode visitFilter(FilterNode node, RewriteContext context) { Expression predicate = combineConjuncts(node.getPredicate(), context.inheritedPredicate); // when exist diff function, predicate can not be pushed down into DeviceTableScanNode - if (containsDiffFunction(predicate)) { + if (containsDiffFunction(predicate) || canNotPushDownBelowProjectNode(node, predicate)) { node.setChild(node.getChild().accept(this, new RewriteContext())); node.setPredicate(predicate); context.inheritedPredicate = TRUE_LITERAL; @@ -231,6 +231,35 @@ public PlanNode visitFilter(FilterNode node, RewriteContext context) { return node; } + private boolean canNotPushDownBelowProjectNode(FilterNode node, Expression predicate) { + PlanNode child = node.getChild(); + if (child instanceof ProjectNode) { + // if the inheritedPredicate is not in the output of the child of ProjectNode, we can not + // push it down for now. + // (predicate will be computed by the ProjectNode, Trino will rewrite the predicate in + // visitProject, but we have not implemented this for now.) + Set outputSymbolsOfProjectChild = + new HashSet<>(((ProjectNode) child).getChild().getOutputSymbols()); + return missingTermsInOutputSymbols(predicate, outputSymbolsOfProjectChild); + } + return false; + } + + private boolean missingTermsInOutputSymbols(Expression expression, Set outputSymbols) { + if (expression instanceof SymbolReference) { + return !outputSymbols.contains(Symbol.from(expression)); + } + if (!expression.getChildren().isEmpty()) { + for (Node node : expression.getChildren()) { + if (missingTermsInOutputSymbols((Expression) node, outputSymbols)) { + return true; + } + } + } + + return false; + } + // private boolean areExpressionsEquivalent( // Expression leftExpression, Expression rightExpression) { // return false; diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/planner/optimizations/TransformQuantifiedComparisonApplyToCorrelatedJoin.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/planner/optimizations/TransformQuantifiedComparisonApplyToCorrelatedJoin.java new file mode 100644 index 000000000000..c56ed0b706ed --- /dev/null +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/planner/optimizations/TransformQuantifiedComparisonApplyToCorrelatedJoin.java @@ -0,0 +1,341 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.iotdb.db.queryengine.plan.relational.planner.optimizations; + +import org.apache.iotdb.db.queryengine.common.QueryId; +import org.apache.iotdb.db.queryengine.plan.planner.plan.node.PlanNode; +import org.apache.iotdb.db.queryengine.plan.relational.function.BoundSignature; +import org.apache.iotdb.db.queryengine.plan.relational.function.FunctionId; +import org.apache.iotdb.db.queryengine.plan.relational.function.FunctionKind; +import org.apache.iotdb.db.queryengine.plan.relational.metadata.FunctionNullability; +import org.apache.iotdb.db.queryengine.plan.relational.metadata.Metadata; +import org.apache.iotdb.db.queryengine.plan.relational.metadata.ResolvedFunction; +import org.apache.iotdb.db.queryengine.plan.relational.planner.Assignments; +import org.apache.iotdb.db.queryengine.plan.relational.planner.SimplePlanRewriter; +import org.apache.iotdb.db.queryengine.plan.relational.planner.Symbol; +import org.apache.iotdb.db.queryengine.plan.relational.planner.SymbolAllocator; +import org.apache.iotdb.db.queryengine.plan.relational.planner.ir.IrUtils; +import org.apache.iotdb.db.queryengine.plan.relational.planner.node.AggregationNode; +import org.apache.iotdb.db.queryengine.plan.relational.planner.node.ApplyNode; +import org.apache.iotdb.db.queryengine.plan.relational.planner.node.CorrelatedJoinNode; +import org.apache.iotdb.db.queryengine.plan.relational.planner.node.JoinNode; +import org.apache.iotdb.db.queryengine.plan.relational.planner.node.ProjectNode; +import org.apache.iotdb.db.queryengine.plan.relational.sql.ast.BooleanLiteral; +import org.apache.iotdb.db.queryengine.plan.relational.sql.ast.Cast; +import org.apache.iotdb.db.queryengine.plan.relational.sql.ast.ComparisonExpression; +import org.apache.iotdb.db.queryengine.plan.relational.sql.ast.Expression; +import org.apache.iotdb.db.queryengine.plan.relational.sql.ast.LongLiteral; +import org.apache.iotdb.db.queryengine.plan.relational.sql.ast.NullLiteral; +import org.apache.iotdb.db.queryengine.plan.relational.sql.ast.SearchedCaseExpression; +import org.apache.iotdb.db.queryengine.plan.relational.sql.ast.SimpleCaseExpression; +import org.apache.iotdb.db.queryengine.plan.relational.sql.ast.WhenClause; + +import com.google.common.collect.ImmutableList; +import com.google.common.collect.ImmutableMap; +import org.apache.tsfile.read.common.type.LongType; +import org.apache.tsfile.read.common.type.Type; + +import java.util.EnumSet; +import java.util.List; +import java.util.Locale; +import java.util.Optional; +import java.util.function.Function; + +import static com.google.common.base.Preconditions.checkState; +import static com.google.common.collect.Iterables.getOnlyElement; +import static java.util.Objects.requireNonNull; +import static org.apache.iotdb.db.queryengine.plan.relational.planner.SimplePlanRewriter.rewriteWith; +import static org.apache.iotdb.db.queryengine.plan.relational.planner.ir.IrUtils.combineConjuncts; +import static org.apache.iotdb.db.queryengine.plan.relational.planner.node.AggregationNode.globalAggregation; +import static org.apache.iotdb.db.queryengine.plan.relational.planner.node.AggregationNode.singleAggregation; +import static org.apache.iotdb.db.queryengine.plan.relational.planner.node.ApplyNode.Quantifier.ALL; +import static org.apache.iotdb.db.queryengine.plan.relational.sql.ast.BooleanLiteral.FALSE_LITERAL; +import static org.apache.iotdb.db.queryengine.plan.relational.sql.ast.BooleanLiteral.TRUE_LITERAL; +import static org.apache.iotdb.db.queryengine.plan.relational.sql.ast.ComparisonExpression.Operator.EQUAL; +import static org.apache.iotdb.db.queryengine.plan.relational.sql.ast.ComparisonExpression.Operator.GREATER_THAN; +import static org.apache.iotdb.db.queryengine.plan.relational.sql.ast.ComparisonExpression.Operator.GREATER_THAN_OR_EQUAL; +import static org.apache.iotdb.db.queryengine.plan.relational.sql.ast.ComparisonExpression.Operator.LESS_THAN; +import static org.apache.iotdb.db.queryengine.plan.relational.sql.ast.ComparisonExpression.Operator.LESS_THAN_OR_EQUAL; +import static org.apache.iotdb.db.queryengine.plan.relational.sql.ast.ComparisonExpression.Operator.NOT_EQUAL; +import static org.apache.iotdb.db.queryengine.plan.relational.type.TypeSignatureTranslator.toSqlType; +import static org.apache.tsfile.read.common.type.BooleanType.BOOLEAN; + +public class TransformQuantifiedComparisonApplyToCorrelatedJoin implements PlanOptimizer { + private final Metadata metadata; + + public TransformQuantifiedComparisonApplyToCorrelatedJoin(Metadata metadata) { + this.metadata = requireNonNull(metadata, "metadata is null"); + } + + @Override + public PlanNode optimize(PlanNode plan, Context context) { + return rewriteWith( + new Rewriter(context.idAllocator(), context.getSymbolAllocator(), metadata), plan, null); + } + + private static class Rewriter extends SimplePlanRewriter { + private final QueryId idAllocator; + private final SymbolAllocator symbolAllocator; + private final Metadata metadata; + + public Rewriter(QueryId idAllocator, SymbolAllocator symbolAllocator, Metadata metadata) { + this.idAllocator = requireNonNull(idAllocator, "idAllocator is null"); + this.symbolAllocator = requireNonNull(symbolAllocator, "symbolAllocator is null"); + this.metadata = requireNonNull(metadata, "metadata is null"); + } + + @Override + public PlanNode visitApply(ApplyNode node, RewriteContext context) { + if (node.getSubqueryAssignments().size() != 1) { + return context.defaultRewrite(node); + } + + ApplyNode.SetExpression expression = getOnlyElement(node.getSubqueryAssignments().values()); + if (expression instanceof ApplyNode.QuantifiedComparison) { + return rewriteQuantifiedApplyNode( + node, (ApplyNode.QuantifiedComparison) expression, context); + } + + return context.defaultRewrite(node); + } + + private PlanNode rewriteQuantifiedApplyNode( + ApplyNode node, + ApplyNode.QuantifiedComparison quantifiedComparison, + RewriteContext context) { + PlanNode subqueryPlan = context.rewrite(node.getSubquery()); + + Symbol outputColumn = getOnlyElement(subqueryPlan.getOutputSymbols()); + Type outputColumnType = symbolAllocator.getTypes().getTableModelType(outputColumn); + checkState(outputColumnType.isOrderable(), "Subquery result type must be orderable"); + + Symbol minValue = symbolAllocator.newSymbol("min", outputColumnType); + Symbol maxValue = symbolAllocator.newSymbol("max", outputColumnType); + Symbol countAllValue = symbolAllocator.newSymbol("count_all", LongType.getInstance()); + Symbol countNonNullValue = + symbolAllocator.newSymbol("count_non_null", LongType.getInstance()); + + List outputColumnReferences = ImmutableList.of(outputColumn.toSymbolReference()); + + subqueryPlan = + singleAggregation( + idAllocator.genPlanNodeId(), + subqueryPlan, + ImmutableMap.of( + minValue, + new AggregationNode.Aggregation( + getResolvedBuiltInAggregateFunction( + "min", ImmutableList.of(outputColumnType)), + outputColumnReferences, + false, + Optional.empty(), + Optional.empty(), + Optional.empty()), + maxValue, + new AggregationNode.Aggregation( + getResolvedBuiltInAggregateFunction( + "max", ImmutableList.of(outputColumnType)), + outputColumnReferences, + false, + Optional.empty(), + Optional.empty(), + Optional.empty()), + countAllValue, + new AggregationNode.Aggregation( + getResolvedBuiltInAggregateFunction( + "count_all", ImmutableList.of(outputColumnType)), + outputColumnReferences, + false, + Optional.empty(), + Optional.empty(), + Optional.empty()), + countNonNullValue, + new AggregationNode.Aggregation( + getResolvedBuiltInAggregateFunction( + "count", ImmutableList.of(outputColumnType)), + outputColumnReferences, + false, + Optional.empty(), + Optional.empty(), + Optional.empty())), + globalAggregation()); + + PlanNode join = + new CorrelatedJoinNode( + node.getPlanNodeId(), + context.rewrite(node.getInput()), + subqueryPlan, + node.getCorrelation(), + JoinNode.JoinType.INNER, + TRUE_LITERAL, + node.getOriginSubquery()); + + Expression valueComparedToSubquery = + rewriteUsingBounds( + quantifiedComparison, minValue, maxValue, countAllValue, countNonNullValue); + + Symbol quantifiedComparisonSymbol = getOnlyElement(node.getSubqueryAssignments().keySet()); + + return projectExpressions( + join, Assignments.of(quantifiedComparisonSymbol, valueComparedToSubquery)); + } + + private ResolvedFunction getResolvedBuiltInAggregateFunction( + String functionName, List argumentTypes) { + // The same as the code in ExpressionAnalyzer + Type type = metadata.getFunctionReturnType(functionName, argumentTypes); + return new ResolvedFunction( + new BoundSignature(functionName.toLowerCase(Locale.ENGLISH), type, argumentTypes), + new FunctionId("noop"), + FunctionKind.AGGREGATE, + true, + FunctionNullability.getAggregationFunctionNullability(argumentTypes.size())); + } + + public Expression rewriteUsingBounds( + ApplyNode.QuantifiedComparison quantifiedComparison, + Symbol minValue, + Symbol maxValue, + Symbol countAllValue, + Symbol countNonNullValue) { + BooleanLiteral emptySetResult; + Function, Expression> quantifier; + if (quantifiedComparison.getQuantifier() == ALL) { + emptySetResult = TRUE_LITERAL; + quantifier = IrUtils::combineConjuncts; + } else { + emptySetResult = FALSE_LITERAL; + quantifier = IrUtils::combineDisjuncts; + } + Expression comparisonWithExtremeValue = + getBoundComparisons(quantifiedComparison, minValue, maxValue); + + return new SimpleCaseExpression( + countAllValue.toSymbolReference(), + ImmutableList.of(new WhenClause(new LongLiteral("0"), emptySetResult)), + quantifier.apply( + ImmutableList.of( + comparisonWithExtremeValue, + new SearchedCaseExpression( + ImmutableList.of( + new WhenClause( + new ComparisonExpression( + NOT_EQUAL, + countAllValue.toSymbolReference(), + countNonNullValue.toSymbolReference()), + new Cast(new NullLiteral(), toSqlType(BOOLEAN)))), + emptySetResult)))); + } + + private Expression getBoundComparisons( + ApplyNode.QuantifiedComparison quantifiedComparison, Symbol minValue, Symbol maxValue) { + if (mapOperator(quantifiedComparison) == EQUAL + && quantifiedComparison.getQuantifier() == ALL) { + // A = ALL B <=> min B = max B && A = min B + return combineConjuncts( + new ComparisonExpression( + EQUAL, minValue.toSymbolReference(), maxValue.toSymbolReference()), + new ComparisonExpression( + EQUAL, + quantifiedComparison.getValue().toSymbolReference(), + maxValue.toSymbolReference())); + } + + if (EnumSet.of(LESS_THAN, LESS_THAN_OR_EQUAL, GREATER_THAN, GREATER_THAN_OR_EQUAL) + .contains(mapOperator(quantifiedComparison))) { + // A < ALL B <=> A < min B + // A > ALL B <=> A > max B + // A < ANY B <=> A < max B + // A > ANY B <=> A > min B + Symbol boundValue = + shouldCompareValueWithLowerBound(quantifiedComparison) ? minValue : maxValue; + return new ComparisonExpression( + mapOperator(quantifiedComparison), + quantifiedComparison.getValue().toSymbolReference(), + boundValue.toSymbolReference()); + } + throw new IllegalArgumentException( + "Unsupported quantified comparison: " + quantifiedComparison); + } + + private static ComparisonExpression.Operator mapOperator( + ApplyNode.QuantifiedComparison quantifiedComparison) { + switch (quantifiedComparison.getOperator()) { + case EQUAL: + return EQUAL; + case NOT_EQUAL: + return NOT_EQUAL; + case LESS_THAN: + return LESS_THAN; + case LESS_THAN_OR_EQUAL: + return LESS_THAN_OR_EQUAL; + case GREATER_THAN: + return GREATER_THAN; + case GREATER_THAN_OR_EQUAL: + return GREATER_THAN_OR_EQUAL; + default: + throw new IllegalArgumentException( + "Unexpected quantifiedComparison: " + quantifiedComparison.getOperator()); + } + } + + private static boolean shouldCompareValueWithLowerBound( + ApplyNode.QuantifiedComparison quantifiedComparison) { + ComparisonExpression.Operator operator = mapOperator(quantifiedComparison); + switch (quantifiedComparison.getQuantifier()) { + case ALL: + switch (operator) { + case LESS_THAN: + case LESS_THAN_OR_EQUAL: + return true; + case GREATER_THAN: + case GREATER_THAN_OR_EQUAL: + return false; + default: + throw new IllegalArgumentException("Unexpected value: " + operator); + } + case ANY: + case SOME: + switch (operator) { + case LESS_THAN: + case LESS_THAN_OR_EQUAL: + return false; + case GREATER_THAN: + case GREATER_THAN_OR_EQUAL: + return true; + default: + throw new IllegalArgumentException("Unexpected value: " + operator); + } + default: + throw new IllegalArgumentException( + "Unexpected Quantifier: " + quantifiedComparison.getQuantifier()); + } + } + + private ProjectNode projectExpressions(PlanNode input, Assignments subqueryAssignments) { + Assignments assignments = + Assignments.builder() + .putIdentities(input.getOutputSymbols()) + .putAll(subqueryAssignments) + .build(); + return new ProjectNode(idAllocator.genPlanNodeId(), input, assignments); + } + } +} diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/utils/constant/SqlConstant.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/utils/constant/SqlConstant.java index 43267f9e48e1..cad8f229c4eb 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/utils/constant/SqlConstant.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/utils/constant/SqlConstant.java @@ -62,6 +62,7 @@ protected SqlConstant() { public static final String LAST_AGGREGATION = "last"; public static final String FIRST_AGGREGATION = "first"; public static final String COUNT = "count"; + public static final String COUNT_ALL = "count_all"; public static final String AVG = "avg"; public static final String SUM = "sum"; public static final String COUNT_IF = "count_if"; diff --git a/iotdb-core/node-commons/src/main/java/org/apache/iotdb/commons/udf/builtin/relational/TableBuiltinAggregationFunction.java b/iotdb-core/node-commons/src/main/java/org/apache/iotdb/commons/udf/builtin/relational/TableBuiltinAggregationFunction.java index 49db27110c7b..ae006f6751da 100644 --- a/iotdb-core/node-commons/src/main/java/org/apache/iotdb/commons/udf/builtin/relational/TableBuiltinAggregationFunction.java +++ b/iotdb-core/node-commons/src/main/java/org/apache/iotdb/commons/udf/builtin/relational/TableBuiltinAggregationFunction.java @@ -37,6 +37,7 @@ public enum TableBuiltinAggregationFunction { SUM("sum"), COUNT("count"), + COUNT_ALL("count_all"), AVG("avg"), EXTREME("extreme"), MAX("max"), @@ -80,6 +81,7 @@ public static Type getIntermediateType(String name, List originalArgumentT final String functionName = name.toLowerCase(); switch (functionName) { case "count": + case "count_all": return INT64; case "sum": return DOUBLE; diff --git a/iotdb-protocol/thrift-commons/src/main/thrift/common.thrift b/iotdb-protocol/thrift-commons/src/main/thrift/common.thrift index c46c4c0a65a0..93eafedc118e 100644 --- a/iotdb-protocol/thrift-commons/src/main/thrift/common.thrift +++ b/iotdb-protocol/thrift-commons/src/main/thrift/common.thrift @@ -281,7 +281,8 @@ enum TAggregationType { FIRST_BY, LAST_BY, MIN, - MAX + MAX, + COUNT_ALL } struct TShowConfigurationTemplateResp { From 3e815a40311ff1c82209dbaad3d4262fe2f0e5d4 Mon Sep 17 00:00:00 2001 From: lancelly <1435078631@qq.com> Date: Wed, 18 Dec 2024 17:44:17 +0800 Subject: [PATCH 16/26] add some IT --- ...TDBUncorrelatedQuantifiedComparisonIT.java | 555 ++++++++++++++++++ .../optimizations/LogicalOptimizeFactory.java | 7 + 2 files changed, 562 insertions(+) create mode 100644 integration-test/src/test/java/org/apache/iotdb/relational/it/query/recent/subquery/uncorrelated/IoTDBUncorrelatedQuantifiedComparisonIT.java diff --git a/integration-test/src/test/java/org/apache/iotdb/relational/it/query/recent/subquery/uncorrelated/IoTDBUncorrelatedQuantifiedComparisonIT.java b/integration-test/src/test/java/org/apache/iotdb/relational/it/query/recent/subquery/uncorrelated/IoTDBUncorrelatedQuantifiedComparisonIT.java new file mode 100644 index 000000000000..25dc92645f06 --- /dev/null +++ b/integration-test/src/test/java/org/apache/iotdb/relational/it/query/recent/subquery/uncorrelated/IoTDBUncorrelatedQuantifiedComparisonIT.java @@ -0,0 +1,555 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.iotdb.relational.it.query.recent.subquery.uncorrelated; + +import org.apache.iotdb.it.env.EnvFactory; +import org.apache.iotdb.it.framework.IoTDBTestRunner; +import org.apache.iotdb.itbase.category.TableClusterIT; +import org.apache.iotdb.itbase.category.TableLocalStandaloneIT; + +import org.junit.AfterClass; +import org.junit.BeforeClass; +import org.junit.Test; +import org.junit.experimental.categories.Category; +import org.junit.runner.RunWith; + +import static org.apache.iotdb.db.it.utils.TestUtils.prepareTableData; +import static org.apache.iotdb.db.it.utils.TestUtils.tableAssertTestFail; +import static org.apache.iotdb.db.it.utils.TestUtils.tableResultSetEqualTest; +import static org.apache.iotdb.relational.it.query.recent.subquery.SubqueryDataSetUtils.CREATE_SQLS; +import static org.apache.iotdb.relational.it.query.recent.subquery.SubqueryDataSetUtils.DATABASE_NAME; +import static org.apache.iotdb.relational.it.query.recent.subquery.SubqueryDataSetUtils.NUMERIC_MEASUREMENTS; + +@RunWith(IoTDBTestRunner.class) +@Category({TableLocalStandaloneIT.class, TableClusterIT.class}) +public class IoTDBUncorrelatedQuantifiedComparisonIT { + + @BeforeClass + public static void setUp() throws Exception { + EnvFactory.getEnv().getConfig().getCommonConfig().setSortBufferSize(128 * 1024); + EnvFactory.getEnv().getConfig().getCommonConfig().setMaxTsBlockSizeInByte(4 * 1024); + EnvFactory.getEnv().initClusterEnvironment(); + prepareTableData(CREATE_SQLS); + } + + @AfterClass + public static void tearDown() throws Exception { + EnvFactory.getEnv().cleanClusterEnvironment(); + } + + @Test + public void testAnyAndSomeComparisonInWhereClauseWithoutNull() { + String sql; + String[] expectedHeader; + String[] retArray; + + // Test case: where s > any (subquery), s does not contain null value + sql = + "SELECT cast(%s AS INT32) as %s FROM table1 WHERE device_id = 'd01' and %s > any (SELECT %s FROM table1 WHERE device_id = 'd01')"; + retArray = new String[] {"40,", "50,", "60,", "70,"}; + for (String measurement : NUMERIC_MEASUREMENTS) { + expectedHeader = new String[] {measurement}; + tableResultSetEqualTest( + String.format(sql, measurement, measurement, measurement, measurement), + expectedHeader, + retArray, + DATABASE_NAME); + } + + // Test case: where s > some (subquery), s does not contain null value + sql = + "SELECT cast(%s AS INT32) as %s FROM table1 WHERE device_id = 'd01' and %s > some (SELECT %s FROM table1 WHERE device_id = 'd01')"; + retArray = new String[] {"40,", "50,", "60,", "70,"}; + for (String measurement : NUMERIC_MEASUREMENTS) { + expectedHeader = new String[] {measurement}; + tableResultSetEqualTest( + String.format(sql, measurement, measurement, measurement, measurement), + expectedHeader, + retArray, + DATABASE_NAME); + } + + // Test case: where s >= any (subquery), s does not contain null value + sql = + "SELECT cast(%s AS INT32) as %s FROM table1 WHERE device_id = 'd01' and %s >= any (SELECT %s FROM table1 WHERE device_id = 'd01')"; + retArray = new String[] {"30,", "40,", "50,", "60,", "70,"}; + for (String measurement : NUMERIC_MEASUREMENTS) { + expectedHeader = new String[] {measurement}; + tableResultSetEqualTest( + String.format(sql, measurement, measurement, measurement, measurement), + expectedHeader, + retArray, + DATABASE_NAME); + } + + // Test case: where s >= some (subquery), s does not contain null value + sql = + "SELECT cast(%s AS INT32) as %s FROM table1 WHERE device_id = 'd01' and %s >= some (SELECT %s FROM table1 WHERE device_id = 'd01')"; + retArray = new String[] {"30,", "40,", "50,", "60,", "70,"}; + for (String measurement : NUMERIC_MEASUREMENTS) { + expectedHeader = new String[] {measurement}; + tableResultSetEqualTest( + String.format(sql, measurement, measurement, measurement, measurement), + expectedHeader, + retArray, + DATABASE_NAME); + } + + // Test case: where s < any (subquery), s does not contain null value + sql = + "SELECT cast(%s AS INT32) as %s FROM table1 WHERE device_id = 'd01' and %s < any (SELECT %s FROM table1 WHERE device_id = 'd01')"; + retArray = new String[] {"30,", "40,", "50,", "60,"}; + for (String measurement : NUMERIC_MEASUREMENTS) { + expectedHeader = new String[] {measurement}; + tableResultSetEqualTest( + String.format(sql, measurement, measurement, measurement, measurement), + expectedHeader, + retArray, + DATABASE_NAME); + } + + // Test case: where < some (subquery), s does not contain null value + sql = + "SELECT cast(%s AS INT32) as %s FROM table1 WHERE device_id = 'd01' and %s < some (SELECT %s FROM table1 WHERE device_id = 'd01')"; + retArray = new String[] {"30,", "40,", "50,", "60,"}; + for (String measurement : NUMERIC_MEASUREMENTS) { + expectedHeader = new String[] {measurement}; + tableResultSetEqualTest( + String.format(sql, measurement, measurement, measurement, measurement), + expectedHeader, + retArray, + DATABASE_NAME); + } + + // Test case: where s <= any (subquery), s does not contain null value + sql = + "SELECT cast(%s AS INT32) as %s FROM table1 WHERE device_id = 'd01' and %s <= any (SELECT %s FROM table1 WHERE device_id = 'd01')"; + retArray = new String[] {"30,", "40,", "50,", "60,", "70,"}; + for (String measurement : NUMERIC_MEASUREMENTS) { + expectedHeader = new String[] {measurement}; + tableResultSetEqualTest( + String.format(sql, measurement, measurement, measurement, measurement), + expectedHeader, + retArray, + DATABASE_NAME); + } + + // Test case: where s <= some (subquery), s does not contain null value + sql = + "SELECT cast(%s AS INT32) as %s FROM table1 WHERE device_id = 'd01' and %s <= some (SELECT %s FROM table1 WHERE device_id = 'd01')"; + retArray = new String[] {"30,", "40,", "50,", "60,", "70,"}; + for (String measurement : NUMERIC_MEASUREMENTS) { + expectedHeader = new String[] {measurement}; + tableResultSetEqualTest( + String.format(sql, measurement, measurement, measurement, measurement), + expectedHeader, + retArray, + DATABASE_NAME); + } + + // Test case: where s = any (subquery), s does not contain null value + sql = + "SELECT cast(%s AS INT32) as %s FROM table1 WHERE device_id = 'd01' and %s = any (SELECT %s FROM table1 WHERE device_id = 'd01')"; + retArray = new String[] {"30,", "40,", "50,", "60,", "70,"}; + for (String measurement : NUMERIC_MEASUREMENTS) { + expectedHeader = new String[] {measurement}; + tableResultSetEqualTest( + String.format(sql, measurement, measurement, measurement, measurement), + expectedHeader, + retArray, + DATABASE_NAME); + } + + // Test case: where s = some (subquery), s does not contain null value + sql = + "SELECT cast(%s AS INT32) as %s FROM table1 WHERE device_id = 'd01' and %s = some (SELECT %s FROM table1 WHERE device_id = 'd01')"; + retArray = new String[] {"30,", "40,", "50,", "60,", "70,"}; + for (String measurement : NUMERIC_MEASUREMENTS) { + expectedHeader = new String[] {measurement}; + tableResultSetEqualTest( + String.format(sql, measurement, measurement, measurement, measurement), + expectedHeader, + retArray, + DATABASE_NAME); + } + + // Test case: where s != any (subquery), s does not contain null value + sql = + "SELECT cast(%s AS INT32) as %s FROM table1 WHERE device_id = 'd01' and %s != any (SELECT %s FROM table1 WHERE device_id = 'd01')"; + retArray = new String[] {"30,", "40,", "50,", "60,", "70,"}; + for (String measurement : NUMERIC_MEASUREMENTS) { + expectedHeader = new String[] {measurement}; + tableResultSetEqualTest( + String.format(sql, measurement, measurement, measurement, measurement), + expectedHeader, + retArray, + DATABASE_NAME); + } + + // Test case: where s != some (subquery), s does not contain null value + sql = + "SELECT cast(%s AS INT32) as %s FROM table1 WHERE device_id = 'd01' and %s != some (SELECT %s FROM table1 WHERE device_id = 'd01')"; + retArray = new String[] {"30,", "40,", "50,", "60,", "70,"}; + for (String measurement : NUMERIC_MEASUREMENTS) { + expectedHeader = new String[] {measurement}; + tableResultSetEqualTest( + String.format(sql, measurement, measurement, measurement, measurement), + expectedHeader, + retArray, + DATABASE_NAME); + } + } + + @Test + public void testAllComparisonInWhereClauseWithoutNull() { + String sql; + String[] expectedHeader; + String[] retArray; + + // Test case: where s > all (subquery), s does not contain null value + sql = + "SELECT cast(%s AS INT32) as %s FROM table1 WHERE device_id = 'd01' and %s > all (SELECT %s FROM table3 WHERE device_id = 'd01')"; + retArray = new String[] {"50,", "60,", "70,"}; + for (String measurement : NUMERIC_MEASUREMENTS) { + expectedHeader = new String[] {measurement}; + tableResultSetEqualTest( + String.format(sql, measurement, measurement, measurement, measurement), + expectedHeader, + retArray, + DATABASE_NAME); + } + + // Test case: where s >= all (subquery), s does not contain null value + sql = + "SELECT cast(%s AS INT32) as %s FROM table1 WHERE device_id = 'd01' and %s >= all (SELECT %s FROM table3 WHERE device_id = 'd01')"; + retArray = new String[] {"40,", "50,", "60,", "70,"}; + for (String measurement : NUMERIC_MEASUREMENTS) { + expectedHeader = new String[] {measurement}; + tableResultSetEqualTest( + String.format(sql, measurement, measurement, measurement, measurement), + expectedHeader, + retArray, + DATABASE_NAME); + } + + // Test case: where s < all (subquery), s does not contain null value + sql = + "SELECT cast(%s AS INT32) as %s FROM table1 WHERE device_id = 'd01' and %s < all (SELECT %s FROM table3 WHERE device_id = 'd01')"; + retArray = new String[] {}; + for (String measurement : NUMERIC_MEASUREMENTS) { + expectedHeader = new String[] {measurement}; + tableResultSetEqualTest( + String.format(sql, measurement, measurement, measurement, measurement), + expectedHeader, + retArray, + DATABASE_NAME); + } + + // Test case: where s <= all (subquery), s does not contain null value + sql = + "SELECT cast(%s AS INT32) as %s FROM table1 WHERE device_id = 'd01' and %s <= all (SELECT %s FROM table3 WHERE device_id = 'd01')"; + retArray = new String[] {"30,"}; + for (String measurement : NUMERIC_MEASUREMENTS) { + expectedHeader = new String[] {measurement}; + tableResultSetEqualTest( + String.format(sql, measurement, measurement, measurement, measurement), + expectedHeader, + retArray, + DATABASE_NAME); + } + + // Test case: where s = all (subquery), s does not contain null value + sql = + "SELECT cast(%s AS INT32) as %s FROM table1 WHERE device_id = 'd01' and %s = all (SELECT %s FROM table3 WHERE device_id = 'd01')"; + retArray = new String[] {}; + for (String measurement : NUMERIC_MEASUREMENTS) { + expectedHeader = new String[] {measurement}; + tableResultSetEqualTest( + String.format(sql, measurement, measurement, measurement, measurement), + expectedHeader, + retArray, + DATABASE_NAME); + } + + // Test case: where s != all (subquery), s does not contain null value + sql = + "SELECT cast(%s AS INT32) as %s FROM table1 WHERE device_id = 'd01' and %s != all (SELECT %s FROM table3 WHERE device_id = 'd01')"; + retArray = new String[] {"50,", "60,", "70,"}; + for (String measurement : NUMERIC_MEASUREMENTS) { + expectedHeader = new String[] {measurement}; + tableResultSetEqualTest( + String.format(sql, measurement, measurement, measurement, measurement), + expectedHeader, + retArray, + DATABASE_NAME); + } + } + + @Test + public void testAnyAndSomeComparisonInWhereClauseWithNull() { + String sql; + String[] expectedHeader; + String[] retArray; + + // Test case: where s1 > any (subquery), s1 in table3 contains null value + sql = + "SELECT cast(%s AS INT32) as %s FROM table1 WHERE device_id = 'd01' and %s > any (SELECT s1 FROM table3)"; + retArray = new String[] {"40,", "50,", "60,", "70,"}; + for (String measurement : NUMERIC_MEASUREMENTS) { + expectedHeader = new String[] {measurement}; + tableResultSetEqualTest( + String.format(sql, measurement, measurement, measurement, measurement), + expectedHeader, + retArray, + DATABASE_NAME); + } + + // Test case: where s1 > some (subquery), s1 in table3 contains null value + sql = + "SELECT cast(%s AS INT32) as %s FROM table1 WHERE device_id = 'd01' and %s > some (SELECT s1 FROM table3)"; + retArray = new String[] {"40,", "50,", "60,", "70,"}; + for (String measurement : NUMERIC_MEASUREMENTS) { + expectedHeader = new String[] {measurement}; + tableResultSetEqualTest( + String.format(sql, measurement, measurement, measurement, measurement), + expectedHeader, + retArray, + DATABASE_NAME); + } + + // Test case: where s1 >= any (subquery), s1 in table3 contains null value + sql = + "SELECT cast(%s AS INT32) as %s FROM table1 WHERE device_id = 'd01' and %s >= any (SELECT s1 FROM table3)"; + retArray = new String[] {"30,", "40,", "50,", "60,", "70,"}; + for (String measurement : NUMERIC_MEASUREMENTS) { + expectedHeader = new String[] {measurement}; + tableResultSetEqualTest( + String.format(sql, measurement, measurement, measurement, measurement), + expectedHeader, + retArray, + DATABASE_NAME); + } + + // Test case: where s1 >= some (subquery), s1 in table3 contains null value + sql = + "SELECT cast(%s AS INT32) as %s FROM table1 WHERE device_id = 'd01' and %s >= some (SELECT s1 FROM table3)"; + retArray = new String[] {"30,", "40,", "50,", "60,", "70,"}; + for (String measurement : NUMERIC_MEASUREMENTS) { + expectedHeader = new String[] {measurement}; + tableResultSetEqualTest( + String.format(sql, measurement, measurement, measurement, measurement), + expectedHeader, + retArray, + DATABASE_NAME); + } + } + + @Test + public void testAllComparisonInWhereClauseWithNull() { + String sql; + String[] expectedHeader; + String[] retArray; + + // Test case: where s1 > all (subquery), s1 in table3 contains null value. + sql = + "SELECT cast(%s AS INT32) as %s FROM table1 WHERE device_id = 'd01' and %s > all (SELECT s1 FROM table3)"; + retArray = new String[] {}; + for (String measurement : NUMERIC_MEASUREMENTS) { + expectedHeader = new String[] {measurement}; + tableResultSetEqualTest( + String.format(sql, measurement, measurement, measurement, measurement), + expectedHeader, + retArray, + DATABASE_NAME); + } + + // Test case: where s1 >= all (subquery), s1 in table3 contains null value. + sql = + "SELECT cast(%s AS INT32) as %s FROM table1 WHERE device_id = 'd01' and %s >= all (SELECT s1 FROM table3)"; + retArray = new String[] {}; + for (String measurement : NUMERIC_MEASUREMENTS) { + expectedHeader = new String[] {measurement}; + tableResultSetEqualTest( + String.format(sql, measurement, measurement, measurement, measurement), + expectedHeader, + retArray, + DATABASE_NAME); + } + + // Test case: where s1 < all (subquery), s1 in table3 contains null value. + sql = + "SELECT cast(%s AS INT32) as %s FROM table1 WHERE device_id = 'd01' and %s < all (SELECT s1 FROM table3)"; + retArray = new String[] {}; + for (String measurement : NUMERIC_MEASUREMENTS) { + expectedHeader = new String[] {measurement}; + tableResultSetEqualTest( + String.format(sql, measurement, measurement, measurement, measurement), + expectedHeader, + retArray, + DATABASE_NAME); + } + + // Test case: where s1 <= all (subquery), s1 in table3 contains null value. + sql = + "SELECT cast(%s AS INT32) as %s FROM table1 WHERE device_id = 'd01' and %s <= all (SELECT s1 FROM table3)"; + retArray = new String[] {}; + for (String measurement : NUMERIC_MEASUREMENTS) { + expectedHeader = new String[] {measurement}; + tableResultSetEqualTest( + String.format(sql, measurement, measurement, measurement, measurement), + expectedHeader, + retArray, + DATABASE_NAME); + } + + // Test case: where s1 = all (subquery), s1 in table3 contains null value. + sql = + "SELECT cast(%s AS INT32) as %s FROM table1 WHERE device_id = 'd01' and %s = all (SELECT s1 FROM table3)"; + retArray = new String[] {}; + for (String measurement : NUMERIC_MEASUREMENTS) { + expectedHeader = new String[] {measurement}; + tableResultSetEqualTest( + String.format(sql, measurement, measurement, measurement, measurement), + expectedHeader, + retArray, + DATABASE_NAME); + } + + // Test case: where s1 != all (subquery), s1 in table3 contains null value. + sql = + "SELECT cast(%s AS INT32) as %s FROM table1 WHERE device_id = 'd01' and cast(%s as INT32) != all (SELECT s1 FROM table3)"; + retArray = new String[] {"50,", "60,", "70,"}; + for (String measurement : NUMERIC_MEASUREMENTS) { + expectedHeader = new String[] {measurement}; + tableResultSetEqualTest( + String.format(sql, measurement, measurement, measurement, measurement), + expectedHeader, + retArray, + DATABASE_NAME); + } + } + + @Test + public void testQuantifiedComparisonInWhereWithExpression() { + String sql; + String[] expectedHeader; + String[] retArray; + + sql = + "SELECT cast(%s + 10 AS INT32) as %s FROM table1 WHERE device_id = 'd01' and %s + 10 > any (SELECT %s + 10 FROM table1 WHERE device_id = 'd01')"; + retArray = new String[] {"50,", "60,", "70,", "80,"}; + for (String measurement : NUMERIC_MEASUREMENTS) { + expectedHeader = new String[] {measurement}; + tableResultSetEqualTest( + String.format(sql, measurement, measurement, measurement, measurement), + expectedHeader, + retArray, + DATABASE_NAME); + } + + sql = + "SELECT cast(%s + 10 AS INT32) as %s FROM table1 WHERE device_id = 'd01' and %s + 10 > some (SELECT %s + 10 FROM table1 WHERE device_id = 'd01')"; + retArray = new String[] {"50,", "60,", "70,", "80,"}; + for (String measurement : NUMERIC_MEASUREMENTS) { + expectedHeader = new String[] {measurement}; + tableResultSetEqualTest( + String.format(sql, measurement, measurement, measurement, measurement), + expectedHeader, + retArray, + DATABASE_NAME); + } + + sql = + "SELECT cast(%s + 10 AS INT32) as %s FROM table1 WHERE device_id = 'd01' and %s + 10 >= all (SELECT %s + 10 FROM table1 WHERE device_id = 'd01')"; + retArray = new String[] {"80,"}; + for (String measurement : NUMERIC_MEASUREMENTS) { + expectedHeader = new String[] {measurement}; + tableResultSetEqualTest( + String.format(sql, measurement, measurement, measurement, measurement), + expectedHeader, + retArray, + DATABASE_NAME); + } + } + + @Test + public void testQuantifiedComparisonInHavingClause() { + String sql; + String[] expectedHeader; + String[] retArray; + + // Test case: having s >= any(subquery) + sql = + "SELECT device_id, count(*) from table1 group by device_id having count(*) + 25 >= any(SELECT cast(s1 as INT64) from table3 where device_id = 'd01')"; + expectedHeader = new String[] {"device_id", "_col1"}; + retArray = + new String[] { + "d01,5,", "d03,5,", "d05,5,", "d07,5,", "d09,5,", "d11,5,", "d13,5,", "d15,5," + }; + for (String measurement : NUMERIC_MEASUREMENTS) { + tableResultSetEqualTest( + String.format(sql, measurement, measurement, measurement, measurement), + expectedHeader, + retArray, + DATABASE_NAME); + } + + // Test case: having s >= some(subquery) + sql = + "SELECT device_id, count(*) from table1 group by device_id having count(*) + 25 >= some(SELECT cast(s1 as INT64) from table3 where device_id = 'd01')"; + expectedHeader = new String[] {"device_id", "_col1"}; + retArray = + new String[] { + "d01,5,", "d03,5,", "d05,5,", "d07,5,", "d09,5,", "d11,5,", "d13,5,", "d15,5," + }; + for (String measurement : NUMERIC_MEASUREMENTS) { + tableResultSetEqualTest( + String.format(sql, measurement, measurement, measurement, measurement), + expectedHeader, + retArray, + DATABASE_NAME); + } + + // Test case: having s >= all(subquery) + sql = + "SELECT device_id, count(*) from table1 group by device_id having count(*) + 35 >= all(SELECT cast(s1 as INT64) from table3 where device_id = 'd01')"; + expectedHeader = new String[] {"device_id", "_col1"}; + retArray = + new String[] { + "d01,5,", "d03,5,", "d05,5,", "d07,5,", "d09,5,", "d11,5,", "d13,5,", "d15,5," + }; + for (String measurement : NUMERIC_MEASUREMENTS) { + tableResultSetEqualTest( + String.format(sql, measurement, measurement, measurement, measurement), + expectedHeader, + retArray, + DATABASE_NAME); + } + } + + @Test + public void testQuantifiedComparisonLegalityCheck() { + // Legality check: only support any/some/all quantifier + tableAssertTestFail( + "select s1 from table1 where s1 > any_value (select s1 from table3)", + "mismatched input", + DATABASE_NAME); + } +} diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/planner/optimizations/LogicalOptimizeFactory.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/planner/optimizations/LogicalOptimizeFactory.java index d95d246a20a9..b3766383c180 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/planner/optimizations/LogicalOptimizeFactory.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/planner/optimizations/LogicalOptimizeFactory.java @@ -216,6 +216,13 @@ public LogicalOptimizeFactory(PlannerContext plannerContext) { new RemoveRedundantEnforceSingleRowNode(), new RemoveUnreferencedScalarSubqueries(), new TransformUncorrelatedSubqueryToJoin(), new TransformUncorrelatedInPredicateSubqueryToSemiJoin())), + new IterativeOptimizer( + plannerContext, + ruleStats, + ImmutableSet.of( + new InlineProjections(plannerContext), new RemoveRedundantIdentityProjections() + /*new TransformCorrelatedSingleRowSubqueryToProject(), + new RemoveAggregationInSemiJoin())*/ )), new CheckSubqueryNodesAreRewritten(), new IterativeOptimizer( plannerContext, ruleStats, ImmutableSet.of(new PruneDistinctAggregation())), From 3280f84092e1605c7b44511e0befb6c15acd3762 Mon Sep 17 00:00:00 2001 From: lancelly <1435078631@qq.com> Date: Wed, 18 Dec 2024 18:25:58 +0800 Subject: [PATCH 17/26] fix semi join operator with right sort null first --- ...oTDBUncorrelatedInPredicateSubqueryIT.java | 22 +++++- .../relational/MergeSortSemiJoinOperator.java | 67 ++++++++++++------- .../PushPredicateIntoTableScan.java | 7 +- 3 files changed, 68 insertions(+), 28 deletions(-) diff --git a/integration-test/src/test/java/org/apache/iotdb/relational/it/query/recent/subquery/uncorrelated/IoTDBUncorrelatedInPredicateSubqueryIT.java b/integration-test/src/test/java/org/apache/iotdb/relational/it/query/recent/subquery/uncorrelated/IoTDBUncorrelatedInPredicateSubqueryIT.java index a4accf01fb12..f49e2320dcf7 100644 --- a/integration-test/src/test/java/org/apache/iotdb/relational/it/query/recent/subquery/uncorrelated/IoTDBUncorrelatedInPredicateSubqueryIT.java +++ b/integration-test/src/test/java/org/apache/iotdb/relational/it/query/recent/subquery/uncorrelated/IoTDBUncorrelatedInPredicateSubqueryIT.java @@ -152,10 +152,16 @@ public void testInPredicateSubqueryInWhereClause() { } // Test case: where s in (subquery), s contains null value - sql = "SELECT s1 FROM table3 WHERE device_id = 'd_null' and s1 in (SELECT (s1) from table3)"; + sql = "SELECT s1 FROM table3 WHERE device_id = 'd_null' and s1 in (SELECT s1 from table3)"; expectedHeader = new String[] {"s1"}; retArray = new String[] {"30,"}; tableResultSetEqualTest(sql, expectedHeader, retArray, DATABASE_NAME); + + // Test case: where s not in (subquery), s contains null value, the resutl should be empty + sql = "SELECT s1 FROM table3 WHERE device_id = 'd_null' and s1 not in (SELECT s1 from table3)"; + expectedHeader = new String[] {"s1"}; + retArray = new String[] {}; + tableResultSetEqualTest(sql, expectedHeader, retArray, DATABASE_NAME); } @Test @@ -258,6 +264,20 @@ public void testInPredicateSubqueryInSelectClause() { expectedHeader = new String[] {"_col0"}; retArray = new String[] {"true,", "null,"}; tableResultSetEqualTest(sql, expectedHeader, retArray, DATABASE_NAME); + + // Test case: select s in (subquery), s contains null value. The result should also be null when + // s is null. + sql = "SELECT s1 not in (SELECT s1 from table3) from table3 where device_id = 'd_null'"; + expectedHeader = new String[] {"_col0"}; + retArray = new String[] {"false,", "null,"}; + tableResultSetEqualTest(sql, expectedHeader, retArray, DATABASE_NAME); + + // Test case: select s in (subquery), s contains null value. 36 not in(30, 40, null) returns + // null + sql = "SELECT s1 not in (SELECT s1 from table3) from table1 where device_id = 'd02'"; + expectedHeader = new String[] {"_col0"}; + retArray = new String[] {"null,", "false,", "null,"}; + tableResultSetEqualTest(sql, expectedHeader, retArray, DATABASE_NAME); } @Test diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/execution/operator/source/relational/MergeSortSemiJoinOperator.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/execution/operator/source/relational/MergeSortSemiJoinOperator.java index 48daa48525e5..b21ed578965f 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/execution/operator/source/relational/MergeSortSemiJoinOperator.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/execution/operator/source/relational/MergeSortSemiJoinOperator.java @@ -37,6 +37,8 @@ public class MergeSortSemiJoinOperator extends AbstractMergeSortJoinOperator { private final int outputColumnNum; + private boolean rightHasNullValue = false; + public MergeSortSemiJoinOperator( OperatorContext operatorContext, Operator leftChild, @@ -83,27 +85,27 @@ protected boolean processFinished() { appendAllLeftBlock(); return true; } + + // skip all NULL values in right, because NULL value will not match the left value + while (currentRightHasNullValue()) { + rightHasNullValue = true; + if (rightFinishedWithIncIndex()) { + return true; + } + } // all the join keys in rightTsBlock are less than leftTsBlock, just skip right if (allRightLessThanLeft()) { resetRightBlockList(); return true; } - // all the join Keys in leftTsBlock are less than rightTsBlock, just append the left value with - // match flag as false + // all the join Keys in leftTsBlock are less than rightTsBlock, just append the left value if (allLeftLessThanRight()) { appendAllLeftBlock(); resetLeftBlock(); return true; } - // skip all NULL values in right, because NULL value will not match the left value - while (currentRightHasNullValue()) { - if (rightFinishedWithIncIndex()) { - return true; - } - } - // continue right < left, until right >= left while (lessThan( rightBlockList.get(rightBlockListIdx), @@ -136,8 +138,7 @@ protected boolean processFinished() { rightBlockList.get(rightBlockListIdx), rightJoinKeyPositions, rightIndex)) { - // current left won't match any right, append left with false SemiJoin result - appendValueToResult(false); + appendWhenNotMatch(); leftIndex++; if (leftIndex >= leftBlock.getPositionCount()) { resetLeftBlock(); @@ -162,36 +163,50 @@ protected boolean hasMatchedRightValueToProbeLeft() { rightBlockList.get(rightBlockListIdx), rightJoinKeyPositions, rightIndex); - appendValueToResult(matches); + if (matches) { + appendValueToResult(true); + } else { + appendWhenNotMatch(); + } + return matches; } - private void appendValueToResult(boolean matches) { - appendLeftBlockData(leftOutputSymbolIdx, resultBuilder, leftBlock, leftIndex); - appendSemiJoinOutput(matches); - resultBuilder.declarePosition(); + private void appendWhenNotMatch() { + // current left won't match any right, append left with false SemiJoin result + if (!rightHasNullValue) { + appendValueToResult(false); + } else { + // if right has null value, append null to result. This behaves like MySQL and Trino. + appendNullValueToResult(); + } } - private void appendSemiJoinOutput(boolean value) { + private void appendValueToResult(boolean matches) { + appendLeftBlockData(leftOutputSymbolIdx, resultBuilder, leftBlock, leftIndex); ColumnBuilder columnBuilder = resultBuilder.getColumnBuilder(outputColumnNum - 1); - columnBuilder.writeBoolean(value); + columnBuilder.writeBoolean(matches); + resultBuilder.declarePosition(); } private void appendNullValueToResult() { appendLeftBlockData(leftOutputSymbolIdx, resultBuilder, leftBlock, leftIndex); - appendNullToSemiJoinOutput(); - resultBuilder.declarePosition(); - } - - private void appendNullToSemiJoinOutput() { ColumnBuilder columnBuilder = resultBuilder.getColumnBuilder(outputColumnNum - 1); columnBuilder.appendNull(); + resultBuilder.declarePosition(); } private void appendAllLeftBlock() { - while (leftBlockNotEmpty()) { - appendValueToResult(false); - leftIndex++; + if (rightHasNullValue) { + while (leftBlockNotEmpty()) { + appendNullValueToResult(); + leftIndex++; + } + } else { + while (leftBlockNotEmpty()) { + appendValueToResult(false); + leftIndex++; + } } } diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/planner/optimizations/PushPredicateIntoTableScan.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/planner/optimizations/PushPredicateIntoTableScan.java index f5313b1a5374..9cb71138a123 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/planner/optimizations/PushPredicateIntoTableScan.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/planner/optimizations/PushPredicateIntoTableScan.java @@ -88,6 +88,7 @@ import static org.apache.iotdb.db.queryengine.metric.QueryPlanCostMetricSet.SCHEMA_FETCHER; import static org.apache.iotdb.db.queryengine.metric.QueryPlanCostMetricSet.TABLE_TYPE; import static org.apache.iotdb.db.queryengine.plan.analyze.AnalyzeVisitor.getTimePartitionSlotList; +import static org.apache.iotdb.db.queryengine.plan.relational.planner.SortOrder.ASC_NULLS_FIRST; import static org.apache.iotdb.db.queryengine.plan.relational.planner.SortOrder.ASC_NULLS_LAST; import static org.apache.iotdb.db.queryengine.plan.relational.planner.SymbolsExtractor.extractUnique; import static org.apache.iotdb.db.queryengine.plan.relational.planner.ir.DeterminismEvaluator.isDeterministic; @@ -810,7 +811,11 @@ private SemiJoinNode appendSortNodeForSemiJoin( OrderingScheme filteringSourceOrderingScheme = new OrderingScheme( ImmutableList.of(node.getFilteringSourceJoinSymbol()), - ImmutableMap.of(node.getFilteringSourceJoinSymbol(), ASC_NULLS_LAST)); + // NULL first is used to make sure that we can know if there's null value in the + // result set of right table. + // For x in (subquery), if subquery returns null and some value a,b, and x is not + // in(a,b), the result of SemiJoinOutput should be NULL. + ImmutableMap.of(node.getFilteringSourceJoinSymbol(), ASC_NULLS_FIRST)); SortNode sourceSortNode = new SortNode( queryId.genPlanNodeId(), rewrittenSource, sourceOrderingScheme, false, false); From 7ae6f9ed69b22a717aa31d431a52f5fb2fc03115 Mon Sep 17 00:00:00 2001 From: lancelly <1435078631@qq.com> Date: Wed, 18 Dec 2024 19:48:22 +0800 Subject: [PATCH 18/26] add it for != all with null value --- ...TDBUncorrelatedQuantifiedComparisonIT.java | 120 +++++++++++++++++- 1 file changed, 119 insertions(+), 1 deletion(-) diff --git a/integration-test/src/test/java/org/apache/iotdb/relational/it/query/recent/subquery/uncorrelated/IoTDBUncorrelatedQuantifiedComparisonIT.java b/integration-test/src/test/java/org/apache/iotdb/relational/it/query/recent/subquery/uncorrelated/IoTDBUncorrelatedQuantifiedComparisonIT.java index 25dc92645f06..37c6eaafc351 100644 --- a/integration-test/src/test/java/org/apache/iotdb/relational/it/query/recent/subquery/uncorrelated/IoTDBUncorrelatedQuantifiedComparisonIT.java +++ b/integration-test/src/test/java/org/apache/iotdb/relational/it/query/recent/subquery/uncorrelated/IoTDBUncorrelatedQuantifiedComparisonIT.java @@ -359,6 +359,32 @@ public void testAnyAndSomeComparisonInWhereClauseWithNull() { retArray, DATABASE_NAME); } + + // Test case: where s1 < any (subquery), s1 in table3 contains null value + sql = + "SELECT cast(%s AS INT32) as %s FROM table1 WHERE device_id = 'd01' and %s < any (SELECT s1 FROM table3)"; + retArray = new String[] {"30,"}; + for (String measurement : NUMERIC_MEASUREMENTS) { + expectedHeader = new String[] {measurement}; + tableResultSetEqualTest( + String.format(sql, measurement, measurement, measurement, measurement), + expectedHeader, + retArray, + DATABASE_NAME); + } + + // Test case: where s1 < some (subquery), s1 in table3 contains null value + sql = + "SELECT cast(%s AS INT32) as %s FROM table1 WHERE device_id = 'd01' and %s < some (SELECT s1 FROM table3)"; + retArray = new String[] {"30,"}; + for (String measurement : NUMERIC_MEASUREMENTS) { + expectedHeader = new String[] {measurement}; + tableResultSetEqualTest( + String.format(sql, measurement, measurement, measurement, measurement), + expectedHeader, + retArray, + DATABASE_NAME); + } } @Test @@ -435,7 +461,7 @@ public void testAllComparisonInWhereClauseWithNull() { // Test case: where s1 != all (subquery), s1 in table3 contains null value. sql = "SELECT cast(%s AS INT32) as %s FROM table1 WHERE device_id = 'd01' and cast(%s as INT32) != all (SELECT s1 FROM table3)"; - retArray = new String[] {"50,", "60,", "70,"}; + retArray = new String[] {}; for (String measurement : NUMERIC_MEASUREMENTS) { expectedHeader = new String[] {measurement}; tableResultSetEqualTest( @@ -544,6 +570,98 @@ public void testQuantifiedComparisonInHavingClause() { } } + public void testQuantifiedComparisonInSelectClause() { + String sql; + String[] expectedHeader; + String[] retArray; + + // Test case: select s > any(subquery) + sql = + "SELECT %s > any(SELECT (%s) from table3 WHERE device_id = 'd01') from table1 where device_id = 'd01'"; + expectedHeader = new String[] {"_col0"}; + retArray = new String[] {"false,", "true,", "true,", "true,", "true,"}; + for (String measurement : NUMERIC_MEASUREMENTS) { + tableResultSetEqualTest( + String.format(sql, measurement, measurement), expectedHeader, retArray, DATABASE_NAME); + } + + // Test case: select s > some(subquery) + sql = + "SELECT %s > some(SELECT (%s) from table3 WHERE device_id = 'd01') from table1 where device_id = 'd01'"; + expectedHeader = new String[] {"_col0"}; + retArray = new String[] {"false,", "true,", "true,", "true,", "true,"}; + for (String measurement : NUMERIC_MEASUREMENTS) { + tableResultSetEqualTest( + String.format(sql, measurement, measurement), expectedHeader, retArray, DATABASE_NAME); + } + + // Test case: select s > all(subquery) + sql = + "SELECT %s > all(SELECT (%s) from table3 WHERE device_id = 'd01') from table1 where device_id = 'd01'"; + expectedHeader = new String[] {"_col0"}; + retArray = new String[] {"false,", "false,", "false,", "false,", "false,"}; + for (String measurement : NUMERIC_MEASUREMENTS) { + tableResultSetEqualTest( + String.format(sql, measurement, measurement), expectedHeader, retArray, DATABASE_NAME); + } + + // Test case: select s < any(subquery), subquery contains null value + sql = "SELECT %s < any(SELECT (%s) from table3) from table1 where device_id = 'd01'"; + expectedHeader = new String[] {"_col0"}; + retArray = new String[] {"null,", "null,", "null,", "null,", "null,"}; + for (String measurement : NUMERIC_MEASUREMENTS) { + tableResultSetEqualTest( + String.format(sql, measurement, measurement), expectedHeader, retArray, DATABASE_NAME); + } + + // Test case: select s < some(subquery), subquery contains null value + sql = "SELECT %s < some(SELECT (%s) from table3) from table1 where device_id = 'd01'"; + expectedHeader = new String[] {"_col0"}; + retArray = new String[] {"null,", "null,", "null,", "null,", "null,"}; + for (String measurement : NUMERIC_MEASUREMENTS) { + tableResultSetEqualTest( + String.format(sql, measurement, measurement), expectedHeader, retArray, DATABASE_NAME); + } + + // Test case: select s <= any(subquery), subquery contains null value + sql = "SELECT %s <= any(SELECT (%s) from table3) from table1 where device_id = 'd01'"; + expectedHeader = new String[] {"_col0"}; + retArray = new String[] {"true,", "null,", "null,", "null,", "null,"}; + for (String measurement : NUMERIC_MEASUREMENTS) { + tableResultSetEqualTest( + String.format(sql, measurement, measurement), expectedHeader, retArray, DATABASE_NAME); + } + + // Test case: select s <= some(subquery), subquery contains null value + sql = "SELECT %s <= some(SELECT (%s) from table3) from table1 where device_id = 'd01'"; + expectedHeader = new String[] {"_col0"}; + retArray = new String[] {"true,", "null,", "null,", "null,", "null,"}; + for (String measurement : NUMERIC_MEASUREMENTS) { + tableResultSetEqualTest( + String.format(sql, measurement, measurement), expectedHeader, retArray, DATABASE_NAME); + } + + // Test case: select s != all(subquery), subquery contains null value + sql = "SELECT %s != all(SELECT (%s) from table3) from table1 where device_id = 'd01'"; + expectedHeader = new String[] {"_col0"}; + retArray = new String[] {"false,", "false,", "null,", "null,", "null,"}; + for (String measurement : NUMERIC_MEASUREMENTS) { + tableResultSetEqualTest( + String.format(sql, measurement, measurement), expectedHeader, retArray, DATABASE_NAME); + } + + // Test case: select s != all(subquery), subquery contains null value and s not in non-null + // value result set + sql = + "SELECT %s != all(SELECT (%s) from table3 where device_id = 'd_null') from table1 where device_id = 'd02' and %s != 30"; + expectedHeader = new String[] {"_col0"}; + retArray = new String[] {"null,", "null,"}; + for (String measurement : NUMERIC_MEASUREMENTS) { + tableResultSetEqualTest( + String.format(sql, measurement, measurement), expectedHeader, retArray, DATABASE_NAME); + } + } + @Test public void testQuantifiedComparisonLegalityCheck() { // Legality check: only support any/some/all quantifier From 2048eec7132e272df5bf8aa629b2113f87ac3786 Mon Sep 17 00:00:00 2001 From: lancelly <1435078631@qq.com> Date: Thu, 19 Dec 2024 10:23:34 +0800 Subject: [PATCH 19/26] add some UTs --- .../plan/relational/planner/SubqueryTest.java | 212 ++++++++++++++++++ 1 file changed, 212 insertions(+) diff --git a/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/queryengine/plan/relational/planner/SubqueryTest.java b/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/queryengine/plan/relational/planner/SubqueryTest.java index 52321265280b..96dbe9d2b149 100644 --- a/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/queryengine/plan/relational/planner/SubqueryTest.java +++ b/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/queryengine/plan/relational/planner/SubqueryTest.java @@ -56,6 +56,7 @@ import static org.apache.iotdb.db.queryengine.plan.relational.planner.node.AggregationNode.Step.FINAL; import static org.apache.iotdb.db.queryengine.plan.relational.planner.node.AggregationNode.Step.INTERMEDIATE; import static org.apache.iotdb.db.queryengine.plan.relational.planner.node.AggregationNode.Step.PARTIAL; +import static org.apache.iotdb.db.queryengine.plan.relational.planner.node.AggregationNode.Step.SINGLE; import static org.apache.iotdb.db.queryengine.plan.relational.sql.ast.ComparisonExpression.Operator.EQUAL; public class SubqueryTest { @@ -322,4 +323,215 @@ public void testUncorrelatedNotInPredicateSubquery() { filterPredicate, semiJoin("s1", "s1_6", "expr", sort(tableScan1), sort(tableScan2)))))); } + + @Test + public void testUncorrelatedAnyComparisonSubquery() { + PlanTester planTester = new PlanTester(); + + String sql = "SELECT s1 FROM table1 where s1 > any (select s1 from table1)"; + + LogicalQueryPlan logicalQueryPlan = planTester.createPlan(sql); + + PlanMatchPattern tableScan1 = + tableScan("testdb.table1", ImmutableList.of("s1"), ImmutableSet.of("s1")); + + PlanMatchPattern tableScan2 = tableScan("testdb.table1", ImmutableMap.of("s1_7", "s1")); + + PlanMatchPattern tableScan3 = tableScan("testdb.table1", ImmutableMap.of("s1_6", "s1")); + + // Verify full LogicalPlan + /* + * └──OutputNode + * └──ProjectNode + * └──FilterNode + * └──ProjectNode + * └──JoinNode + * |──TableScanNode + * ├──AggregationNode + * │ └──TableScanNode + + */ + assertPlan( + logicalQueryPlan, + output( + project( + anyTree( + project( + join( + JoinNode.JoinType.INNER, + builder -> + builder + .left(tableScan1) + .right( + aggregation( + singleGroupingSet(), + ImmutableMap.of( + Optional.of("min"), + aggregationFunction( + "min", ImmutableList.of("s1_7")), + Optional.of("count_all"), + aggregationFunction( + "count_all", ImmutableList.of("s1_7")), + Optional.of("count_non_null"), + aggregationFunction( + "count", ImmutableList.of("s1_7"))), + Collections.emptyList(), + Optional.empty(), + SINGLE, + tableScan2)))))))); + + // Verify DistributionPlan + assertPlan( + planTester.getFragmentPlan(0), + output( + project( + anyTree( + project( + join( + JoinNode.JoinType.INNER, + builder -> + builder + .left(collect(exchange(), tableScan1, exchange())) + .right( + aggregation( + singleGroupingSet(), + ImmutableMap.of( + Optional.of("min"), + aggregationFunction( + "min", ImmutableList.of("min_9")), + Optional.of("count_all"), + aggregationFunction( + "count_all", ImmutableList.of("count_all_10")), + Optional.of("count_non_null"), + aggregationFunction( + "count", ImmutableList.of("count"))), + Collections.emptyList(), + Optional.empty(), + FINAL, + collect( + exchange(), + aggregation( + singleGroupingSet(), + ImmutableMap.of( + Optional.of("min_9"), + aggregationFunction( + "min", ImmutableList.of("s1_6")), + Optional.of("count_all_10"), + aggregationFunction( + "count_all", ImmutableList.of("s1_6")), + Optional.of("count"), + aggregationFunction( + "count", ImmutableList.of("s1_6"))), + Collections.emptyList(), + Optional.empty(), + PARTIAL, + tableScan3), + exchange()))))))))); + + assertPlan(planTester.getFragmentPlan(1), tableScan1); + + assertPlan(planTester.getFragmentPlan(2), tableScan1); + + assertPlan( + planTester.getFragmentPlan(3), + aggregation( + singleGroupingSet(), + ImmutableMap.of( + Optional.of("min_9"), + aggregationFunction("min", ImmutableList.of("s1_6")), + Optional.of("count_all_10"), + aggregationFunction("count_all", ImmutableList.of("s1_6")), + Optional.of("count"), + aggregationFunction("count", ImmutableList.of("s1_6"))), + Collections.emptyList(), + Optional.empty(), + PARTIAL, + tableScan3)); + + assertPlan( + planTester.getFragmentPlan(4), + aggregation( + singleGroupingSet(), + ImmutableMap.of( + Optional.of("min_9"), + aggregationFunction("min", ImmutableList.of("s1_6")), + Optional.of("count_all_10"), + aggregationFunction("count_all", ImmutableList.of("s1_6")), + Optional.of("count"), + aggregationFunction("count", ImmutableList.of("s1_6"))), + Collections.emptyList(), + Optional.empty(), + PARTIAL, + tableScan3)); + } + + @Test + public void testUncorrelatedEqualsSomeComparisonSubquery() { + PlanTester planTester = new PlanTester(); + + String sql = "SELECT s1 FROM table1 where s1 = some (select s1 from table1)"; + + LogicalQueryPlan logicalQueryPlan = planTester.createPlan(sql); + + Expression filterPredicate = new SymbolReference("expr"); + + PlanMatchPattern tableScan1 = + tableScan("testdb.table1", ImmutableList.of("s1"), ImmutableSet.of("s1")); + + PlanMatchPattern tableScan2 = tableScan("testdb.table1", ImmutableMap.of("s1_6", "s1")); + + // Verify full LogicalPlan + /* + * └──OutputNode + * └──ProjectNode + * └──FilterNode + * └──SemiJoinNode + * |──SortNode + * | └──TableScanNode + * ├──SortNode + * │ └──TableScanNode + + */ + assertPlan( + logicalQueryPlan, + output( + project( + filter( + filterPredicate, + semiJoin("s1", "s1_6", "expr", sort(tableScan1), sort(tableScan2)))))); + } + + @Test + public void testUncorrelatedAllComparisonSubquery() { + PlanTester planTester = new PlanTester(); + + String sql = "SELECT s1 FROM table1 where s1 != all (select s1 from table1)"; + + LogicalQueryPlan logicalQueryPlan = planTester.createPlan(sql); + + PlanMatchPattern tableScan1 = + tableScan("testdb.table1", ImmutableList.of("s1"), ImmutableSet.of("s1")); + + PlanMatchPattern tableScan2 = tableScan("testdb.table1", ImmutableMap.of("s1_6", "s1")); + + // Verify full LogicalPlan + /* + * └──OutputNode + * └──ProjectNode + * └──FilterNode + * └──ProjectNode + * └──SemiJoinNode + * |──SortNode + * | └──TableScanNode + * ├──SortNode + * │ └──TableScanNode + + */ + assertPlan( + logicalQueryPlan, + output( + project( + anyTree( + project(semiJoin("s1", "s1_6", "expr", sort(tableScan1), sort(tableScan2))))))); + } } From 723775b0c43b1b5467658a22655070351b229a3e Mon Sep 17 00:00:00 2001 From: lancelly <1435078631@qq.com> Date: Tue, 24 Dec 2024 13:26:50 +0800 Subject: [PATCH 20/26] IT --- .../uncorrelated/IoTDBUncorrelatedQuantifiedComparisonIT.java | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/integration-test/src/test/java/org/apache/iotdb/relational/it/query/recent/subquery/uncorrelated/IoTDBUncorrelatedQuantifiedComparisonIT.java b/integration-test/src/test/java/org/apache/iotdb/relational/it/query/recent/subquery/uncorrelated/IoTDBUncorrelatedQuantifiedComparisonIT.java index 37c6eaafc351..11eb1feab901 100644 --- a/integration-test/src/test/java/org/apache/iotdb/relational/it/query/recent/subquery/uncorrelated/IoTDBUncorrelatedQuantifiedComparisonIT.java +++ b/integration-test/src/test/java/org/apache/iotdb/relational/it/query/recent/subquery/uncorrelated/IoTDBUncorrelatedQuantifiedComparisonIT.java @@ -650,7 +650,8 @@ public void testQuantifiedComparisonInSelectClause() { String.format(sql, measurement, measurement), expectedHeader, retArray, DATABASE_NAME); } - // Test case: select s != all(subquery), subquery contains null value and s not in non-null + // Test case: select s != all(subquery), subquery result contains null value and s not in + // non-null // value result set sql = "SELECT %s != all(SELECT (%s) from table3 where device_id = 'd_null') from table1 where device_id = 'd02' and %s != 30"; From 4c2986079acfe2c00e662c6ad1d6843ca57c66d3 Mon Sep 17 00:00:00 2001 From: lancelly <1435078631@qq.com> Date: Fri, 27 Dec 2024 15:55:50 +0800 Subject: [PATCH 21/26] merge master --- .../queryengine/plan/relational/sql/parser/AstBuilder.java | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/sql/parser/AstBuilder.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/sql/parser/AstBuilder.java index d3b38af0f581..821b9c50e43c 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/sql/parser/AstBuilder.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/sql/parser/AstBuilder.java @@ -1855,8 +1855,7 @@ public Node visitInList(RelationalSqlParser.InListContext ctx) { @Override public Node visitInSubquery(RelationalSqlParser.InSubqueryContext ctx) { - throw new SemanticException("Only TableSubquery is supported now"); - /*Expression result = + Expression result = new InPredicate( getLocation(ctx), (Expression) visit(ctx.value), @@ -1866,7 +1865,7 @@ public Node visitInSubquery(RelationalSqlParser.InSubqueryContext ctx) { result = new NotExpression(getLocation(ctx), result); } - return result;*/ + return result; } @Override From ed4e416aebc7e4043e992635a89cc0cd65f84754 Mon Sep 17 00:00:00 2001 From: lancelly <1435078631@qq.com> Date: Wed, 1 Jan 2025 11:48:31 +0800 Subject: [PATCH 22/26] 301 -> 701 error code --- .../uncorrelated/IoTDBUncorrelatedInPredicateSubqueryIT.java | 2 +- .../uncorrelated/IoTDBUncorrelatedScalarSubqueryIT.java | 2 +- .../execution/operator/process/EnforceSingleRowOperator.java | 5 +++-- 3 files changed, 5 insertions(+), 4 deletions(-) diff --git a/integration-test/src/test/java/org/apache/iotdb/relational/it/query/recent/subquery/uncorrelated/IoTDBUncorrelatedInPredicateSubqueryIT.java b/integration-test/src/test/java/org/apache/iotdb/relational/it/query/recent/subquery/uncorrelated/IoTDBUncorrelatedInPredicateSubqueryIT.java index f49e2320dcf7..232c0060a2cf 100644 --- a/integration-test/src/test/java/org/apache/iotdb/relational/it/query/recent/subquery/uncorrelated/IoTDBUncorrelatedInPredicateSubqueryIT.java +++ b/integration-test/src/test/java/org/apache/iotdb/relational/it/query/recent/subquery/uncorrelated/IoTDBUncorrelatedInPredicateSubqueryIT.java @@ -285,7 +285,7 @@ public void testInPredicateSubqueryLegalityCheck() { // Legality check: Multiple parentheses around subquery, this behaves the same as Trino tableAssertTestFail( "select s1 from table1 where device_id = 'd01' and s1 not in ((select s1 from table3 where device_id = 'd01'))", - "301: Scalar sub-query has returned multiple rows.", + "701: Scalar sub-query has returned multiple rows.", DATABASE_NAME); // Legality check: Join key type mismatch.(left key is int and right key is double) diff --git a/integration-test/src/test/java/org/apache/iotdb/relational/it/query/recent/subquery/uncorrelated/IoTDBUncorrelatedScalarSubqueryIT.java b/integration-test/src/test/java/org/apache/iotdb/relational/it/query/recent/subquery/uncorrelated/IoTDBUncorrelatedScalarSubqueryIT.java index b715b5cfeb97..d2bce2e0d4ae 100644 --- a/integration-test/src/test/java/org/apache/iotdb/relational/it/query/recent/subquery/uncorrelated/IoTDBUncorrelatedScalarSubqueryIT.java +++ b/integration-test/src/test/java/org/apache/iotdb/relational/it/query/recent/subquery/uncorrelated/IoTDBUncorrelatedScalarSubqueryIT.java @@ -262,7 +262,7 @@ public void testScalarSubqueryAfterComparisonLegalityCheck() { // Legality check: subquery returns multiple rows (should fail) tableAssertTestFail( "select s1 from table1 where s1 = (select s1 from table1)", - "301: Scalar sub-query has returned multiple rows.", + "701: Scalar sub-query has returned multiple rows.", DATABASE_NAME); // Legality check: subquery can not be parsed diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/execution/operator/process/EnforceSingleRowOperator.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/execution/operator/process/EnforceSingleRowOperator.java index e9f6fba26f68..bbc9b2c155ca 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/execution/operator/process/EnforceSingleRowOperator.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/execution/operator/process/EnforceSingleRowOperator.java @@ -19,6 +19,7 @@ package org.apache.iotdb.db.queryengine.execution.operator.process; +import org.apache.iotdb.db.exception.sql.SemanticException; import org.apache.iotdb.db.queryengine.execution.MemoryEstimationHelper; import org.apache.iotdb.db.queryengine.execution.operator.Operator; import org.apache.iotdb.db.queryengine.execution.operator.OperatorContext; @@ -59,7 +60,7 @@ public TsBlock next() throws Exception { return tsBlock; } if (tsBlock.getPositionCount() > 1 || finished) { - throw new IllegalStateException(MULTIPLE_ROWS_ERROR_MESSAGE); + throw new SemanticException(MULTIPLE_ROWS_ERROR_MESSAGE); } finished = true; return tsBlock; @@ -83,7 +84,7 @@ public boolean isFinished() throws Exception { if (childFinished && !finished) { // finished == false means the child has no result returned up to now, but we need at least // one result. - throw new IllegalStateException(NO_RESULT_ERROR_MESSAGE); + throw new SemanticException(NO_RESULT_ERROR_MESSAGE); } // Even if finished == true, we can not return true here, we need to call child.next() to check // if child has more data. From ecf580dcc88a27362159b9f39da7d71ace68ea86 Mon Sep 17 00:00:00 2001 From: lancelly <1435078631@qq.com> Date: Wed, 1 Jan 2025 17:32:46 +0800 Subject: [PATCH 23/26] fix it sql --- .../it/query/recent/subquery/SubqueryDataSetUtils.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/integration-test/src/test/java/org/apache/iotdb/relational/it/query/recent/subquery/SubqueryDataSetUtils.java b/integration-test/src/test/java/org/apache/iotdb/relational/it/query/recent/subquery/SubqueryDataSetUtils.java index 80eab629c8fb..0e6e2124f328 100644 --- a/integration-test/src/test/java/org/apache/iotdb/relational/it/query/recent/subquery/SubqueryDataSetUtils.java +++ b/integration-test/src/test/java/org/apache/iotdb/relational/it/query/recent/subquery/SubqueryDataSetUtils.java @@ -105,7 +105,7 @@ public class SubqueryDataSetUtils { "INSERT INTO table2(time,device_id,s1,s2,s3,s4,s5) " + " values(5, 'd1', 5, 55, 5.5, 55.5, false)", // table3 - "CREATE TABLE table3(device_id STRING ID, s1 INT32 MEASUREMENT, s2 INT64 MEASUREMENT, s3 FLOAT MEASUREMENT, s4 DOUBLE MEASUREMENT, s5 BOOLEAN MEASUREMENT, s6 TEXT MEASUREMENT, s7 STRING MEASUREMENT, s8 BLOB MEASUREMENT, s9 TIMESTAMP MEASUREMENT, s10 DATE MEASUREMENT)", + "CREATE TABLE table3(device_id STRING TAG, s1 INT32 FIELD, s2 INT64 FIELD, s3 FLOAT FIELD, s4 DOUBLE FIELD, s5 BOOLEAN FIELD, s6 TEXT FIELD, s7 STRING FIELD, s8 BLOB FIELD, s9 TIMESTAMP FIELD, s10 DATE FIELD)", "INSERT INTO table3(time,device_id,s1,s2,s3,s4,s5,s6,s7,s8,s9,s10) values (2024-09-24T06:13:30.000+00:00,'d01',30,30,30.0,30.0,true,'shanghai_huangpu_red_A_d01_30','shanghai_huangpu_red_A_d01_30',X'cafebabe30',2024-09-24T06:13:00.000+00:00,'2024-09-23')", "INSERT INTO table3(time,device_id,s1,s2,s3,s4,s5,s6,s7,s8,s9,s10) values (2024-09-24T06:14:30.000+00:00,'d01',40,40,40.0,40.0,false,'shanghai_huangpu_red_A_d01_40','shanghai_huangpu_red_A_d01_40',X'cafebabe40',2024-09-24T06:14:00.000+00:00,'2024-09-24')", "INSERT INTO table3(time,device_id,s1,s2,s3,s4,s5,s6,s7,s8,s9,s10) values (2024-09-24T06:13:30.000+00:00,'d_null',30,30,30.0,30.0,true,'shanghai_huangpu_red_A_d01_30','shanghai_huangpu_red_A_d01_30',X'cafebabe30',2024-09-24T06:13:00.000+00:00,'2024-09-23')", From 395313fec98c4f38dc02e34c1213f40a1bf54d77 Mon Sep 17 00:00:00 2001 From: lancelly <1435078631@qq.com> Date: Wed, 1 Jan 2025 17:33:57 +0800 Subject: [PATCH 24/26] fix it sql --- .../it/query/recent/subquery/SubqueryDataSetUtils.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/integration-test/src/test/java/org/apache/iotdb/relational/it/query/recent/subquery/SubqueryDataSetUtils.java b/integration-test/src/test/java/org/apache/iotdb/relational/it/query/recent/subquery/SubqueryDataSetUtils.java index 80eab629c8fb..0e6e2124f328 100644 --- a/integration-test/src/test/java/org/apache/iotdb/relational/it/query/recent/subquery/SubqueryDataSetUtils.java +++ b/integration-test/src/test/java/org/apache/iotdb/relational/it/query/recent/subquery/SubqueryDataSetUtils.java @@ -105,7 +105,7 @@ public class SubqueryDataSetUtils { "INSERT INTO table2(time,device_id,s1,s2,s3,s4,s5) " + " values(5, 'd1', 5, 55, 5.5, 55.5, false)", // table3 - "CREATE TABLE table3(device_id STRING ID, s1 INT32 MEASUREMENT, s2 INT64 MEASUREMENT, s3 FLOAT MEASUREMENT, s4 DOUBLE MEASUREMENT, s5 BOOLEAN MEASUREMENT, s6 TEXT MEASUREMENT, s7 STRING MEASUREMENT, s8 BLOB MEASUREMENT, s9 TIMESTAMP MEASUREMENT, s10 DATE MEASUREMENT)", + "CREATE TABLE table3(device_id STRING TAG, s1 INT32 FIELD, s2 INT64 FIELD, s3 FLOAT FIELD, s4 DOUBLE FIELD, s5 BOOLEAN FIELD, s6 TEXT FIELD, s7 STRING FIELD, s8 BLOB FIELD, s9 TIMESTAMP FIELD, s10 DATE FIELD)", "INSERT INTO table3(time,device_id,s1,s2,s3,s4,s5,s6,s7,s8,s9,s10) values (2024-09-24T06:13:30.000+00:00,'d01',30,30,30.0,30.0,true,'shanghai_huangpu_red_A_d01_30','shanghai_huangpu_red_A_d01_30',X'cafebabe30',2024-09-24T06:13:00.000+00:00,'2024-09-23')", "INSERT INTO table3(time,device_id,s1,s2,s3,s4,s5,s6,s7,s8,s9,s10) values (2024-09-24T06:14:30.000+00:00,'d01',40,40,40.0,40.0,false,'shanghai_huangpu_red_A_d01_40','shanghai_huangpu_red_A_d01_40',X'cafebabe40',2024-09-24T06:14:00.000+00:00,'2024-09-24')", "INSERT INTO table3(time,device_id,s1,s2,s3,s4,s5,s6,s7,s8,s9,s10) values (2024-09-24T06:13:30.000+00:00,'d_null',30,30,30.0,30.0,true,'shanghai_huangpu_red_A_d01_30','shanghai_huangpu_red_A_d01_30',X'cafebabe30',2024-09-24T06:13:00.000+00:00,'2024-09-23')", From 44dab8ecc27911745705f35cef409bf08283f676 Mon Sep 17 00:00:00 2001 From: lancelly <1435078631@qq.com> Date: Sat, 4 Jan 2025 16:05:39 +0800 Subject: [PATCH 25/26] remove todo and merge master --- .../execution/relational/ColumnTransformerBuilder.java | 6 +++++- .../plan/relational/planner/IrTypeAnalyzer.java | 7 +++++-- ...TransformQuantifiedComparisonApplyToCorrelatedJoin.java | 4 ++-- 3 files changed, 12 insertions(+), 5 deletions(-) diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/execution/relational/ColumnTransformerBuilder.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/execution/relational/ColumnTransformerBuilder.java index 142aef701990..3682a124f985 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/execution/relational/ColumnTransformerBuilder.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/execution/relational/ColumnTransformerBuilder.java @@ -493,7 +493,7 @@ protected ColumnTransformer visitGenericLiteral(GenericLiteral node, Context con return res; } - // currently, we only support Date and Timestamp + // currently, we only support Date/Timestamp/INT64 // for Date, GenericLiteral.value is an int value // for Timestamp, GenericLiteral.value is a long value private static ConstantColumnTransformer getColumnTransformerForGenericLiteral( @@ -506,6 +506,10 @@ private static ConstantColumnTransformer getColumnTransformerForGenericLiteral( return new ConstantColumnTransformer( TimestampType.TIMESTAMP, new LongColumn(1, Optional.empty(), new long[] {Long.parseLong(literal.getValue())})); + } else if (INT64.getTypeEnum().name().equals(literal.getType())) { + return new ConstantColumnTransformer( + INT64, + new LongColumn(1, Optional.empty(), new long[] {Long.parseLong(literal.getValue())})); } else { throw new SemanticException("Unsupported type in GenericLiteral: " + literal.getType()); } diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/planner/IrTypeAnalyzer.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/planner/IrTypeAnalyzer.java index ab84934d166e..1ecf2dad77dc 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/planner/IrTypeAnalyzer.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/planner/IrTypeAnalyzer.java @@ -82,6 +82,7 @@ import static org.apache.iotdb.db.queryengine.plan.relational.type.TypeSignatureTranslator.toTypeSignature; import static org.apache.tsfile.read.common.type.BooleanType.BOOLEAN; import static org.apache.tsfile.read.common.type.DoubleType.DOUBLE; +import static org.apache.tsfile.read.common.type.IntType.INT32; import static org.apache.tsfile.read.common.type.LongType.INT64; import static org.apache.tsfile.read.common.type.UnknownType.UNKNOWN; @@ -335,10 +336,10 @@ protected Type visitBinaryLiteral(BinaryLiteral node, Context context) { @Override protected Type visitLongLiteral(LongLiteral node, Context context) { - /*if (node.getParsedValue() >= Integer.MIN_VALUE + if (node.getParsedValue() >= Integer.MIN_VALUE && node.getParsedValue() <= Integer.MAX_VALUE) { return setExpressionType(node, INT32); - }*/ + } // keep the original type return setExpressionType(node, INT64); } @@ -360,6 +361,8 @@ protected Type visitGenericLiteral(GenericLiteral node, Context context) { type = DateType.DATE; } else if (TimestampType.TIMESTAMP.getTypeEnum().name().equals(node.getType())) { type = TimestampType.TIMESTAMP; + } else if (INT64.getTypeEnum().name().equals(node.getType())) { + type = INT64; } else { throw new SemanticException("Unsupported type in GenericLiteral: " + node.getType()); } diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/planner/optimizations/TransformQuantifiedComparisonApplyToCorrelatedJoin.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/planner/optimizations/TransformQuantifiedComparisonApplyToCorrelatedJoin.java index c56ed0b706ed..00fecc1a69a4 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/planner/optimizations/TransformQuantifiedComparisonApplyToCorrelatedJoin.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/planner/optimizations/TransformQuantifiedComparisonApplyToCorrelatedJoin.java @@ -41,7 +41,7 @@ import org.apache.iotdb.db.queryengine.plan.relational.sql.ast.Cast; import org.apache.iotdb.db.queryengine.plan.relational.sql.ast.ComparisonExpression; import org.apache.iotdb.db.queryengine.plan.relational.sql.ast.Expression; -import org.apache.iotdb.db.queryengine.plan.relational.sql.ast.LongLiteral; +import org.apache.iotdb.db.queryengine.plan.relational.sql.ast.GenericLiteral; import org.apache.iotdb.db.queryengine.plan.relational.sql.ast.NullLiteral; import org.apache.iotdb.db.queryengine.plan.relational.sql.ast.SearchedCaseExpression; import org.apache.iotdb.db.queryengine.plan.relational.sql.ast.SimpleCaseExpression; @@ -229,7 +229,7 @@ public Expression rewriteUsingBounds( return new SimpleCaseExpression( countAllValue.toSymbolReference(), - ImmutableList.of(new WhenClause(new LongLiteral("0"), emptySetResult)), + ImmutableList.of(new WhenClause(new GenericLiteral("INT64", "0"), emptySetResult)), quantifier.apply( ImmutableList.of( comparisonWithExtremeValue, From 55123de96e546d8d7b061e4db8e38a2cc2b918cf Mon Sep 17 00:00:00 2001 From: lancelly <1435078631@qq.com> Date: Fri, 17 Jan 2025 16:44:07 +0800 Subject: [PATCH 26/26] merge master --- .../source/relational/aggregation/CountAllAccumulator.java | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/execution/operator/source/relational/aggregation/CountAllAccumulator.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/execution/operator/source/relational/aggregation/CountAllAccumulator.java index 78c94f3e6e69..27867e387305 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/execution/operator/source/relational/aggregation/CountAllAccumulator.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/execution/operator/source/relational/aggregation/CountAllAccumulator.java @@ -42,9 +42,9 @@ public TableAccumulator copy() { } @Override - public void addInput(Column[] arguments) { - checkArgument(arguments.length == 1, "argument of Count should be one column"); - int count = arguments[0].getPositionCount(); + public void addInput(Column[] arguments, AggregationMask mask) { + checkArgument(arguments.length == 1, "argument of CountAll should be one column"); + int count = mask.getSelectedPositionCount(); countState += count; }