Merge branch 'master' into fix-bvar-leak-when-alter-table-failed

xy720 · web-flow · commit aaff74e04765 · 2025-09-21T21:07:39.000+08:00
diff --git a/be/src/vec/exec/scan/file_scanner.cpp b/be/src/vec/exec/scan/file_scanner.cpp
@@ -1154,7 +1154,17 @@ Status FileScanner::_get_next_reader() {
         }
 
         _cur_reader->set_push_down_agg_type(_get_push_down_agg_type());
-        RETURN_IF_ERROR(_set_fill_or_truncate_columns(need_to_get_parsed_schema));
+        if (_get_push_down_agg_type() == TPushAggOp::type::COUNT &&
+            range.__isset.table_format_params &&
+            range.table_format_params.table_level_row_count >= 0) {
+            // This is a table level count push down operation, no need to call
+            // _set_fill_or_truncate_columns.
+            // in _set_fill_or_truncate_columns, we will use [range.start_offset, end offset]
+            // to filter the row group. But if this is count push down, the offset is undefined,
+            // causing incorrect row group filter and may return empty result.
+        } else {
+            RETURN_IF_ERROR(_set_fill_or_truncate_columns(need_to_get_parsed_schema));
+        }
         _cur_reader_eof = false;
         break;
     }
diff --git a/cloud/script/start.sh b/cloud/script/start.sh
@@ -129,7 +129,8 @@ if [[ ${enable_hdfs} -eq 1 ]]; then
         done
     fi
 
-    export CLASSPATH="${DORIS_CLASSPATH}"
+    # and conf/ dir so that hadoop libhdfs can read .xml config file in conf/
+    export CLASSPATH="${DORIS_HOME}/conf/:${DORIS_CLASSPATH}:${CLASSPATH}"
 
     export LD_LIBRARY_PATH="${JAVA_HOME}/lib/server:${LD_LIBRARY_PATH}"
 
diff --git a/fe/fe-core/src/main/java/org/apache/doris/datasource/hive/source/HiveScanNode.java b/fe/fe-core/src/main/java/org/apache/doris/datasource/hive/source/HiveScanNode.java
@@ -467,10 +467,12 @@ protected void setScanParams(TFileRangeDesc rangeDesc, Split split) {
                 }
                 transactionalHiveDesc.setDeleteDeltas(deleteDeltaDescs);
                 tableFormatFileDesc.setTransactionalHiveParams(transactionalHiveDesc);
+                tableFormatFileDesc.setTableLevelRowCount(-1);
                 rangeDesc.setTableFormatParams(tableFormatFileDesc);
             } else {
                 TTableFormatFileDesc tableFormatFileDesc = new TTableFormatFileDesc();
                 tableFormatFileDesc.setTableFormatType(TableFormatType.HIVE.value());
+                tableFormatFileDesc.setTableLevelRowCount(-1);
                 rangeDesc.setTableFormatParams(tableFormatFileDesc);
             }
         }
@@ -593,3 +595,4 @@ protected TFileCompressType getFileCompressType(FileSplit fileSplit) throws User
     }
 }
 
+
diff --git a/fe/fe-core/src/main/java/org/apache/doris/datasource/iceberg/source/IcebergScanNode.java b/fe/fe-core/src/main/java/org/apache/doris/datasource/iceberg/source/IcebergScanNode.java
@@ -181,6 +181,9 @@ private void setIcebergParams(TFileRangeDesc rangeDesc, IcebergSplit icebergSpli
         tableFormatFileDesc.setTableFormatType(icebergSplit.getTableFormatType().value());
         if (tableLevelPushDownCount) {
             tableFormatFileDesc.setTableLevelRowCount(icebergSplit.getTableLevelRowCount());
+        } else {
+            // MUST explicitly set to -1, to be distinct from valid row count >= 0
+            tableFormatFileDesc.setTableLevelRowCount(-1);
         }
         TIcebergFileDesc fileDesc = new TIcebergFileDesc();
         fileDesc.setFormatVersion(formatVersion);
@@ -621,3 +624,4 @@ private Optional<NotSupportedException> checkNotSupportedException(Exception e)
         return Optional.empty();
     }
 }
+
diff --git a/fe/fe-core/src/main/java/org/apache/doris/datasource/paimon/source/PaimonScanNode.java b/fe/fe-core/src/main/java/org/apache/doris/datasource/paimon/source/PaimonScanNode.java
@@ -243,6 +243,9 @@ private void setPaimonParams(TFileRangeDesc rangeDesc, PaimonSplit paimonSplit)
         }
         if (paimonSplit.getRowCount().isPresent()) {
             tableFormatFileDesc.setTableLevelRowCount(paimonSplit.getRowCount().get());
+        } else {
+            // MUST explicitly set to -1, to be distinct from valid row count >= 0
+            tableFormatFileDesc.setTableLevelRowCount(-1);
         }
         tableFormatFileDesc.setPaimonParams(fileDesc);
         Map<String, String> partitionValues = paimonSplit.getPaimonPartitionValues();
@@ -714,3 +717,4 @@ private Table getProcessedTable() throws UserException {
         return baseTable;
     }
 }
+
diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/NereidsPlanner.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/NereidsPlanner.java
@@ -983,7 +983,7 @@ public static PhysicalProperties buildInitRequireProperties() {
         return PhysicalProperties.GATHER;
     }
 
-    private ExplainLevel getExplainLevel(ExplainOptions explainOptions) {
+    protected ExplainLevel getExplainLevel(ExplainOptions explainOptions) {
         if (explainOptions == null) {
             return ExplainLevel.NONE;
         }
diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/analysis/ExpressionAnalyzer.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/analysis/ExpressionAnalyzer.java
@@ -74,13 +74,11 @@
 import org.apache.doris.nereids.trees.expressions.functions.agg.NullableAggregateFunction;
 import org.apache.doris.nereids.trees.expressions.functions.agg.SupportMultiDistinct;
 import org.apache.doris.nereids.trees.expressions.functions.scalar.Lambda;
-import org.apache.doris.nereids.trees.expressions.functions.scalar.PreparePlaceholder;
 import org.apache.doris.nereids.trees.expressions.functions.udf.AliasUdfBuilder;
 import org.apache.doris.nereids.trees.expressions.functions.udf.JavaUdaf;
 import org.apache.doris.nereids.trees.expressions.functions.udf.JavaUdf;
 import org.apache.doris.nereids.trees.expressions.functions.udf.UdfBuilder;
 import org.apache.doris.nereids.trees.expressions.literal.IntegerLikeLiteral;
-import org.apache.doris.nereids.trees.expressions.literal.IntegerLiteral;
 import org.apache.doris.nereids.trees.expressions.literal.Literal;
 import org.apache.doris.nereids.trees.expressions.literal.NullLiteral;
 import org.apache.doris.nereids.trees.expressions.literal.StringLiteral;
@@ -652,7 +650,7 @@ public Expression visitPlaceholder(Placeholder placeholder, ExpressionRewriteCon
         // In prepare stage, the realExpr has not been set, set it to StringLiteral so that we can plan the statement
         // and get the output slots in prepare stage, which is required by Mysql api definition.
         if (realExpr == null && context.cascadesContext.getStatementContext().isPrepareStage()) {
-            realExpr = new PreparePlaceholder(new IntegerLiteral(1));
+            realExpr = new StringLiteral(String.valueOf(placeholder.getPlaceholderId().asInt()));
         }
         return visit(realExpr, context);
     }
diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/rewrite/LogicalResultSinkToShortCircuitPointQuery.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/rewrite/LogicalResultSinkToShortCircuitPointQuery.java
@@ -26,7 +26,6 @@
 import org.apache.doris.nereids.trees.expressions.EqualTo;
 import org.apache.doris.nereids.trees.expressions.Expression;
 import org.apache.doris.nereids.trees.expressions.SlotReference;
-import org.apache.doris.nereids.trees.expressions.functions.scalar.PreparePlaceholder;
 import org.apache.doris.nereids.trees.plans.Plan;
 import org.apache.doris.nereids.trees.plans.logical.LogicalFilter;
 import org.apache.doris.nereids.trees.plans.logical.LogicalOlapScan;
@@ -58,8 +57,7 @@ private boolean filterMatchShortCircuitCondition(LogicalFilter<LogicalOlapScan>
                         && (removeCast(expression.child(0)).isKeyColumnFromTable()
                         || (expression.child(0) instanceof SlotReference
                         && ((SlotReference) expression.child(0)).getName().equals(Column.DELETE_SIGN)))
-                        && (expression.child(1).isLiteral()
-                        || removeCast(expression.child(1)) instanceof PreparePlaceholder));
+                        && expression.child(1).isLiteral());
     }
 
     private boolean scanMatchShortCircuitCondition(LogicalOlapScan olapScan) {
diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/Placeholder.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/Placeholder.java
@@ -24,7 +24,7 @@
 import org.apache.doris.nereids.trees.expressions.visitor.ExpressionVisitor;
 import org.apache.doris.nereids.trees.plans.PlaceholderId;
 import org.apache.doris.nereids.types.DataType;
-import org.apache.doris.nereids.types.NullType;
+import org.apache.doris.nereids.types.StringType;
 
 import java.util.Optional;
 
@@ -78,7 +78,7 @@ public int fastChildrenHashCode() {
 
     @Override
     public DataType getDataType() throws UnboundException {
-        return NullType.INSTANCE;
+        return StringType.INSTANCE;
     }
 
     public Placeholder withNewMysqlColType(int mysqlTypeCode) {
diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/functions/scalar/PreparePlaceholder.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/functions/scalar/PreparePlaceholder.java
diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/visitor/ScalarFunctionVisitor.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/visitor/ScalarFunctionVisitor.java
@@ -369,7 +369,6 @@
 import org.apache.doris.nereids.trees.expressions.functions.scalar.Positive;
 import org.apache.doris.nereids.trees.expressions.functions.scalar.Pow;
 import org.apache.doris.nereids.trees.expressions.functions.scalar.Power;
-import org.apache.doris.nereids.trees.expressions.functions.scalar.PreparePlaceholder;
 import org.apache.doris.nereids.trees.expressions.functions.scalar.Protocol;
 import org.apache.doris.nereids.trees.expressions.functions.scalar.QuantilePercent;
 import org.apache.doris.nereids.trees.expressions.functions.scalar.QuantileStateEmpty;
@@ -1823,10 +1822,6 @@ default R visitParseDataSize(ParseDataSize parseDataSize, C context) {
         return visitScalarFunction(parseDataSize, context);
     }
 
-    default R visitPreparePlaceholer(PreparePlaceholder preparePlaceholder, C context) {
-        return visitScalarFunction(preparePlaceholder, context);
-    }
-
     default R visitPassword(Password password, C context) {
         return visitScalarFunction(password, context);
     }
diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/PrepareCommandPlanner.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/PrepareCommandPlanner.java
@@ -0,0 +1,39 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements.  See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership.  The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License.  You may obtain a copy of the License at
+//
+//   http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied.  See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+package org.apache.doris.nereids.trees.plans;
+
+import org.apache.doris.analysis.ExplainOptions;
+import org.apache.doris.nereids.NereidsPlanner;
+import org.apache.doris.nereids.StatementContext;
+import org.apache.doris.nereids.trees.plans.commands.ExplainCommand.ExplainLevel;
+
+/**
+ * Planner for prepare command. Used for PrepareCommand to get output slots metadata.
+ * In prepare stage, we only need to do analyze stage of plan to get the output slots metadata.
+ * And we skip all the other stages to avoid errors.
+ */
+public class PrepareCommandPlanner extends NereidsPlanner {
+    public PrepareCommandPlanner(StatementContext statementContext) {
+        super(statementContext);
+    }
+
+    protected ExplainLevel getExplainLevel(ExplainOptions explainOptions) {
+        return ExplainLevel.ANALYZED_PLAN;
+    }
+
+}
diff --git a/fe/fe-core/src/main/java/org/apache/doris/qe/StmtExecutor.java b/fe/fe-core/src/main/java/org/apache/doris/qe/StmtExecutor.java
@@ -84,6 +84,7 @@
 import org.apache.doris.nereids.trees.expressions.SlotReference;
 import org.apache.doris.nereids.trees.expressions.literal.DateTimeV2Literal;
 import org.apache.doris.nereids.trees.plans.Plan;
+import org.apache.doris.nereids.trees.plans.PrepareCommandPlanner;
 import org.apache.doris.nereids.trees.plans.algebra.InlineTable;
 import org.apache.doris.nereids.trees.plans.commands.Command;
 import org.apache.doris.nereids.trees.plans.commands.CreateTableCommand;
@@ -1854,9 +1855,9 @@ public List<Slot> planPrepareStatementSlots() throws Exception {
         Preconditions.checkState(parsedStmt instanceof LogicalPlanAdapter,
                 "Nereids only process LogicalPlanAdapter,"
                         + " but parsedStmt is " + parsedStmt.getClass().getName());
-        NereidsPlanner nereidsPlanner = new NereidsPlanner(statementContext);
+        NereidsPlanner nereidsPlanner = new PrepareCommandPlanner(statementContext);
         nereidsPlanner.plan(parsedStmt, context.getSessionVariable().toThrift());
-        return nereidsPlanner.getPhysicalPlan().getOutput();
+        return nereidsPlanner.getCascadesContext().getRewritePlan().getOutput();
     }
 
     public List<ResultRow> executeInternalQuery() {
diff --git a/gensrc/thrift/PlanNodes.thrift b/gensrc/thrift/PlanNodes.thrift
@@ -398,7 +398,7 @@ struct TTableFormatFileDesc {
     6: optional TMaxComputeFileDesc max_compute_params
     7: optional TTrinoConnectorFileDesc trino_connector_params
     8: optional TLakeSoulFileDesc lakesoul_params
-    9: optional i64 table_level_row_count
+    9: optional i64 table_level_row_count = -1
 }
 
 // Deprecated, hive text talbe is a special format, not a serde type
diff --git a/regression-test/data/external_table_p0/iceberg/test_iceberg_optimize_count.out b/regression-test/data/external_table_p0/iceberg/test_iceberg_optimize_count.out
@@ -11,6 +11,18 @@
 -- !q04 --
 1000
 
+-- !q01 --
+1000
+
+-- !q02 --
+1000
+
+-- !q03 --
+1000
+
+-- !q04 --
+1000
+
 -- !q05 --
 1000
 
diff --git a/regression-test/data/prepared_stmt_p0/prepared_stmt.out b/regression-test/data/prepared_stmt_p0/prepared_stmt.out
@@ -138,6 +138,10 @@ a
 -- !select25 --
 2025-08-15T00:00
 
+-- !select26 --
+0	1
+1	2
+
 -- !overflow_2 --
 2
 
diff --git a/regression-test/suites/external_table_p0/iceberg/test_iceberg_optimize_count.groovy b/regression-test/suites/external_table_p0/iceberg/test_iceberg_optimize_count.groovy
@@ -50,10 +50,14 @@ suite("test_iceberg_optimize_count", "p0,external,doris,external_docker,external
         // use push down count
         sql """ set enable_count_push_down_for_external_table=true; """
 
-        qt_q01 """${sqlstr1}""" 
-        qt_q02 """${sqlstr2}""" 
-        qt_q03 """${sqlstr3}""" 
-        qt_q04 """${sqlstr4}""" 
+        for (String val: ["1K", "0"]) {
+            sql "set file_split_size=${val}"
+            qt_q01 """${sqlstr1}""" 
+            qt_q02 """${sqlstr2}""" 
+            qt_q03 """${sqlstr3}""" 
+            qt_q04 """${sqlstr4}""" 
+        }
+        sql "unset variable file_split_size;"
 
         // traditional mode
         sql """set num_files_in_batch_mode=100000"""
diff --git a/regression-test/suites/prepared_stmt_p0/prepared_stmt.groovy b/regression-test/suites/prepared_stmt_p0/prepared_stmt.groovy
@@ -327,6 +327,35 @@ suite("test_prepared_stmt", "nonConcurrent") {
         stmt_read.setString(1, "2025-08-15 11:22:33")
         stmt_read.setString(2, "DAY")
         qe_select25 stmt_read
+
+        sql """drop table if exists table_20_undef_partitions2_keys3_properties4_distributed_by54"""
+        sql """ CREATE TABLE IF NOT EXISTS `table_20_undef_partitions2_keys3_properties4_distributed_by54` (
+              `col_int_undef_signed` int NULL,
+              `col_int_undef_signed2` int NULL,
+              `pk` int NULL
+            ) ENGINE=OLAP
+            DUPLICATE KEY(`col_int_undef_signed`, `col_int_undef_signed2`, `pk`)
+            DISTRIBUTED BY HASH(`pk`) BUCKETS 10
+            PROPERTIES (
+            "replication_allocation" = "tag.location.default: 1",
+            "min_load_replica_num" = "-1",
+            "is_being_synced" = "false",
+            "storage_medium" = "hdd",
+            "storage_format" = "V2",
+            "inverted_index_storage_format" = "V2",
+            "light_schema_change" = "true",
+            "disable_auto_compaction" = "false",
+            "enable_single_replica_compaction" = "false",
+            "group_commit_interval_ms" = "10000",
+            "group_commit_data_bytes" = "134217728"
+            ); 
+        """
+        sql """insert into table_20_undef_partitions2_keys3_properties4_distributed_by54 values (1, 1, 1), (2, 2, 2)"""
+        stmt_read = prepareStatement "select min ( pk - ? ) pk , pk as pk from table_20_undef_partitions2_keys3_properties4_distributed_by54 tbl_alias1 group by pk having ( pk >= pk ) or ( round ( sign ( sign ( pk ) ) ) - ? < ? ) order by pk "
+        stmt_read.setString(1, "1")
+        stmt_read.setString(2, "2")
+        stmt_read.setString(3, "3")
+        qe_select26 stmt_read
     }
 
     // test stmtId overflow

Original file line number	Diff line number	Diff line change
`@@ -467,10 +467,12 @@ protected void setScanParams(TFileRangeDesc rangeDesc, Split split) {`
`467`	`467`	`}`
`468`	`468`	`transactionalHiveDesc.setDeleteDeltas(deleteDeltaDescs);`
`469`	`469`	`tableFormatFileDesc.setTransactionalHiveParams(transactionalHiveDesc);`
	`470`	`+ tableFormatFileDesc.setTableLevelRowCount(-1);`
`470`	`471`	`rangeDesc.setTableFormatParams(tableFormatFileDesc);`
`471`	`472`	`} else {`
`472`	`473`	`TTableFormatFileDesc tableFormatFileDesc = new TTableFormatFileDesc();`
`473`	`474`	`tableFormatFileDesc.setTableFormatType(TableFormatType.HIVE.value());`
	`475`	`+ tableFormatFileDesc.setTableLevelRowCount(-1);`
`474`	`476`	`rangeDesc.setTableFormatParams(tableFormatFileDesc);`
`475`	`477`	`}`
`476`	`478`	`}`
`@@ -593,3 +595,4 @@ protected TFileCompressType getFileCompressType(FileSplit fileSplit) throws User`
`593`	`595`	`}`
`594`	`596`	`}`
`595`	`597`
	`598`	`+`
Original file line number	Diff line number	Diff line change
`@@ -181,6 +181,9 @@ private void setIcebergParams(TFileRangeDesc rangeDesc, IcebergSplit icebergSpli`
`181`	`181`	`tableFormatFileDesc.setTableFormatType(icebergSplit.getTableFormatType().value());`
`182`	`182`	`if (tableLevelPushDownCount) {`
`183`	`183`	`tableFormatFileDesc.setTableLevelRowCount(icebergSplit.getTableLevelRowCount());`
	`184`	`+ } else {`
	`185`	`+ // MUST explicitly set to -1, to be distinct from valid row count >= 0`
	`186`	`+ tableFormatFileDesc.setTableLevelRowCount(-1);`
`184`	`187`	`}`
`185`	`188`	`TIcebergFileDesc fileDesc = new TIcebergFileDesc();`
`186`	`189`	`fileDesc.setFormatVersion(formatVersion);`
`@@ -621,3 +624,4 @@ private Optional<NotSupportedException> checkNotSupportedException(Exception e)`
`621`	`624`	`return Optional.empty();`
`622`	`625`	`}`
`623`	`626`	`}`
	`627`	`+`
Original file line number	Diff line number	Diff line change
`@@ -243,6 +243,9 @@ private void setPaimonParams(TFileRangeDesc rangeDesc, PaimonSplit paimonSplit)`
`243`	`243`	`}`
`244`	`244`	`if (paimonSplit.getRowCount().isPresent()) {`
`245`	`245`	`tableFormatFileDesc.setTableLevelRowCount(paimonSplit.getRowCount().get());`
	`246`	`+ } else {`
	`247`	`+ // MUST explicitly set to -1, to be distinct from valid row count >= 0`
	`248`	`+ tableFormatFileDesc.setTableLevelRowCount(-1);`
`246`	`249`	`}`
`247`	`250`	`tableFormatFileDesc.setPaimonParams(fileDesc);`
`248`	`251`	`Map<String, String> partitionValues = paimonSplit.getPaimonPartitionValues();`
`@@ -714,3 +717,4 @@ private Table getProcessedTable() throws UserException {`
`714`	`717`	`return baseTable;`
`715`	`718`	`}`
`716`	`719`	`}`
	`720`	`+`
Original file line number	Diff line number	Diff line change
`@@ -983,7 +983,7 @@ public static PhysicalProperties buildInitRequireProperties() {`
`983`	`983`	`return PhysicalProperties.GATHER;`
`984`	`984`	`}`
`985`	`985`
`986`		`- private ExplainLevel getExplainLevel(ExplainOptions explainOptions) {`
	`986`	`+ protected ExplainLevel getExplainLevel(ExplainOptions explainOptions) {`
`987`	`987`	`if (explainOptions == null) {`
`988`	`988`	`return ExplainLevel.NONE;`
`989`	`989`	`}`