From 6ee6defef6dff91d06ed80feb0b7507539507afe Mon Sep 17 00:00:00 2001 From: Andy Grove Date: Sat, 8 Nov 2025 06:51:23 -0700 Subject: [PATCH 01/44] filter --- .../org/apache/comet/serde/CometFilter.scala | 51 +++++++++++++++++++ .../apache/comet/serde/QueryPlanSerde.scala | 18 ++----- 2 files changed, 55 insertions(+), 14 deletions(-) create mode 100644 spark/src/main/scala/org/apache/comet/serde/CometFilter.scala diff --git a/spark/src/main/scala/org/apache/comet/serde/CometFilter.scala b/spark/src/main/scala/org/apache/comet/serde/CometFilter.scala new file mode 100644 index 0000000000..1638750b5f --- /dev/null +++ b/spark/src/main/scala/org/apache/comet/serde/CometFilter.scala @@ -0,0 +1,51 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.comet.serde + +import org.apache.spark.sql.execution.FilterExec + +import org.apache.comet.{CometConf, ConfigEntry} +import org.apache.comet.CometSparkSessionExtensions.withInfo +import org.apache.comet.serde.OperatorOuterClass.Operator +import org.apache.comet.serde.QueryPlanSerde.exprToProto + +object CometFilter extends CometOperatorSerde[FilterExec] { + + override def enabledConfig: Option[ConfigEntry[Boolean]] = + Some(CometConf.COMET_EXEC_FILTER_ENABLED) + + override def convert( + op: FilterExec, + builder: Operator.Builder, + childOp: OperatorOuterClass.Operator*): Option[OperatorOuterClass.Operator] = { + val cond = exprToProto(op.condition, op.child.output) + + if (cond.isDefined && childOp.nonEmpty) { + val filterBuilder = OperatorOuterClass.Filter + .newBuilder() + .setPredicate(cond.get) + Some(builder.setFilter(filterBuilder).build()) + } else { + withInfo(op, op.condition, op.child) + None + } + } + +} diff --git a/spark/src/main/scala/org/apache/comet/serde/QueryPlanSerde.scala b/spark/src/main/scala/org/apache/comet/serde/QueryPlanSerde.scala index 63e18c145a..8ffec22d15 100644 --- a/spark/src/main/scala/org/apache/comet/serde/QueryPlanSerde.scala +++ b/spark/src/main/scala/org/apache/comet/serde/QueryPlanSerde.scala @@ -66,7 +66,10 @@ object QueryPlanSerde extends Logging with CometExprShim { * Mapping of Spark operator class to Comet operator handler. */ private val opSerdeMap: Map[Class[_ <: SparkPlan], CometOperatorSerde[_]] = - Map(classOf[ProjectExec] -> CometProject, classOf[SortExec] -> CometSort) + Map( + classOf[ProjectExec] -> CometProject, + classOf[FilterExec] -> CometFilter, + classOf[SortExec] -> CometSort) private val arrayExpressions: Map[Class[_ <: Expression], CometExpressionSerde[_]] = Map( classOf[ArrayAppend] -> CometArrayAppend, @@ -1065,19 +1068,6 @@ object QueryPlanSerde extends Logging with CometExprShim { None } - case FilterExec(condition, child) if CometConf.COMET_EXEC_FILTER_ENABLED.get(conf) => - val cond = exprToProto(condition, child.output) - - if (cond.isDefined && childOp.nonEmpty) { - val filterBuilder = OperatorOuterClass.Filter - .newBuilder() - .setPredicate(cond.get) - Some(builder.setFilter(filterBuilder).build()) - } else { - withInfo(op, condition, child) - None - } - case LocalLimitExec(limit, _) if CometConf.COMET_EXEC_LOCAL_LIMIT_ENABLED.get(conf) => if (childOp.nonEmpty) { // LocalLimit doesn't use offset, but it shares same operator serde class. From 3a80a1f1bc7c953b16fd9f07a31601ffc1f4b788 Mon Sep 17 00:00:00 2001 From: Andy Grove Date: Sat, 8 Nov 2025 06:56:22 -0700 Subject: [PATCH 02/44] limit --- .../apache/comet/serde/CometGlobalLimit.scala | 49 ++++++++++++++++++ .../apache/comet/serde/CometLocalLimit.scala | 50 +++++++++++++++++++ .../apache/comet/serde/QueryPlanSerde.scala | 29 +---------- 3 files changed, 101 insertions(+), 27 deletions(-) create mode 100644 spark/src/main/scala/org/apache/comet/serde/CometGlobalLimit.scala create mode 100644 spark/src/main/scala/org/apache/comet/serde/CometLocalLimit.scala diff --git a/spark/src/main/scala/org/apache/comet/serde/CometGlobalLimit.scala b/spark/src/main/scala/org/apache/comet/serde/CometGlobalLimit.scala new file mode 100644 index 0000000000..774e1ad77e --- /dev/null +++ b/spark/src/main/scala/org/apache/comet/serde/CometGlobalLimit.scala @@ -0,0 +1,49 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.comet.serde + +import org.apache.spark.sql.execution.GlobalLimitExec + +import org.apache.comet.{CometConf, ConfigEntry} +import org.apache.comet.CometSparkSessionExtensions.withInfo +import org.apache.comet.serde.OperatorOuterClass.Operator + +object CometGlobalLimit extends CometOperatorSerde[GlobalLimitExec] { + + override def enabledConfig: Option[ConfigEntry[Boolean]] = + Some(CometConf.COMET_EXEC_GLOBAL_LIMIT_ENABLED) + + override def convert( + op: GlobalLimitExec, + builder: Operator.Builder, + childOp: OperatorOuterClass.Operator*): Option[OperatorOuterClass.Operator] = { + if (childOp.nonEmpty) { + val limitBuilder = OperatorOuterClass.Limit.newBuilder() + + limitBuilder.setLimit(op.limit).setOffset(op.offset) + + Some(builder.setLimit(limitBuilder).build()) + } else { + withInfo(op, "No child operator") + None + } + + } +} diff --git a/spark/src/main/scala/org/apache/comet/serde/CometLocalLimit.scala b/spark/src/main/scala/org/apache/comet/serde/CometLocalLimit.scala new file mode 100644 index 0000000000..1347b12907 --- /dev/null +++ b/spark/src/main/scala/org/apache/comet/serde/CometLocalLimit.scala @@ -0,0 +1,50 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.comet.serde + +import org.apache.spark.sql.execution.LocalLimitExec + +import org.apache.comet.{CometConf, ConfigEntry} +import org.apache.comet.CometSparkSessionExtensions.withInfo +import org.apache.comet.serde.OperatorOuterClass.Operator + +object CometLocalLimit extends CometOperatorSerde[LocalLimitExec] { + + override def enabledConfig: Option[ConfigEntry[Boolean]] = + Some(CometConf.COMET_EXEC_LOCAL_LIMIT_ENABLED) + + override def convert( + op: LocalLimitExec, + builder: Operator.Builder, + childOp: OperatorOuterClass.Operator*): Option[OperatorOuterClass.Operator] = { + if (childOp.nonEmpty) { + // LocalLimit doesn't use offset, but it shares same operator serde class. + // Just set it to zero. + val limitBuilder = OperatorOuterClass.Limit + .newBuilder() + .setLimit(op.limit) + .setOffset(0) + Some(builder.setLimit(limitBuilder).build()) + } else { + withInfo(op, "No child operator") + None + } + } +} diff --git a/spark/src/main/scala/org/apache/comet/serde/QueryPlanSerde.scala b/spark/src/main/scala/org/apache/comet/serde/QueryPlanSerde.scala index 8ffec22d15..7a4b457cdd 100644 --- a/spark/src/main/scala/org/apache/comet/serde/QueryPlanSerde.scala +++ b/spark/src/main/scala/org/apache/comet/serde/QueryPlanSerde.scala @@ -69,6 +69,8 @@ object QueryPlanSerde extends Logging with CometExprShim { Map( classOf[ProjectExec] -> CometProject, classOf[FilterExec] -> CometFilter, + classOf[LocalLimitExec] -> CometLocalLimit, + classOf[GlobalLimitExec] -> CometGlobalLimit, classOf[SortExec] -> CometSort) private val arrayExpressions: Map[Class[_ <: Expression], CometExpressionSerde[_]] = Map( @@ -1068,33 +1070,6 @@ object QueryPlanSerde extends Logging with CometExprShim { None } - case LocalLimitExec(limit, _) if CometConf.COMET_EXEC_LOCAL_LIMIT_ENABLED.get(conf) => - if (childOp.nonEmpty) { - // LocalLimit doesn't use offset, but it shares same operator serde class. - // Just set it to zero. - val limitBuilder = OperatorOuterClass.Limit - .newBuilder() - .setLimit(limit) - .setOffset(0) - Some(builder.setLimit(limitBuilder).build()) - } else { - withInfo(op, "No child operator") - None - } - - case globalLimitExec: GlobalLimitExec - if CometConf.COMET_EXEC_GLOBAL_LIMIT_ENABLED.get(conf) => - if (childOp.nonEmpty) { - val limitBuilder = OperatorOuterClass.Limit.newBuilder() - - limitBuilder.setLimit(globalLimitExec.limit).setOffset(globalLimitExec.offset) - - Some(builder.setLimit(limitBuilder).build()) - } else { - withInfo(op, "No child operator") - None - } - case ExpandExec(projections, _, child) if CometConf.COMET_EXEC_EXPAND_ENABLED.get(conf) => var allProjExprs: Seq[Expression] = Seq() val projExprs = projections.flatMap(_.map(e => { From 51e0aa2b206d7f13aba4dc3fc1bf3b7e62d1fa9d Mon Sep 17 00:00:00 2001 From: Andy Grove Date: Sat, 8 Nov 2025 07:03:03 -0700 Subject: [PATCH 03/44] hash join --- .../apache/comet/serde/CometHashJoin.scala | 102 ++++++++++++++++++ .../apache/comet/serde/QueryPlanSerde.scala | 67 +----------- 2 files changed, 106 insertions(+), 63 deletions(-) create mode 100644 spark/src/main/scala/org/apache/comet/serde/CometHashJoin.scala diff --git a/spark/src/main/scala/org/apache/comet/serde/CometHashJoin.scala b/spark/src/main/scala/org/apache/comet/serde/CometHashJoin.scala new file mode 100644 index 0000000000..94fd880f8f --- /dev/null +++ b/spark/src/main/scala/org/apache/comet/serde/CometHashJoin.scala @@ -0,0 +1,102 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.comet.serde + +import scala.jdk.CollectionConverters._ + +import org.apache.spark.sql.catalyst.expressions.Expression +import org.apache.spark.sql.catalyst.optimizer.{BuildLeft, BuildRight} +import org.apache.spark.sql.catalyst.plans.{FullOuter, Inner, LeftAnti, LeftOuter, LeftSemi, RightOuter} +import org.apache.spark.sql.execution.joins.{BroadcastHashJoinExec, HashJoin, ShuffledHashJoinExec} + +import org.apache.comet.{CometConf, ConfigEntry} +import org.apache.comet.CometSparkSessionExtensions.withInfo +import org.apache.comet.serde.OperatorOuterClass.{BuildSide, JoinType, Operator} +import org.apache.comet.serde.QueryPlanSerde.exprToProto + +object CometHashJoin extends CometOperatorSerde[HashJoin] { + + override def enabledConfig: Option[ConfigEntry[Boolean]] = + Some(CometConf.COMET_EXEC_HASH_JOIN_ENABLED) + + override def convert( + join: HashJoin, + builder: Operator.Builder, + childOp: OperatorOuterClass.Operator*): Option[OperatorOuterClass.Operator] = { + // `HashJoin` has only two implementations in Spark, but we check the type of the join to + // make sure we are handling the correct join type. + if (!(CometConf.COMET_EXEC_HASH_JOIN_ENABLED.get(join.conf) && + join.isInstanceOf[ShuffledHashJoinExec]) && + !(CometConf.COMET_EXEC_BROADCAST_HASH_JOIN_ENABLED.get(join.conf) && + join.isInstanceOf[BroadcastHashJoinExec])) { + withInfo(join, s"Invalid hash join type ${join.nodeName}") + return None + } + + if (join.buildSide == BuildRight && join.joinType == LeftAnti) { + // https://github.com/apache/datafusion-comet/issues/457 + withInfo(join, "BuildRight with LeftAnti is not supported") + return None + } + + val condition = join.condition.map { cond => + val condProto = exprToProto(cond, join.left.output ++ join.right.output) + if (condProto.isEmpty) { + withInfo(join, cond) + return None + } + condProto.get + } + + val joinType = join.joinType match { + case Inner => JoinType.Inner + case LeftOuter => JoinType.LeftOuter + case RightOuter => JoinType.RightOuter + case FullOuter => JoinType.FullOuter + case LeftSemi => JoinType.LeftSemi + case LeftAnti => JoinType.LeftAnti + case _ => + // Spark doesn't support other join types + withInfo(join, s"Unsupported join type ${join.joinType}") + return None + } + + val leftKeys = join.leftKeys.map(exprToProto(_, join.left.output)) + val rightKeys = join.rightKeys.map(exprToProto(_, join.right.output)) + + if (leftKeys.forall(_.isDefined) && + rightKeys.forall(_.isDefined) && + childOp.nonEmpty) { + val joinBuilder = OperatorOuterClass.HashJoin + .newBuilder() + .setJoinType(joinType) + .addAllLeftJoinKeys(leftKeys.map(_.get).asJava) + .addAllRightJoinKeys(rightKeys.map(_.get).asJava) + .setBuildSide( + if (join.buildSide == BuildLeft) BuildSide.BuildLeft else BuildSide.BuildRight) + condition.foreach(joinBuilder.setCondition) + Some(builder.setHashJoin(joinBuilder).build()) + } else { + val allExprs: Seq[Expression] = join.leftKeys ++ join.rightKeys + withInfo(join, allExprs: _*) + None + } + } +} diff --git a/spark/src/main/scala/org/apache/comet/serde/QueryPlanSerde.scala b/spark/src/main/scala/org/apache/comet/serde/QueryPlanSerde.scala index 7a4b457cdd..e51f69adb0 100644 --- a/spark/src/main/scala/org/apache/comet/serde/QueryPlanSerde.scala +++ b/spark/src/main/scala/org/apache/comet/serde/QueryPlanSerde.scala @@ -26,7 +26,7 @@ import org.apache.spark.internal.Logging import org.apache.spark.sql.catalyst.expressions._ import org.apache.spark.sql.catalyst.expressions.aggregate._ import org.apache.spark.sql.catalyst.expressions.objects.StaticInvoke -import org.apache.spark.sql.catalyst.optimizer.{BuildLeft, BuildRight, NormalizeNaNAndZero} +import org.apache.spark.sql.catalyst.optimizer.NormalizeNaNAndZero import org.apache.spark.sql.catalyst.plans._ import org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils import org.apache.spark.sql.catalyst.util.ResolveDefaultColumns.getExistenceDefaultValues @@ -39,7 +39,7 @@ import org.apache.spark.sql.execution.aggregate.{BaseAggregateExec, HashAggregat import org.apache.spark.sql.execution.datasources.{FilePartition, FileScanRDD, PartitionedFile} import org.apache.spark.sql.execution.datasources.v2.{DataSourceRDD, DataSourceRDDPartition} import org.apache.spark.sql.execution.exchange.{BroadcastExchangeExec, ReusedExchangeExec, ShuffleExchangeExec} -import org.apache.spark.sql.execution.joins.{BroadcastHashJoinExec, HashJoin, ShuffledHashJoinExec, SortMergeJoinExec} +import org.apache.spark.sql.execution.joins.{HashJoin, SortMergeJoinExec} import org.apache.spark.sql.execution.window.WindowExec import org.apache.spark.sql.internal.SQLConf import org.apache.spark.sql.types._ @@ -50,7 +50,7 @@ import org.apache.comet.expressions._ import org.apache.comet.objectstore.NativeConfig import org.apache.comet.parquet.CometParquetUtils import org.apache.comet.serde.ExprOuterClass.{AggExpr, Expr, ScalarFunc} -import org.apache.comet.serde.OperatorOuterClass.{AggregateMode => CometAggregateMode, BuildSide, JoinType, Operator} +import org.apache.comet.serde.OperatorOuterClass.{AggregateMode => CometAggregateMode, JoinType, Operator} import org.apache.comet.serde.QueryPlanSerde.{exprToProtoInternal, optExprWithInfo, scalarFunctionExprToProto} import org.apache.comet.serde.Types.{DataType => ProtoDataType} import org.apache.comet.serde.Types.DataType._ @@ -71,6 +71,7 @@ object QueryPlanSerde extends Logging with CometExprShim { classOf[FilterExec] -> CometFilter, classOf[LocalLimitExec] -> CometLocalLimit, classOf[GlobalLimitExec] -> CometGlobalLimit, + classOf[HashJoin] -> CometHashJoin, classOf[SortExec] -> CometSort) private val arrayExpressions: Map[Class[_ <: Expression], CometExpressionSerde[_]] = Map( @@ -1259,66 +1260,6 @@ object QueryPlanSerde extends Logging with CometExprShim { } } - case join: HashJoin => - // `HashJoin` has only two implementations in Spark, but we check the type of the join to - // make sure we are handling the correct join type. - if (!(CometConf.COMET_EXEC_HASH_JOIN_ENABLED.get(conf) && - join.isInstanceOf[ShuffledHashJoinExec]) && - !(CometConf.COMET_EXEC_BROADCAST_HASH_JOIN_ENABLED.get(conf) && - join.isInstanceOf[BroadcastHashJoinExec])) { - withInfo(join, s"Invalid hash join type ${join.nodeName}") - return None - } - - if (join.buildSide == BuildRight && join.joinType == LeftAnti) { - // https://github.com/apache/datafusion-comet/issues/457 - withInfo(join, "BuildRight with LeftAnti is not supported") - return None - } - - val condition = join.condition.map { cond => - val condProto = exprToProto(cond, join.left.output ++ join.right.output) - if (condProto.isEmpty) { - withInfo(join, cond) - return None - } - condProto.get - } - - val joinType = join.joinType match { - case Inner => JoinType.Inner - case LeftOuter => JoinType.LeftOuter - case RightOuter => JoinType.RightOuter - case FullOuter => JoinType.FullOuter - case LeftSemi => JoinType.LeftSemi - case LeftAnti => JoinType.LeftAnti - case _ => - // Spark doesn't support other join types - withInfo(join, s"Unsupported join type ${join.joinType}") - return None - } - - val leftKeys = join.leftKeys.map(exprToProto(_, join.left.output)) - val rightKeys = join.rightKeys.map(exprToProto(_, join.right.output)) - - if (leftKeys.forall(_.isDefined) && - rightKeys.forall(_.isDefined) && - childOp.nonEmpty) { - val joinBuilder = OperatorOuterClass.HashJoin - .newBuilder() - .setJoinType(joinType) - .addAllLeftJoinKeys(leftKeys.map(_.get).asJava) - .addAllRightJoinKeys(rightKeys.map(_.get).asJava) - .setBuildSide( - if (join.buildSide == BuildLeft) BuildSide.BuildLeft else BuildSide.BuildRight) - condition.foreach(joinBuilder.setCondition) - Some(builder.setHashJoin(joinBuilder).build()) - } else { - val allExprs: Seq[Expression] = join.leftKeys ++ join.rightKeys - withInfo(join, allExprs: _*) - None - } - case join: SortMergeJoinExec if CometConf.COMET_EXEC_SORT_MERGE_JOIN_ENABLED.get(conf) => // `requiredOrders` and `getKeyOrdering` are copied from Spark's SortMergeJoinExec. def requiredOrders(keys: Seq[Expression]): Seq[SortOrder] = { From 8c0338e764a8a2a5786aa89b10841d29ebdb9dae Mon Sep 17 00:00:00 2001 From: Andy Grove Date: Sat, 8 Nov 2025 07:07:26 -0700 Subject: [PATCH 04/44] sort merge join --- .../comet/serde/CometSortMergeJoin.scala | 144 ++++++++++++++++++ .../apache/comet/serde/QueryPlanSerde.scala | 109 +------------ 2 files changed, 146 insertions(+), 107 deletions(-) create mode 100644 spark/src/main/scala/org/apache/comet/serde/CometSortMergeJoin.scala diff --git a/spark/src/main/scala/org/apache/comet/serde/CometSortMergeJoin.scala b/spark/src/main/scala/org/apache/comet/serde/CometSortMergeJoin.scala new file mode 100644 index 0000000000..5f926f06e8 --- /dev/null +++ b/spark/src/main/scala/org/apache/comet/serde/CometSortMergeJoin.scala @@ -0,0 +1,144 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.comet.serde + +import scala.jdk.CollectionConverters._ + +import org.apache.spark.sql.catalyst.expressions.{Ascending, Expression, ExpressionSet, SortOrder} +import org.apache.spark.sql.catalyst.plans.{FullOuter, Inner, LeftAnti, LeftOuter, LeftSemi, RightOuter} +import org.apache.spark.sql.execution.joins.SortMergeJoinExec +import org.apache.spark.sql.types.{BooleanType, ByteType, DataType, DateType, DecimalType, DoubleType, FloatType, IntegerType, LongType, ShortType, StringType, TimestampNTZType} + +import org.apache.comet.{CometConf, ConfigEntry} +import org.apache.comet.CometSparkSessionExtensions.withInfo +import org.apache.comet.serde.OperatorOuterClass.{JoinType, Operator} +import org.apache.comet.serde.QueryPlanSerde.exprToProto + +object CometSortMergeJoin extends CometOperatorSerde[SortMergeJoinExec] { + override def enabledConfig: Option[ConfigEntry[Boolean]] = Some( + CometConf.COMET_EXEC_SORT_MERGE_JOIN_ENABLED) + + override def convert( + join: SortMergeJoinExec, + builder: Operator.Builder, + childOp: OperatorOuterClass.Operator*): Option[OperatorOuterClass.Operator] = { + // `requiredOrders` and `getKeyOrdering` are copied from Spark's SortMergeJoinExec. + def requiredOrders(keys: Seq[Expression]): Seq[SortOrder] = { + keys.map(SortOrder(_, Ascending)) + } + + def getKeyOrdering( + keys: Seq[Expression], + childOutputOrdering: Seq[SortOrder]): Seq[SortOrder] = { + val requiredOrdering = requiredOrders(keys) + if (SortOrder.orderingSatisfies(childOutputOrdering, requiredOrdering)) { + keys.zip(childOutputOrdering).map { case (key, childOrder) => + val sameOrderExpressionsSet = ExpressionSet(childOrder.children) - key + SortOrder(key, Ascending, sameOrderExpressionsSet.toSeq) + } + } else { + requiredOrdering + } + } + + if (join.condition.isDefined && + !CometConf.COMET_EXEC_SORT_MERGE_JOIN_WITH_JOIN_FILTER_ENABLED + .get(join.conf)) { + withInfo( + join, + s"${CometConf.COMET_EXEC_SORT_MERGE_JOIN_WITH_JOIN_FILTER_ENABLED.key} is not enabled", + join.condition.get) + return None + } + + val condition = join.condition.map { cond => + val condProto = exprToProto(cond, join.left.output ++ join.right.output) + if (condProto.isEmpty) { + withInfo(join, cond) + return None + } + condProto.get + } + + val joinType = join.joinType match { + case Inner => JoinType.Inner + case LeftOuter => JoinType.LeftOuter + case RightOuter => JoinType.RightOuter + case FullOuter => JoinType.FullOuter + case LeftSemi => JoinType.LeftSemi + case LeftAnti => JoinType.LeftAnti + case _ => + // Spark doesn't support other join types + withInfo(join, s"Unsupported join type ${join.joinType}") + return None + } + + // Checks if the join keys are supported by DataFusion SortMergeJoin. + val errorMsgs = join.leftKeys.flatMap { key => + if (!supportedSortMergeJoinEqualType(key.dataType)) { + Some(s"Unsupported join key type ${key.dataType} on key: ${key.sql}") + } else { + None + } + } + + if (errorMsgs.nonEmpty) { + withInfo(join, errorMsgs.flatten.mkString("\n")) + return None + } + + val leftKeys = join.leftKeys.map(exprToProto(_, join.left.output)) + val rightKeys = join.rightKeys.map(exprToProto(_, join.right.output)) + + val sortOptions = getKeyOrdering(join.leftKeys, join.left.outputOrdering) + .map(exprToProto(_, join.left.output)) + + if (sortOptions.forall(_.isDefined) && + leftKeys.forall(_.isDefined) && + rightKeys.forall(_.isDefined) && + childOp.nonEmpty) { + val joinBuilder = OperatorOuterClass.SortMergeJoin + .newBuilder() + .setJoinType(joinType) + .addAllSortOptions(sortOptions.map(_.get).asJava) + .addAllLeftJoinKeys(leftKeys.map(_.get).asJava) + .addAllRightJoinKeys(rightKeys.map(_.get).asJava) + condition.map(joinBuilder.setCondition) + Some(builder.setSortMergeJoin(joinBuilder).build()) + } else { + val allExprs: Seq[Expression] = join.leftKeys ++ join.rightKeys + withInfo(join, allExprs: _*) + None + } + + } + + /** + * Returns true if given datatype is supported as a key in DataFusion sort merge join. + */ + private def supportedSortMergeJoinEqualType(dataType: DataType): Boolean = dataType match { + case _: ByteType | _: ShortType | _: IntegerType | _: LongType | _: FloatType | + _: DoubleType | _: StringType | _: DateType | _: DecimalType | _: BooleanType => + true + case TimestampNTZType => true + case _ => false + } + +} diff --git a/spark/src/main/scala/org/apache/comet/serde/QueryPlanSerde.scala b/spark/src/main/scala/org/apache/comet/serde/QueryPlanSerde.scala index e51f69adb0..62ebd96c75 100644 --- a/spark/src/main/scala/org/apache/comet/serde/QueryPlanSerde.scala +++ b/spark/src/main/scala/org/apache/comet/serde/QueryPlanSerde.scala @@ -27,7 +27,6 @@ import org.apache.spark.sql.catalyst.expressions._ import org.apache.spark.sql.catalyst.expressions.aggregate._ import org.apache.spark.sql.catalyst.expressions.objects.StaticInvoke import org.apache.spark.sql.catalyst.optimizer.NormalizeNaNAndZero -import org.apache.spark.sql.catalyst.plans._ import org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils import org.apache.spark.sql.catalyst.util.ResolveDefaultColumns.getExistenceDefaultValues import org.apache.spark.sql.comet._ @@ -50,7 +49,7 @@ import org.apache.comet.expressions._ import org.apache.comet.objectstore.NativeConfig import org.apache.comet.parquet.CometParquetUtils import org.apache.comet.serde.ExprOuterClass.{AggExpr, Expr, ScalarFunc} -import org.apache.comet.serde.OperatorOuterClass.{AggregateMode => CometAggregateMode, JoinType, Operator} +import org.apache.comet.serde.OperatorOuterClass.{AggregateMode => CometAggregateMode, Operator} import org.apache.comet.serde.QueryPlanSerde.{exprToProtoInternal, optExprWithInfo, scalarFunctionExprToProto} import org.apache.comet.serde.Types.{DataType => ProtoDataType} import org.apache.comet.serde.Types.DataType._ @@ -72,6 +71,7 @@ object QueryPlanSerde extends Logging with CometExprShim { classOf[LocalLimitExec] -> CometLocalLimit, classOf[GlobalLimitExec] -> CometGlobalLimit, classOf[HashJoin] -> CometHashJoin, + classOf[SortMergeJoinExec] -> CometSortMergeJoin, classOf[SortExec] -> CometSort) private val arrayExpressions: Map[Class[_ <: Expression], CometExpressionSerde[_]] = Map( @@ -917,17 +917,6 @@ object QueryPlanSerde extends Logging with CometExprShim { Some(ExprOuterClass.Expr.newBuilder().setScalarFunc(builder).build()) } - /** - * Returns true if given datatype is supported as a key in DataFusion sort merge join. - */ - private def supportedSortMergeJoinEqualType(dataType: DataType): Boolean = dataType match { - case _: ByteType | _: ShortType | _: IntegerType | _: LongType | _: FloatType | - _: DoubleType | _: StringType | _: DateType | _: DecimalType | _: BooleanType => - true - case TimestampNTZType => true - case _ => false - } - /** * Convert a Spark plan operator to a protobuf Comet operator. * @@ -1260,100 +1249,6 @@ object QueryPlanSerde extends Logging with CometExprShim { } } - case join: SortMergeJoinExec if CometConf.COMET_EXEC_SORT_MERGE_JOIN_ENABLED.get(conf) => - // `requiredOrders` and `getKeyOrdering` are copied from Spark's SortMergeJoinExec. - def requiredOrders(keys: Seq[Expression]): Seq[SortOrder] = { - keys.map(SortOrder(_, Ascending)) - } - - def getKeyOrdering( - keys: Seq[Expression], - childOutputOrdering: Seq[SortOrder]): Seq[SortOrder] = { - val requiredOrdering = requiredOrders(keys) - if (SortOrder.orderingSatisfies(childOutputOrdering, requiredOrdering)) { - keys.zip(childOutputOrdering).map { case (key, childOrder) => - val sameOrderExpressionsSet = ExpressionSet(childOrder.children) - key - SortOrder(key, Ascending, sameOrderExpressionsSet.toSeq) - } - } else { - requiredOrdering - } - } - - if (join.condition.isDefined && - !CometConf.COMET_EXEC_SORT_MERGE_JOIN_WITH_JOIN_FILTER_ENABLED - .get(conf)) { - withInfo( - join, - s"${CometConf.COMET_EXEC_SORT_MERGE_JOIN_WITH_JOIN_FILTER_ENABLED.key} is not enabled", - join.condition.get) - return None - } - - val condition = join.condition.map { cond => - val condProto = exprToProto(cond, join.left.output ++ join.right.output) - if (condProto.isEmpty) { - withInfo(join, cond) - return None - } - condProto.get - } - - val joinType = join.joinType match { - case Inner => JoinType.Inner - case LeftOuter => JoinType.LeftOuter - case RightOuter => JoinType.RightOuter - case FullOuter => JoinType.FullOuter - case LeftSemi => JoinType.LeftSemi - case LeftAnti => JoinType.LeftAnti - case _ => - // Spark doesn't support other join types - withInfo(op, s"Unsupported join type ${join.joinType}") - return None - } - - // Checks if the join keys are supported by DataFusion SortMergeJoin. - val errorMsgs = join.leftKeys.flatMap { key => - if (!supportedSortMergeJoinEqualType(key.dataType)) { - Some(s"Unsupported join key type ${key.dataType} on key: ${key.sql}") - } else { - None - } - } - - if (errorMsgs.nonEmpty) { - withInfo(op, errorMsgs.flatten.mkString("\n")) - return None - } - - val leftKeys = join.leftKeys.map(exprToProto(_, join.left.output)) - val rightKeys = join.rightKeys.map(exprToProto(_, join.right.output)) - - val sortOptions = getKeyOrdering(join.leftKeys, join.left.outputOrdering) - .map(exprToProto(_, join.left.output)) - - if (sortOptions.forall(_.isDefined) && - leftKeys.forall(_.isDefined) && - rightKeys.forall(_.isDefined) && - childOp.nonEmpty) { - val joinBuilder = OperatorOuterClass.SortMergeJoin - .newBuilder() - .setJoinType(joinType) - .addAllSortOptions(sortOptions.map(_.get).asJava) - .addAllLeftJoinKeys(leftKeys.map(_.get).asJava) - .addAllRightJoinKeys(rightKeys.map(_.get).asJava) - condition.map(joinBuilder.setCondition) - Some(builder.setSortMergeJoin(joinBuilder).build()) - } else { - val allExprs: Seq[Expression] = join.leftKeys ++ join.rightKeys - withInfo(join, allExprs: _*) - None - } - - case join: SortMergeJoinExec if !CometConf.COMET_EXEC_SORT_MERGE_JOIN_ENABLED.get(conf) => - withInfo(join, "SortMergeJoin is not enabled") - None - case op if isCometSink(op) => val supportedTypes = op.output.forall(a => supportedDataType(a.dataType, allowComplex = true)) From e30e7767f3bca008c07a635cb47f4d2a6d735e1c Mon Sep 17 00:00:00 2001 From: Andy Grove Date: Sat, 8 Nov 2025 07:26:31 -0700 Subject: [PATCH 05/44] save --- .../apache/comet/serde/CometNativeScan.scala | 218 ++++++++++++ .../apache/comet/serde/QueryPlanSerde.scala | 319 +++++------------- 2 files changed, 298 insertions(+), 239 deletions(-) create mode 100644 spark/src/main/scala/org/apache/comet/serde/CometNativeScan.scala diff --git a/spark/src/main/scala/org/apache/comet/serde/CometNativeScan.scala b/spark/src/main/scala/org/apache/comet/serde/CometNativeScan.scala new file mode 100644 index 0000000000..476313a9d1 --- /dev/null +++ b/spark/src/main/scala/org/apache/comet/serde/CometNativeScan.scala @@ -0,0 +1,218 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.comet.serde + +import scala.collection.mutable.ListBuffer +import scala.jdk.CollectionConverters._ + +import org.apache.spark.internal.Logging +import org.apache.spark.sql.catalyst.expressions.Literal +import org.apache.spark.sql.catalyst.util.ResolveDefaultColumns.getExistenceDefaultValues +import org.apache.spark.sql.comet.CometScanExec +import org.apache.spark.sql.execution.datasources.{FilePartition, FileScanRDD, PartitionedFile} +import org.apache.spark.sql.execution.datasources.v2.{DataSourceRDD, DataSourceRDDPartition} +import org.apache.spark.sql.internal.SQLConf +import org.apache.spark.sql.types.{StructField, StructType} + +import org.apache.comet.{CometConf, ConfigEntry} +import org.apache.comet.CometSparkSessionExtensions.withInfo +import org.apache.comet.objectstore.NativeConfig +import org.apache.comet.parquet.CometParquetUtils +import org.apache.comet.serde.ExprOuterClass.Expr +import org.apache.comet.serde.OperatorOuterClass.Operator +import org.apache.comet.serde.QueryPlanSerde.{exprToProto, serializeDataType} + +object CometNativeScan extends CometOperatorSerde[CometScanExec] with Logging { + + override def enabledConfig: Option[ConfigEntry[Boolean]] = None + + override def convert( + scan: CometScanExec, + builder: Operator.Builder, + childOp: OperatorOuterClass.Operator*): Option[OperatorOuterClass.Operator] = { + val nativeScanBuilder = OperatorOuterClass.NativeScan.newBuilder() + nativeScanBuilder.setSource(scan.simpleStringWithNodeId()) + + val scanTypes = scan.output.flatten { attr => + serializeDataType(attr.dataType) + } + + if (scanTypes.length == scan.output.length) { + nativeScanBuilder.addAllFields(scanTypes.asJava) + + // Sink operators don't have children + builder.clearChildren() + + if (scan.conf.getConf(SQLConf.PARQUET_FILTER_PUSHDOWN_ENABLED) && + CometConf.COMET_RESPECT_PARQUET_FILTER_PUSHDOWN.get(scan.conf)) { + + val dataFilters = new ListBuffer[Expr]() + for (filter <- scan.dataFilters) { + exprToProto(filter, scan.output) match { + case Some(proto) => dataFilters += proto + case _ => + logWarning(s"Unsupported data filter $filter") + } + } + nativeScanBuilder.addAllDataFilters(dataFilters.asJava) + } + + val possibleDefaultValues = getExistenceDefaultValues(scan.requiredSchema) + if (possibleDefaultValues.exists(_ != null)) { + // Our schema has default values. Serialize two lists, one with the default values + // and another with the indexes in the schema so the native side can map missing + // columns to these default values. + val (defaultValues, indexes) = possibleDefaultValues.zipWithIndex + .filter { case (expr, _) => expr != null } + .map { case (expr, index) => + // ResolveDefaultColumnsUtil.getExistenceDefaultValues has evaluated these + // expressions and they should now just be literals. + (Literal(expr), index.toLong.asInstanceOf[java.lang.Long]) + } + .unzip + nativeScanBuilder.addAllDefaultValues( + defaultValues.flatMap(exprToProto(_, scan.output)).toIterable.asJava) + nativeScanBuilder.addAllDefaultValuesIndexes(indexes.toIterable.asJava) + } + + // TODO: modify CometNativeScan to generate the file partitions without instantiating RDD. + var firstPartition: Option[PartitionedFile] = None + scan.inputRDD match { + case rdd: DataSourceRDD => + val partitions = rdd.partitions + partitions.foreach(p => { + val inputPartitions = p.asInstanceOf[DataSourceRDDPartition].inputPartitions + inputPartitions.foreach(partition => { + if (firstPartition.isEmpty) { + firstPartition = partition.asInstanceOf[FilePartition].files.headOption + } + partition2Proto( + partition.asInstanceOf[FilePartition], + nativeScanBuilder, + scan.relation.partitionSchema) + }) + }) + case rdd: FileScanRDD => + rdd.filePartitions.foreach(partition => { + if (firstPartition.isEmpty) { + firstPartition = partition.files.headOption + } + partition2Proto(partition, nativeScanBuilder, scan.relation.partitionSchema) + }) + case _ => + } + + val partitionSchema = schema2Proto(scan.relation.partitionSchema.fields) + val requiredSchema = schema2Proto(scan.requiredSchema.fields) + val dataSchema = schema2Proto(scan.relation.dataSchema.fields) + + val dataSchemaIndexes = scan.requiredSchema.fields.map(field => { + scan.relation.dataSchema.fieldIndex(field.name) + }) + val partitionSchemaIndexes = Array + .range( + scan.relation.dataSchema.fields.length, + scan.relation.dataSchema.length + scan.relation.partitionSchema.fields.length) + + val projectionVector = (dataSchemaIndexes ++ partitionSchemaIndexes).map(idx => + idx.toLong.asInstanceOf[java.lang.Long]) + + nativeScanBuilder.addAllProjectionVector(projectionVector.toIterable.asJava) + + // In `CometScanRule`, we ensure partitionSchema is supported. + assert(partitionSchema.length == scan.relation.partitionSchema.fields.length) + + nativeScanBuilder.addAllDataSchema(dataSchema.toIterable.asJava) + nativeScanBuilder.addAllRequiredSchema(requiredSchema.toIterable.asJava) + nativeScanBuilder.addAllPartitionSchema(partitionSchema.toIterable.asJava) + nativeScanBuilder.setSessionTimezone(scan.conf.getConfString("spark.sql.session.timeZone")) + nativeScanBuilder.setCaseSensitive(scan.conf.getConf[Boolean](SQLConf.CASE_SENSITIVE)) + + // Collect S3/cloud storage configurations + val hadoopConf = scan.relation.sparkSession.sessionState + .newHadoopConfWithOptions(scan.relation.options) + + nativeScanBuilder.setEncryptionEnabled(CometParquetUtils.encryptionEnabled(hadoopConf)) + + firstPartition.foreach { partitionFile => + val objectStoreOptions = + NativeConfig.extractObjectStoreOptions(hadoopConf, partitionFile.pathUri) + objectStoreOptions.foreach { case (key, value) => + nativeScanBuilder.putObjectStoreOptions(key, value) + } + } + + Some(builder.setNativeScan(nativeScanBuilder).build()) + + } else { + // There are unsupported scan type + withInfo( + scan, + s"unsupported Comet operator: ${scan.nodeName}, due to unsupported data types above") + None + } + + } + + private def schema2Proto( + fields: Array[StructField]): Array[OperatorOuterClass.SparkStructField] = { + val fieldBuilder = OperatorOuterClass.SparkStructField.newBuilder() + fields.map(field => { + fieldBuilder.setName(field.name) + fieldBuilder.setDataType(serializeDataType(field.dataType).get) + fieldBuilder.setNullable(field.nullable) + fieldBuilder.build() + }) + } + + private def partition2Proto( + partition: FilePartition, + nativeScanBuilder: OperatorOuterClass.NativeScan.Builder, + partitionSchema: StructType): Unit = { + val partitionBuilder = OperatorOuterClass.SparkFilePartition.newBuilder() + partition.files.foreach(file => { + // Process the partition values + val partitionValues = file.partitionValues + assert(partitionValues.numFields == partitionSchema.length) + val partitionVals = + partitionValues.toSeq(partitionSchema).zipWithIndex.map { case (value, i) => + val attr = partitionSchema(i) + val valueProto = exprToProto(Literal(value, attr.dataType), Seq.empty) + // In `CometScanRule`, we have already checked that all partition values are + // supported. So, we can safely use `get` here. + assert( + valueProto.isDefined, + s"Unsupported partition value: $value, type: ${attr.dataType}") + valueProto.get + } + + val fileBuilder = OperatorOuterClass.SparkPartitionedFile.newBuilder() + partitionVals.foreach(fileBuilder.addPartitionValues) + fileBuilder + .setFilePath(file.filePath.toString) + .setStart(file.start) + .setLength(file.length) + .setFileSize(file.fileSize) + partitionBuilder.addPartitionedFile(fileBuilder.build()) + }) + nativeScanBuilder.addFilePartitions(partitionBuilder.build()) + } + +} diff --git a/spark/src/main/scala/org/apache/comet/serde/QueryPlanSerde.scala b/spark/src/main/scala/org/apache/comet/serde/QueryPlanSerde.scala index 62ebd96c75..d2a767a797 100644 --- a/spark/src/main/scala/org/apache/comet/serde/QueryPlanSerde.scala +++ b/spark/src/main/scala/org/apache/comet/serde/QueryPlanSerde.scala @@ -19,7 +19,6 @@ package org.apache.comet.serde -import scala.collection.mutable.ListBuffer import scala.jdk.CollectionConverters._ import org.apache.spark.internal.Logging @@ -28,15 +27,12 @@ import org.apache.spark.sql.catalyst.expressions.aggregate._ import org.apache.spark.sql.catalyst.expressions.objects.StaticInvoke import org.apache.spark.sql.catalyst.optimizer.NormalizeNaNAndZero import org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils -import org.apache.spark.sql.catalyst.util.ResolveDefaultColumns.getExistenceDefaultValues import org.apache.spark.sql.comet._ import org.apache.spark.sql.comet.execution.shuffle.CometShuffleExchangeExec import org.apache.spark.sql.execution import org.apache.spark.sql.execution._ import org.apache.spark.sql.execution.adaptive.{BroadcastQueryStageExec, ShuffleQueryStageExec} import org.apache.spark.sql.execution.aggregate.{BaseAggregateExec, HashAggregateExec, ObjectHashAggregateExec} -import org.apache.spark.sql.execution.datasources.{FilePartition, FileScanRDD, PartitionedFile} -import org.apache.spark.sql.execution.datasources.v2.{DataSourceRDD, DataSourceRDDPartition} import org.apache.spark.sql.execution.exchange.{BroadcastExchangeExec, ReusedExchangeExec, ShuffleExchangeExec} import org.apache.spark.sql.execution.joins.{HashJoin, SortMergeJoinExec} import org.apache.spark.sql.execution.window.WindowExec @@ -46,8 +42,6 @@ import org.apache.spark.sql.types._ import org.apache.comet.{CometConf, ConfigEntry} import org.apache.comet.CometSparkSessionExtensions.{isCometScan, withInfo} import org.apache.comet.expressions._ -import org.apache.comet.objectstore.NativeConfig -import org.apache.comet.parquet.CometParquetUtils import org.apache.comet.serde.ExprOuterClass.{AggExpr, Expr, ScalarFunc} import org.apache.comet.serde.OperatorOuterClass.{AggregateMode => CometAggregateMode, Operator} import org.apache.comet.serde.QueryPlanSerde.{exprToProtoInternal, optExprWithInfo, scalarFunctionExprToProto} @@ -938,127 +932,7 @@ object QueryPlanSerde extends Logging with CometExprShim { // Fully native scan for V1 case scan: CometScanExec if scan.scanImpl == CometConf.SCAN_NATIVE_DATAFUSION => - val nativeScanBuilder = OperatorOuterClass.NativeScan.newBuilder() - nativeScanBuilder.setSource(op.simpleStringWithNodeId()) - - val scanTypes = op.output.flatten { attr => - serializeDataType(attr.dataType) - } - - if (scanTypes.length == op.output.length) { - nativeScanBuilder.addAllFields(scanTypes.asJava) - - // Sink operators don't have children - builder.clearChildren() - - if (conf.getConf(SQLConf.PARQUET_FILTER_PUSHDOWN_ENABLED) && - CometConf.COMET_RESPECT_PARQUET_FILTER_PUSHDOWN.get(conf)) { - - val dataFilters = new ListBuffer[Expr]() - for (filter <- scan.dataFilters) { - exprToProto(filter, scan.output) match { - case Some(proto) => dataFilters += proto - case _ => - logWarning(s"Unsupported data filter $filter") - } - } - nativeScanBuilder.addAllDataFilters(dataFilters.asJava) - } - - val possibleDefaultValues = getExistenceDefaultValues(scan.requiredSchema) - if (possibleDefaultValues.exists(_ != null)) { - // Our schema has default values. Serialize two lists, one with the default values - // and another with the indexes in the schema so the native side can map missing - // columns to these default values. - val (defaultValues, indexes) = possibleDefaultValues.zipWithIndex - .filter { case (expr, _) => expr != null } - .map { case (expr, index) => - // ResolveDefaultColumnsUtil.getExistenceDefaultValues has evaluated these - // expressions and they should now just be literals. - (Literal(expr), index.toLong.asInstanceOf[java.lang.Long]) - } - .unzip - nativeScanBuilder.addAllDefaultValues( - defaultValues.flatMap(exprToProto(_, scan.output)).toIterable.asJava) - nativeScanBuilder.addAllDefaultValuesIndexes(indexes.toIterable.asJava) - } - - // TODO: modify CometNativeScan to generate the file partitions without instantiating RDD. - var firstPartition: Option[PartitionedFile] = None - scan.inputRDD match { - case rdd: DataSourceRDD => - val partitions = rdd.partitions - partitions.foreach(p => { - val inputPartitions = p.asInstanceOf[DataSourceRDDPartition].inputPartitions - inputPartitions.foreach(partition => { - if (firstPartition.isEmpty) { - firstPartition = partition.asInstanceOf[FilePartition].files.headOption - } - partition2Proto( - partition.asInstanceOf[FilePartition], - nativeScanBuilder, - scan.relation.partitionSchema) - }) - }) - case rdd: FileScanRDD => - rdd.filePartitions.foreach(partition => { - if (firstPartition.isEmpty) { - firstPartition = partition.files.headOption - } - partition2Proto(partition, nativeScanBuilder, scan.relation.partitionSchema) - }) - case _ => - } - - val partitionSchema = schema2Proto(scan.relation.partitionSchema.fields) - val requiredSchema = schema2Proto(scan.requiredSchema.fields) - val dataSchema = schema2Proto(scan.relation.dataSchema.fields) - - val dataSchemaIndexes = scan.requiredSchema.fields.map(field => { - scan.relation.dataSchema.fieldIndex(field.name) - }) - val partitionSchemaIndexes = Array - .range( - scan.relation.dataSchema.fields.length, - scan.relation.dataSchema.length + scan.relation.partitionSchema.fields.length) - - val projectionVector = (dataSchemaIndexes ++ partitionSchemaIndexes).map(idx => - idx.toLong.asInstanceOf[java.lang.Long]) - - nativeScanBuilder.addAllProjectionVector(projectionVector.toIterable.asJava) - - // In `CometScanRule`, we ensure partitionSchema is supported. - assert(partitionSchema.length == scan.relation.partitionSchema.fields.length) - - nativeScanBuilder.addAllDataSchema(dataSchema.toIterable.asJava) - nativeScanBuilder.addAllRequiredSchema(requiredSchema.toIterable.asJava) - nativeScanBuilder.addAllPartitionSchema(partitionSchema.toIterable.asJava) - nativeScanBuilder.setSessionTimezone(conf.getConfString("spark.sql.session.timeZone")) - nativeScanBuilder.setCaseSensitive(conf.getConf[Boolean](SQLConf.CASE_SENSITIVE)) - - // Collect S3/cloud storage configurations - val hadoopConf = scan.relation.sparkSession.sessionState - .newHadoopConfWithOptions(scan.relation.options) - - nativeScanBuilder.setEncryptionEnabled(CometParquetUtils.encryptionEnabled(hadoopConf)) - - firstPartition.foreach { partitionFile => - val objectStoreOptions = - NativeConfig.extractObjectStoreOptions(hadoopConf, partitionFile.pathUri) - objectStoreOptions.foreach { case (key, value) => - nativeScanBuilder.putObjectStoreOptions(key, value) - } - } - - Some(builder.setNativeScan(nativeScanBuilder).build()) - - } else { - // There are unsupported scan type - withInfo( - op, - s"unsupported Comet operator: ${op.nodeName}, due to unsupported data types above") - None - } + CometNativeScan.convert(scan, builder, childOp: _*) case ExpandExec(projections, _, child) if CometConf.COMET_EXEC_EXPAND_ENABLED.get(conf) => var allProjExprs: Seq[Expression] = Seq() @@ -1079,7 +953,7 @@ object QueryPlanSerde extends Logging with CometExprShim { } case WindowExec(windowExpression, partitionSpec, orderSpec, child) - if CometConf.COMET_EXEC_WINDOW_ENABLED.get(conf) => + if CometConf.COMET_EXEC_WINDOW_ENABLED.get(conf) => val output = child.output val winExprs: Array[WindowExpression] = windowExpression.flatMap { expr => @@ -1123,9 +997,9 @@ object QueryPlanSerde extends Logging with CometExprShim { } case aggregate: BaseAggregateExec - if (aggregate.isInstanceOf[HashAggregateExec] || - aggregate.isInstanceOf[ObjectHashAggregateExec]) && - CometConf.COMET_EXEC_AGGREGATE_ENABLED.get(conf) => + if (aggregate.isInstanceOf[HashAggregateExec] || + aggregate.isInstanceOf[ObjectHashAggregateExec]) && + CometConf.COMET_EXEC_AGGREGATE_ENABLED.get(conf) => val groupingExpressions = aggregate.groupingExpressions val aggregateExpressions = aggregate.aggregateExpressions val aggregateAttributes = aggregate.aggregateAttributes @@ -1144,10 +1018,10 @@ object QueryPlanSerde extends Logging with CometExprShim { } if (groupingExpressions.exists(expr => - expr.dataType match { - case _: MapType => true - case _ => false - })) { + expr.dataType match { + case _: MapType => true + case _ => false + })) { withInfo(op, "Grouping on map types is not supported") return None } @@ -1249,60 +1123,6 @@ object QueryPlanSerde extends Logging with CometExprShim { } } - case op if isCometSink(op) => - val supportedTypes = - op.output.forall(a => supportedDataType(a.dataType, allowComplex = true)) - - if (!supportedTypes) { - withInfo(op, "Unsupported data type") - return None - } - - // These operators are source of Comet native execution chain - val scanBuilder = OperatorOuterClass.Scan.newBuilder() - val source = op.simpleStringWithNodeId() - if (source.isEmpty) { - scanBuilder.setSource(op.getClass.getSimpleName) - } else { - scanBuilder.setSource(source) - } - - val ffiSafe = op match { - case _ if isExchangeSink(op) => - // Source of broadcast exchange batches is ArrowStreamReader - // Source of shuffle exchange batches is NativeBatchDecoderIterator - true - case scan: CometScanExec if scan.scanImpl == CometConf.SCAN_NATIVE_COMET => - // native_comet scan reuses mutable buffers - false - case scan: CometScanExec if scan.scanImpl == CometConf.SCAN_NATIVE_ICEBERG_COMPAT => - // native_iceberg_compat scan reuses mutable buffers for constant columns - // https://github.com/apache/datafusion-comet/issues/2152 - false - case _ => - false - } - scanBuilder.setArrowFfiSafe(ffiSafe) - - val scanTypes = op.output.flatten { attr => - serializeDataType(attr.dataType) - } - - if (scanTypes.length == op.output.length) { - scanBuilder.addAllFields(scanTypes.asJava) - - // Sink operators don't have children - builder.clearChildren() - - Some(builder.setScan(scanBuilder).build()) - } else { - // There are unsupported scan type - withInfo( - op, - s"unsupported Comet operator: ${op.nodeName}, due to unsupported data types above") - None - } - case op => opSerdeMap.get(op.getClass) match { case Some(handler) => @@ -1312,22 +1132,86 @@ object QueryPlanSerde extends Logging with CometExprShim { op, s"Native support for operator ${op.getClass.getSimpleName} is disabled. " + s"Set ${enabledConfig.key}=true to enable it.") - return None + if (isCometSink(op)) { + return cometSink(op) + } else { + return None + } } } handler.asInstanceOf[CometOperatorSerde[SparkPlan]].convert(op, builder, childOp: _*) case _ => - // Emit warning if: - // 1. it is not Spark shuffle operator, which is handled separately - // 2. it is not a Comet operator - if (!op.nodeName.contains("Comet") && !op.isInstanceOf[ShuffleExchangeExec]) { - withInfo(op, s"unsupported Spark operator: ${op.nodeName}") + if (isCometSink(op)) { + cometSink(op) + } else { + // Emit warning if: + // 1. it is not Spark shuffle operator, which is handled separately + // 2. it is not a Comet operator + if (!op.nodeName.contains("Comet") && !op.isInstanceOf[ShuffleExchangeExec]) { + withInfo(op, s"unsupported Spark operator: ${op.nodeName}") + } + None } - None } } } + def cometSink(op: SparkPlan): Option[Operator] = { + val supportedTypes = + op.output.forall(a => supportedDataType(a.dataType, allowComplex = true)) + + if (!supportedTypes) { + withInfo(op, "Unsupported data type") + return None + } + + // These operators are source of Comet native execution chain + val scanBuilder = OperatorOuterClass.Scan.newBuilder() + val source = op.simpleStringWithNodeId() + if (source.isEmpty) { + scanBuilder.setSource(op.getClass.getSimpleName) + } else { + scanBuilder.setSource(source) + } + + val ffiSafe = op match { + case _ if isExchangeSink(op) => + // Source of broadcast exchange batches is ArrowStreamReader + // Source of shuffle exchange batches is NativeBatchDecoderIterator + true + case scan: CometScanExec if scan.scanImpl == CometConf.SCAN_NATIVE_COMET => + // native_comet scan reuses mutable buffers + false + case scan: CometScanExec if scan.scanImpl == CometConf.SCAN_NATIVE_ICEBERG_COMPAT => + // native_iceberg_compat scan reuses mutable buffers for constant columns + // https://github.com/apache/datafusion-comet/issues/2152 + false + case _ => + false + } + scanBuilder.setArrowFfiSafe(ffiSafe) + + val scanTypes = op.output.flatten { attr => + serializeDataType(attr.dataType) + } + + if (scanTypes.length == op.output.length) { + scanBuilder.addAllFields(scanTypes.asJava) + + // Sink operators don't have children + builder.clearChildren() + + Some(builder.setScan(scanBuilder).build()) + } else { + // There are unsupported scan type + withInfo( + op, + s"unsupported Comet operator: ${op.nodeName}, due to unsupported data types above") + None + } + + } + /** * Whether the input Spark operator `op` can be considered as a Comet sink, i.e., the start of * native execution. If it is true, we'll wrap `op` with `CometScanWrapper` or @@ -1458,49 +1342,6 @@ object QueryPlanSerde extends Logging with CometExprShim { true } - private def schema2Proto( - fields: Array[StructField]): Array[OperatorOuterClass.SparkStructField] = { - val fieldBuilder = OperatorOuterClass.SparkStructField.newBuilder() - fields.map(field => { - fieldBuilder.setName(field.name) - fieldBuilder.setDataType(serializeDataType(field.dataType).get) - fieldBuilder.setNullable(field.nullable) - fieldBuilder.build() - }) - } - - private def partition2Proto( - partition: FilePartition, - nativeScanBuilder: OperatorOuterClass.NativeScan.Builder, - partitionSchema: StructType): Unit = { - val partitionBuilder = OperatorOuterClass.SparkFilePartition.newBuilder() - partition.files.foreach(file => { - // Process the partition values - val partitionValues = file.partitionValues - assert(partitionValues.numFields == partitionSchema.length) - val partitionVals = - partitionValues.toSeq(partitionSchema).zipWithIndex.map { case (value, i) => - val attr = partitionSchema(i) - val valueProto = exprToProto(Literal(value, attr.dataType), Seq.empty) - // In `CometScanRule`, we have already checked that all partition values are - // supported. So, we can safely use `get` here. - assert( - valueProto.isDefined, - s"Unsupported partition value: $value, type: ${attr.dataType}") - valueProto.get - } - - val fileBuilder = OperatorOuterClass.SparkPartitionedFile.newBuilder() - partitionVals.foreach(fileBuilder.addPartitionValues) - fileBuilder - .setFilePath(file.filePath.toString) - .setStart(file.start) - .setLength(file.length) - .setFileSize(file.fileSize) - partitionBuilder.addPartitionedFile(fileBuilder.build()) - }) - nativeScanBuilder.addFilePartitions(partitionBuilder.build()) - } } sealed trait SupportLevel From ac30640b052ae3af0fe7a4bb12f8b84a4432f760 Mon Sep 17 00:00:00 2001 From: Andy Grove Date: Sat, 8 Nov 2025 07:37:49 -0700 Subject: [PATCH 06/44] agg --- .../apache/comet/serde/CometAggregate.scala | 168 ++++++++++++++++++ .../org/apache/comet/serde/CometExpand.scala | 60 +++++++ ...ometWindowExec.scala => CometWindow.scala} | 2 +- .../apache/comet/serde/QueryPlanSerde.scala | 146 +-------------- 4 files changed, 233 insertions(+), 143 deletions(-) create mode 100644 spark/src/main/scala/org/apache/comet/serde/CometAggregate.scala create mode 100644 spark/src/main/scala/org/apache/comet/serde/CometExpand.scala rename spark/src/main/scala/org/apache/comet/serde/{CometWindowExec.scala => CometWindow.scala} (98%) diff --git a/spark/src/main/scala/org/apache/comet/serde/CometAggregate.scala b/spark/src/main/scala/org/apache/comet/serde/CometAggregate.scala new file mode 100644 index 0000000000..9b577d5a91 --- /dev/null +++ b/spark/src/main/scala/org/apache/comet/serde/CometAggregate.scala @@ -0,0 +1,168 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.comet.serde + +import scala.jdk.CollectionConverters._ + +import org.apache.spark.sql.catalyst.expressions.Expression +import org.apache.spark.sql.catalyst.expressions.aggregate.{Final, Partial} +import org.apache.spark.sql.execution.aggregate.BaseAggregateExec +import org.apache.spark.sql.types.MapType + +import org.apache.comet.{CometConf, ConfigEntry} +import org.apache.comet.CometSparkSessionExtensions.withInfo +import org.apache.comet.serde.OperatorOuterClass.{AggregateMode => CometAggregateMode, Operator} +import org.apache.comet.serde.QueryPlanSerde.{aggExprToProto, exprToProto} + +object CometAggregate extends CometOperatorSerde[BaseAggregateExec] { + + override def enabledConfig: Option[ConfigEntry[Boolean]] = Some( + CometConf.COMET_EXEC_AGGREGATE_ENABLED) + + override def convert( + aggregate: BaseAggregateExec, + builder: Operator.Builder, + childOp: OperatorOuterClass.Operator*): Option[OperatorOuterClass.Operator] = { + val groupingExpressions = aggregate.groupingExpressions + val aggregateExpressions = aggregate.aggregateExpressions + val aggregateAttributes = aggregate.aggregateAttributes + val resultExpressions = aggregate.resultExpressions + val child = aggregate.child + + if (groupingExpressions.isEmpty && aggregateExpressions.isEmpty) { + withInfo(aggregate, "No group by or aggregation") + return None + } + + // Aggregate expressions with filter are not supported yet. + if (aggregateExpressions.exists(_.filter.isDefined)) { + withInfo(aggregate, "Aggregate expression with filter is not supported") + return None + } + + if (groupingExpressions.exists(expr => + expr.dataType match { + case _: MapType => true + case _ => false + })) { + withInfo(aggregate, "Grouping on map types is not supported") + return None + } + + val groupingExprsWithInput = + groupingExpressions.map(expr => expr.name -> exprToProto(expr, child.output)) + + val emptyExprs = groupingExprsWithInput.collect { + case (expr, proto) if proto.isEmpty => expr + } + + if (emptyExprs.nonEmpty) { + withInfo(aggregate, s"Unsupported group expressions: ${emptyExprs.mkString(", ")}") + return None + } + + val groupingExprs = groupingExprsWithInput.map(_._2) + + // In some of the cases, the aggregateExpressions could be empty. + // For example, if the aggregate functions only have group by or if the aggregate + // functions only have distinct aggregate functions: + // + // SELECT COUNT(distinct col2), col1 FROM test group by col1 + // +- HashAggregate (keys =[col1# 6], functions =[count (distinct col2#7)] ) + // +- Exchange hashpartitioning (col1#6, 10), ENSURE_REQUIREMENTS, [plan_id = 36] + // +- HashAggregate (keys =[col1#6], functions =[partial_count (distinct col2#7)] ) + // +- HashAggregate (keys =[col1#6, col2#7], functions =[] ) + // +- Exchange hashpartitioning (col1#6, col2#7, 10), ENSURE_REQUIREMENTS, ... + // +- HashAggregate (keys =[col1#6, col2#7], functions =[] ) + // +- FileScan parquet spark_catalog.default.test[col1#6, col2#7] ...... + // If the aggregateExpressions is empty, we only want to build groupingExpressions, + // and skip processing of aggregateExpressions. + if (aggregateExpressions.isEmpty) { + val hashAggBuilder = OperatorOuterClass.HashAggregate.newBuilder() + hashAggBuilder.addAllGroupingExprs(groupingExprs.map(_.get).asJava) + val attributes = groupingExpressions.map(_.toAttribute) ++ aggregateAttributes + val resultExprs = resultExpressions.map(exprToProto(_, attributes)) + if (resultExprs.exists(_.isEmpty)) { + withInfo( + aggregate, + s"Unsupported result expressions found in: $resultExpressions", + resultExpressions: _*) + return None + } + hashAggBuilder.addAllResultExprs(resultExprs.map(_.get).asJava) + Some(builder.setHashAgg(hashAggBuilder).build()) + } else { + val modes = aggregateExpressions.map(_.mode).distinct + + if (modes.size != 1) { + // This shouldn't happen as all aggregation expressions should share the same mode. + // Fallback to Spark nevertheless here. + withInfo(aggregate, "All aggregate expressions do not have the same mode") + return None + } + + val mode = modes.head match { + case Partial => CometAggregateMode.Partial + case Final => CometAggregateMode.Final + case _ => + withInfo(aggregate, s"Unsupported aggregation mode ${modes.head}") + return None + } + + // In final mode, the aggregate expressions are bound to the output of the + // child and partial aggregate expressions buffer attributes produced by partial + // aggregation. This is done in Spark `HashAggregateExec` internally. In Comet, + // we don't have to do this because we don't use the merging expression. + val binding = mode != CometAggregateMode.Final + // `output` is only used when `binding` is true (i.e., non-Final) + val output = child.output + + val aggExprs = + aggregateExpressions.map(aggExprToProto(_, output, binding, aggregate.conf)) + if (childOp.nonEmpty && groupingExprs.forall(_.isDefined) && + aggExprs.forall(_.isDefined)) { + val hashAggBuilder = OperatorOuterClass.HashAggregate.newBuilder() + hashAggBuilder.addAllGroupingExprs(groupingExprs.map(_.get).asJava) + hashAggBuilder.addAllAggExprs(aggExprs.map(_.get).asJava) + if (mode == CometAggregateMode.Final) { + val attributes = groupingExpressions.map(_.toAttribute) ++ aggregateAttributes + val resultExprs = resultExpressions.map(exprToProto(_, attributes)) + if (resultExprs.exists(_.isEmpty)) { + withInfo( + aggregate, + s"Unsupported result expressions found in: $resultExpressions", + resultExpressions: _*) + return None + } + hashAggBuilder.addAllResultExprs(resultExprs.map(_.get).asJava) + } + hashAggBuilder.setModeValue(mode.getNumber) + Some(builder.setHashAgg(hashAggBuilder).build()) + } else { + val allChildren: Seq[Expression] = + groupingExpressions ++ aggregateExpressions ++ aggregateAttributes + withInfo(aggregate, allChildren: _*) + None + } + } + + } + +} diff --git a/spark/src/main/scala/org/apache/comet/serde/CometExpand.scala b/spark/src/main/scala/org/apache/comet/serde/CometExpand.scala new file mode 100644 index 0000000000..5979eed4dc --- /dev/null +++ b/spark/src/main/scala/org/apache/comet/serde/CometExpand.scala @@ -0,0 +1,60 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.comet.serde + +import scala.jdk.CollectionConverters._ + +import org.apache.spark.sql.catalyst.expressions.Expression +import org.apache.spark.sql.execution.ExpandExec + +import org.apache.comet.{CometConf, ConfigEntry} +import org.apache.comet.CometSparkSessionExtensions.withInfo +import org.apache.comet.serde.OperatorOuterClass.Operator +import org.apache.comet.serde.QueryPlanSerde.exprToProto + +object CometExpand extends CometOperatorSerde[ExpandExec] { + + override def enabledConfig: Option[ConfigEntry[Boolean]] = Some( + CometConf.COMET_EXEC_EXPAND_ENABLED) + + override def convert( + op: ExpandExec, + builder: Operator.Builder, + childOp: OperatorOuterClass.Operator*): Option[OperatorOuterClass.Operator] = { + var allProjExprs: Seq[Expression] = Seq() + val projExprs = op.projections.flatMap(_.map(e => { + allProjExprs = allProjExprs :+ e + exprToProto(e, op.child.output) + })) + + if (projExprs.forall(_.isDefined) && childOp.nonEmpty) { + val expandBuilder = OperatorOuterClass.Expand + .newBuilder() + .addAllProjectList(projExprs.map(_.get).asJava) + .setNumExprPerProject(op.projections.head.size) + Some(builder.setExpand(expandBuilder).build()) + } else { + withInfo(op, allProjExprs: _*) + None + } + + } + +} diff --git a/spark/src/main/scala/org/apache/comet/serde/CometWindowExec.scala b/spark/src/main/scala/org/apache/comet/serde/CometWindow.scala similarity index 98% rename from spark/src/main/scala/org/apache/comet/serde/CometWindowExec.scala rename to spark/src/main/scala/org/apache/comet/serde/CometWindow.scala index dafe019421..7e963d6326 100644 --- a/spark/src/main/scala/org/apache/comet/serde/CometWindowExec.scala +++ b/spark/src/main/scala/org/apache/comet/serde/CometWindow.scala @@ -30,7 +30,7 @@ import org.apache.comet.CometSparkSessionExtensions.withInfo import org.apache.comet.serde.OperatorOuterClass.Operator import org.apache.comet.serde.QueryPlanSerde.{exprToProto, windowExprToProto} -object CometWindowExec extends CometOperatorSerde[WindowExec] { +object CometWindow extends CometOperatorSerde[WindowExec] { override def enabledConfig: Option[ConfigEntry[Boolean]] = Some( CometConf.COMET_EXEC_WINDOW_ENABLED) diff --git a/spark/src/main/scala/org/apache/comet/serde/QueryPlanSerde.scala b/spark/src/main/scala/org/apache/comet/serde/QueryPlanSerde.scala index b6d44496dd..8b1c10ee88 100644 --- a/spark/src/main/scala/org/apache/comet/serde/QueryPlanSerde.scala +++ b/spark/src/main/scala/org/apache/comet/serde/QueryPlanSerde.scala @@ -43,7 +43,7 @@ import org.apache.comet.{CometConf, ConfigEntry} import org.apache.comet.CometSparkSessionExtensions.{isCometScan, withInfo} import org.apache.comet.expressions._ import org.apache.comet.serde.ExprOuterClass.{AggExpr, Expr, ScalarFunc} -import org.apache.comet.serde.OperatorOuterClass.{AggregateMode => CometAggregateMode, Operator} +import org.apache.comet.serde.OperatorOuterClass.Operator import org.apache.comet.serde.QueryPlanSerde.{exprToProtoInternal, optExprWithInfo, scalarFunctionExprToProto} import org.apache.comet.serde.Types.{DataType => ProtoDataType} import org.apache.comet.serde.Types.DataType._ @@ -66,7 +66,8 @@ object QueryPlanSerde extends Logging with CometExprShim { classOf[GlobalLimitExec] -> CometGlobalLimit, classOf[HashJoin] -> CometHashJoin, classOf[SortMergeJoinExec] -> CometSortMergeJoin, - classOf[WindowExec] -> CometWindowExec, + classOf[ExpandExec] -> CometExpand, + classOf[WindowExec] -> CometWindow, classOf[SortExec] -> CometSort) private val arrayExpressions: Map[Class[_ <: Expression], CometExpressionSerde[_]] = Map( @@ -935,150 +936,11 @@ object QueryPlanSerde extends Logging with CometExprShim { case scan: CometScanExec if scan.scanImpl == CometConf.SCAN_NATIVE_DATAFUSION => CometNativeScan.convert(scan, builder, childOp: _*) - case ExpandExec(projections, _, child) if CometConf.COMET_EXEC_EXPAND_ENABLED.get(conf) => - var allProjExprs: Seq[Expression] = Seq() - val projExprs = projections.flatMap(_.map(e => { - allProjExprs = allProjExprs :+ e - exprToProto(e, child.output) - })) - - if (projExprs.forall(_.isDefined) && childOp.nonEmpty) { - val expandBuilder = OperatorOuterClass.Expand - .newBuilder() - .addAllProjectList(projExprs.map(_.get).asJava) - .setNumExprPerProject(projections.head.size) - Some(builder.setExpand(expandBuilder).build()) - } else { - withInfo(op, allProjExprs: _*) - None - } - case aggregate: BaseAggregateExec if (aggregate.isInstanceOf[HashAggregateExec] || aggregate.isInstanceOf[ObjectHashAggregateExec]) && CometConf.COMET_EXEC_AGGREGATE_ENABLED.get(conf) => - val groupingExpressions = aggregate.groupingExpressions - val aggregateExpressions = aggregate.aggregateExpressions - val aggregateAttributes = aggregate.aggregateAttributes - val resultExpressions = aggregate.resultExpressions - val child = aggregate.child - - if (groupingExpressions.isEmpty && aggregateExpressions.isEmpty) { - withInfo(op, "No group by or aggregation") - return None - } - - // Aggregate expressions with filter are not supported yet. - if (aggregateExpressions.exists(_.filter.isDefined)) { - withInfo(op, "Aggregate expression with filter is not supported") - return None - } - - if (groupingExpressions.exists(expr => - expr.dataType match { - case _: MapType => true - case _ => false - })) { - withInfo(op, "Grouping on map types is not supported") - return None - } - - val groupingExprsWithInput = - groupingExpressions.map(expr => expr.name -> exprToProto(expr, child.output)) - - val emptyExprs = groupingExprsWithInput.collect { - case (expr, proto) if proto.isEmpty => expr - } - - if (emptyExprs.nonEmpty) { - withInfo(op, s"Unsupported group expressions: ${emptyExprs.mkString(", ")}") - return None - } - - val groupingExprs = groupingExprsWithInput.map(_._2) - - // In some of the cases, the aggregateExpressions could be empty. - // For example, if the aggregate functions only have group by or if the aggregate - // functions only have distinct aggregate functions: - // - // SELECT COUNT(distinct col2), col1 FROM test group by col1 - // +- HashAggregate (keys =[col1# 6], functions =[count (distinct col2#7)] ) - // +- Exchange hashpartitioning (col1#6, 10), ENSURE_REQUIREMENTS, [plan_id = 36] - // +- HashAggregate (keys =[col1#6], functions =[partial_count (distinct col2#7)] ) - // +- HashAggregate (keys =[col1#6, col2#7], functions =[] ) - // +- Exchange hashpartitioning (col1#6, col2#7, 10), ENSURE_REQUIREMENTS, ... - // +- HashAggregate (keys =[col1#6, col2#7], functions =[] ) - // +- FileScan parquet spark_catalog.default.test[col1#6, col2#7] ...... - // If the aggregateExpressions is empty, we only want to build groupingExpressions, - // and skip processing of aggregateExpressions. - if (aggregateExpressions.isEmpty) { - val hashAggBuilder = OperatorOuterClass.HashAggregate.newBuilder() - hashAggBuilder.addAllGroupingExprs(groupingExprs.map(_.get).asJava) - val attributes = groupingExpressions.map(_.toAttribute) ++ aggregateAttributes - val resultExprs = resultExpressions.map(exprToProto(_, attributes)) - if (resultExprs.exists(_.isEmpty)) { - withInfo( - op, - s"Unsupported result expressions found in: $resultExpressions", - resultExpressions: _*) - return None - } - hashAggBuilder.addAllResultExprs(resultExprs.map(_.get).asJava) - Some(builder.setHashAgg(hashAggBuilder).build()) - } else { - val modes = aggregateExpressions.map(_.mode).distinct - - if (modes.size != 1) { - // This shouldn't happen as all aggregation expressions should share the same mode. - // Fallback to Spark nevertheless here. - withInfo(op, "All aggregate expressions do not have the same mode") - return None - } - - val mode = modes.head match { - case Partial => CometAggregateMode.Partial - case Final => CometAggregateMode.Final - case _ => - withInfo(op, s"Unsupported aggregation mode ${modes.head}") - return None - } - - // In final mode, the aggregate expressions are bound to the output of the - // child and partial aggregate expressions buffer attributes produced by partial - // aggregation. This is done in Spark `HashAggregateExec` internally. In Comet, - // we don't have to do this because we don't use the merging expression. - val binding = mode != CometAggregateMode.Final - // `output` is only used when `binding` is true (i.e., non-Final) - val output = child.output - - val aggExprs = - aggregateExpressions.map(aggExprToProto(_, output, binding, op.conf)) - if (childOp.nonEmpty && groupingExprs.forall(_.isDefined) && - aggExprs.forall(_.isDefined)) { - val hashAggBuilder = OperatorOuterClass.HashAggregate.newBuilder() - hashAggBuilder.addAllGroupingExprs(groupingExprs.map(_.get).asJava) - hashAggBuilder.addAllAggExprs(aggExprs.map(_.get).asJava) - if (mode == CometAggregateMode.Final) { - val attributes = groupingExpressions.map(_.toAttribute) ++ aggregateAttributes - val resultExprs = resultExpressions.map(exprToProto(_, attributes)) - if (resultExprs.exists(_.isEmpty)) { - withInfo( - op, - s"Unsupported result expressions found in: $resultExpressions", - resultExpressions: _*) - return None - } - hashAggBuilder.addAllResultExprs(resultExprs.map(_.get).asJava) - } - hashAggBuilder.setModeValue(mode.getNumber) - Some(builder.setHashAgg(hashAggBuilder).build()) - } else { - val allChildren: Seq[Expression] = - groupingExpressions ++ aggregateExpressions ++ aggregateAttributes - withInfo(op, allChildren: _*) - None - } - } + CometAggregate.convert(aggregate, builder, childOp: _*) case op => opSerdeMap.get(op.getClass) match { From b515d8fe3b779e43cf2bdb7be81d048a450cff0a Mon Sep 17 00:00:00 2001 From: Andy Grove Date: Sat, 8 Nov 2025 07:41:53 -0700 Subject: [PATCH 07/44] docs --- docs/source/user-guide/latest/configs.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/source/user-guide/latest/configs.md b/docs/source/user-guide/latest/configs.md index 1cc05dfc78..bf6388549c 100644 --- a/docs/source/user-guide/latest/configs.md +++ b/docs/source/user-guide/latest/configs.md @@ -155,7 +155,7 @@ These settings can be used to determine which parts of the plan are accelerated | `spark.comet.exec.sortMergeJoinWithJoinFilter.enabled` | Experimental support for Sort Merge Join with filter | false | | `spark.comet.exec.takeOrderedAndProject.enabled` | Whether to enable takeOrderedAndProject by default. | true | | `spark.comet.exec.union.enabled` | Whether to enable union by default. | true | -| `spark.comet.exec.window.enabled` | Whether to enable window by default. | true | +| `spark.comet.exec.window.enabled` | Whether to enable window by default. | false | ## Enabling or Disabling Individual Scalar Expressions From fcab57ecf40ed1ec50e05fa84e00313c1ffcb221 Mon Sep 17 00:00:00 2001 From: Andy Grove Date: Sat, 8 Nov 2025 07:52:17 -0700 Subject: [PATCH 08/44] refactor --- .../apache/comet/serde/CometAggregate.scala | 37 +++++++-- .../apache/comet/serde/QueryPlanSerde.scala | 75 +++++++++---------- 2 files changed, 67 insertions(+), 45 deletions(-) diff --git a/spark/src/main/scala/org/apache/comet/serde/CometAggregate.scala b/spark/src/main/scala/org/apache/comet/serde/CometAggregate.scala index 9b577d5a91..f0cf244f1e 100644 --- a/spark/src/main/scala/org/apache/comet/serde/CometAggregate.scala +++ b/spark/src/main/scala/org/apache/comet/serde/CometAggregate.scala @@ -23,7 +23,7 @@ import scala.jdk.CollectionConverters._ import org.apache.spark.sql.catalyst.expressions.Expression import org.apache.spark.sql.catalyst.expressions.aggregate.{Final, Partial} -import org.apache.spark.sql.execution.aggregate.BaseAggregateExec +import org.apache.spark.sql.execution.aggregate.{BaseAggregateExec, HashAggregateExec, ObjectHashAggregateExec} import org.apache.spark.sql.types.MapType import org.apache.comet.{CometConf, ConfigEntry} @@ -31,12 +31,9 @@ import org.apache.comet.CometSparkSessionExtensions.withInfo import org.apache.comet.serde.OperatorOuterClass.{AggregateMode => CometAggregateMode, Operator} import org.apache.comet.serde.QueryPlanSerde.{aggExprToProto, exprToProto} -object CometAggregate extends CometOperatorSerde[BaseAggregateExec] { +trait CometBaseAggregate { - override def enabledConfig: Option[ConfigEntry[Boolean]] = Some( - CometConf.COMET_EXEC_AGGREGATE_ENABLED) - - override def convert( + def doConvert( aggregate: BaseAggregateExec, builder: Operator.Builder, childOp: OperatorOuterClass.Operator*): Option[OperatorOuterClass.Operator] = { @@ -166,3 +163,31 @@ object CometAggregate extends CometOperatorSerde[BaseAggregateExec] { } } + +object CometHashAggregate extends CometOperatorSerde[HashAggregateExec] with CometBaseAggregate { + + override def enabledConfig: Option[ConfigEntry[Boolean]] = Some( + CometConf.COMET_EXEC_AGGREGATE_ENABLED) + + override def convert( + aggregate: HashAggregateExec, + builder: Operator.Builder, + childOp: OperatorOuterClass.Operator*): Option[OperatorOuterClass.Operator] = { + doConvert(aggregate, builder, childOp: _*) + } +} + +object CometObjectHashAggregate + extends CometOperatorSerde[ObjectHashAggregateExec] + with CometBaseAggregate { + + override def enabledConfig: Option[ConfigEntry[Boolean]] = Some( + CometConf.COMET_EXEC_AGGREGATE_ENABLED) + + override def convert( + aggregate: ObjectHashAggregateExec, + builder: Operator.Builder, + childOp: OperatorOuterClass.Operator*): Option[OperatorOuterClass.Operator] = { + doConvert(aggregate, builder, childOp: _*) + } +} diff --git a/spark/src/main/scala/org/apache/comet/serde/QueryPlanSerde.scala b/spark/src/main/scala/org/apache/comet/serde/QueryPlanSerde.scala index 8b1c10ee88..a5f3805322 100644 --- a/spark/src/main/scala/org/apache/comet/serde/QueryPlanSerde.scala +++ b/spark/src/main/scala/org/apache/comet/serde/QueryPlanSerde.scala @@ -32,7 +32,7 @@ import org.apache.spark.sql.comet.execution.shuffle.CometShuffleExchangeExec import org.apache.spark.sql.execution import org.apache.spark.sql.execution._ import org.apache.spark.sql.execution.adaptive.{BroadcastQueryStageExec, ShuffleQueryStageExec} -import org.apache.spark.sql.execution.aggregate.{BaseAggregateExec, HashAggregateExec, ObjectHashAggregateExec} +import org.apache.spark.sql.execution.aggregate.{HashAggregateExec, ObjectHashAggregateExec} import org.apache.spark.sql.execution.exchange.{BroadcastExchangeExec, ReusedExchangeExec, ShuffleExchangeExec} import org.apache.spark.sql.execution.joins.{HashJoin, SortMergeJoinExec} import org.apache.spark.sql.execution.window.WindowExec @@ -64,6 +64,8 @@ object QueryPlanSerde extends Logging with CometExprShim { classOf[FilterExec] -> CometFilter, classOf[LocalLimitExec] -> CometLocalLimit, classOf[GlobalLimitExec] -> CometGlobalLimit, + classOf[HashAggregateExec] -> CometHashAggregate, + classOf[ObjectHashAggregateExec] -> CometObjectHashAggregate, classOf[HashJoin] -> CometHashJoin, classOf[SortMergeJoinExec] -> CometSortMergeJoin, classOf[ExpandExec] -> CometExpand, @@ -926,52 +928,47 @@ object QueryPlanSerde extends Logging with CometExprShim { * converted to a native operator. */ def operator2Proto(op: SparkPlan, childOp: Operator*): Option[Operator] = { - val conf = op.conf val builder = OperatorOuterClass.Operator.newBuilder().setPlanId(op.id) childOp.foreach(builder.addChildren) - op match { - - // Fully native scan for V1 - case scan: CometScanExec if scan.scanImpl == CometConf.SCAN_NATIVE_DATAFUSION => - CometNativeScan.convert(scan, builder, childOp: _*) - - case aggregate: BaseAggregateExec - if (aggregate.isInstanceOf[HashAggregateExec] || - aggregate.isInstanceOf[ObjectHashAggregateExec]) && - CometConf.COMET_EXEC_AGGREGATE_ENABLED.get(conf) => - CometAggregate.convert(aggregate, builder, childOp: _*) + def getOperatorSerde: Option[CometOperatorSerde[_]] = { + op match { + case scan: CometScanExec if scan.scanImpl == CometConf.SCAN_NATIVE_DATAFUSION => + Some(CometNativeScan) + case _ => + opSerdeMap.get(op.getClass) + } + } - case op => - opSerdeMap.get(op.getClass) match { - case Some(handler) => - handler.enabledConfig.foreach { enabledConfig => - if (!enabledConfig.get(op.conf)) { - withInfo( - op, - s"Native support for operator ${op.getClass.getSimpleName} is disabled. " + - s"Set ${enabledConfig.key}=true to enable it.") - if (isCometSink(op)) { - return cometSink(op, builder) - } else { - return None - } - } - } - handler.asInstanceOf[CometOperatorSerde[SparkPlan]].convert(op, builder, childOp: _*) - case _ => + getOperatorSerde match { + case Some(handler) => + handler.enabledConfig.foreach { enabledConfig => + if (!enabledConfig.get(op.conf)) { + withInfo( + op, + s"Native support for operator ${op.getClass.getSimpleName} is disabled. " + + s"Set ${enabledConfig.key}=true to enable it.") if (isCometSink(op)) { - cometSink(op, builder) + return cometSink(op, builder) } else { - // Emit warning if: - // 1. it is not Spark shuffle operator, which is handled separately - // 2. it is not a Comet operator - if (!op.nodeName.contains("Comet") && !op.isInstanceOf[ShuffleExchangeExec]) { - withInfo(op, s"unsupported Spark operator: ${op.nodeName}") - } - None + return None } + } + } + handler.asInstanceOf[CometOperatorSerde[SparkPlan]].convert(op, builder, childOp: _*) + case _ => + if (isCometSink(op)) { + cometSink(op, builder) + } else { + // Emit warning if: + // 1. it is not Spark shuffle operator, which is handled separately + // 2. it is not a Comet operator + if (!op.nodeName.contains("Comet") && !op.isInstanceOf[ShuffleExchangeExec]) { + withInfo(op, s"unsupported Spark operator: ${op.nodeName}") + } + None } + } } From 4cde2867ff7308d2cb439d9ae442fe7308d51eaa Mon Sep 17 00:00:00 2001 From: Andy Grove Date: Sat, 8 Nov 2025 07:54:26 -0700 Subject: [PATCH 09/44] refactor --- .../serde/CometAggregateExpressionSerde.scala | 67 +++++++++ .../comet/serde/CometExpressionSerde.scala | 66 +++++++++ .../comet/serde/CometOperatorSerde.scala | 57 ++++++++ .../comet/serde/CometScalarFunction.scala | 34 +++++ .../apache/comet/serde/QueryPlanSerde.scala | 134 +----------------- 5 files changed, 226 insertions(+), 132 deletions(-) create mode 100644 spark/src/main/scala/org/apache/comet/serde/CometAggregateExpressionSerde.scala create mode 100644 spark/src/main/scala/org/apache/comet/serde/CometExpressionSerde.scala create mode 100644 spark/src/main/scala/org/apache/comet/serde/CometOperatorSerde.scala create mode 100644 spark/src/main/scala/org/apache/comet/serde/CometScalarFunction.scala diff --git a/spark/src/main/scala/org/apache/comet/serde/CometAggregateExpressionSerde.scala b/spark/src/main/scala/org/apache/comet/serde/CometAggregateExpressionSerde.scala new file mode 100644 index 0000000000..c0c2b07284 --- /dev/null +++ b/spark/src/main/scala/org/apache/comet/serde/CometAggregateExpressionSerde.scala @@ -0,0 +1,67 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.comet.serde + +import org.apache.spark.sql.catalyst.expressions.Attribute +import org.apache.spark.sql.catalyst.expressions.aggregate.{AggregateExpression, AggregateFunction} +import org.apache.spark.sql.internal.SQLConf + +/** + * Trait for providing serialization logic for aggregate expressions. + */ +trait CometAggregateExpressionSerde[T <: AggregateFunction] { + + /** + * Get a short name for the expression that can be used as part of a config key related to the + * expression, such as enabling or disabling that expression. + * + * @param expr + * The Spark expression. + * @return + * Short name for the expression, defaulting to the Spark class name + */ + def getExprConfigName(expr: T): String = expr.getClass.getSimpleName + + /** + * Convert a Spark expression into a protocol buffer representation that can be passed into + * native code. + * + * @param aggExpr + * The aggregate expression. + * @param expr + * The aggregate function. + * @param inputs + * The input attributes. + * @param binding + * Whether the attributes are bound (this is only relevant in aggregate expressions). + * @param conf + * SQLConf + * @return + * Protocol buffer representation, or None if the expression could not be converted. In this + * case it is expected that the input expression will have been tagged with reasons why it + * could not be converted. + */ + def convert( + aggExpr: AggregateExpression, + expr: T, + inputs: Seq[Attribute], + binding: Boolean, + conf: SQLConf): Option[ExprOuterClass.AggExpr] +} diff --git a/spark/src/main/scala/org/apache/comet/serde/CometExpressionSerde.scala b/spark/src/main/scala/org/apache/comet/serde/CometExpressionSerde.scala new file mode 100644 index 0000000000..20c0343037 --- /dev/null +++ b/spark/src/main/scala/org/apache/comet/serde/CometExpressionSerde.scala @@ -0,0 +1,66 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.comet.serde + +import org.apache.spark.sql.catalyst.expressions.{Attribute, Expression} + +/** + * Trait for providing serialization logic for expressions. + */ +trait CometExpressionSerde[T <: Expression] { + + /** + * Get a short name for the expression that can be used as part of a config key related to the + * expression, such as enabling or disabling that expression. + * + * @param expr + * The Spark expression. + * @return + * Short name for the expression, defaulting to the Spark class name + */ + def getExprConfigName(expr: T): String = expr.getClass.getSimpleName + + /** + * Determine the support level of the expression based on its attributes. + * + * @param expr + * The Spark expression. + * @return + * Support level (Compatible, Incompatible, or Unsupported). + */ + def getSupportLevel(expr: T): SupportLevel = Compatible(None) + + /** + * Convert a Spark expression into a protocol buffer representation that can be passed into + * native code. + * + * @param expr + * The Spark expression. + * @param inputs + * The input attributes. + * @param binding + * Whether the attributes are bound (this is only relevant in aggregate expressions). + * @return + * Protocol buffer representation, or None if the expression could not be converted. In this + * case it is expected that the input expression will have been tagged with reasons why it + * could not be converted. + */ + def convert(expr: T, inputs: Seq[Attribute], binding: Boolean): Option[ExprOuterClass.Expr] +} diff --git a/spark/src/main/scala/org/apache/comet/serde/CometOperatorSerde.scala b/spark/src/main/scala/org/apache/comet/serde/CometOperatorSerde.scala new file mode 100644 index 0000000000..c6a95ec88a --- /dev/null +++ b/spark/src/main/scala/org/apache/comet/serde/CometOperatorSerde.scala @@ -0,0 +1,57 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.comet.serde + +import org.apache.spark.sql.execution.SparkPlan + +import org.apache.comet.ConfigEntry +import org.apache.comet.serde.OperatorOuterClass.Operator + +/** + * Trait for providing serialization logic for operators. + */ +trait CometOperatorSerde[T <: SparkPlan] { + + /** + * Convert a Spark operator into a protocol buffer representation that can be passed into native + * code. + * + * @param op + * The Spark operator. + * @param builder + * The protobuf builder for the operator. + * @param childOp + * Child operators that have already been converted to Comet. + * @return + * Protocol buffer representation, or None if the operator could not be converted. In this + * case it is expected that the input operator will have been tagged with reasons why it could + * not be converted. + */ + def convert( + op: T, + builder: Operator.Builder, + childOp: Operator*): Option[OperatorOuterClass.Operator] + + /** + * Get the optional Comet configuration entry that is used to enable or disable native support + * for this operator. + */ + def enabledConfig: Option[ConfigEntry[Boolean]] +} diff --git a/spark/src/main/scala/org/apache/comet/serde/CometScalarFunction.scala b/spark/src/main/scala/org/apache/comet/serde/CometScalarFunction.scala new file mode 100644 index 0000000000..aa3bf775fb --- /dev/null +++ b/spark/src/main/scala/org/apache/comet/serde/CometScalarFunction.scala @@ -0,0 +1,34 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.comet.serde + +import org.apache.spark.sql.catalyst.expressions.{Attribute, Expression} + +import org.apache.comet.serde.ExprOuterClass.Expr +import org.apache.comet.serde.QueryPlanSerde.{exprToProtoInternal, optExprWithInfo, scalarFunctionExprToProto} + +/** Serde for scalar function. */ +case class CometScalarFunction[T <: Expression](name: String) extends CometExpressionSerde[T] { + override def convert(expr: T, inputs: Seq[Attribute], binding: Boolean): Option[Expr] = { + val childExpr = expr.children.map(exprToProtoInternal(_, inputs, binding)) + val optExpr = scalarFunctionExprToProto(name, childExpr: _*) + optExprWithInfo(optExpr, expr, expr.children: _*) + } +} diff --git a/spark/src/main/scala/org/apache/comet/serde/QueryPlanSerde.scala b/spark/src/main/scala/org/apache/comet/serde/QueryPlanSerde.scala index a5f3805322..d11ba363aa 100644 --- a/spark/src/main/scala/org/apache/comet/serde/QueryPlanSerde.scala +++ b/spark/src/main/scala/org/apache/comet/serde/QueryPlanSerde.scala @@ -39,12 +39,11 @@ import org.apache.spark.sql.execution.window.WindowExec import org.apache.spark.sql.internal.SQLConf import org.apache.spark.sql.types._ -import org.apache.comet.{CometConf, ConfigEntry} +import org.apache.comet.CometConf import org.apache.comet.CometSparkSessionExtensions.{isCometScan, withInfo} import org.apache.comet.expressions._ import org.apache.comet.serde.ExprOuterClass.{AggExpr, Expr, ScalarFunc} import org.apache.comet.serde.OperatorOuterClass.Operator -import org.apache.comet.serde.QueryPlanSerde.{exprToProtoInternal, optExprWithInfo, scalarFunctionExprToProto} import org.apache.comet.serde.Types.{DataType => ProtoDataType} import org.apache.comet.serde.Types.DataType._ import org.apache.comet.serde.literals.CometLiteral @@ -1079,7 +1078,6 @@ object QueryPlanSerde extends Logging with CometExprShim { } // scalastyle:off - /** * Align w/ Arrow's * [[https://github.com/apache/arrow-rs/blob/55.2.0/arrow-ord/src/rank.rs#L30-L40 can_rank]] and @@ -1087,7 +1085,7 @@ object QueryPlanSerde extends Logging with CometExprShim { * * TODO: Include SparkSQL's [[YearMonthIntervalType]] and [[DayTimeIntervalType]] */ - // scalastyle:off + // scalastyle:on def supportedSortType(op: SparkPlan, sortOrder: Seq[SortOrder]): Boolean = { def canRank(dt: DataType): Boolean = { dt match { @@ -1147,131 +1145,3 @@ case class Incompatible(notes: Option[String] = None) extends SupportLevel /** Comet does not support this feature */ case class Unsupported(notes: Option[String] = None) extends SupportLevel - -/** - * Trait for providing serialization logic for operators. - */ -trait CometOperatorSerde[T <: SparkPlan] { - - /** - * Convert a Spark operator into a protocol buffer representation that can be passed into native - * code. - * - * @param op - * The Spark operator. - * @param builder - * The protobuf builder for the operator. - * @param childOp - * Child operators that have already been converted to Comet. - * @return - * Protocol buffer representation, or None if the operator could not be converted. In this - * case it is expected that the input operator will have been tagged with reasons why it could - * not be converted. - */ - def convert( - op: T, - builder: Operator.Builder, - childOp: Operator*): Option[OperatorOuterClass.Operator] - - /** - * Get the optional Comet configuration entry that is used to enable or disable native support - * for this operator. - */ - def enabledConfig: Option[ConfigEntry[Boolean]] -} - -/** - * Trait for providing serialization logic for expressions. - */ -trait CometExpressionSerde[T <: Expression] { - - /** - * Get a short name for the expression that can be used as part of a config key related to the - * expression, such as enabling or disabling that expression. - * - * @param expr - * The Spark expression. - * @return - * Short name for the expression, defaulting to the Spark class name - */ - def getExprConfigName(expr: T): String = expr.getClass.getSimpleName - - /** - * Determine the support level of the expression based on its attributes. - * - * @param expr - * The Spark expression. - * @return - * Support level (Compatible, Incompatible, or Unsupported). - */ - def getSupportLevel(expr: T): SupportLevel = Compatible(None) - - /** - * Convert a Spark expression into a protocol buffer representation that can be passed into - * native code. - * - * @param expr - * The Spark expression. - * @param inputs - * The input attributes. - * @param binding - * Whether the attributes are bound (this is only relevant in aggregate expressions). - * @return - * Protocol buffer representation, or None if the expression could not be converted. In this - * case it is expected that the input expression will have been tagged with reasons why it - * could not be converted. - */ - def convert(expr: T, inputs: Seq[Attribute], binding: Boolean): Option[ExprOuterClass.Expr] -} - -/** - * Trait for providing serialization logic for aggregate expressions. - */ -trait CometAggregateExpressionSerde[T <: AggregateFunction] { - - /** - * Get a short name for the expression that can be used as part of a config key related to the - * expression, such as enabling or disabling that expression. - * - * @param expr - * The Spark expression. - * @return - * Short name for the expression, defaulting to the Spark class name - */ - def getExprConfigName(expr: T): String = expr.getClass.getSimpleName - - /** - * Convert a Spark expression into a protocol buffer representation that can be passed into - * native code. - * - * @param aggExpr - * The aggregate expression. - * @param expr - * The aggregate function. - * @param inputs - * The input attributes. - * @param binding - * Whether the attributes are bound (this is only relevant in aggregate expressions). - * @param conf - * SQLConf - * @return - * Protocol buffer representation, or None if the expression could not be converted. In this - * case it is expected that the input expression will have been tagged with reasons why it - * could not be converted. - */ - def convert( - aggExpr: AggregateExpression, - expr: T, - inputs: Seq[Attribute], - binding: Boolean, - conf: SQLConf): Option[ExprOuterClass.AggExpr] -} - -/** Serde for scalar function. */ -case class CometScalarFunction[T <: Expression](name: String) extends CometExpressionSerde[T] { - override def convert(expr: T, inputs: Seq[Attribute], binding: Boolean): Option[Expr] = { - val childExpr = expr.children.map(exprToProtoInternal(_, inputs, binding)) - val optExpr = scalarFunctionExprToProto(name, childExpr: _*) - optExprWithInfo(optExpr, expr, expr.children: _*) - } -} From 8afadf7add676e241ab2c3b6bc0fbd075d1ac79a Mon Sep 17 00:00:00 2001 From: Andy Grove Date: Sat, 8 Nov 2025 08:02:52 -0700 Subject: [PATCH 10/44] fix --- .../apache/comet/serde/CometHashJoin.scala | 33 ++++++++++++++----- .../apache/comet/serde/QueryPlanSerde.scala | 7 ++-- 2 files changed, 27 insertions(+), 13 deletions(-) diff --git a/spark/src/main/scala/org/apache/comet/serde/CometHashJoin.scala b/spark/src/main/scala/org/apache/comet/serde/CometHashJoin.scala index 94fd880f8f..937ea66571 100644 --- a/spark/src/main/scala/org/apache/comet/serde/CometHashJoin.scala +++ b/spark/src/main/scala/org/apache/comet/serde/CometHashJoin.scala @@ -31,19 +31,16 @@ import org.apache.comet.CometSparkSessionExtensions.withInfo import org.apache.comet.serde.OperatorOuterClass.{BuildSide, JoinType, Operator} import org.apache.comet.serde.QueryPlanSerde.exprToProto -object CometHashJoin extends CometOperatorSerde[HashJoin] { +trait CometHashJoin { - override def enabledConfig: Option[ConfigEntry[Boolean]] = - Some(CometConf.COMET_EXEC_HASH_JOIN_ENABLED) - - override def convert( - join: HashJoin, - builder: Operator.Builder, - childOp: OperatorOuterClass.Operator*): Option[OperatorOuterClass.Operator] = { + def doConvert( + join: HashJoin, + builder: Operator.Builder, + childOp: OperatorOuterClass.Operator*): Option[OperatorOuterClass.Operator] = { // `HashJoin` has only two implementations in Spark, but we check the type of the join to // make sure we are handling the correct join type. if (!(CometConf.COMET_EXEC_HASH_JOIN_ENABLED.get(join.conf) && - join.isInstanceOf[ShuffledHashJoinExec]) && + join.isInstanceOf[ShuffledHashJoinExec]) && !(CometConf.COMET_EXEC_BROADCAST_HASH_JOIN_ENABLED.get(join.conf) && join.isInstanceOf[BroadcastHashJoinExec])) { withInfo(join, s"Invalid hash join type ${join.nodeName}") @@ -100,3 +97,21 @@ object CometHashJoin extends CometOperatorSerde[HashJoin] { } } } + +object CometBroadcastHashJoin extends CometOperatorSerde[HashJoin] with CometHashJoin { + + override def enabledConfig: Option[ConfigEntry[Boolean]] = + Some(CometConf.COMET_EXEC_HASH_JOIN_ENABLED) + + override def convert(join: HashJoin, builder: Operator.Builder, childOp: Operator*): Option[Operator] = + doConvert(join, builder, childOp: _*) +} + +object CometShuffleHashJoin extends CometOperatorSerde[HashJoin] with CometHashJoin { + + override def enabledConfig: Option[ConfigEntry[Boolean]] = + Some(CometConf.COMET_EXEC_HASH_JOIN_ENABLED) + + override def convert(join: HashJoin, builder: Operator.Builder, childOp: Operator*): Option[Operator] = + doConvert(join, builder, childOp: _*) +} diff --git a/spark/src/main/scala/org/apache/comet/serde/QueryPlanSerde.scala b/spark/src/main/scala/org/apache/comet/serde/QueryPlanSerde.scala index d11ba363aa..e70a55e060 100644 --- a/spark/src/main/scala/org/apache/comet/serde/QueryPlanSerde.scala +++ b/spark/src/main/scala/org/apache/comet/serde/QueryPlanSerde.scala @@ -20,7 +20,6 @@ package org.apache.comet.serde import scala.jdk.CollectionConverters._ - import org.apache.spark.internal.Logging import org.apache.spark.sql.catalyst.expressions._ import org.apache.spark.sql.catalyst.expressions.aggregate._ @@ -34,11 +33,10 @@ import org.apache.spark.sql.execution._ import org.apache.spark.sql.execution.adaptive.{BroadcastQueryStageExec, ShuffleQueryStageExec} import org.apache.spark.sql.execution.aggregate.{HashAggregateExec, ObjectHashAggregateExec} import org.apache.spark.sql.execution.exchange.{BroadcastExchangeExec, ReusedExchangeExec, ShuffleExchangeExec} -import org.apache.spark.sql.execution.joins.{HashJoin, SortMergeJoinExec} +import org.apache.spark.sql.execution.joins.{BroadcastHashJoinExec, HashJoin, ShuffledHashJoinExec, SortMergeJoinExec} import org.apache.spark.sql.execution.window.WindowExec import org.apache.spark.sql.internal.SQLConf import org.apache.spark.sql.types._ - import org.apache.comet.CometConf import org.apache.comet.CometSparkSessionExtensions.{isCometScan, withInfo} import org.apache.comet.expressions._ @@ -65,7 +63,8 @@ object QueryPlanSerde extends Logging with CometExprShim { classOf[GlobalLimitExec] -> CometGlobalLimit, classOf[HashAggregateExec] -> CometHashAggregate, classOf[ObjectHashAggregateExec] -> CometObjectHashAggregate, - classOf[HashJoin] -> CometHashJoin, + classOf[BroadcastHashJoinExec] -> CometBroadcastHashJoin, + classOf[ShuffledHashJoinExec] -> CometShuffleHashJoin, classOf[SortMergeJoinExec] -> CometSortMergeJoin, classOf[ExpandExec] -> CometExpand, classOf[WindowExec] -> CometWindow, From ae2254ebca2efe0d83cbadba05f05ca7bc921483 Mon Sep 17 00:00:00 2001 From: Andy Grove Date: Sat, 8 Nov 2025 08:04:22 -0700 Subject: [PATCH 11/44] scalastyle --- .../org/apache/comet/serde/CometHashJoin.scala | 18 ++++++++++++------ .../apache/comet/serde/QueryPlanSerde.scala | 4 +++- 2 files changed, 15 insertions(+), 7 deletions(-) diff --git a/spark/src/main/scala/org/apache/comet/serde/CometHashJoin.scala b/spark/src/main/scala/org/apache/comet/serde/CometHashJoin.scala index 937ea66571..67fb67a2e7 100644 --- a/spark/src/main/scala/org/apache/comet/serde/CometHashJoin.scala +++ b/spark/src/main/scala/org/apache/comet/serde/CometHashJoin.scala @@ -34,13 +34,13 @@ import org.apache.comet.serde.QueryPlanSerde.exprToProto trait CometHashJoin { def doConvert( - join: HashJoin, - builder: Operator.Builder, - childOp: OperatorOuterClass.Operator*): Option[OperatorOuterClass.Operator] = { + join: HashJoin, + builder: Operator.Builder, + childOp: OperatorOuterClass.Operator*): Option[OperatorOuterClass.Operator] = { // `HashJoin` has only two implementations in Spark, but we check the type of the join to // make sure we are handling the correct join type. if (!(CometConf.COMET_EXEC_HASH_JOIN_ENABLED.get(join.conf) && - join.isInstanceOf[ShuffledHashJoinExec]) && + join.isInstanceOf[ShuffledHashJoinExec]) && !(CometConf.COMET_EXEC_BROADCAST_HASH_JOIN_ENABLED.get(join.conf) && join.isInstanceOf[BroadcastHashJoinExec])) { withInfo(join, s"Invalid hash join type ${join.nodeName}") @@ -103,7 +103,10 @@ object CometBroadcastHashJoin extends CometOperatorSerde[HashJoin] with CometHas override def enabledConfig: Option[ConfigEntry[Boolean]] = Some(CometConf.COMET_EXEC_HASH_JOIN_ENABLED) - override def convert(join: HashJoin, builder: Operator.Builder, childOp: Operator*): Option[Operator] = + override def convert( + join: HashJoin, + builder: Operator.Builder, + childOp: Operator*): Option[Operator] = doConvert(join, builder, childOp: _*) } @@ -112,6 +115,9 @@ object CometShuffleHashJoin extends CometOperatorSerde[HashJoin] with CometHashJ override def enabledConfig: Option[ConfigEntry[Boolean]] = Some(CometConf.COMET_EXEC_HASH_JOIN_ENABLED) - override def convert(join: HashJoin, builder: Operator.Builder, childOp: Operator*): Option[Operator] = + override def convert( + join: HashJoin, + builder: Operator.Builder, + childOp: Operator*): Option[Operator] = doConvert(join, builder, childOp: _*) } diff --git a/spark/src/main/scala/org/apache/comet/serde/QueryPlanSerde.scala b/spark/src/main/scala/org/apache/comet/serde/QueryPlanSerde.scala index e70a55e060..1e63b70b81 100644 --- a/spark/src/main/scala/org/apache/comet/serde/QueryPlanSerde.scala +++ b/spark/src/main/scala/org/apache/comet/serde/QueryPlanSerde.scala @@ -20,6 +20,7 @@ package org.apache.comet.serde import scala.jdk.CollectionConverters._ + import org.apache.spark.internal.Logging import org.apache.spark.sql.catalyst.expressions._ import org.apache.spark.sql.catalyst.expressions.aggregate._ @@ -33,10 +34,11 @@ import org.apache.spark.sql.execution._ import org.apache.spark.sql.execution.adaptive.{BroadcastQueryStageExec, ShuffleQueryStageExec} import org.apache.spark.sql.execution.aggregate.{HashAggregateExec, ObjectHashAggregateExec} import org.apache.spark.sql.execution.exchange.{BroadcastExchangeExec, ReusedExchangeExec, ShuffleExchangeExec} -import org.apache.spark.sql.execution.joins.{BroadcastHashJoinExec, HashJoin, ShuffledHashJoinExec, SortMergeJoinExec} +import org.apache.spark.sql.execution.joins.{BroadcastHashJoinExec, ShuffledHashJoinExec, SortMergeJoinExec} import org.apache.spark.sql.execution.window.WindowExec import org.apache.spark.sql.internal.SQLConf import org.apache.spark.sql.types._ + import org.apache.comet.CometConf import org.apache.comet.CometSparkSessionExtensions.{isCometScan, withInfo} import org.apache.comet.expressions._ From 3dbb10dbd0eed1d42c23d53e5a9286eb70bf1fb2 Mon Sep 17 00:00:00 2001 From: Andy Grove Date: Sat, 8 Nov 2025 08:49:46 -0700 Subject: [PATCH 12/44] remove map approach --- .../apache/comet/serde/QueryPlanSerde.scala | 209 ++++++++++-------- 1 file changed, 112 insertions(+), 97 deletions(-) diff --git a/spark/src/main/scala/org/apache/comet/serde/QueryPlanSerde.scala b/spark/src/main/scala/org/apache/comet/serde/QueryPlanSerde.scala index 1e63b70b81..f512aec0f9 100644 --- a/spark/src/main/scala/org/apache/comet/serde/QueryPlanSerde.scala +++ b/spark/src/main/scala/org/apache/comet/serde/QueryPlanSerde.scala @@ -58,19 +58,7 @@ object QueryPlanSerde extends Logging with CometExprShim { * Mapping of Spark operator class to Comet operator handler. */ private val opSerdeMap: Map[Class[_ <: SparkPlan], CometOperatorSerde[_]] = - Map( - classOf[ProjectExec] -> CometProject, - classOf[FilterExec] -> CometFilter, - classOf[LocalLimitExec] -> CometLocalLimit, - classOf[GlobalLimitExec] -> CometGlobalLimit, - classOf[HashAggregateExec] -> CometHashAggregate, - classOf[ObjectHashAggregateExec] -> CometObjectHashAggregate, - classOf[BroadcastHashJoinExec] -> CometBroadcastHashJoin, - classOf[ShuffledHashJoinExec] -> CometShuffleHashJoin, - classOf[SortMergeJoinExec] -> CometSortMergeJoin, - classOf[ExpandExec] -> CometExpand, - classOf[WindowExec] -> CometWindow, - classOf[SortExec] -> CometSort) + Map(classOf[ProjectExec] -> CometProject, classOf[SortExec] -> CometSort) private val arrayExpressions: Map[Class[_ <: Expression], CometExpressionSerde[_]] = Map( classOf[ArrayAppend] -> CometArrayAppend, @@ -928,104 +916,131 @@ object QueryPlanSerde extends Logging with CometExprShim { * converted to a native operator. */ def operator2Proto(op: SparkPlan, childOp: Operator*): Option[Operator] = { + val conf = op.conf val builder = OperatorOuterClass.Operator.newBuilder().setPlanId(op.id) childOp.foreach(builder.addChildren) - def getOperatorSerde: Option[CometOperatorSerde[_]] = { - op match { - case scan: CometScanExec if scan.scanImpl == CometConf.SCAN_NATIVE_DATAFUSION => - Some(CometNativeScan) - case _ => - opSerdeMap.get(op.getClass) - } - } + op match { - getOperatorSerde match { - case Some(handler) => - handler.enabledConfig.foreach { enabledConfig => - if (!enabledConfig.get(op.conf)) { - withInfo( - op, - s"Native support for operator ${op.getClass.getSimpleName} is disabled. " + - s"Set ${enabledConfig.key}=true to enable it.") - if (isCometSink(op)) { - return cometSink(op, builder) - } else { - return None - } - } - } - handler.asInstanceOf[CometOperatorSerde[SparkPlan]].convert(op, builder, childOp: _*) - case _ => - if (isCometSink(op)) { - cometSink(op, builder) - } else { - // Emit warning if: - // 1. it is not Spark shuffle operator, which is handled separately - // 2. it is not a Comet operator - if (!op.nodeName.contains("Comet") && !op.isInstanceOf[ShuffleExchangeExec]) { - withInfo(op, s"unsupported Spark operator: ${op.nodeName}") - } - None - } + // Fully native scan for V1 + case scan: CometScanExec if scan.scanImpl == CometConf.SCAN_NATIVE_DATAFUSION => + CometNativeScan.convert(scan, builder, childOp: _*) - } - } + case filter: FilterExec if CometConf.COMET_EXEC_FILTER_ENABLED.get(conf) => + CometFilter.convert(filter, builder, childOp: _*) - def cometSink(op: SparkPlan, builder: Operator.Builder): Option[Operator] = { - val supportedTypes = - op.output.forall(a => supportedDataType(a.dataType, allowComplex = true)) + case limit: LocalLimitExec if CometConf.COMET_EXEC_LOCAL_LIMIT_ENABLED.get(conf) => + CometLocalLimit.convert(limit, builder, childOp: _*) - if (!supportedTypes) { - withInfo(op, "Unsupported data type") - return None - } + case globalLimitExec: GlobalLimitExec + if CometConf.COMET_EXEC_GLOBAL_LIMIT_ENABLED.get(conf) => + CometGlobalLimit.convert(globalLimitExec, builder, childOp: _*) - // These operators are source of Comet native execution chain - val scanBuilder = OperatorOuterClass.Scan.newBuilder() - val source = op.simpleStringWithNodeId() - if (source.isEmpty) { - scanBuilder.setSource(op.getClass.getSimpleName) - } else { - scanBuilder.setSource(source) - } + case expand: ExpandExec if CometConf.COMET_EXEC_EXPAND_ENABLED.get(conf) => + CometExpand.convert(expand, builder, childOp: _*) - val ffiSafe = op match { - case _ if isExchangeSink(op) => - // Source of broadcast exchange batches is ArrowStreamReader - // Source of shuffle exchange batches is NativeBatchDecoderIterator - true - case scan: CometScanExec if scan.scanImpl == CometConf.SCAN_NATIVE_COMET => - // native_comet scan reuses mutable buffers - false - case scan: CometScanExec if scan.scanImpl == CometConf.SCAN_NATIVE_ICEBERG_COMPAT => - // native_iceberg_compat scan reuses mutable buffers for constant columns - // https://github.com/apache/datafusion-comet/issues/2152 - false - case _ => - false - } - scanBuilder.setArrowFfiSafe(ffiSafe) + case _: WindowExec + if CometConf.COMET_EXEC_WINDOW_ENABLED.get(conf) => + withInfo(op, "Window expressions are not supported") + None - val scanTypes = op.output.flatten { attr => - serializeDataType(attr.dataType) - } + case aggregate: HashAggregateExec if CometConf.COMET_EXEC_AGGREGATE_ENABLED.get(conf) => + CometHashAggregate.convert(aggregate, builder, childOp: _*) - if (scanTypes.length == op.output.length) { - scanBuilder.addAllFields(scanTypes.asJava) + case aggregate: ObjectHashAggregateExec + if CometConf.COMET_EXEC_AGGREGATE_ENABLED.get(conf) => + CometObjectHashAggregate.convert(aggregate, builder, childOp: _*) - // Sink operators don't have children - builder.clearChildren() + case join: BroadcastHashJoinExec => + CometBroadcastHashJoin.convert(join, builder, childOp: _*) - Some(builder.setScan(scanBuilder).build()) - } else { - // There are unsupported scan type - withInfo( - op, - s"unsupported Comet operator: ${op.nodeName}, due to unsupported data types above") - None - } + case join: ShuffledHashJoinExec => + CometShuffleHashJoin.convert(join, builder, childOp: _*) + case join: SortMergeJoinExec if CometConf.COMET_EXEC_SORT_MERGE_JOIN_ENABLED.get(conf) => + CometSortMergeJoin.convert(join, builder, childOp: _*) + + case join: SortMergeJoinExec if !CometConf.COMET_EXEC_SORT_MERGE_JOIN_ENABLED.get(conf) => + withInfo(join, "SortMergeJoin is not enabled") + None + + case op if isCometSink(op) => + val supportedTypes = + op.output.forall(a => supportedDataType(a.dataType, allowComplex = true)) + + if (!supportedTypes) { + withInfo(op, "Unsupported data type") + return None + } + + // These operators are source of Comet native execution chain + val scanBuilder = OperatorOuterClass.Scan.newBuilder() + val source = op.simpleStringWithNodeId() + if (source.isEmpty) { + scanBuilder.setSource(op.getClass.getSimpleName) + } else { + scanBuilder.setSource(source) + } + + val ffiSafe = op match { + case _ if isExchangeSink(op) => + // Source of broadcast exchange batches is ArrowStreamReader + // Source of shuffle exchange batches is NativeBatchDecoderIterator + true + case scan: CometScanExec if scan.scanImpl == CometConf.SCAN_NATIVE_COMET => + // native_comet scan reuses mutable buffers + false + case scan: CometScanExec if scan.scanImpl == CometConf.SCAN_NATIVE_ICEBERG_COMPAT => + // native_iceberg_compat scan reuses mutable buffers for constant columns + // https://github.com/apache/datafusion-comet/issues/2152 + false + case _ => + false + } + scanBuilder.setArrowFfiSafe(ffiSafe) + + val scanTypes = op.output.flatten { attr => + serializeDataType(attr.dataType) + } + + if (scanTypes.length == op.output.length) { + scanBuilder.addAllFields(scanTypes.asJava) + + // Sink operators don't have children + builder.clearChildren() + + Some(builder.setScan(scanBuilder).build()) + } else { + // There are unsupported scan type + withInfo( + op, + s"unsupported Comet operator: ${op.nodeName}, due to unsupported data types above") + None + } + + case op => + opSerdeMap.get(op.getClass) match { + case Some(handler) => + handler.enabledConfig.foreach { enabledConfig => + if (!enabledConfig.get(op.conf)) { + withInfo( + op, + s"Native support for operator ${op.getClass.getSimpleName} is disabled. " + + s"Set ${enabledConfig.key}=true to enable it.") + return None + } + } + handler.asInstanceOf[CometOperatorSerde[SparkPlan]].convert(op, builder, childOp: _*) + case _ => + // Emit warning if: + // 1. it is not Spark shuffle operator, which is handled separately + // 2. it is not a Comet operator + if (!op.nodeName.contains("Comet") && !op.isInstanceOf[ShuffleExchangeExec]) { + withInfo(op, s"unsupported Spark operator: ${op.nodeName}") + } + None + } + } } /** From 30360ab864c41e83a7dcebfe0a2438c0cf1f7867 Mon Sep 17 00:00:00 2001 From: Andy Grove Date: Sat, 8 Nov 2025 09:12:51 -0700 Subject: [PATCH 13/44] scalastyle --- common/src/main/scala/org/apache/comet/CometConf.scala | 2 +- .../src/main/scala/org/apache/comet/serde/QueryPlanSerde.scala | 3 +-- 2 files changed, 2 insertions(+), 3 deletions(-) diff --git a/common/src/main/scala/org/apache/comet/CometConf.scala b/common/src/main/scala/org/apache/comet/CometConf.scala index 7bd075eb50..60fd1940bc 100644 --- a/common/src/main/scala/org/apache/comet/CometConf.scala +++ b/common/src/main/scala/org/apache/comet/CometConf.scala @@ -248,7 +248,7 @@ object CometConf extends ShimCometConf { val COMET_EXEC_EXPAND_ENABLED: ConfigEntry[Boolean] = createExecEnabledConfig("expand", defaultValue = true) val COMET_EXEC_WINDOW_ENABLED: ConfigEntry[Boolean] = - createExecEnabledConfig("window", defaultValue = false) + createExecEnabledConfig("window", defaultValue = true) val COMET_EXEC_TAKE_ORDERED_AND_PROJECT_ENABLED: ConfigEntry[Boolean] = createExecEnabledConfig("takeOrderedAndProject", defaultValue = true) diff --git a/spark/src/main/scala/org/apache/comet/serde/QueryPlanSerde.scala b/spark/src/main/scala/org/apache/comet/serde/QueryPlanSerde.scala index f512aec0f9..e75038a143 100644 --- a/spark/src/main/scala/org/apache/comet/serde/QueryPlanSerde.scala +++ b/spark/src/main/scala/org/apache/comet/serde/QueryPlanSerde.scala @@ -939,8 +939,7 @@ object QueryPlanSerde extends Logging with CometExprShim { case expand: ExpandExec if CometConf.COMET_EXEC_EXPAND_ENABLED.get(conf) => CometExpand.convert(expand, builder, childOp: _*) - case _: WindowExec - if CometConf.COMET_EXEC_WINDOW_ENABLED.get(conf) => + case _: WindowExec if CometConf.COMET_EXEC_WINDOW_ENABLED.get(conf) => withInfo(op, "Window expressions are not supported") None From a6e9023af9cf524ca8e422c9f6dabb3b1b0559ae Mon Sep 17 00:00:00 2001 From: Andy Grove Date: Sat, 8 Nov 2025 09:13:01 -0700 Subject: [PATCH 14/44] revert config --- docs/source/user-guide/latest/configs.md | 34 ++++++++++++------------ 1 file changed, 17 insertions(+), 17 deletions(-) diff --git a/docs/source/user-guide/latest/configs.md b/docs/source/user-guide/latest/configs.md index bf6388549c..9490079480 100644 --- a/docs/source/user-guide/latest/configs.md +++ b/docs/source/user-guide/latest/configs.md @@ -139,23 +139,23 @@ These settings can be used to determine which parts of the plan are accelerated | Config | Description | Default Value | |--------|-------------|---------------| -| `spark.comet.exec.aggregate.enabled` | Whether to enable aggregate by default. | true | -| `spark.comet.exec.broadcastExchange.enabled` | Whether to enable broadcastExchange by default. | true | -| `spark.comet.exec.broadcastHashJoin.enabled` | Whether to enable broadcastHashJoin by default. | true | -| `spark.comet.exec.coalesce.enabled` | Whether to enable coalesce by default. | true | -| `spark.comet.exec.collectLimit.enabled` | Whether to enable collectLimit by default. | true | -| `spark.comet.exec.expand.enabled` | Whether to enable expand by default. | true | -| `spark.comet.exec.filter.enabled` | Whether to enable filter by default. | true | -| `spark.comet.exec.globalLimit.enabled` | Whether to enable globalLimit by default. | true | -| `spark.comet.exec.hashJoin.enabled` | Whether to enable hashJoin by default. | true | -| `spark.comet.exec.localLimit.enabled` | Whether to enable localLimit by default. | true | -| `spark.comet.exec.project.enabled` | Whether to enable project by default. | true | -| `spark.comet.exec.sort.enabled` | Whether to enable sort by default. | true | -| `spark.comet.exec.sortMergeJoin.enabled` | Whether to enable sortMergeJoin by default. | true | -| `spark.comet.exec.sortMergeJoinWithJoinFilter.enabled` | Experimental support for Sort Merge Join with filter | false | -| `spark.comet.exec.takeOrderedAndProject.enabled` | Whether to enable takeOrderedAndProject by default. | true | -| `spark.comet.exec.union.enabled` | Whether to enable union by default. | true | -| `spark.comet.exec.window.enabled` | Whether to enable window by default. | false | +| `spark.comet.exec.aggregate.enabled` | Whether to enable aggregate by default. | true | +| `spark.comet.exec.broadcastExchange.enabled` | Whether to enable broadcastExchange by default. | true | +| `spark.comet.exec.broadcastHashJoin.enabled` | Whether to enable broadcastHashJoin by default. | true | +| `spark.comet.exec.coalesce.enabled` | Whether to enable coalesce by default. | true | +| `spark.comet.exec.collectLimit.enabled` | Whether to enable collectLimit by default. | true | +| `spark.comet.exec.expand.enabled` | Whether to enable expand by default. | true | +| `spark.comet.exec.filter.enabled` | Whether to enable filter by default. | true | +| `spark.comet.exec.globalLimit.enabled` | Whether to enable globalLimit by default. | true | +| `spark.comet.exec.hashJoin.enabled` | Whether to enable hashJoin by default. | true | +| `spark.comet.exec.localLimit.enabled` | Whether to enable localLimit by default. | true | +| `spark.comet.exec.project.enabled` | Whether to enable project by default. | true | +| `spark.comet.exec.sort.enabled` | Whether to enable sort by default. | true | +| `spark.comet.exec.sortMergeJoin.enabled` | Whether to enable sortMergeJoin by default. | true | +| `spark.comet.exec.sortMergeJoinWithJoinFilter.enabled` | Experimental support for Sort Merge Join with filter | false | +| `spark.comet.exec.takeOrderedAndProject.enabled` | Whether to enable takeOrderedAndProject by default. | true | +| `spark.comet.exec.union.enabled` | Whether to enable union by default. | true | +| `spark.comet.exec.window.enabled` | Whether to enable window by default. | true | ## Enabling or Disabling Individual Scalar Expressions From c719eeba3bfb5c9e07b3d3dcd9e838d928e68894 Mon Sep 17 00:00:00 2001 From: Andy Grove Date: Sat, 8 Nov 2025 09:16:04 -0700 Subject: [PATCH 15/44] revert docs --- docs/source/user-guide/latest/configs.md | 34 ++++++++++++------------ 1 file changed, 17 insertions(+), 17 deletions(-) diff --git a/docs/source/user-guide/latest/configs.md b/docs/source/user-guide/latest/configs.md index 9490079480..1cc05dfc78 100644 --- a/docs/source/user-guide/latest/configs.md +++ b/docs/source/user-guide/latest/configs.md @@ -139,23 +139,23 @@ These settings can be used to determine which parts of the plan are accelerated | Config | Description | Default Value | |--------|-------------|---------------| -| `spark.comet.exec.aggregate.enabled` | Whether to enable aggregate by default. | true | -| `spark.comet.exec.broadcastExchange.enabled` | Whether to enable broadcastExchange by default. | true | -| `spark.comet.exec.broadcastHashJoin.enabled` | Whether to enable broadcastHashJoin by default. | true | -| `spark.comet.exec.coalesce.enabled` | Whether to enable coalesce by default. | true | -| `spark.comet.exec.collectLimit.enabled` | Whether to enable collectLimit by default. | true | -| `spark.comet.exec.expand.enabled` | Whether to enable expand by default. | true | -| `spark.comet.exec.filter.enabled` | Whether to enable filter by default. | true | -| `spark.comet.exec.globalLimit.enabled` | Whether to enable globalLimit by default. | true | -| `spark.comet.exec.hashJoin.enabled` | Whether to enable hashJoin by default. | true | -| `spark.comet.exec.localLimit.enabled` | Whether to enable localLimit by default. | true | -| `spark.comet.exec.project.enabled` | Whether to enable project by default. | true | -| `spark.comet.exec.sort.enabled` | Whether to enable sort by default. | true | -| `spark.comet.exec.sortMergeJoin.enabled` | Whether to enable sortMergeJoin by default. | true | -| `spark.comet.exec.sortMergeJoinWithJoinFilter.enabled` | Experimental support for Sort Merge Join with filter | false | -| `spark.comet.exec.takeOrderedAndProject.enabled` | Whether to enable takeOrderedAndProject by default. | true | -| `spark.comet.exec.union.enabled` | Whether to enable union by default. | true | -| `spark.comet.exec.window.enabled` | Whether to enable window by default. | true | +| `spark.comet.exec.aggregate.enabled` | Whether to enable aggregate by default. | true | +| `spark.comet.exec.broadcastExchange.enabled` | Whether to enable broadcastExchange by default. | true | +| `spark.comet.exec.broadcastHashJoin.enabled` | Whether to enable broadcastHashJoin by default. | true | +| `spark.comet.exec.coalesce.enabled` | Whether to enable coalesce by default. | true | +| `spark.comet.exec.collectLimit.enabled` | Whether to enable collectLimit by default. | true | +| `spark.comet.exec.expand.enabled` | Whether to enable expand by default. | true | +| `spark.comet.exec.filter.enabled` | Whether to enable filter by default. | true | +| `spark.comet.exec.globalLimit.enabled` | Whether to enable globalLimit by default. | true | +| `spark.comet.exec.hashJoin.enabled` | Whether to enable hashJoin by default. | true | +| `spark.comet.exec.localLimit.enabled` | Whether to enable localLimit by default. | true | +| `spark.comet.exec.project.enabled` | Whether to enable project by default. | true | +| `spark.comet.exec.sort.enabled` | Whether to enable sort by default. | true | +| `spark.comet.exec.sortMergeJoin.enabled` | Whether to enable sortMergeJoin by default. | true | +| `spark.comet.exec.sortMergeJoinWithJoinFilter.enabled` | Experimental support for Sort Merge Join with filter | false | +| `spark.comet.exec.takeOrderedAndProject.enabled` | Whether to enable takeOrderedAndProject by default. | true | +| `spark.comet.exec.union.enabled` | Whether to enable union by default. | true | +| `spark.comet.exec.window.enabled` | Whether to enable window by default. | true | ## Enabling or Disabling Individual Scalar Expressions From 812dd7fc1cf8efab2c5a347687895aaa8ea7588f Mon Sep 17 00:00:00 2001 From: Andy Grove Date: Sat, 8 Nov 2025 09:21:38 -0700 Subject: [PATCH 16/44] revert a change --- .../apache/comet/serde/QueryPlanSerde.scala | 18 ++++++++++-------- 1 file changed, 10 insertions(+), 8 deletions(-) diff --git a/spark/src/main/scala/org/apache/comet/serde/QueryPlanSerde.scala b/spark/src/main/scala/org/apache/comet/serde/QueryPlanSerde.scala index e75038a143..3e0e837c9c 100644 --- a/spark/src/main/scala/org/apache/comet/serde/QueryPlanSerde.scala +++ b/spark/src/main/scala/org/apache/comet/serde/QueryPlanSerde.scala @@ -950,18 +950,20 @@ object QueryPlanSerde extends Logging with CometExprShim { if CometConf.COMET_EXEC_AGGREGATE_ENABLED.get(conf) => CometObjectHashAggregate.convert(aggregate, builder, childOp: _*) - case join: BroadcastHashJoinExec => + case join: BroadcastHashJoinExec + if CometConf.COMET_EXEC_BROADCAST_HASH_JOIN_ENABLED.get(conf) => CometBroadcastHashJoin.convert(join, builder, childOp: _*) - case join: ShuffledHashJoinExec => + case join: ShuffledHashJoinExec if CometConf.COMET_EXEC_HASH_JOIN_ENABLED.get(conf) => CometShuffleHashJoin.convert(join, builder, childOp: _*) - case join: SortMergeJoinExec if CometConf.COMET_EXEC_SORT_MERGE_JOIN_ENABLED.get(conf) => - CometSortMergeJoin.convert(join, builder, childOp: _*) - - case join: SortMergeJoinExec if !CometConf.COMET_EXEC_SORT_MERGE_JOIN_ENABLED.get(conf) => - withInfo(join, "SortMergeJoin is not enabled") - None + case join: SortMergeJoinExec => + if (CometConf.COMET_EXEC_SORT_MERGE_JOIN_ENABLED.get(conf)) { + CometSortMergeJoin.convert(join, builder, childOp: _*) + } else { + withInfo(join, "SortMergeJoin is not enabled") + None + } case op if isCometSink(op) => val supportedTypes = From 0343323567dbb49d2a5bff3cec7a5722e4b8f68d Mon Sep 17 00:00:00 2001 From: Andy Grove Date: Sat, 8 Nov 2025 09:38:05 -0700 Subject: [PATCH 17/44] refactor sink serde --- .../org/apache/comet/serde/CometSink.scala | 134 ++++++++++++++++++ .../apache/comet/serde/QueryPlanSerde.scala | 97 +------------ 2 files changed, 138 insertions(+), 93 deletions(-) create mode 100644 spark/src/main/scala/org/apache/comet/serde/CometSink.scala diff --git a/spark/src/main/scala/org/apache/comet/serde/CometSink.scala b/spark/src/main/scala/org/apache/comet/serde/CometSink.scala new file mode 100644 index 0000000000..ec7aac8ec2 --- /dev/null +++ b/spark/src/main/scala/org/apache/comet/serde/CometSink.scala @@ -0,0 +1,134 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.comet.serde + +import scala.jdk.CollectionConverters._ + +import org.apache.spark.sql.comet.{CometBroadcastExchangeExec, CometScanExec, CometSinkPlaceHolder, CometSparkToColumnarExec} +import org.apache.spark.sql.comet.execution.shuffle.CometShuffleExchangeExec +import org.apache.spark.sql.execution.{CoalesceExec, CollectLimitExec, SparkPlan, TakeOrderedAndProjectExec, UnionExec} +import org.apache.spark.sql.execution.adaptive.{BroadcastQueryStageExec, ShuffleQueryStageExec} +import org.apache.spark.sql.execution.exchange.{BroadcastExchangeExec, ReusedExchangeExec, ShuffleExchangeExec} +import org.apache.spark.sql.execution.window.WindowExec + +import org.apache.comet.{CometConf, ConfigEntry} +import org.apache.comet.CometSparkSessionExtensions.{isCometScan, withInfo} +import org.apache.comet.serde.OperatorOuterClass.Operator +import org.apache.comet.serde.QueryPlanSerde.{serializeDataType, supportedDataType} + +object CometSink extends CometOperatorSerde[SparkPlan] { + + override def enabledConfig: Option[ConfigEntry[Boolean]] = None + + override def convert( + op: SparkPlan, + builder: Operator.Builder, + childOp: OperatorOuterClass.Operator*): Option[OperatorOuterClass.Operator] = { + val supportedTypes = + op.output.forall(a => supportedDataType(a.dataType, allowComplex = true)) + + if (!supportedTypes) { + withInfo(op, "Unsupported data type") + return None + } + + // These operators are source of Comet native execution chain + val scanBuilder = OperatorOuterClass.Scan.newBuilder() + val source = op.simpleStringWithNodeId() + if (source.isEmpty) { + scanBuilder.setSource(op.getClass.getSimpleName) + } else { + scanBuilder.setSource(source) + } + + val ffiSafe = op match { + case _ if isExchangeSink(op) => + // Source of broadcast exchange batches is ArrowStreamReader + // Source of shuffle exchange batches is NativeBatchDecoderIterator + true + case scan: CometScanExec if scan.scanImpl == CometConf.SCAN_NATIVE_COMET => + // native_comet scan reuses mutable buffers + false + case scan: CometScanExec if scan.scanImpl == CometConf.SCAN_NATIVE_ICEBERG_COMPAT => + // native_iceberg_compat scan reuses mutable buffers for constant columns + // https://github.com/apache/datafusion-comet/issues/2152 + false + case _ => + false + } + scanBuilder.setArrowFfiSafe(ffiSafe) + + val scanTypes = op.output.flatten { attr => + serializeDataType(attr.dataType) + } + + if (scanTypes.length == op.output.length) { + scanBuilder.addAllFields(scanTypes.asJava) + + // Sink operators don't have children + builder.clearChildren() + + Some(builder.setScan(scanBuilder).build()) + } else { + // There are unsupported scan type + withInfo( + op, + s"unsupported Comet operator: ${op.nodeName}, due to unsupported data types above") + None + } + + } + + /** + * Whether the input Spark operator `op` can be considered as a Comet sink, i.e., the start of + * native execution. If it is true, we'll wrap `op` with `CometScanWrapper` or + * `CometSinkPlaceHolder` later in `CometSparkSessionExtensions` after `operator2proto` is + * called. + */ + def isCometSink(op: SparkPlan): Boolean = { + if (isExchangeSink(op)) { + return true + } + op match { + case s if isCometScan(s) => true + case _: CometSparkToColumnarExec => true + case _: CometSinkPlaceHolder => true + case _: CoalesceExec => true + case _: CollectLimitExec => true + case _: UnionExec => true + case _: TakeOrderedAndProjectExec => true + case _: WindowExec => true + case _ => false + } + } + + private def isExchangeSink(op: SparkPlan): Boolean = { + op match { + case _: ShuffleExchangeExec => true + case ShuffleQueryStageExec(_, _: CometShuffleExchangeExec, _) => true + case ShuffleQueryStageExec(_, ReusedExchangeExec(_, _: CometShuffleExchangeExec), _) => true + case BroadcastQueryStageExec(_, _: CometBroadcastExchangeExec, _) => true + case BroadcastQueryStageExec(_, ReusedExchangeExec(_, _: CometBroadcastExchangeExec), _) => + true + case _: BroadcastExchangeExec => true + case _ => false + } + } +} diff --git a/spark/src/main/scala/org/apache/comet/serde/QueryPlanSerde.scala b/spark/src/main/scala/org/apache/comet/serde/QueryPlanSerde.scala index 3e0e837c9c..820c4fa80c 100644 --- a/spark/src/main/scala/org/apache/comet/serde/QueryPlanSerde.scala +++ b/spark/src/main/scala/org/apache/comet/serde/QueryPlanSerde.scala @@ -28,19 +28,17 @@ import org.apache.spark.sql.catalyst.expressions.objects.StaticInvoke import org.apache.spark.sql.catalyst.optimizer.NormalizeNaNAndZero import org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils import org.apache.spark.sql.comet._ -import org.apache.spark.sql.comet.execution.shuffle.CometShuffleExchangeExec import org.apache.spark.sql.execution import org.apache.spark.sql.execution._ -import org.apache.spark.sql.execution.adaptive.{BroadcastQueryStageExec, ShuffleQueryStageExec} import org.apache.spark.sql.execution.aggregate.{HashAggregateExec, ObjectHashAggregateExec} -import org.apache.spark.sql.execution.exchange.{BroadcastExchangeExec, ReusedExchangeExec, ShuffleExchangeExec} +import org.apache.spark.sql.execution.exchange.ShuffleExchangeExec import org.apache.spark.sql.execution.joins.{BroadcastHashJoinExec, ShuffledHashJoinExec, SortMergeJoinExec} import org.apache.spark.sql.execution.window.WindowExec import org.apache.spark.sql.internal.SQLConf import org.apache.spark.sql.types._ import org.apache.comet.CometConf -import org.apache.comet.CometSparkSessionExtensions.{isCometScan, withInfo} +import org.apache.comet.CometSparkSessionExtensions.withInfo import org.apache.comet.expressions._ import org.apache.comet.serde.ExprOuterClass.{AggExpr, Expr, ScalarFunc} import org.apache.comet.serde.OperatorOuterClass.Operator @@ -965,59 +963,8 @@ object QueryPlanSerde extends Logging with CometExprShim { None } - case op if isCometSink(op) => - val supportedTypes = - op.output.forall(a => supportedDataType(a.dataType, allowComplex = true)) - - if (!supportedTypes) { - withInfo(op, "Unsupported data type") - return None - } - - // These operators are source of Comet native execution chain - val scanBuilder = OperatorOuterClass.Scan.newBuilder() - val source = op.simpleStringWithNodeId() - if (source.isEmpty) { - scanBuilder.setSource(op.getClass.getSimpleName) - } else { - scanBuilder.setSource(source) - } - - val ffiSafe = op match { - case _ if isExchangeSink(op) => - // Source of broadcast exchange batches is ArrowStreamReader - // Source of shuffle exchange batches is NativeBatchDecoderIterator - true - case scan: CometScanExec if scan.scanImpl == CometConf.SCAN_NATIVE_COMET => - // native_comet scan reuses mutable buffers - false - case scan: CometScanExec if scan.scanImpl == CometConf.SCAN_NATIVE_ICEBERG_COMPAT => - // native_iceberg_compat scan reuses mutable buffers for constant columns - // https://github.com/apache/datafusion-comet/issues/2152 - false - case _ => - false - } - scanBuilder.setArrowFfiSafe(ffiSafe) - - val scanTypes = op.output.flatten { attr => - serializeDataType(attr.dataType) - } - - if (scanTypes.length == op.output.length) { - scanBuilder.addAllFields(scanTypes.asJava) - - // Sink operators don't have children - builder.clearChildren() - - Some(builder.setScan(scanBuilder).build()) - } else { - // There are unsupported scan type - withInfo( - op, - s"unsupported Comet operator: ${op.nodeName}, due to unsupported data types above") - None - } + case op if CometSink.isCometSink(op) => + CometSink.convert(op, builder, childOp: _*) case op => opSerdeMap.get(op.getClass) match { @@ -1044,42 +991,6 @@ object QueryPlanSerde extends Logging with CometExprShim { } } - /** - * Whether the input Spark operator `op` can be considered as a Comet sink, i.e., the start of - * native execution. If it is true, we'll wrap `op` with `CometScanWrapper` or - * `CometSinkPlaceHolder` later in `CometSparkSessionExtensions` after `operator2proto` is - * called. - */ - private def isCometSink(op: SparkPlan): Boolean = { - if (isExchangeSink(op)) { - return true - } - op match { - case s if isCometScan(s) => true - case _: CometSparkToColumnarExec => true - case _: CometSinkPlaceHolder => true - case _: CoalesceExec => true - case _: CollectLimitExec => true - case _: UnionExec => true - case _: TakeOrderedAndProjectExec => true - case _: WindowExec => true - case _ => false - } - } - - private def isExchangeSink(op: SparkPlan): Boolean = { - op match { - case _: ShuffleExchangeExec => true - case ShuffleQueryStageExec(_, _: CometShuffleExchangeExec, _) => true - case ShuffleQueryStageExec(_, ReusedExchangeExec(_, _: CometShuffleExchangeExec), _) => true - case BroadcastQueryStageExec(_, _: CometBroadcastExchangeExec, _) => true - case BroadcastQueryStageExec(_, ReusedExchangeExec(_, _: CometBroadcastExchangeExec), _) => - true - case _: BroadcastExchangeExec => true - case _ => false - } - } - // Utility method. Adds explain info if the result of calling exprToProto is None def optExprWithInfo( optExpr: Option[Expr], From 7aeea75c6ea356d28b1917c6686263bc6a1e855a Mon Sep 17 00:00:00 2001 From: Andy Grove Date: Sat, 8 Nov 2025 09:51:03 -0700 Subject: [PATCH 18/44] refactor --- .../apache/comet/serde/QueryPlanSerde.scala | 48 +++++++++++-------- 1 file changed, 27 insertions(+), 21 deletions(-) diff --git a/spark/src/main/scala/org/apache/comet/serde/QueryPlanSerde.scala b/spark/src/main/scala/org/apache/comet/serde/QueryPlanSerde.scala index 820c4fa80c..73fb431f2a 100644 --- a/spark/src/main/scala/org/apache/comet/serde/QueryPlanSerde.scala +++ b/spark/src/main/scala/org/apache/comet/serde/QueryPlanSerde.scala @@ -918,6 +918,26 @@ object QueryPlanSerde extends Logging with CometExprShim { val builder = OperatorOuterClass.Operator.newBuilder().setPlanId(op.id) childOp.foreach(builder.addChildren) + // look for registered handler first + opSerdeMap.get(op.getClass) match { + case Some(handler) => + val enabled = handler.enabledConfig.map(_.get(op.conf)).getOrElse(true) + if (enabled) { + return handler + .asInstanceOf[CometOperatorSerde[SparkPlan]] + .convert(op, builder, childOp: _*) + } + // if the config was explicitly disabled, tag the operator + if (handler.enabledConfig.isDefined) { + withInfo( + op, + s"Native support for operator ${op.getClass.getSimpleName} is disabled. " + + s"Set ${handler.enabledConfig.get.key}=true to enable it.") + } + case _ => + } + + // now handle special cases that cannot be handled as a simple mapping from class name op match { // Fully native scan for V1 @@ -966,28 +986,14 @@ object QueryPlanSerde extends Logging with CometExprShim { case op if CometSink.isCometSink(op) => CometSink.convert(op, builder, childOp: _*) - case op => - opSerdeMap.get(op.getClass) match { - case Some(handler) => - handler.enabledConfig.foreach { enabledConfig => - if (!enabledConfig.get(op.conf)) { - withInfo( - op, - s"Native support for operator ${op.getClass.getSimpleName} is disabled. " + - s"Set ${enabledConfig.key}=true to enable it.") - return None - } - } - handler.asInstanceOf[CometOperatorSerde[SparkPlan]].convert(op, builder, childOp: _*) - case _ => - // Emit warning if: - // 1. it is not Spark shuffle operator, which is handled separately - // 2. it is not a Comet operator - if (!op.nodeName.contains("Comet") && !op.isInstanceOf[ShuffleExchangeExec]) { - withInfo(op, s"unsupported Spark operator: ${op.nodeName}") - } - None + case _ => + // Emit warning if: + // 1. it is not Spark shuffle operator, which is handled separately + // 2. it is not a Comet operator + if (!op.nodeName.contains("Comet") && !op.isInstanceOf[ShuffleExchangeExec]) { + withInfo(op, s"unsupported Spark operator: ${op.nodeName}") } + None } } From e811e7a11613c4018118c97b770d3af0cf1cc7d4 Mon Sep 17 00:00:00 2001 From: Andy Grove Date: Sat, 8 Nov 2025 09:53:55 -0700 Subject: [PATCH 19/44] fix --- .../main/scala/org/apache/comet/serde/QueryPlanSerde.scala | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/spark/src/main/scala/org/apache/comet/serde/QueryPlanSerde.scala b/spark/src/main/scala/org/apache/comet/serde/QueryPlanSerde.scala index 73fb431f2a..ab1e827138 100644 --- a/spark/src/main/scala/org/apache/comet/serde/QueryPlanSerde.scala +++ b/spark/src/main/scala/org/apache/comet/serde/QueryPlanSerde.scala @@ -923,9 +923,12 @@ object QueryPlanSerde extends Logging with CometExprShim { case Some(handler) => val enabled = handler.enabledConfig.map(_.get(op.conf)).getOrElse(true) if (enabled) { - return handler + val maybeConverted = handler .asInstanceOf[CometOperatorSerde[SparkPlan]] .convert(op, builder, childOp: _*) + if (maybeConverted.isDefined) { + return maybeConverted + } } // if the config was explicitly disabled, tag the operator if (handler.enabledConfig.isDefined) { @@ -938,6 +941,7 @@ object QueryPlanSerde extends Logging with CometExprShim { } // now handle special cases that cannot be handled as a simple mapping from class name + // and see if operator can be used as a sink op match { // Fully native scan for V1 From 7c59a0ca38088e90c7e7b49ed98186514662a37d Mon Sep 17 00:00:00 2001 From: Andy Grove Date: Sat, 8 Nov 2025 09:58:41 -0700 Subject: [PATCH 20/44] map lookup --- .../apache/comet/serde/QueryPlanSerde.scala | 50 ++++++------------- 1 file changed, 14 insertions(+), 36 deletions(-) diff --git a/spark/src/main/scala/org/apache/comet/serde/QueryPlanSerde.scala b/spark/src/main/scala/org/apache/comet/serde/QueryPlanSerde.scala index ab1e827138..fa2dc432c2 100644 --- a/spark/src/main/scala/org/apache/comet/serde/QueryPlanSerde.scala +++ b/spark/src/main/scala/org/apache/comet/serde/QueryPlanSerde.scala @@ -56,7 +56,20 @@ object QueryPlanSerde extends Logging with CometExprShim { * Mapping of Spark operator class to Comet operator handler. */ private val opSerdeMap: Map[Class[_ <: SparkPlan], CometOperatorSerde[_]] = - Map(classOf[ProjectExec] -> CometProject, classOf[SortExec] -> CometSort) + Map( + classOf[ProjectExec] -> CometProject, + classOf[FilterExec] -> CometFilter, + classOf[LocalLimitExec] -> CometLocalLimit, + classOf[GlobalLimitExec] -> CometGlobalLimit, + classOf[ExpandExec] -> CometExpand, + classOf[HashAggregateExec] -> CometHashAggregate, + classOf[ObjectHashAggregateExec] -> CometObjectHashAggregate, + classOf[BroadcastHashJoinExec] -> CometBroadcastHashJoin, + classOf[ShuffledHashJoinExec] -> CometShuffleHashJoin, + classOf[SortMergeJoinExec] -> CometSortMergeJoin, + // Window support is disabled due to correctness issues + // classOf[WindowExec] -> CometWindow, + classOf[SortExec] -> CometSort) private val arrayExpressions: Map[Class[_ <: Expression], CometExpressionSerde[_]] = Map( classOf[ArrayAppend] -> CometArrayAppend, @@ -948,45 +961,10 @@ object QueryPlanSerde extends Logging with CometExprShim { case scan: CometScanExec if scan.scanImpl == CometConf.SCAN_NATIVE_DATAFUSION => CometNativeScan.convert(scan, builder, childOp: _*) - case filter: FilterExec if CometConf.COMET_EXEC_FILTER_ENABLED.get(conf) => - CometFilter.convert(filter, builder, childOp: _*) - - case limit: LocalLimitExec if CometConf.COMET_EXEC_LOCAL_LIMIT_ENABLED.get(conf) => - CometLocalLimit.convert(limit, builder, childOp: _*) - - case globalLimitExec: GlobalLimitExec - if CometConf.COMET_EXEC_GLOBAL_LIMIT_ENABLED.get(conf) => - CometGlobalLimit.convert(globalLimitExec, builder, childOp: _*) - - case expand: ExpandExec if CometConf.COMET_EXEC_EXPAND_ENABLED.get(conf) => - CometExpand.convert(expand, builder, childOp: _*) - case _: WindowExec if CometConf.COMET_EXEC_WINDOW_ENABLED.get(conf) => withInfo(op, "Window expressions are not supported") None - case aggregate: HashAggregateExec if CometConf.COMET_EXEC_AGGREGATE_ENABLED.get(conf) => - CometHashAggregate.convert(aggregate, builder, childOp: _*) - - case aggregate: ObjectHashAggregateExec - if CometConf.COMET_EXEC_AGGREGATE_ENABLED.get(conf) => - CometObjectHashAggregate.convert(aggregate, builder, childOp: _*) - - case join: BroadcastHashJoinExec - if CometConf.COMET_EXEC_BROADCAST_HASH_JOIN_ENABLED.get(conf) => - CometBroadcastHashJoin.convert(join, builder, childOp: _*) - - case join: ShuffledHashJoinExec if CometConf.COMET_EXEC_HASH_JOIN_ENABLED.get(conf) => - CometShuffleHashJoin.convert(join, builder, childOp: _*) - - case join: SortMergeJoinExec => - if (CometConf.COMET_EXEC_SORT_MERGE_JOIN_ENABLED.get(conf)) { - CometSortMergeJoin.convert(join, builder, childOp: _*) - } else { - withInfo(join, "SortMergeJoin is not enabled") - None - } - case op if CometSink.isCometSink(op) => CometSink.convert(op, builder, childOp: _*) From 83cc2c363684652436b65fc26686bc52f093bb5f Mon Sep 17 00:00:00 2001 From: Andy Grove Date: Sat, 8 Nov 2025 10:22:29 -0700 Subject: [PATCH 21/44] more refactoring --- .../apache/comet/serde/QueryPlanSerde.scala | 25 +---------- .../org/apache/comet/serde/SupportLevel.scala | 42 +++++++++++++++++++ 2 files changed, 44 insertions(+), 23 deletions(-) create mode 100644 spark/src/main/scala/org/apache/comet/serde/SupportLevel.scala diff --git a/spark/src/main/scala/org/apache/comet/serde/QueryPlanSerde.scala b/spark/src/main/scala/org/apache/comet/serde/QueryPlanSerde.scala index fa2dc432c2..e9fe23dac8 100644 --- a/spark/src/main/scala/org/apache/comet/serde/QueryPlanSerde.scala +++ b/spark/src/main/scala/org/apache/comet/serde/QueryPlanSerde.scala @@ -934,7 +934,7 @@ object QueryPlanSerde extends Logging with CometExprShim { // look for registered handler first opSerdeMap.get(op.getClass) match { case Some(handler) => - val enabled = handler.enabledConfig.map(_.get(op.conf)).getOrElse(true) + val enabled = handler.enabledConfig.forall(_.get(op.conf)) if (enabled) { val maybeConverted = handler .asInstanceOf[CometOperatorSerde[SparkPlan]] @@ -961,6 +961,7 @@ object QueryPlanSerde extends Logging with CometExprShim { case scan: CometScanExec if scan.scanImpl == CometConf.SCAN_NATIVE_DATAFUSION => CometNativeScan.convert(scan, builder, childOp: _*) + // this is kept for backwards compatibility with current golden files case _: WindowExec if CometConf.COMET_EXEC_WINDOW_ENABLED.get(conf) => withInfo(op, "Window expressions are not supported") None @@ -1039,25 +1040,3 @@ object QueryPlanSerde extends Logging with CometExprShim { } } - -sealed trait SupportLevel - -/** - * Comet either supports this feature with full compatibility with Spark, or may have known - * differences in some specific edge cases that are unlikely to be an issue for most users. - * - * Any compatibility differences are noted in the - * [[https://datafusion.apache.org/comet/user-guide/compatibility.html Comet Compatibility Guide]]. - */ -case class Compatible(notes: Option[String] = None) extends SupportLevel - -/** - * Comet supports this feature but results can be different from Spark. - * - * Any compatibility differences are noted in the - * [[https://datafusion.apache.org/comet/user-guide/compatibility.html Comet Compatibility Guide]]. - */ -case class Incompatible(notes: Option[String] = None) extends SupportLevel - -/** Comet does not support this feature */ -case class Unsupported(notes: Option[String] = None) extends SupportLevel diff --git a/spark/src/main/scala/org/apache/comet/serde/SupportLevel.scala b/spark/src/main/scala/org/apache/comet/serde/SupportLevel.scala new file mode 100644 index 0000000000..d5a524077d --- /dev/null +++ b/spark/src/main/scala/org/apache/comet/serde/SupportLevel.scala @@ -0,0 +1,42 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.comet.serde + +sealed trait SupportLevel + +/** + * Comet either supports this feature with full compatibility with Spark, or may have known + * differences in some specific edge cases that are unlikely to be an issue for most users. + * + * Any compatibility differences are noted in the + * [[https://datafusion.apache.org/comet/user-guide/compatibility.html Comet Compatibility Guide]]. + */ +case class Compatible(notes: Option[String] = None) extends SupportLevel + +/** + * Comet supports this feature but results can be different from Spark. + * + * Any compatibility differences are noted in the + * [[https://datafusion.apache.org/comet/user-guide/compatibility.html Comet Compatibility Guide]]. + */ +case class Incompatible(notes: Option[String] = None) extends SupportLevel + +/** Comet does not support this feature */ +case class Unsupported(notes: Option[String] = None) extends SupportLevel From 5131395bfa372a23c6f8a3001b0e1cbfb099b6a1 Mon Sep 17 00:00:00 2001 From: Andy Grove Date: Sat, 8 Nov 2025 10:27:52 -0700 Subject: [PATCH 22/44] more refactoring --- .../org/apache/comet/serde/CometWindow.scala | 179 +++++++++++++++++- .../apache/comet/serde/QueryPlanSerde.scala | 172 ----------------- 2 files changed, 177 insertions(+), 174 deletions(-) diff --git a/spark/src/main/scala/org/apache/comet/serde/CometWindow.scala b/spark/src/main/scala/org/apache/comet/serde/CometWindow.scala index 7e963d6326..e1ea3ff388 100644 --- a/spark/src/main/scala/org/apache/comet/serde/CometWindow.scala +++ b/spark/src/main/scala/org/apache/comet/serde/CometWindow.scala @@ -21,14 +21,17 @@ package org.apache.comet.serde import scala.jdk.CollectionConverters._ -import org.apache.spark.sql.catalyst.expressions.{Alias, AttributeReference, Expression, SortOrder, WindowExpression} +import org.apache.spark.sql.catalyst.expressions.{Alias, Attribute, AttributeReference, CurrentRow, Expression, RangeFrame, RowFrame, SortOrder, SpecifiedWindowFrame, UnboundedFollowing, UnboundedPreceding, WindowExpression} +import org.apache.spark.sql.catalyst.expressions.aggregate.{AggregateExpression, Complete, Count, Max, Min, Sum} import org.apache.spark.sql.execution.SparkPlan import org.apache.spark.sql.execution.window.WindowExec +import org.apache.spark.sql.internal.SQLConf +import org.apache.spark.sql.types.DecimalType import org.apache.comet.{CometConf, ConfigEntry} import org.apache.comet.CometSparkSessionExtensions.withInfo import org.apache.comet.serde.OperatorOuterClass.Operator -import org.apache.comet.serde.QueryPlanSerde.{exprToProto, windowExprToProto} +import org.apache.comet.serde.QueryPlanSerde.{aggExprToProto, exprToProto} object CometWindow extends CometOperatorSerde[WindowExec] { @@ -83,6 +86,178 @@ object CometWindow extends CometOperatorSerde[WindowExec] { } + def windowExprToProto( + windowExpr: WindowExpression, + output: Seq[Attribute], + conf: SQLConf): Option[OperatorOuterClass.WindowExpr] = { + + val aggregateExpressions: Array[AggregateExpression] = windowExpr.flatMap { expr => + expr match { + case agg: AggregateExpression => + agg.aggregateFunction match { + case _: Count => + Some(agg) + case min: Min => + if (AggSerde.minMaxDataTypeSupported(min.dataType)) { + Some(agg) + } else { + withInfo(windowExpr, s"datatype ${min.dataType} is not supported", expr) + None + } + case max: Max => + if (AggSerde.minMaxDataTypeSupported(max.dataType)) { + Some(agg) + } else { + withInfo(windowExpr, s"datatype ${max.dataType} is not supported", expr) + None + } + case s: Sum => + if (AggSerde.sumDataTypeSupported(s.dataType) && !s.dataType + .isInstanceOf[DecimalType]) { + Some(agg) + } else { + withInfo(windowExpr, s"datatype ${s.dataType} is not supported", expr) + None + } + case _ => + withInfo( + windowExpr, + s"aggregate ${agg.aggregateFunction}" + + " is not supported for window function", + expr) + None + } + case _ => + None + } + }.toArray + + val (aggExpr, builtinFunc) = if (aggregateExpressions.nonEmpty) { + val modes = aggregateExpressions.map(_.mode).distinct + assert(modes.size == 1 && modes.head == Complete) + (aggExprToProto(aggregateExpressions.head, output, true, conf), None) + } else { + (None, exprToProto(windowExpr.windowFunction, output)) + } + + if (aggExpr.isEmpty && builtinFunc.isEmpty) { + return None + } + + val f = windowExpr.windowSpec.frameSpecification + + val (frameType, lowerBound, upperBound) = f match { + case SpecifiedWindowFrame(frameType, lBound, uBound) => + val frameProto = frameType match { + case RowFrame => OperatorOuterClass.WindowFrameType.Rows + case RangeFrame => OperatorOuterClass.WindowFrameType.Range + } + + val lBoundProto = lBound match { + case UnboundedPreceding => + OperatorOuterClass.LowerWindowFrameBound + .newBuilder() + .setUnboundedPreceding(OperatorOuterClass.UnboundedPreceding.newBuilder().build()) + .build() + case CurrentRow => + OperatorOuterClass.LowerWindowFrameBound + .newBuilder() + .setCurrentRow(OperatorOuterClass.CurrentRow.newBuilder().build()) + .build() + case e if frameType == RowFrame => + val offset = e.eval() match { + case i: Integer => i.toLong + case l: Long => l + case _ => return None + } + OperatorOuterClass.LowerWindowFrameBound + .newBuilder() + .setPreceding( + OperatorOuterClass.Preceding + .newBuilder() + .setOffset(offset) + .build()) + .build() + case _ => + // TODO add support for numeric and temporal RANGE BETWEEN expressions + // see https://github.com/apache/datafusion-comet/issues/1246 + return None + } + + val uBoundProto = uBound match { + case UnboundedFollowing => + OperatorOuterClass.UpperWindowFrameBound + .newBuilder() + .setUnboundedFollowing(OperatorOuterClass.UnboundedFollowing.newBuilder().build()) + .build() + case CurrentRow => + OperatorOuterClass.UpperWindowFrameBound + .newBuilder() + .setCurrentRow(OperatorOuterClass.CurrentRow.newBuilder().build()) + .build() + case e if frameType == RowFrame => + val offset = e.eval() match { + case i: Integer => i.toLong + case l: Long => l + case _ => return None + } + OperatorOuterClass.UpperWindowFrameBound + .newBuilder() + .setFollowing( + OperatorOuterClass.Following + .newBuilder() + .setOffset(offset) + .build()) + .build() + case _ => + // TODO add support for numeric and temporal RANGE BETWEEN expressions + // see https://github.com/apache/datafusion-comet/issues/1246 + return None + } + + (frameProto, lBoundProto, uBoundProto) + case _ => + ( + OperatorOuterClass.WindowFrameType.Rows, + OperatorOuterClass.LowerWindowFrameBound + .newBuilder() + .setUnboundedPreceding(OperatorOuterClass.UnboundedPreceding.newBuilder().build()) + .build(), + OperatorOuterClass.UpperWindowFrameBound + .newBuilder() + .setUnboundedFollowing(OperatorOuterClass.UnboundedFollowing.newBuilder().build()) + .build()) + } + + val frame = OperatorOuterClass.WindowFrame + .newBuilder() + .setFrameType(frameType) + .setLowerBound(lowerBound) + .setUpperBound(upperBound) + .build() + + val spec = + OperatorOuterClass.WindowSpecDefinition.newBuilder().setFrameSpecification(frame).build() + + if (builtinFunc.isDefined) { + Some( + OperatorOuterClass.WindowExpr + .newBuilder() + .setBuiltInWindowFunction(builtinFunc.get) + .setSpec(spec) + .build()) + } else if (aggExpr.isDefined) { + Some( + OperatorOuterClass.WindowExpr + .newBuilder() + .setAggFunc(aggExpr.get) + .setSpec(spec) + .build()) + } else { + None + } + } + private def validatePartitionAndSortSpecsForWindowFunc( partitionSpec: Seq[Expression], orderSpec: Seq[SortOrder], diff --git a/spark/src/main/scala/org/apache/comet/serde/QueryPlanSerde.scala b/spark/src/main/scala/org/apache/comet/serde/QueryPlanSerde.scala index e9fe23dac8..a42108f105 100644 --- a/spark/src/main/scala/org/apache/comet/serde/QueryPlanSerde.scala +++ b/spark/src/main/scala/org/apache/comet/serde/QueryPlanSerde.scala @@ -390,178 +390,6 @@ object QueryPlanSerde extends Logging with CometExprShim { Some(dataType) } - def windowExprToProto( - windowExpr: WindowExpression, - output: Seq[Attribute], - conf: SQLConf): Option[OperatorOuterClass.WindowExpr] = { - - val aggregateExpressions: Array[AggregateExpression] = windowExpr.flatMap { expr => - expr match { - case agg: AggregateExpression => - agg.aggregateFunction match { - case _: Count => - Some(agg) - case min: Min => - if (AggSerde.minMaxDataTypeSupported(min.dataType)) { - Some(agg) - } else { - withInfo(windowExpr, s"datatype ${min.dataType} is not supported", expr) - None - } - case max: Max => - if (AggSerde.minMaxDataTypeSupported(max.dataType)) { - Some(agg) - } else { - withInfo(windowExpr, s"datatype ${max.dataType} is not supported", expr) - None - } - case s: Sum => - if (AggSerde.sumDataTypeSupported(s.dataType) && !s.dataType - .isInstanceOf[DecimalType]) { - Some(agg) - } else { - withInfo(windowExpr, s"datatype ${s.dataType} is not supported", expr) - None - } - case _ => - withInfo( - windowExpr, - s"aggregate ${agg.aggregateFunction}" + - " is not supported for window function", - expr) - None - } - case _ => - None - } - }.toArray - - val (aggExpr, builtinFunc) = if (aggregateExpressions.nonEmpty) { - val modes = aggregateExpressions.map(_.mode).distinct - assert(modes.size == 1 && modes.head == Complete) - (aggExprToProto(aggregateExpressions.head, output, true, conf), None) - } else { - (None, exprToProto(windowExpr.windowFunction, output)) - } - - if (aggExpr.isEmpty && builtinFunc.isEmpty) { - return None - } - - val f = windowExpr.windowSpec.frameSpecification - - val (frameType, lowerBound, upperBound) = f match { - case SpecifiedWindowFrame(frameType, lBound, uBound) => - val frameProto = frameType match { - case RowFrame => OperatorOuterClass.WindowFrameType.Rows - case RangeFrame => OperatorOuterClass.WindowFrameType.Range - } - - val lBoundProto = lBound match { - case UnboundedPreceding => - OperatorOuterClass.LowerWindowFrameBound - .newBuilder() - .setUnboundedPreceding(OperatorOuterClass.UnboundedPreceding.newBuilder().build()) - .build() - case CurrentRow => - OperatorOuterClass.LowerWindowFrameBound - .newBuilder() - .setCurrentRow(OperatorOuterClass.CurrentRow.newBuilder().build()) - .build() - case e if frameType == RowFrame => - val offset = e.eval() match { - case i: Integer => i.toLong - case l: Long => l - case _ => return None - } - OperatorOuterClass.LowerWindowFrameBound - .newBuilder() - .setPreceding( - OperatorOuterClass.Preceding - .newBuilder() - .setOffset(offset) - .build()) - .build() - case _ => - // TODO add support for numeric and temporal RANGE BETWEEN expressions - // see https://github.com/apache/datafusion-comet/issues/1246 - return None - } - - val uBoundProto = uBound match { - case UnboundedFollowing => - OperatorOuterClass.UpperWindowFrameBound - .newBuilder() - .setUnboundedFollowing(OperatorOuterClass.UnboundedFollowing.newBuilder().build()) - .build() - case CurrentRow => - OperatorOuterClass.UpperWindowFrameBound - .newBuilder() - .setCurrentRow(OperatorOuterClass.CurrentRow.newBuilder().build()) - .build() - case e if frameType == RowFrame => - val offset = e.eval() match { - case i: Integer => i.toLong - case l: Long => l - case _ => return None - } - OperatorOuterClass.UpperWindowFrameBound - .newBuilder() - .setFollowing( - OperatorOuterClass.Following - .newBuilder() - .setOffset(offset) - .build()) - .build() - case _ => - // TODO add support for numeric and temporal RANGE BETWEEN expressions - // see https://github.com/apache/datafusion-comet/issues/1246 - return None - } - - (frameProto, lBoundProto, uBoundProto) - case _ => - ( - OperatorOuterClass.WindowFrameType.Rows, - OperatorOuterClass.LowerWindowFrameBound - .newBuilder() - .setUnboundedPreceding(OperatorOuterClass.UnboundedPreceding.newBuilder().build()) - .build(), - OperatorOuterClass.UpperWindowFrameBound - .newBuilder() - .setUnboundedFollowing(OperatorOuterClass.UnboundedFollowing.newBuilder().build()) - .build()) - } - - val frame = OperatorOuterClass.WindowFrame - .newBuilder() - .setFrameType(frameType) - .setLowerBound(lowerBound) - .setUpperBound(upperBound) - .build() - - val spec = - OperatorOuterClass.WindowSpecDefinition.newBuilder().setFrameSpecification(frame).build() - - if (builtinFunc.isDefined) { - Some( - OperatorOuterClass.WindowExpr - .newBuilder() - .setBuiltInWindowFunction(builtinFunc.get) - .setSpec(spec) - .build()) - } else if (aggExpr.isDefined) { - Some( - OperatorOuterClass.WindowExpr - .newBuilder() - .setAggFunc(aggExpr.get) - .setSpec(spec) - .build()) - } else { - None - } - } - def aggExprToProto( aggExpr: AggregateExpression, inputs: Seq[Attribute], From 82da68f920f960b6f140d5123f201d04f0a19d7a Mon Sep 17 00:00:00 2001 From: Andy Grove Date: Sat, 8 Nov 2025 10:39:30 -0700 Subject: [PATCH 23/44] comments --- .../main/scala/org/apache/comet/serde/QueryPlanSerde.scala | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/spark/src/main/scala/org/apache/comet/serde/QueryPlanSerde.scala b/spark/src/main/scala/org/apache/comet/serde/QueryPlanSerde.scala index a42108f105..aae25c53b0 100644 --- a/spark/src/main/scala/org/apache/comet/serde/QueryPlanSerde.scala +++ b/spark/src/main/scala/org/apache/comet/serde/QueryPlanSerde.scala @@ -67,7 +67,9 @@ object QueryPlanSerde extends Logging with CometExprShim { classOf[BroadcastHashJoinExec] -> CometBroadcastHashJoin, classOf[ShuffledHashJoinExec] -> CometShuffleHashJoin, classOf[SortMergeJoinExec] -> CometSortMergeJoin, - // Window support is disabled due to correctness issues + // Window support is disabled due to correctness issues and + // cannot be disabled via config due to bug + // https://github.com/apache/datafusion-comet/issues/2737 // classOf[WindowExec] -> CometWindow, classOf[SortExec] -> CometSort) @@ -790,6 +792,7 @@ object QueryPlanSerde extends Logging with CometExprShim { CometNativeScan.convert(scan, builder, childOp: _*) // this is kept for backwards compatibility with current golden files + // and also as a workaround for https://github.com/apache/datafusion-comet/issues/2737 case _: WindowExec if CometConf.COMET_EXEC_WINDOW_ENABLED.get(conf) => withInfo(op, "Window expressions are not supported") None From 54e9d172c265cd772fa971305cc58271b1a75276 Mon Sep 17 00:00:00 2001 From: Andy Grove Date: Sat, 8 Nov 2025 10:47:54 -0700 Subject: [PATCH 24/44] fix --- .../src/main/scala/org/apache/comet/CometConf.scala | 2 +- .../main/scala/org/apache/comet/serde/CometSink.scala | 4 ++-- .../scala/org/apache/comet/serde/QueryPlanSerde.scala | 11 +---------- 3 files changed, 4 insertions(+), 13 deletions(-) diff --git a/common/src/main/scala/org/apache/comet/CometConf.scala b/common/src/main/scala/org/apache/comet/CometConf.scala index 60fd1940bc..7bd075eb50 100644 --- a/common/src/main/scala/org/apache/comet/CometConf.scala +++ b/common/src/main/scala/org/apache/comet/CometConf.scala @@ -248,7 +248,7 @@ object CometConf extends ShimCometConf { val COMET_EXEC_EXPAND_ENABLED: ConfigEntry[Boolean] = createExecEnabledConfig("expand", defaultValue = true) val COMET_EXEC_WINDOW_ENABLED: ConfigEntry[Boolean] = - createExecEnabledConfig("window", defaultValue = true) + createExecEnabledConfig("window", defaultValue = false) val COMET_EXEC_TAKE_ORDERED_AND_PROJECT_ENABLED: ConfigEntry[Boolean] = createExecEnabledConfig("takeOrderedAndProject", defaultValue = true) diff --git a/spark/src/main/scala/org/apache/comet/serde/CometSink.scala b/spark/src/main/scala/org/apache/comet/serde/CometSink.scala index ec7aac8ec2..270208a0a5 100644 --- a/spark/src/main/scala/org/apache/comet/serde/CometSink.scala +++ b/spark/src/main/scala/org/apache/comet/serde/CometSink.scala @@ -26,7 +26,6 @@ import org.apache.spark.sql.comet.execution.shuffle.CometShuffleExchangeExec import org.apache.spark.sql.execution.{CoalesceExec, CollectLimitExec, SparkPlan, TakeOrderedAndProjectExec, UnionExec} import org.apache.spark.sql.execution.adaptive.{BroadcastQueryStageExec, ShuffleQueryStageExec} import org.apache.spark.sql.execution.exchange.{BroadcastExchangeExec, ReusedExchangeExec, ShuffleExchangeExec} -import org.apache.spark.sql.execution.window.WindowExec import org.apache.comet.{CometConf, ConfigEntry} import org.apache.comet.CometSparkSessionExtensions.{isCometScan, withInfo} @@ -114,7 +113,8 @@ object CometSink extends CometOperatorSerde[SparkPlan] { case _: CollectLimitExec => true case _: UnionExec => true case _: TakeOrderedAndProjectExec => true - case _: WindowExec => true + // https://github.com/apache/datafusion-comet/issues/2737 + // case _: WindowExec => true case _ => false } } diff --git a/spark/src/main/scala/org/apache/comet/serde/QueryPlanSerde.scala b/spark/src/main/scala/org/apache/comet/serde/QueryPlanSerde.scala index aae25c53b0..3e47c59134 100644 --- a/spark/src/main/scala/org/apache/comet/serde/QueryPlanSerde.scala +++ b/spark/src/main/scala/org/apache/comet/serde/QueryPlanSerde.scala @@ -67,10 +67,7 @@ object QueryPlanSerde extends Logging with CometExprShim { classOf[BroadcastHashJoinExec] -> CometBroadcastHashJoin, classOf[ShuffledHashJoinExec] -> CometShuffleHashJoin, classOf[SortMergeJoinExec] -> CometSortMergeJoin, - // Window support is disabled due to correctness issues and - // cannot be disabled via config due to bug - // https://github.com/apache/datafusion-comet/issues/2737 - // classOf[WindowExec] -> CometWindow, + classOf[WindowExec] -> CometWindow, classOf[SortExec] -> CometSort) private val arrayExpressions: Map[Class[_ <: Expression], CometExpressionSerde[_]] = Map( @@ -791,12 +788,6 @@ object QueryPlanSerde extends Logging with CometExprShim { case scan: CometScanExec if scan.scanImpl == CometConf.SCAN_NATIVE_DATAFUSION => CometNativeScan.convert(scan, builder, childOp: _*) - // this is kept for backwards compatibility with current golden files - // and also as a workaround for https://github.com/apache/datafusion-comet/issues/2737 - case _: WindowExec if CometConf.COMET_EXEC_WINDOW_ENABLED.get(conf) => - withInfo(op, "Window expressions are not supported") - None - case op if CometSink.isCometSink(op) => CometSink.convert(op, builder, childOp: _*) From 266ff4d4c959586199426f44d0fa36e5917e1f6e Mon Sep 17 00:00:00 2001 From: Andy Grove Date: Sat, 8 Nov 2025 10:49:16 -0700 Subject: [PATCH 25/44] fix --- spark/src/main/scala/org/apache/comet/serde/QueryPlanSerde.scala | 1 - 1 file changed, 1 deletion(-) diff --git a/spark/src/main/scala/org/apache/comet/serde/QueryPlanSerde.scala b/spark/src/main/scala/org/apache/comet/serde/QueryPlanSerde.scala index 3e47c59134..49a44f7b16 100644 --- a/spark/src/main/scala/org/apache/comet/serde/QueryPlanSerde.scala +++ b/spark/src/main/scala/org/apache/comet/serde/QueryPlanSerde.scala @@ -754,7 +754,6 @@ object QueryPlanSerde extends Logging with CometExprShim { * converted to a native operator. */ def operator2Proto(op: SparkPlan, childOp: Operator*): Option[Operator] = { - val conf = op.conf val builder = OperatorOuterClass.Operator.newBuilder().setPlanId(op.id) childOp.foreach(builder.addChildren) From ff2b9ad1fff34920deed5950957a9d69fc70bab7 Mon Sep 17 00:00:00 2001 From: Andy Grove Date: Sat, 8 Nov 2025 11:00:40 -0700 Subject: [PATCH 26/44] docs --- docs/source/user-guide/latest/configs.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/source/user-guide/latest/configs.md b/docs/source/user-guide/latest/configs.md index 1cc05dfc78..bf6388549c 100644 --- a/docs/source/user-guide/latest/configs.md +++ b/docs/source/user-guide/latest/configs.md @@ -155,7 +155,7 @@ These settings can be used to determine which parts of the plan are accelerated | `spark.comet.exec.sortMergeJoinWithJoinFilter.enabled` | Experimental support for Sort Merge Join with filter | false | | `spark.comet.exec.takeOrderedAndProject.enabled` | Whether to enable takeOrderedAndProject by default. | true | | `spark.comet.exec.union.enabled` | Whether to enable union by default. | true | -| `spark.comet.exec.window.enabled` | Whether to enable window by default. | true | +| `spark.comet.exec.window.enabled` | Whether to enable window by default. | false | ## Enabling or Disabling Individual Scalar Expressions From ba2ca5a08b02b3f63159c86a3734222830ef27d5 Mon Sep 17 00:00:00 2001 From: Andy Grove Date: Sat, 8 Nov 2025 11:28:03 -0700 Subject: [PATCH 27/44] docs --- docs/source/user-guide/latest/operators.md | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/docs/source/user-guide/latest/operators.md b/docs/source/user-guide/latest/operators.md index 066daf4b95..805eb47157 100644 --- a/docs/source/user-guide/latest/operators.md +++ b/docs/source/user-guide/latest/operators.md @@ -23,7 +23,7 @@ The following Spark operators are currently replaced with native versions. Query not supported by Comet will fall back to regular Spark execution. | Operator | Spark-Compatible? | Compatibility Notes | -| ----------------------- | ----------------- | ------------------------------------------------------------------------------------------------------------------ | +| ----------------------- |-------------------|--------------------------------------------------------------------------------------------------------------------| | BatchScanExec | Yes | Supports Parquet files and Apache Iceberg Parquet scans. See the [Comet Compatibility Guide] for more information. | | BroadcastExchangeExec | Yes | | | BroadcastHashJoinExec | Yes | | @@ -40,6 +40,6 @@ not supported by Comet will fall back to regular Spark execution. | SortExec | Yes | | | SortMergeJoinExec | Yes | | | UnionExec | Yes | | -| WindowExec | Yes | | +| WindowExec | No | Disabled by default due to known correctness issues. | [Comet Compatibility Guide]: compatibility.md From 91407426a6aa0300c6bd4f527c68bfe77407bea6 Mon Sep 17 00:00:00 2001 From: Andy Grove Date: Sat, 8 Nov 2025 11:34:17 -0700 Subject: [PATCH 28/44] docs --- docs/source/user-guide/latest/expressions.md | 17 +++++++++++++++++ 1 file changed, 17 insertions(+) diff --git a/docs/source/user-guide/latest/expressions.md b/docs/source/user-guide/latest/expressions.md index fe42f49a47..548941f7a0 100644 --- a/docs/source/user-guide/latest/expressions.md +++ b/docs/source/user-guide/latest/expressions.md @@ -202,6 +202,23 @@ incompatible expressions. | VariancePop | | Yes | | | VarianceSamp | | Yes | | +## Window Functions + +```{warning} +Window support is disabled by default due to known correctness issues. Tracking issue: [#2721](https://github.com/apache/datafusion-comet/issues/2721). +``` + +Comet supports using the following aggregate functions within window contexts with PARTITION BY and ORDER BY clauses. + +| Expression | Spark-Compatible? | Compatibility Notes | +| ---------- | ----------------- | -------------------------------------------- | +| Count | Yes | | +| Max | Yes | | +| Min | Yes | | +| Sum | Yes | | + +**Note:** Dedicated window functions such as `rank`, `dense_rank`, `row_number`, `lag`, `lead`, `ntile`, `cume_dist`, `percent_rank`, and `nth_value` are not currently supported and will fall back to Spark. + ## Array Expressions | Expression | Spark-Compatible? | Compatibility Notes | From 7216f3c2ffa7829c96302398d1d4bb3a696249de Mon Sep 17 00:00:00 2001 From: Andy Grove Date: Sat, 8 Nov 2025 12:28:41 -0700 Subject: [PATCH 29/44] improve withInfo --- .../scala/org/apache/comet/serde/QueryPlanSerde.scala | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/spark/src/main/scala/org/apache/comet/serde/QueryPlanSerde.scala b/spark/src/main/scala/org/apache/comet/serde/QueryPlanSerde.scala index 49a44f7b16..410dda8ba0 100644 --- a/spark/src/main/scala/org/apache/comet/serde/QueryPlanSerde.scala +++ b/spark/src/main/scala/org/apache/comet/serde/QueryPlanSerde.scala @@ -758,7 +758,8 @@ object QueryPlanSerde extends Logging with CometExprShim { childOp.foreach(builder.addChildren) // look for registered handler first - opSerdeMap.get(op.getClass) match { + val serde = opSerdeMap.get(op.getClass) + serde match { case Some(handler) => val enabled = handler.enabledConfig.forall(_.get(op.conf)) if (enabled) { @@ -795,7 +796,10 @@ object QueryPlanSerde extends Logging with CometExprShim { // 1. it is not Spark shuffle operator, which is handled separately // 2. it is not a Comet operator if (!op.nodeName.contains("Comet") && !op.isInstanceOf[ShuffleExchangeExec]) { - withInfo(op, s"unsupported Spark operator: ${op.nodeName}") + // skip unsupported operator tag if operator is supported but disabled + if (serde.isEmpty) { + withInfo(op, s"unsupported Spark operator: ${op.nodeName}") + } } None } From 1a4807b9c646bbfa669992c7d7e5e226ec2be91e Mon Sep 17 00:00:00 2001 From: Andy Grove Date: Sat, 8 Nov 2025 12:44:12 -0700 Subject: [PATCH 30/44] add operator incompat handling --- .../scala/org/apache/comet/CometConf.scala | 16 ++++++++- .../comet/serde/CometOperatorSerde.scala | 21 +++++++++--- .../org/apache/comet/serde/CometWindow.scala | 4 +++ .../apache/comet/serde/QueryPlanSerde.scala | 34 ++++++++++++++++--- .../q12.native_iceberg_compat/extended.txt | 2 +- .../q12/extended.txt | 2 +- .../q16.native_iceberg_compat/extended.txt | 2 +- .../q16/extended.txt | 2 +- .../q20.native_iceberg_compat/extended.txt | 2 +- .../q20/extended.txt | 2 +- .../q28.native_iceberg_compat/extended.txt | 12 +++---- .../q28/extended.txt | 12 +++---- .../q36.native_iceberg_compat/extended.txt | 2 +- .../q36/extended.txt | 2 +- .../q47.native_iceberg_compat/extended.txt | 6 ++-- .../q47/extended.txt | 6 ++-- .../q49.native_iceberg_compat/extended.txt | 6 ++-- .../q49/extended.txt | 6 ++-- .../q51.native_iceberg_compat/extended.txt | 6 ++-- .../q51/extended.txt | 6 ++-- .../q53.native_iceberg_compat/extended.txt | 2 +- .../q53/extended.txt | 2 +- .../q57.native_iceberg_compat/extended.txt | 6 ++-- .../q57/extended.txt | 6 ++-- .../q63.native_iceberg_compat/extended.txt | 2 +- .../q63/extended.txt | 2 +- .../q70.native_iceberg_compat/extended.txt | 2 +- .../q70/extended.txt | 2 +- .../q78.native_iceberg_compat/extended.txt | 2 +- .../q78/extended.txt | 2 +- .../q86.native_iceberg_compat/extended.txt | 2 +- .../q86/extended.txt | 2 +- .../q89.native_iceberg_compat/extended.txt | 2 +- .../q89/extended.txt | 2 +- .../q9.native_iceberg_compat/extended.txt | 2 +- .../q9/extended.txt | 2 +- .../q94.native_iceberg_compat/extended.txt | 2 +- .../q94/extended.txt | 2 +- .../q95.native_iceberg_compat/extended.txt | 2 +- .../q95/extended.txt | 2 +- .../q98.native_iceberg_compat/extended.txt | 2 +- .../q98/extended.txt | 2 +- 42 files changed, 129 insertions(+), 74 deletions(-) diff --git a/common/src/main/scala/org/apache/comet/CometConf.scala b/common/src/main/scala/org/apache/comet/CometConf.scala index 7bd075eb50..7922e5167a 100644 --- a/common/src/main/scala/org/apache/comet/CometConf.scala +++ b/common/src/main/scala/org/apache/comet/CometConf.scala @@ -79,6 +79,8 @@ object CometConf extends ShimCometConf { val COMET_EXPR_CONFIG_PREFIX: String = s"$COMET_PREFIX.expression"; + val COMET_OPERATOR_CONFIG_PREFIX: String = s"$COMET_PREFIX.operator"; + val COMET_ENABLED: ConfigEntry[Boolean] = conf("spark.comet.enabled") .category(CATEGORY_EXEC) .doc( @@ -248,7 +250,7 @@ object CometConf extends ShimCometConf { val COMET_EXEC_EXPAND_ENABLED: ConfigEntry[Boolean] = createExecEnabledConfig("expand", defaultValue = true) val COMET_EXEC_WINDOW_ENABLED: ConfigEntry[Boolean] = - createExecEnabledConfig("window", defaultValue = false) + createExecEnabledConfig("window", defaultValue = true) val COMET_EXEC_TAKE_ORDERED_AND_PROJECT_ENABLED: ConfigEntry[Boolean] = createExecEnabledConfig("takeOrderedAndProject", defaultValue = true) @@ -745,6 +747,18 @@ object CometConf extends ShimCometConf { s"${CometConf.COMET_EXPR_CONFIG_PREFIX}.${exprClass.getSimpleName}.allowIncompatible" } + def isOperatorAllowIncompat(name: String, conf: SQLConf = SQLConf.get): Boolean = { + getBooleanConf(getOperatorAllowIncompatConfigKey(name), defaultValue = false, conf) + } + + def getOperatorAllowIncompatConfigKey(name: String): String = { + s"${CometConf.COMET_OPERATOR_CONFIG_PREFIX}.$name.allowIncompatible" + } + + def getOperatorAllowIncompatConfigKey(exprClass: Class[_]): String = { + s"${CometConf.COMET_OPERATOR_CONFIG_PREFIX}.${exprClass.getSimpleName}.allowIncompatible" + } + def getBooleanConf(name: String, defaultValue: Boolean, conf: SQLConf): Boolean = { conf.getConfString(name, defaultValue.toString).toLowerCase(Locale.ROOT) == "true" } diff --git a/spark/src/main/scala/org/apache/comet/serde/CometOperatorSerde.scala b/spark/src/main/scala/org/apache/comet/serde/CometOperatorSerde.scala index c6a95ec88a..25ea2c1233 100644 --- a/spark/src/main/scala/org/apache/comet/serde/CometOperatorSerde.scala +++ b/spark/src/main/scala/org/apache/comet/serde/CometOperatorSerde.scala @@ -29,6 +29,22 @@ import org.apache.comet.serde.OperatorOuterClass.Operator */ trait CometOperatorSerde[T <: SparkPlan] { + /** + * Get the optional Comet configuration entry that is used to enable or disable native support + * for this operator. + */ + def enabledConfig: Option[ConfigEntry[Boolean]] + + /** + * Determine the support level of the operator based on its attributes. + * + * @param operator + * The Spark operator. + * @return + * Support level (Compatible, Incompatible, or Unsupported). + */ + def getSupportLevel(operator: T): SupportLevel = Compatible(None) + /** * Convert a Spark operator into a protocol buffer representation that can be passed into native * code. @@ -49,9 +65,4 @@ trait CometOperatorSerde[T <: SparkPlan] { builder: Operator.Builder, childOp: Operator*): Option[OperatorOuterClass.Operator] - /** - * Get the optional Comet configuration entry that is used to enable or disable native support - * for this operator. - */ - def enabledConfig: Option[ConfigEntry[Boolean]] } diff --git a/spark/src/main/scala/org/apache/comet/serde/CometWindow.scala b/spark/src/main/scala/org/apache/comet/serde/CometWindow.scala index e1ea3ff388..a234c1d320 100644 --- a/spark/src/main/scala/org/apache/comet/serde/CometWindow.scala +++ b/spark/src/main/scala/org/apache/comet/serde/CometWindow.scala @@ -38,6 +38,10 @@ object CometWindow extends CometOperatorSerde[WindowExec] { override def enabledConfig: Option[ConfigEntry[Boolean]] = Some( CometConf.COMET_EXEC_WINDOW_ENABLED) + override def getSupportLevel(op: WindowExec): SupportLevel = { + Incompatible(Some("Native WindowExec has known correctness issues")) + } + override def convert( op: WindowExec, builder: Operator.Builder, diff --git a/spark/src/main/scala/org/apache/comet/serde/QueryPlanSerde.scala b/spark/src/main/scala/org/apache/comet/serde/QueryPlanSerde.scala index 410dda8ba0..c6f426db88 100644 --- a/spark/src/main/scala/org/apache/comet/serde/QueryPlanSerde.scala +++ b/spark/src/main/scala/org/apache/comet/serde/QueryPlanSerde.scala @@ -762,19 +762,45 @@ object QueryPlanSerde extends Logging with CometExprShim { serde match { case Some(handler) => val enabled = handler.enabledConfig.forall(_.get(op.conf)) + val opName = op.getClass.getSimpleName if (enabled) { + val opSerde = handler.asInstanceOf[CometOperatorSerde[SparkPlan]] + opSerde.getSupportLevel(op) match { + case Unsupported(notes) => + withInfo(op, notes.getOrElse("")) + case Incompatible(notes) => + val allowIncompat = CometConf.isOperatorAllowIncompat(opName) + if (allowIncompat) { + if (notes.isDefined) { + logWarning( + s"Comet supports $op when ${CometConf.getOperatorAllowIncompatConfigKey(opName)}=true " + + s"but has notes: ${notes.get}") + } + return opSerde.convert(op, builder, childOp: _*) + } else { + val optionalNotes = notes.map(str => s" ($str)").getOrElse("") + withInfo( + op, + s"$op is not fully compatible with Spark$optionalNotes. To enable it anyway, " + + s"set ${CometConf.getOperatorAllowIncompatConfigKey(opName)}=true. " + + s"${CometConf.COMPAT_GUIDE}.") + } + case Compatible(notes) => + if (notes.isDefined) { + logWarning(s"Comet supports $op but has notes: ${notes.get}") + } + return opSerde.convert(op, builder, childOp: _*) + } val maybeConverted = handler .asInstanceOf[CometOperatorSerde[SparkPlan]] .convert(op, builder, childOp: _*) if (maybeConverted.isDefined) { return maybeConverted } - } - // if the config was explicitly disabled, tag the operator - if (handler.enabledConfig.isDefined) { + } else { withInfo( op, - s"Native support for operator ${op.getClass.getSimpleName} is disabled. " + + s"Native support for operator $opName is disabled. " + s"Set ${handler.enabledConfig.get.key}=true to enable it.") } case _ => diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q12.native_iceberg_compat/extended.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q12.native_iceberg_compat/extended.txt index 9ddd3aeb38..831200c6ff 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q12.native_iceberg_compat/extended.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q12.native_iceberg_compat/extended.txt @@ -1,6 +1,6 @@ TakeOrderedAndProject [COMET: ] +- Project - +- Window [COMET: Window expressions are not supported] + +- Window [COMET: Native support for operator WindowExec is disabled. Set spark.comet.exec.window.enabled=true to enable it.] +- CometColumnarToRow +- CometSort +- CometExchange diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q12/extended.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q12/extended.txt index 9ddd3aeb38..831200c6ff 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q12/extended.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q12/extended.txt @@ -1,6 +1,6 @@ TakeOrderedAndProject [COMET: ] +- Project - +- Window [COMET: Window expressions are not supported] + +- Window [COMET: Native support for operator WindowExec is disabled. Set spark.comet.exec.window.enabled=true to enable it.] +- CometColumnarToRow +- CometSort +- CometExchange diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q16.native_iceberg_compat/extended.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q16.native_iceberg_compat/extended.txt index 2ad029e444..a8f535db5a 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q16.native_iceberg_compat/extended.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q16.native_iceberg_compat/extended.txt @@ -2,7 +2,7 @@ CometColumnarToRow +- CometHashAggregate +- CometColumnarExchange +- HashAggregate - +- HashAggregate [COMET: Unsupported aggregation mode PartialMerge] + +- HashAggregate [COMET: Unsupported aggregation mode PartialMerge, Native support for operator HashAggregateExec is disabled. Set spark.comet.exec.aggregate.enabled=true to enable it.] +- CometColumnarToRow +- CometHashAggregate +- CometProject diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q16/extended.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q16/extended.txt index 2ad029e444..a8f535db5a 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q16/extended.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q16/extended.txt @@ -2,7 +2,7 @@ CometColumnarToRow +- CometHashAggregate +- CometColumnarExchange +- HashAggregate - +- HashAggregate [COMET: Unsupported aggregation mode PartialMerge] + +- HashAggregate [COMET: Unsupported aggregation mode PartialMerge, Native support for operator HashAggregateExec is disabled. Set spark.comet.exec.aggregate.enabled=true to enable it.] +- CometColumnarToRow +- CometHashAggregate +- CometProject diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q20.native_iceberg_compat/extended.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q20.native_iceberg_compat/extended.txt index 400633ed9f..f5da783a44 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q20.native_iceberg_compat/extended.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q20.native_iceberg_compat/extended.txt @@ -1,6 +1,6 @@ TakeOrderedAndProject [COMET: ] +- Project - +- Window [COMET: Window expressions are not supported] + +- Window [COMET: Native support for operator WindowExec is disabled. Set spark.comet.exec.window.enabled=true to enable it.] +- CometColumnarToRow +- CometSort +- CometExchange diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q20/extended.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q20/extended.txt index 400633ed9f..f5da783a44 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q20/extended.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q20/extended.txt @@ -1,6 +1,6 @@ TakeOrderedAndProject [COMET: ] +- Project - +- Window [COMET: Window expressions are not supported] + +- Window [COMET: Native support for operator WindowExec is disabled. Set spark.comet.exec.window.enabled=true to enable it.] +- CometColumnarToRow +- CometSort +- CometExchange diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q28.native_iceberg_compat/extended.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q28.native_iceberg_compat/extended.txt index 841b4c9822..4b94cbb91f 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q28.native_iceberg_compat/extended.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q28.native_iceberg_compat/extended.txt @@ -7,7 +7,7 @@ BroadcastNestedLoopJoin : : : : : +- CometHashAggregate : : : : : +- CometColumnarExchange : : : : : +- HashAggregate -: : : : : +- HashAggregate [COMET: Unsupported aggregation mode PartialMerge] +: : : : : +- HashAggregate [COMET: Unsupported aggregation mode PartialMerge, Native support for operator HashAggregateExec is disabled. Set spark.comet.exec.aggregate.enabled=true to enable it.] : : : : : +- CometColumnarToRow : : : : : +- CometExchange : : : : : +- CometHashAggregate @@ -19,7 +19,7 @@ BroadcastNestedLoopJoin : : : : +- CometHashAggregate : : : : +- CometColumnarExchange : : : : +- HashAggregate -: : : : +- HashAggregate [COMET: Unsupported aggregation mode PartialMerge] +: : : : +- HashAggregate [COMET: Unsupported aggregation mode PartialMerge, Native support for operator HashAggregateExec is disabled. Set spark.comet.exec.aggregate.enabled=true to enable it.] : : : : +- CometColumnarToRow : : : : +- CometExchange : : : : +- CometHashAggregate @@ -31,7 +31,7 @@ BroadcastNestedLoopJoin : : : +- CometHashAggregate : : : +- CometColumnarExchange : : : +- HashAggregate -: : : +- HashAggregate [COMET: Unsupported aggregation mode PartialMerge] +: : : +- HashAggregate [COMET: Unsupported aggregation mode PartialMerge, Native support for operator HashAggregateExec is disabled. Set spark.comet.exec.aggregate.enabled=true to enable it.] : : : +- CometColumnarToRow : : : +- CometExchange : : : +- CometHashAggregate @@ -43,7 +43,7 @@ BroadcastNestedLoopJoin : : +- CometHashAggregate : : +- CometColumnarExchange : : +- HashAggregate -: : +- HashAggregate [COMET: Unsupported aggregation mode PartialMerge] +: : +- HashAggregate [COMET: Unsupported aggregation mode PartialMerge, Native support for operator HashAggregateExec is disabled. Set spark.comet.exec.aggregate.enabled=true to enable it.] : : +- CometColumnarToRow : : +- CometExchange : : +- CometHashAggregate @@ -55,7 +55,7 @@ BroadcastNestedLoopJoin : +- CometHashAggregate : +- CometColumnarExchange : +- HashAggregate -: +- HashAggregate [COMET: Unsupported aggregation mode PartialMerge] +: +- HashAggregate [COMET: Unsupported aggregation mode PartialMerge, Native support for operator HashAggregateExec is disabled. Set spark.comet.exec.aggregate.enabled=true to enable it.] : +- CometColumnarToRow : +- CometExchange : +- CometHashAggregate @@ -67,7 +67,7 @@ BroadcastNestedLoopJoin +- CometHashAggregate +- CometColumnarExchange +- HashAggregate - +- HashAggregate [COMET: Unsupported aggregation mode PartialMerge] + +- HashAggregate [COMET: Unsupported aggregation mode PartialMerge, Native support for operator HashAggregateExec is disabled. Set spark.comet.exec.aggregate.enabled=true to enable it.] +- CometColumnarToRow +- CometExchange +- CometHashAggregate diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q28/extended.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q28/extended.txt index 841b4c9822..4b94cbb91f 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q28/extended.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q28/extended.txt @@ -7,7 +7,7 @@ BroadcastNestedLoopJoin : : : : : +- CometHashAggregate : : : : : +- CometColumnarExchange : : : : : +- HashAggregate -: : : : : +- HashAggregate [COMET: Unsupported aggregation mode PartialMerge] +: : : : : +- HashAggregate [COMET: Unsupported aggregation mode PartialMerge, Native support for operator HashAggregateExec is disabled. Set spark.comet.exec.aggregate.enabled=true to enable it.] : : : : : +- CometColumnarToRow : : : : : +- CometExchange : : : : : +- CometHashAggregate @@ -19,7 +19,7 @@ BroadcastNestedLoopJoin : : : : +- CometHashAggregate : : : : +- CometColumnarExchange : : : : +- HashAggregate -: : : : +- HashAggregate [COMET: Unsupported aggregation mode PartialMerge] +: : : : +- HashAggregate [COMET: Unsupported aggregation mode PartialMerge, Native support for operator HashAggregateExec is disabled. Set spark.comet.exec.aggregate.enabled=true to enable it.] : : : : +- CometColumnarToRow : : : : +- CometExchange : : : : +- CometHashAggregate @@ -31,7 +31,7 @@ BroadcastNestedLoopJoin : : : +- CometHashAggregate : : : +- CometColumnarExchange : : : +- HashAggregate -: : : +- HashAggregate [COMET: Unsupported aggregation mode PartialMerge] +: : : +- HashAggregate [COMET: Unsupported aggregation mode PartialMerge, Native support for operator HashAggregateExec is disabled. Set spark.comet.exec.aggregate.enabled=true to enable it.] : : : +- CometColumnarToRow : : : +- CometExchange : : : +- CometHashAggregate @@ -43,7 +43,7 @@ BroadcastNestedLoopJoin : : +- CometHashAggregate : : +- CometColumnarExchange : : +- HashAggregate -: : +- HashAggregate [COMET: Unsupported aggregation mode PartialMerge] +: : +- HashAggregate [COMET: Unsupported aggregation mode PartialMerge, Native support for operator HashAggregateExec is disabled. Set spark.comet.exec.aggregate.enabled=true to enable it.] : : +- CometColumnarToRow : : +- CometExchange : : +- CometHashAggregate @@ -55,7 +55,7 @@ BroadcastNestedLoopJoin : +- CometHashAggregate : +- CometColumnarExchange : +- HashAggregate -: +- HashAggregate [COMET: Unsupported aggregation mode PartialMerge] +: +- HashAggregate [COMET: Unsupported aggregation mode PartialMerge, Native support for operator HashAggregateExec is disabled. Set spark.comet.exec.aggregate.enabled=true to enable it.] : +- CometColumnarToRow : +- CometExchange : +- CometHashAggregate @@ -67,7 +67,7 @@ BroadcastNestedLoopJoin +- CometHashAggregate +- CometColumnarExchange +- HashAggregate - +- HashAggregate [COMET: Unsupported aggregation mode PartialMerge] + +- HashAggregate [COMET: Unsupported aggregation mode PartialMerge, Native support for operator HashAggregateExec is disabled. Set spark.comet.exec.aggregate.enabled=true to enable it.] +- CometColumnarToRow +- CometExchange +- CometHashAggregate diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q36.native_iceberg_compat/extended.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q36.native_iceberg_compat/extended.txt index 31fe2cc028..4ddf55dee5 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q36.native_iceberg_compat/extended.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q36.native_iceberg_compat/extended.txt @@ -1,6 +1,6 @@ TakeOrderedAndProject [COMET: ] +- Project - +- Window [COMET: Window expressions are not supported] + +- Window [COMET: Native support for operator WindowExec is disabled. Set spark.comet.exec.window.enabled=true to enable it.] +- CometColumnarToRow +- CometSort +- CometExchange diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q36/extended.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q36/extended.txt index 31fe2cc028..4ddf55dee5 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q36/extended.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q36/extended.txt @@ -1,6 +1,6 @@ TakeOrderedAndProject [COMET: ] +- Project - +- Window [COMET: Window expressions are not supported] + +- Window [COMET: Native support for operator WindowExec is disabled. Set spark.comet.exec.window.enabled=true to enable it.] +- CometColumnarToRow +- CometSort +- CometExchange diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q47.native_iceberg_compat/extended.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q47.native_iceberg_compat/extended.txt index 11fbf5bd11..56c60c0d74 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q47.native_iceberg_compat/extended.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q47.native_iceberg_compat/extended.txt @@ -7,7 +7,7 @@ : : +- Filter : : +- Window : : +- Filter - : : +- Window [COMET: Window expressions are not supported] + : : +- Window [COMET: Native support for operator WindowExec is disabled. Set spark.comet.exec.window.enabled=true to enable it.] : : +- CometColumnarToRow : : +- CometSort : : +- CometExchange @@ -39,7 +39,7 @@ : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.store : +- BroadcastExchange : +- Project - : +- Window [COMET: Window expressions are not supported] + : +- Window [COMET: Native support for operator WindowExec is disabled. Set spark.comet.exec.window.enabled=true to enable it.] : +- CometColumnarToRow : +- CometSort : +- CometExchange @@ -71,7 +71,7 @@ : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.store +- BroadcastExchange +- Project - +- Window [COMET: Window expressions are not supported] + +- Window [COMET: Native support for operator WindowExec is disabled. Set spark.comet.exec.window.enabled=true to enable it.] +- CometColumnarToRow +- CometSort +- CometExchange diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q47/extended.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q47/extended.txt index 11fbf5bd11..56c60c0d74 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q47/extended.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q47/extended.txt @@ -7,7 +7,7 @@ : : +- Filter : : +- Window : : +- Filter - : : +- Window [COMET: Window expressions are not supported] + : : +- Window [COMET: Native support for operator WindowExec is disabled. Set spark.comet.exec.window.enabled=true to enable it.] : : +- CometColumnarToRow : : +- CometSort : : +- CometExchange @@ -39,7 +39,7 @@ : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.store : +- BroadcastExchange : +- Project - : +- Window [COMET: Window expressions are not supported] + : +- Window [COMET: Native support for operator WindowExec is disabled. Set spark.comet.exec.window.enabled=true to enable it.] : +- CometColumnarToRow : +- CometSort : +- CometExchange @@ -71,7 +71,7 @@ : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.store +- BroadcastExchange +- Project - +- Window [COMET: Window expressions are not supported] + +- Window [COMET: Native support for operator WindowExec is disabled. Set spark.comet.exec.window.enabled=true to enable it.] +- CometColumnarToRow +- CometSort +- CometExchange diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q49.native_iceberg_compat/extended.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q49.native_iceberg_compat/extended.txt index 734b0ce7bb..e6af7fd5e0 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q49.native_iceberg_compat/extended.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q49.native_iceberg_compat/extended.txt @@ -8,7 +8,7 @@ CometColumnarToRow : +- Filter : +- Window : +- Sort - : +- Window [COMET: Window expressions are not supported] + : +- Window [COMET: Native support for operator WindowExec is disabled. Set spark.comet.exec.window.enabled=true to enable it.] : +- CometColumnarToRow : +- CometSort : +- CometExchange @@ -40,7 +40,7 @@ CometColumnarToRow : +- Filter : +- Window : +- Sort - : +- Window [COMET: Window expressions are not supported] + : +- Window [COMET: Native support for operator WindowExec is disabled. Set spark.comet.exec.window.enabled=true to enable it.] : +- CometColumnarToRow : +- CometSort : +- CometExchange @@ -67,7 +67,7 @@ CometColumnarToRow +- Filter +- Window +- Sort - +- Window [COMET: Window expressions are not supported] + +- Window [COMET: Native support for operator WindowExec is disabled. Set spark.comet.exec.window.enabled=true to enable it.] +- CometColumnarToRow +- CometSort +- CometExchange diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q49/extended.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q49/extended.txt index 734b0ce7bb..e6af7fd5e0 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q49/extended.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q49/extended.txt @@ -8,7 +8,7 @@ CometColumnarToRow : +- Filter : +- Window : +- Sort - : +- Window [COMET: Window expressions are not supported] + : +- Window [COMET: Native support for operator WindowExec is disabled. Set spark.comet.exec.window.enabled=true to enable it.] : +- CometColumnarToRow : +- CometSort : +- CometExchange @@ -40,7 +40,7 @@ CometColumnarToRow : +- Filter : +- Window : +- Sort - : +- Window [COMET: Window expressions are not supported] + : +- Window [COMET: Native support for operator WindowExec is disabled. Set spark.comet.exec.window.enabled=true to enable it.] : +- CometColumnarToRow : +- CometSort : +- CometExchange @@ -67,7 +67,7 @@ CometColumnarToRow +- Filter +- Window +- Sort - +- Window [COMET: Window expressions are not supported] + +- Window [COMET: Native support for operator WindowExec is disabled. Set spark.comet.exec.window.enabled=true to enable it.] +- CometColumnarToRow +- CometSort +- CometExchange diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q51.native_iceberg_compat/extended.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q51.native_iceberg_compat/extended.txt index da1ac9c0a3..5766386853 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q51.native_iceberg_compat/extended.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q51.native_iceberg_compat/extended.txt @@ -1,6 +1,6 @@ TakeOrderedAndProject [COMET: ] +- Filter - +- Window [COMET: Window expressions are not supported] + +- Window [COMET: Native support for operator WindowExec is disabled. Set spark.comet.exec.window.enabled=true to enable it.] +- CometColumnarToRow +- CometSort +- CometExchange @@ -9,7 +9,7 @@ :- CometSort : +- CometColumnarExchange : +- Project - : +- Window [COMET: Window expressions are not supported] + : +- Window [COMET: Native support for operator WindowExec is disabled. Set spark.comet.exec.window.enabled=true to enable it.] : +- CometColumnarToRow : +- CometSort : +- CometExchange @@ -33,7 +33,7 @@ +- CometSort +- CometColumnarExchange +- Project - +- Window [COMET: Window expressions are not supported] + +- Window [COMET: Native support for operator WindowExec is disabled. Set spark.comet.exec.window.enabled=true to enable it.] +- CometColumnarToRow +- CometSort +- CometExchange diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q51/extended.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q51/extended.txt index da1ac9c0a3..5766386853 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q51/extended.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q51/extended.txt @@ -1,6 +1,6 @@ TakeOrderedAndProject [COMET: ] +- Filter - +- Window [COMET: Window expressions are not supported] + +- Window [COMET: Native support for operator WindowExec is disabled. Set spark.comet.exec.window.enabled=true to enable it.] +- CometColumnarToRow +- CometSort +- CometExchange @@ -9,7 +9,7 @@ :- CometSort : +- CometColumnarExchange : +- Project - : +- Window [COMET: Window expressions are not supported] + : +- Window [COMET: Native support for operator WindowExec is disabled. Set spark.comet.exec.window.enabled=true to enable it.] : +- CometColumnarToRow : +- CometSort : +- CometExchange @@ -33,7 +33,7 @@ +- CometSort +- CometColumnarExchange +- Project - +- Window [COMET: Window expressions are not supported] + +- Window [COMET: Native support for operator WindowExec is disabled. Set spark.comet.exec.window.enabled=true to enable it.] +- CometColumnarToRow +- CometSort +- CometExchange diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q53.native_iceberg_compat/extended.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q53.native_iceberg_compat/extended.txt index 004828cdfb..c3eff7fb0f 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q53.native_iceberg_compat/extended.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q53.native_iceberg_compat/extended.txt @@ -1,7 +1,7 @@ TakeOrderedAndProject [COMET: ] +- Project +- Filter - +- Window [COMET: Window expressions are not supported] + +- Window [COMET: Native support for operator WindowExec is disabled. Set spark.comet.exec.window.enabled=true to enable it.] +- CometColumnarToRow +- CometSort +- CometExchange diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q53/extended.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q53/extended.txt index 004828cdfb..c3eff7fb0f 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q53/extended.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q53/extended.txt @@ -1,7 +1,7 @@ TakeOrderedAndProject [COMET: ] +- Project +- Filter - +- Window [COMET: Window expressions are not supported] + +- Window [COMET: Native support for operator WindowExec is disabled. Set spark.comet.exec.window.enabled=true to enable it.] +- CometColumnarToRow +- CometSort +- CometExchange diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q57.native_iceberg_compat/extended.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q57.native_iceberg_compat/extended.txt index 7bee0e61e8..5e24da49b7 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q57.native_iceberg_compat/extended.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q57.native_iceberg_compat/extended.txt @@ -7,7 +7,7 @@ : : +- Filter : : +- Window : : +- Filter - : : +- Window [COMET: Window expressions are not supported] + : : +- Window [COMET: Native support for operator WindowExec is disabled. Set spark.comet.exec.window.enabled=true to enable it.] : : +- CometColumnarToRow : : +- CometSort : : +- CometExchange @@ -39,7 +39,7 @@ : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.call_center : +- BroadcastExchange : +- Project - : +- Window [COMET: Window expressions are not supported] + : +- Window [COMET: Native support for operator WindowExec is disabled. Set spark.comet.exec.window.enabled=true to enable it.] : +- CometColumnarToRow : +- CometSort : +- CometExchange @@ -71,7 +71,7 @@ : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.call_center +- BroadcastExchange +- Project - +- Window [COMET: Window expressions are not supported] + +- Window [COMET: Native support for operator WindowExec is disabled. Set spark.comet.exec.window.enabled=true to enable it.] +- CometColumnarToRow +- CometSort +- CometExchange diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q57/extended.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q57/extended.txt index 7bee0e61e8..5e24da49b7 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q57/extended.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q57/extended.txt @@ -7,7 +7,7 @@ : : +- Filter : : +- Window : : +- Filter - : : +- Window [COMET: Window expressions are not supported] + : : +- Window [COMET: Native support for operator WindowExec is disabled. Set spark.comet.exec.window.enabled=true to enable it.] : : +- CometColumnarToRow : : +- CometSort : : +- CometExchange @@ -39,7 +39,7 @@ : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.call_center : +- BroadcastExchange : +- Project - : +- Window [COMET: Window expressions are not supported] + : +- Window [COMET: Native support for operator WindowExec is disabled. Set spark.comet.exec.window.enabled=true to enable it.] : +- CometColumnarToRow : +- CometSort : +- CometExchange @@ -71,7 +71,7 @@ : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.call_center +- BroadcastExchange +- Project - +- Window [COMET: Window expressions are not supported] + +- Window [COMET: Native support for operator WindowExec is disabled. Set spark.comet.exec.window.enabled=true to enable it.] +- CometColumnarToRow +- CometSort +- CometExchange diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q63.native_iceberg_compat/extended.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q63.native_iceberg_compat/extended.txt index 004828cdfb..c3eff7fb0f 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q63.native_iceberg_compat/extended.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q63.native_iceberg_compat/extended.txt @@ -1,7 +1,7 @@ TakeOrderedAndProject [COMET: ] +- Project +- Filter - +- Window [COMET: Window expressions are not supported] + +- Window [COMET: Native support for operator WindowExec is disabled. Set spark.comet.exec.window.enabled=true to enable it.] +- CometColumnarToRow +- CometSort +- CometExchange diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q63/extended.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q63/extended.txt index 004828cdfb..c3eff7fb0f 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q63/extended.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q63/extended.txt @@ -1,7 +1,7 @@ TakeOrderedAndProject [COMET: ] +- Project +- Filter - +- Window [COMET: Window expressions are not supported] + +- Window [COMET: Native support for operator WindowExec is disabled. Set spark.comet.exec.window.enabled=true to enable it.] +- CometColumnarToRow +- CometSort +- CometExchange diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q70.native_iceberg_compat/extended.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q70.native_iceberg_compat/extended.txt index 010caafdc0..97fefb4f00 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q70.native_iceberg_compat/extended.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q70.native_iceberg_compat/extended.txt @@ -1,6 +1,6 @@ TakeOrderedAndProject [COMET: ] +- Project - +- Window [COMET: Window expressions are not supported] + +- Window [COMET: Native support for operator WindowExec is disabled. Set spark.comet.exec.window.enabled=true to enable it.] +- CometColumnarToRow +- CometSort +- CometColumnarExchange diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q70/extended.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q70/extended.txt index 010caafdc0..97fefb4f00 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q70/extended.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q70/extended.txt @@ -1,6 +1,6 @@ TakeOrderedAndProject [COMET: ] +- Project - +- Window [COMET: Window expressions are not supported] + +- Window [COMET: Native support for operator WindowExec is disabled. Set spark.comet.exec.window.enabled=true to enable it.] +- CometColumnarToRow +- CometSort +- CometColumnarExchange diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q78.native_iceberg_compat/extended.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q78.native_iceberg_compat/extended.txt index 97a1bdbbaf..8053f01bc6 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q78.native_iceberg_compat/extended.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q78.native_iceberg_compat/extended.txt @@ -1,5 +1,5 @@ TakeOrderedAndProject [COMET: ] -+- Project [COMET: Comet does not support Spark's BigDecimal rounding] ++- Project [COMET: Comet does not support Spark's BigDecimal rounding, Native support for operator ProjectExec is disabled. Set spark.comet.exec.project.enabled=true to enable it.] +- CometColumnarToRow +- CometSortMergeJoin :- CometProject diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q78/extended.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q78/extended.txt index 97a1bdbbaf..8053f01bc6 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q78/extended.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q78/extended.txt @@ -1,5 +1,5 @@ TakeOrderedAndProject [COMET: ] -+- Project [COMET: Comet does not support Spark's BigDecimal rounding] ++- Project [COMET: Comet does not support Spark's BigDecimal rounding, Native support for operator ProjectExec is disabled. Set spark.comet.exec.project.enabled=true to enable it.] +- CometColumnarToRow +- CometSortMergeJoin :- CometProject diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q86.native_iceberg_compat/extended.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q86.native_iceberg_compat/extended.txt index 0b05b63e05..ef6698a475 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q86.native_iceberg_compat/extended.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q86.native_iceberg_compat/extended.txt @@ -1,6 +1,6 @@ TakeOrderedAndProject [COMET: ] +- Project - +- Window [COMET: Window expressions are not supported] + +- Window [COMET: Native support for operator WindowExec is disabled. Set spark.comet.exec.window.enabled=true to enable it.] +- CometColumnarToRow +- CometSort +- CometExchange diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q86/extended.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q86/extended.txt index 0b05b63e05..ef6698a475 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q86/extended.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q86/extended.txt @@ -1,6 +1,6 @@ TakeOrderedAndProject [COMET: ] +- Project - +- Window [COMET: Window expressions are not supported] + +- Window [COMET: Native support for operator WindowExec is disabled. Set spark.comet.exec.window.enabled=true to enable it.] +- CometColumnarToRow +- CometSort +- CometExchange diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q89.native_iceberg_compat/extended.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q89.native_iceberg_compat/extended.txt index 004828cdfb..c3eff7fb0f 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q89.native_iceberg_compat/extended.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q89.native_iceberg_compat/extended.txt @@ -1,7 +1,7 @@ TakeOrderedAndProject [COMET: ] +- Project +- Filter - +- Window [COMET: Window expressions are not supported] + +- Window [COMET: Native support for operator WindowExec is disabled. Set spark.comet.exec.window.enabled=true to enable it.] +- CometColumnarToRow +- CometSort +- CometExchange diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q89/extended.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q89/extended.txt index 004828cdfb..c3eff7fb0f 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q89/extended.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q89/extended.txt @@ -1,7 +1,7 @@ TakeOrderedAndProject [COMET: ] +- Project +- Filter - +- Window [COMET: Window expressions are not supported] + +- Window [COMET: Native support for operator WindowExec is disabled. Set spark.comet.exec.window.enabled=true to enable it.] +- CometColumnarToRow +- CometSort +- CometExchange diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q9.native_iceberg_compat/extended.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q9.native_iceberg_compat/extended.txt index ed71033b26..148b9d4491 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q9.native_iceberg_compat/extended.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q9.native_iceberg_compat/extended.txt @@ -1,4 +1,4 @@ - Project [COMET: ] + Project [COMET: Native support for operator ProjectExec is disabled. Set spark.comet.exec.project.enabled=true to enable it.] : :- Subquery : : +- CometColumnarToRow : : +- CometProject diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q9/extended.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q9/extended.txt index ed71033b26..148b9d4491 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q9/extended.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q9/extended.txt @@ -1,4 +1,4 @@ - Project [COMET: ] + Project [COMET: Native support for operator ProjectExec is disabled. Set spark.comet.exec.project.enabled=true to enable it.] : :- Subquery : : +- CometColumnarToRow : : +- CometProject diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q94.native_iceberg_compat/extended.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q94.native_iceberg_compat/extended.txt index eac4939621..e42c9a2062 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q94.native_iceberg_compat/extended.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q94.native_iceberg_compat/extended.txt @@ -2,7 +2,7 @@ CometColumnarToRow +- CometHashAggregate +- CometColumnarExchange +- HashAggregate - +- HashAggregate [COMET: Unsupported aggregation mode PartialMerge] + +- HashAggregate [COMET: Unsupported aggregation mode PartialMerge, Native support for operator HashAggregateExec is disabled. Set spark.comet.exec.aggregate.enabled=true to enable it.] +- CometColumnarToRow +- CometHashAggregate +- CometProject diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q94/extended.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q94/extended.txt index eac4939621..e42c9a2062 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q94/extended.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q94/extended.txt @@ -2,7 +2,7 @@ CometColumnarToRow +- CometHashAggregate +- CometColumnarExchange +- HashAggregate - +- HashAggregate [COMET: Unsupported aggregation mode PartialMerge] + +- HashAggregate [COMET: Unsupported aggregation mode PartialMerge, Native support for operator HashAggregateExec is disabled. Set spark.comet.exec.aggregate.enabled=true to enable it.] +- CometColumnarToRow +- CometHashAggregate +- CometProject diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q95.native_iceberg_compat/extended.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q95.native_iceberg_compat/extended.txt index 6ff8eba58f..5a1b14b632 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q95.native_iceberg_compat/extended.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q95.native_iceberg_compat/extended.txt @@ -2,7 +2,7 @@ CometColumnarToRow +- CometHashAggregate +- CometColumnarExchange +- HashAggregate - +- HashAggregate [COMET: Unsupported aggregation mode PartialMerge] + +- HashAggregate [COMET: Unsupported aggregation mode PartialMerge, Native support for operator HashAggregateExec is disabled. Set spark.comet.exec.aggregate.enabled=true to enable it.] +- CometColumnarToRow +- CometHashAggregate +- CometProject diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q95/extended.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q95/extended.txt index 6ff8eba58f..5a1b14b632 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q95/extended.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q95/extended.txt @@ -2,7 +2,7 @@ CometColumnarToRow +- CometHashAggregate +- CometColumnarExchange +- HashAggregate - +- HashAggregate [COMET: Unsupported aggregation mode PartialMerge] + +- HashAggregate [COMET: Unsupported aggregation mode PartialMerge, Native support for operator HashAggregateExec is disabled. Set spark.comet.exec.aggregate.enabled=true to enable it.] +- CometColumnarToRow +- CometHashAggregate +- CometProject diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q98.native_iceberg_compat/extended.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q98.native_iceberg_compat/extended.txt index c34e3cfede..001fedfc83 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q98.native_iceberg_compat/extended.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q98.native_iceberg_compat/extended.txt @@ -3,7 +3,7 @@ CometColumnarToRow +- CometSort +- CometColumnarExchange +- Project - +- Window [COMET: Window expressions are not supported] + +- Window [COMET: Native support for operator WindowExec is disabled. Set spark.comet.exec.window.enabled=true to enable it.] +- CometColumnarToRow +- CometSort +- CometExchange diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q98/extended.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q98/extended.txt index c34e3cfede..001fedfc83 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q98/extended.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q98/extended.txt @@ -3,7 +3,7 @@ CometColumnarToRow +- CometSort +- CometColumnarExchange +- Project - +- Window [COMET: Window expressions are not supported] + +- Window [COMET: Native support for operator WindowExec is disabled. Set spark.comet.exec.window.enabled=true to enable it.] +- CometColumnarToRow +- CometSort +- CometExchange From 865274f8b2c3466e83fe668970823af941d6fbc4 Mon Sep 17 00:00:00 2001 From: Andy Grove Date: Sat, 8 Nov 2025 13:04:28 -0700 Subject: [PATCH 31/44] update 3.5 golden files --- docs/source/user-guide/latest/configs.md | 2 +- .../apache/comet/serde/QueryPlanSerde.scala | 9 +++++---- .../q12.native_iceberg_compat/extended.txt | 2 +- .../q12/extended.txt | 2 +- .../q16.native_iceberg_compat/extended.txt | 2 +- .../q16/extended.txt | 2 +- .../q20.native_iceberg_compat/extended.txt | 2 +- .../q20/extended.txt | 2 +- .../q28.native_iceberg_compat/extended.txt | 12 +++++------ .../q28/extended.txt | 12 +++++------ .../q36.native_iceberg_compat/extended.txt | 2 +- .../q36/extended.txt | 2 +- .../q47.native_iceberg_compat/extended.txt | 6 +++--- .../q47/extended.txt | 6 +++--- .../q49.native_iceberg_compat/extended.txt | 6 +++--- .../q49/extended.txt | 6 +++--- .../q51.native_iceberg_compat/extended.txt | 6 +++--- .../q51/extended.txt | 6 +++--- .../q53.native_iceberg_compat/extended.txt | 2 +- .../q53/extended.txt | 2 +- .../q57.native_iceberg_compat/extended.txt | 6 +++--- .../q57/extended.txt | 6 +++--- .../q63.native_iceberg_compat/extended.txt | 2 +- .../q63/extended.txt | 2 +- .../q70.native_iceberg_compat/extended.txt | 2 +- .../q70/extended.txt | 2 +- .../q78.native_iceberg_compat/extended.txt | 2 +- .../q78/extended.txt | 2 +- .../q86.native_iceberg_compat/extended.txt | 2 +- .../q86/extended.txt | 2 +- .../q89.native_iceberg_compat/extended.txt | 2 +- .../q89/extended.txt | 2 +- .../q9.native_iceberg_compat/extended.txt | 2 +- .../q9/extended.txt | 2 +- .../q94.native_iceberg_compat/extended.txt | 2 +- .../q94/extended.txt | 2 +- .../q95.native_iceberg_compat/extended.txt | 2 +- .../q95/extended.txt | 2 +- .../q98.native_iceberg_compat/extended.txt | 2 +- .../q98/extended.txt | 2 +- .../q12.native_iceberg_compat/extended.txt | 2 +- .../q12/extended.txt | 2 +- .../q20.native_iceberg_compat/extended.txt | 2 +- .../q20/extended.txt | 2 +- .../q36a.native_iceberg_compat/extended.txt | 2 +- .../q36a/extended.txt | 2 +- .../q47.native_iceberg_compat/extended.txt | 6 +++--- .../q47/extended.txt | 6 +++--- .../q49.native_iceberg_compat/extended.txt | 6 +++--- .../q49/extended.txt | 6 +++--- .../q51a.native_iceberg_compat/extended.txt | 20 +++++++++---------- .../q51a/extended.txt | 20 +++++++++---------- .../q57.native_iceberg_compat/extended.txt | 6 +++--- .../q57/extended.txt | 6 +++--- .../q70a.native_iceberg_compat/extended.txt | 2 +- .../q70a/extended.txt | 2 +- .../q86a.native_iceberg_compat/extended.txt | 2 +- .../q86a/extended.txt | 2 +- .../q98.native_iceberg_compat/extended.txt | 2 +- .../q98/extended.txt | 2 +- 60 files changed, 120 insertions(+), 119 deletions(-) diff --git a/docs/source/user-guide/latest/configs.md b/docs/source/user-guide/latest/configs.md index bf6388549c..1cc05dfc78 100644 --- a/docs/source/user-guide/latest/configs.md +++ b/docs/source/user-guide/latest/configs.md @@ -155,7 +155,7 @@ These settings can be used to determine which parts of the plan are accelerated | `spark.comet.exec.sortMergeJoinWithJoinFilter.enabled` | Experimental support for Sort Merge Join with filter | false | | `spark.comet.exec.takeOrderedAndProject.enabled` | Whether to enable takeOrderedAndProject by default. | true | | `spark.comet.exec.union.enabled` | Whether to enable union by default. | true | -| `spark.comet.exec.window.enabled` | Whether to enable window by default. | false | +| `spark.comet.exec.window.enabled` | Whether to enable window by default. | true | ## Enabling or Disabling Individual Scalar Expressions diff --git a/spark/src/main/scala/org/apache/comet/serde/QueryPlanSerde.scala b/spark/src/main/scala/org/apache/comet/serde/QueryPlanSerde.scala index c6f426db88..101e30e97d 100644 --- a/spark/src/main/scala/org/apache/comet/serde/QueryPlanSerde.scala +++ b/spark/src/main/scala/org/apache/comet/serde/QueryPlanSerde.scala @@ -770,10 +770,11 @@ object QueryPlanSerde extends Logging with CometExprShim { withInfo(op, notes.getOrElse("")) case Incompatible(notes) => val allowIncompat = CometConf.isOperatorAllowIncompat(opName) + val incompatConf = CometConf.getOperatorAllowIncompatConfigKey(opName) if (allowIncompat) { if (notes.isDefined) { logWarning( - s"Comet supports $op when ${CometConf.getOperatorAllowIncompatConfigKey(opName)}=true " + + s"Comet supports $opName when $incompatConf=true " + s"but has notes: ${notes.get}") } return opSerde.convert(op, builder, childOp: _*) @@ -781,13 +782,13 @@ object QueryPlanSerde extends Logging with CometExprShim { val optionalNotes = notes.map(str => s" ($str)").getOrElse("") withInfo( op, - s"$op is not fully compatible with Spark$optionalNotes. To enable it anyway, " + - s"set ${CometConf.getOperatorAllowIncompatConfigKey(opName)}=true. " + + s"$opName is not fully compatible with Spark$optionalNotes. " + + s"To enable it anyway, set $incompatConf=true. " + s"${CometConf.COMPAT_GUIDE}.") } case Compatible(notes) => if (notes.isDefined) { - logWarning(s"Comet supports $op but has notes: ${notes.get}") + logWarning(s"Comet supports $opName but has notes: ${notes.get}") } return opSerde.convert(op, builder, childOp: _*) } diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q12.native_iceberg_compat/extended.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q12.native_iceberg_compat/extended.txt index 831200c6ff..9ea26caab6 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q12.native_iceberg_compat/extended.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q12.native_iceberg_compat/extended.txt @@ -1,6 +1,6 @@ TakeOrderedAndProject [COMET: ] +- Project - +- Window [COMET: Native support for operator WindowExec is disabled. Set spark.comet.exec.window.enabled=true to enable it.] + +- Window [COMET: WindowExec is not fully compatible with Spark (Native WindowExec has known correctness issues). To enable it anyway, set spark.comet.operator.WindowExec.allowIncompatible=true. For more information, refer to the Comet Compatibility Guide (https://datafusion.apache.org/comet/user-guide/compatibility.html).] +- CometColumnarToRow +- CometSort +- CometExchange diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q12/extended.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q12/extended.txt index 831200c6ff..9ea26caab6 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q12/extended.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q12/extended.txt @@ -1,6 +1,6 @@ TakeOrderedAndProject [COMET: ] +- Project - +- Window [COMET: Native support for operator WindowExec is disabled. Set spark.comet.exec.window.enabled=true to enable it.] + +- Window [COMET: WindowExec is not fully compatible with Spark (Native WindowExec has known correctness issues). To enable it anyway, set spark.comet.operator.WindowExec.allowIncompatible=true. For more information, refer to the Comet Compatibility Guide (https://datafusion.apache.org/comet/user-guide/compatibility.html).] +- CometColumnarToRow +- CometSort +- CometExchange diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q16.native_iceberg_compat/extended.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q16.native_iceberg_compat/extended.txt index a8f535db5a..2ad029e444 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q16.native_iceberg_compat/extended.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q16.native_iceberg_compat/extended.txt @@ -2,7 +2,7 @@ CometColumnarToRow +- CometHashAggregate +- CometColumnarExchange +- HashAggregate - +- HashAggregate [COMET: Unsupported aggregation mode PartialMerge, Native support for operator HashAggregateExec is disabled. Set spark.comet.exec.aggregate.enabled=true to enable it.] + +- HashAggregate [COMET: Unsupported aggregation mode PartialMerge] +- CometColumnarToRow +- CometHashAggregate +- CometProject diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q16/extended.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q16/extended.txt index a8f535db5a..2ad029e444 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q16/extended.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q16/extended.txt @@ -2,7 +2,7 @@ CometColumnarToRow +- CometHashAggregate +- CometColumnarExchange +- HashAggregate - +- HashAggregate [COMET: Unsupported aggregation mode PartialMerge, Native support for operator HashAggregateExec is disabled. Set spark.comet.exec.aggregate.enabled=true to enable it.] + +- HashAggregate [COMET: Unsupported aggregation mode PartialMerge] +- CometColumnarToRow +- CometHashAggregate +- CometProject diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q20.native_iceberg_compat/extended.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q20.native_iceberg_compat/extended.txt index f5da783a44..1b24b68f14 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q20.native_iceberg_compat/extended.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q20.native_iceberg_compat/extended.txt @@ -1,6 +1,6 @@ TakeOrderedAndProject [COMET: ] +- Project - +- Window [COMET: Native support for operator WindowExec is disabled. Set spark.comet.exec.window.enabled=true to enable it.] + +- Window [COMET: WindowExec is not fully compatible with Spark (Native WindowExec has known correctness issues). To enable it anyway, set spark.comet.operator.WindowExec.allowIncompatible=true. For more information, refer to the Comet Compatibility Guide (https://datafusion.apache.org/comet/user-guide/compatibility.html).] +- CometColumnarToRow +- CometSort +- CometExchange diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q20/extended.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q20/extended.txt index f5da783a44..1b24b68f14 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q20/extended.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q20/extended.txt @@ -1,6 +1,6 @@ TakeOrderedAndProject [COMET: ] +- Project - +- Window [COMET: Native support for operator WindowExec is disabled. Set spark.comet.exec.window.enabled=true to enable it.] + +- Window [COMET: WindowExec is not fully compatible with Spark (Native WindowExec has known correctness issues). To enable it anyway, set spark.comet.operator.WindowExec.allowIncompatible=true. For more information, refer to the Comet Compatibility Guide (https://datafusion.apache.org/comet/user-guide/compatibility.html).] +- CometColumnarToRow +- CometSort +- CometExchange diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q28.native_iceberg_compat/extended.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q28.native_iceberg_compat/extended.txt index 4b94cbb91f..841b4c9822 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q28.native_iceberg_compat/extended.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q28.native_iceberg_compat/extended.txt @@ -7,7 +7,7 @@ BroadcastNestedLoopJoin : : : : : +- CometHashAggregate : : : : : +- CometColumnarExchange : : : : : +- HashAggregate -: : : : : +- HashAggregate [COMET: Unsupported aggregation mode PartialMerge, Native support for operator HashAggregateExec is disabled. Set spark.comet.exec.aggregate.enabled=true to enable it.] +: : : : : +- HashAggregate [COMET: Unsupported aggregation mode PartialMerge] : : : : : +- CometColumnarToRow : : : : : +- CometExchange : : : : : +- CometHashAggregate @@ -19,7 +19,7 @@ BroadcastNestedLoopJoin : : : : +- CometHashAggregate : : : : +- CometColumnarExchange : : : : +- HashAggregate -: : : : +- HashAggregate [COMET: Unsupported aggregation mode PartialMerge, Native support for operator HashAggregateExec is disabled. Set spark.comet.exec.aggregate.enabled=true to enable it.] +: : : : +- HashAggregate [COMET: Unsupported aggregation mode PartialMerge] : : : : +- CometColumnarToRow : : : : +- CometExchange : : : : +- CometHashAggregate @@ -31,7 +31,7 @@ BroadcastNestedLoopJoin : : : +- CometHashAggregate : : : +- CometColumnarExchange : : : +- HashAggregate -: : : +- HashAggregate [COMET: Unsupported aggregation mode PartialMerge, Native support for operator HashAggregateExec is disabled. Set spark.comet.exec.aggregate.enabled=true to enable it.] +: : : +- HashAggregate [COMET: Unsupported aggregation mode PartialMerge] : : : +- CometColumnarToRow : : : +- CometExchange : : : +- CometHashAggregate @@ -43,7 +43,7 @@ BroadcastNestedLoopJoin : : +- CometHashAggregate : : +- CometColumnarExchange : : +- HashAggregate -: : +- HashAggregate [COMET: Unsupported aggregation mode PartialMerge, Native support for operator HashAggregateExec is disabled. Set spark.comet.exec.aggregate.enabled=true to enable it.] +: : +- HashAggregate [COMET: Unsupported aggregation mode PartialMerge] : : +- CometColumnarToRow : : +- CometExchange : : +- CometHashAggregate @@ -55,7 +55,7 @@ BroadcastNestedLoopJoin : +- CometHashAggregate : +- CometColumnarExchange : +- HashAggregate -: +- HashAggregate [COMET: Unsupported aggregation mode PartialMerge, Native support for operator HashAggregateExec is disabled. Set spark.comet.exec.aggregate.enabled=true to enable it.] +: +- HashAggregate [COMET: Unsupported aggregation mode PartialMerge] : +- CometColumnarToRow : +- CometExchange : +- CometHashAggregate @@ -67,7 +67,7 @@ BroadcastNestedLoopJoin +- CometHashAggregate +- CometColumnarExchange +- HashAggregate - +- HashAggregate [COMET: Unsupported aggregation mode PartialMerge, Native support for operator HashAggregateExec is disabled. Set spark.comet.exec.aggregate.enabled=true to enable it.] + +- HashAggregate [COMET: Unsupported aggregation mode PartialMerge] +- CometColumnarToRow +- CometExchange +- CometHashAggregate diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q28/extended.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q28/extended.txt index 4b94cbb91f..841b4c9822 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q28/extended.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q28/extended.txt @@ -7,7 +7,7 @@ BroadcastNestedLoopJoin : : : : : +- CometHashAggregate : : : : : +- CometColumnarExchange : : : : : +- HashAggregate -: : : : : +- HashAggregate [COMET: Unsupported aggregation mode PartialMerge, Native support for operator HashAggregateExec is disabled. Set spark.comet.exec.aggregate.enabled=true to enable it.] +: : : : : +- HashAggregate [COMET: Unsupported aggregation mode PartialMerge] : : : : : +- CometColumnarToRow : : : : : +- CometExchange : : : : : +- CometHashAggregate @@ -19,7 +19,7 @@ BroadcastNestedLoopJoin : : : : +- CometHashAggregate : : : : +- CometColumnarExchange : : : : +- HashAggregate -: : : : +- HashAggregate [COMET: Unsupported aggregation mode PartialMerge, Native support for operator HashAggregateExec is disabled. Set spark.comet.exec.aggregate.enabled=true to enable it.] +: : : : +- HashAggregate [COMET: Unsupported aggregation mode PartialMerge] : : : : +- CometColumnarToRow : : : : +- CometExchange : : : : +- CometHashAggregate @@ -31,7 +31,7 @@ BroadcastNestedLoopJoin : : : +- CometHashAggregate : : : +- CometColumnarExchange : : : +- HashAggregate -: : : +- HashAggregate [COMET: Unsupported aggregation mode PartialMerge, Native support for operator HashAggregateExec is disabled. Set spark.comet.exec.aggregate.enabled=true to enable it.] +: : : +- HashAggregate [COMET: Unsupported aggregation mode PartialMerge] : : : +- CometColumnarToRow : : : +- CometExchange : : : +- CometHashAggregate @@ -43,7 +43,7 @@ BroadcastNestedLoopJoin : : +- CometHashAggregate : : +- CometColumnarExchange : : +- HashAggregate -: : +- HashAggregate [COMET: Unsupported aggregation mode PartialMerge, Native support for operator HashAggregateExec is disabled. Set spark.comet.exec.aggregate.enabled=true to enable it.] +: : +- HashAggregate [COMET: Unsupported aggregation mode PartialMerge] : : +- CometColumnarToRow : : +- CometExchange : : +- CometHashAggregate @@ -55,7 +55,7 @@ BroadcastNestedLoopJoin : +- CometHashAggregate : +- CometColumnarExchange : +- HashAggregate -: +- HashAggregate [COMET: Unsupported aggregation mode PartialMerge, Native support for operator HashAggregateExec is disabled. Set spark.comet.exec.aggregate.enabled=true to enable it.] +: +- HashAggregate [COMET: Unsupported aggregation mode PartialMerge] : +- CometColumnarToRow : +- CometExchange : +- CometHashAggregate @@ -67,7 +67,7 @@ BroadcastNestedLoopJoin +- CometHashAggregate +- CometColumnarExchange +- HashAggregate - +- HashAggregate [COMET: Unsupported aggregation mode PartialMerge, Native support for operator HashAggregateExec is disabled. Set spark.comet.exec.aggregate.enabled=true to enable it.] + +- HashAggregate [COMET: Unsupported aggregation mode PartialMerge] +- CometColumnarToRow +- CometExchange +- CometHashAggregate diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q36.native_iceberg_compat/extended.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q36.native_iceberg_compat/extended.txt index 4ddf55dee5..63050a1b8f 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q36.native_iceberg_compat/extended.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q36.native_iceberg_compat/extended.txt @@ -1,6 +1,6 @@ TakeOrderedAndProject [COMET: ] +- Project - +- Window [COMET: Native support for operator WindowExec is disabled. Set spark.comet.exec.window.enabled=true to enable it.] + +- Window [COMET: WindowExec is not fully compatible with Spark (Native WindowExec has known correctness issues). To enable it anyway, set spark.comet.operator.WindowExec.allowIncompatible=true. For more information, refer to the Comet Compatibility Guide (https://datafusion.apache.org/comet/user-guide/compatibility.html).] +- CometColumnarToRow +- CometSort +- CometExchange diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q36/extended.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q36/extended.txt index 4ddf55dee5..63050a1b8f 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q36/extended.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q36/extended.txt @@ -1,6 +1,6 @@ TakeOrderedAndProject [COMET: ] +- Project - +- Window [COMET: Native support for operator WindowExec is disabled. Set spark.comet.exec.window.enabled=true to enable it.] + +- Window [COMET: WindowExec is not fully compatible with Spark (Native WindowExec has known correctness issues). To enable it anyway, set spark.comet.operator.WindowExec.allowIncompatible=true. For more information, refer to the Comet Compatibility Guide (https://datafusion.apache.org/comet/user-guide/compatibility.html).] +- CometColumnarToRow +- CometSort +- CometExchange diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q47.native_iceberg_compat/extended.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q47.native_iceberg_compat/extended.txt index 56c60c0d74..d9f357c1c1 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q47.native_iceberg_compat/extended.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q47.native_iceberg_compat/extended.txt @@ -7,7 +7,7 @@ : : +- Filter : : +- Window : : +- Filter - : : +- Window [COMET: Native support for operator WindowExec is disabled. Set spark.comet.exec.window.enabled=true to enable it.] + : : +- Window [COMET: WindowExec is not fully compatible with Spark (Native WindowExec has known correctness issues). To enable it anyway, set spark.comet.operator.WindowExec.allowIncompatible=true. For more information, refer to the Comet Compatibility Guide (https://datafusion.apache.org/comet/user-guide/compatibility.html).] : : +- CometColumnarToRow : : +- CometSort : : +- CometExchange @@ -39,7 +39,7 @@ : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.store : +- BroadcastExchange : +- Project - : +- Window [COMET: Native support for operator WindowExec is disabled. Set spark.comet.exec.window.enabled=true to enable it.] + : +- Window [COMET: WindowExec is not fully compatible with Spark (Native WindowExec has known correctness issues). To enable it anyway, set spark.comet.operator.WindowExec.allowIncompatible=true. For more information, refer to the Comet Compatibility Guide (https://datafusion.apache.org/comet/user-guide/compatibility.html).] : +- CometColumnarToRow : +- CometSort : +- CometExchange @@ -71,7 +71,7 @@ : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.store +- BroadcastExchange +- Project - +- Window [COMET: Native support for operator WindowExec is disabled. Set spark.comet.exec.window.enabled=true to enable it.] + +- Window [COMET: WindowExec is not fully compatible with Spark (Native WindowExec has known correctness issues). To enable it anyway, set spark.comet.operator.WindowExec.allowIncompatible=true. For more information, refer to the Comet Compatibility Guide (https://datafusion.apache.org/comet/user-guide/compatibility.html).] +- CometColumnarToRow +- CometSort +- CometExchange diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q47/extended.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q47/extended.txt index 56c60c0d74..d9f357c1c1 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q47/extended.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q47/extended.txt @@ -7,7 +7,7 @@ : : +- Filter : : +- Window : : +- Filter - : : +- Window [COMET: Native support for operator WindowExec is disabled. Set spark.comet.exec.window.enabled=true to enable it.] + : : +- Window [COMET: WindowExec is not fully compatible with Spark (Native WindowExec has known correctness issues). To enable it anyway, set spark.comet.operator.WindowExec.allowIncompatible=true. For more information, refer to the Comet Compatibility Guide (https://datafusion.apache.org/comet/user-guide/compatibility.html).] : : +- CometColumnarToRow : : +- CometSort : : +- CometExchange @@ -39,7 +39,7 @@ : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.store : +- BroadcastExchange : +- Project - : +- Window [COMET: Native support for operator WindowExec is disabled. Set spark.comet.exec.window.enabled=true to enable it.] + : +- Window [COMET: WindowExec is not fully compatible with Spark (Native WindowExec has known correctness issues). To enable it anyway, set spark.comet.operator.WindowExec.allowIncompatible=true. For more information, refer to the Comet Compatibility Guide (https://datafusion.apache.org/comet/user-guide/compatibility.html).] : +- CometColumnarToRow : +- CometSort : +- CometExchange @@ -71,7 +71,7 @@ : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.store +- BroadcastExchange +- Project - +- Window [COMET: Native support for operator WindowExec is disabled. Set spark.comet.exec.window.enabled=true to enable it.] + +- Window [COMET: WindowExec is not fully compatible with Spark (Native WindowExec has known correctness issues). To enable it anyway, set spark.comet.operator.WindowExec.allowIncompatible=true. For more information, refer to the Comet Compatibility Guide (https://datafusion.apache.org/comet/user-guide/compatibility.html).] +- CometColumnarToRow +- CometSort +- CometExchange diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q49.native_iceberg_compat/extended.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q49.native_iceberg_compat/extended.txt index e6af7fd5e0..75684a9669 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q49.native_iceberg_compat/extended.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q49.native_iceberg_compat/extended.txt @@ -8,7 +8,7 @@ CometColumnarToRow : +- Filter : +- Window : +- Sort - : +- Window [COMET: Native support for operator WindowExec is disabled. Set spark.comet.exec.window.enabled=true to enable it.] + : +- Window [COMET: WindowExec is not fully compatible with Spark (Native WindowExec has known correctness issues). To enable it anyway, set spark.comet.operator.WindowExec.allowIncompatible=true. For more information, refer to the Comet Compatibility Guide (https://datafusion.apache.org/comet/user-guide/compatibility.html).] : +- CometColumnarToRow : +- CometSort : +- CometExchange @@ -40,7 +40,7 @@ CometColumnarToRow : +- Filter : +- Window : +- Sort - : +- Window [COMET: Native support for operator WindowExec is disabled. Set spark.comet.exec.window.enabled=true to enable it.] + : +- Window [COMET: WindowExec is not fully compatible with Spark (Native WindowExec has known correctness issues). To enable it anyway, set spark.comet.operator.WindowExec.allowIncompatible=true. For more information, refer to the Comet Compatibility Guide (https://datafusion.apache.org/comet/user-guide/compatibility.html).] : +- CometColumnarToRow : +- CometSort : +- CometExchange @@ -67,7 +67,7 @@ CometColumnarToRow +- Filter +- Window +- Sort - +- Window [COMET: Native support for operator WindowExec is disabled. Set spark.comet.exec.window.enabled=true to enable it.] + +- Window [COMET: WindowExec is not fully compatible with Spark (Native WindowExec has known correctness issues). To enable it anyway, set spark.comet.operator.WindowExec.allowIncompatible=true. For more information, refer to the Comet Compatibility Guide (https://datafusion.apache.org/comet/user-guide/compatibility.html).] +- CometColumnarToRow +- CometSort +- CometExchange diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q49/extended.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q49/extended.txt index e6af7fd5e0..75684a9669 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q49/extended.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q49/extended.txt @@ -8,7 +8,7 @@ CometColumnarToRow : +- Filter : +- Window : +- Sort - : +- Window [COMET: Native support for operator WindowExec is disabled. Set spark.comet.exec.window.enabled=true to enable it.] + : +- Window [COMET: WindowExec is not fully compatible with Spark (Native WindowExec has known correctness issues). To enable it anyway, set spark.comet.operator.WindowExec.allowIncompatible=true. For more information, refer to the Comet Compatibility Guide (https://datafusion.apache.org/comet/user-guide/compatibility.html).] : +- CometColumnarToRow : +- CometSort : +- CometExchange @@ -40,7 +40,7 @@ CometColumnarToRow : +- Filter : +- Window : +- Sort - : +- Window [COMET: Native support for operator WindowExec is disabled. Set spark.comet.exec.window.enabled=true to enable it.] + : +- Window [COMET: WindowExec is not fully compatible with Spark (Native WindowExec has known correctness issues). To enable it anyway, set spark.comet.operator.WindowExec.allowIncompatible=true. For more information, refer to the Comet Compatibility Guide (https://datafusion.apache.org/comet/user-guide/compatibility.html).] : +- CometColumnarToRow : +- CometSort : +- CometExchange @@ -67,7 +67,7 @@ CometColumnarToRow +- Filter +- Window +- Sort - +- Window [COMET: Native support for operator WindowExec is disabled. Set spark.comet.exec.window.enabled=true to enable it.] + +- Window [COMET: WindowExec is not fully compatible with Spark (Native WindowExec has known correctness issues). To enable it anyway, set spark.comet.operator.WindowExec.allowIncompatible=true. For more information, refer to the Comet Compatibility Guide (https://datafusion.apache.org/comet/user-guide/compatibility.html).] +- CometColumnarToRow +- CometSort +- CometExchange diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q51.native_iceberg_compat/extended.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q51.native_iceberg_compat/extended.txt index 5766386853..d55771ac8a 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q51.native_iceberg_compat/extended.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q51.native_iceberg_compat/extended.txt @@ -1,6 +1,6 @@ TakeOrderedAndProject [COMET: ] +- Filter - +- Window [COMET: Native support for operator WindowExec is disabled. Set spark.comet.exec.window.enabled=true to enable it.] + +- Window [COMET: WindowExec is not fully compatible with Spark (Native WindowExec has known correctness issues). To enable it anyway, set spark.comet.operator.WindowExec.allowIncompatible=true. For more information, refer to the Comet Compatibility Guide (https://datafusion.apache.org/comet/user-guide/compatibility.html)., Partitioning and sorting specifications must be the same.] +- CometColumnarToRow +- CometSort +- CometExchange @@ -9,7 +9,7 @@ :- CometSort : +- CometColumnarExchange : +- Project - : +- Window [COMET: Native support for operator WindowExec is disabled. Set spark.comet.exec.window.enabled=true to enable it.] + : +- Window [COMET: WindowExec is not fully compatible with Spark (Native WindowExec has known correctness issues). To enable it anyway, set spark.comet.operator.WindowExec.allowIncompatible=true. For more information, refer to the Comet Compatibility Guide (https://datafusion.apache.org/comet/user-guide/compatibility.html)., Partitioning and sorting specifications must be the same.] : +- CometColumnarToRow : +- CometSort : +- CometExchange @@ -33,7 +33,7 @@ +- CometSort +- CometColumnarExchange +- Project - +- Window [COMET: Native support for operator WindowExec is disabled. Set spark.comet.exec.window.enabled=true to enable it.] + +- Window [COMET: WindowExec is not fully compatible with Spark (Native WindowExec has known correctness issues). To enable it anyway, set spark.comet.operator.WindowExec.allowIncompatible=true. For more information, refer to the Comet Compatibility Guide (https://datafusion.apache.org/comet/user-guide/compatibility.html)., Partitioning and sorting specifications must be the same.] +- CometColumnarToRow +- CometSort +- CometExchange diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q51/extended.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q51/extended.txt index 5766386853..d55771ac8a 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q51/extended.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q51/extended.txt @@ -1,6 +1,6 @@ TakeOrderedAndProject [COMET: ] +- Filter - +- Window [COMET: Native support for operator WindowExec is disabled. Set spark.comet.exec.window.enabled=true to enable it.] + +- Window [COMET: WindowExec is not fully compatible with Spark (Native WindowExec has known correctness issues). To enable it anyway, set spark.comet.operator.WindowExec.allowIncompatible=true. For more information, refer to the Comet Compatibility Guide (https://datafusion.apache.org/comet/user-guide/compatibility.html)., Partitioning and sorting specifications must be the same.] +- CometColumnarToRow +- CometSort +- CometExchange @@ -9,7 +9,7 @@ :- CometSort : +- CometColumnarExchange : +- Project - : +- Window [COMET: Native support for operator WindowExec is disabled. Set spark.comet.exec.window.enabled=true to enable it.] + : +- Window [COMET: WindowExec is not fully compatible with Spark (Native WindowExec has known correctness issues). To enable it anyway, set spark.comet.operator.WindowExec.allowIncompatible=true. For more information, refer to the Comet Compatibility Guide (https://datafusion.apache.org/comet/user-guide/compatibility.html)., Partitioning and sorting specifications must be the same.] : +- CometColumnarToRow : +- CometSort : +- CometExchange @@ -33,7 +33,7 @@ +- CometSort +- CometColumnarExchange +- Project - +- Window [COMET: Native support for operator WindowExec is disabled. Set spark.comet.exec.window.enabled=true to enable it.] + +- Window [COMET: WindowExec is not fully compatible with Spark (Native WindowExec has known correctness issues). To enable it anyway, set spark.comet.operator.WindowExec.allowIncompatible=true. For more information, refer to the Comet Compatibility Guide (https://datafusion.apache.org/comet/user-guide/compatibility.html)., Partitioning and sorting specifications must be the same.] +- CometColumnarToRow +- CometSort +- CometExchange diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q53.native_iceberg_compat/extended.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q53.native_iceberg_compat/extended.txt index c3eff7fb0f..3a68550e3d 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q53.native_iceberg_compat/extended.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q53.native_iceberg_compat/extended.txt @@ -1,7 +1,7 @@ TakeOrderedAndProject [COMET: ] +- Project +- Filter - +- Window [COMET: Native support for operator WindowExec is disabled. Set spark.comet.exec.window.enabled=true to enable it.] + +- Window [COMET: WindowExec is not fully compatible with Spark (Native WindowExec has known correctness issues). To enable it anyway, set spark.comet.operator.WindowExec.allowIncompatible=true. For more information, refer to the Comet Compatibility Guide (https://datafusion.apache.org/comet/user-guide/compatibility.html).] +- CometColumnarToRow +- CometSort +- CometExchange diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q53/extended.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q53/extended.txt index c3eff7fb0f..3a68550e3d 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q53/extended.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q53/extended.txt @@ -1,7 +1,7 @@ TakeOrderedAndProject [COMET: ] +- Project +- Filter - +- Window [COMET: Native support for operator WindowExec is disabled. Set spark.comet.exec.window.enabled=true to enable it.] + +- Window [COMET: WindowExec is not fully compatible with Spark (Native WindowExec has known correctness issues). To enable it anyway, set spark.comet.operator.WindowExec.allowIncompatible=true. For more information, refer to the Comet Compatibility Guide (https://datafusion.apache.org/comet/user-guide/compatibility.html).] +- CometColumnarToRow +- CometSort +- CometExchange diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q57.native_iceberg_compat/extended.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q57.native_iceberg_compat/extended.txt index 5e24da49b7..eb8f4ab303 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q57.native_iceberg_compat/extended.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q57.native_iceberg_compat/extended.txt @@ -7,7 +7,7 @@ : : +- Filter : : +- Window : : +- Filter - : : +- Window [COMET: Native support for operator WindowExec is disabled. Set spark.comet.exec.window.enabled=true to enable it.] + : : +- Window [COMET: WindowExec is not fully compatible with Spark (Native WindowExec has known correctness issues). To enable it anyway, set spark.comet.operator.WindowExec.allowIncompatible=true. For more information, refer to the Comet Compatibility Guide (https://datafusion.apache.org/comet/user-guide/compatibility.html).] : : +- CometColumnarToRow : : +- CometSort : : +- CometExchange @@ -39,7 +39,7 @@ : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.call_center : +- BroadcastExchange : +- Project - : +- Window [COMET: Native support for operator WindowExec is disabled. Set spark.comet.exec.window.enabled=true to enable it.] + : +- Window [COMET: WindowExec is not fully compatible with Spark (Native WindowExec has known correctness issues). To enable it anyway, set spark.comet.operator.WindowExec.allowIncompatible=true. For more information, refer to the Comet Compatibility Guide (https://datafusion.apache.org/comet/user-guide/compatibility.html).] : +- CometColumnarToRow : +- CometSort : +- CometExchange @@ -71,7 +71,7 @@ : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.call_center +- BroadcastExchange +- Project - +- Window [COMET: Native support for operator WindowExec is disabled. Set spark.comet.exec.window.enabled=true to enable it.] + +- Window [COMET: WindowExec is not fully compatible with Spark (Native WindowExec has known correctness issues). To enable it anyway, set spark.comet.operator.WindowExec.allowIncompatible=true. For more information, refer to the Comet Compatibility Guide (https://datafusion.apache.org/comet/user-guide/compatibility.html).] +- CometColumnarToRow +- CometSort +- CometExchange diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q57/extended.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q57/extended.txt index 5e24da49b7..eb8f4ab303 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q57/extended.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q57/extended.txt @@ -7,7 +7,7 @@ : : +- Filter : : +- Window : : +- Filter - : : +- Window [COMET: Native support for operator WindowExec is disabled. Set spark.comet.exec.window.enabled=true to enable it.] + : : +- Window [COMET: WindowExec is not fully compatible with Spark (Native WindowExec has known correctness issues). To enable it anyway, set spark.comet.operator.WindowExec.allowIncompatible=true. For more information, refer to the Comet Compatibility Guide (https://datafusion.apache.org/comet/user-guide/compatibility.html).] : : +- CometColumnarToRow : : +- CometSort : : +- CometExchange @@ -39,7 +39,7 @@ : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.call_center : +- BroadcastExchange : +- Project - : +- Window [COMET: Native support for operator WindowExec is disabled. Set spark.comet.exec.window.enabled=true to enable it.] + : +- Window [COMET: WindowExec is not fully compatible with Spark (Native WindowExec has known correctness issues). To enable it anyway, set spark.comet.operator.WindowExec.allowIncompatible=true. For more information, refer to the Comet Compatibility Guide (https://datafusion.apache.org/comet/user-guide/compatibility.html).] : +- CometColumnarToRow : +- CometSort : +- CometExchange @@ -71,7 +71,7 @@ : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.call_center +- BroadcastExchange +- Project - +- Window [COMET: Native support for operator WindowExec is disabled. Set spark.comet.exec.window.enabled=true to enable it.] + +- Window [COMET: WindowExec is not fully compatible with Spark (Native WindowExec has known correctness issues). To enable it anyway, set spark.comet.operator.WindowExec.allowIncompatible=true. For more information, refer to the Comet Compatibility Guide (https://datafusion.apache.org/comet/user-guide/compatibility.html).] +- CometColumnarToRow +- CometSort +- CometExchange diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q63.native_iceberg_compat/extended.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q63.native_iceberg_compat/extended.txt index c3eff7fb0f..3a68550e3d 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q63.native_iceberg_compat/extended.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q63.native_iceberg_compat/extended.txt @@ -1,7 +1,7 @@ TakeOrderedAndProject [COMET: ] +- Project +- Filter - +- Window [COMET: Native support for operator WindowExec is disabled. Set spark.comet.exec.window.enabled=true to enable it.] + +- Window [COMET: WindowExec is not fully compatible with Spark (Native WindowExec has known correctness issues). To enable it anyway, set spark.comet.operator.WindowExec.allowIncompatible=true. For more information, refer to the Comet Compatibility Guide (https://datafusion.apache.org/comet/user-guide/compatibility.html).] +- CometColumnarToRow +- CometSort +- CometExchange diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q63/extended.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q63/extended.txt index c3eff7fb0f..3a68550e3d 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q63/extended.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q63/extended.txt @@ -1,7 +1,7 @@ TakeOrderedAndProject [COMET: ] +- Project +- Filter - +- Window [COMET: Native support for operator WindowExec is disabled. Set spark.comet.exec.window.enabled=true to enable it.] + +- Window [COMET: WindowExec is not fully compatible with Spark (Native WindowExec has known correctness issues). To enable it anyway, set spark.comet.operator.WindowExec.allowIncompatible=true. For more information, refer to the Comet Compatibility Guide (https://datafusion.apache.org/comet/user-guide/compatibility.html).] +- CometColumnarToRow +- CometSort +- CometExchange diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q70.native_iceberg_compat/extended.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q70.native_iceberg_compat/extended.txt index 97fefb4f00..68ff906018 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q70.native_iceberg_compat/extended.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q70.native_iceberg_compat/extended.txt @@ -1,6 +1,6 @@ TakeOrderedAndProject [COMET: ] +- Project - +- Window [COMET: Native support for operator WindowExec is disabled. Set spark.comet.exec.window.enabled=true to enable it.] + +- Window [COMET: WindowExec is not fully compatible with Spark (Native WindowExec has known correctness issues). To enable it anyway, set spark.comet.operator.WindowExec.allowIncompatible=true. For more information, refer to the Comet Compatibility Guide (https://datafusion.apache.org/comet/user-guide/compatibility.html).] +- CometColumnarToRow +- CometSort +- CometColumnarExchange diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q70/extended.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q70/extended.txt index 97fefb4f00..68ff906018 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q70/extended.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q70/extended.txt @@ -1,6 +1,6 @@ TakeOrderedAndProject [COMET: ] +- Project - +- Window [COMET: Native support for operator WindowExec is disabled. Set spark.comet.exec.window.enabled=true to enable it.] + +- Window [COMET: WindowExec is not fully compatible with Spark (Native WindowExec has known correctness issues). To enable it anyway, set spark.comet.operator.WindowExec.allowIncompatible=true. For more information, refer to the Comet Compatibility Guide (https://datafusion.apache.org/comet/user-guide/compatibility.html).] +- CometColumnarToRow +- CometSort +- CometColumnarExchange diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q78.native_iceberg_compat/extended.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q78.native_iceberg_compat/extended.txt index 8053f01bc6..97a1bdbbaf 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q78.native_iceberg_compat/extended.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q78.native_iceberg_compat/extended.txt @@ -1,5 +1,5 @@ TakeOrderedAndProject [COMET: ] -+- Project [COMET: Comet does not support Spark's BigDecimal rounding, Native support for operator ProjectExec is disabled. Set spark.comet.exec.project.enabled=true to enable it.] ++- Project [COMET: Comet does not support Spark's BigDecimal rounding] +- CometColumnarToRow +- CometSortMergeJoin :- CometProject diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q78/extended.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q78/extended.txt index 8053f01bc6..97a1bdbbaf 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q78/extended.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q78/extended.txt @@ -1,5 +1,5 @@ TakeOrderedAndProject [COMET: ] -+- Project [COMET: Comet does not support Spark's BigDecimal rounding, Native support for operator ProjectExec is disabled. Set spark.comet.exec.project.enabled=true to enable it.] ++- Project [COMET: Comet does not support Spark's BigDecimal rounding] +- CometColumnarToRow +- CometSortMergeJoin :- CometProject diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q86.native_iceberg_compat/extended.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q86.native_iceberg_compat/extended.txt index ef6698a475..1193bed826 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q86.native_iceberg_compat/extended.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q86.native_iceberg_compat/extended.txt @@ -1,6 +1,6 @@ TakeOrderedAndProject [COMET: ] +- Project - +- Window [COMET: Native support for operator WindowExec is disabled. Set spark.comet.exec.window.enabled=true to enable it.] + +- Window [COMET: WindowExec is not fully compatible with Spark (Native WindowExec has known correctness issues). To enable it anyway, set spark.comet.operator.WindowExec.allowIncompatible=true. For more information, refer to the Comet Compatibility Guide (https://datafusion.apache.org/comet/user-guide/compatibility.html).] +- CometColumnarToRow +- CometSort +- CometExchange diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q86/extended.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q86/extended.txt index ef6698a475..1193bed826 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q86/extended.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q86/extended.txt @@ -1,6 +1,6 @@ TakeOrderedAndProject [COMET: ] +- Project - +- Window [COMET: Native support for operator WindowExec is disabled. Set spark.comet.exec.window.enabled=true to enable it.] + +- Window [COMET: WindowExec is not fully compatible with Spark (Native WindowExec has known correctness issues). To enable it anyway, set spark.comet.operator.WindowExec.allowIncompatible=true. For more information, refer to the Comet Compatibility Guide (https://datafusion.apache.org/comet/user-guide/compatibility.html).] +- CometColumnarToRow +- CometSort +- CometExchange diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q89.native_iceberg_compat/extended.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q89.native_iceberg_compat/extended.txt index c3eff7fb0f..3a68550e3d 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q89.native_iceberg_compat/extended.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q89.native_iceberg_compat/extended.txt @@ -1,7 +1,7 @@ TakeOrderedAndProject [COMET: ] +- Project +- Filter - +- Window [COMET: Native support for operator WindowExec is disabled. Set spark.comet.exec.window.enabled=true to enable it.] + +- Window [COMET: WindowExec is not fully compatible with Spark (Native WindowExec has known correctness issues). To enable it anyway, set spark.comet.operator.WindowExec.allowIncompatible=true. For more information, refer to the Comet Compatibility Guide (https://datafusion.apache.org/comet/user-guide/compatibility.html).] +- CometColumnarToRow +- CometSort +- CometExchange diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q89/extended.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q89/extended.txt index c3eff7fb0f..3a68550e3d 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q89/extended.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q89/extended.txt @@ -1,7 +1,7 @@ TakeOrderedAndProject [COMET: ] +- Project +- Filter - +- Window [COMET: Native support for operator WindowExec is disabled. Set spark.comet.exec.window.enabled=true to enable it.] + +- Window [COMET: WindowExec is not fully compatible with Spark (Native WindowExec has known correctness issues). To enable it anyway, set spark.comet.operator.WindowExec.allowIncompatible=true. For more information, refer to the Comet Compatibility Guide (https://datafusion.apache.org/comet/user-guide/compatibility.html).] +- CometColumnarToRow +- CometSort +- CometExchange diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q9.native_iceberg_compat/extended.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q9.native_iceberg_compat/extended.txt index 148b9d4491..ed71033b26 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q9.native_iceberg_compat/extended.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q9.native_iceberg_compat/extended.txt @@ -1,4 +1,4 @@ - Project [COMET: Native support for operator ProjectExec is disabled. Set spark.comet.exec.project.enabled=true to enable it.] + Project [COMET: ] : :- Subquery : : +- CometColumnarToRow : : +- CometProject diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q9/extended.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q9/extended.txt index 148b9d4491..ed71033b26 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q9/extended.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q9/extended.txt @@ -1,4 +1,4 @@ - Project [COMET: Native support for operator ProjectExec is disabled. Set spark.comet.exec.project.enabled=true to enable it.] + Project [COMET: ] : :- Subquery : : +- CometColumnarToRow : : +- CometProject diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q94.native_iceberg_compat/extended.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q94.native_iceberg_compat/extended.txt index e42c9a2062..eac4939621 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q94.native_iceberg_compat/extended.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q94.native_iceberg_compat/extended.txt @@ -2,7 +2,7 @@ CometColumnarToRow +- CometHashAggregate +- CometColumnarExchange +- HashAggregate - +- HashAggregate [COMET: Unsupported aggregation mode PartialMerge, Native support for operator HashAggregateExec is disabled. Set spark.comet.exec.aggregate.enabled=true to enable it.] + +- HashAggregate [COMET: Unsupported aggregation mode PartialMerge] +- CometColumnarToRow +- CometHashAggregate +- CometProject diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q94/extended.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q94/extended.txt index e42c9a2062..eac4939621 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q94/extended.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q94/extended.txt @@ -2,7 +2,7 @@ CometColumnarToRow +- CometHashAggregate +- CometColumnarExchange +- HashAggregate - +- HashAggregate [COMET: Unsupported aggregation mode PartialMerge, Native support for operator HashAggregateExec is disabled. Set spark.comet.exec.aggregate.enabled=true to enable it.] + +- HashAggregate [COMET: Unsupported aggregation mode PartialMerge] +- CometColumnarToRow +- CometHashAggregate +- CometProject diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q95.native_iceberg_compat/extended.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q95.native_iceberg_compat/extended.txt index 5a1b14b632..6ff8eba58f 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q95.native_iceberg_compat/extended.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q95.native_iceberg_compat/extended.txt @@ -2,7 +2,7 @@ CometColumnarToRow +- CometHashAggregate +- CometColumnarExchange +- HashAggregate - +- HashAggregate [COMET: Unsupported aggregation mode PartialMerge, Native support for operator HashAggregateExec is disabled. Set spark.comet.exec.aggregate.enabled=true to enable it.] + +- HashAggregate [COMET: Unsupported aggregation mode PartialMerge] +- CometColumnarToRow +- CometHashAggregate +- CometProject diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q95/extended.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q95/extended.txt index 5a1b14b632..6ff8eba58f 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q95/extended.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q95/extended.txt @@ -2,7 +2,7 @@ CometColumnarToRow +- CometHashAggregate +- CometColumnarExchange +- HashAggregate - +- HashAggregate [COMET: Unsupported aggregation mode PartialMerge, Native support for operator HashAggregateExec is disabled. Set spark.comet.exec.aggregate.enabled=true to enable it.] + +- HashAggregate [COMET: Unsupported aggregation mode PartialMerge] +- CometColumnarToRow +- CometHashAggregate +- CometProject diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q98.native_iceberg_compat/extended.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q98.native_iceberg_compat/extended.txt index 001fedfc83..4c972848e7 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q98.native_iceberg_compat/extended.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q98.native_iceberg_compat/extended.txt @@ -3,7 +3,7 @@ CometColumnarToRow +- CometSort +- CometColumnarExchange +- Project - +- Window [COMET: Native support for operator WindowExec is disabled. Set spark.comet.exec.window.enabled=true to enable it.] + +- Window [COMET: WindowExec is not fully compatible with Spark (Native WindowExec has known correctness issues). To enable it anyway, set spark.comet.operator.WindowExec.allowIncompatible=true. For more information, refer to the Comet Compatibility Guide (https://datafusion.apache.org/comet/user-guide/compatibility.html).] +- CometColumnarToRow +- CometSort +- CometExchange diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q98/extended.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q98/extended.txt index 001fedfc83..4c972848e7 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q98/extended.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q98/extended.txt @@ -3,7 +3,7 @@ CometColumnarToRow +- CometSort +- CometColumnarExchange +- Project - +- Window [COMET: Native support for operator WindowExec is disabled. Set spark.comet.exec.window.enabled=true to enable it.] + +- Window [COMET: WindowExec is not fully compatible with Spark (Native WindowExec has known correctness issues). To enable it anyway, set spark.comet.operator.WindowExec.allowIncompatible=true. For more information, refer to the Comet Compatibility Guide (https://datafusion.apache.org/comet/user-guide/compatibility.html).] +- CometColumnarToRow +- CometSort +- CometExchange diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7-spark3_5/q12.native_iceberg_compat/extended.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7-spark3_5/q12.native_iceberg_compat/extended.txt index 9ddd3aeb38..9ea26caab6 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7-spark3_5/q12.native_iceberg_compat/extended.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7-spark3_5/q12.native_iceberg_compat/extended.txt @@ -1,6 +1,6 @@ TakeOrderedAndProject [COMET: ] +- Project - +- Window [COMET: Window expressions are not supported] + +- Window [COMET: WindowExec is not fully compatible with Spark (Native WindowExec has known correctness issues). To enable it anyway, set spark.comet.operator.WindowExec.allowIncompatible=true. For more information, refer to the Comet Compatibility Guide (https://datafusion.apache.org/comet/user-guide/compatibility.html).] +- CometColumnarToRow +- CometSort +- CometExchange diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7-spark3_5/q12/extended.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7-spark3_5/q12/extended.txt index 9ddd3aeb38..9ea26caab6 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7-spark3_5/q12/extended.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7-spark3_5/q12/extended.txt @@ -1,6 +1,6 @@ TakeOrderedAndProject [COMET: ] +- Project - +- Window [COMET: Window expressions are not supported] + +- Window [COMET: WindowExec is not fully compatible with Spark (Native WindowExec has known correctness issues). To enable it anyway, set spark.comet.operator.WindowExec.allowIncompatible=true. For more information, refer to the Comet Compatibility Guide (https://datafusion.apache.org/comet/user-guide/compatibility.html).] +- CometColumnarToRow +- CometSort +- CometExchange diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7-spark3_5/q20.native_iceberg_compat/extended.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7-spark3_5/q20.native_iceberg_compat/extended.txt index 400633ed9f..1b24b68f14 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7-spark3_5/q20.native_iceberg_compat/extended.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7-spark3_5/q20.native_iceberg_compat/extended.txt @@ -1,6 +1,6 @@ TakeOrderedAndProject [COMET: ] +- Project - +- Window [COMET: Window expressions are not supported] + +- Window [COMET: WindowExec is not fully compatible with Spark (Native WindowExec has known correctness issues). To enable it anyway, set spark.comet.operator.WindowExec.allowIncompatible=true. For more information, refer to the Comet Compatibility Guide (https://datafusion.apache.org/comet/user-guide/compatibility.html).] +- CometColumnarToRow +- CometSort +- CometExchange diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7-spark3_5/q20/extended.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7-spark3_5/q20/extended.txt index 400633ed9f..1b24b68f14 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7-spark3_5/q20/extended.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7-spark3_5/q20/extended.txt @@ -1,6 +1,6 @@ TakeOrderedAndProject [COMET: ] +- Project - +- Window [COMET: Window expressions are not supported] + +- Window [COMET: WindowExec is not fully compatible with Spark (Native WindowExec has known correctness issues). To enable it anyway, set spark.comet.operator.WindowExec.allowIncompatible=true. For more information, refer to the Comet Compatibility Guide (https://datafusion.apache.org/comet/user-guide/compatibility.html).] +- CometColumnarToRow +- CometSort +- CometExchange diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7-spark3_5/q36a.native_iceberg_compat/extended.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7-spark3_5/q36a.native_iceberg_compat/extended.txt index dee6669b47..1de0c1ea43 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7-spark3_5/q36a.native_iceberg_compat/extended.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7-spark3_5/q36a.native_iceberg_compat/extended.txt @@ -1,6 +1,6 @@ TakeOrderedAndProject [COMET: ] +- Project - +- Window [COMET: Window expressions are not supported] + +- Window [COMET: WindowExec is not fully compatible with Spark (Native WindowExec has known correctness issues). To enable it anyway, set spark.comet.operator.WindowExec.allowIncompatible=true. For more information, refer to the Comet Compatibility Guide (https://datafusion.apache.org/comet/user-guide/compatibility.html).] +- CometColumnarToRow +- CometSort +- CometExchange diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7-spark3_5/q36a/extended.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7-spark3_5/q36a/extended.txt index dee6669b47..1de0c1ea43 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7-spark3_5/q36a/extended.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7-spark3_5/q36a/extended.txt @@ -1,6 +1,6 @@ TakeOrderedAndProject [COMET: ] +- Project - +- Window [COMET: Window expressions are not supported] + +- Window [COMET: WindowExec is not fully compatible with Spark (Native WindowExec has known correctness issues). To enable it anyway, set spark.comet.operator.WindowExec.allowIncompatible=true. For more information, refer to the Comet Compatibility Guide (https://datafusion.apache.org/comet/user-guide/compatibility.html).] +- CometColumnarToRow +- CometSort +- CometExchange diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7-spark3_5/q47.native_iceberg_compat/extended.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7-spark3_5/q47.native_iceberg_compat/extended.txt index 11fbf5bd11..d9f357c1c1 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7-spark3_5/q47.native_iceberg_compat/extended.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7-spark3_5/q47.native_iceberg_compat/extended.txt @@ -7,7 +7,7 @@ : : +- Filter : : +- Window : : +- Filter - : : +- Window [COMET: Window expressions are not supported] + : : +- Window [COMET: WindowExec is not fully compatible with Spark (Native WindowExec has known correctness issues). To enable it anyway, set spark.comet.operator.WindowExec.allowIncompatible=true. For more information, refer to the Comet Compatibility Guide (https://datafusion.apache.org/comet/user-guide/compatibility.html).] : : +- CometColumnarToRow : : +- CometSort : : +- CometExchange @@ -39,7 +39,7 @@ : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.store : +- BroadcastExchange : +- Project - : +- Window [COMET: Window expressions are not supported] + : +- Window [COMET: WindowExec is not fully compatible with Spark (Native WindowExec has known correctness issues). To enable it anyway, set spark.comet.operator.WindowExec.allowIncompatible=true. For more information, refer to the Comet Compatibility Guide (https://datafusion.apache.org/comet/user-guide/compatibility.html).] : +- CometColumnarToRow : +- CometSort : +- CometExchange @@ -71,7 +71,7 @@ : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.store +- BroadcastExchange +- Project - +- Window [COMET: Window expressions are not supported] + +- Window [COMET: WindowExec is not fully compatible with Spark (Native WindowExec has known correctness issues). To enable it anyway, set spark.comet.operator.WindowExec.allowIncompatible=true. For more information, refer to the Comet Compatibility Guide (https://datafusion.apache.org/comet/user-guide/compatibility.html).] +- CometColumnarToRow +- CometSort +- CometExchange diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7-spark3_5/q47/extended.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7-spark3_5/q47/extended.txt index 11fbf5bd11..d9f357c1c1 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7-spark3_5/q47/extended.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7-spark3_5/q47/extended.txt @@ -7,7 +7,7 @@ : : +- Filter : : +- Window : : +- Filter - : : +- Window [COMET: Window expressions are not supported] + : : +- Window [COMET: WindowExec is not fully compatible with Spark (Native WindowExec has known correctness issues). To enable it anyway, set spark.comet.operator.WindowExec.allowIncompatible=true. For more information, refer to the Comet Compatibility Guide (https://datafusion.apache.org/comet/user-guide/compatibility.html).] : : +- CometColumnarToRow : : +- CometSort : : +- CometExchange @@ -39,7 +39,7 @@ : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.store : +- BroadcastExchange : +- Project - : +- Window [COMET: Window expressions are not supported] + : +- Window [COMET: WindowExec is not fully compatible with Spark (Native WindowExec has known correctness issues). To enable it anyway, set spark.comet.operator.WindowExec.allowIncompatible=true. For more information, refer to the Comet Compatibility Guide (https://datafusion.apache.org/comet/user-guide/compatibility.html).] : +- CometColumnarToRow : +- CometSort : +- CometExchange @@ -71,7 +71,7 @@ : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.store +- BroadcastExchange +- Project - +- Window [COMET: Window expressions are not supported] + +- Window [COMET: WindowExec is not fully compatible with Spark (Native WindowExec has known correctness issues). To enable it anyway, set spark.comet.operator.WindowExec.allowIncompatible=true. For more information, refer to the Comet Compatibility Guide (https://datafusion.apache.org/comet/user-guide/compatibility.html).] +- CometColumnarToRow +- CometSort +- CometExchange diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7-spark3_5/q49.native_iceberg_compat/extended.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7-spark3_5/q49.native_iceberg_compat/extended.txt index 734b0ce7bb..75684a9669 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7-spark3_5/q49.native_iceberg_compat/extended.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7-spark3_5/q49.native_iceberg_compat/extended.txt @@ -8,7 +8,7 @@ CometColumnarToRow : +- Filter : +- Window : +- Sort - : +- Window [COMET: Window expressions are not supported] + : +- Window [COMET: WindowExec is not fully compatible with Spark (Native WindowExec has known correctness issues). To enable it anyway, set spark.comet.operator.WindowExec.allowIncompatible=true. For more information, refer to the Comet Compatibility Guide (https://datafusion.apache.org/comet/user-guide/compatibility.html).] : +- CometColumnarToRow : +- CometSort : +- CometExchange @@ -40,7 +40,7 @@ CometColumnarToRow : +- Filter : +- Window : +- Sort - : +- Window [COMET: Window expressions are not supported] + : +- Window [COMET: WindowExec is not fully compatible with Spark (Native WindowExec has known correctness issues). To enable it anyway, set spark.comet.operator.WindowExec.allowIncompatible=true. For more information, refer to the Comet Compatibility Guide (https://datafusion.apache.org/comet/user-guide/compatibility.html).] : +- CometColumnarToRow : +- CometSort : +- CometExchange @@ -67,7 +67,7 @@ CometColumnarToRow +- Filter +- Window +- Sort - +- Window [COMET: Window expressions are not supported] + +- Window [COMET: WindowExec is not fully compatible with Spark (Native WindowExec has known correctness issues). To enable it anyway, set spark.comet.operator.WindowExec.allowIncompatible=true. For more information, refer to the Comet Compatibility Guide (https://datafusion.apache.org/comet/user-guide/compatibility.html).] +- CometColumnarToRow +- CometSort +- CometExchange diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7-spark3_5/q49/extended.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7-spark3_5/q49/extended.txt index 734b0ce7bb..75684a9669 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7-spark3_5/q49/extended.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7-spark3_5/q49/extended.txt @@ -8,7 +8,7 @@ CometColumnarToRow : +- Filter : +- Window : +- Sort - : +- Window [COMET: Window expressions are not supported] + : +- Window [COMET: WindowExec is not fully compatible with Spark (Native WindowExec has known correctness issues). To enable it anyway, set spark.comet.operator.WindowExec.allowIncompatible=true. For more information, refer to the Comet Compatibility Guide (https://datafusion.apache.org/comet/user-guide/compatibility.html).] : +- CometColumnarToRow : +- CometSort : +- CometExchange @@ -40,7 +40,7 @@ CometColumnarToRow : +- Filter : +- Window : +- Sort - : +- Window [COMET: Window expressions are not supported] + : +- Window [COMET: WindowExec is not fully compatible with Spark (Native WindowExec has known correctness issues). To enable it anyway, set spark.comet.operator.WindowExec.allowIncompatible=true. For more information, refer to the Comet Compatibility Guide (https://datafusion.apache.org/comet/user-guide/compatibility.html).] : +- CometColumnarToRow : +- CometSort : +- CometExchange @@ -67,7 +67,7 @@ CometColumnarToRow +- Filter +- Window +- Sort - +- Window [COMET: Window expressions are not supported] + +- Window [COMET: WindowExec is not fully compatible with Spark (Native WindowExec has known correctness issues). To enable it anyway, set spark.comet.operator.WindowExec.allowIncompatible=true. For more information, refer to the Comet Compatibility Guide (https://datafusion.apache.org/comet/user-guide/compatibility.html).] +- CometColumnarToRow +- CometSort +- CometExchange diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7-spark3_5/q51a.native_iceberg_compat/extended.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7-spark3_5/q51a.native_iceberg_compat/extended.txt index 15a87f66b6..b192555935 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7-spark3_5/q51a.native_iceberg_compat/extended.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7-spark3_5/q51a.native_iceberg_compat/extended.txt @@ -4,7 +4,7 @@ +- HashAggregate +- Project +- BroadcastHashJoin - :- Window [COMET: Window expressions are not supported] + :- Window [COMET: WindowExec is not fully compatible with Spark (Native WindowExec has known correctness issues). To enable it anyway, set spark.comet.operator.WindowExec.allowIncompatible=true. For more information, refer to the Comet Compatibility Guide (https://datafusion.apache.org/comet/user-guide/compatibility.html)., Partitioning and sorting specifications must be the same.] : +- CometColumnarToRow : +- CometSort : +- CometExchange @@ -20,7 +20,7 @@ : : +- Project : : +- BroadcastHashJoin : : :- Project - : : : +- Window [COMET: Window expressions are not supported] + : : : +- Window [COMET: WindowExec is not fully compatible with Spark (Native WindowExec has known correctness issues). To enable it anyway, set spark.comet.operator.WindowExec.allowIncompatible=true. For more information, refer to the Comet Compatibility Guide (https://datafusion.apache.org/comet/user-guide/compatibility.html)., Partitioning and sorting specifications must be the same.] : : : +- CometColumnarToRow : : : +- CometSort : : : +- CometExchange @@ -43,7 +43,7 @@ : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim : : +- BroadcastExchange : : +- Project - : : +- Window [COMET: Window expressions are not supported] + : : +- Window [COMET: WindowExec is not fully compatible with Spark (Native WindowExec has known correctness issues). To enable it anyway, set spark.comet.operator.WindowExec.allowIncompatible=true. For more information, refer to the Comet Compatibility Guide (https://datafusion.apache.org/comet/user-guide/compatibility.html)., Partitioning and sorting specifications must be the same.] : : +- CometColumnarToRow : : +- CometSort : : +- CometExchange @@ -73,7 +73,7 @@ : +- Project : +- BroadcastHashJoin : :- Project - : : +- Window [COMET: Window expressions are not supported] + : : +- Window [COMET: WindowExec is not fully compatible with Spark (Native WindowExec has known correctness issues). To enable it anyway, set spark.comet.operator.WindowExec.allowIncompatible=true. For more information, refer to the Comet Compatibility Guide (https://datafusion.apache.org/comet/user-guide/compatibility.html)., Partitioning and sorting specifications must be the same.] : : +- CometColumnarToRow : : +- CometSort : : +- CometExchange @@ -91,7 +91,7 @@ : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim : +- BroadcastExchange : +- Project - : +- Window [COMET: Window expressions are not supported] + : +- Window [COMET: WindowExec is not fully compatible with Spark (Native WindowExec has known correctness issues). To enable it anyway, set spark.comet.operator.WindowExec.allowIncompatible=true. For more information, refer to the Comet Compatibility Guide (https://datafusion.apache.org/comet/user-guide/compatibility.html)., Partitioning and sorting specifications must be the same.] : +- CometColumnarToRow : +- CometSort : +- CometExchange @@ -109,7 +109,7 @@ : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim +- BroadcastExchange +- Project - +- Window [COMET: Window expressions are not supported] + +- Window [COMET: WindowExec is not fully compatible with Spark (Native WindowExec has known correctness issues). To enable it anyway, set spark.comet.operator.WindowExec.allowIncompatible=true. For more information, refer to the Comet Compatibility Guide (https://datafusion.apache.org/comet/user-guide/compatibility.html)., Partitioning and sorting specifications must be the same.] +- CometColumnarToRow +- CometSort +- CometExchange @@ -125,7 +125,7 @@ : +- Project : +- BroadcastHashJoin : :- Project - : : +- Window [COMET: Window expressions are not supported] + : : +- Window [COMET: WindowExec is not fully compatible with Spark (Native WindowExec has known correctness issues). To enable it anyway, set spark.comet.operator.WindowExec.allowIncompatible=true. For more information, refer to the Comet Compatibility Guide (https://datafusion.apache.org/comet/user-guide/compatibility.html)., Partitioning and sorting specifications must be the same.] : : +- CometColumnarToRow : : +- CometSort : : +- CometExchange @@ -148,7 +148,7 @@ : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim : +- BroadcastExchange : +- Project - : +- Window [COMET: Window expressions are not supported] + : +- Window [COMET: WindowExec is not fully compatible with Spark (Native WindowExec has known correctness issues). To enable it anyway, set spark.comet.operator.WindowExec.allowIncompatible=true. For more information, refer to the Comet Compatibility Guide (https://datafusion.apache.org/comet/user-guide/compatibility.html)., Partitioning and sorting specifications must be the same.] : +- CometColumnarToRow : +- CometSort : +- CometExchange @@ -178,7 +178,7 @@ +- Project +- BroadcastHashJoin :- Project - : +- Window [COMET: Window expressions are not supported] + : +- Window [COMET: WindowExec is not fully compatible with Spark (Native WindowExec has known correctness issues). To enable it anyway, set spark.comet.operator.WindowExec.allowIncompatible=true. For more information, refer to the Comet Compatibility Guide (https://datafusion.apache.org/comet/user-guide/compatibility.html)., Partitioning and sorting specifications must be the same.] : +- CometColumnarToRow : +- CometSort : +- CometExchange @@ -196,7 +196,7 @@ : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim +- BroadcastExchange +- Project - +- Window [COMET: Window expressions are not supported] + +- Window [COMET: WindowExec is not fully compatible with Spark (Native WindowExec has known correctness issues). To enable it anyway, set spark.comet.operator.WindowExec.allowIncompatible=true. For more information, refer to the Comet Compatibility Guide (https://datafusion.apache.org/comet/user-guide/compatibility.html)., Partitioning and sorting specifications must be the same.] +- CometColumnarToRow +- CometSort +- CometExchange diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7-spark3_5/q51a/extended.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7-spark3_5/q51a/extended.txt index 15a87f66b6..b192555935 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7-spark3_5/q51a/extended.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7-spark3_5/q51a/extended.txt @@ -4,7 +4,7 @@ +- HashAggregate +- Project +- BroadcastHashJoin - :- Window [COMET: Window expressions are not supported] + :- Window [COMET: WindowExec is not fully compatible with Spark (Native WindowExec has known correctness issues). To enable it anyway, set spark.comet.operator.WindowExec.allowIncompatible=true. For more information, refer to the Comet Compatibility Guide (https://datafusion.apache.org/comet/user-guide/compatibility.html)., Partitioning and sorting specifications must be the same.] : +- CometColumnarToRow : +- CometSort : +- CometExchange @@ -20,7 +20,7 @@ : : +- Project : : +- BroadcastHashJoin : : :- Project - : : : +- Window [COMET: Window expressions are not supported] + : : : +- Window [COMET: WindowExec is not fully compatible with Spark (Native WindowExec has known correctness issues). To enable it anyway, set spark.comet.operator.WindowExec.allowIncompatible=true. For more information, refer to the Comet Compatibility Guide (https://datafusion.apache.org/comet/user-guide/compatibility.html)., Partitioning and sorting specifications must be the same.] : : : +- CometColumnarToRow : : : +- CometSort : : : +- CometExchange @@ -43,7 +43,7 @@ : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim : : +- BroadcastExchange : : +- Project - : : +- Window [COMET: Window expressions are not supported] + : : +- Window [COMET: WindowExec is not fully compatible with Spark (Native WindowExec has known correctness issues). To enable it anyway, set spark.comet.operator.WindowExec.allowIncompatible=true. For more information, refer to the Comet Compatibility Guide (https://datafusion.apache.org/comet/user-guide/compatibility.html)., Partitioning and sorting specifications must be the same.] : : +- CometColumnarToRow : : +- CometSort : : +- CometExchange @@ -73,7 +73,7 @@ : +- Project : +- BroadcastHashJoin : :- Project - : : +- Window [COMET: Window expressions are not supported] + : : +- Window [COMET: WindowExec is not fully compatible with Spark (Native WindowExec has known correctness issues). To enable it anyway, set spark.comet.operator.WindowExec.allowIncompatible=true. For more information, refer to the Comet Compatibility Guide (https://datafusion.apache.org/comet/user-guide/compatibility.html)., Partitioning and sorting specifications must be the same.] : : +- CometColumnarToRow : : +- CometSort : : +- CometExchange @@ -91,7 +91,7 @@ : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim : +- BroadcastExchange : +- Project - : +- Window [COMET: Window expressions are not supported] + : +- Window [COMET: WindowExec is not fully compatible with Spark (Native WindowExec has known correctness issues). To enable it anyway, set spark.comet.operator.WindowExec.allowIncompatible=true. For more information, refer to the Comet Compatibility Guide (https://datafusion.apache.org/comet/user-guide/compatibility.html)., Partitioning and sorting specifications must be the same.] : +- CometColumnarToRow : +- CometSort : +- CometExchange @@ -109,7 +109,7 @@ : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim +- BroadcastExchange +- Project - +- Window [COMET: Window expressions are not supported] + +- Window [COMET: WindowExec is not fully compatible with Spark (Native WindowExec has known correctness issues). To enable it anyway, set spark.comet.operator.WindowExec.allowIncompatible=true. For more information, refer to the Comet Compatibility Guide (https://datafusion.apache.org/comet/user-guide/compatibility.html)., Partitioning and sorting specifications must be the same.] +- CometColumnarToRow +- CometSort +- CometExchange @@ -125,7 +125,7 @@ : +- Project : +- BroadcastHashJoin : :- Project - : : +- Window [COMET: Window expressions are not supported] + : : +- Window [COMET: WindowExec is not fully compatible with Spark (Native WindowExec has known correctness issues). To enable it anyway, set spark.comet.operator.WindowExec.allowIncompatible=true. For more information, refer to the Comet Compatibility Guide (https://datafusion.apache.org/comet/user-guide/compatibility.html)., Partitioning and sorting specifications must be the same.] : : +- CometColumnarToRow : : +- CometSort : : +- CometExchange @@ -148,7 +148,7 @@ : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim : +- BroadcastExchange : +- Project - : +- Window [COMET: Window expressions are not supported] + : +- Window [COMET: WindowExec is not fully compatible with Spark (Native WindowExec has known correctness issues). To enable it anyway, set spark.comet.operator.WindowExec.allowIncompatible=true. For more information, refer to the Comet Compatibility Guide (https://datafusion.apache.org/comet/user-guide/compatibility.html)., Partitioning and sorting specifications must be the same.] : +- CometColumnarToRow : +- CometSort : +- CometExchange @@ -178,7 +178,7 @@ +- Project +- BroadcastHashJoin :- Project - : +- Window [COMET: Window expressions are not supported] + : +- Window [COMET: WindowExec is not fully compatible with Spark (Native WindowExec has known correctness issues). To enable it anyway, set spark.comet.operator.WindowExec.allowIncompatible=true. For more information, refer to the Comet Compatibility Guide (https://datafusion.apache.org/comet/user-guide/compatibility.html)., Partitioning and sorting specifications must be the same.] : +- CometColumnarToRow : +- CometSort : +- CometExchange @@ -196,7 +196,7 @@ : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim +- BroadcastExchange +- Project - +- Window [COMET: Window expressions are not supported] + +- Window [COMET: WindowExec is not fully compatible with Spark (Native WindowExec has known correctness issues). To enable it anyway, set spark.comet.operator.WindowExec.allowIncompatible=true. For more information, refer to the Comet Compatibility Guide (https://datafusion.apache.org/comet/user-guide/compatibility.html)., Partitioning and sorting specifications must be the same.] +- CometColumnarToRow +- CometSort +- CometExchange diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7-spark3_5/q57.native_iceberg_compat/extended.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7-spark3_5/q57.native_iceberg_compat/extended.txt index 7bee0e61e8..eb8f4ab303 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7-spark3_5/q57.native_iceberg_compat/extended.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7-spark3_5/q57.native_iceberg_compat/extended.txt @@ -7,7 +7,7 @@ : : +- Filter : : +- Window : : +- Filter - : : +- Window [COMET: Window expressions are not supported] + : : +- Window [COMET: WindowExec is not fully compatible with Spark (Native WindowExec has known correctness issues). To enable it anyway, set spark.comet.operator.WindowExec.allowIncompatible=true. For more information, refer to the Comet Compatibility Guide (https://datafusion.apache.org/comet/user-guide/compatibility.html).] : : +- CometColumnarToRow : : +- CometSort : : +- CometExchange @@ -39,7 +39,7 @@ : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.call_center : +- BroadcastExchange : +- Project - : +- Window [COMET: Window expressions are not supported] + : +- Window [COMET: WindowExec is not fully compatible with Spark (Native WindowExec has known correctness issues). To enable it anyway, set spark.comet.operator.WindowExec.allowIncompatible=true. For more information, refer to the Comet Compatibility Guide (https://datafusion.apache.org/comet/user-guide/compatibility.html).] : +- CometColumnarToRow : +- CometSort : +- CometExchange @@ -71,7 +71,7 @@ : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.call_center +- BroadcastExchange +- Project - +- Window [COMET: Window expressions are not supported] + +- Window [COMET: WindowExec is not fully compatible with Spark (Native WindowExec has known correctness issues). To enable it anyway, set spark.comet.operator.WindowExec.allowIncompatible=true. For more information, refer to the Comet Compatibility Guide (https://datafusion.apache.org/comet/user-guide/compatibility.html).] +- CometColumnarToRow +- CometSort +- CometExchange diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7-spark3_5/q57/extended.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7-spark3_5/q57/extended.txt index 7bee0e61e8..eb8f4ab303 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7-spark3_5/q57/extended.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7-spark3_5/q57/extended.txt @@ -7,7 +7,7 @@ : : +- Filter : : +- Window : : +- Filter - : : +- Window [COMET: Window expressions are not supported] + : : +- Window [COMET: WindowExec is not fully compatible with Spark (Native WindowExec has known correctness issues). To enable it anyway, set spark.comet.operator.WindowExec.allowIncompatible=true. For more information, refer to the Comet Compatibility Guide (https://datafusion.apache.org/comet/user-guide/compatibility.html).] : : +- CometColumnarToRow : : +- CometSort : : +- CometExchange @@ -39,7 +39,7 @@ : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.call_center : +- BroadcastExchange : +- Project - : +- Window [COMET: Window expressions are not supported] + : +- Window [COMET: WindowExec is not fully compatible with Spark (Native WindowExec has known correctness issues). To enable it anyway, set spark.comet.operator.WindowExec.allowIncompatible=true. For more information, refer to the Comet Compatibility Guide (https://datafusion.apache.org/comet/user-guide/compatibility.html).] : +- CometColumnarToRow : +- CometSort : +- CometExchange @@ -71,7 +71,7 @@ : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.call_center +- BroadcastExchange +- Project - +- Window [COMET: Window expressions are not supported] + +- Window [COMET: WindowExec is not fully compatible with Spark (Native WindowExec has known correctness issues). To enable it anyway, set spark.comet.operator.WindowExec.allowIncompatible=true. For more information, refer to the Comet Compatibility Guide (https://datafusion.apache.org/comet/user-guide/compatibility.html).] +- CometColumnarToRow +- CometSort +- CometExchange diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7-spark3_5/q70a.native_iceberg_compat/extended.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7-spark3_5/q70a.native_iceberg_compat/extended.txt index 1574f09c56..9dbf07afd1 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7-spark3_5/q70a.native_iceberg_compat/extended.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7-spark3_5/q70a.native_iceberg_compat/extended.txt @@ -1,6 +1,6 @@ TakeOrderedAndProject [COMET: ] +- Project - +- Window [COMET: Window expressions are not supported] + +- Window [COMET: WindowExec is not fully compatible with Spark (Native WindowExec has known correctness issues). To enable it anyway, set spark.comet.operator.WindowExec.allowIncompatible=true. For more information, refer to the Comet Compatibility Guide (https://datafusion.apache.org/comet/user-guide/compatibility.html).] +- CometColumnarToRow +- CometSort +- CometExchange diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7-spark3_5/q70a/extended.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7-spark3_5/q70a/extended.txt index 1574f09c56..9dbf07afd1 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7-spark3_5/q70a/extended.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7-spark3_5/q70a/extended.txt @@ -1,6 +1,6 @@ TakeOrderedAndProject [COMET: ] +- Project - +- Window [COMET: Window expressions are not supported] + +- Window [COMET: WindowExec is not fully compatible with Spark (Native WindowExec has known correctness issues). To enable it anyway, set spark.comet.operator.WindowExec.allowIncompatible=true. For more information, refer to the Comet Compatibility Guide (https://datafusion.apache.org/comet/user-guide/compatibility.html).] +- CometColumnarToRow +- CometSort +- CometExchange diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7-spark3_5/q86a.native_iceberg_compat/extended.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7-spark3_5/q86a.native_iceberg_compat/extended.txt index 6dd65034ce..b2b5b4d63d 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7-spark3_5/q86a.native_iceberg_compat/extended.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7-spark3_5/q86a.native_iceberg_compat/extended.txt @@ -1,6 +1,6 @@ TakeOrderedAndProject [COMET: ] +- Project - +- Window [COMET: Window expressions are not supported] + +- Window [COMET: WindowExec is not fully compatible with Spark (Native WindowExec has known correctness issues). To enable it anyway, set spark.comet.operator.WindowExec.allowIncompatible=true. For more information, refer to the Comet Compatibility Guide (https://datafusion.apache.org/comet/user-guide/compatibility.html).] +- CometColumnarToRow +- CometSort +- CometExchange diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7-spark3_5/q86a/extended.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7-spark3_5/q86a/extended.txt index 6dd65034ce..b2b5b4d63d 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7-spark3_5/q86a/extended.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7-spark3_5/q86a/extended.txt @@ -1,6 +1,6 @@ TakeOrderedAndProject [COMET: ] +- Project - +- Window [COMET: Window expressions are not supported] + +- Window [COMET: WindowExec is not fully compatible with Spark (Native WindowExec has known correctness issues). To enable it anyway, set spark.comet.operator.WindowExec.allowIncompatible=true. For more information, refer to the Comet Compatibility Guide (https://datafusion.apache.org/comet/user-guide/compatibility.html).] +- CometColumnarToRow +- CometSort +- CometExchange diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7-spark3_5/q98.native_iceberg_compat/extended.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7-spark3_5/q98.native_iceberg_compat/extended.txt index 54a4d3c1b7..3dbaf2e346 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7-spark3_5/q98.native_iceberg_compat/extended.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7-spark3_5/q98.native_iceberg_compat/extended.txt @@ -2,7 +2,7 @@ CometColumnarToRow +- CometSort +- CometColumnarExchange +- Project - +- Window [COMET: Window expressions are not supported] + +- Window [COMET: WindowExec is not fully compatible with Spark (Native WindowExec has known correctness issues). To enable it anyway, set spark.comet.operator.WindowExec.allowIncompatible=true. For more information, refer to the Comet Compatibility Guide (https://datafusion.apache.org/comet/user-guide/compatibility.html).] +- CometColumnarToRow +- CometSort +- CometExchange diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7-spark3_5/q98/extended.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7-spark3_5/q98/extended.txt index 54a4d3c1b7..3dbaf2e346 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7-spark3_5/q98/extended.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7-spark3_5/q98/extended.txt @@ -2,7 +2,7 @@ CometColumnarToRow +- CometSort +- CometColumnarExchange +- Project - +- Window [COMET: Window expressions are not supported] + +- Window [COMET: WindowExec is not fully compatible with Spark (Native WindowExec has known correctness issues). To enable it anyway, set spark.comet.operator.WindowExec.allowIncompatible=true. For more information, refer to the Comet Compatibility Guide (https://datafusion.apache.org/comet/user-guide/compatibility.html).] +- CometColumnarToRow +- CometSort +- CometExchange From 74ee98e0b90c040b4889bcc24f0691c6940374af Mon Sep 17 00:00:00 2001 From: Andy Grove Date: Sat, 8 Nov 2025 13:11:15 -0700 Subject: [PATCH 32/44] 3.4 golden files --- .../q12.native_iceberg_compat/extended.txt | 2 +- .../approved-plans-v1_4/q12/extended.txt | 2 +- .../q20.native_iceberg_compat/extended.txt | 2 +- .../approved-plans-v1_4/q20/extended.txt | 2 +- .../q36.native_iceberg_compat/extended.txt | 2 +- .../approved-plans-v1_4/q36/extended.txt | 2 +- .../q44.native_iceberg_compat/extended.txt | 4 ++-- .../approved-plans-v1_4/q44/extended.txt | 4 ++-- .../q47.native_iceberg_compat/extended.txt | 6 +++--- .../approved-plans-v1_4/q47/extended.txt | 6 +++--- .../q49.native_iceberg_compat/extended.txt | 6 +++--- .../approved-plans-v1_4/q49/extended.txt | 6 +++--- .../q51.native_iceberg_compat/extended.txt | 6 +++--- .../approved-plans-v1_4/q51/extended.txt | 6 +++--- .../q53.native_iceberg_compat/extended.txt | 2 +- .../approved-plans-v1_4/q53/extended.txt | 2 +- .../q57.native_iceberg_compat/extended.txt | 6 +++--- .../approved-plans-v1_4/q57/extended.txt | 6 +++--- .../q63.native_iceberg_compat/extended.txt | 2 +- .../approved-plans-v1_4/q63/extended.txt | 2 +- .../q67.native_iceberg_compat/extended.txt | 2 +- .../approved-plans-v1_4/q67/extended.txt | 2 +- .../q70.native_iceberg_compat/extended.txt | 4 ++-- .../approved-plans-v1_4/q70/extended.txt | 4 ++-- .../q86.native_iceberg_compat/extended.txt | 2 +- .../approved-plans-v1_4/q86/extended.txt | 2 +- .../q89.native_iceberg_compat/extended.txt | 2 +- .../approved-plans-v1_4/q89/extended.txt | 2 +- .../q98.native_iceberg_compat/extended.txt | 2 +- .../approved-plans-v1_4/q98/extended.txt | 2 +- .../q12.native_iceberg_compat/extended.txt | 2 +- .../approved-plans-v2_7/q12/extended.txt | 2 +- .../q20.native_iceberg_compat/extended.txt | 2 +- .../approved-plans-v2_7/q20/extended.txt | 2 +- .../q36a.native_iceberg_compat/extended.txt | 2 +- .../approved-plans-v2_7/q36a/extended.txt | 2 +- .../q47.native_iceberg_compat/extended.txt | 6 +++--- .../approved-plans-v2_7/q47/extended.txt | 6 +++--- .../q49.native_iceberg_compat/extended.txt | 6 +++--- .../approved-plans-v2_7/q49/extended.txt | 6 +++--- .../q51a.native_iceberg_compat/extended.txt | 20 +++++++++---------- .../approved-plans-v2_7/q51a/extended.txt | 20 +++++++++---------- .../q57.native_iceberg_compat/extended.txt | 6 +++--- .../approved-plans-v2_7/q57/extended.txt | 6 +++--- .../q67a.native_iceberg_compat/extended.txt | 2 +- .../approved-plans-v2_7/q67a/extended.txt | 2 +- .../q70a.native_iceberg_compat/extended.txt | 8 ++++---- .../approved-plans-v2_7/q70a/extended.txt | 8 ++++---- .../q86a.native_iceberg_compat/extended.txt | 2 +- .../approved-plans-v2_7/q86a/extended.txt | 2 +- .../q98.native_iceberg_compat/extended.txt | 2 +- .../approved-plans-v2_7/q98/extended.txt | 2 +- 52 files changed, 108 insertions(+), 108 deletions(-) diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q12.native_iceberg_compat/extended.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q12.native_iceberg_compat/extended.txt index 9ddd3aeb38..9ea26caab6 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q12.native_iceberg_compat/extended.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q12.native_iceberg_compat/extended.txt @@ -1,6 +1,6 @@ TakeOrderedAndProject [COMET: ] +- Project - +- Window [COMET: Window expressions are not supported] + +- Window [COMET: WindowExec is not fully compatible with Spark (Native WindowExec has known correctness issues). To enable it anyway, set spark.comet.operator.WindowExec.allowIncompatible=true. For more information, refer to the Comet Compatibility Guide (https://datafusion.apache.org/comet/user-guide/compatibility.html).] +- CometColumnarToRow +- CometSort +- CometExchange diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q12/extended.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q12/extended.txt index 9ddd3aeb38..9ea26caab6 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q12/extended.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q12/extended.txt @@ -1,6 +1,6 @@ TakeOrderedAndProject [COMET: ] +- Project - +- Window [COMET: Window expressions are not supported] + +- Window [COMET: WindowExec is not fully compatible with Spark (Native WindowExec has known correctness issues). To enable it anyway, set spark.comet.operator.WindowExec.allowIncompatible=true. For more information, refer to the Comet Compatibility Guide (https://datafusion.apache.org/comet/user-guide/compatibility.html).] +- CometColumnarToRow +- CometSort +- CometExchange diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q20.native_iceberg_compat/extended.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q20.native_iceberg_compat/extended.txt index 400633ed9f..1b24b68f14 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q20.native_iceberg_compat/extended.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q20.native_iceberg_compat/extended.txt @@ -1,6 +1,6 @@ TakeOrderedAndProject [COMET: ] +- Project - +- Window [COMET: Window expressions are not supported] + +- Window [COMET: WindowExec is not fully compatible with Spark (Native WindowExec has known correctness issues). To enable it anyway, set spark.comet.operator.WindowExec.allowIncompatible=true. For more information, refer to the Comet Compatibility Guide (https://datafusion.apache.org/comet/user-guide/compatibility.html).] +- CometColumnarToRow +- CometSort +- CometExchange diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q20/extended.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q20/extended.txt index 400633ed9f..1b24b68f14 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q20/extended.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q20/extended.txt @@ -1,6 +1,6 @@ TakeOrderedAndProject [COMET: ] +- Project - +- Window [COMET: Window expressions are not supported] + +- Window [COMET: WindowExec is not fully compatible with Spark (Native WindowExec has known correctness issues). To enable it anyway, set spark.comet.operator.WindowExec.allowIncompatible=true. For more information, refer to the Comet Compatibility Guide (https://datafusion.apache.org/comet/user-guide/compatibility.html).] +- CometColumnarToRow +- CometSort +- CometExchange diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q36.native_iceberg_compat/extended.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q36.native_iceberg_compat/extended.txt index 31fe2cc028..63050a1b8f 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q36.native_iceberg_compat/extended.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q36.native_iceberg_compat/extended.txt @@ -1,6 +1,6 @@ TakeOrderedAndProject [COMET: ] +- Project - +- Window [COMET: Window expressions are not supported] + +- Window [COMET: WindowExec is not fully compatible with Spark (Native WindowExec has known correctness issues). To enable it anyway, set spark.comet.operator.WindowExec.allowIncompatible=true. For more information, refer to the Comet Compatibility Guide (https://datafusion.apache.org/comet/user-guide/compatibility.html).] +- CometColumnarToRow +- CometSort +- CometExchange diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q36/extended.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q36/extended.txt index 31fe2cc028..63050a1b8f 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q36/extended.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q36/extended.txt @@ -1,6 +1,6 @@ TakeOrderedAndProject [COMET: ] +- Project - +- Window [COMET: Window expressions are not supported] + +- Window [COMET: WindowExec is not fully compatible with Spark (Native WindowExec has known correctness issues). To enable it anyway, set spark.comet.operator.WindowExec.allowIncompatible=true. For more information, refer to the Comet Compatibility Guide (https://datafusion.apache.org/comet/user-guide/compatibility.html).] +- CometColumnarToRow +- CometSort +- CometExchange diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q44.native_iceberg_compat/extended.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q44.native_iceberg_compat/extended.txt index ff284b0f37..a966fcdaec 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q44.native_iceberg_compat/extended.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q44.native_iceberg_compat/extended.txt @@ -8,7 +8,7 @@ : : :- Sort : : : +- Project : : : +- Filter - : : : +- Window [COMET: Window expressions are not supported] + : : : +- Window [COMET: WindowExec is not fully compatible with Spark (Native WindowExec has known correctness issues). To enable it anyway, set spark.comet.operator.WindowExec.allowIncompatible=true. For more information, refer to the Comet Compatibility Guide (https://datafusion.apache.org/comet/user-guide/compatibility.html).] : : : +- CometColumnarToRow : : : +- CometSort : : : +- CometExchange @@ -30,7 +30,7 @@ : : +- Sort : : +- Project : : +- Filter - : : +- Window [COMET: Window expressions are not supported] + : : +- Window [COMET: WindowExec is not fully compatible with Spark (Native WindowExec has known correctness issues). To enable it anyway, set spark.comet.operator.WindowExec.allowIncompatible=true. For more information, refer to the Comet Compatibility Guide (https://datafusion.apache.org/comet/user-guide/compatibility.html).] : : +- CometColumnarToRow : : +- CometSort : : +- CometExchange diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q44/extended.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q44/extended.txt index ff284b0f37..a966fcdaec 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q44/extended.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q44/extended.txt @@ -8,7 +8,7 @@ : : :- Sort : : : +- Project : : : +- Filter - : : : +- Window [COMET: Window expressions are not supported] + : : : +- Window [COMET: WindowExec is not fully compatible with Spark (Native WindowExec has known correctness issues). To enable it anyway, set spark.comet.operator.WindowExec.allowIncompatible=true. For more information, refer to the Comet Compatibility Guide (https://datafusion.apache.org/comet/user-guide/compatibility.html).] : : : +- CometColumnarToRow : : : +- CometSort : : : +- CometExchange @@ -30,7 +30,7 @@ : : +- Sort : : +- Project : : +- Filter - : : +- Window [COMET: Window expressions are not supported] + : : +- Window [COMET: WindowExec is not fully compatible with Spark (Native WindowExec has known correctness issues). To enable it anyway, set spark.comet.operator.WindowExec.allowIncompatible=true. For more information, refer to the Comet Compatibility Guide (https://datafusion.apache.org/comet/user-guide/compatibility.html).] : : +- CometColumnarToRow : : +- CometSort : : +- CometExchange diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q47.native_iceberg_compat/extended.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q47.native_iceberg_compat/extended.txt index 11fbf5bd11..d9f357c1c1 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q47.native_iceberg_compat/extended.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q47.native_iceberg_compat/extended.txt @@ -7,7 +7,7 @@ : : +- Filter : : +- Window : : +- Filter - : : +- Window [COMET: Window expressions are not supported] + : : +- Window [COMET: WindowExec is not fully compatible with Spark (Native WindowExec has known correctness issues). To enable it anyway, set spark.comet.operator.WindowExec.allowIncompatible=true. For more information, refer to the Comet Compatibility Guide (https://datafusion.apache.org/comet/user-guide/compatibility.html).] : : +- CometColumnarToRow : : +- CometSort : : +- CometExchange @@ -39,7 +39,7 @@ : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.store : +- BroadcastExchange : +- Project - : +- Window [COMET: Window expressions are not supported] + : +- Window [COMET: WindowExec is not fully compatible with Spark (Native WindowExec has known correctness issues). To enable it anyway, set spark.comet.operator.WindowExec.allowIncompatible=true. For more information, refer to the Comet Compatibility Guide (https://datafusion.apache.org/comet/user-guide/compatibility.html).] : +- CometColumnarToRow : +- CometSort : +- CometExchange @@ -71,7 +71,7 @@ : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.store +- BroadcastExchange +- Project - +- Window [COMET: Window expressions are not supported] + +- Window [COMET: WindowExec is not fully compatible with Spark (Native WindowExec has known correctness issues). To enable it anyway, set spark.comet.operator.WindowExec.allowIncompatible=true. For more information, refer to the Comet Compatibility Guide (https://datafusion.apache.org/comet/user-guide/compatibility.html).] +- CometColumnarToRow +- CometSort +- CometExchange diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q47/extended.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q47/extended.txt index 11fbf5bd11..d9f357c1c1 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q47/extended.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q47/extended.txt @@ -7,7 +7,7 @@ : : +- Filter : : +- Window : : +- Filter - : : +- Window [COMET: Window expressions are not supported] + : : +- Window [COMET: WindowExec is not fully compatible with Spark (Native WindowExec has known correctness issues). To enable it anyway, set spark.comet.operator.WindowExec.allowIncompatible=true. For more information, refer to the Comet Compatibility Guide (https://datafusion.apache.org/comet/user-guide/compatibility.html).] : : +- CometColumnarToRow : : +- CometSort : : +- CometExchange @@ -39,7 +39,7 @@ : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.store : +- BroadcastExchange : +- Project - : +- Window [COMET: Window expressions are not supported] + : +- Window [COMET: WindowExec is not fully compatible with Spark (Native WindowExec has known correctness issues). To enable it anyway, set spark.comet.operator.WindowExec.allowIncompatible=true. For more information, refer to the Comet Compatibility Guide (https://datafusion.apache.org/comet/user-guide/compatibility.html).] : +- CometColumnarToRow : +- CometSort : +- CometExchange @@ -71,7 +71,7 @@ : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.store +- BroadcastExchange +- Project - +- Window [COMET: Window expressions are not supported] + +- Window [COMET: WindowExec is not fully compatible with Spark (Native WindowExec has known correctness issues). To enable it anyway, set spark.comet.operator.WindowExec.allowIncompatible=true. For more information, refer to the Comet Compatibility Guide (https://datafusion.apache.org/comet/user-guide/compatibility.html).] +- CometColumnarToRow +- CometSort +- CometExchange diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q49.native_iceberg_compat/extended.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q49.native_iceberg_compat/extended.txt index 734b0ce7bb..75684a9669 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q49.native_iceberg_compat/extended.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q49.native_iceberg_compat/extended.txt @@ -8,7 +8,7 @@ CometColumnarToRow : +- Filter : +- Window : +- Sort - : +- Window [COMET: Window expressions are not supported] + : +- Window [COMET: WindowExec is not fully compatible with Spark (Native WindowExec has known correctness issues). To enable it anyway, set spark.comet.operator.WindowExec.allowIncompatible=true. For more information, refer to the Comet Compatibility Guide (https://datafusion.apache.org/comet/user-guide/compatibility.html).] : +- CometColumnarToRow : +- CometSort : +- CometExchange @@ -40,7 +40,7 @@ CometColumnarToRow : +- Filter : +- Window : +- Sort - : +- Window [COMET: Window expressions are not supported] + : +- Window [COMET: WindowExec is not fully compatible with Spark (Native WindowExec has known correctness issues). To enable it anyway, set spark.comet.operator.WindowExec.allowIncompatible=true. For more information, refer to the Comet Compatibility Guide (https://datafusion.apache.org/comet/user-guide/compatibility.html).] : +- CometColumnarToRow : +- CometSort : +- CometExchange @@ -67,7 +67,7 @@ CometColumnarToRow +- Filter +- Window +- Sort - +- Window [COMET: Window expressions are not supported] + +- Window [COMET: WindowExec is not fully compatible with Spark (Native WindowExec has known correctness issues). To enable it anyway, set spark.comet.operator.WindowExec.allowIncompatible=true. For more information, refer to the Comet Compatibility Guide (https://datafusion.apache.org/comet/user-guide/compatibility.html).] +- CometColumnarToRow +- CometSort +- CometExchange diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q49/extended.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q49/extended.txt index 734b0ce7bb..75684a9669 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q49/extended.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q49/extended.txt @@ -8,7 +8,7 @@ CometColumnarToRow : +- Filter : +- Window : +- Sort - : +- Window [COMET: Window expressions are not supported] + : +- Window [COMET: WindowExec is not fully compatible with Spark (Native WindowExec has known correctness issues). To enable it anyway, set spark.comet.operator.WindowExec.allowIncompatible=true. For more information, refer to the Comet Compatibility Guide (https://datafusion.apache.org/comet/user-guide/compatibility.html).] : +- CometColumnarToRow : +- CometSort : +- CometExchange @@ -40,7 +40,7 @@ CometColumnarToRow : +- Filter : +- Window : +- Sort - : +- Window [COMET: Window expressions are not supported] + : +- Window [COMET: WindowExec is not fully compatible with Spark (Native WindowExec has known correctness issues). To enable it anyway, set spark.comet.operator.WindowExec.allowIncompatible=true. For more information, refer to the Comet Compatibility Guide (https://datafusion.apache.org/comet/user-guide/compatibility.html).] : +- CometColumnarToRow : +- CometSort : +- CometExchange @@ -67,7 +67,7 @@ CometColumnarToRow +- Filter +- Window +- Sort - +- Window [COMET: Window expressions are not supported] + +- Window [COMET: WindowExec is not fully compatible with Spark (Native WindowExec has known correctness issues). To enable it anyway, set spark.comet.operator.WindowExec.allowIncompatible=true. For more information, refer to the Comet Compatibility Guide (https://datafusion.apache.org/comet/user-guide/compatibility.html).] +- CometColumnarToRow +- CometSort +- CometExchange diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q51.native_iceberg_compat/extended.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q51.native_iceberg_compat/extended.txt index da1ac9c0a3..d55771ac8a 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q51.native_iceberg_compat/extended.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q51.native_iceberg_compat/extended.txt @@ -1,6 +1,6 @@ TakeOrderedAndProject [COMET: ] +- Filter - +- Window [COMET: Window expressions are not supported] + +- Window [COMET: WindowExec is not fully compatible with Spark (Native WindowExec has known correctness issues). To enable it anyway, set spark.comet.operator.WindowExec.allowIncompatible=true. For more information, refer to the Comet Compatibility Guide (https://datafusion.apache.org/comet/user-guide/compatibility.html)., Partitioning and sorting specifications must be the same.] +- CometColumnarToRow +- CometSort +- CometExchange @@ -9,7 +9,7 @@ :- CometSort : +- CometColumnarExchange : +- Project - : +- Window [COMET: Window expressions are not supported] + : +- Window [COMET: WindowExec is not fully compatible with Spark (Native WindowExec has known correctness issues). To enable it anyway, set spark.comet.operator.WindowExec.allowIncompatible=true. For more information, refer to the Comet Compatibility Guide (https://datafusion.apache.org/comet/user-guide/compatibility.html)., Partitioning and sorting specifications must be the same.] : +- CometColumnarToRow : +- CometSort : +- CometExchange @@ -33,7 +33,7 @@ +- CometSort +- CometColumnarExchange +- Project - +- Window [COMET: Window expressions are not supported] + +- Window [COMET: WindowExec is not fully compatible with Spark (Native WindowExec has known correctness issues). To enable it anyway, set spark.comet.operator.WindowExec.allowIncompatible=true. For more information, refer to the Comet Compatibility Guide (https://datafusion.apache.org/comet/user-guide/compatibility.html)., Partitioning and sorting specifications must be the same.] +- CometColumnarToRow +- CometSort +- CometExchange diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q51/extended.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q51/extended.txt index da1ac9c0a3..d55771ac8a 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q51/extended.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q51/extended.txt @@ -1,6 +1,6 @@ TakeOrderedAndProject [COMET: ] +- Filter - +- Window [COMET: Window expressions are not supported] + +- Window [COMET: WindowExec is not fully compatible with Spark (Native WindowExec has known correctness issues). To enable it anyway, set spark.comet.operator.WindowExec.allowIncompatible=true. For more information, refer to the Comet Compatibility Guide (https://datafusion.apache.org/comet/user-guide/compatibility.html)., Partitioning and sorting specifications must be the same.] +- CometColumnarToRow +- CometSort +- CometExchange @@ -9,7 +9,7 @@ :- CometSort : +- CometColumnarExchange : +- Project - : +- Window [COMET: Window expressions are not supported] + : +- Window [COMET: WindowExec is not fully compatible with Spark (Native WindowExec has known correctness issues). To enable it anyway, set spark.comet.operator.WindowExec.allowIncompatible=true. For more information, refer to the Comet Compatibility Guide (https://datafusion.apache.org/comet/user-guide/compatibility.html)., Partitioning and sorting specifications must be the same.] : +- CometColumnarToRow : +- CometSort : +- CometExchange @@ -33,7 +33,7 @@ +- CometSort +- CometColumnarExchange +- Project - +- Window [COMET: Window expressions are not supported] + +- Window [COMET: WindowExec is not fully compatible with Spark (Native WindowExec has known correctness issues). To enable it anyway, set spark.comet.operator.WindowExec.allowIncompatible=true. For more information, refer to the Comet Compatibility Guide (https://datafusion.apache.org/comet/user-guide/compatibility.html)., Partitioning and sorting specifications must be the same.] +- CometColumnarToRow +- CometSort +- CometExchange diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q53.native_iceberg_compat/extended.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q53.native_iceberg_compat/extended.txt index 004828cdfb..3a68550e3d 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q53.native_iceberg_compat/extended.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q53.native_iceberg_compat/extended.txt @@ -1,7 +1,7 @@ TakeOrderedAndProject [COMET: ] +- Project +- Filter - +- Window [COMET: Window expressions are not supported] + +- Window [COMET: WindowExec is not fully compatible with Spark (Native WindowExec has known correctness issues). To enable it anyway, set spark.comet.operator.WindowExec.allowIncompatible=true. For more information, refer to the Comet Compatibility Guide (https://datafusion.apache.org/comet/user-guide/compatibility.html).] +- CometColumnarToRow +- CometSort +- CometExchange diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q53/extended.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q53/extended.txt index 004828cdfb..3a68550e3d 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q53/extended.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q53/extended.txt @@ -1,7 +1,7 @@ TakeOrderedAndProject [COMET: ] +- Project +- Filter - +- Window [COMET: Window expressions are not supported] + +- Window [COMET: WindowExec is not fully compatible with Spark (Native WindowExec has known correctness issues). To enable it anyway, set spark.comet.operator.WindowExec.allowIncompatible=true. For more information, refer to the Comet Compatibility Guide (https://datafusion.apache.org/comet/user-guide/compatibility.html).] +- CometColumnarToRow +- CometSort +- CometExchange diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q57.native_iceberg_compat/extended.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q57.native_iceberg_compat/extended.txt index 7bee0e61e8..eb8f4ab303 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q57.native_iceberg_compat/extended.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q57.native_iceberg_compat/extended.txt @@ -7,7 +7,7 @@ : : +- Filter : : +- Window : : +- Filter - : : +- Window [COMET: Window expressions are not supported] + : : +- Window [COMET: WindowExec is not fully compatible with Spark (Native WindowExec has known correctness issues). To enable it anyway, set spark.comet.operator.WindowExec.allowIncompatible=true. For more information, refer to the Comet Compatibility Guide (https://datafusion.apache.org/comet/user-guide/compatibility.html).] : : +- CometColumnarToRow : : +- CometSort : : +- CometExchange @@ -39,7 +39,7 @@ : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.call_center : +- BroadcastExchange : +- Project - : +- Window [COMET: Window expressions are not supported] + : +- Window [COMET: WindowExec is not fully compatible with Spark (Native WindowExec has known correctness issues). To enable it anyway, set spark.comet.operator.WindowExec.allowIncompatible=true. For more information, refer to the Comet Compatibility Guide (https://datafusion.apache.org/comet/user-guide/compatibility.html).] : +- CometColumnarToRow : +- CometSort : +- CometExchange @@ -71,7 +71,7 @@ : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.call_center +- BroadcastExchange +- Project - +- Window [COMET: Window expressions are not supported] + +- Window [COMET: WindowExec is not fully compatible with Spark (Native WindowExec has known correctness issues). To enable it anyway, set spark.comet.operator.WindowExec.allowIncompatible=true. For more information, refer to the Comet Compatibility Guide (https://datafusion.apache.org/comet/user-guide/compatibility.html).] +- CometColumnarToRow +- CometSort +- CometExchange diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q57/extended.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q57/extended.txt index 7bee0e61e8..eb8f4ab303 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q57/extended.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q57/extended.txt @@ -7,7 +7,7 @@ : : +- Filter : : +- Window : : +- Filter - : : +- Window [COMET: Window expressions are not supported] + : : +- Window [COMET: WindowExec is not fully compatible with Spark (Native WindowExec has known correctness issues). To enable it anyway, set spark.comet.operator.WindowExec.allowIncompatible=true. For more information, refer to the Comet Compatibility Guide (https://datafusion.apache.org/comet/user-guide/compatibility.html).] : : +- CometColumnarToRow : : +- CometSort : : +- CometExchange @@ -39,7 +39,7 @@ : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.call_center : +- BroadcastExchange : +- Project - : +- Window [COMET: Window expressions are not supported] + : +- Window [COMET: WindowExec is not fully compatible with Spark (Native WindowExec has known correctness issues). To enable it anyway, set spark.comet.operator.WindowExec.allowIncompatible=true. For more information, refer to the Comet Compatibility Guide (https://datafusion.apache.org/comet/user-guide/compatibility.html).] : +- CometColumnarToRow : +- CometSort : +- CometExchange @@ -71,7 +71,7 @@ : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.call_center +- BroadcastExchange +- Project - +- Window [COMET: Window expressions are not supported] + +- Window [COMET: WindowExec is not fully compatible with Spark (Native WindowExec has known correctness issues). To enable it anyway, set spark.comet.operator.WindowExec.allowIncompatible=true. For more information, refer to the Comet Compatibility Guide (https://datafusion.apache.org/comet/user-guide/compatibility.html).] +- CometColumnarToRow +- CometSort +- CometExchange diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q63.native_iceberg_compat/extended.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q63.native_iceberg_compat/extended.txt index 004828cdfb..3a68550e3d 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q63.native_iceberg_compat/extended.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q63.native_iceberg_compat/extended.txt @@ -1,7 +1,7 @@ TakeOrderedAndProject [COMET: ] +- Project +- Filter - +- Window [COMET: Window expressions are not supported] + +- Window [COMET: WindowExec is not fully compatible with Spark (Native WindowExec has known correctness issues). To enable it anyway, set spark.comet.operator.WindowExec.allowIncompatible=true. For more information, refer to the Comet Compatibility Guide (https://datafusion.apache.org/comet/user-guide/compatibility.html).] +- CometColumnarToRow +- CometSort +- CometExchange diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q63/extended.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q63/extended.txt index 004828cdfb..3a68550e3d 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q63/extended.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q63/extended.txt @@ -1,7 +1,7 @@ TakeOrderedAndProject [COMET: ] +- Project +- Filter - +- Window [COMET: Window expressions are not supported] + +- Window [COMET: WindowExec is not fully compatible with Spark (Native WindowExec has known correctness issues). To enable it anyway, set spark.comet.operator.WindowExec.allowIncompatible=true. For more information, refer to the Comet Compatibility Guide (https://datafusion.apache.org/comet/user-guide/compatibility.html).] +- CometColumnarToRow +- CometSort +- CometExchange diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q67.native_iceberg_compat/extended.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q67.native_iceberg_compat/extended.txt index 2d55d60ddc..ac51af90b6 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q67.native_iceberg_compat/extended.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q67.native_iceberg_compat/extended.txt @@ -1,6 +1,6 @@ TakeOrderedAndProject [COMET: ] +- Filter - +- Window [COMET: Window expressions are not supported] + +- Window [COMET: WindowExec is not fully compatible with Spark (Native WindowExec has known correctness issues). To enable it anyway, set spark.comet.operator.WindowExec.allowIncompatible=true. For more information, refer to the Comet Compatibility Guide (https://datafusion.apache.org/comet/user-guide/compatibility.html)., Partitioning and sorting specifications must be the same.] +- CometColumnarToRow +- CometSort +- CometExchange diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q67/extended.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q67/extended.txt index 2d55d60ddc..ac51af90b6 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q67/extended.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q67/extended.txt @@ -1,6 +1,6 @@ TakeOrderedAndProject [COMET: ] +- Filter - +- Window [COMET: Window expressions are not supported] + +- Window [COMET: WindowExec is not fully compatible with Spark (Native WindowExec has known correctness issues). To enable it anyway, set spark.comet.operator.WindowExec.allowIncompatible=true. For more information, refer to the Comet Compatibility Guide (https://datafusion.apache.org/comet/user-guide/compatibility.html)., Partitioning and sorting specifications must be the same.] +- CometColumnarToRow +- CometSort +- CometExchange diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q70.native_iceberg_compat/extended.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q70.native_iceberg_compat/extended.txt index a67bf27b57..9141654d2a 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q70.native_iceberg_compat/extended.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q70.native_iceberg_compat/extended.txt @@ -1,6 +1,6 @@ TakeOrderedAndProject [COMET: ] +- Project - +- Window [COMET: Window expressions are not supported] + +- Window [COMET: WindowExec is not fully compatible with Spark (Native WindowExec has known correctness issues). To enable it anyway, set spark.comet.operator.WindowExec.allowIncompatible=true. For more information, refer to the Comet Compatibility Guide (https://datafusion.apache.org/comet/user-guide/compatibility.html).] +- CometColumnarToRow +- CometSort +- CometColumnarExchange @@ -35,7 +35,7 @@ +- BroadcastExchange +- Project +- Filter - +- Window [COMET: Window expressions are not supported] + +- Window [COMET: WindowExec is not fully compatible with Spark (Native WindowExec has known correctness issues). To enable it anyway, set spark.comet.operator.WindowExec.allowIncompatible=true. For more information, refer to the Comet Compatibility Guide (https://datafusion.apache.org/comet/user-guide/compatibility.html)., Partitioning and sorting specifications must be the same.] +- CometColumnarToRow +- CometSort +- CometHashAggregate diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q70/extended.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q70/extended.txt index a67bf27b57..9141654d2a 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q70/extended.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q70/extended.txt @@ -1,6 +1,6 @@ TakeOrderedAndProject [COMET: ] +- Project - +- Window [COMET: Window expressions are not supported] + +- Window [COMET: WindowExec is not fully compatible with Spark (Native WindowExec has known correctness issues). To enable it anyway, set spark.comet.operator.WindowExec.allowIncompatible=true. For more information, refer to the Comet Compatibility Guide (https://datafusion.apache.org/comet/user-guide/compatibility.html).] +- CometColumnarToRow +- CometSort +- CometColumnarExchange @@ -35,7 +35,7 @@ +- BroadcastExchange +- Project +- Filter - +- Window [COMET: Window expressions are not supported] + +- Window [COMET: WindowExec is not fully compatible with Spark (Native WindowExec has known correctness issues). To enable it anyway, set spark.comet.operator.WindowExec.allowIncompatible=true. For more information, refer to the Comet Compatibility Guide (https://datafusion.apache.org/comet/user-guide/compatibility.html)., Partitioning and sorting specifications must be the same.] +- CometColumnarToRow +- CometSort +- CometHashAggregate diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q86.native_iceberg_compat/extended.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q86.native_iceberg_compat/extended.txt index 0b05b63e05..1193bed826 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q86.native_iceberg_compat/extended.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q86.native_iceberg_compat/extended.txt @@ -1,6 +1,6 @@ TakeOrderedAndProject [COMET: ] +- Project - +- Window [COMET: Window expressions are not supported] + +- Window [COMET: WindowExec is not fully compatible with Spark (Native WindowExec has known correctness issues). To enable it anyway, set spark.comet.operator.WindowExec.allowIncompatible=true. For more information, refer to the Comet Compatibility Guide (https://datafusion.apache.org/comet/user-guide/compatibility.html).] +- CometColumnarToRow +- CometSort +- CometExchange diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q86/extended.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q86/extended.txt index 0b05b63e05..1193bed826 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q86/extended.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q86/extended.txt @@ -1,6 +1,6 @@ TakeOrderedAndProject [COMET: ] +- Project - +- Window [COMET: Window expressions are not supported] + +- Window [COMET: WindowExec is not fully compatible with Spark (Native WindowExec has known correctness issues). To enable it anyway, set spark.comet.operator.WindowExec.allowIncompatible=true. For more information, refer to the Comet Compatibility Guide (https://datafusion.apache.org/comet/user-guide/compatibility.html).] +- CometColumnarToRow +- CometSort +- CometExchange diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q89.native_iceberg_compat/extended.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q89.native_iceberg_compat/extended.txt index 004828cdfb..3a68550e3d 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q89.native_iceberg_compat/extended.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q89.native_iceberg_compat/extended.txt @@ -1,7 +1,7 @@ TakeOrderedAndProject [COMET: ] +- Project +- Filter - +- Window [COMET: Window expressions are not supported] + +- Window [COMET: WindowExec is not fully compatible with Spark (Native WindowExec has known correctness issues). To enable it anyway, set spark.comet.operator.WindowExec.allowIncompatible=true. For more information, refer to the Comet Compatibility Guide (https://datafusion.apache.org/comet/user-guide/compatibility.html).] +- CometColumnarToRow +- CometSort +- CometExchange diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q89/extended.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q89/extended.txt index 004828cdfb..3a68550e3d 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q89/extended.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q89/extended.txt @@ -1,7 +1,7 @@ TakeOrderedAndProject [COMET: ] +- Project +- Filter - +- Window [COMET: Window expressions are not supported] + +- Window [COMET: WindowExec is not fully compatible with Spark (Native WindowExec has known correctness issues). To enable it anyway, set spark.comet.operator.WindowExec.allowIncompatible=true. For more information, refer to the Comet Compatibility Guide (https://datafusion.apache.org/comet/user-guide/compatibility.html).] +- CometColumnarToRow +- CometSort +- CometExchange diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q98.native_iceberg_compat/extended.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q98.native_iceberg_compat/extended.txt index c34e3cfede..4c972848e7 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q98.native_iceberg_compat/extended.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q98.native_iceberg_compat/extended.txt @@ -3,7 +3,7 @@ CometColumnarToRow +- CometSort +- CometColumnarExchange +- Project - +- Window [COMET: Window expressions are not supported] + +- Window [COMET: WindowExec is not fully compatible with Spark (Native WindowExec has known correctness issues). To enable it anyway, set spark.comet.operator.WindowExec.allowIncompatible=true. For more information, refer to the Comet Compatibility Guide (https://datafusion.apache.org/comet/user-guide/compatibility.html).] +- CometColumnarToRow +- CometSort +- CometExchange diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q98/extended.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q98/extended.txt index c34e3cfede..4c972848e7 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q98/extended.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q98/extended.txt @@ -3,7 +3,7 @@ CometColumnarToRow +- CometSort +- CometColumnarExchange +- Project - +- Window [COMET: Window expressions are not supported] + +- Window [COMET: WindowExec is not fully compatible with Spark (Native WindowExec has known correctness issues). To enable it anyway, set spark.comet.operator.WindowExec.allowIncompatible=true. For more information, refer to the Comet Compatibility Guide (https://datafusion.apache.org/comet/user-guide/compatibility.html).] +- CometColumnarToRow +- CometSort +- CometExchange diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7/q12.native_iceberg_compat/extended.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7/q12.native_iceberg_compat/extended.txt index 9ddd3aeb38..9ea26caab6 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7/q12.native_iceberg_compat/extended.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7/q12.native_iceberg_compat/extended.txt @@ -1,6 +1,6 @@ TakeOrderedAndProject [COMET: ] +- Project - +- Window [COMET: Window expressions are not supported] + +- Window [COMET: WindowExec is not fully compatible with Spark (Native WindowExec has known correctness issues). To enable it anyway, set spark.comet.operator.WindowExec.allowIncompatible=true. For more information, refer to the Comet Compatibility Guide (https://datafusion.apache.org/comet/user-guide/compatibility.html).] +- CometColumnarToRow +- CometSort +- CometExchange diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7/q12/extended.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7/q12/extended.txt index 9ddd3aeb38..9ea26caab6 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7/q12/extended.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7/q12/extended.txt @@ -1,6 +1,6 @@ TakeOrderedAndProject [COMET: ] +- Project - +- Window [COMET: Window expressions are not supported] + +- Window [COMET: WindowExec is not fully compatible with Spark (Native WindowExec has known correctness issues). To enable it anyway, set spark.comet.operator.WindowExec.allowIncompatible=true. For more information, refer to the Comet Compatibility Guide (https://datafusion.apache.org/comet/user-guide/compatibility.html).] +- CometColumnarToRow +- CometSort +- CometExchange diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7/q20.native_iceberg_compat/extended.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7/q20.native_iceberg_compat/extended.txt index 400633ed9f..1b24b68f14 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7/q20.native_iceberg_compat/extended.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7/q20.native_iceberg_compat/extended.txt @@ -1,6 +1,6 @@ TakeOrderedAndProject [COMET: ] +- Project - +- Window [COMET: Window expressions are not supported] + +- Window [COMET: WindowExec is not fully compatible with Spark (Native WindowExec has known correctness issues). To enable it anyway, set spark.comet.operator.WindowExec.allowIncompatible=true. For more information, refer to the Comet Compatibility Guide (https://datafusion.apache.org/comet/user-guide/compatibility.html).] +- CometColumnarToRow +- CometSort +- CometExchange diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7/q20/extended.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7/q20/extended.txt index 400633ed9f..1b24b68f14 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7/q20/extended.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7/q20/extended.txt @@ -1,6 +1,6 @@ TakeOrderedAndProject [COMET: ] +- Project - +- Window [COMET: Window expressions are not supported] + +- Window [COMET: WindowExec is not fully compatible with Spark (Native WindowExec has known correctness issues). To enable it anyway, set spark.comet.operator.WindowExec.allowIncompatible=true. For more information, refer to the Comet Compatibility Guide (https://datafusion.apache.org/comet/user-guide/compatibility.html).] +- CometColumnarToRow +- CometSort +- CometExchange diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7/q36a.native_iceberg_compat/extended.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7/q36a.native_iceberg_compat/extended.txt index dee6669b47..1de0c1ea43 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7/q36a.native_iceberg_compat/extended.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7/q36a.native_iceberg_compat/extended.txt @@ -1,6 +1,6 @@ TakeOrderedAndProject [COMET: ] +- Project - +- Window [COMET: Window expressions are not supported] + +- Window [COMET: WindowExec is not fully compatible with Spark (Native WindowExec has known correctness issues). To enable it anyway, set spark.comet.operator.WindowExec.allowIncompatible=true. For more information, refer to the Comet Compatibility Guide (https://datafusion.apache.org/comet/user-guide/compatibility.html).] +- CometColumnarToRow +- CometSort +- CometExchange diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7/q36a/extended.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7/q36a/extended.txt index dee6669b47..1de0c1ea43 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7/q36a/extended.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7/q36a/extended.txt @@ -1,6 +1,6 @@ TakeOrderedAndProject [COMET: ] +- Project - +- Window [COMET: Window expressions are not supported] + +- Window [COMET: WindowExec is not fully compatible with Spark (Native WindowExec has known correctness issues). To enable it anyway, set spark.comet.operator.WindowExec.allowIncompatible=true. For more information, refer to the Comet Compatibility Guide (https://datafusion.apache.org/comet/user-guide/compatibility.html).] +- CometColumnarToRow +- CometSort +- CometExchange diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7/q47.native_iceberg_compat/extended.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7/q47.native_iceberg_compat/extended.txt index 11fbf5bd11..d9f357c1c1 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7/q47.native_iceberg_compat/extended.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7/q47.native_iceberg_compat/extended.txt @@ -7,7 +7,7 @@ : : +- Filter : : +- Window : : +- Filter - : : +- Window [COMET: Window expressions are not supported] + : : +- Window [COMET: WindowExec is not fully compatible with Spark (Native WindowExec has known correctness issues). To enable it anyway, set spark.comet.operator.WindowExec.allowIncompatible=true. For more information, refer to the Comet Compatibility Guide (https://datafusion.apache.org/comet/user-guide/compatibility.html).] : : +- CometColumnarToRow : : +- CometSort : : +- CometExchange @@ -39,7 +39,7 @@ : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.store : +- BroadcastExchange : +- Project - : +- Window [COMET: Window expressions are not supported] + : +- Window [COMET: WindowExec is not fully compatible with Spark (Native WindowExec has known correctness issues). To enable it anyway, set spark.comet.operator.WindowExec.allowIncompatible=true. For more information, refer to the Comet Compatibility Guide (https://datafusion.apache.org/comet/user-guide/compatibility.html).] : +- CometColumnarToRow : +- CometSort : +- CometExchange @@ -71,7 +71,7 @@ : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.store +- BroadcastExchange +- Project - +- Window [COMET: Window expressions are not supported] + +- Window [COMET: WindowExec is not fully compatible with Spark (Native WindowExec has known correctness issues). To enable it anyway, set spark.comet.operator.WindowExec.allowIncompatible=true. For more information, refer to the Comet Compatibility Guide (https://datafusion.apache.org/comet/user-guide/compatibility.html).] +- CometColumnarToRow +- CometSort +- CometExchange diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7/q47/extended.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7/q47/extended.txt index 11fbf5bd11..d9f357c1c1 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7/q47/extended.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7/q47/extended.txt @@ -7,7 +7,7 @@ : : +- Filter : : +- Window : : +- Filter - : : +- Window [COMET: Window expressions are not supported] + : : +- Window [COMET: WindowExec is not fully compatible with Spark (Native WindowExec has known correctness issues). To enable it anyway, set spark.comet.operator.WindowExec.allowIncompatible=true. For more information, refer to the Comet Compatibility Guide (https://datafusion.apache.org/comet/user-guide/compatibility.html).] : : +- CometColumnarToRow : : +- CometSort : : +- CometExchange @@ -39,7 +39,7 @@ : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.store : +- BroadcastExchange : +- Project - : +- Window [COMET: Window expressions are not supported] + : +- Window [COMET: WindowExec is not fully compatible with Spark (Native WindowExec has known correctness issues). To enable it anyway, set spark.comet.operator.WindowExec.allowIncompatible=true. For more information, refer to the Comet Compatibility Guide (https://datafusion.apache.org/comet/user-guide/compatibility.html).] : +- CometColumnarToRow : +- CometSort : +- CometExchange @@ -71,7 +71,7 @@ : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.store +- BroadcastExchange +- Project - +- Window [COMET: Window expressions are not supported] + +- Window [COMET: WindowExec is not fully compatible with Spark (Native WindowExec has known correctness issues). To enable it anyway, set spark.comet.operator.WindowExec.allowIncompatible=true. For more information, refer to the Comet Compatibility Guide (https://datafusion.apache.org/comet/user-guide/compatibility.html).] +- CometColumnarToRow +- CometSort +- CometExchange diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7/q49.native_iceberg_compat/extended.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7/q49.native_iceberg_compat/extended.txt index 734b0ce7bb..75684a9669 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7/q49.native_iceberg_compat/extended.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7/q49.native_iceberg_compat/extended.txt @@ -8,7 +8,7 @@ CometColumnarToRow : +- Filter : +- Window : +- Sort - : +- Window [COMET: Window expressions are not supported] + : +- Window [COMET: WindowExec is not fully compatible with Spark (Native WindowExec has known correctness issues). To enable it anyway, set spark.comet.operator.WindowExec.allowIncompatible=true. For more information, refer to the Comet Compatibility Guide (https://datafusion.apache.org/comet/user-guide/compatibility.html).] : +- CometColumnarToRow : +- CometSort : +- CometExchange @@ -40,7 +40,7 @@ CometColumnarToRow : +- Filter : +- Window : +- Sort - : +- Window [COMET: Window expressions are not supported] + : +- Window [COMET: WindowExec is not fully compatible with Spark (Native WindowExec has known correctness issues). To enable it anyway, set spark.comet.operator.WindowExec.allowIncompatible=true. For more information, refer to the Comet Compatibility Guide (https://datafusion.apache.org/comet/user-guide/compatibility.html).] : +- CometColumnarToRow : +- CometSort : +- CometExchange @@ -67,7 +67,7 @@ CometColumnarToRow +- Filter +- Window +- Sort - +- Window [COMET: Window expressions are not supported] + +- Window [COMET: WindowExec is not fully compatible with Spark (Native WindowExec has known correctness issues). To enable it anyway, set spark.comet.operator.WindowExec.allowIncompatible=true. For more information, refer to the Comet Compatibility Guide (https://datafusion.apache.org/comet/user-guide/compatibility.html).] +- CometColumnarToRow +- CometSort +- CometExchange diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7/q49/extended.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7/q49/extended.txt index 734b0ce7bb..75684a9669 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7/q49/extended.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7/q49/extended.txt @@ -8,7 +8,7 @@ CometColumnarToRow : +- Filter : +- Window : +- Sort - : +- Window [COMET: Window expressions are not supported] + : +- Window [COMET: WindowExec is not fully compatible with Spark (Native WindowExec has known correctness issues). To enable it anyway, set spark.comet.operator.WindowExec.allowIncompatible=true. For more information, refer to the Comet Compatibility Guide (https://datafusion.apache.org/comet/user-guide/compatibility.html).] : +- CometColumnarToRow : +- CometSort : +- CometExchange @@ -40,7 +40,7 @@ CometColumnarToRow : +- Filter : +- Window : +- Sort - : +- Window [COMET: Window expressions are not supported] + : +- Window [COMET: WindowExec is not fully compatible with Spark (Native WindowExec has known correctness issues). To enable it anyway, set spark.comet.operator.WindowExec.allowIncompatible=true. For more information, refer to the Comet Compatibility Guide (https://datafusion.apache.org/comet/user-guide/compatibility.html).] : +- CometColumnarToRow : +- CometSort : +- CometExchange @@ -67,7 +67,7 @@ CometColumnarToRow +- Filter +- Window +- Sort - +- Window [COMET: Window expressions are not supported] + +- Window [COMET: WindowExec is not fully compatible with Spark (Native WindowExec has known correctness issues). To enable it anyway, set spark.comet.operator.WindowExec.allowIncompatible=true. For more information, refer to the Comet Compatibility Guide (https://datafusion.apache.org/comet/user-guide/compatibility.html).] +- CometColumnarToRow +- CometSort +- CometExchange diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7/q51a.native_iceberg_compat/extended.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7/q51a.native_iceberg_compat/extended.txt index 15a87f66b6..b192555935 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7/q51a.native_iceberg_compat/extended.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7/q51a.native_iceberg_compat/extended.txt @@ -4,7 +4,7 @@ +- HashAggregate +- Project +- BroadcastHashJoin - :- Window [COMET: Window expressions are not supported] + :- Window [COMET: WindowExec is not fully compatible with Spark (Native WindowExec has known correctness issues). To enable it anyway, set spark.comet.operator.WindowExec.allowIncompatible=true. For more information, refer to the Comet Compatibility Guide (https://datafusion.apache.org/comet/user-guide/compatibility.html)., Partitioning and sorting specifications must be the same.] : +- CometColumnarToRow : +- CometSort : +- CometExchange @@ -20,7 +20,7 @@ : : +- Project : : +- BroadcastHashJoin : : :- Project - : : : +- Window [COMET: Window expressions are not supported] + : : : +- Window [COMET: WindowExec is not fully compatible with Spark (Native WindowExec has known correctness issues). To enable it anyway, set spark.comet.operator.WindowExec.allowIncompatible=true. For more information, refer to the Comet Compatibility Guide (https://datafusion.apache.org/comet/user-guide/compatibility.html)., Partitioning and sorting specifications must be the same.] : : : +- CometColumnarToRow : : : +- CometSort : : : +- CometExchange @@ -43,7 +43,7 @@ : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim : : +- BroadcastExchange : : +- Project - : : +- Window [COMET: Window expressions are not supported] + : : +- Window [COMET: WindowExec is not fully compatible with Spark (Native WindowExec has known correctness issues). To enable it anyway, set spark.comet.operator.WindowExec.allowIncompatible=true. For more information, refer to the Comet Compatibility Guide (https://datafusion.apache.org/comet/user-guide/compatibility.html)., Partitioning and sorting specifications must be the same.] : : +- CometColumnarToRow : : +- CometSort : : +- CometExchange @@ -73,7 +73,7 @@ : +- Project : +- BroadcastHashJoin : :- Project - : : +- Window [COMET: Window expressions are not supported] + : : +- Window [COMET: WindowExec is not fully compatible with Spark (Native WindowExec has known correctness issues). To enable it anyway, set spark.comet.operator.WindowExec.allowIncompatible=true. For more information, refer to the Comet Compatibility Guide (https://datafusion.apache.org/comet/user-guide/compatibility.html)., Partitioning and sorting specifications must be the same.] : : +- CometColumnarToRow : : +- CometSort : : +- CometExchange @@ -91,7 +91,7 @@ : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim : +- BroadcastExchange : +- Project - : +- Window [COMET: Window expressions are not supported] + : +- Window [COMET: WindowExec is not fully compatible with Spark (Native WindowExec has known correctness issues). To enable it anyway, set spark.comet.operator.WindowExec.allowIncompatible=true. For more information, refer to the Comet Compatibility Guide (https://datafusion.apache.org/comet/user-guide/compatibility.html)., Partitioning and sorting specifications must be the same.] : +- CometColumnarToRow : +- CometSort : +- CometExchange @@ -109,7 +109,7 @@ : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim +- BroadcastExchange +- Project - +- Window [COMET: Window expressions are not supported] + +- Window [COMET: WindowExec is not fully compatible with Spark (Native WindowExec has known correctness issues). To enable it anyway, set spark.comet.operator.WindowExec.allowIncompatible=true. For more information, refer to the Comet Compatibility Guide (https://datafusion.apache.org/comet/user-guide/compatibility.html)., Partitioning and sorting specifications must be the same.] +- CometColumnarToRow +- CometSort +- CometExchange @@ -125,7 +125,7 @@ : +- Project : +- BroadcastHashJoin : :- Project - : : +- Window [COMET: Window expressions are not supported] + : : +- Window [COMET: WindowExec is not fully compatible with Spark (Native WindowExec has known correctness issues). To enable it anyway, set spark.comet.operator.WindowExec.allowIncompatible=true. For more information, refer to the Comet Compatibility Guide (https://datafusion.apache.org/comet/user-guide/compatibility.html)., Partitioning and sorting specifications must be the same.] : : +- CometColumnarToRow : : +- CometSort : : +- CometExchange @@ -148,7 +148,7 @@ : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim : +- BroadcastExchange : +- Project - : +- Window [COMET: Window expressions are not supported] + : +- Window [COMET: WindowExec is not fully compatible with Spark (Native WindowExec has known correctness issues). To enable it anyway, set spark.comet.operator.WindowExec.allowIncompatible=true. For more information, refer to the Comet Compatibility Guide (https://datafusion.apache.org/comet/user-guide/compatibility.html)., Partitioning and sorting specifications must be the same.] : +- CometColumnarToRow : +- CometSort : +- CometExchange @@ -178,7 +178,7 @@ +- Project +- BroadcastHashJoin :- Project - : +- Window [COMET: Window expressions are not supported] + : +- Window [COMET: WindowExec is not fully compatible with Spark (Native WindowExec has known correctness issues). To enable it anyway, set spark.comet.operator.WindowExec.allowIncompatible=true. For more information, refer to the Comet Compatibility Guide (https://datafusion.apache.org/comet/user-guide/compatibility.html)., Partitioning and sorting specifications must be the same.] : +- CometColumnarToRow : +- CometSort : +- CometExchange @@ -196,7 +196,7 @@ : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim +- BroadcastExchange +- Project - +- Window [COMET: Window expressions are not supported] + +- Window [COMET: WindowExec is not fully compatible with Spark (Native WindowExec has known correctness issues). To enable it anyway, set spark.comet.operator.WindowExec.allowIncompatible=true. For more information, refer to the Comet Compatibility Guide (https://datafusion.apache.org/comet/user-guide/compatibility.html)., Partitioning and sorting specifications must be the same.] +- CometColumnarToRow +- CometSort +- CometExchange diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7/q51a/extended.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7/q51a/extended.txt index 15a87f66b6..b192555935 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7/q51a/extended.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7/q51a/extended.txt @@ -4,7 +4,7 @@ +- HashAggregate +- Project +- BroadcastHashJoin - :- Window [COMET: Window expressions are not supported] + :- Window [COMET: WindowExec is not fully compatible with Spark (Native WindowExec has known correctness issues). To enable it anyway, set spark.comet.operator.WindowExec.allowIncompatible=true. For more information, refer to the Comet Compatibility Guide (https://datafusion.apache.org/comet/user-guide/compatibility.html)., Partitioning and sorting specifications must be the same.] : +- CometColumnarToRow : +- CometSort : +- CometExchange @@ -20,7 +20,7 @@ : : +- Project : : +- BroadcastHashJoin : : :- Project - : : : +- Window [COMET: Window expressions are not supported] + : : : +- Window [COMET: WindowExec is not fully compatible with Spark (Native WindowExec has known correctness issues). To enable it anyway, set spark.comet.operator.WindowExec.allowIncompatible=true. For more information, refer to the Comet Compatibility Guide (https://datafusion.apache.org/comet/user-guide/compatibility.html)., Partitioning and sorting specifications must be the same.] : : : +- CometColumnarToRow : : : +- CometSort : : : +- CometExchange @@ -43,7 +43,7 @@ : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim : : +- BroadcastExchange : : +- Project - : : +- Window [COMET: Window expressions are not supported] + : : +- Window [COMET: WindowExec is not fully compatible with Spark (Native WindowExec has known correctness issues). To enable it anyway, set spark.comet.operator.WindowExec.allowIncompatible=true. For more information, refer to the Comet Compatibility Guide (https://datafusion.apache.org/comet/user-guide/compatibility.html)., Partitioning and sorting specifications must be the same.] : : +- CometColumnarToRow : : +- CometSort : : +- CometExchange @@ -73,7 +73,7 @@ : +- Project : +- BroadcastHashJoin : :- Project - : : +- Window [COMET: Window expressions are not supported] + : : +- Window [COMET: WindowExec is not fully compatible with Spark (Native WindowExec has known correctness issues). To enable it anyway, set spark.comet.operator.WindowExec.allowIncompatible=true. For more information, refer to the Comet Compatibility Guide (https://datafusion.apache.org/comet/user-guide/compatibility.html)., Partitioning and sorting specifications must be the same.] : : +- CometColumnarToRow : : +- CometSort : : +- CometExchange @@ -91,7 +91,7 @@ : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim : +- BroadcastExchange : +- Project - : +- Window [COMET: Window expressions are not supported] + : +- Window [COMET: WindowExec is not fully compatible with Spark (Native WindowExec has known correctness issues). To enable it anyway, set spark.comet.operator.WindowExec.allowIncompatible=true. For more information, refer to the Comet Compatibility Guide (https://datafusion.apache.org/comet/user-guide/compatibility.html)., Partitioning and sorting specifications must be the same.] : +- CometColumnarToRow : +- CometSort : +- CometExchange @@ -109,7 +109,7 @@ : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim +- BroadcastExchange +- Project - +- Window [COMET: Window expressions are not supported] + +- Window [COMET: WindowExec is not fully compatible with Spark (Native WindowExec has known correctness issues). To enable it anyway, set spark.comet.operator.WindowExec.allowIncompatible=true. For more information, refer to the Comet Compatibility Guide (https://datafusion.apache.org/comet/user-guide/compatibility.html)., Partitioning and sorting specifications must be the same.] +- CometColumnarToRow +- CometSort +- CometExchange @@ -125,7 +125,7 @@ : +- Project : +- BroadcastHashJoin : :- Project - : : +- Window [COMET: Window expressions are not supported] + : : +- Window [COMET: WindowExec is not fully compatible with Spark (Native WindowExec has known correctness issues). To enable it anyway, set spark.comet.operator.WindowExec.allowIncompatible=true. For more information, refer to the Comet Compatibility Guide (https://datafusion.apache.org/comet/user-guide/compatibility.html)., Partitioning and sorting specifications must be the same.] : : +- CometColumnarToRow : : +- CometSort : : +- CometExchange @@ -148,7 +148,7 @@ : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim : +- BroadcastExchange : +- Project - : +- Window [COMET: Window expressions are not supported] + : +- Window [COMET: WindowExec is not fully compatible with Spark (Native WindowExec has known correctness issues). To enable it anyway, set spark.comet.operator.WindowExec.allowIncompatible=true. For more information, refer to the Comet Compatibility Guide (https://datafusion.apache.org/comet/user-guide/compatibility.html)., Partitioning and sorting specifications must be the same.] : +- CometColumnarToRow : +- CometSort : +- CometExchange @@ -178,7 +178,7 @@ +- Project +- BroadcastHashJoin :- Project - : +- Window [COMET: Window expressions are not supported] + : +- Window [COMET: WindowExec is not fully compatible with Spark (Native WindowExec has known correctness issues). To enable it anyway, set spark.comet.operator.WindowExec.allowIncompatible=true. For more information, refer to the Comet Compatibility Guide (https://datafusion.apache.org/comet/user-guide/compatibility.html)., Partitioning and sorting specifications must be the same.] : +- CometColumnarToRow : +- CometSort : +- CometExchange @@ -196,7 +196,7 @@ : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim +- BroadcastExchange +- Project - +- Window [COMET: Window expressions are not supported] + +- Window [COMET: WindowExec is not fully compatible with Spark (Native WindowExec has known correctness issues). To enable it anyway, set spark.comet.operator.WindowExec.allowIncompatible=true. For more information, refer to the Comet Compatibility Guide (https://datafusion.apache.org/comet/user-guide/compatibility.html)., Partitioning and sorting specifications must be the same.] +- CometColumnarToRow +- CometSort +- CometExchange diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7/q57.native_iceberg_compat/extended.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7/q57.native_iceberg_compat/extended.txt index 7bee0e61e8..eb8f4ab303 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7/q57.native_iceberg_compat/extended.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7/q57.native_iceberg_compat/extended.txt @@ -7,7 +7,7 @@ : : +- Filter : : +- Window : : +- Filter - : : +- Window [COMET: Window expressions are not supported] + : : +- Window [COMET: WindowExec is not fully compatible with Spark (Native WindowExec has known correctness issues). To enable it anyway, set spark.comet.operator.WindowExec.allowIncompatible=true. For more information, refer to the Comet Compatibility Guide (https://datafusion.apache.org/comet/user-guide/compatibility.html).] : : +- CometColumnarToRow : : +- CometSort : : +- CometExchange @@ -39,7 +39,7 @@ : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.call_center : +- BroadcastExchange : +- Project - : +- Window [COMET: Window expressions are not supported] + : +- Window [COMET: WindowExec is not fully compatible with Spark (Native WindowExec has known correctness issues). To enable it anyway, set spark.comet.operator.WindowExec.allowIncompatible=true. For more information, refer to the Comet Compatibility Guide (https://datafusion.apache.org/comet/user-guide/compatibility.html).] : +- CometColumnarToRow : +- CometSort : +- CometExchange @@ -71,7 +71,7 @@ : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.call_center +- BroadcastExchange +- Project - +- Window [COMET: Window expressions are not supported] + +- Window [COMET: WindowExec is not fully compatible with Spark (Native WindowExec has known correctness issues). To enable it anyway, set spark.comet.operator.WindowExec.allowIncompatible=true. For more information, refer to the Comet Compatibility Guide (https://datafusion.apache.org/comet/user-guide/compatibility.html).] +- CometColumnarToRow +- CometSort +- CometExchange diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7/q57/extended.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7/q57/extended.txt index 7bee0e61e8..eb8f4ab303 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7/q57/extended.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7/q57/extended.txt @@ -7,7 +7,7 @@ : : +- Filter : : +- Window : : +- Filter - : : +- Window [COMET: Window expressions are not supported] + : : +- Window [COMET: WindowExec is not fully compatible with Spark (Native WindowExec has known correctness issues). To enable it anyway, set spark.comet.operator.WindowExec.allowIncompatible=true. For more information, refer to the Comet Compatibility Guide (https://datafusion.apache.org/comet/user-guide/compatibility.html).] : : +- CometColumnarToRow : : +- CometSort : : +- CometExchange @@ -39,7 +39,7 @@ : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.call_center : +- BroadcastExchange : +- Project - : +- Window [COMET: Window expressions are not supported] + : +- Window [COMET: WindowExec is not fully compatible with Spark (Native WindowExec has known correctness issues). To enable it anyway, set spark.comet.operator.WindowExec.allowIncompatible=true. For more information, refer to the Comet Compatibility Guide (https://datafusion.apache.org/comet/user-guide/compatibility.html).] : +- CometColumnarToRow : +- CometSort : +- CometExchange @@ -71,7 +71,7 @@ : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.call_center +- BroadcastExchange +- Project - +- Window [COMET: Window expressions are not supported] + +- Window [COMET: WindowExec is not fully compatible with Spark (Native WindowExec has known correctness issues). To enable it anyway, set spark.comet.operator.WindowExec.allowIncompatible=true. For more information, refer to the Comet Compatibility Guide (https://datafusion.apache.org/comet/user-guide/compatibility.html).] +- CometColumnarToRow +- CometSort +- CometExchange diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7/q67a.native_iceberg_compat/extended.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7/q67a.native_iceberg_compat/extended.txt index e1884afb56..c619ba85a1 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7/q67a.native_iceberg_compat/extended.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7/q67a.native_iceberg_compat/extended.txt @@ -1,6 +1,6 @@ TakeOrderedAndProject [COMET: ] +- Filter - +- Window [COMET: Window expressions are not supported] + +- Window [COMET: WindowExec is not fully compatible with Spark (Native WindowExec has known correctness issues). To enable it anyway, set spark.comet.operator.WindowExec.allowIncompatible=true. For more information, refer to the Comet Compatibility Guide (https://datafusion.apache.org/comet/user-guide/compatibility.html)., Partitioning and sorting specifications must be the same.] +- CometColumnarToRow +- CometSort +- CometExchange diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7/q67a/extended.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7/q67a/extended.txt index e1884afb56..c619ba85a1 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7/q67a/extended.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7/q67a/extended.txt @@ -1,6 +1,6 @@ TakeOrderedAndProject [COMET: ] +- Filter - +- Window [COMET: Window expressions are not supported] + +- Window [COMET: WindowExec is not fully compatible with Spark (Native WindowExec has known correctness issues). To enable it anyway, set spark.comet.operator.WindowExec.allowIncompatible=true. For more information, refer to the Comet Compatibility Guide (https://datafusion.apache.org/comet/user-guide/compatibility.html)., Partitioning and sorting specifications must be the same.] +- CometColumnarToRow +- CometSort +- CometExchange diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7/q70a.native_iceberg_compat/extended.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7/q70a.native_iceberg_compat/extended.txt index a187c52d0d..0a2ecaa017 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7/q70a.native_iceberg_compat/extended.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7/q70a.native_iceberg_compat/extended.txt @@ -1,6 +1,6 @@ TakeOrderedAndProject [COMET: ] +- Project - +- Window [COMET: Window expressions are not supported] + +- Window [COMET: WindowExec is not fully compatible with Spark (Native WindowExec has known correctness issues). To enable it anyway, set spark.comet.operator.WindowExec.allowIncompatible=true. For more information, refer to the Comet Compatibility Guide (https://datafusion.apache.org/comet/user-guide/compatibility.html).] +- CometColumnarToRow +- CometSort +- CometExchange @@ -38,7 +38,7 @@ : +- BroadcastExchange : +- Project : +- Filter - : +- Window [COMET: Window expressions are not supported] + : +- Window [COMET: WindowExec is not fully compatible with Spark (Native WindowExec has known correctness issues). To enable it anyway, set spark.comet.operator.WindowExec.allowIncompatible=true. For more information, refer to the Comet Compatibility Guide (https://datafusion.apache.org/comet/user-guide/compatibility.html)., Partitioning and sorting specifications must be the same.] : +- CometColumnarToRow : +- CometSort : +- CometHashAggregate @@ -93,7 +93,7 @@ : +- BroadcastExchange : +- Project : +- Filter - : +- Window [COMET: Window expressions are not supported] + : +- Window [COMET: WindowExec is not fully compatible with Spark (Native WindowExec has known correctness issues). To enable it anyway, set spark.comet.operator.WindowExec.allowIncompatible=true. For more information, refer to the Comet Compatibility Guide (https://datafusion.apache.org/comet/user-guide/compatibility.html)., Partitioning and sorting specifications must be the same.] : +- CometColumnarToRow : +- CometSort : +- CometHashAggregate @@ -148,7 +148,7 @@ +- BroadcastExchange +- Project +- Filter - +- Window [COMET: Window expressions are not supported] + +- Window [COMET: WindowExec is not fully compatible with Spark (Native WindowExec has known correctness issues). To enable it anyway, set spark.comet.operator.WindowExec.allowIncompatible=true. For more information, refer to the Comet Compatibility Guide (https://datafusion.apache.org/comet/user-guide/compatibility.html)., Partitioning and sorting specifications must be the same.] +- CometColumnarToRow +- CometSort +- CometHashAggregate diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7/q70a/extended.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7/q70a/extended.txt index a187c52d0d..0a2ecaa017 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7/q70a/extended.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7/q70a/extended.txt @@ -1,6 +1,6 @@ TakeOrderedAndProject [COMET: ] +- Project - +- Window [COMET: Window expressions are not supported] + +- Window [COMET: WindowExec is not fully compatible with Spark (Native WindowExec has known correctness issues). To enable it anyway, set spark.comet.operator.WindowExec.allowIncompatible=true. For more information, refer to the Comet Compatibility Guide (https://datafusion.apache.org/comet/user-guide/compatibility.html).] +- CometColumnarToRow +- CometSort +- CometExchange @@ -38,7 +38,7 @@ : +- BroadcastExchange : +- Project : +- Filter - : +- Window [COMET: Window expressions are not supported] + : +- Window [COMET: WindowExec is not fully compatible with Spark (Native WindowExec has known correctness issues). To enable it anyway, set spark.comet.operator.WindowExec.allowIncompatible=true. For more information, refer to the Comet Compatibility Guide (https://datafusion.apache.org/comet/user-guide/compatibility.html)., Partitioning and sorting specifications must be the same.] : +- CometColumnarToRow : +- CometSort : +- CometHashAggregate @@ -93,7 +93,7 @@ : +- BroadcastExchange : +- Project : +- Filter - : +- Window [COMET: Window expressions are not supported] + : +- Window [COMET: WindowExec is not fully compatible with Spark (Native WindowExec has known correctness issues). To enable it anyway, set spark.comet.operator.WindowExec.allowIncompatible=true. For more information, refer to the Comet Compatibility Guide (https://datafusion.apache.org/comet/user-guide/compatibility.html)., Partitioning and sorting specifications must be the same.] : +- CometColumnarToRow : +- CometSort : +- CometHashAggregate @@ -148,7 +148,7 @@ +- BroadcastExchange +- Project +- Filter - +- Window [COMET: Window expressions are not supported] + +- Window [COMET: WindowExec is not fully compatible with Spark (Native WindowExec has known correctness issues). To enable it anyway, set spark.comet.operator.WindowExec.allowIncompatible=true. For more information, refer to the Comet Compatibility Guide (https://datafusion.apache.org/comet/user-guide/compatibility.html)., Partitioning and sorting specifications must be the same.] +- CometColumnarToRow +- CometSort +- CometHashAggregate diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7/q86a.native_iceberg_compat/extended.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7/q86a.native_iceberg_compat/extended.txt index 6dd65034ce..b2b5b4d63d 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7/q86a.native_iceberg_compat/extended.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7/q86a.native_iceberg_compat/extended.txt @@ -1,6 +1,6 @@ TakeOrderedAndProject [COMET: ] +- Project - +- Window [COMET: Window expressions are not supported] + +- Window [COMET: WindowExec is not fully compatible with Spark (Native WindowExec has known correctness issues). To enable it anyway, set spark.comet.operator.WindowExec.allowIncompatible=true. For more information, refer to the Comet Compatibility Guide (https://datafusion.apache.org/comet/user-guide/compatibility.html).] +- CometColumnarToRow +- CometSort +- CometExchange diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7/q86a/extended.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7/q86a/extended.txt index 6dd65034ce..b2b5b4d63d 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7/q86a/extended.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7/q86a/extended.txt @@ -1,6 +1,6 @@ TakeOrderedAndProject [COMET: ] +- Project - +- Window [COMET: Window expressions are not supported] + +- Window [COMET: WindowExec is not fully compatible with Spark (Native WindowExec has known correctness issues). To enable it anyway, set spark.comet.operator.WindowExec.allowIncompatible=true. For more information, refer to the Comet Compatibility Guide (https://datafusion.apache.org/comet/user-guide/compatibility.html).] +- CometColumnarToRow +- CometSort +- CometExchange diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7/q98.native_iceberg_compat/extended.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7/q98.native_iceberg_compat/extended.txt index 54a4d3c1b7..3dbaf2e346 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7/q98.native_iceberg_compat/extended.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7/q98.native_iceberg_compat/extended.txt @@ -2,7 +2,7 @@ CometColumnarToRow +- CometSort +- CometColumnarExchange +- Project - +- Window [COMET: Window expressions are not supported] + +- Window [COMET: WindowExec is not fully compatible with Spark (Native WindowExec has known correctness issues). To enable it anyway, set spark.comet.operator.WindowExec.allowIncompatible=true. For more information, refer to the Comet Compatibility Guide (https://datafusion.apache.org/comet/user-guide/compatibility.html).] +- CometColumnarToRow +- CometSort +- CometExchange diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7/q98/extended.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7/q98/extended.txt index 54a4d3c1b7..3dbaf2e346 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7/q98/extended.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7/q98/extended.txt @@ -2,7 +2,7 @@ CometColumnarToRow +- CometSort +- CometColumnarExchange +- Project - +- Window [COMET: Window expressions are not supported] + +- Window [COMET: WindowExec is not fully compatible with Spark (Native WindowExec has known correctness issues). To enable it anyway, set spark.comet.operator.WindowExec.allowIncompatible=true. For more information, refer to the Comet Compatibility Guide (https://datafusion.apache.org/comet/user-guide/compatibility.html).] +- CometColumnarToRow +- CometSort +- CometExchange From 468ea5667abd6c1e0e0edd4d63a6590d8ea7190a Mon Sep 17 00:00:00 2001 From: Andy Grove Date: Sat, 8 Nov 2025 13:17:01 -0700 Subject: [PATCH 33/44] 4.0 golden files --- .../q12.native_iceberg_compat/extended.txt | 2 +- .../q12/extended.txt | 2 +- .../q20.native_iceberg_compat/extended.txt | 2 +- .../q20/extended.txt | 2 +- .../q36.native_iceberg_compat/extended.txt | 2 +- .../q36/extended.txt | 2 +- .../q47.native_iceberg_compat/extended.txt | 6 +++--- .../q47/extended.txt | 6 +++--- .../q49.native_iceberg_compat/extended.txt | 6 +++--- .../q49/extended.txt | 6 +++--- .../q51.native_iceberg_compat/extended.txt | 6 +++--- .../q51/extended.txt | 6 +++--- .../q53.native_iceberg_compat/extended.txt | 2 +- .../q53/extended.txt | 2 +- .../q57.native_iceberg_compat/extended.txt | 6 +++--- .../q57/extended.txt | 6 +++--- .../q63.native_iceberg_compat/extended.txt | 2 +- .../q63/extended.txt | 2 +- .../q70.native_iceberg_compat/extended.txt | 2 +- .../q70/extended.txt | 2 +- .../q86.native_iceberg_compat/extended.txt | 2 +- .../q86/extended.txt | 2 +- .../q89.native_iceberg_compat/extended.txt | 2 +- .../q89/extended.txt | 2 +- .../q98.native_iceberg_compat/extended.txt | 2 +- .../q98/extended.txt | 2 +- .../q12.native_iceberg_compat/extended.txt | 2 +- .../q12/extended.txt | 2 +- .../q20.native_iceberg_compat/extended.txt | 2 +- .../q20/extended.txt | 2 +- .../q36a.native_iceberg_compat/extended.txt | 2 +- .../q36a/extended.txt | 2 +- .../q47.native_iceberg_compat/extended.txt | 6 +++--- .../q47/extended.txt | 6 +++--- .../q49.native_iceberg_compat/extended.txt | 6 +++--- .../q49/extended.txt | 6 +++--- .../q51a.native_iceberg_compat/extended.txt | 20 +++++++++---------- .../q51a/extended.txt | 20 +++++++++---------- .../q57.native_iceberg_compat/extended.txt | 6 +++--- .../q57/extended.txt | 6 +++--- .../q70a.native_iceberg_compat/extended.txt | 2 +- .../q70a/extended.txt | 2 +- .../q86a.native_iceberg_compat/extended.txt | 2 +- .../q86a/extended.txt | 2 +- .../q98.native_iceberg_compat/extended.txt | 2 +- .../q98/extended.txt | 2 +- 46 files changed, 92 insertions(+), 92 deletions(-) diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q12.native_iceberg_compat/extended.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q12.native_iceberg_compat/extended.txt index f99382d8c3..745c04cc07 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q12.native_iceberg_compat/extended.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q12.native_iceberg_compat/extended.txt @@ -1,6 +1,6 @@ TakeOrderedAndProject [COMET: ] +- Project - +- Window [COMET: Window expressions are not supported] + +- Window [COMET: WindowExec is not fully compatible with Spark (Native WindowExec has known correctness issues). To enable it anyway, set spark.comet.operator.WindowExec.allowIncompatible=true. For more information, refer to the Comet Compatibility Guide (https://datafusion.apache.org/comet/user-guide/compatibility.html).] +- CometColumnarToRow +- CometSort +- CometColumnarExchange diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q12/extended.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q12/extended.txt index f99382d8c3..745c04cc07 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q12/extended.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q12/extended.txt @@ -1,6 +1,6 @@ TakeOrderedAndProject [COMET: ] +- Project - +- Window [COMET: Window expressions are not supported] + +- Window [COMET: WindowExec is not fully compatible with Spark (Native WindowExec has known correctness issues). To enable it anyway, set spark.comet.operator.WindowExec.allowIncompatible=true. For more information, refer to the Comet Compatibility Guide (https://datafusion.apache.org/comet/user-guide/compatibility.html).] +- CometColumnarToRow +- CometSort +- CometColumnarExchange diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q20.native_iceberg_compat/extended.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q20.native_iceberg_compat/extended.txt index b45bca05b8..c87e12f843 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q20.native_iceberg_compat/extended.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q20.native_iceberg_compat/extended.txt @@ -1,6 +1,6 @@ TakeOrderedAndProject [COMET: ] +- Project - +- Window [COMET: Window expressions are not supported] + +- Window [COMET: WindowExec is not fully compatible with Spark (Native WindowExec has known correctness issues). To enable it anyway, set spark.comet.operator.WindowExec.allowIncompatible=true. For more information, refer to the Comet Compatibility Guide (https://datafusion.apache.org/comet/user-guide/compatibility.html).] +- CometColumnarToRow +- CometSort +- CometColumnarExchange diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q20/extended.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q20/extended.txt index b45bca05b8..c87e12f843 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q20/extended.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q20/extended.txt @@ -1,6 +1,6 @@ TakeOrderedAndProject [COMET: ] +- Project - +- Window [COMET: Window expressions are not supported] + +- Window [COMET: WindowExec is not fully compatible with Spark (Native WindowExec has known correctness issues). To enable it anyway, set spark.comet.operator.WindowExec.allowIncompatible=true. For more information, refer to the Comet Compatibility Guide (https://datafusion.apache.org/comet/user-guide/compatibility.html).] +- CometColumnarToRow +- CometSort +- CometColumnarExchange diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q36.native_iceberg_compat/extended.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q36.native_iceberg_compat/extended.txt index 3e8236fe57..c5ee3b2822 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q36.native_iceberg_compat/extended.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q36.native_iceberg_compat/extended.txt @@ -1,6 +1,6 @@ TakeOrderedAndProject [COMET: ] +- Project - +- Window [COMET: Window expressions are not supported] + +- Window [COMET: WindowExec is not fully compatible with Spark (Native WindowExec has known correctness issues). To enable it anyway, set spark.comet.operator.WindowExec.allowIncompatible=true. For more information, refer to the Comet Compatibility Guide (https://datafusion.apache.org/comet/user-guide/compatibility.html).] +- CometColumnarToRow +- CometSort +- CometColumnarExchange diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q36/extended.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q36/extended.txt index 3e8236fe57..c5ee3b2822 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q36/extended.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q36/extended.txt @@ -1,6 +1,6 @@ TakeOrderedAndProject [COMET: ] +- Project - +- Window [COMET: Window expressions are not supported] + +- Window [COMET: WindowExec is not fully compatible with Spark (Native WindowExec has known correctness issues). To enable it anyway, set spark.comet.operator.WindowExec.allowIncompatible=true. For more information, refer to the Comet Compatibility Guide (https://datafusion.apache.org/comet/user-guide/compatibility.html).] +- CometColumnarToRow +- CometSort +- CometColumnarExchange diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q47.native_iceberg_compat/extended.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q47.native_iceberg_compat/extended.txt index 1a4201332c..1a62aa48d9 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q47.native_iceberg_compat/extended.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q47.native_iceberg_compat/extended.txt @@ -7,7 +7,7 @@ : : +- Filter : : +- Window : : +- Filter - : : +- Window [COMET: Window expressions are not supported] + : : +- Window [COMET: WindowExec is not fully compatible with Spark (Native WindowExec has known correctness issues). To enable it anyway, set spark.comet.operator.WindowExec.allowIncompatible=true. For more information, refer to the Comet Compatibility Guide (https://datafusion.apache.org/comet/user-guide/compatibility.html).] : : +- CometColumnarToRow : : +- CometSort : : +- CometColumnarExchange @@ -40,7 +40,7 @@ : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.store : +- BroadcastExchange : +- Project - : +- Window [COMET: Window expressions are not supported] + : +- Window [COMET: WindowExec is not fully compatible with Spark (Native WindowExec has known correctness issues). To enable it anyway, set spark.comet.operator.WindowExec.allowIncompatible=true. For more information, refer to the Comet Compatibility Guide (https://datafusion.apache.org/comet/user-guide/compatibility.html).] : +- CometColumnarToRow : +- CometSort : +- CometColumnarExchange @@ -73,7 +73,7 @@ : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.store +- BroadcastExchange +- Project - +- Window [COMET: Window expressions are not supported] + +- Window [COMET: WindowExec is not fully compatible with Spark (Native WindowExec has known correctness issues). To enable it anyway, set spark.comet.operator.WindowExec.allowIncompatible=true. For more information, refer to the Comet Compatibility Guide (https://datafusion.apache.org/comet/user-guide/compatibility.html).] +- CometColumnarToRow +- CometSort +- CometColumnarExchange diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q47/extended.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q47/extended.txt index 1a4201332c..1a62aa48d9 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q47/extended.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q47/extended.txt @@ -7,7 +7,7 @@ : : +- Filter : : +- Window : : +- Filter - : : +- Window [COMET: Window expressions are not supported] + : : +- Window [COMET: WindowExec is not fully compatible with Spark (Native WindowExec has known correctness issues). To enable it anyway, set spark.comet.operator.WindowExec.allowIncompatible=true. For more information, refer to the Comet Compatibility Guide (https://datafusion.apache.org/comet/user-guide/compatibility.html).] : : +- CometColumnarToRow : : +- CometSort : : +- CometColumnarExchange @@ -40,7 +40,7 @@ : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.store : +- BroadcastExchange : +- Project - : +- Window [COMET: Window expressions are not supported] + : +- Window [COMET: WindowExec is not fully compatible with Spark (Native WindowExec has known correctness issues). To enable it anyway, set spark.comet.operator.WindowExec.allowIncompatible=true. For more information, refer to the Comet Compatibility Guide (https://datafusion.apache.org/comet/user-guide/compatibility.html).] : +- CometColumnarToRow : +- CometSort : +- CometColumnarExchange @@ -73,7 +73,7 @@ : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.store +- BroadcastExchange +- Project - +- Window [COMET: Window expressions are not supported] + +- Window [COMET: WindowExec is not fully compatible with Spark (Native WindowExec has known correctness issues). To enable it anyway, set spark.comet.operator.WindowExec.allowIncompatible=true. For more information, refer to the Comet Compatibility Guide (https://datafusion.apache.org/comet/user-guide/compatibility.html).] +- CometColumnarToRow +- CometSort +- CometColumnarExchange diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q49.native_iceberg_compat/extended.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q49.native_iceberg_compat/extended.txt index 734b0ce7bb..75684a9669 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q49.native_iceberg_compat/extended.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q49.native_iceberg_compat/extended.txt @@ -8,7 +8,7 @@ CometColumnarToRow : +- Filter : +- Window : +- Sort - : +- Window [COMET: Window expressions are not supported] + : +- Window [COMET: WindowExec is not fully compatible with Spark (Native WindowExec has known correctness issues). To enable it anyway, set spark.comet.operator.WindowExec.allowIncompatible=true. For more information, refer to the Comet Compatibility Guide (https://datafusion.apache.org/comet/user-guide/compatibility.html).] : +- CometColumnarToRow : +- CometSort : +- CometExchange @@ -40,7 +40,7 @@ CometColumnarToRow : +- Filter : +- Window : +- Sort - : +- Window [COMET: Window expressions are not supported] + : +- Window [COMET: WindowExec is not fully compatible with Spark (Native WindowExec has known correctness issues). To enable it anyway, set spark.comet.operator.WindowExec.allowIncompatible=true. For more information, refer to the Comet Compatibility Guide (https://datafusion.apache.org/comet/user-guide/compatibility.html).] : +- CometColumnarToRow : +- CometSort : +- CometExchange @@ -67,7 +67,7 @@ CometColumnarToRow +- Filter +- Window +- Sort - +- Window [COMET: Window expressions are not supported] + +- Window [COMET: WindowExec is not fully compatible with Spark (Native WindowExec has known correctness issues). To enable it anyway, set spark.comet.operator.WindowExec.allowIncompatible=true. For more information, refer to the Comet Compatibility Guide (https://datafusion.apache.org/comet/user-guide/compatibility.html).] +- CometColumnarToRow +- CometSort +- CometExchange diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q49/extended.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q49/extended.txt index 734b0ce7bb..75684a9669 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q49/extended.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q49/extended.txt @@ -8,7 +8,7 @@ CometColumnarToRow : +- Filter : +- Window : +- Sort - : +- Window [COMET: Window expressions are not supported] + : +- Window [COMET: WindowExec is not fully compatible with Spark (Native WindowExec has known correctness issues). To enable it anyway, set spark.comet.operator.WindowExec.allowIncompatible=true. For more information, refer to the Comet Compatibility Guide (https://datafusion.apache.org/comet/user-guide/compatibility.html).] : +- CometColumnarToRow : +- CometSort : +- CometExchange @@ -40,7 +40,7 @@ CometColumnarToRow : +- Filter : +- Window : +- Sort - : +- Window [COMET: Window expressions are not supported] + : +- Window [COMET: WindowExec is not fully compatible with Spark (Native WindowExec has known correctness issues). To enable it anyway, set spark.comet.operator.WindowExec.allowIncompatible=true. For more information, refer to the Comet Compatibility Guide (https://datafusion.apache.org/comet/user-guide/compatibility.html).] : +- CometColumnarToRow : +- CometSort : +- CometExchange @@ -67,7 +67,7 @@ CometColumnarToRow +- Filter +- Window +- Sort - +- Window [COMET: Window expressions are not supported] + +- Window [COMET: WindowExec is not fully compatible with Spark (Native WindowExec has known correctness issues). To enable it anyway, set spark.comet.operator.WindowExec.allowIncompatible=true. For more information, refer to the Comet Compatibility Guide (https://datafusion.apache.org/comet/user-guide/compatibility.html).] +- CometColumnarToRow +- CometSort +- CometExchange diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q51.native_iceberg_compat/extended.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q51.native_iceberg_compat/extended.txt index d5de75daf6..f35e106153 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q51.native_iceberg_compat/extended.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q51.native_iceberg_compat/extended.txt @@ -1,6 +1,6 @@ TakeOrderedAndProject [COMET: ] +- Filter - +- Window [COMET: Window expressions are not supported] + +- Window [COMET: WindowExec is not fully compatible with Spark (Native WindowExec has known correctness issues). To enable it anyway, set spark.comet.operator.WindowExec.allowIncompatible=true. For more information, refer to the Comet Compatibility Guide (https://datafusion.apache.org/comet/user-guide/compatibility.html)., Partitioning and sorting specifications must be the same.] +- CometColumnarToRow +- CometSort +- CometExchange @@ -9,7 +9,7 @@ :- CometSort : +- CometColumnarExchange : +- Project - : +- Window [COMET: Window expressions are not supported] + : +- Window [COMET: WindowExec is not fully compatible with Spark (Native WindowExec has known correctness issues). To enable it anyway, set spark.comet.operator.WindowExec.allowIncompatible=true. For more information, refer to the Comet Compatibility Guide (https://datafusion.apache.org/comet/user-guide/compatibility.html)., Partitioning and sorting specifications must be the same.] : +- CometColumnarToRow : +- CometSort : +- CometColumnarExchange @@ -34,7 +34,7 @@ +- CometSort +- CometColumnarExchange +- Project - +- Window [COMET: Window expressions are not supported] + +- Window [COMET: WindowExec is not fully compatible with Spark (Native WindowExec has known correctness issues). To enable it anyway, set spark.comet.operator.WindowExec.allowIncompatible=true. For more information, refer to the Comet Compatibility Guide (https://datafusion.apache.org/comet/user-guide/compatibility.html)., Partitioning and sorting specifications must be the same.] +- CometColumnarToRow +- CometSort +- CometColumnarExchange diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q51/extended.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q51/extended.txt index d5de75daf6..f35e106153 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q51/extended.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q51/extended.txt @@ -1,6 +1,6 @@ TakeOrderedAndProject [COMET: ] +- Filter - +- Window [COMET: Window expressions are not supported] + +- Window [COMET: WindowExec is not fully compatible with Spark (Native WindowExec has known correctness issues). To enable it anyway, set spark.comet.operator.WindowExec.allowIncompatible=true. For more information, refer to the Comet Compatibility Guide (https://datafusion.apache.org/comet/user-guide/compatibility.html)., Partitioning and sorting specifications must be the same.] +- CometColumnarToRow +- CometSort +- CometExchange @@ -9,7 +9,7 @@ :- CometSort : +- CometColumnarExchange : +- Project - : +- Window [COMET: Window expressions are not supported] + : +- Window [COMET: WindowExec is not fully compatible with Spark (Native WindowExec has known correctness issues). To enable it anyway, set spark.comet.operator.WindowExec.allowIncompatible=true. For more information, refer to the Comet Compatibility Guide (https://datafusion.apache.org/comet/user-guide/compatibility.html)., Partitioning and sorting specifications must be the same.] : +- CometColumnarToRow : +- CometSort : +- CometColumnarExchange @@ -34,7 +34,7 @@ +- CometSort +- CometColumnarExchange +- Project - +- Window [COMET: Window expressions are not supported] + +- Window [COMET: WindowExec is not fully compatible with Spark (Native WindowExec has known correctness issues). To enable it anyway, set spark.comet.operator.WindowExec.allowIncompatible=true. For more information, refer to the Comet Compatibility Guide (https://datafusion.apache.org/comet/user-guide/compatibility.html)., Partitioning and sorting specifications must be the same.] +- CometColumnarToRow +- CometSort +- CometColumnarExchange diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q53.native_iceberg_compat/extended.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q53.native_iceberg_compat/extended.txt index 577a05e76d..25ab91b3b3 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q53.native_iceberg_compat/extended.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q53.native_iceberg_compat/extended.txt @@ -1,7 +1,7 @@ TakeOrderedAndProject [COMET: ] +- Project +- Filter - +- Window [COMET: Window expressions are not supported] + +- Window [COMET: WindowExec is not fully compatible with Spark (Native WindowExec has known correctness issues). To enable it anyway, set spark.comet.operator.WindowExec.allowIncompatible=true. For more information, refer to the Comet Compatibility Guide (https://datafusion.apache.org/comet/user-guide/compatibility.html).] +- CometColumnarToRow +- CometSort +- CometColumnarExchange diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q53/extended.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q53/extended.txt index 577a05e76d..25ab91b3b3 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q53/extended.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q53/extended.txt @@ -1,7 +1,7 @@ TakeOrderedAndProject [COMET: ] +- Project +- Filter - +- Window [COMET: Window expressions are not supported] + +- Window [COMET: WindowExec is not fully compatible with Spark (Native WindowExec has known correctness issues). To enable it anyway, set spark.comet.operator.WindowExec.allowIncompatible=true. For more information, refer to the Comet Compatibility Guide (https://datafusion.apache.org/comet/user-guide/compatibility.html).] +- CometColumnarToRow +- CometSort +- CometColumnarExchange diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q57.native_iceberg_compat/extended.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q57.native_iceberg_compat/extended.txt index 1c4e1f9890..2888ddcfac 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q57.native_iceberg_compat/extended.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q57.native_iceberg_compat/extended.txt @@ -7,7 +7,7 @@ : : +- Filter : : +- Window : : +- Filter - : : +- Window [COMET: Window expressions are not supported] + : : +- Window [COMET: WindowExec is not fully compatible with Spark (Native WindowExec has known correctness issues). To enable it anyway, set spark.comet.operator.WindowExec.allowIncompatible=true. For more information, refer to the Comet Compatibility Guide (https://datafusion.apache.org/comet/user-guide/compatibility.html).] : : +- CometColumnarToRow : : +- CometSort : : +- CometColumnarExchange @@ -40,7 +40,7 @@ : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.call_center : +- BroadcastExchange : +- Project - : +- Window [COMET: Window expressions are not supported] + : +- Window [COMET: WindowExec is not fully compatible with Spark (Native WindowExec has known correctness issues). To enable it anyway, set spark.comet.operator.WindowExec.allowIncompatible=true. For more information, refer to the Comet Compatibility Guide (https://datafusion.apache.org/comet/user-guide/compatibility.html).] : +- CometColumnarToRow : +- CometSort : +- CometColumnarExchange @@ -73,7 +73,7 @@ : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.call_center +- BroadcastExchange +- Project - +- Window [COMET: Window expressions are not supported] + +- Window [COMET: WindowExec is not fully compatible with Spark (Native WindowExec has known correctness issues). To enable it anyway, set spark.comet.operator.WindowExec.allowIncompatible=true. For more information, refer to the Comet Compatibility Guide (https://datafusion.apache.org/comet/user-guide/compatibility.html).] +- CometColumnarToRow +- CometSort +- CometColumnarExchange diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q57/extended.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q57/extended.txt index 1c4e1f9890..2888ddcfac 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q57/extended.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q57/extended.txt @@ -7,7 +7,7 @@ : : +- Filter : : +- Window : : +- Filter - : : +- Window [COMET: Window expressions are not supported] + : : +- Window [COMET: WindowExec is not fully compatible with Spark (Native WindowExec has known correctness issues). To enable it anyway, set spark.comet.operator.WindowExec.allowIncompatible=true. For more information, refer to the Comet Compatibility Guide (https://datafusion.apache.org/comet/user-guide/compatibility.html).] : : +- CometColumnarToRow : : +- CometSort : : +- CometColumnarExchange @@ -40,7 +40,7 @@ : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.call_center : +- BroadcastExchange : +- Project - : +- Window [COMET: Window expressions are not supported] + : +- Window [COMET: WindowExec is not fully compatible with Spark (Native WindowExec has known correctness issues). To enable it anyway, set spark.comet.operator.WindowExec.allowIncompatible=true. For more information, refer to the Comet Compatibility Guide (https://datafusion.apache.org/comet/user-guide/compatibility.html).] : +- CometColumnarToRow : +- CometSort : +- CometColumnarExchange @@ -73,7 +73,7 @@ : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.call_center +- BroadcastExchange +- Project - +- Window [COMET: Window expressions are not supported] + +- Window [COMET: WindowExec is not fully compatible with Spark (Native WindowExec has known correctness issues). To enable it anyway, set spark.comet.operator.WindowExec.allowIncompatible=true. For more information, refer to the Comet Compatibility Guide (https://datafusion.apache.org/comet/user-guide/compatibility.html).] +- CometColumnarToRow +- CometSort +- CometColumnarExchange diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q63.native_iceberg_compat/extended.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q63.native_iceberg_compat/extended.txt index 2d0e88e77e..ce29988210 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q63.native_iceberg_compat/extended.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q63.native_iceberg_compat/extended.txt @@ -1,7 +1,7 @@ TakeOrderedAndProject [COMET: ] +- Project +- Filter - +- Window [COMET: Window expressions are not supported] + +- Window [COMET: WindowExec is not fully compatible with Spark (Native WindowExec has known correctness issues). To enable it anyway, set spark.comet.operator.WindowExec.allowIncompatible=true. For more information, refer to the Comet Compatibility Guide (https://datafusion.apache.org/comet/user-guide/compatibility.html).] +- CometColumnarToRow +- CometSort +- CometColumnarExchange diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q63/extended.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q63/extended.txt index 2d0e88e77e..ce29988210 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q63/extended.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q63/extended.txt @@ -1,7 +1,7 @@ TakeOrderedAndProject [COMET: ] +- Project +- Filter - +- Window [COMET: Window expressions are not supported] + +- Window [COMET: WindowExec is not fully compatible with Spark (Native WindowExec has known correctness issues). To enable it anyway, set spark.comet.operator.WindowExec.allowIncompatible=true. For more information, refer to the Comet Compatibility Guide (https://datafusion.apache.org/comet/user-guide/compatibility.html).] +- CometColumnarToRow +- CometSort +- CometColumnarExchange diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q70.native_iceberg_compat/extended.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q70.native_iceberg_compat/extended.txt index 4cf87804d7..829e46834b 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q70.native_iceberg_compat/extended.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q70.native_iceberg_compat/extended.txt @@ -1,6 +1,6 @@ TakeOrderedAndProject [COMET: ] +- Project - +- Window [COMET: Window expressions are not supported] + +- Window [COMET: WindowExec is not fully compatible with Spark (Native WindowExec has known correctness issues). To enable it anyway, set spark.comet.operator.WindowExec.allowIncompatible=true. For more information, refer to the Comet Compatibility Guide (https://datafusion.apache.org/comet/user-guide/compatibility.html).] +- CometColumnarToRow +- CometSort +- CometColumnarExchange diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q70/extended.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q70/extended.txt index 4cf87804d7..829e46834b 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q70/extended.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q70/extended.txt @@ -1,6 +1,6 @@ TakeOrderedAndProject [COMET: ] +- Project - +- Window [COMET: Window expressions are not supported] + +- Window [COMET: WindowExec is not fully compatible with Spark (Native WindowExec has known correctness issues). To enable it anyway, set spark.comet.operator.WindowExec.allowIncompatible=true. For more information, refer to the Comet Compatibility Guide (https://datafusion.apache.org/comet/user-guide/compatibility.html).] +- CometColumnarToRow +- CometSort +- CometColumnarExchange diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q86.native_iceberg_compat/extended.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q86.native_iceberg_compat/extended.txt index 7653245f61..4deba7fb9b 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q86.native_iceberg_compat/extended.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q86.native_iceberg_compat/extended.txt @@ -1,6 +1,6 @@ TakeOrderedAndProject [COMET: ] +- Project - +- Window [COMET: Window expressions are not supported] + +- Window [COMET: WindowExec is not fully compatible with Spark (Native WindowExec has known correctness issues). To enable it anyway, set spark.comet.operator.WindowExec.allowIncompatible=true. For more information, refer to the Comet Compatibility Guide (https://datafusion.apache.org/comet/user-guide/compatibility.html).] +- CometColumnarToRow +- CometSort +- CometColumnarExchange diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q86/extended.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q86/extended.txt index 7653245f61..4deba7fb9b 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q86/extended.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q86/extended.txt @@ -1,6 +1,6 @@ TakeOrderedAndProject [COMET: ] +- Project - +- Window [COMET: Window expressions are not supported] + +- Window [COMET: WindowExec is not fully compatible with Spark (Native WindowExec has known correctness issues). To enable it anyway, set spark.comet.operator.WindowExec.allowIncompatible=true. For more information, refer to the Comet Compatibility Guide (https://datafusion.apache.org/comet/user-guide/compatibility.html).] +- CometColumnarToRow +- CometSort +- CometColumnarExchange diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q89.native_iceberg_compat/extended.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q89.native_iceberg_compat/extended.txt index 451a45a044..99421ae087 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q89.native_iceberg_compat/extended.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q89.native_iceberg_compat/extended.txt @@ -1,7 +1,7 @@ TakeOrderedAndProject [COMET: ] +- Project +- Filter - +- Window [COMET: Window expressions are not supported] + +- Window [COMET: WindowExec is not fully compatible with Spark (Native WindowExec has known correctness issues). To enable it anyway, set spark.comet.operator.WindowExec.allowIncompatible=true. For more information, refer to the Comet Compatibility Guide (https://datafusion.apache.org/comet/user-guide/compatibility.html).] +- CometColumnarToRow +- CometSort +- CometColumnarExchange diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q89/extended.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q89/extended.txt index 451a45a044..99421ae087 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q89/extended.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q89/extended.txt @@ -1,7 +1,7 @@ TakeOrderedAndProject [COMET: ] +- Project +- Filter - +- Window [COMET: Window expressions are not supported] + +- Window [COMET: WindowExec is not fully compatible with Spark (Native WindowExec has known correctness issues). To enable it anyway, set spark.comet.operator.WindowExec.allowIncompatible=true. For more information, refer to the Comet Compatibility Guide (https://datafusion.apache.org/comet/user-guide/compatibility.html).] +- CometColumnarToRow +- CometSort +- CometColumnarExchange diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q98.native_iceberg_compat/extended.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q98.native_iceberg_compat/extended.txt index 434f1aa99e..aaefeae86c 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q98.native_iceberg_compat/extended.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q98.native_iceberg_compat/extended.txt @@ -3,7 +3,7 @@ CometColumnarToRow +- CometSort +- CometColumnarExchange +- Project - +- Window [COMET: Window expressions are not supported] + +- Window [COMET: WindowExec is not fully compatible with Spark (Native WindowExec has known correctness issues). To enable it anyway, set spark.comet.operator.WindowExec.allowIncompatible=true. For more information, refer to the Comet Compatibility Guide (https://datafusion.apache.org/comet/user-guide/compatibility.html).] +- CometColumnarToRow +- CometSort +- CometColumnarExchange diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q98/extended.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q98/extended.txt index 434f1aa99e..aaefeae86c 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q98/extended.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q98/extended.txt @@ -3,7 +3,7 @@ CometColumnarToRow +- CometSort +- CometColumnarExchange +- Project - +- Window [COMET: Window expressions are not supported] + +- Window [COMET: WindowExec is not fully compatible with Spark (Native WindowExec has known correctness issues). To enable it anyway, set spark.comet.operator.WindowExec.allowIncompatible=true. For more information, refer to the Comet Compatibility Guide (https://datafusion.apache.org/comet/user-guide/compatibility.html).] +- CometColumnarToRow +- CometSort +- CometColumnarExchange diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7-spark4_0/q12.native_iceberg_compat/extended.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7-spark4_0/q12.native_iceberg_compat/extended.txt index 6a604adb85..cea46c263c 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7-spark4_0/q12.native_iceberg_compat/extended.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7-spark4_0/q12.native_iceberg_compat/extended.txt @@ -1,6 +1,6 @@ TakeOrderedAndProject [COMET: ] +- Project - +- Window [COMET: Window expressions are not supported] + +- Window [COMET: WindowExec is not fully compatible with Spark (Native WindowExec has known correctness issues). To enable it anyway, set spark.comet.operator.WindowExec.allowIncompatible=true. For more information, refer to the Comet Compatibility Guide (https://datafusion.apache.org/comet/user-guide/compatibility.html).] +- CometColumnarToRow +- CometSort +- CometColumnarExchange diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7-spark4_0/q12/extended.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7-spark4_0/q12/extended.txt index 6a604adb85..cea46c263c 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7-spark4_0/q12/extended.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7-spark4_0/q12/extended.txt @@ -1,6 +1,6 @@ TakeOrderedAndProject [COMET: ] +- Project - +- Window [COMET: Window expressions are not supported] + +- Window [COMET: WindowExec is not fully compatible with Spark (Native WindowExec has known correctness issues). To enable it anyway, set spark.comet.operator.WindowExec.allowIncompatible=true. For more information, refer to the Comet Compatibility Guide (https://datafusion.apache.org/comet/user-guide/compatibility.html).] +- CometColumnarToRow +- CometSort +- CometColumnarExchange diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7-spark4_0/q20.native_iceberg_compat/extended.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7-spark4_0/q20.native_iceberg_compat/extended.txt index 956b109103..cd64190012 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7-spark4_0/q20.native_iceberg_compat/extended.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7-spark4_0/q20.native_iceberg_compat/extended.txt @@ -1,6 +1,6 @@ TakeOrderedAndProject [COMET: ] +- Project - +- Window [COMET: Window expressions are not supported] + +- Window [COMET: WindowExec is not fully compatible with Spark (Native WindowExec has known correctness issues). To enable it anyway, set spark.comet.operator.WindowExec.allowIncompatible=true. For more information, refer to the Comet Compatibility Guide (https://datafusion.apache.org/comet/user-guide/compatibility.html).] +- CometColumnarToRow +- CometSort +- CometColumnarExchange diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7-spark4_0/q20/extended.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7-spark4_0/q20/extended.txt index 956b109103..cd64190012 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7-spark4_0/q20/extended.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7-spark4_0/q20/extended.txt @@ -1,6 +1,6 @@ TakeOrderedAndProject [COMET: ] +- Project - +- Window [COMET: Window expressions are not supported] + +- Window [COMET: WindowExec is not fully compatible with Spark (Native WindowExec has known correctness issues). To enable it anyway, set spark.comet.operator.WindowExec.allowIncompatible=true. For more information, refer to the Comet Compatibility Guide (https://datafusion.apache.org/comet/user-guide/compatibility.html).] +- CometColumnarToRow +- CometSort +- CometColumnarExchange diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7-spark4_0/q36a.native_iceberg_compat/extended.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7-spark4_0/q36a.native_iceberg_compat/extended.txt index 39168c9476..8e08c2aa7d 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7-spark4_0/q36a.native_iceberg_compat/extended.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7-spark4_0/q36a.native_iceberg_compat/extended.txt @@ -1,6 +1,6 @@ TakeOrderedAndProject [COMET: ] +- Project - +- Window [COMET: Window expressions are not supported] + +- Window [COMET: WindowExec is not fully compatible with Spark (Native WindowExec has known correctness issues). To enable it anyway, set spark.comet.operator.WindowExec.allowIncompatible=true. For more information, refer to the Comet Compatibility Guide (https://datafusion.apache.org/comet/user-guide/compatibility.html).] +- CometColumnarToRow +- CometSort +- CometExchange diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7-spark4_0/q36a/extended.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7-spark4_0/q36a/extended.txt index 39168c9476..8e08c2aa7d 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7-spark4_0/q36a/extended.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7-spark4_0/q36a/extended.txt @@ -1,6 +1,6 @@ TakeOrderedAndProject [COMET: ] +- Project - +- Window [COMET: Window expressions are not supported] + +- Window [COMET: WindowExec is not fully compatible with Spark (Native WindowExec has known correctness issues). To enable it anyway, set spark.comet.operator.WindowExec.allowIncompatible=true. For more information, refer to the Comet Compatibility Guide (https://datafusion.apache.org/comet/user-guide/compatibility.html).] +- CometColumnarToRow +- CometSort +- CometExchange diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7-spark4_0/q47.native_iceberg_compat/extended.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7-spark4_0/q47.native_iceberg_compat/extended.txt index 1a4201332c..1a62aa48d9 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7-spark4_0/q47.native_iceberg_compat/extended.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7-spark4_0/q47.native_iceberg_compat/extended.txt @@ -7,7 +7,7 @@ : : +- Filter : : +- Window : : +- Filter - : : +- Window [COMET: Window expressions are not supported] + : : +- Window [COMET: WindowExec is not fully compatible with Spark (Native WindowExec has known correctness issues). To enable it anyway, set spark.comet.operator.WindowExec.allowIncompatible=true. For more information, refer to the Comet Compatibility Guide (https://datafusion.apache.org/comet/user-guide/compatibility.html).] : : +- CometColumnarToRow : : +- CometSort : : +- CometColumnarExchange @@ -40,7 +40,7 @@ : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.store : +- BroadcastExchange : +- Project - : +- Window [COMET: Window expressions are not supported] + : +- Window [COMET: WindowExec is not fully compatible with Spark (Native WindowExec has known correctness issues). To enable it anyway, set spark.comet.operator.WindowExec.allowIncompatible=true. For more information, refer to the Comet Compatibility Guide (https://datafusion.apache.org/comet/user-guide/compatibility.html).] : +- CometColumnarToRow : +- CometSort : +- CometColumnarExchange @@ -73,7 +73,7 @@ : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.store +- BroadcastExchange +- Project - +- Window [COMET: Window expressions are not supported] + +- Window [COMET: WindowExec is not fully compatible with Spark (Native WindowExec has known correctness issues). To enable it anyway, set spark.comet.operator.WindowExec.allowIncompatible=true. For more information, refer to the Comet Compatibility Guide (https://datafusion.apache.org/comet/user-guide/compatibility.html).] +- CometColumnarToRow +- CometSort +- CometColumnarExchange diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7-spark4_0/q47/extended.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7-spark4_0/q47/extended.txt index 1a4201332c..1a62aa48d9 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7-spark4_0/q47/extended.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7-spark4_0/q47/extended.txt @@ -7,7 +7,7 @@ : : +- Filter : : +- Window : : +- Filter - : : +- Window [COMET: Window expressions are not supported] + : : +- Window [COMET: WindowExec is not fully compatible with Spark (Native WindowExec has known correctness issues). To enable it anyway, set spark.comet.operator.WindowExec.allowIncompatible=true. For more information, refer to the Comet Compatibility Guide (https://datafusion.apache.org/comet/user-guide/compatibility.html).] : : +- CometColumnarToRow : : +- CometSort : : +- CometColumnarExchange @@ -40,7 +40,7 @@ : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.store : +- BroadcastExchange : +- Project - : +- Window [COMET: Window expressions are not supported] + : +- Window [COMET: WindowExec is not fully compatible with Spark (Native WindowExec has known correctness issues). To enable it anyway, set spark.comet.operator.WindowExec.allowIncompatible=true. For more information, refer to the Comet Compatibility Guide (https://datafusion.apache.org/comet/user-guide/compatibility.html).] : +- CometColumnarToRow : +- CometSort : +- CometColumnarExchange @@ -73,7 +73,7 @@ : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.store +- BroadcastExchange +- Project - +- Window [COMET: Window expressions are not supported] + +- Window [COMET: WindowExec is not fully compatible with Spark (Native WindowExec has known correctness issues). To enable it anyway, set spark.comet.operator.WindowExec.allowIncompatible=true. For more information, refer to the Comet Compatibility Guide (https://datafusion.apache.org/comet/user-guide/compatibility.html).] +- CometColumnarToRow +- CometSort +- CometColumnarExchange diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7-spark4_0/q49.native_iceberg_compat/extended.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7-spark4_0/q49.native_iceberg_compat/extended.txt index 734b0ce7bb..75684a9669 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7-spark4_0/q49.native_iceberg_compat/extended.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7-spark4_0/q49.native_iceberg_compat/extended.txt @@ -8,7 +8,7 @@ CometColumnarToRow : +- Filter : +- Window : +- Sort - : +- Window [COMET: Window expressions are not supported] + : +- Window [COMET: WindowExec is not fully compatible with Spark (Native WindowExec has known correctness issues). To enable it anyway, set spark.comet.operator.WindowExec.allowIncompatible=true. For more information, refer to the Comet Compatibility Guide (https://datafusion.apache.org/comet/user-guide/compatibility.html).] : +- CometColumnarToRow : +- CometSort : +- CometExchange @@ -40,7 +40,7 @@ CometColumnarToRow : +- Filter : +- Window : +- Sort - : +- Window [COMET: Window expressions are not supported] + : +- Window [COMET: WindowExec is not fully compatible with Spark (Native WindowExec has known correctness issues). To enable it anyway, set spark.comet.operator.WindowExec.allowIncompatible=true. For more information, refer to the Comet Compatibility Guide (https://datafusion.apache.org/comet/user-guide/compatibility.html).] : +- CometColumnarToRow : +- CometSort : +- CometExchange @@ -67,7 +67,7 @@ CometColumnarToRow +- Filter +- Window +- Sort - +- Window [COMET: Window expressions are not supported] + +- Window [COMET: WindowExec is not fully compatible with Spark (Native WindowExec has known correctness issues). To enable it anyway, set spark.comet.operator.WindowExec.allowIncompatible=true. For more information, refer to the Comet Compatibility Guide (https://datafusion.apache.org/comet/user-guide/compatibility.html).] +- CometColumnarToRow +- CometSort +- CometExchange diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7-spark4_0/q49/extended.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7-spark4_0/q49/extended.txt index 734b0ce7bb..75684a9669 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7-spark4_0/q49/extended.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7-spark4_0/q49/extended.txt @@ -8,7 +8,7 @@ CometColumnarToRow : +- Filter : +- Window : +- Sort - : +- Window [COMET: Window expressions are not supported] + : +- Window [COMET: WindowExec is not fully compatible with Spark (Native WindowExec has known correctness issues). To enable it anyway, set spark.comet.operator.WindowExec.allowIncompatible=true. For more information, refer to the Comet Compatibility Guide (https://datafusion.apache.org/comet/user-guide/compatibility.html).] : +- CometColumnarToRow : +- CometSort : +- CometExchange @@ -40,7 +40,7 @@ CometColumnarToRow : +- Filter : +- Window : +- Sort - : +- Window [COMET: Window expressions are not supported] + : +- Window [COMET: WindowExec is not fully compatible with Spark (Native WindowExec has known correctness issues). To enable it anyway, set spark.comet.operator.WindowExec.allowIncompatible=true. For more information, refer to the Comet Compatibility Guide (https://datafusion.apache.org/comet/user-guide/compatibility.html).] : +- CometColumnarToRow : +- CometSort : +- CometExchange @@ -67,7 +67,7 @@ CometColumnarToRow +- Filter +- Window +- Sort - +- Window [COMET: Window expressions are not supported] + +- Window [COMET: WindowExec is not fully compatible with Spark (Native WindowExec has known correctness issues). To enable it anyway, set spark.comet.operator.WindowExec.allowIncompatible=true. For more information, refer to the Comet Compatibility Guide (https://datafusion.apache.org/comet/user-guide/compatibility.html).] +- CometColumnarToRow +- CometSort +- CometExchange diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7-spark4_0/q51a.native_iceberg_compat/extended.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7-spark4_0/q51a.native_iceberg_compat/extended.txt index f7546ea154..0c6c1fe8dd 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7-spark4_0/q51a.native_iceberg_compat/extended.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7-spark4_0/q51a.native_iceberg_compat/extended.txt @@ -4,7 +4,7 @@ +- HashAggregate +- Project +- BroadcastHashJoin - :- Window [COMET: Window expressions are not supported] + :- Window [COMET: WindowExec is not fully compatible with Spark (Native WindowExec has known correctness issues). To enable it anyway, set spark.comet.operator.WindowExec.allowIncompatible=true. For more information, refer to the Comet Compatibility Guide (https://datafusion.apache.org/comet/user-guide/compatibility.html)., Partitioning and sorting specifications must be the same.] : +- CometColumnarToRow : +- CometSort : +- CometExchange @@ -20,7 +20,7 @@ : : +- Project : : +- BroadcastHashJoin : : :- Project - : : : +- Window [COMET: Window expressions are not supported] + : : : +- Window [COMET: WindowExec is not fully compatible with Spark (Native WindowExec has known correctness issues). To enable it anyway, set spark.comet.operator.WindowExec.allowIncompatible=true. For more information, refer to the Comet Compatibility Guide (https://datafusion.apache.org/comet/user-guide/compatibility.html)., Partitioning and sorting specifications must be the same.] : : : +- CometColumnarToRow : : : +- CometSort : : : +- CometColumnarExchange @@ -44,7 +44,7 @@ : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim : : +- BroadcastExchange : : +- Project - : : +- Window [COMET: Window expressions are not supported] + : : +- Window [COMET: WindowExec is not fully compatible with Spark (Native WindowExec has known correctness issues). To enable it anyway, set spark.comet.operator.WindowExec.allowIncompatible=true. For more information, refer to the Comet Compatibility Guide (https://datafusion.apache.org/comet/user-guide/compatibility.html)., Partitioning and sorting specifications must be the same.] : : +- CometColumnarToRow : : +- CometSort : : +- CometColumnarExchange @@ -75,7 +75,7 @@ : +- Project : +- BroadcastHashJoin : :- Project - : : +- Window [COMET: Window expressions are not supported] + : : +- Window [COMET: WindowExec is not fully compatible with Spark (Native WindowExec has known correctness issues). To enable it anyway, set spark.comet.operator.WindowExec.allowIncompatible=true. For more information, refer to the Comet Compatibility Guide (https://datafusion.apache.org/comet/user-guide/compatibility.html)., Partitioning and sorting specifications must be the same.] : : +- CometColumnarToRow : : +- CometSort : : +- CometColumnarExchange @@ -94,7 +94,7 @@ : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim : +- BroadcastExchange : +- Project - : +- Window [COMET: Window expressions are not supported] + : +- Window [COMET: WindowExec is not fully compatible with Spark (Native WindowExec has known correctness issues). To enable it anyway, set spark.comet.operator.WindowExec.allowIncompatible=true. For more information, refer to the Comet Compatibility Guide (https://datafusion.apache.org/comet/user-guide/compatibility.html)., Partitioning and sorting specifications must be the same.] : +- CometColumnarToRow : +- CometSort : +- CometColumnarExchange @@ -113,7 +113,7 @@ : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim +- BroadcastExchange +- Project - +- Window [COMET: Window expressions are not supported] + +- Window [COMET: WindowExec is not fully compatible with Spark (Native WindowExec has known correctness issues). To enable it anyway, set spark.comet.operator.WindowExec.allowIncompatible=true. For more information, refer to the Comet Compatibility Guide (https://datafusion.apache.org/comet/user-guide/compatibility.html)., Partitioning and sorting specifications must be the same.] +- CometColumnarToRow +- CometSort +- CometExchange @@ -129,7 +129,7 @@ : +- Project : +- BroadcastHashJoin : :- Project - : : +- Window [COMET: Window expressions are not supported] + : : +- Window [COMET: WindowExec is not fully compatible with Spark (Native WindowExec has known correctness issues). To enable it anyway, set spark.comet.operator.WindowExec.allowIncompatible=true. For more information, refer to the Comet Compatibility Guide (https://datafusion.apache.org/comet/user-guide/compatibility.html)., Partitioning and sorting specifications must be the same.] : : +- CometColumnarToRow : : +- CometSort : : +- CometColumnarExchange @@ -153,7 +153,7 @@ : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim : +- BroadcastExchange : +- Project - : +- Window [COMET: Window expressions are not supported] + : +- Window [COMET: WindowExec is not fully compatible with Spark (Native WindowExec has known correctness issues). To enable it anyway, set spark.comet.operator.WindowExec.allowIncompatible=true. For more information, refer to the Comet Compatibility Guide (https://datafusion.apache.org/comet/user-guide/compatibility.html)., Partitioning and sorting specifications must be the same.] : +- CometColumnarToRow : +- CometSort : +- CometColumnarExchange @@ -184,7 +184,7 @@ +- Project +- BroadcastHashJoin :- Project - : +- Window [COMET: Window expressions are not supported] + : +- Window [COMET: WindowExec is not fully compatible with Spark (Native WindowExec has known correctness issues). To enable it anyway, set spark.comet.operator.WindowExec.allowIncompatible=true. For more information, refer to the Comet Compatibility Guide (https://datafusion.apache.org/comet/user-guide/compatibility.html)., Partitioning and sorting specifications must be the same.] : +- CometColumnarToRow : +- CometSort : +- CometColumnarExchange @@ -203,7 +203,7 @@ : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim +- BroadcastExchange +- Project - +- Window [COMET: Window expressions are not supported] + +- Window [COMET: WindowExec is not fully compatible with Spark (Native WindowExec has known correctness issues). To enable it anyway, set spark.comet.operator.WindowExec.allowIncompatible=true. For more information, refer to the Comet Compatibility Guide (https://datafusion.apache.org/comet/user-guide/compatibility.html)., Partitioning and sorting specifications must be the same.] +- CometColumnarToRow +- CometSort +- CometColumnarExchange diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7-spark4_0/q51a/extended.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7-spark4_0/q51a/extended.txt index f7546ea154..0c6c1fe8dd 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7-spark4_0/q51a/extended.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7-spark4_0/q51a/extended.txt @@ -4,7 +4,7 @@ +- HashAggregate +- Project +- BroadcastHashJoin - :- Window [COMET: Window expressions are not supported] + :- Window [COMET: WindowExec is not fully compatible with Spark (Native WindowExec has known correctness issues). To enable it anyway, set spark.comet.operator.WindowExec.allowIncompatible=true. For more information, refer to the Comet Compatibility Guide (https://datafusion.apache.org/comet/user-guide/compatibility.html)., Partitioning and sorting specifications must be the same.] : +- CometColumnarToRow : +- CometSort : +- CometExchange @@ -20,7 +20,7 @@ : : +- Project : : +- BroadcastHashJoin : : :- Project - : : : +- Window [COMET: Window expressions are not supported] + : : : +- Window [COMET: WindowExec is not fully compatible with Spark (Native WindowExec has known correctness issues). To enable it anyway, set spark.comet.operator.WindowExec.allowIncompatible=true. For more information, refer to the Comet Compatibility Guide (https://datafusion.apache.org/comet/user-guide/compatibility.html)., Partitioning and sorting specifications must be the same.] : : : +- CometColumnarToRow : : : +- CometSort : : : +- CometColumnarExchange @@ -44,7 +44,7 @@ : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim : : +- BroadcastExchange : : +- Project - : : +- Window [COMET: Window expressions are not supported] + : : +- Window [COMET: WindowExec is not fully compatible with Spark (Native WindowExec has known correctness issues). To enable it anyway, set spark.comet.operator.WindowExec.allowIncompatible=true. For more information, refer to the Comet Compatibility Guide (https://datafusion.apache.org/comet/user-guide/compatibility.html)., Partitioning and sorting specifications must be the same.] : : +- CometColumnarToRow : : +- CometSort : : +- CometColumnarExchange @@ -75,7 +75,7 @@ : +- Project : +- BroadcastHashJoin : :- Project - : : +- Window [COMET: Window expressions are not supported] + : : +- Window [COMET: WindowExec is not fully compatible with Spark (Native WindowExec has known correctness issues). To enable it anyway, set spark.comet.operator.WindowExec.allowIncompatible=true. For more information, refer to the Comet Compatibility Guide (https://datafusion.apache.org/comet/user-guide/compatibility.html)., Partitioning and sorting specifications must be the same.] : : +- CometColumnarToRow : : +- CometSort : : +- CometColumnarExchange @@ -94,7 +94,7 @@ : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim : +- BroadcastExchange : +- Project - : +- Window [COMET: Window expressions are not supported] + : +- Window [COMET: WindowExec is not fully compatible with Spark (Native WindowExec has known correctness issues). To enable it anyway, set spark.comet.operator.WindowExec.allowIncompatible=true. For more information, refer to the Comet Compatibility Guide (https://datafusion.apache.org/comet/user-guide/compatibility.html)., Partitioning and sorting specifications must be the same.] : +- CometColumnarToRow : +- CometSort : +- CometColumnarExchange @@ -113,7 +113,7 @@ : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim +- BroadcastExchange +- Project - +- Window [COMET: Window expressions are not supported] + +- Window [COMET: WindowExec is not fully compatible with Spark (Native WindowExec has known correctness issues). To enable it anyway, set spark.comet.operator.WindowExec.allowIncompatible=true. For more information, refer to the Comet Compatibility Guide (https://datafusion.apache.org/comet/user-guide/compatibility.html)., Partitioning and sorting specifications must be the same.] +- CometColumnarToRow +- CometSort +- CometExchange @@ -129,7 +129,7 @@ : +- Project : +- BroadcastHashJoin : :- Project - : : +- Window [COMET: Window expressions are not supported] + : : +- Window [COMET: WindowExec is not fully compatible with Spark (Native WindowExec has known correctness issues). To enable it anyway, set spark.comet.operator.WindowExec.allowIncompatible=true. For more information, refer to the Comet Compatibility Guide (https://datafusion.apache.org/comet/user-guide/compatibility.html)., Partitioning and sorting specifications must be the same.] : : +- CometColumnarToRow : : +- CometSort : : +- CometColumnarExchange @@ -153,7 +153,7 @@ : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim : +- BroadcastExchange : +- Project - : +- Window [COMET: Window expressions are not supported] + : +- Window [COMET: WindowExec is not fully compatible with Spark (Native WindowExec has known correctness issues). To enable it anyway, set spark.comet.operator.WindowExec.allowIncompatible=true. For more information, refer to the Comet Compatibility Guide (https://datafusion.apache.org/comet/user-guide/compatibility.html)., Partitioning and sorting specifications must be the same.] : +- CometColumnarToRow : +- CometSort : +- CometColumnarExchange @@ -184,7 +184,7 @@ +- Project +- BroadcastHashJoin :- Project - : +- Window [COMET: Window expressions are not supported] + : +- Window [COMET: WindowExec is not fully compatible with Spark (Native WindowExec has known correctness issues). To enable it anyway, set spark.comet.operator.WindowExec.allowIncompatible=true. For more information, refer to the Comet Compatibility Guide (https://datafusion.apache.org/comet/user-guide/compatibility.html)., Partitioning and sorting specifications must be the same.] : +- CometColumnarToRow : +- CometSort : +- CometColumnarExchange @@ -203,7 +203,7 @@ : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim +- BroadcastExchange +- Project - +- Window [COMET: Window expressions are not supported] + +- Window [COMET: WindowExec is not fully compatible with Spark (Native WindowExec has known correctness issues). To enable it anyway, set spark.comet.operator.WindowExec.allowIncompatible=true. For more information, refer to the Comet Compatibility Guide (https://datafusion.apache.org/comet/user-guide/compatibility.html)., Partitioning and sorting specifications must be the same.] +- CometColumnarToRow +- CometSort +- CometColumnarExchange diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7-spark4_0/q57.native_iceberg_compat/extended.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7-spark4_0/q57.native_iceberg_compat/extended.txt index 1c4e1f9890..2888ddcfac 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7-spark4_0/q57.native_iceberg_compat/extended.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7-spark4_0/q57.native_iceberg_compat/extended.txt @@ -7,7 +7,7 @@ : : +- Filter : : +- Window : : +- Filter - : : +- Window [COMET: Window expressions are not supported] + : : +- Window [COMET: WindowExec is not fully compatible with Spark (Native WindowExec has known correctness issues). To enable it anyway, set spark.comet.operator.WindowExec.allowIncompatible=true. For more information, refer to the Comet Compatibility Guide (https://datafusion.apache.org/comet/user-guide/compatibility.html).] : : +- CometColumnarToRow : : +- CometSort : : +- CometColumnarExchange @@ -40,7 +40,7 @@ : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.call_center : +- BroadcastExchange : +- Project - : +- Window [COMET: Window expressions are not supported] + : +- Window [COMET: WindowExec is not fully compatible with Spark (Native WindowExec has known correctness issues). To enable it anyway, set spark.comet.operator.WindowExec.allowIncompatible=true. For more information, refer to the Comet Compatibility Guide (https://datafusion.apache.org/comet/user-guide/compatibility.html).] : +- CometColumnarToRow : +- CometSort : +- CometColumnarExchange @@ -73,7 +73,7 @@ : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.call_center +- BroadcastExchange +- Project - +- Window [COMET: Window expressions are not supported] + +- Window [COMET: WindowExec is not fully compatible with Spark (Native WindowExec has known correctness issues). To enable it anyway, set spark.comet.operator.WindowExec.allowIncompatible=true. For more information, refer to the Comet Compatibility Guide (https://datafusion.apache.org/comet/user-guide/compatibility.html).] +- CometColumnarToRow +- CometSort +- CometColumnarExchange diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7-spark4_0/q57/extended.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7-spark4_0/q57/extended.txt index 1c4e1f9890..2888ddcfac 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7-spark4_0/q57/extended.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7-spark4_0/q57/extended.txt @@ -7,7 +7,7 @@ : : +- Filter : : +- Window : : +- Filter - : : +- Window [COMET: Window expressions are not supported] + : : +- Window [COMET: WindowExec is not fully compatible with Spark (Native WindowExec has known correctness issues). To enable it anyway, set spark.comet.operator.WindowExec.allowIncompatible=true. For more information, refer to the Comet Compatibility Guide (https://datafusion.apache.org/comet/user-guide/compatibility.html).] : : +- CometColumnarToRow : : +- CometSort : : +- CometColumnarExchange @@ -40,7 +40,7 @@ : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.call_center : +- BroadcastExchange : +- Project - : +- Window [COMET: Window expressions are not supported] + : +- Window [COMET: WindowExec is not fully compatible with Spark (Native WindowExec has known correctness issues). To enable it anyway, set spark.comet.operator.WindowExec.allowIncompatible=true. For more information, refer to the Comet Compatibility Guide (https://datafusion.apache.org/comet/user-guide/compatibility.html).] : +- CometColumnarToRow : +- CometSort : +- CometColumnarExchange @@ -73,7 +73,7 @@ : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.call_center +- BroadcastExchange +- Project - +- Window [COMET: Window expressions are not supported] + +- Window [COMET: WindowExec is not fully compatible with Spark (Native WindowExec has known correctness issues). To enable it anyway, set spark.comet.operator.WindowExec.allowIncompatible=true. For more information, refer to the Comet Compatibility Guide (https://datafusion.apache.org/comet/user-guide/compatibility.html).] +- CometColumnarToRow +- CometSort +- CometColumnarExchange diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7-spark4_0/q70a.native_iceberg_compat/extended.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7-spark4_0/q70a.native_iceberg_compat/extended.txt index fda8aba347..269ae36faa 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7-spark4_0/q70a.native_iceberg_compat/extended.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7-spark4_0/q70a.native_iceberg_compat/extended.txt @@ -1,6 +1,6 @@ TakeOrderedAndProject [COMET: ] +- Project - +- Window [COMET: Window expressions are not supported] + +- Window [COMET: WindowExec is not fully compatible with Spark (Native WindowExec has known correctness issues). To enable it anyway, set spark.comet.operator.WindowExec.allowIncompatible=true. For more information, refer to the Comet Compatibility Guide (https://datafusion.apache.org/comet/user-guide/compatibility.html).] +- CometColumnarToRow +- CometSort +- CometExchange diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7-spark4_0/q70a/extended.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7-spark4_0/q70a/extended.txt index fda8aba347..269ae36faa 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7-spark4_0/q70a/extended.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7-spark4_0/q70a/extended.txt @@ -1,6 +1,6 @@ TakeOrderedAndProject [COMET: ] +- Project - +- Window [COMET: Window expressions are not supported] + +- Window [COMET: WindowExec is not fully compatible with Spark (Native WindowExec has known correctness issues). To enable it anyway, set spark.comet.operator.WindowExec.allowIncompatible=true. For more information, refer to the Comet Compatibility Guide (https://datafusion.apache.org/comet/user-guide/compatibility.html).] +- CometColumnarToRow +- CometSort +- CometExchange diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7-spark4_0/q86a.native_iceberg_compat/extended.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7-spark4_0/q86a.native_iceberg_compat/extended.txt index 2beaad68b0..c5c1247e1b 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7-spark4_0/q86a.native_iceberg_compat/extended.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7-spark4_0/q86a.native_iceberg_compat/extended.txt @@ -1,6 +1,6 @@ TakeOrderedAndProject [COMET: ] +- Project - +- Window [COMET: Window expressions are not supported] + +- Window [COMET: WindowExec is not fully compatible with Spark (Native WindowExec has known correctness issues). To enable it anyway, set spark.comet.operator.WindowExec.allowIncompatible=true. For more information, refer to the Comet Compatibility Guide (https://datafusion.apache.org/comet/user-guide/compatibility.html).] +- CometColumnarToRow +- CometSort +- CometExchange diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7-spark4_0/q86a/extended.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7-spark4_0/q86a/extended.txt index 2beaad68b0..c5c1247e1b 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7-spark4_0/q86a/extended.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7-spark4_0/q86a/extended.txt @@ -1,6 +1,6 @@ TakeOrderedAndProject [COMET: ] +- Project - +- Window [COMET: Window expressions are not supported] + +- Window [COMET: WindowExec is not fully compatible with Spark (Native WindowExec has known correctness issues). To enable it anyway, set spark.comet.operator.WindowExec.allowIncompatible=true. For more information, refer to the Comet Compatibility Guide (https://datafusion.apache.org/comet/user-guide/compatibility.html).] +- CometColumnarToRow +- CometSort +- CometExchange diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7-spark4_0/q98.native_iceberg_compat/extended.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7-spark4_0/q98.native_iceberg_compat/extended.txt index 7517dafcbe..f00a5af531 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7-spark4_0/q98.native_iceberg_compat/extended.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7-spark4_0/q98.native_iceberg_compat/extended.txt @@ -2,7 +2,7 @@ CometColumnarToRow +- CometSort +- CometColumnarExchange +- Project - +- Window [COMET: Window expressions are not supported] + +- Window [COMET: WindowExec is not fully compatible with Spark (Native WindowExec has known correctness issues). To enable it anyway, set spark.comet.operator.WindowExec.allowIncompatible=true. For more information, refer to the Comet Compatibility Guide (https://datafusion.apache.org/comet/user-guide/compatibility.html).] +- CometColumnarToRow +- CometSort +- CometColumnarExchange diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7-spark4_0/q98/extended.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7-spark4_0/q98/extended.txt index 7517dafcbe..f00a5af531 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7-spark4_0/q98/extended.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7-spark4_0/q98/extended.txt @@ -2,7 +2,7 @@ CometColumnarToRow +- CometSort +- CometColumnarExchange +- Project - +- Window [COMET: Window expressions are not supported] + +- Window [COMET: WindowExec is not fully compatible with Spark (Native WindowExec has known correctness issues). To enable it anyway, set spark.comet.operator.WindowExec.allowIncompatible=true. For more information, refer to the Comet Compatibility Guide (https://datafusion.apache.org/comet/user-guide/compatibility.html).] +- CometColumnarToRow +- CometSort +- CometColumnarExchange From 26d24808dbf22ddb0a1711c7d4f0781f6e3d69eb Mon Sep 17 00:00:00 2001 From: Andy Grove Date: Sat, 8 Nov 2025 13:56:04 -0700 Subject: [PATCH 34/44] update expected fallback reason in test --- .../org/apache/comet/exec/CometWindowExecSuite.scala | 12 +++++++----- 1 file changed, 7 insertions(+), 5 deletions(-) diff --git a/spark/src/test/scala/org/apache/comet/exec/CometWindowExecSuite.scala b/spark/src/test/scala/org/apache/comet/exec/CometWindowExecSuite.scala index 60f2b9b500..9f225105fd 100644 --- a/spark/src/test/scala/org/apache/comet/exec/CometWindowExecSuite.scala +++ b/spark/src/test/scala/org/apache/comet/exec/CometWindowExecSuite.scala @@ -284,7 +284,9 @@ class CometWindowExecSuite extends CometTestBase { s"SELECT $function OVER(order by _2 rows between current row and 1 following) FROM t1") queries.foreach { query => - checkSparkAnswerAndFallbackReason(query, "Window expressions are not supported") + checkSparkAnswerAndFallbackReason( + query, + "Native WindowExec has known correctness issues") } } } @@ -303,7 +305,7 @@ class CometWindowExecSuite extends CometTestBase { spark.read.parquet(dir.toString).createOrReplaceTempView("window_test") val df = sql("SELECT a, b, c, COUNT(*) OVER () as cnt FROM window_test") - checkSparkAnswerAndFallbackReason(df, "Window expressions are not supported") + checkSparkAnswerAndFallbackReason(df, "Native WindowExec has known correctness issues") } } @@ -319,11 +321,11 @@ class CometWindowExecSuite extends CometTestBase { spark.read.parquet(dir.toString).createOrReplaceTempView("window_test") val df = sql("SELECT a, b, c, SUM(c) OVER (PARTITION BY a) as sum_c FROM window_test") - checkSparkAnswerAndFallbackReason(df, "Window expressions are not supported") + checkSparkAnswerAndFallbackReason(df, "Native WindowExec has known correctness issues") } } - // TODO: AVG with PARTITION BY and ORDER BY not supported + // TODO: AVG with PARTITION BY and ORDER BY not supportedu // Falls back to Spark Window operator - "Partitioning and sorting specifications must be the same" ignore("window: AVG with PARTITION BY and ORDER BY") { withTempDir { dir => @@ -359,7 +361,7 @@ class CometWindowExecSuite extends CometTestBase { MAX(c) OVER (ORDER BY b) as max_c FROM window_test """) - checkSparkAnswerAndFallbackReason(df, "Window expressions are not supported") + checkSparkAnswerAndFallbackReason(df, "Native WindowExec has known correctness issues") } } From c997b95eeef9f0159917e7002e44eca31fbe7bc4 Mon Sep 17 00:00:00 2001 From: Andy Grove Date: Sat, 8 Nov 2025 14:13:16 -0700 Subject: [PATCH 35/44] hacky fix --- .../apache/comet/rules/CometExecRule.scala | 31 ++++++++++++------- .../apache/comet/serde/QueryPlanSerde.scala | 4 +++ .../comet/exec/CometWindowExecSuite.scala | 2 +- 3 files changed, 25 insertions(+), 12 deletions(-) diff --git a/spark/src/main/scala/org/apache/comet/rules/CometExecRule.scala b/spark/src/main/scala/org/apache/comet/rules/CometExecRule.scala index 873a1c55c9..381ae59539 100644 --- a/spark/src/main/scala/org/apache/comet/rules/CometExecRule.scala +++ b/spark/src/main/scala/org/apache/comet/rules/CometExecRule.scala @@ -392,17 +392,26 @@ case class CometExecRule(session: SparkSession) extends Rule[SparkPlan] { withInfo(s, Seq(info1, info2).flatten.mkString(",")) case w: WindowExec => - newPlanWithProto( - w, - CometWindowExec( - _, - w, - w.output, - w.windowExpression, - w.partitionSpec, - w.orderSpec, - w.child, - SerializedPlan(None))) + if (CometConf.COMET_EXEC_WINDOW_ENABLED.get(conf)) { + // TODO this duplicates logic in QueryPlanSerde + if (CometConf.isOperatorAllowIncompat("WindowExec")) { + newPlanWithProto( + w, + CometWindowExec( + _, + w, + w.output, + w.windowExpression, + w.partitionSpec, + w.orderSpec, + w.child, + SerializedPlan(None))) + } else { + withInfo(w, "Native WindowExec has known correctness issues") + } + } else { + withInfo(w, "WindowExec is not enabled") + } case u: UnionExec if CometConf.COMET_EXEC_UNION_ENABLED.get(conf) && diff --git a/spark/src/main/scala/org/apache/comet/serde/QueryPlanSerde.scala b/spark/src/main/scala/org/apache/comet/serde/QueryPlanSerde.scala index 101e30e97d..464905cd44 100644 --- a/spark/src/main/scala/org/apache/comet/serde/QueryPlanSerde.scala +++ b/spark/src/main/scala/org/apache/comet/serde/QueryPlanSerde.scala @@ -767,11 +767,14 @@ object QueryPlanSerde extends Logging with CometExprShim { val opSerde = handler.asInstanceOf[CometOperatorSerde[SparkPlan]] opSerde.getSupportLevel(op) match { case Unsupported(notes) => + println("Unsupported") withInfo(op, notes.getOrElse("")) case Incompatible(notes) => + println(s"Incompatible ${notes}") val allowIncompat = CometConf.isOperatorAllowIncompat(opName) val incompatConf = CometConf.getOperatorAllowIncompatConfigKey(opName) if (allowIncompat) { + println("allowIncompat") if (notes.isDefined) { logWarning( s"Comet supports $opName when $incompatConf=true " + @@ -787,6 +790,7 @@ object QueryPlanSerde extends Logging with CometExprShim { s"${CometConf.COMPAT_GUIDE}.") } case Compatible(notes) => + println(s"Compatible ${notes}") if (notes.isDefined) { logWarning(s"Comet supports $opName but has notes: ${notes.get}") } diff --git a/spark/src/test/scala/org/apache/comet/exec/CometWindowExecSuite.scala b/spark/src/test/scala/org/apache/comet/exec/CometWindowExecSuite.scala index 9f225105fd..8132459750 100644 --- a/spark/src/test/scala/org/apache/comet/exec/CometWindowExecSuite.scala +++ b/spark/src/test/scala/org/apache/comet/exec/CometWindowExecSuite.scala @@ -206,7 +206,7 @@ class CometWindowExecSuite extends CometTestBase { } } - test("aggregate window function for all types") { + ignore("aggregate window function for all types") { val numValues = 2048 Seq(1, 100, numValues).foreach { numGroups => From 50b62b8a948f1436adefa9b08747550611dd9079 Mon Sep 17 00:00:00 2001 From: Andy Grove Date: Sat, 8 Nov 2025 14:19:54 -0700 Subject: [PATCH 36/44] less hacky fix --- .../apache/comet/rules/CometExecRule.scala | 29 +++-- .../apache/comet/serde/QueryPlanSerde.scala | 100 ++++++++++-------- .../comet/exec/CometWindowExecSuite.scala | 2 +- 3 files changed, 66 insertions(+), 65 deletions(-) diff --git a/spark/src/main/scala/org/apache/comet/rules/CometExecRule.scala b/spark/src/main/scala/org/apache/comet/rules/CometExecRule.scala index 381ae59539..6f9dbf8e7e 100644 --- a/spark/src/main/scala/org/apache/comet/rules/CometExecRule.scala +++ b/spark/src/main/scala/org/apache/comet/rules/CometExecRule.scala @@ -44,8 +44,8 @@ import org.apache.spark.sql.types._ import org.apache.comet.{CometConf, ExtendedExplainInfo} import org.apache.comet.CometConf.COMET_EXEC_SHUFFLE_ENABLED import org.apache.comet.CometSparkSessionExtensions._ +import org.apache.comet.serde.{CometWindow, QueryPlanSerde} import org.apache.comet.serde.OperatorOuterClass.Operator -import org.apache.comet.serde.QueryPlanSerde /** * Spark physical optimizer rule for replacing Spark operators with Comet operators. @@ -392,23 +392,18 @@ case class CometExecRule(session: SparkSession) extends Rule[SparkPlan] { withInfo(s, Seq(info1, info2).flatten.mkString(",")) case w: WindowExec => - if (CometConf.COMET_EXEC_WINDOW_ENABLED.get(conf)) { - // TODO this duplicates logic in QueryPlanSerde - if (CometConf.isOperatorAllowIncompat("WindowExec")) { - newPlanWithProto( + if (QueryPlanSerde.isOperatorEnabled(CometWindow, w)) { + newPlanWithProto( + w, + CometWindowExec( + _, w, - CometWindowExec( - _, - w, - w.output, - w.windowExpression, - w.partitionSpec, - w.orderSpec, - w.child, - SerializedPlan(None))) - } else { - withInfo(w, "Native WindowExec has known correctness issues") - } + w.output, + w.windowExpression, + w.partitionSpec, + w.orderSpec, + w.child, + SerializedPlan(None))) } else { withInfo(w, "WindowExec is not enabled") } diff --git a/spark/src/main/scala/org/apache/comet/serde/QueryPlanSerde.scala b/spark/src/main/scala/org/apache/comet/serde/QueryPlanSerde.scala index 464905cd44..8d043510a2 100644 --- a/spark/src/main/scala/org/apache/comet/serde/QueryPlanSerde.scala +++ b/spark/src/main/scala/org/apache/comet/serde/QueryPlanSerde.scala @@ -760,53 +760,11 @@ object QueryPlanSerde extends Logging with CometExprShim { // look for registered handler first val serde = opSerdeMap.get(op.getClass) serde match { - case Some(handler) => - val enabled = handler.enabledConfig.forall(_.get(op.conf)) - val opName = op.getClass.getSimpleName - if (enabled) { - val opSerde = handler.asInstanceOf[CometOperatorSerde[SparkPlan]] - opSerde.getSupportLevel(op) match { - case Unsupported(notes) => - println("Unsupported") - withInfo(op, notes.getOrElse("")) - case Incompatible(notes) => - println(s"Incompatible ${notes}") - val allowIncompat = CometConf.isOperatorAllowIncompat(opName) - val incompatConf = CometConf.getOperatorAllowIncompatConfigKey(opName) - if (allowIncompat) { - println("allowIncompat") - if (notes.isDefined) { - logWarning( - s"Comet supports $opName when $incompatConf=true " + - s"but has notes: ${notes.get}") - } - return opSerde.convert(op, builder, childOp: _*) - } else { - val optionalNotes = notes.map(str => s" ($str)").getOrElse("") - withInfo( - op, - s"$opName is not fully compatible with Spark$optionalNotes. " + - s"To enable it anyway, set $incompatConf=true. " + - s"${CometConf.COMPAT_GUIDE}.") - } - case Compatible(notes) => - println(s"Compatible ${notes}") - if (notes.isDefined) { - logWarning(s"Comet supports $opName but has notes: ${notes.get}") - } - return opSerde.convert(op, builder, childOp: _*) - } - val maybeConverted = handler - .asInstanceOf[CometOperatorSerde[SparkPlan]] - .convert(op, builder, childOp: _*) - if (maybeConverted.isDefined) { - return maybeConverted - } - } else { - withInfo( - op, - s"Native support for operator $opName is disabled. " + - s"Set ${handler.enabledConfig.get.key}=true to enable it.") + case Some(handler) if isOperatorEnabled(handler, op) => + val opSerde = handler.asInstanceOf[CometOperatorSerde[SparkPlan]] + val maybeConverted = opSerde.convert(op, builder, childOp: _*) + if (maybeConverted.isDefined) { + return maybeConverted } case _ => } @@ -836,6 +794,54 @@ object QueryPlanSerde extends Logging with CometExprShim { } } + def isOperatorEnabled(handler: CometOperatorSerde[_], op: SparkPlan): Boolean = { + val enabled = handler.enabledConfig.forall(_.get(op.conf)) + val opName = op.getClass.getSimpleName + if (enabled) { + val opSerde = handler.asInstanceOf[CometOperatorSerde[SparkPlan]] + opSerde.getSupportLevel(op) match { + case Unsupported(notes) => + println("Unsupported") + withInfo(op, notes.getOrElse("")) + false + case Incompatible(notes) => + println(s"Incompatible ${notes}") + val allowIncompat = CometConf.isOperatorAllowIncompat(opName) + val incompatConf = CometConf.getOperatorAllowIncompatConfigKey(opName) + if (allowIncompat) { + println("allowIncompat") + if (notes.isDefined) { + logWarning( + s"Comet supports $opName when $incompatConf=true " + + s"but has notes: ${notes.get}") + } + true + } else { + val optionalNotes = notes.map(str => s" ($str)").getOrElse("") + withInfo( + op, + s"$opName is not fully compatible with Spark$optionalNotes. " + + s"To enable it anyway, set $incompatConf=true. " + + s"${CometConf.COMPAT_GUIDE}.") + false + } + case Compatible(notes) => + println(s"Compatible ${notes}") + if (notes.isDefined) { + logWarning(s"Comet supports $opName but has notes: ${notes.get}") + } + true + } + } else { + withInfo( + op, + s"Native support for operator $opName is disabled. " + + s"Set ${handler.enabledConfig.get.key}=true to enable it.") + false + } + + } + // Utility method. Adds explain info if the result of calling exprToProto is None def optExprWithInfo( optExpr: Option[Expr], diff --git a/spark/src/test/scala/org/apache/comet/exec/CometWindowExecSuite.scala b/spark/src/test/scala/org/apache/comet/exec/CometWindowExecSuite.scala index 8132459750..9f225105fd 100644 --- a/spark/src/test/scala/org/apache/comet/exec/CometWindowExecSuite.scala +++ b/spark/src/test/scala/org/apache/comet/exec/CometWindowExecSuite.scala @@ -206,7 +206,7 @@ class CometWindowExecSuite extends CometTestBase { } } - ignore("aggregate window function for all types") { + test("aggregate window function for all types") { val numValues = 2048 Seq(1, 100, numValues).foreach { numGroups => From de2c6fd1a6b9393d04b00fa85ca9380dd2b21c50 Mon Sep 17 00:00:00 2001 From: Andy Grove Date: Sat, 8 Nov 2025 14:28:32 -0700 Subject: [PATCH 37/44] refine --- .../apache/comet/rules/CometExecRule.scala | 26 ++++++++----------- 1 file changed, 11 insertions(+), 15 deletions(-) diff --git a/spark/src/main/scala/org/apache/comet/rules/CometExecRule.scala b/spark/src/main/scala/org/apache/comet/rules/CometExecRule.scala index 6f9dbf8e7e..2172022158 100644 --- a/spark/src/main/scala/org/apache/comet/rules/CometExecRule.scala +++ b/spark/src/main/scala/org/apache/comet/rules/CometExecRule.scala @@ -391,22 +391,18 @@ case class CometExecRule(session: SparkSession) extends Rule[SparkPlan] { "TakeOrderedAndProject requires shuffle to be enabled") withInfo(s, Seq(info1, info2).flatten.mkString(",")) - case w: WindowExec => - if (QueryPlanSerde.isOperatorEnabled(CometWindow, w)) { - newPlanWithProto( + case w: WindowExec if QueryPlanSerde.isOperatorEnabled(CometWindow, w) => + newPlanWithProto( + w, + CometWindowExec( + _, w, - CometWindowExec( - _, - w, - w.output, - w.windowExpression, - w.partitionSpec, - w.orderSpec, - w.child, - SerializedPlan(None))) - } else { - withInfo(w, "WindowExec is not enabled") - } + w.output, + w.windowExpression, + w.partitionSpec, + w.orderSpec, + w.child, + SerializedPlan(None))) case u: UnionExec if CometConf.COMET_EXEC_UNION_ENABLED.get(conf) && From 618366384d9a46d502212b3d16a5eac795099f48 Mon Sep 17 00:00:00 2001 From: Andy Grove Date: Sat, 8 Nov 2025 15:43:14 -0700 Subject: [PATCH 38/44] remove println --- .../main/scala/org/apache/comet/serde/QueryPlanSerde.scala | 4 ---- 1 file changed, 4 deletions(-) diff --git a/spark/src/main/scala/org/apache/comet/serde/QueryPlanSerde.scala b/spark/src/main/scala/org/apache/comet/serde/QueryPlanSerde.scala index 8d043510a2..73fa578527 100644 --- a/spark/src/main/scala/org/apache/comet/serde/QueryPlanSerde.scala +++ b/spark/src/main/scala/org/apache/comet/serde/QueryPlanSerde.scala @@ -801,15 +801,12 @@ object QueryPlanSerde extends Logging with CometExprShim { val opSerde = handler.asInstanceOf[CometOperatorSerde[SparkPlan]] opSerde.getSupportLevel(op) match { case Unsupported(notes) => - println("Unsupported") withInfo(op, notes.getOrElse("")) false case Incompatible(notes) => - println(s"Incompatible ${notes}") val allowIncompat = CometConf.isOperatorAllowIncompat(opName) val incompatConf = CometConf.getOperatorAllowIncompatConfigKey(opName) if (allowIncompat) { - println("allowIncompat") if (notes.isDefined) { logWarning( s"Comet supports $opName when $incompatConf=true " + @@ -826,7 +823,6 @@ object QueryPlanSerde extends Logging with CometExprShim { false } case Compatible(notes) => - println(s"Compatible ${notes}") if (notes.isDefined) { logWarning(s"Comet supports $opName but has notes: ${notes.get}") } From 3a5935980c9c481aec63ea293d25892ede869a82 Mon Sep 17 00:00:00 2001 From: Andy Grove Date: Sat, 8 Nov 2025 16:55:06 -0700 Subject: [PATCH 39/44] fix --- .../apache/comet/rules/CometExecRule.scala | 28 +++++++++++-------- .../apache/comet/serde/QueryPlanSerde.scala | 3 +- .../comet/exec/CometWindowExecSuite.scala | 2 +- 3 files changed, 18 insertions(+), 15 deletions(-) diff --git a/spark/src/main/scala/org/apache/comet/rules/CometExecRule.scala b/spark/src/main/scala/org/apache/comet/rules/CometExecRule.scala index 2172022158..fff995d1b6 100644 --- a/spark/src/main/scala/org/apache/comet/rules/CometExecRule.scala +++ b/spark/src/main/scala/org/apache/comet/rules/CometExecRule.scala @@ -44,7 +44,7 @@ import org.apache.spark.sql.types._ import org.apache.comet.{CometConf, ExtendedExplainInfo} import org.apache.comet.CometConf.COMET_EXEC_SHUFFLE_ENABLED import org.apache.comet.CometSparkSessionExtensions._ -import org.apache.comet.serde.{CometWindow, QueryPlanSerde} +import org.apache.comet.serde.QueryPlanSerde import org.apache.comet.serde.OperatorOuterClass.Operator /** @@ -391,18 +391,22 @@ case class CometExecRule(session: SparkSession) extends Rule[SparkPlan] { "TakeOrderedAndProject requires shuffle to be enabled") withInfo(s, Seq(info1, info2).flatten.mkString(",")) - case w: WindowExec if QueryPlanSerde.isOperatorEnabled(CometWindow, w) => - newPlanWithProto( - w, - CometWindowExec( - _, + case w: WindowExec => + if (w.children.forall(isCometNative)) { + newPlanWithProto( w, - w.output, - w.windowExpression, - w.partitionSpec, - w.orderSpec, - w.child, - SerializedPlan(None))) + CometWindowExec( + _, + w, + w.output, + w.windowExpression, + w.partitionSpec, + w.orderSpec, + w.child, + SerializedPlan(None))) + } else { + op + } case u: UnionExec if CometConf.COMET_EXEC_UNION_ENABLED.get(conf) && diff --git a/spark/src/main/scala/org/apache/comet/serde/QueryPlanSerde.scala b/spark/src/main/scala/org/apache/comet/serde/QueryPlanSerde.scala index 73fa578527..0c0014cb2a 100644 --- a/spark/src/main/scala/org/apache/comet/serde/QueryPlanSerde.scala +++ b/spark/src/main/scala/org/apache/comet/serde/QueryPlanSerde.scala @@ -794,7 +794,7 @@ object QueryPlanSerde extends Logging with CometExprShim { } } - def isOperatorEnabled(handler: CometOperatorSerde[_], op: SparkPlan): Boolean = { + private def isOperatorEnabled(handler: CometOperatorSerde[_], op: SparkPlan): Boolean = { val enabled = handler.enabledConfig.forall(_.get(op.conf)) val opName = op.getClass.getSimpleName if (enabled) { @@ -835,7 +835,6 @@ object QueryPlanSerde extends Logging with CometExprShim { s"Set ${handler.enabledConfig.get.key}=true to enable it.") false } - } // Utility method. Adds explain info if the result of calling exprToProto is None diff --git a/spark/src/test/scala/org/apache/comet/exec/CometWindowExecSuite.scala b/spark/src/test/scala/org/apache/comet/exec/CometWindowExecSuite.scala index 9f225105fd..be14658cfb 100644 --- a/spark/src/test/scala/org/apache/comet/exec/CometWindowExecSuite.scala +++ b/spark/src/test/scala/org/apache/comet/exec/CometWindowExecSuite.scala @@ -325,7 +325,7 @@ class CometWindowExecSuite extends CometTestBase { } } - // TODO: AVG with PARTITION BY and ORDER BY not supportedu + // TODO: AVG with PARTITION BY and ORDER BY not supported // Falls back to Spark Window operator - "Partitioning and sorting specifications must be the same" ignore("window: AVG with PARTITION BY and ORDER BY") { withTempDir { dir => From 12ff8056dfcf4df458eccffac2e2a686c6d88365 Mon Sep 17 00:00:00 2001 From: Andy Grove Date: Sat, 8 Nov 2025 16:57:59 -0700 Subject: [PATCH 40/44] scalastyle --- spark/src/main/scala/org/apache/comet/rules/CometExecRule.scala | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/spark/src/main/scala/org/apache/comet/rules/CometExecRule.scala b/spark/src/main/scala/org/apache/comet/rules/CometExecRule.scala index fff995d1b6..6db5c807bb 100644 --- a/spark/src/main/scala/org/apache/comet/rules/CometExecRule.scala +++ b/spark/src/main/scala/org/apache/comet/rules/CometExecRule.scala @@ -44,8 +44,8 @@ import org.apache.spark.sql.types._ import org.apache.comet.{CometConf, ExtendedExplainInfo} import org.apache.comet.CometConf.COMET_EXEC_SHUFFLE_ENABLED import org.apache.comet.CometSparkSessionExtensions._ -import org.apache.comet.serde.QueryPlanSerde import org.apache.comet.serde.OperatorOuterClass.Operator +import org.apache.comet.serde.QueryPlanSerde /** * Spark physical optimizer rule for replacing Spark operators with Comet operators. From fd4385acf2dbac4374bd6fd2d723582ee1ee0dd4 Mon Sep 17 00:00:00 2001 From: Andy Grove Date: Sat, 8 Nov 2025 17:01:53 -0700 Subject: [PATCH 41/44] update 4.0 golden files --- .../q51.native_iceberg_compat/extended.txt | 6 +++--- .../approved-plans-v1_4-spark4_0/q51/extended.txt | 6 +++--- 2 files changed, 6 insertions(+), 6 deletions(-) diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q51.native_iceberg_compat/extended.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q51.native_iceberg_compat/extended.txt index f35e106153..03282d997d 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q51.native_iceberg_compat/extended.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q51.native_iceberg_compat/extended.txt @@ -1,6 +1,6 @@ TakeOrderedAndProject [COMET: ] +- Filter - +- Window [COMET: WindowExec is not fully compatible with Spark (Native WindowExec has known correctness issues). To enable it anyway, set spark.comet.operator.WindowExec.allowIncompatible=true. For more information, refer to the Comet Compatibility Guide (https://datafusion.apache.org/comet/user-guide/compatibility.html)., Partitioning and sorting specifications must be the same.] + +- Window [COMET: WindowExec is not fully compatible with Spark (Native WindowExec has known correctness issues). To enable it anyway, set spark.comet.operator.WindowExec.allowIncompatible=true. For more information, refer to the Comet Compatibility Guide (https://datafusion.apache.org/comet/user-guide/compatibility.html).] +- CometColumnarToRow +- CometSort +- CometExchange @@ -9,7 +9,7 @@ :- CometSort : +- CometColumnarExchange : +- Project - : +- Window [COMET: WindowExec is not fully compatible with Spark (Native WindowExec has known correctness issues). To enable it anyway, set spark.comet.operator.WindowExec.allowIncompatible=true. For more information, refer to the Comet Compatibility Guide (https://datafusion.apache.org/comet/user-guide/compatibility.html)., Partitioning and sorting specifications must be the same.] + : +- Window [COMET: WindowExec is not fully compatible with Spark (Native WindowExec has known correctness issues). To enable it anyway, set spark.comet.operator.WindowExec.allowIncompatible=true. For more information, refer to the Comet Compatibility Guide (https://datafusion.apache.org/comet/user-guide/compatibility.html).] : +- CometColumnarToRow : +- CometSort : +- CometColumnarExchange @@ -34,7 +34,7 @@ +- CometSort +- CometColumnarExchange +- Project - +- Window [COMET: WindowExec is not fully compatible with Spark (Native WindowExec has known correctness issues). To enable it anyway, set spark.comet.operator.WindowExec.allowIncompatible=true. For more information, refer to the Comet Compatibility Guide (https://datafusion.apache.org/comet/user-guide/compatibility.html)., Partitioning and sorting specifications must be the same.] + +- Window [COMET: WindowExec is not fully compatible with Spark (Native WindowExec has known correctness issues). To enable it anyway, set spark.comet.operator.WindowExec.allowIncompatible=true. For more information, refer to the Comet Compatibility Guide (https://datafusion.apache.org/comet/user-guide/compatibility.html).] +- CometColumnarToRow +- CometSort +- CometColumnarExchange diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q51/extended.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q51/extended.txt index f35e106153..03282d997d 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q51/extended.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q51/extended.txt @@ -1,6 +1,6 @@ TakeOrderedAndProject [COMET: ] +- Filter - +- Window [COMET: WindowExec is not fully compatible with Spark (Native WindowExec has known correctness issues). To enable it anyway, set spark.comet.operator.WindowExec.allowIncompatible=true. For more information, refer to the Comet Compatibility Guide (https://datafusion.apache.org/comet/user-guide/compatibility.html)., Partitioning and sorting specifications must be the same.] + +- Window [COMET: WindowExec is not fully compatible with Spark (Native WindowExec has known correctness issues). To enable it anyway, set spark.comet.operator.WindowExec.allowIncompatible=true. For more information, refer to the Comet Compatibility Guide (https://datafusion.apache.org/comet/user-guide/compatibility.html).] +- CometColumnarToRow +- CometSort +- CometExchange @@ -9,7 +9,7 @@ :- CometSort : +- CometColumnarExchange : +- Project - : +- Window [COMET: WindowExec is not fully compatible with Spark (Native WindowExec has known correctness issues). To enable it anyway, set spark.comet.operator.WindowExec.allowIncompatible=true. For more information, refer to the Comet Compatibility Guide (https://datafusion.apache.org/comet/user-guide/compatibility.html)., Partitioning and sorting specifications must be the same.] + : +- Window [COMET: WindowExec is not fully compatible with Spark (Native WindowExec has known correctness issues). To enable it anyway, set spark.comet.operator.WindowExec.allowIncompatible=true. For more information, refer to the Comet Compatibility Guide (https://datafusion.apache.org/comet/user-guide/compatibility.html).] : +- CometColumnarToRow : +- CometSort : +- CometColumnarExchange @@ -34,7 +34,7 @@ +- CometSort +- CometColumnarExchange +- Project - +- Window [COMET: WindowExec is not fully compatible with Spark (Native WindowExec has known correctness issues). To enable it anyway, set spark.comet.operator.WindowExec.allowIncompatible=true. For more information, refer to the Comet Compatibility Guide (https://datafusion.apache.org/comet/user-guide/compatibility.html)., Partitioning and sorting specifications must be the same.] + +- Window [COMET: WindowExec is not fully compatible with Spark (Native WindowExec has known correctness issues). To enable it anyway, set spark.comet.operator.WindowExec.allowIncompatible=true. For more information, refer to the Comet Compatibility Guide (https://datafusion.apache.org/comet/user-guide/compatibility.html).] +- CometColumnarToRow +- CometSort +- CometColumnarExchange From 839f6c71ca4e6f022e2637ea167fab23c6a76a2c Mon Sep 17 00:00:00 2001 From: Andy Grove Date: Sat, 8 Nov 2025 17:07:05 -0700 Subject: [PATCH 42/44] update 4.0 golden files --- .../q51a.native_iceberg_compat/extended.txt | 20 +++++++++---------- .../q51a/extended.txt | 20 +++++++++---------- 2 files changed, 20 insertions(+), 20 deletions(-) diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7-spark4_0/q51a.native_iceberg_compat/extended.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7-spark4_0/q51a.native_iceberg_compat/extended.txt index 0c6c1fe8dd..aa31cfd74e 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7-spark4_0/q51a.native_iceberg_compat/extended.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7-spark4_0/q51a.native_iceberg_compat/extended.txt @@ -4,7 +4,7 @@ +- HashAggregate +- Project +- BroadcastHashJoin - :- Window [COMET: WindowExec is not fully compatible with Spark (Native WindowExec has known correctness issues). To enable it anyway, set spark.comet.operator.WindowExec.allowIncompatible=true. For more information, refer to the Comet Compatibility Guide (https://datafusion.apache.org/comet/user-guide/compatibility.html)., Partitioning and sorting specifications must be the same.] + :- Window [COMET: WindowExec is not fully compatible with Spark (Native WindowExec has known correctness issues). To enable it anyway, set spark.comet.operator.WindowExec.allowIncompatible=true. For more information, refer to the Comet Compatibility Guide (https://datafusion.apache.org/comet/user-guide/compatibility.html).] : +- CometColumnarToRow : +- CometSort : +- CometExchange @@ -20,7 +20,7 @@ : : +- Project : : +- BroadcastHashJoin : : :- Project - : : : +- Window [COMET: WindowExec is not fully compatible with Spark (Native WindowExec has known correctness issues). To enable it anyway, set spark.comet.operator.WindowExec.allowIncompatible=true. For more information, refer to the Comet Compatibility Guide (https://datafusion.apache.org/comet/user-guide/compatibility.html)., Partitioning and sorting specifications must be the same.] + : : : +- Window [COMET: WindowExec is not fully compatible with Spark (Native WindowExec has known correctness issues). To enable it anyway, set spark.comet.operator.WindowExec.allowIncompatible=true. For more information, refer to the Comet Compatibility Guide (https://datafusion.apache.org/comet/user-guide/compatibility.html).] : : : +- CometColumnarToRow : : : +- CometSort : : : +- CometColumnarExchange @@ -44,7 +44,7 @@ : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim : : +- BroadcastExchange : : +- Project - : : +- Window [COMET: WindowExec is not fully compatible with Spark (Native WindowExec has known correctness issues). To enable it anyway, set spark.comet.operator.WindowExec.allowIncompatible=true. For more information, refer to the Comet Compatibility Guide (https://datafusion.apache.org/comet/user-guide/compatibility.html)., Partitioning and sorting specifications must be the same.] + : : +- Window [COMET: WindowExec is not fully compatible with Spark (Native WindowExec has known correctness issues). To enable it anyway, set spark.comet.operator.WindowExec.allowIncompatible=true. For more information, refer to the Comet Compatibility Guide (https://datafusion.apache.org/comet/user-guide/compatibility.html).] : : +- CometColumnarToRow : : +- CometSort : : +- CometColumnarExchange @@ -75,7 +75,7 @@ : +- Project : +- BroadcastHashJoin : :- Project - : : +- Window [COMET: WindowExec is not fully compatible with Spark (Native WindowExec has known correctness issues). To enable it anyway, set spark.comet.operator.WindowExec.allowIncompatible=true. For more information, refer to the Comet Compatibility Guide (https://datafusion.apache.org/comet/user-guide/compatibility.html)., Partitioning and sorting specifications must be the same.] + : : +- Window [COMET: WindowExec is not fully compatible with Spark (Native WindowExec has known correctness issues). To enable it anyway, set spark.comet.operator.WindowExec.allowIncompatible=true. For more information, refer to the Comet Compatibility Guide (https://datafusion.apache.org/comet/user-guide/compatibility.html).] : : +- CometColumnarToRow : : +- CometSort : : +- CometColumnarExchange @@ -94,7 +94,7 @@ : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim : +- BroadcastExchange : +- Project - : +- Window [COMET: WindowExec is not fully compatible with Spark (Native WindowExec has known correctness issues). To enable it anyway, set spark.comet.operator.WindowExec.allowIncompatible=true. For more information, refer to the Comet Compatibility Guide (https://datafusion.apache.org/comet/user-guide/compatibility.html)., Partitioning and sorting specifications must be the same.] + : +- Window [COMET: WindowExec is not fully compatible with Spark (Native WindowExec has known correctness issues). To enable it anyway, set spark.comet.operator.WindowExec.allowIncompatible=true. For more information, refer to the Comet Compatibility Guide (https://datafusion.apache.org/comet/user-guide/compatibility.html).] : +- CometColumnarToRow : +- CometSort : +- CometColumnarExchange @@ -113,7 +113,7 @@ : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim +- BroadcastExchange +- Project - +- Window [COMET: WindowExec is not fully compatible with Spark (Native WindowExec has known correctness issues). To enable it anyway, set spark.comet.operator.WindowExec.allowIncompatible=true. For more information, refer to the Comet Compatibility Guide (https://datafusion.apache.org/comet/user-guide/compatibility.html)., Partitioning and sorting specifications must be the same.] + +- Window [COMET: WindowExec is not fully compatible with Spark (Native WindowExec has known correctness issues). To enable it anyway, set spark.comet.operator.WindowExec.allowIncompatible=true. For more information, refer to the Comet Compatibility Guide (https://datafusion.apache.org/comet/user-guide/compatibility.html).] +- CometColumnarToRow +- CometSort +- CometExchange @@ -129,7 +129,7 @@ : +- Project : +- BroadcastHashJoin : :- Project - : : +- Window [COMET: WindowExec is not fully compatible with Spark (Native WindowExec has known correctness issues). To enable it anyway, set spark.comet.operator.WindowExec.allowIncompatible=true. For more information, refer to the Comet Compatibility Guide (https://datafusion.apache.org/comet/user-guide/compatibility.html)., Partitioning and sorting specifications must be the same.] + : : +- Window [COMET: WindowExec is not fully compatible with Spark (Native WindowExec has known correctness issues). To enable it anyway, set spark.comet.operator.WindowExec.allowIncompatible=true. For more information, refer to the Comet Compatibility Guide (https://datafusion.apache.org/comet/user-guide/compatibility.html).] : : +- CometColumnarToRow : : +- CometSort : : +- CometColumnarExchange @@ -153,7 +153,7 @@ : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim : +- BroadcastExchange : +- Project - : +- Window [COMET: WindowExec is not fully compatible with Spark (Native WindowExec has known correctness issues). To enable it anyway, set spark.comet.operator.WindowExec.allowIncompatible=true. For more information, refer to the Comet Compatibility Guide (https://datafusion.apache.org/comet/user-guide/compatibility.html)., Partitioning and sorting specifications must be the same.] + : +- Window [COMET: WindowExec is not fully compatible with Spark (Native WindowExec has known correctness issues). To enable it anyway, set spark.comet.operator.WindowExec.allowIncompatible=true. For more information, refer to the Comet Compatibility Guide (https://datafusion.apache.org/comet/user-guide/compatibility.html).] : +- CometColumnarToRow : +- CometSort : +- CometColumnarExchange @@ -184,7 +184,7 @@ +- Project +- BroadcastHashJoin :- Project - : +- Window [COMET: WindowExec is not fully compatible with Spark (Native WindowExec has known correctness issues). To enable it anyway, set spark.comet.operator.WindowExec.allowIncompatible=true. For more information, refer to the Comet Compatibility Guide (https://datafusion.apache.org/comet/user-guide/compatibility.html)., Partitioning and sorting specifications must be the same.] + : +- Window [COMET: WindowExec is not fully compatible with Spark (Native WindowExec has known correctness issues). To enable it anyway, set spark.comet.operator.WindowExec.allowIncompatible=true. For more information, refer to the Comet Compatibility Guide (https://datafusion.apache.org/comet/user-guide/compatibility.html).] : +- CometColumnarToRow : +- CometSort : +- CometColumnarExchange @@ -203,7 +203,7 @@ : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim +- BroadcastExchange +- Project - +- Window [COMET: WindowExec is not fully compatible with Spark (Native WindowExec has known correctness issues). To enable it anyway, set spark.comet.operator.WindowExec.allowIncompatible=true. For more information, refer to the Comet Compatibility Guide (https://datafusion.apache.org/comet/user-guide/compatibility.html)., Partitioning and sorting specifications must be the same.] + +- Window [COMET: WindowExec is not fully compatible with Spark (Native WindowExec has known correctness issues). To enable it anyway, set spark.comet.operator.WindowExec.allowIncompatible=true. For more information, refer to the Comet Compatibility Guide (https://datafusion.apache.org/comet/user-guide/compatibility.html).] +- CometColumnarToRow +- CometSort +- CometColumnarExchange diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7-spark4_0/q51a/extended.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7-spark4_0/q51a/extended.txt index 0c6c1fe8dd..aa31cfd74e 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7-spark4_0/q51a/extended.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7-spark4_0/q51a/extended.txt @@ -4,7 +4,7 @@ +- HashAggregate +- Project +- BroadcastHashJoin - :- Window [COMET: WindowExec is not fully compatible with Spark (Native WindowExec has known correctness issues). To enable it anyway, set spark.comet.operator.WindowExec.allowIncompatible=true. For more information, refer to the Comet Compatibility Guide (https://datafusion.apache.org/comet/user-guide/compatibility.html)., Partitioning and sorting specifications must be the same.] + :- Window [COMET: WindowExec is not fully compatible with Spark (Native WindowExec has known correctness issues). To enable it anyway, set spark.comet.operator.WindowExec.allowIncompatible=true. For more information, refer to the Comet Compatibility Guide (https://datafusion.apache.org/comet/user-guide/compatibility.html).] : +- CometColumnarToRow : +- CometSort : +- CometExchange @@ -20,7 +20,7 @@ : : +- Project : : +- BroadcastHashJoin : : :- Project - : : : +- Window [COMET: WindowExec is not fully compatible with Spark (Native WindowExec has known correctness issues). To enable it anyway, set spark.comet.operator.WindowExec.allowIncompatible=true. For more information, refer to the Comet Compatibility Guide (https://datafusion.apache.org/comet/user-guide/compatibility.html)., Partitioning and sorting specifications must be the same.] + : : : +- Window [COMET: WindowExec is not fully compatible with Spark (Native WindowExec has known correctness issues). To enable it anyway, set spark.comet.operator.WindowExec.allowIncompatible=true. For more information, refer to the Comet Compatibility Guide (https://datafusion.apache.org/comet/user-guide/compatibility.html).] : : : +- CometColumnarToRow : : : +- CometSort : : : +- CometColumnarExchange @@ -44,7 +44,7 @@ : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim : : +- BroadcastExchange : : +- Project - : : +- Window [COMET: WindowExec is not fully compatible with Spark (Native WindowExec has known correctness issues). To enable it anyway, set spark.comet.operator.WindowExec.allowIncompatible=true. For more information, refer to the Comet Compatibility Guide (https://datafusion.apache.org/comet/user-guide/compatibility.html)., Partitioning and sorting specifications must be the same.] + : : +- Window [COMET: WindowExec is not fully compatible with Spark (Native WindowExec has known correctness issues). To enable it anyway, set spark.comet.operator.WindowExec.allowIncompatible=true. For more information, refer to the Comet Compatibility Guide (https://datafusion.apache.org/comet/user-guide/compatibility.html).] : : +- CometColumnarToRow : : +- CometSort : : +- CometColumnarExchange @@ -75,7 +75,7 @@ : +- Project : +- BroadcastHashJoin : :- Project - : : +- Window [COMET: WindowExec is not fully compatible with Spark (Native WindowExec has known correctness issues). To enable it anyway, set spark.comet.operator.WindowExec.allowIncompatible=true. For more information, refer to the Comet Compatibility Guide (https://datafusion.apache.org/comet/user-guide/compatibility.html)., Partitioning and sorting specifications must be the same.] + : : +- Window [COMET: WindowExec is not fully compatible with Spark (Native WindowExec has known correctness issues). To enable it anyway, set spark.comet.operator.WindowExec.allowIncompatible=true. For more information, refer to the Comet Compatibility Guide (https://datafusion.apache.org/comet/user-guide/compatibility.html).] : : +- CometColumnarToRow : : +- CometSort : : +- CometColumnarExchange @@ -94,7 +94,7 @@ : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim : +- BroadcastExchange : +- Project - : +- Window [COMET: WindowExec is not fully compatible with Spark (Native WindowExec has known correctness issues). To enable it anyway, set spark.comet.operator.WindowExec.allowIncompatible=true. For more information, refer to the Comet Compatibility Guide (https://datafusion.apache.org/comet/user-guide/compatibility.html)., Partitioning and sorting specifications must be the same.] + : +- Window [COMET: WindowExec is not fully compatible with Spark (Native WindowExec has known correctness issues). To enable it anyway, set spark.comet.operator.WindowExec.allowIncompatible=true. For more information, refer to the Comet Compatibility Guide (https://datafusion.apache.org/comet/user-guide/compatibility.html).] : +- CometColumnarToRow : +- CometSort : +- CometColumnarExchange @@ -113,7 +113,7 @@ : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim +- BroadcastExchange +- Project - +- Window [COMET: WindowExec is not fully compatible with Spark (Native WindowExec has known correctness issues). To enable it anyway, set spark.comet.operator.WindowExec.allowIncompatible=true. For more information, refer to the Comet Compatibility Guide (https://datafusion.apache.org/comet/user-guide/compatibility.html)., Partitioning and sorting specifications must be the same.] + +- Window [COMET: WindowExec is not fully compatible with Spark (Native WindowExec has known correctness issues). To enable it anyway, set spark.comet.operator.WindowExec.allowIncompatible=true. For more information, refer to the Comet Compatibility Guide (https://datafusion.apache.org/comet/user-guide/compatibility.html).] +- CometColumnarToRow +- CometSort +- CometExchange @@ -129,7 +129,7 @@ : +- Project : +- BroadcastHashJoin : :- Project - : : +- Window [COMET: WindowExec is not fully compatible with Spark (Native WindowExec has known correctness issues). To enable it anyway, set spark.comet.operator.WindowExec.allowIncompatible=true. For more information, refer to the Comet Compatibility Guide (https://datafusion.apache.org/comet/user-guide/compatibility.html)., Partitioning and sorting specifications must be the same.] + : : +- Window [COMET: WindowExec is not fully compatible with Spark (Native WindowExec has known correctness issues). To enable it anyway, set spark.comet.operator.WindowExec.allowIncompatible=true. For more information, refer to the Comet Compatibility Guide (https://datafusion.apache.org/comet/user-guide/compatibility.html).] : : +- CometColumnarToRow : : +- CometSort : : +- CometColumnarExchange @@ -153,7 +153,7 @@ : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim : +- BroadcastExchange : +- Project - : +- Window [COMET: WindowExec is not fully compatible with Spark (Native WindowExec has known correctness issues). To enable it anyway, set spark.comet.operator.WindowExec.allowIncompatible=true. For more information, refer to the Comet Compatibility Guide (https://datafusion.apache.org/comet/user-guide/compatibility.html)., Partitioning and sorting specifications must be the same.] + : +- Window [COMET: WindowExec is not fully compatible with Spark (Native WindowExec has known correctness issues). To enable it anyway, set spark.comet.operator.WindowExec.allowIncompatible=true. For more information, refer to the Comet Compatibility Guide (https://datafusion.apache.org/comet/user-guide/compatibility.html).] : +- CometColumnarToRow : +- CometSort : +- CometColumnarExchange @@ -184,7 +184,7 @@ +- Project +- BroadcastHashJoin :- Project - : +- Window [COMET: WindowExec is not fully compatible with Spark (Native WindowExec has known correctness issues). To enable it anyway, set spark.comet.operator.WindowExec.allowIncompatible=true. For more information, refer to the Comet Compatibility Guide (https://datafusion.apache.org/comet/user-guide/compatibility.html)., Partitioning and sorting specifications must be the same.] + : +- Window [COMET: WindowExec is not fully compatible with Spark (Native WindowExec has known correctness issues). To enable it anyway, set spark.comet.operator.WindowExec.allowIncompatible=true. For more information, refer to the Comet Compatibility Guide (https://datafusion.apache.org/comet/user-guide/compatibility.html).] : +- CometColumnarToRow : +- CometSort : +- CometColumnarExchange @@ -203,7 +203,7 @@ : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim +- BroadcastExchange +- Project - +- Window [COMET: WindowExec is not fully compatible with Spark (Native WindowExec has known correctness issues). To enable it anyway, set spark.comet.operator.WindowExec.allowIncompatible=true. For more information, refer to the Comet Compatibility Guide (https://datafusion.apache.org/comet/user-guide/compatibility.html)., Partitioning and sorting specifications must be the same.] + +- Window [COMET: WindowExec is not fully compatible with Spark (Native WindowExec has known correctness issues). To enable it anyway, set spark.comet.operator.WindowExec.allowIncompatible=true. For more information, refer to the Comet Compatibility Guide (https://datafusion.apache.org/comet/user-guide/compatibility.html).] +- CometColumnarToRow +- CometSort +- CometColumnarExchange From 5f1fbf45c07eaace3e55789d31967686ba2f2977 Mon Sep 17 00:00:00 2001 From: Andy Grove Date: Sat, 8 Nov 2025 17:09:01 -0700 Subject: [PATCH 43/44] update 3.5 golden files --- .../q51.native_iceberg_compat/extended.txt | 6 +++--- .../q51/extended.txt | 6 +++--- .../q51a.native_iceberg_compat/extended.txt | 20 +++++++++---------- .../q51a/extended.txt | 20 +++++++++---------- 4 files changed, 26 insertions(+), 26 deletions(-) diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q51.native_iceberg_compat/extended.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q51.native_iceberg_compat/extended.txt index d55771ac8a..17d5d53446 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q51.native_iceberg_compat/extended.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q51.native_iceberg_compat/extended.txt @@ -1,6 +1,6 @@ TakeOrderedAndProject [COMET: ] +- Filter - +- Window [COMET: WindowExec is not fully compatible with Spark (Native WindowExec has known correctness issues). To enable it anyway, set spark.comet.operator.WindowExec.allowIncompatible=true. For more information, refer to the Comet Compatibility Guide (https://datafusion.apache.org/comet/user-guide/compatibility.html)., Partitioning and sorting specifications must be the same.] + +- Window [COMET: WindowExec is not fully compatible with Spark (Native WindowExec has known correctness issues). To enable it anyway, set spark.comet.operator.WindowExec.allowIncompatible=true. For more information, refer to the Comet Compatibility Guide (https://datafusion.apache.org/comet/user-guide/compatibility.html).] +- CometColumnarToRow +- CometSort +- CometExchange @@ -9,7 +9,7 @@ :- CometSort : +- CometColumnarExchange : +- Project - : +- Window [COMET: WindowExec is not fully compatible with Spark (Native WindowExec has known correctness issues). To enable it anyway, set spark.comet.operator.WindowExec.allowIncompatible=true. For more information, refer to the Comet Compatibility Guide (https://datafusion.apache.org/comet/user-guide/compatibility.html)., Partitioning and sorting specifications must be the same.] + : +- Window [COMET: WindowExec is not fully compatible with Spark (Native WindowExec has known correctness issues). To enable it anyway, set spark.comet.operator.WindowExec.allowIncompatible=true. For more information, refer to the Comet Compatibility Guide (https://datafusion.apache.org/comet/user-guide/compatibility.html).] : +- CometColumnarToRow : +- CometSort : +- CometExchange @@ -33,7 +33,7 @@ +- CometSort +- CometColumnarExchange +- Project - +- Window [COMET: WindowExec is not fully compatible with Spark (Native WindowExec has known correctness issues). To enable it anyway, set spark.comet.operator.WindowExec.allowIncompatible=true. For more information, refer to the Comet Compatibility Guide (https://datafusion.apache.org/comet/user-guide/compatibility.html)., Partitioning and sorting specifications must be the same.] + +- Window [COMET: WindowExec is not fully compatible with Spark (Native WindowExec has known correctness issues). To enable it anyway, set spark.comet.operator.WindowExec.allowIncompatible=true. For more information, refer to the Comet Compatibility Guide (https://datafusion.apache.org/comet/user-guide/compatibility.html).] +- CometColumnarToRow +- CometSort +- CometExchange diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q51/extended.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q51/extended.txt index d55771ac8a..17d5d53446 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q51/extended.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q51/extended.txt @@ -1,6 +1,6 @@ TakeOrderedAndProject [COMET: ] +- Filter - +- Window [COMET: WindowExec is not fully compatible with Spark (Native WindowExec has known correctness issues). To enable it anyway, set spark.comet.operator.WindowExec.allowIncompatible=true. For more information, refer to the Comet Compatibility Guide (https://datafusion.apache.org/comet/user-guide/compatibility.html)., Partitioning and sorting specifications must be the same.] + +- Window [COMET: WindowExec is not fully compatible with Spark (Native WindowExec has known correctness issues). To enable it anyway, set spark.comet.operator.WindowExec.allowIncompatible=true. For more information, refer to the Comet Compatibility Guide (https://datafusion.apache.org/comet/user-guide/compatibility.html).] +- CometColumnarToRow +- CometSort +- CometExchange @@ -9,7 +9,7 @@ :- CometSort : +- CometColumnarExchange : +- Project - : +- Window [COMET: WindowExec is not fully compatible with Spark (Native WindowExec has known correctness issues). To enable it anyway, set spark.comet.operator.WindowExec.allowIncompatible=true. For more information, refer to the Comet Compatibility Guide (https://datafusion.apache.org/comet/user-guide/compatibility.html)., Partitioning and sorting specifications must be the same.] + : +- Window [COMET: WindowExec is not fully compatible with Spark (Native WindowExec has known correctness issues). To enable it anyway, set spark.comet.operator.WindowExec.allowIncompatible=true. For more information, refer to the Comet Compatibility Guide (https://datafusion.apache.org/comet/user-guide/compatibility.html).] : +- CometColumnarToRow : +- CometSort : +- CometExchange @@ -33,7 +33,7 @@ +- CometSort +- CometColumnarExchange +- Project - +- Window [COMET: WindowExec is not fully compatible with Spark (Native WindowExec has known correctness issues). To enable it anyway, set spark.comet.operator.WindowExec.allowIncompatible=true. For more information, refer to the Comet Compatibility Guide (https://datafusion.apache.org/comet/user-guide/compatibility.html)., Partitioning and sorting specifications must be the same.] + +- Window [COMET: WindowExec is not fully compatible with Spark (Native WindowExec has known correctness issues). To enable it anyway, set spark.comet.operator.WindowExec.allowIncompatible=true. For more information, refer to the Comet Compatibility Guide (https://datafusion.apache.org/comet/user-guide/compatibility.html).] +- CometColumnarToRow +- CometSort +- CometExchange diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7-spark3_5/q51a.native_iceberg_compat/extended.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7-spark3_5/q51a.native_iceberg_compat/extended.txt index b192555935..b8822715ca 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7-spark3_5/q51a.native_iceberg_compat/extended.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7-spark3_5/q51a.native_iceberg_compat/extended.txt @@ -4,7 +4,7 @@ +- HashAggregate +- Project +- BroadcastHashJoin - :- Window [COMET: WindowExec is not fully compatible with Spark (Native WindowExec has known correctness issues). To enable it anyway, set spark.comet.operator.WindowExec.allowIncompatible=true. For more information, refer to the Comet Compatibility Guide (https://datafusion.apache.org/comet/user-guide/compatibility.html)., Partitioning and sorting specifications must be the same.] + :- Window [COMET: WindowExec is not fully compatible with Spark (Native WindowExec has known correctness issues). To enable it anyway, set spark.comet.operator.WindowExec.allowIncompatible=true. For more information, refer to the Comet Compatibility Guide (https://datafusion.apache.org/comet/user-guide/compatibility.html).] : +- CometColumnarToRow : +- CometSort : +- CometExchange @@ -20,7 +20,7 @@ : : +- Project : : +- BroadcastHashJoin : : :- Project - : : : +- Window [COMET: WindowExec is not fully compatible with Spark (Native WindowExec has known correctness issues). To enable it anyway, set spark.comet.operator.WindowExec.allowIncompatible=true. For more information, refer to the Comet Compatibility Guide (https://datafusion.apache.org/comet/user-guide/compatibility.html)., Partitioning and sorting specifications must be the same.] + : : : +- Window [COMET: WindowExec is not fully compatible with Spark (Native WindowExec has known correctness issues). To enable it anyway, set spark.comet.operator.WindowExec.allowIncompatible=true. For more information, refer to the Comet Compatibility Guide (https://datafusion.apache.org/comet/user-guide/compatibility.html).] : : : +- CometColumnarToRow : : : +- CometSort : : : +- CometExchange @@ -43,7 +43,7 @@ : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim : : +- BroadcastExchange : : +- Project - : : +- Window [COMET: WindowExec is not fully compatible with Spark (Native WindowExec has known correctness issues). To enable it anyway, set spark.comet.operator.WindowExec.allowIncompatible=true. For more information, refer to the Comet Compatibility Guide (https://datafusion.apache.org/comet/user-guide/compatibility.html)., Partitioning and sorting specifications must be the same.] + : : +- Window [COMET: WindowExec is not fully compatible with Spark (Native WindowExec has known correctness issues). To enable it anyway, set spark.comet.operator.WindowExec.allowIncompatible=true. For more information, refer to the Comet Compatibility Guide (https://datafusion.apache.org/comet/user-guide/compatibility.html).] : : +- CometColumnarToRow : : +- CometSort : : +- CometExchange @@ -73,7 +73,7 @@ : +- Project : +- BroadcastHashJoin : :- Project - : : +- Window [COMET: WindowExec is not fully compatible with Spark (Native WindowExec has known correctness issues). To enable it anyway, set spark.comet.operator.WindowExec.allowIncompatible=true. For more information, refer to the Comet Compatibility Guide (https://datafusion.apache.org/comet/user-guide/compatibility.html)., Partitioning and sorting specifications must be the same.] + : : +- Window [COMET: WindowExec is not fully compatible with Spark (Native WindowExec has known correctness issues). To enable it anyway, set spark.comet.operator.WindowExec.allowIncompatible=true. For more information, refer to the Comet Compatibility Guide (https://datafusion.apache.org/comet/user-guide/compatibility.html).] : : +- CometColumnarToRow : : +- CometSort : : +- CometExchange @@ -91,7 +91,7 @@ : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim : +- BroadcastExchange : +- Project - : +- Window [COMET: WindowExec is not fully compatible with Spark (Native WindowExec has known correctness issues). To enable it anyway, set spark.comet.operator.WindowExec.allowIncompatible=true. For more information, refer to the Comet Compatibility Guide (https://datafusion.apache.org/comet/user-guide/compatibility.html)., Partitioning and sorting specifications must be the same.] + : +- Window [COMET: WindowExec is not fully compatible with Spark (Native WindowExec has known correctness issues). To enable it anyway, set spark.comet.operator.WindowExec.allowIncompatible=true. For more information, refer to the Comet Compatibility Guide (https://datafusion.apache.org/comet/user-guide/compatibility.html).] : +- CometColumnarToRow : +- CometSort : +- CometExchange @@ -109,7 +109,7 @@ : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim +- BroadcastExchange +- Project - +- Window [COMET: WindowExec is not fully compatible with Spark (Native WindowExec has known correctness issues). To enable it anyway, set spark.comet.operator.WindowExec.allowIncompatible=true. For more information, refer to the Comet Compatibility Guide (https://datafusion.apache.org/comet/user-guide/compatibility.html)., Partitioning and sorting specifications must be the same.] + +- Window [COMET: WindowExec is not fully compatible with Spark (Native WindowExec has known correctness issues). To enable it anyway, set spark.comet.operator.WindowExec.allowIncompatible=true. For more information, refer to the Comet Compatibility Guide (https://datafusion.apache.org/comet/user-guide/compatibility.html).] +- CometColumnarToRow +- CometSort +- CometExchange @@ -125,7 +125,7 @@ : +- Project : +- BroadcastHashJoin : :- Project - : : +- Window [COMET: WindowExec is not fully compatible with Spark (Native WindowExec has known correctness issues). To enable it anyway, set spark.comet.operator.WindowExec.allowIncompatible=true. For more information, refer to the Comet Compatibility Guide (https://datafusion.apache.org/comet/user-guide/compatibility.html)., Partitioning and sorting specifications must be the same.] + : : +- Window [COMET: WindowExec is not fully compatible with Spark (Native WindowExec has known correctness issues). To enable it anyway, set spark.comet.operator.WindowExec.allowIncompatible=true. For more information, refer to the Comet Compatibility Guide (https://datafusion.apache.org/comet/user-guide/compatibility.html).] : : +- CometColumnarToRow : : +- CometSort : : +- CometExchange @@ -148,7 +148,7 @@ : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim : +- BroadcastExchange : +- Project - : +- Window [COMET: WindowExec is not fully compatible with Spark (Native WindowExec has known correctness issues). To enable it anyway, set spark.comet.operator.WindowExec.allowIncompatible=true. For more information, refer to the Comet Compatibility Guide (https://datafusion.apache.org/comet/user-guide/compatibility.html)., Partitioning and sorting specifications must be the same.] + : +- Window [COMET: WindowExec is not fully compatible with Spark (Native WindowExec has known correctness issues). To enable it anyway, set spark.comet.operator.WindowExec.allowIncompatible=true. For more information, refer to the Comet Compatibility Guide (https://datafusion.apache.org/comet/user-guide/compatibility.html).] : +- CometColumnarToRow : +- CometSort : +- CometExchange @@ -178,7 +178,7 @@ +- Project +- BroadcastHashJoin :- Project - : +- Window [COMET: WindowExec is not fully compatible with Spark (Native WindowExec has known correctness issues). To enable it anyway, set spark.comet.operator.WindowExec.allowIncompatible=true. For more information, refer to the Comet Compatibility Guide (https://datafusion.apache.org/comet/user-guide/compatibility.html)., Partitioning and sorting specifications must be the same.] + : +- Window [COMET: WindowExec is not fully compatible with Spark (Native WindowExec has known correctness issues). To enable it anyway, set spark.comet.operator.WindowExec.allowIncompatible=true. For more information, refer to the Comet Compatibility Guide (https://datafusion.apache.org/comet/user-guide/compatibility.html).] : +- CometColumnarToRow : +- CometSort : +- CometExchange @@ -196,7 +196,7 @@ : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim +- BroadcastExchange +- Project - +- Window [COMET: WindowExec is not fully compatible with Spark (Native WindowExec has known correctness issues). To enable it anyway, set spark.comet.operator.WindowExec.allowIncompatible=true. For more information, refer to the Comet Compatibility Guide (https://datafusion.apache.org/comet/user-guide/compatibility.html)., Partitioning and sorting specifications must be the same.] + +- Window [COMET: WindowExec is not fully compatible with Spark (Native WindowExec has known correctness issues). To enable it anyway, set spark.comet.operator.WindowExec.allowIncompatible=true. For more information, refer to the Comet Compatibility Guide (https://datafusion.apache.org/comet/user-guide/compatibility.html).] +- CometColumnarToRow +- CometSort +- CometExchange diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7-spark3_5/q51a/extended.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7-spark3_5/q51a/extended.txt index b192555935..b8822715ca 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7-spark3_5/q51a/extended.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7-spark3_5/q51a/extended.txt @@ -4,7 +4,7 @@ +- HashAggregate +- Project +- BroadcastHashJoin - :- Window [COMET: WindowExec is not fully compatible with Spark (Native WindowExec has known correctness issues). To enable it anyway, set spark.comet.operator.WindowExec.allowIncompatible=true. For more information, refer to the Comet Compatibility Guide (https://datafusion.apache.org/comet/user-guide/compatibility.html)., Partitioning and sorting specifications must be the same.] + :- Window [COMET: WindowExec is not fully compatible with Spark (Native WindowExec has known correctness issues). To enable it anyway, set spark.comet.operator.WindowExec.allowIncompatible=true. For more information, refer to the Comet Compatibility Guide (https://datafusion.apache.org/comet/user-guide/compatibility.html).] : +- CometColumnarToRow : +- CometSort : +- CometExchange @@ -20,7 +20,7 @@ : : +- Project : : +- BroadcastHashJoin : : :- Project - : : : +- Window [COMET: WindowExec is not fully compatible with Spark (Native WindowExec has known correctness issues). To enable it anyway, set spark.comet.operator.WindowExec.allowIncompatible=true. For more information, refer to the Comet Compatibility Guide (https://datafusion.apache.org/comet/user-guide/compatibility.html)., Partitioning and sorting specifications must be the same.] + : : : +- Window [COMET: WindowExec is not fully compatible with Spark (Native WindowExec has known correctness issues). To enable it anyway, set spark.comet.operator.WindowExec.allowIncompatible=true. For more information, refer to the Comet Compatibility Guide (https://datafusion.apache.org/comet/user-guide/compatibility.html).] : : : +- CometColumnarToRow : : : +- CometSort : : : +- CometExchange @@ -43,7 +43,7 @@ : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim : : +- BroadcastExchange : : +- Project - : : +- Window [COMET: WindowExec is not fully compatible with Spark (Native WindowExec has known correctness issues). To enable it anyway, set spark.comet.operator.WindowExec.allowIncompatible=true. For more information, refer to the Comet Compatibility Guide (https://datafusion.apache.org/comet/user-guide/compatibility.html)., Partitioning and sorting specifications must be the same.] + : : +- Window [COMET: WindowExec is not fully compatible with Spark (Native WindowExec has known correctness issues). To enable it anyway, set spark.comet.operator.WindowExec.allowIncompatible=true. For more information, refer to the Comet Compatibility Guide (https://datafusion.apache.org/comet/user-guide/compatibility.html).] : : +- CometColumnarToRow : : +- CometSort : : +- CometExchange @@ -73,7 +73,7 @@ : +- Project : +- BroadcastHashJoin : :- Project - : : +- Window [COMET: WindowExec is not fully compatible with Spark (Native WindowExec has known correctness issues). To enable it anyway, set spark.comet.operator.WindowExec.allowIncompatible=true. For more information, refer to the Comet Compatibility Guide (https://datafusion.apache.org/comet/user-guide/compatibility.html)., Partitioning and sorting specifications must be the same.] + : : +- Window [COMET: WindowExec is not fully compatible with Spark (Native WindowExec has known correctness issues). To enable it anyway, set spark.comet.operator.WindowExec.allowIncompatible=true. For more information, refer to the Comet Compatibility Guide (https://datafusion.apache.org/comet/user-guide/compatibility.html).] : : +- CometColumnarToRow : : +- CometSort : : +- CometExchange @@ -91,7 +91,7 @@ : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim : +- BroadcastExchange : +- Project - : +- Window [COMET: WindowExec is not fully compatible with Spark (Native WindowExec has known correctness issues). To enable it anyway, set spark.comet.operator.WindowExec.allowIncompatible=true. For more information, refer to the Comet Compatibility Guide (https://datafusion.apache.org/comet/user-guide/compatibility.html)., Partitioning and sorting specifications must be the same.] + : +- Window [COMET: WindowExec is not fully compatible with Spark (Native WindowExec has known correctness issues). To enable it anyway, set spark.comet.operator.WindowExec.allowIncompatible=true. For more information, refer to the Comet Compatibility Guide (https://datafusion.apache.org/comet/user-guide/compatibility.html).] : +- CometColumnarToRow : +- CometSort : +- CometExchange @@ -109,7 +109,7 @@ : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim +- BroadcastExchange +- Project - +- Window [COMET: WindowExec is not fully compatible with Spark (Native WindowExec has known correctness issues). To enable it anyway, set spark.comet.operator.WindowExec.allowIncompatible=true. For more information, refer to the Comet Compatibility Guide (https://datafusion.apache.org/comet/user-guide/compatibility.html)., Partitioning and sorting specifications must be the same.] + +- Window [COMET: WindowExec is not fully compatible with Spark (Native WindowExec has known correctness issues). To enable it anyway, set spark.comet.operator.WindowExec.allowIncompatible=true. For more information, refer to the Comet Compatibility Guide (https://datafusion.apache.org/comet/user-guide/compatibility.html).] +- CometColumnarToRow +- CometSort +- CometExchange @@ -125,7 +125,7 @@ : +- Project : +- BroadcastHashJoin : :- Project - : : +- Window [COMET: WindowExec is not fully compatible with Spark (Native WindowExec has known correctness issues). To enable it anyway, set spark.comet.operator.WindowExec.allowIncompatible=true. For more information, refer to the Comet Compatibility Guide (https://datafusion.apache.org/comet/user-guide/compatibility.html)., Partitioning and sorting specifications must be the same.] + : : +- Window [COMET: WindowExec is not fully compatible with Spark (Native WindowExec has known correctness issues). To enable it anyway, set spark.comet.operator.WindowExec.allowIncompatible=true. For more information, refer to the Comet Compatibility Guide (https://datafusion.apache.org/comet/user-guide/compatibility.html).] : : +- CometColumnarToRow : : +- CometSort : : +- CometExchange @@ -148,7 +148,7 @@ : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim : +- BroadcastExchange : +- Project - : +- Window [COMET: WindowExec is not fully compatible with Spark (Native WindowExec has known correctness issues). To enable it anyway, set spark.comet.operator.WindowExec.allowIncompatible=true. For more information, refer to the Comet Compatibility Guide (https://datafusion.apache.org/comet/user-guide/compatibility.html)., Partitioning and sorting specifications must be the same.] + : +- Window [COMET: WindowExec is not fully compatible with Spark (Native WindowExec has known correctness issues). To enable it anyway, set spark.comet.operator.WindowExec.allowIncompatible=true. For more information, refer to the Comet Compatibility Guide (https://datafusion.apache.org/comet/user-guide/compatibility.html).] : +- CometColumnarToRow : +- CometSort : +- CometExchange @@ -178,7 +178,7 @@ +- Project +- BroadcastHashJoin :- Project - : +- Window [COMET: WindowExec is not fully compatible with Spark (Native WindowExec has known correctness issues). To enable it anyway, set spark.comet.operator.WindowExec.allowIncompatible=true. For more information, refer to the Comet Compatibility Guide (https://datafusion.apache.org/comet/user-guide/compatibility.html)., Partitioning and sorting specifications must be the same.] + : +- Window [COMET: WindowExec is not fully compatible with Spark (Native WindowExec has known correctness issues). To enable it anyway, set spark.comet.operator.WindowExec.allowIncompatible=true. For more information, refer to the Comet Compatibility Guide (https://datafusion.apache.org/comet/user-guide/compatibility.html).] : +- CometColumnarToRow : +- CometSort : +- CometExchange @@ -196,7 +196,7 @@ : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim +- BroadcastExchange +- Project - +- Window [COMET: WindowExec is not fully compatible with Spark (Native WindowExec has known correctness issues). To enable it anyway, set spark.comet.operator.WindowExec.allowIncompatible=true. For more information, refer to the Comet Compatibility Guide (https://datafusion.apache.org/comet/user-guide/compatibility.html)., Partitioning and sorting specifications must be the same.] + +- Window [COMET: WindowExec is not fully compatible with Spark (Native WindowExec has known correctness issues). To enable it anyway, set spark.comet.operator.WindowExec.allowIncompatible=true. For more information, refer to the Comet Compatibility Guide (https://datafusion.apache.org/comet/user-guide/compatibility.html).] +- CometColumnarToRow +- CometSort +- CometExchange From 04899f29bc99e8d1593018a510c3486dd3a6ddd7 Mon Sep 17 00:00:00 2001 From: Andy Grove Date: Sat, 8 Nov 2025 17:13:28 -0700 Subject: [PATCH 44/44] update 3.4 golden files --- .../q51.native_iceberg_compat/extended.txt | 6 +++--- .../approved-plans-v1_4/q51/extended.txt | 6 +++--- .../q67.native_iceberg_compat/extended.txt | 2 +- .../approved-plans-v1_4/q67/extended.txt | 2 +- .../q70.native_iceberg_compat/extended.txt | 2 +- .../approved-plans-v1_4/q70/extended.txt | 2 +- .../q51a.native_iceberg_compat/extended.txt | 20 +++++++++---------- .../approved-plans-v2_7/q51a/extended.txt | 20 +++++++++---------- .../q67a.native_iceberg_compat/extended.txt | 2 +- .../approved-plans-v2_7/q67a/extended.txt | 2 +- .../q70a.native_iceberg_compat/extended.txt | 6 +++--- .../approved-plans-v2_7/q70a/extended.txt | 6 +++--- 12 files changed, 38 insertions(+), 38 deletions(-) diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q51.native_iceberg_compat/extended.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q51.native_iceberg_compat/extended.txt index d55771ac8a..17d5d53446 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q51.native_iceberg_compat/extended.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q51.native_iceberg_compat/extended.txt @@ -1,6 +1,6 @@ TakeOrderedAndProject [COMET: ] +- Filter - +- Window [COMET: WindowExec is not fully compatible with Spark (Native WindowExec has known correctness issues). To enable it anyway, set spark.comet.operator.WindowExec.allowIncompatible=true. For more information, refer to the Comet Compatibility Guide (https://datafusion.apache.org/comet/user-guide/compatibility.html)., Partitioning and sorting specifications must be the same.] + +- Window [COMET: WindowExec is not fully compatible with Spark (Native WindowExec has known correctness issues). To enable it anyway, set spark.comet.operator.WindowExec.allowIncompatible=true. For more information, refer to the Comet Compatibility Guide (https://datafusion.apache.org/comet/user-guide/compatibility.html).] +- CometColumnarToRow +- CometSort +- CometExchange @@ -9,7 +9,7 @@ :- CometSort : +- CometColumnarExchange : +- Project - : +- Window [COMET: WindowExec is not fully compatible with Spark (Native WindowExec has known correctness issues). To enable it anyway, set spark.comet.operator.WindowExec.allowIncompatible=true. For more information, refer to the Comet Compatibility Guide (https://datafusion.apache.org/comet/user-guide/compatibility.html)., Partitioning and sorting specifications must be the same.] + : +- Window [COMET: WindowExec is not fully compatible with Spark (Native WindowExec has known correctness issues). To enable it anyway, set spark.comet.operator.WindowExec.allowIncompatible=true. For more information, refer to the Comet Compatibility Guide (https://datafusion.apache.org/comet/user-guide/compatibility.html).] : +- CometColumnarToRow : +- CometSort : +- CometExchange @@ -33,7 +33,7 @@ +- CometSort +- CometColumnarExchange +- Project - +- Window [COMET: WindowExec is not fully compatible with Spark (Native WindowExec has known correctness issues). To enable it anyway, set spark.comet.operator.WindowExec.allowIncompatible=true. For more information, refer to the Comet Compatibility Guide (https://datafusion.apache.org/comet/user-guide/compatibility.html)., Partitioning and sorting specifications must be the same.] + +- Window [COMET: WindowExec is not fully compatible with Spark (Native WindowExec has known correctness issues). To enable it anyway, set spark.comet.operator.WindowExec.allowIncompatible=true. For more information, refer to the Comet Compatibility Guide (https://datafusion.apache.org/comet/user-guide/compatibility.html).] +- CometColumnarToRow +- CometSort +- CometExchange diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q51/extended.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q51/extended.txt index d55771ac8a..17d5d53446 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q51/extended.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q51/extended.txt @@ -1,6 +1,6 @@ TakeOrderedAndProject [COMET: ] +- Filter - +- Window [COMET: WindowExec is not fully compatible with Spark (Native WindowExec has known correctness issues). To enable it anyway, set spark.comet.operator.WindowExec.allowIncompatible=true. For more information, refer to the Comet Compatibility Guide (https://datafusion.apache.org/comet/user-guide/compatibility.html)., Partitioning and sorting specifications must be the same.] + +- Window [COMET: WindowExec is not fully compatible with Spark (Native WindowExec has known correctness issues). To enable it anyway, set spark.comet.operator.WindowExec.allowIncompatible=true. For more information, refer to the Comet Compatibility Guide (https://datafusion.apache.org/comet/user-guide/compatibility.html).] +- CometColumnarToRow +- CometSort +- CometExchange @@ -9,7 +9,7 @@ :- CometSort : +- CometColumnarExchange : +- Project - : +- Window [COMET: WindowExec is not fully compatible with Spark (Native WindowExec has known correctness issues). To enable it anyway, set spark.comet.operator.WindowExec.allowIncompatible=true. For more information, refer to the Comet Compatibility Guide (https://datafusion.apache.org/comet/user-guide/compatibility.html)., Partitioning and sorting specifications must be the same.] + : +- Window [COMET: WindowExec is not fully compatible with Spark (Native WindowExec has known correctness issues). To enable it anyway, set spark.comet.operator.WindowExec.allowIncompatible=true. For more information, refer to the Comet Compatibility Guide (https://datafusion.apache.org/comet/user-guide/compatibility.html).] : +- CometColumnarToRow : +- CometSort : +- CometExchange @@ -33,7 +33,7 @@ +- CometSort +- CometColumnarExchange +- Project - +- Window [COMET: WindowExec is not fully compatible with Spark (Native WindowExec has known correctness issues). To enable it anyway, set spark.comet.operator.WindowExec.allowIncompatible=true. For more information, refer to the Comet Compatibility Guide (https://datafusion.apache.org/comet/user-guide/compatibility.html)., Partitioning and sorting specifications must be the same.] + +- Window [COMET: WindowExec is not fully compatible with Spark (Native WindowExec has known correctness issues). To enable it anyway, set spark.comet.operator.WindowExec.allowIncompatible=true. For more information, refer to the Comet Compatibility Guide (https://datafusion.apache.org/comet/user-guide/compatibility.html).] +- CometColumnarToRow +- CometSort +- CometExchange diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q67.native_iceberg_compat/extended.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q67.native_iceberg_compat/extended.txt index ac51af90b6..cbce322169 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q67.native_iceberg_compat/extended.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q67.native_iceberg_compat/extended.txt @@ -1,6 +1,6 @@ TakeOrderedAndProject [COMET: ] +- Filter - +- Window [COMET: WindowExec is not fully compatible with Spark (Native WindowExec has known correctness issues). To enable it anyway, set spark.comet.operator.WindowExec.allowIncompatible=true. For more information, refer to the Comet Compatibility Guide (https://datafusion.apache.org/comet/user-guide/compatibility.html)., Partitioning and sorting specifications must be the same.] + +- Window [COMET: WindowExec is not fully compatible with Spark (Native WindowExec has known correctness issues). To enable it anyway, set spark.comet.operator.WindowExec.allowIncompatible=true. For more information, refer to the Comet Compatibility Guide (https://datafusion.apache.org/comet/user-guide/compatibility.html).] +- CometColumnarToRow +- CometSort +- CometExchange diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q67/extended.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q67/extended.txt index ac51af90b6..cbce322169 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q67/extended.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q67/extended.txt @@ -1,6 +1,6 @@ TakeOrderedAndProject [COMET: ] +- Filter - +- Window [COMET: WindowExec is not fully compatible with Spark (Native WindowExec has known correctness issues). To enable it anyway, set spark.comet.operator.WindowExec.allowIncompatible=true. For more information, refer to the Comet Compatibility Guide (https://datafusion.apache.org/comet/user-guide/compatibility.html)., Partitioning and sorting specifications must be the same.] + +- Window [COMET: WindowExec is not fully compatible with Spark (Native WindowExec has known correctness issues). To enable it anyway, set spark.comet.operator.WindowExec.allowIncompatible=true. For more information, refer to the Comet Compatibility Guide (https://datafusion.apache.org/comet/user-guide/compatibility.html).] +- CometColumnarToRow +- CometSort +- CometExchange diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q70.native_iceberg_compat/extended.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q70.native_iceberg_compat/extended.txt index 9141654d2a..7535cbc088 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q70.native_iceberg_compat/extended.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q70.native_iceberg_compat/extended.txt @@ -35,7 +35,7 @@ +- BroadcastExchange +- Project +- Filter - +- Window [COMET: WindowExec is not fully compatible with Spark (Native WindowExec has known correctness issues). To enable it anyway, set spark.comet.operator.WindowExec.allowIncompatible=true. For more information, refer to the Comet Compatibility Guide (https://datafusion.apache.org/comet/user-guide/compatibility.html)., Partitioning and sorting specifications must be the same.] + +- Window [COMET: WindowExec is not fully compatible with Spark (Native WindowExec has known correctness issues). To enable it anyway, set spark.comet.operator.WindowExec.allowIncompatible=true. For more information, refer to the Comet Compatibility Guide (https://datafusion.apache.org/comet/user-guide/compatibility.html).] +- CometColumnarToRow +- CometSort +- CometHashAggregate diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q70/extended.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q70/extended.txt index 9141654d2a..7535cbc088 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q70/extended.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q70/extended.txt @@ -35,7 +35,7 @@ +- BroadcastExchange +- Project +- Filter - +- Window [COMET: WindowExec is not fully compatible with Spark (Native WindowExec has known correctness issues). To enable it anyway, set spark.comet.operator.WindowExec.allowIncompatible=true. For more information, refer to the Comet Compatibility Guide (https://datafusion.apache.org/comet/user-guide/compatibility.html)., Partitioning and sorting specifications must be the same.] + +- Window [COMET: WindowExec is not fully compatible with Spark (Native WindowExec has known correctness issues). To enable it anyway, set spark.comet.operator.WindowExec.allowIncompatible=true. For more information, refer to the Comet Compatibility Guide (https://datafusion.apache.org/comet/user-guide/compatibility.html).] +- CometColumnarToRow +- CometSort +- CometHashAggregate diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7/q51a.native_iceberg_compat/extended.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7/q51a.native_iceberg_compat/extended.txt index b192555935..b8822715ca 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7/q51a.native_iceberg_compat/extended.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7/q51a.native_iceberg_compat/extended.txt @@ -4,7 +4,7 @@ +- HashAggregate +- Project +- BroadcastHashJoin - :- Window [COMET: WindowExec is not fully compatible with Spark (Native WindowExec has known correctness issues). To enable it anyway, set spark.comet.operator.WindowExec.allowIncompatible=true. For more information, refer to the Comet Compatibility Guide (https://datafusion.apache.org/comet/user-guide/compatibility.html)., Partitioning and sorting specifications must be the same.] + :- Window [COMET: WindowExec is not fully compatible with Spark (Native WindowExec has known correctness issues). To enable it anyway, set spark.comet.operator.WindowExec.allowIncompatible=true. For more information, refer to the Comet Compatibility Guide (https://datafusion.apache.org/comet/user-guide/compatibility.html).] : +- CometColumnarToRow : +- CometSort : +- CometExchange @@ -20,7 +20,7 @@ : : +- Project : : +- BroadcastHashJoin : : :- Project - : : : +- Window [COMET: WindowExec is not fully compatible with Spark (Native WindowExec has known correctness issues). To enable it anyway, set spark.comet.operator.WindowExec.allowIncompatible=true. For more information, refer to the Comet Compatibility Guide (https://datafusion.apache.org/comet/user-guide/compatibility.html)., Partitioning and sorting specifications must be the same.] + : : : +- Window [COMET: WindowExec is not fully compatible with Spark (Native WindowExec has known correctness issues). To enable it anyway, set spark.comet.operator.WindowExec.allowIncompatible=true. For more information, refer to the Comet Compatibility Guide (https://datafusion.apache.org/comet/user-guide/compatibility.html).] : : : +- CometColumnarToRow : : : +- CometSort : : : +- CometExchange @@ -43,7 +43,7 @@ : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim : : +- BroadcastExchange : : +- Project - : : +- Window [COMET: WindowExec is not fully compatible with Spark (Native WindowExec has known correctness issues). To enable it anyway, set spark.comet.operator.WindowExec.allowIncompatible=true. For more information, refer to the Comet Compatibility Guide (https://datafusion.apache.org/comet/user-guide/compatibility.html)., Partitioning and sorting specifications must be the same.] + : : +- Window [COMET: WindowExec is not fully compatible with Spark (Native WindowExec has known correctness issues). To enable it anyway, set spark.comet.operator.WindowExec.allowIncompatible=true. For more information, refer to the Comet Compatibility Guide (https://datafusion.apache.org/comet/user-guide/compatibility.html).] : : +- CometColumnarToRow : : +- CometSort : : +- CometExchange @@ -73,7 +73,7 @@ : +- Project : +- BroadcastHashJoin : :- Project - : : +- Window [COMET: WindowExec is not fully compatible with Spark (Native WindowExec has known correctness issues). To enable it anyway, set spark.comet.operator.WindowExec.allowIncompatible=true. For more information, refer to the Comet Compatibility Guide (https://datafusion.apache.org/comet/user-guide/compatibility.html)., Partitioning and sorting specifications must be the same.] + : : +- Window [COMET: WindowExec is not fully compatible with Spark (Native WindowExec has known correctness issues). To enable it anyway, set spark.comet.operator.WindowExec.allowIncompatible=true. For more information, refer to the Comet Compatibility Guide (https://datafusion.apache.org/comet/user-guide/compatibility.html).] : : +- CometColumnarToRow : : +- CometSort : : +- CometExchange @@ -91,7 +91,7 @@ : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim : +- BroadcastExchange : +- Project - : +- Window [COMET: WindowExec is not fully compatible with Spark (Native WindowExec has known correctness issues). To enable it anyway, set spark.comet.operator.WindowExec.allowIncompatible=true. For more information, refer to the Comet Compatibility Guide (https://datafusion.apache.org/comet/user-guide/compatibility.html)., Partitioning and sorting specifications must be the same.] + : +- Window [COMET: WindowExec is not fully compatible with Spark (Native WindowExec has known correctness issues). To enable it anyway, set spark.comet.operator.WindowExec.allowIncompatible=true. For more information, refer to the Comet Compatibility Guide (https://datafusion.apache.org/comet/user-guide/compatibility.html).] : +- CometColumnarToRow : +- CometSort : +- CometExchange @@ -109,7 +109,7 @@ : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim +- BroadcastExchange +- Project - +- Window [COMET: WindowExec is not fully compatible with Spark (Native WindowExec has known correctness issues). To enable it anyway, set spark.comet.operator.WindowExec.allowIncompatible=true. For more information, refer to the Comet Compatibility Guide (https://datafusion.apache.org/comet/user-guide/compatibility.html)., Partitioning and sorting specifications must be the same.] + +- Window [COMET: WindowExec is not fully compatible with Spark (Native WindowExec has known correctness issues). To enable it anyway, set spark.comet.operator.WindowExec.allowIncompatible=true. For more information, refer to the Comet Compatibility Guide (https://datafusion.apache.org/comet/user-guide/compatibility.html).] +- CometColumnarToRow +- CometSort +- CometExchange @@ -125,7 +125,7 @@ : +- Project : +- BroadcastHashJoin : :- Project - : : +- Window [COMET: WindowExec is not fully compatible with Spark (Native WindowExec has known correctness issues). To enable it anyway, set spark.comet.operator.WindowExec.allowIncompatible=true. For more information, refer to the Comet Compatibility Guide (https://datafusion.apache.org/comet/user-guide/compatibility.html)., Partitioning and sorting specifications must be the same.] + : : +- Window [COMET: WindowExec is not fully compatible with Spark (Native WindowExec has known correctness issues). To enable it anyway, set spark.comet.operator.WindowExec.allowIncompatible=true. For more information, refer to the Comet Compatibility Guide (https://datafusion.apache.org/comet/user-guide/compatibility.html).] : : +- CometColumnarToRow : : +- CometSort : : +- CometExchange @@ -148,7 +148,7 @@ : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim : +- BroadcastExchange : +- Project - : +- Window [COMET: WindowExec is not fully compatible with Spark (Native WindowExec has known correctness issues). To enable it anyway, set spark.comet.operator.WindowExec.allowIncompatible=true. For more information, refer to the Comet Compatibility Guide (https://datafusion.apache.org/comet/user-guide/compatibility.html)., Partitioning and sorting specifications must be the same.] + : +- Window [COMET: WindowExec is not fully compatible with Spark (Native WindowExec has known correctness issues). To enable it anyway, set spark.comet.operator.WindowExec.allowIncompatible=true. For more information, refer to the Comet Compatibility Guide (https://datafusion.apache.org/comet/user-guide/compatibility.html).] : +- CometColumnarToRow : +- CometSort : +- CometExchange @@ -178,7 +178,7 @@ +- Project +- BroadcastHashJoin :- Project - : +- Window [COMET: WindowExec is not fully compatible with Spark (Native WindowExec has known correctness issues). To enable it anyway, set spark.comet.operator.WindowExec.allowIncompatible=true. For more information, refer to the Comet Compatibility Guide (https://datafusion.apache.org/comet/user-guide/compatibility.html)., Partitioning and sorting specifications must be the same.] + : +- Window [COMET: WindowExec is not fully compatible with Spark (Native WindowExec has known correctness issues). To enable it anyway, set spark.comet.operator.WindowExec.allowIncompatible=true. For more information, refer to the Comet Compatibility Guide (https://datafusion.apache.org/comet/user-guide/compatibility.html).] : +- CometColumnarToRow : +- CometSort : +- CometExchange @@ -196,7 +196,7 @@ : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim +- BroadcastExchange +- Project - +- Window [COMET: WindowExec is not fully compatible with Spark (Native WindowExec has known correctness issues). To enable it anyway, set spark.comet.operator.WindowExec.allowIncompatible=true. For more information, refer to the Comet Compatibility Guide (https://datafusion.apache.org/comet/user-guide/compatibility.html)., Partitioning and sorting specifications must be the same.] + +- Window [COMET: WindowExec is not fully compatible with Spark (Native WindowExec has known correctness issues). To enable it anyway, set spark.comet.operator.WindowExec.allowIncompatible=true. For more information, refer to the Comet Compatibility Guide (https://datafusion.apache.org/comet/user-guide/compatibility.html).] +- CometColumnarToRow +- CometSort +- CometExchange diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7/q51a/extended.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7/q51a/extended.txt index b192555935..b8822715ca 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7/q51a/extended.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7/q51a/extended.txt @@ -4,7 +4,7 @@ +- HashAggregate +- Project +- BroadcastHashJoin - :- Window [COMET: WindowExec is not fully compatible with Spark (Native WindowExec has known correctness issues). To enable it anyway, set spark.comet.operator.WindowExec.allowIncompatible=true. For more information, refer to the Comet Compatibility Guide (https://datafusion.apache.org/comet/user-guide/compatibility.html)., Partitioning and sorting specifications must be the same.] + :- Window [COMET: WindowExec is not fully compatible with Spark (Native WindowExec has known correctness issues). To enable it anyway, set spark.comet.operator.WindowExec.allowIncompatible=true. For more information, refer to the Comet Compatibility Guide (https://datafusion.apache.org/comet/user-guide/compatibility.html).] : +- CometColumnarToRow : +- CometSort : +- CometExchange @@ -20,7 +20,7 @@ : : +- Project : : +- BroadcastHashJoin : : :- Project - : : : +- Window [COMET: WindowExec is not fully compatible with Spark (Native WindowExec has known correctness issues). To enable it anyway, set spark.comet.operator.WindowExec.allowIncompatible=true. For more information, refer to the Comet Compatibility Guide (https://datafusion.apache.org/comet/user-guide/compatibility.html)., Partitioning and sorting specifications must be the same.] + : : : +- Window [COMET: WindowExec is not fully compatible with Spark (Native WindowExec has known correctness issues). To enable it anyway, set spark.comet.operator.WindowExec.allowIncompatible=true. For more information, refer to the Comet Compatibility Guide (https://datafusion.apache.org/comet/user-guide/compatibility.html).] : : : +- CometColumnarToRow : : : +- CometSort : : : +- CometExchange @@ -43,7 +43,7 @@ : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim : : +- BroadcastExchange : : +- Project - : : +- Window [COMET: WindowExec is not fully compatible with Spark (Native WindowExec has known correctness issues). To enable it anyway, set spark.comet.operator.WindowExec.allowIncompatible=true. For more information, refer to the Comet Compatibility Guide (https://datafusion.apache.org/comet/user-guide/compatibility.html)., Partitioning and sorting specifications must be the same.] + : : +- Window [COMET: WindowExec is not fully compatible with Spark (Native WindowExec has known correctness issues). To enable it anyway, set spark.comet.operator.WindowExec.allowIncompatible=true. For more information, refer to the Comet Compatibility Guide (https://datafusion.apache.org/comet/user-guide/compatibility.html).] : : +- CometColumnarToRow : : +- CometSort : : +- CometExchange @@ -73,7 +73,7 @@ : +- Project : +- BroadcastHashJoin : :- Project - : : +- Window [COMET: WindowExec is not fully compatible with Spark (Native WindowExec has known correctness issues). To enable it anyway, set spark.comet.operator.WindowExec.allowIncompatible=true. For more information, refer to the Comet Compatibility Guide (https://datafusion.apache.org/comet/user-guide/compatibility.html)., Partitioning and sorting specifications must be the same.] + : : +- Window [COMET: WindowExec is not fully compatible with Spark (Native WindowExec has known correctness issues). To enable it anyway, set spark.comet.operator.WindowExec.allowIncompatible=true. For more information, refer to the Comet Compatibility Guide (https://datafusion.apache.org/comet/user-guide/compatibility.html).] : : +- CometColumnarToRow : : +- CometSort : : +- CometExchange @@ -91,7 +91,7 @@ : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim : +- BroadcastExchange : +- Project - : +- Window [COMET: WindowExec is not fully compatible with Spark (Native WindowExec has known correctness issues). To enable it anyway, set spark.comet.operator.WindowExec.allowIncompatible=true. For more information, refer to the Comet Compatibility Guide (https://datafusion.apache.org/comet/user-guide/compatibility.html)., Partitioning and sorting specifications must be the same.] + : +- Window [COMET: WindowExec is not fully compatible with Spark (Native WindowExec has known correctness issues). To enable it anyway, set spark.comet.operator.WindowExec.allowIncompatible=true. For more information, refer to the Comet Compatibility Guide (https://datafusion.apache.org/comet/user-guide/compatibility.html).] : +- CometColumnarToRow : +- CometSort : +- CometExchange @@ -109,7 +109,7 @@ : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim +- BroadcastExchange +- Project - +- Window [COMET: WindowExec is not fully compatible with Spark (Native WindowExec has known correctness issues). To enable it anyway, set spark.comet.operator.WindowExec.allowIncompatible=true. For more information, refer to the Comet Compatibility Guide (https://datafusion.apache.org/comet/user-guide/compatibility.html)., Partitioning and sorting specifications must be the same.] + +- Window [COMET: WindowExec is not fully compatible with Spark (Native WindowExec has known correctness issues). To enable it anyway, set spark.comet.operator.WindowExec.allowIncompatible=true. For more information, refer to the Comet Compatibility Guide (https://datafusion.apache.org/comet/user-guide/compatibility.html).] +- CometColumnarToRow +- CometSort +- CometExchange @@ -125,7 +125,7 @@ : +- Project : +- BroadcastHashJoin : :- Project - : : +- Window [COMET: WindowExec is not fully compatible with Spark (Native WindowExec has known correctness issues). To enable it anyway, set spark.comet.operator.WindowExec.allowIncompatible=true. For more information, refer to the Comet Compatibility Guide (https://datafusion.apache.org/comet/user-guide/compatibility.html)., Partitioning and sorting specifications must be the same.] + : : +- Window [COMET: WindowExec is not fully compatible with Spark (Native WindowExec has known correctness issues). To enable it anyway, set spark.comet.operator.WindowExec.allowIncompatible=true. For more information, refer to the Comet Compatibility Guide (https://datafusion.apache.org/comet/user-guide/compatibility.html).] : : +- CometColumnarToRow : : +- CometSort : : +- CometExchange @@ -148,7 +148,7 @@ : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim : +- BroadcastExchange : +- Project - : +- Window [COMET: WindowExec is not fully compatible with Spark (Native WindowExec has known correctness issues). To enable it anyway, set spark.comet.operator.WindowExec.allowIncompatible=true. For more information, refer to the Comet Compatibility Guide (https://datafusion.apache.org/comet/user-guide/compatibility.html)., Partitioning and sorting specifications must be the same.] + : +- Window [COMET: WindowExec is not fully compatible with Spark (Native WindowExec has known correctness issues). To enable it anyway, set spark.comet.operator.WindowExec.allowIncompatible=true. For more information, refer to the Comet Compatibility Guide (https://datafusion.apache.org/comet/user-guide/compatibility.html).] : +- CometColumnarToRow : +- CometSort : +- CometExchange @@ -178,7 +178,7 @@ +- Project +- BroadcastHashJoin :- Project - : +- Window [COMET: WindowExec is not fully compatible with Spark (Native WindowExec has known correctness issues). To enable it anyway, set spark.comet.operator.WindowExec.allowIncompatible=true. For more information, refer to the Comet Compatibility Guide (https://datafusion.apache.org/comet/user-guide/compatibility.html)., Partitioning and sorting specifications must be the same.] + : +- Window [COMET: WindowExec is not fully compatible with Spark (Native WindowExec has known correctness issues). To enable it anyway, set spark.comet.operator.WindowExec.allowIncompatible=true. For more information, refer to the Comet Compatibility Guide (https://datafusion.apache.org/comet/user-guide/compatibility.html).] : +- CometColumnarToRow : +- CometSort : +- CometExchange @@ -196,7 +196,7 @@ : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim +- BroadcastExchange +- Project - +- Window [COMET: WindowExec is not fully compatible with Spark (Native WindowExec has known correctness issues). To enable it anyway, set spark.comet.operator.WindowExec.allowIncompatible=true. For more information, refer to the Comet Compatibility Guide (https://datafusion.apache.org/comet/user-guide/compatibility.html)., Partitioning and sorting specifications must be the same.] + +- Window [COMET: WindowExec is not fully compatible with Spark (Native WindowExec has known correctness issues). To enable it anyway, set spark.comet.operator.WindowExec.allowIncompatible=true. For more information, refer to the Comet Compatibility Guide (https://datafusion.apache.org/comet/user-guide/compatibility.html).] +- CometColumnarToRow +- CometSort +- CometExchange diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7/q67a.native_iceberg_compat/extended.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7/q67a.native_iceberg_compat/extended.txt index c619ba85a1..fcc5c60695 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7/q67a.native_iceberg_compat/extended.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7/q67a.native_iceberg_compat/extended.txt @@ -1,6 +1,6 @@ TakeOrderedAndProject [COMET: ] +- Filter - +- Window [COMET: WindowExec is not fully compatible with Spark (Native WindowExec has known correctness issues). To enable it anyway, set spark.comet.operator.WindowExec.allowIncompatible=true. For more information, refer to the Comet Compatibility Guide (https://datafusion.apache.org/comet/user-guide/compatibility.html)., Partitioning and sorting specifications must be the same.] + +- Window [COMET: WindowExec is not fully compatible with Spark (Native WindowExec has known correctness issues). To enable it anyway, set spark.comet.operator.WindowExec.allowIncompatible=true. For more information, refer to the Comet Compatibility Guide (https://datafusion.apache.org/comet/user-guide/compatibility.html).] +- CometColumnarToRow +- CometSort +- CometExchange diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7/q67a/extended.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7/q67a/extended.txt index c619ba85a1..fcc5c60695 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7/q67a/extended.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7/q67a/extended.txt @@ -1,6 +1,6 @@ TakeOrderedAndProject [COMET: ] +- Filter - +- Window [COMET: WindowExec is not fully compatible with Spark (Native WindowExec has known correctness issues). To enable it anyway, set spark.comet.operator.WindowExec.allowIncompatible=true. For more information, refer to the Comet Compatibility Guide (https://datafusion.apache.org/comet/user-guide/compatibility.html)., Partitioning and sorting specifications must be the same.] + +- Window [COMET: WindowExec is not fully compatible with Spark (Native WindowExec has known correctness issues). To enable it anyway, set spark.comet.operator.WindowExec.allowIncompatible=true. For more information, refer to the Comet Compatibility Guide (https://datafusion.apache.org/comet/user-guide/compatibility.html).] +- CometColumnarToRow +- CometSort +- CometExchange diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7/q70a.native_iceberg_compat/extended.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7/q70a.native_iceberg_compat/extended.txt index 0a2ecaa017..eea54c2694 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7/q70a.native_iceberg_compat/extended.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7/q70a.native_iceberg_compat/extended.txt @@ -38,7 +38,7 @@ : +- BroadcastExchange : +- Project : +- Filter - : +- Window [COMET: WindowExec is not fully compatible with Spark (Native WindowExec has known correctness issues). To enable it anyway, set spark.comet.operator.WindowExec.allowIncompatible=true. For more information, refer to the Comet Compatibility Guide (https://datafusion.apache.org/comet/user-guide/compatibility.html)., Partitioning and sorting specifications must be the same.] + : +- Window [COMET: WindowExec is not fully compatible with Spark (Native WindowExec has known correctness issues). To enable it anyway, set spark.comet.operator.WindowExec.allowIncompatible=true. For more information, refer to the Comet Compatibility Guide (https://datafusion.apache.org/comet/user-guide/compatibility.html).] : +- CometColumnarToRow : +- CometSort : +- CometHashAggregate @@ -93,7 +93,7 @@ : +- BroadcastExchange : +- Project : +- Filter - : +- Window [COMET: WindowExec is not fully compatible with Spark (Native WindowExec has known correctness issues). To enable it anyway, set spark.comet.operator.WindowExec.allowIncompatible=true. For more information, refer to the Comet Compatibility Guide (https://datafusion.apache.org/comet/user-guide/compatibility.html)., Partitioning and sorting specifications must be the same.] + : +- Window [COMET: WindowExec is not fully compatible with Spark (Native WindowExec has known correctness issues). To enable it anyway, set spark.comet.operator.WindowExec.allowIncompatible=true. For more information, refer to the Comet Compatibility Guide (https://datafusion.apache.org/comet/user-guide/compatibility.html).] : +- CometColumnarToRow : +- CometSort : +- CometHashAggregate @@ -148,7 +148,7 @@ +- BroadcastExchange +- Project +- Filter - +- Window [COMET: WindowExec is not fully compatible with Spark (Native WindowExec has known correctness issues). To enable it anyway, set spark.comet.operator.WindowExec.allowIncompatible=true. For more information, refer to the Comet Compatibility Guide (https://datafusion.apache.org/comet/user-guide/compatibility.html)., Partitioning and sorting specifications must be the same.] + +- Window [COMET: WindowExec is not fully compatible with Spark (Native WindowExec has known correctness issues). To enable it anyway, set spark.comet.operator.WindowExec.allowIncompatible=true. For more information, refer to the Comet Compatibility Guide (https://datafusion.apache.org/comet/user-guide/compatibility.html).] +- CometColumnarToRow +- CometSort +- CometHashAggregate diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7/q70a/extended.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7/q70a/extended.txt index 0a2ecaa017..eea54c2694 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7/q70a/extended.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7/q70a/extended.txt @@ -38,7 +38,7 @@ : +- BroadcastExchange : +- Project : +- Filter - : +- Window [COMET: WindowExec is not fully compatible with Spark (Native WindowExec has known correctness issues). To enable it anyway, set spark.comet.operator.WindowExec.allowIncompatible=true. For more information, refer to the Comet Compatibility Guide (https://datafusion.apache.org/comet/user-guide/compatibility.html)., Partitioning and sorting specifications must be the same.] + : +- Window [COMET: WindowExec is not fully compatible with Spark (Native WindowExec has known correctness issues). To enable it anyway, set spark.comet.operator.WindowExec.allowIncompatible=true. For more information, refer to the Comet Compatibility Guide (https://datafusion.apache.org/comet/user-guide/compatibility.html).] : +- CometColumnarToRow : +- CometSort : +- CometHashAggregate @@ -93,7 +93,7 @@ : +- BroadcastExchange : +- Project : +- Filter - : +- Window [COMET: WindowExec is not fully compatible with Spark (Native WindowExec has known correctness issues). To enable it anyway, set spark.comet.operator.WindowExec.allowIncompatible=true. For more information, refer to the Comet Compatibility Guide (https://datafusion.apache.org/comet/user-guide/compatibility.html)., Partitioning and sorting specifications must be the same.] + : +- Window [COMET: WindowExec is not fully compatible with Spark (Native WindowExec has known correctness issues). To enable it anyway, set spark.comet.operator.WindowExec.allowIncompatible=true. For more information, refer to the Comet Compatibility Guide (https://datafusion.apache.org/comet/user-guide/compatibility.html).] : +- CometColumnarToRow : +- CometSort : +- CometHashAggregate @@ -148,7 +148,7 @@ +- BroadcastExchange +- Project +- Filter - +- Window [COMET: WindowExec is not fully compatible with Spark (Native WindowExec has known correctness issues). To enable it anyway, set spark.comet.operator.WindowExec.allowIncompatible=true. For more information, refer to the Comet Compatibility Guide (https://datafusion.apache.org/comet/user-guide/compatibility.html)., Partitioning and sorting specifications must be the same.] + +- Window [COMET: WindowExec is not fully compatible with Spark (Native WindowExec has known correctness issues). To enable it anyway, set spark.comet.operator.WindowExec.allowIncompatible=true. For more information, refer to the Comet Compatibility Guide (https://datafusion.apache.org/comet/user-guide/compatibility.html).] +- CometColumnarToRow +- CometSort +- CometHashAggregate