Skip to content

Commit 00c8f42

Browse files
committed
feat: Remove unnecessary transition for native writes
1 parent 3dcd9ad commit 00c8f42

File tree

1 file changed

+5
-1
lines changed

1 file changed

+5
-1
lines changed

spark/src/main/scala/org/apache/comet/rules/EliminateRedundantTransitions.scala

Lines changed: 5 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -22,7 +22,7 @@ package org.apache.comet.rules
2222
import org.apache.spark.sql.SparkSession
2323
import org.apache.spark.sql.catalyst.rules.Rule
2424
import org.apache.spark.sql.catalyst.util.sideBySide
25-
import org.apache.spark.sql.comet.{CometCollectLimitExec, CometColumnarToRowExec, CometPlan, CometSparkToColumnarExec}
25+
import org.apache.spark.sql.comet.{CometCollectLimitExec, CometColumnarToRowExec, CometNativeWriteExec, CometPlan, CometSparkToColumnarExec}
2626
import org.apache.spark.sql.comet.execution.shuffle.{CometColumnarShuffle, CometShuffleExchangeExec}
2727
import org.apache.spark.sql.execution.{ColumnarToRowExec, RowToColumnarExec, SparkPlan}
2828
import org.apache.spark.sql.execution.adaptive.QueryStageExec
@@ -80,6 +80,10 @@ case class EliminateRedundantTransitions(session: SparkSession) extends Rule[Spa
8080
// and CometSparkToColumnarExec
8181
sparkToColumnar.child
8282
}
83+
// Remove unnecessary transition for native writes
84+
// Write should be final operation in the plan
85+
case ColumnarToRowExec(nativeWrite: CometNativeWriteExec) =>
86+
nativeWrite
8387
case c @ ColumnarToRowExec(child) if hasCometNativeChild(child) =>
8488
val op = CometColumnarToRowExec(child)
8589
if (c.logicalLink.isEmpty) {

0 commit comments

Comments
 (0)