diff --git a/spark/src/main/scala/org/apache/comet/serde/QueryPlanSerde.scala b/spark/src/main/scala/org/apache/comet/serde/QueryPlanSerde.scala index 570c07cb09..a0b89ef95d 100644 --- a/spark/src/main/scala/org/apache/comet/serde/QueryPlanSerde.scala +++ b/spark/src/main/scala/org/apache/comet/serde/QueryPlanSerde.scala @@ -959,7 +959,7 @@ object QueryPlanSerde extends Logging with CometExprShim { CometConf.COMET_RESPECT_PARQUET_FILTER_PUSHDOWN.get(conf)) { val dataFilters = new ListBuffer[Expr]() - for (filter <- scan.dataFilters) { + for (filter <- scan.supportedDataFilters) { exprToProto(filter, scan.output) match { case Some(proto) => dataFilters += proto case _ => diff --git a/spark/src/main/scala/org/apache/spark/sql/comet/CometScanExec.scala b/spark/src/main/scala/org/apache/spark/sql/comet/CometScanExec.scala index 4dd889d231..efc54ab712 100644 --- a/spark/src/main/scala/org/apache/spark/sql/comet/CometScanExec.scala +++ b/spark/src/main/scala/org/apache/spark/sql/comet/CometScanExec.scala @@ -155,16 +155,21 @@ case class CometScanExec( } } - @transient - private lazy val pushedDownFilters = { - val supportedFilters = if (scanImpl == CometConf.SCAN_NATIVE_DATAFUSION) { - // `native_datafusion` scan does not support subquery pushdown filters, - // see: https://github.com/apache/datafusion-comet/issues/2424 + /** + * Returns the data filters that are supported for this scan implementation. For + * native_datafusion scans, this excludes dynamic pruning filters (subqueries) + */ + lazy val supportedDataFilters: Seq[Expression] = { + if (scanImpl == CometConf.SCAN_NATIVE_DATAFUSION) { dataFilters.filterNot(isDynamicPruningFilter) } else { dataFilters } - getPushedDownFilters(relation, supportedFilters) + } + + @transient + private lazy val pushedDownFilters = { + getPushedDownFilters(relation, supportedDataFilters) } override lazy val metadata: Map[String, String] =